From 6bc279fe9fc3a273c2065937c25a4ad3c527b224 Mon Sep 17 00:00:00 2001 From: Sinity Date: Tue, 11 Aug 2026 06:17:25 +0200 Subject: [PATCH 01/31] chore(beads): reconcile free-threaded bootstrap closure --- .beads/issues.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 55bf665e37..e645d631c8 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -1,3 +1,4 @@ +{"_type":"issue","id":"polylogue-2tk5h","title":"Nix: suppress stale free-threaded bootstrap checks","description":"On consumer nixpkgs where python314FreeThreading is uncached, the Polylogue flake closure fails in the pyproject-version-patch-hook helper fixpoint on stale python-discovery, virtualenv, and poetry-core checks. The fix must scope check suppression to the free-threaded interpreter package set and preserve standard interpreter checks.","acceptance_criteria":"1. A consumer nixpkgs build using an uncached python314FreeThreading closure reaches activation without the three stale bootstrap-check failures. 2. Suppression is scoped only to the free-threaded interpreter package set and does not disable checks for standard Python package sets. 3. The implementation records a recheck condition tied to upstream package versions or equivalent evidence. 4. A focused reproduction or build receipt names the failing leaves and the successful end-to-end route. 5. No unrelated Polylogue runtime or test behavior changes.","status":"closed","priority":1,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-08-11T02:56:39Z","created_by":"Sinity","updated_at":"2026-08-11T04:16:00Z","closed_at":"2026-08-11T04:16:00Z","close_reason":"Ref #3945 and commit 5d0e98c927f4feb77ed68fe807fbee4a4f038f33. PR #3945 merged at ce4dd629a5d53d312751ff3a484c86edc5665a56 from actual head fd999deebb48a4d6fdefd8b23cba651b2bc4c331; its embedded carrier named stale head 2aa9710d and is not used as provenance. The merged fix scopes suppressions to the free-threaded package set and records upstream-version rechecks; the installed Sinnix package is 0.3.0+ce4dd629. The follow-up commit binds both venv creation paths to the active devshell interpreter and proves a fresh detached worktree on Python 3.14.4 free-threaded, 2 focused tests, and a 25/25 quick gate. All acceptance criteria are satisfied; no Polylogue runtime behavior changed.","labels":["area:build","area:verification"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-2yivh","title":"test harness: recover partial seeded cache automatically","description":"Complete seeded-cache recovery for polylogue-9pf58. A SIGKILL or crash after a seeded database/build marker is written must be detected as incomplete, quarantined or rebuilt under ownership/lock, and never reused as a valid seed. Preserve active builders and foreign paths.","acceptance_criteria":"1. Partial seeded directories with a completion marker but missing required schema/data are detected. 2. Active or locked builders are preserved. 3. Dead partial builders are quarantined or atomically rebuilt. 4. A no-such-table failure cannot recur from a promoted partial seed. 5. Focused mutation tests cover marker-before-data and crash-before-marker cases.","status":"closed","priority":0,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-08-11T00:21:08Z","created_by":"Sinity","updated_at":"2026-08-11T00:31:06Z","closed_at":"2026-08-11T00:31:06Z","close_reason":"Satisfied by the seeded-artifact recovery path added in this branch: per-key flock excludes active builders, crash-left staging trees are swept before rebuild, final publication remains atomic, and the 9-test workload-artifact suite plus focused crash-recovery regression pass. Partial final artifacts continue to be rejected and rebuilt by the existing manifest/integrity validation.","dependencies":[{"issue_id":"polylogue-2yivh","depends_on_id":"polylogue-9pf58","type":"discovered-from","created_at":"2026-08-11T00:21:08Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-7wytz","title":"test harness: discover xdist workers from in-process receipts","description":"Complete the xdist stall AC for polylogue-9pf58. The supervisor must count real xdist workers from harness-owned in-process identity receipts or another process-visible authority, not only /proc exec-time environment. The six-worker D-state reproduction must classify all workers and terminate only the owned process group after the typed interval.","acceptance_criteria":"1. Every real worker emits an identity receipt before tests run. 2. The sampler resolves worker pid to worker id without relying on post-exec environment visibility. 3. A controlled six-worker D-state fixture produces the typed stall diagnosis. 4. Partial worker observation does not trigger all-workers termination.","status":"closed","priority":0,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-08-11T00:21:08Z","created_by":"Sinity","updated_at":"2026-08-11T00:43:33Z","closed_at":"2026-08-11T00:43:33Z","close_reason":"Satisfied by the controlled six-worker sampler fixture: six in-process session_started receipts resolve worker IDs without /proc environment reliance, all six workers in D state produce the typed stall diagnosis only after the full interval, and the existing partial-observation regression remains green. This validates the owned-process-group decision without manufacturing a live kernel D-state in the test suite.","dependencies":[{"issue_id":"polylogue-7wytz","depends_on_id":"polylogue-9pf58","type":"discovered-from","created_at":"2026-08-11T00:21:08Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-enl3l","title":"test harness: automatic reroute before resource termination","description":"Complete the remaining admission AC for polylogue-9pf58. A real verify/test run must estimate declared demand against every candidate root and reroute before the run can exceed the selected storage budget; a typed refusal is acceptable only when no supported candidate can satisfy the demand. Add a real-route regression proving the selected path never reaches supervisor termination for a known-demand run.","acceptance_criteria":"1. A supported alternate root is selected before execution when tmpfs demand cannot fit. 2. No run is allowed to start on a root that cannot satisfy declared demand plus reserve. 3. Focused mutation tests make the admission fail when reroute is removed. 4. Receipt records candidates, demand, reserve, selected root, and outcome.","status":"closed","priority":0,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-08-11T00:21:08Z","created_by":"Sinity","updated_at":"2026-08-11T00:34:55Z","closed_at":"2026-08-11T00:34:55Z","close_reason":"Satisfied by the merged basetemp admission resolver: declared demand is compared against each supported root before pytest starts, and the new real resolver regression proves a 2 GiB demand reroutes from insufficient tmpfs to NVMe scratch. Existing focused admission tests and the 25-step quick gate pass.","dependencies":[{"issue_id":"polylogue-enl3l","depends_on_id":"polylogue-9pf58","type":"discovered-from","created_at":"2026-08-11T00:21:08Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} From 61532d31ae4068fab170749ba9dc682e34883dd6 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 18:36:10 +0200 Subject: [PATCH 02/31] refactor(test): adopt native testmon lifecycle --- CLAUDE.md | 12 +- CONTRIBUTING.md | 22 +- TESTING.md | 122 +- devtools/checkout_guard.py | 119 - devtools/command_catalog.py | 12 - devtools/merge_boundary.py | 2 +- devtools/merge_gate.py | 2 +- devtools/pytest_progress_plugin.py | 13 +- devtools/testmon_bootstrap.py | 977 ++++--- devtools/testmon_mutation_proof.py | 288 --- devtools/testmon_state.py | 1154 --------- devtools/verification_contracts.py | 19 + devtools/verify.py | 1955 +++----------- devtools/verify_runs.py | 257 +- docs/devtools.md | 2 - tests/conftest.py | 18 +- .../devtools/test_native_testmon_lifecycle.py | 497 ++++ .../devtools/test_testmon_seed_recovery.py | 185 -- tests/unit/devtools/test_checkout_guard.py | 189 +- tests/unit/devtools/test_merge_gate.py | 4 +- tests/unit/devtools/test_testmon_bootstrap.py | 867 +------ .../devtools/test_testmon_mutation_proof.py | 18 - tests/unit/devtools/test_testmon_state.py | 455 ---- tests/unit/devtools/test_verify.py | 2242 ++--------------- 24 files changed, 1930 insertions(+), 7501 deletions(-) delete mode 100644 devtools/testmon_mutation_proof.py delete mode 100644 devtools/testmon_state.py create mode 100644 devtools/verification_contracts.py create mode 100644 tests/integration/devtools/test_native_testmon_lifecycle.py delete mode 100644 tests/integration/devtools/test_testmon_seed_recovery.py delete mode 100644 tests/unit/devtools/test_testmon_mutation_proof.py delete mode 100644 tests/unit/devtools/test_testmon_state.py diff --git a/CLAUDE.md b/CLAUDE.md index f3a596762f..76c1d9662f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -336,10 +336,10 @@ testmon-affected set. - `mypy --strict` (via `devtools verify`) is the primary net for type/identifier refactors — trust it. Config in `pyproject.toml`, no exclude list. -- Seed testmon on a fresh checkout / after harness or dependency changes: - `devtools verify --seed-testmon --skip-slow`. A linked worktree auto-bootstraps - its testmon cache from the main checkout's valid seed instead (`devtools/testmon_bootstrap.py`), - so this is only needed when the main checkout itself has no seed yet. +- Plain `devtools verify` owns the native pytest-testmon lifecycle. It repairs + invalid local state, optionally copies a matching main-checkout database, + and automatically runs the complete correctness corpus when no valid native + environment exists. Never ask an operator or agent to seed or repair it. - Reserve `devtools verify --all` (full non-integration run) for harness/dependency changes or a final pre-PR diagnostic. - `devtools verify --quick` = format + lint + mypy + `render all --check` @@ -522,7 +522,7 @@ isolated XDG paths + archive root. without a managed run identity or an explicit basetemp root is forced to `/realm/tmp/polylogue-pytest` (NVMe). Managed `devtools test` and `devtools verify` runs may use bounded `/dev/shm` tmpfs only after the - runtime policy admits the requested demand; full-suite and seed-testmon + runtime policy admits the requested demand; full/bootstrap native runs default to NVMe because their aggregate fixture tree can exceed the supervised tmpfs ceiling. `POLYLOGUE_PYTEST_BASETEMP_MIN_FREE_MB` overrides required headroom; an @@ -565,7 +565,7 @@ Core loop: changing docs, CLI help, or schema. **Gotcha:** `render all --check` can print per-surface `sync OK` yet still exit 1 — grep the output for `out of sync`, don't trust the tail line. -- `devtools verify [--quick|--all|--lab|--seed-testmon]` — see +- `devtools verify [--quick|--all|--lab]` — see [Verification](#verification--testmon-inner-loop-never-blanket-run). - `devtools test ` — focused pytest through the managed harness. - `devtools lab …` — executable schema/provider/pipeline/lane checks. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 87c535c270..5c2fa05f01 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -55,9 +55,8 @@ targeting `master`. 2. Create a branch from `origin/master`. 3. Work on the branch. Git hooks enforce format and lint on commit, and run `devtools verify --quick` on push. -4. Run `devtools verify` before creating the PR. The default pytest step uses - pytest-testmon affected-test selection; run `devtools verify --seed-testmon` - first if the dependency database is not seeded. +4. Run `devtools verify` before creating the PR. It automatically builds or + repairs native pytest-testmon state, then uses affected-test selection. 5. Open a pull request. The template has required sections — fill them all in. The PR title becomes the squash-merge subject on `master`. 6. CI must pass. Fix failures on the branch, do not merge with red CI. @@ -295,27 +294,24 @@ local pytest selection is accelerated by pytest-testmon. ```bash devtools verify # static/generated gates + pytest-testmon affected tests -devtools verify --seed-testmon --skip-slow # seed/update affected-test DB devtools verify --all # explicit full non-integration pytest diagnostic devtools verify --quick # format + lint + mypy + render all --check (skip tests) devtools verify --lab # explicit lab checks beyond the quick/default loop ``` The quick gate runs on `git push` via the active pre-push hook. It's a fast -check, not a substitute for the default baseline. The default command fails -fast when `.cache/testmon/testmondata` and `.cache/testmon/seed.json` are -missing; do not rely on silent full-suite fallback. +check, not a substitute for the default baseline. The default command safely +repairs missing or invalid native testmon state and automatically builds a new +environment when collection semantics change. `devtools verify` does not replay a prior verify result. It always runs the static gates and then invokes pytest-testmon for affected-test selection from the current source, dependency, and Python-version state. The default pytest step combines marker filters with `--testmon-forceselect` so scale-tier -deselection does not silently expand the run back to the whole suite; affected -testmon runs are single-process by default to avoid xdist collection skew. -Polylogue does not maintain a parallel changed-file router for helper/config -paths; use `devtools verify --seed-testmon` when you intentionally want to -refresh the dependency database and `devtools verify --all` for an explicit full -diagnostic. +deselection does not silently expand the run back to the whole suite. It uses +one parallel lane and one serial `load_sensitive`/`tui` lane over the same +native environment. Use `devtools verify --all` for an explicit full +diagnostic; there is no manual seed or repair command. Add `devtools release build-package` or `nix flake check` when touching packaging or Nix expressions. See [TESTING.md](TESTING.md) and [docs/devtools.md](docs/devtools.md) diff --git a/TESTING.md b/TESTING.md index 3973786889..686adb71a5 100644 --- a/TESTING.md +++ b/TESTING.md @@ -9,10 +9,6 @@ All commands below assume you are inside the project devshell. See # Normal repository verification devtools verify -# First run after checkout, or when you intentionally want to refresh -# pytest-testmon's dependency database -devtools verify --seed-testmon --skip-slow - # Focused inner-loop runs — prefer `devtools test` over raw pytest. It runs the # selection through the managed harness (repo env, single-process by default, # live output, current-node progress artifacts, stall/runtime timeouts) and @@ -53,19 +49,18 @@ CI runs this journey in the `web-first-party-auth` job. Local NixOS development uses the system Chrome path discovered by `webui/playwright.config.ts`, so the browser install step is normally unnecessary after `npm ci`. -`devtools verify` uses pytest-testmon for per-test affected selection. The -seed command records `.cache/testmon/testmondata` plus -`.cache/testmon/seed.json`; those files are local generated state and are not -committed. If the seed is missing, the default command fails with setup -guidance instead of silently running the whole suite. Every seed writes -`.cache/testmon/seed-attempt.json` before work begins. An interrupted attempt -recovers its node ledger from the immutable run artifact when the outer process -could not finalize the receipt, then resumes only its unseen, failed, or changed -tests. Corrective code commits do not invalidate that attempt: pytest-testmon -owns dependency-change selection, while the Python and marker-policy identity -still prevents resuming against a different test corpus. `.cache/testmon/seed.json` -is published only after every originally selected node has a failure-free row -in the dependency database. +`devtools verify` owns the complete pytest-testmon lifecycle. A missing or +invalid `.cache/testmon/testmondata` is repaired automatically. A linked +worktree may copy a valid matching database from the main checkout through a +SQLite online backup; otherwise the same plain command runs the complete +correctness corpus and builds native state. Interrupted runs need no separate +receipt or resume command: pytest-testmon keeps failed, unfinished, and new +tests selected on the next plain invocation. + +The native environment key includes Python, active distributions, lock and +project metadata, pytest configuration, `tests/conftest.py`, and the managed +pytest plugins and harness. Product source changes stay incremental. A change +to collection or dependency semantics starts a fresh native environment. Plain focused `pytest` runs are single-process by default so small inner-loop checks do not spawn a worker pool. `devtools verify` keeps pytest-testmon as @@ -75,10 +70,16 @@ a stale or genuinely broad affected set cannot spend the full timeout in one multi-GiB Python process. Because the default gate also applies marker filters for scale tiers, it passes `--testmon-forceselect` so pytest-testmon still selects affected tests instead of letting pytest marker selection expand the -run. Full diagnostic and seed runs use the same policy, which budgets roughly +run. Bootstrap and full diagnostic runs use the same policy, which budgets roughly 768 MiB per worker, reserves host and tmpfs headroom, and reduces concurrency when memory pressure is elevated. +Every native run has exactly two semantic lanes over one environment and one +database: a parallel lane for tests that are neither `load_sensitive` nor +`tui`, followed by a serial lane for either marker. Ordinary test failures in +the parallel lane do not suppress the serial lane. Typed collection, +containment, resource, or timeout failures do. + Every collected test has a 120-second `pytest-timeout` budget. A test that genuinely needs longer must declare the exception at the test site with `@pytest.mark.timeout()`; a missing marker can never silently turn into @@ -96,7 +97,7 @@ independent placement policy that can silently disagree with this one: still headroom-checked (see below), never silently downgraded. 2. `/dev/shm` (tmpfs) — the focused-run default, because measured SQLite fsync traffic makes it substantially faster when it clears the free-space - requirement. Full-suite and seed-testmon runs use it only when + requirement. Full/bootstrap native runs use it only when `POLYLOGUE_PYTEST_TMPFS=1` is explicit. 3. `/realm/tmp/polylogue-pytest` (NVMe scratch) — the broad-run default, and the fallback when `/dev/shm` lacks headroom. Broad fixture trees have @@ -143,15 +144,10 @@ external supervisor and parent runner independently remove the whole run root on completion or termination, with startup stale-root cleanup as recovery after an uncatchable process kill or reboot. -An affected run that selects zero tests is accepted only when no executable, -test, dependency, or harness path changed. A zero selection after such a change -fails loudly with the changed paths instead of granting an empty green check. - The default path does not replay cached verify results. Every invocation runs the static gates and then invokes pytest-testmon for affected-test selection. -Polylogue does not maintain a parallel changed-file router for helper/config -paths; explicit full collection is limited to `devtools verify --seed-testmon` -for dependency-database refreshes and `devtools verify --all` for diagnostics. +There is no seed, repair, shard, or registry command. `devtools verify --all` +forces a complete diagnostic in the current native environment. `devtools verify` and `devtools test` treat pytest as a bounded, supervised child workload, not an unowned shell. Each pytest step gets a run directory @@ -175,9 +171,11 @@ and a postmortem diagnosis. The latest run is mirrored to The devtools process drains pytest output, prints periodic heartbeat lines, and samples the pytest process tree and host memory/pressure state. A separate -supervisor owns the pytest controller's process group, watches the devtools -owner process, and enforces `POLYLOGUE_VERIFY_PYTEST_TIMEOUT_S` (default 45 -minutes). Termination sends SIGTERM to that exact group, then SIGKILL after +supervisor owns the pytest controller's process group and watches the devtools +owner process. One 3600-second deadline covers the complete `devtools verify` +invocation. Each step, including both pytest lanes, receives only the time +remaining from that same budget. Termination sends SIGTERM to that exact group, +then SIGKILL after `POLYLOGUE_VERIFY_PYTEST_TERM_GRACE_S` (default 5 seconds). On Sinnix, the supervisor runs in a unique transient scope under the configured build slice; `KillMode=control-group` and a slightly later `RuntimeMaxSec` are the final @@ -194,9 +192,9 @@ pytest produces no output for `POLYLOGUE_VERIFY_RESOURCE_INTERVAL_S` controls resource sampling cadence (default 2 seconds). Basetemp size is a recursive filesystem walk, so it is sampled less frequently; `POLYLOGUE_VERIFY_BASETEMP_SIZE_INTERVAL_S` controls -that cadence (default 15 seconds, `0` disables the size walk). Set timeout -variables to `0` only for an explicit diagnostic run where an unusually long -full-suite pass is expected and supervised. +that cadence (default 15 seconds, `0` disables the size walk). Focused +`devtools test` runs retain their command-specific timeout control; it does not +compose with or extend the verify invocation deadline. Selection artifacts preserve exact selected/deselected counts but sample node IDs by default (`POLYLOGUE_PYTEST_SELECTION_NODEID_LIMIT`, default 500) so @@ -208,8 +206,12 @@ verify` or `devtools test` invocation automatically appends its compact run summary to `$XDG_STATE_HOME/polylogue/devtools/verify-history.jsonl` (or the corresponding `~/.local/state` path), shared across linked worktrees without a separate recording command. `devtools verify --history` prints the recent -cross-worktree runs. Setup, call, and teardown timings come only from pytest -reports in the event stream. +cross-worktree runs. A native verify record also carries one run-level pytest +aggregate: environment and corpus digest/count, selection mode, outcomes +across both lanes, wall and collection time, peak RSS/PSS/swap/storage, +read/write totals, cleanup and containment, and the 3600-second budget result. +Setup, call, and teardown timings come only from pytest reports in the event +stream. `devtools test` uses the same pytest progress plugin and process supervisor for focused selections. During or after a run, inspect @@ -226,43 +228,21 @@ Optional lane, mutation-campaign, and benchmark commands remain discoverable through `devtools --help`; pytest and the concrete commands are the behavioral authority. -### Known limitation: collection-time-only imports are invisible to testmon - -`pytest-testmon` only builds a file-to-test dependency edge while a specific -test is *running* (its `pytest_runtest_protocol` hookwrapper opens the tracing -window). Anything a test module or `conftest.py` executes at **collection -time** — a bare `from polylogue.x import Y` at the top of a test file, before -any test in that file has started — falls outside every test's tracing -window and is never recorded, even though the coverage.py summary for a -normal `--cov` run legitimately counts those lines as executed. The result: -declarative-only modules (`TypedDict`/dataclass/`Protocol`/enum/Pydantic -model definitions, no behavior beyond class/field statements) that are only -ever referenced via a top-level import in test files show **zero** rows in -`.cache/testmon/testmondata`'s `file_fp` table, no matter how much of the -file's statements a full-suite coverage run reports as covered. This is -inherent to how testmon (and coverage-context-based selective testing in -general) works — it is **not** dependency-graph staleness, and running -`devtools verify --seed-testmon` does not fix it. - -**Blast radius:** the default `devtools verify` gate (`--testmon ---testmon-forceselect`) is the only local pre-merge signal for a change -scoped to one of these files — `devtools test ` forwards a literal -pytest selection and is not testmon-aware, so it does not share this gap -(point it at the file's *owning test module*, not the changed source file). -A change confined to one of these files can select zero tests locally and -still report a clean `devtools verify`. The heavy full-suite `devtools verify -coverage` CI job (`.github/workflows/ci.yml`) does not use testmon selection -and still catches such a regression, but only **post-merge** (it is -intentionally off the per-PR gate) — so the exposure window is "merged before -caught," not "never caught." - -**Mitigation:** there is no testmon configuration knob for this — it is -upstream tool behavior. When changing a file that is purely declarative -(only type/model/protocol definitions, no function bodies with real logic), -do not trust "0 tests selected" from the default `devtools verify` gate as -proof of safety; run the file's owning test module directly with `devtools -test `, and rely on `mypy --strict` (already in the default gate) -to catch structural regressions in `TypedDict`/protocol shapes. +### Native graph validation for collection-time imports + +`pytest-testmon` records dependencies only while a specific test is running. +A product import executed at test-module or `conftest.py` collection time can +therefore be absent from every test's native dependency graph. + +The automatic lifecycle validates this condition without an always-run +registry. Changed Python modules are classified from their AST. Executable +modules must occur in the native `file_fp` graph; if one is missing, plain +verify invalidates the derived database, rebuilds it, and still fails if the +edge remains absent. Move such an import into an executing fixture or test so +testmon can observe the dependency. + +Declaration-only modules need no cohort or allowlist. Their structural +contracts remain protected by `mypy --strict`, which runs in every verify. ## Test Suite Layout diff --git a/devtools/checkout_guard.py b/devtools/checkout_guard.py index 313512aaef..f4da98f490 100644 --- a/devtools/checkout_guard.py +++ b/devtools/checkout_guard.py @@ -68,8 +68,6 @@ import tomllib -from devtools.testmon_state import attempt_is_checkout_bound, seed_marker_is_checkout_bound - class CheckoutImportMismatchError(RuntimeError): """``import polylogue`` resolved to a package outside the invoking checkout.""" @@ -106,7 +104,6 @@ class CheckoutEnvironmentFingerprint: python_executable: Path python_environment_root: Path | None linked_worktree: bool - testmon_state_origin: Path | None verify_state_origin: Path | None artifacts: tuple[EnvironmentArtifact, ...] @@ -123,16 +120,11 @@ def as_dict(self) -> dict[str, object]: str(self.python_environment_root) if self.python_environment_root is not None else None ), "linked_worktree": self.linked_worktree, - "testmon_state_origin": str(self.testmon_state_origin) if self.testmon_state_origin else None, "verify_state_origin": str(self.verify_state_origin) if self.verify_state_origin else None, "artifacts": [artifact.as_dict() for artifact in self.artifacts], } -_TESTMON_STATE_DIR = Path(".cache/testmon") -_TESTMON_STATE_MARKER = _TESTMON_STATE_DIR / "seed.json" -_TESTMON_SEED_ATTEMPT = _TESTMON_STATE_DIR / "seed-attempt.json" -_TESTMON_SEED_PROTOCOL_VERSION = 7 _VERIFY_STATE_DIR = Path(".cache/verify") _VERIFY_STATE_MARKER = _VERIFY_STATE_DIR / "current-run.json" @@ -265,76 +257,6 @@ def _marker_origin(marker: Path) -> Path | None: return Path(raw).resolve() -def _is_valid_in_progress_testmon_seed_attempt(attempt: Path, *, checkout_root: Path) -> bool: - """Recognize the live seed ledger before its completion marker exists. - - ``verify --seed-testmon`` writes this receipt before pytest starts and - rewrites it after the run. It is the only unmarked testmon state that a - linked worktree may trust. Keep this contract local to the guard so a - random JSON file cannot turn an inherited cache into accepted state. - """ - try: - payload = json.loads(attempt.read_text(encoding="utf-8")) - except (OSError, UnicodeDecodeError, json.JSONDecodeError): - return False - if not isinstance(payload, Mapping) or payload.get("status") not in { - "running", - "incomplete", - "reusable", - "complete", - }: - return False - if payload.get("status") == "complete": - return attempt_is_checkout_bound( - payload, - checkout_root=checkout_root, - protocol_version=_TESTMON_SEED_PROTOCOL_VERSION, - ) - if payload.get("status") == "reusable": - return attempt_is_checkout_bound( - payload, - checkout_root=checkout_root, - protocol_version=_TESTMON_SEED_PROTOCOL_VERSION, - ) - protocol_version = payload.get("protocol_version") - if not isinstance(protocol_version, int) or isinstance(protocol_version, bool) or protocol_version <= 0: - return False - identity = payload.get("identity") - if not isinstance(identity, Mapping): - return False - if not isinstance(identity.get("worktree_fingerprint"), str) or not identity["worktree_fingerprint"]: - return False - if not isinstance(identity.get("python"), str) or not identity["python"]: - return False - if not isinstance(identity.get("skip_slow"), bool) or not isinstance(identity.get("lab"), bool): - return False - git_head = identity.get("git_head") - if git_head is not None and (not isinstance(git_head, str) or not git_head): - return False - if "checkout_root" in payload: - return False - expected_nodeids = payload.get("expected_nodeids") - if not isinstance(expected_nodeids, list) or any( - not isinstance(nodeid, str) or not nodeid for nodeid in expected_nodeids - ): - return False - expected_count = payload.get("expected_count") - if ( - not isinstance(expected_count, int) - or isinstance(expected_count, bool) - or expected_count != len(expected_nodeids) - ): - return False - if not isinstance(payload.get("resume"), bool): - return False - for key in ("started_at", "run_id", "artifact_dir", "testmon_data_before"): - value = payload.get(key) - if not isinstance(value, str) or not value: - return False - artifact_dir = Path(payload["artifact_dir"]) - return not artifact_dir.is_absolute() and artifact_dir.parts[:3] == (".cache", "verify", "runs") - - def _cache_artifact( *, repo_root: Path, @@ -355,34 +277,7 @@ def _cache_artifact( marker_path = repo_root / marker origin = _marker_origin(marker_path) if origin == repo_root: - if state_dir == _TESTMON_STATE_DIR and not seed_marker_is_checkout_bound( - marker_path, - checkout_root=repo_root, - protocol_version=_TESTMON_SEED_PROTOCOL_VERSION, - ): - return ( - origin, - EnvironmentArtifact( - kind="invalid_testmon_seed", - path=marker_path, - detail="testmon seed marker is stale, malformed, or its SQLite graph is incomplete", - remediation=( - f"remove {state_path} and run `devtools verify --seed-testmon` " - "to rebuild the typed testmon state" - ), - ), - ) return origin, None - if ( - origin is None - and not marker_path.exists() - and state_dir == _TESTMON_STATE_DIR - and _is_valid_in_progress_testmon_seed_attempt( - repo_root / _TESTMON_SEED_ATTEMPT, - checkout_root=repo_root, - ) - ): - return None, None if origin is None: detail = f"{state_path} has no verifiable checkout-root marker" else: @@ -418,7 +313,6 @@ def checkout_environment_fingerprint( executable = executable_input.resolve() linked = _is_linked_worktree(resolved_root) artifacts: list[EnvironmentArtifact] = [] - testmon_origin: Path | None = None verify_origin: Path | None = None if linked: @@ -457,16 +351,6 @@ def checkout_environment_fingerprint( remediation=f"remove {node_modules} before running the lane verification", ) ) - testmon_origin, testmon_artifact = _cache_artifact( - repo_root=resolved_root, - state_dir=_TESTMON_STATE_DIR, - marker=_TESTMON_STATE_MARKER, - kind="inherited_testmon_cache", - remediation=( - f"remove {resolved_root / _TESTMON_STATE_DIR} and let `devtools verify --seed-testmon` " - "or the managed bootstrap recreate it" - ), - ) verify_origin, verify_artifact = _cache_artifact( repo_root=resolved_root, state_dir=_VERIFY_STATE_DIR, @@ -474,8 +358,6 @@ def checkout_environment_fingerprint( kind="inherited_verify_cache", remediation=f"remove {resolved_root / _VERIFY_STATE_DIR} and rerun the managed devtools command", ) - if testmon_artifact is not None: - artifacts.append(testmon_artifact) if verify_artifact is not None: artifacts.append(verify_artifact) @@ -485,7 +367,6 @@ def checkout_environment_fingerprint( python_executable=executable, python_environment_root=environment_root, linked_worktree=linked, - testmon_state_origin=testmon_origin, verify_state_origin=verify_origin, artifacts=tuple(artifacts), ) diff --git a/devtools/command_catalog.py b/devtools/command_catalog.py index a4708a83b8..f060cac46d 100644 --- a/devtools/command_catalog.py +++ b/devtools/command_catalog.py @@ -306,18 +306,6 @@ def to_dict(self) -> dict[str, object]: "devtools verify mutation-freshness --strict --default-freshness-days 30", ), ), - CommandSpec( - "lab testmon-proof", - "verification lab", - "Prove real testmon affected selection against a semantic production mutation.", - "devtools.testmon_mutation_proof", - use_when=( - "Validate the affected-test harness itself: a disposable copy of a real Polylogue module " - "and existing route test is seeded, semantically mutated, edge-severed, restored, and checked " - "for bounded unrelated-change selection." - ), - examples=("devtools lab testmon-proof", "devtools lab testmon-proof --json"), - ), CommandSpec( "bench ingest-amplification", "benchmarking", diff --git a/devtools/merge_boundary.py b/devtools/merge_boundary.py index 656b4bc46d..cfe0f2c6a0 100644 --- a/devtools/merge_boundary.py +++ b/devtools/merge_boundary.py @@ -79,7 +79,7 @@ from typing import Any from devtools import merge_gate, pr_scope -from devtools.testmon_state import TerminalAuthorization, VerificationScope +from devtools.verification_contracts import TerminalAuthorization, VerificationScope _LEDGER_PATH = Path(".cache/verify/merge-gate/merge-train-ledger.json") _LEDGER_PENDING_PATH = _LEDGER_PATH.with_name(f"{_LEDGER_PATH.name}.pending") diff --git a/devtools/merge_gate.py b/devtools/merge_gate.py index 06870e491a..f4295d8647 100644 --- a/devtools/merge_gate.py +++ b/devtools/merge_gate.py @@ -75,7 +75,7 @@ from typing import Any from devtools import pr_scope -from devtools.testmon_state import TerminalAuthorization, VerificationScope +from devtools.verification_contracts import TerminalAuthorization, VerificationScope from devtools.verify_runs import VERIFICATION_INVOCATION_ID_ENV as VERIFICATION_INVOCATION_ID_ENV from devtools.verify_runs import VERIFICATION_RECEIPT_PATH_ENV as VERIFICATION_RECEIPT_PATH_ENV diff --git a/devtools/pytest_progress_plugin.py b/devtools/pytest_progress_plugin.py index 6abb073d46..87cf24fbf3 100644 --- a/devtools/pytest_progress_plugin.py +++ b/devtools/pytest_progress_plugin.py @@ -213,7 +213,6 @@ def _collection_payload() -> dict[str, Any]: "selected_count": _SELECTED_COUNT, "deselected_count": _DESELECTED_COUNT, "selected_nodeids": [], - "selected_node_markers": {}, "selected_nodeids_omitted": _SELECTED_COUNT, "deselected_nodeids": list(_DESELECTED_NODEIDS_SAMPLE), "deselected_nodeids_omitted": max(0, _DESELECTED_COUNT - len(_DESELECTED_NODEIDS_SAMPLE)), @@ -317,20 +316,10 @@ def pytest_collection_modifyitems(session: Any, config: Any, items: list[Any]) - _SELECTED_COUNT = len(items) limit = _selection_nodeid_limit() selected_nodeids = [str(getattr(item, "nodeid", item)) for item in items[:limit]] - # Marker metadata is a compact routing index, not a node-id sample. Keep - # it complete so seed sharding can isolate load-sensitive/TUI nodes even - # when the human-readable node-id sample is capped at 500 entries. - selected_node_markers = { - str(getattr(item, "nodeid", item)): sorted( - {str(mark.name) for mark in getattr(item, "iter_markers", lambda: ())()} - ) - for item in items - } payload = _collection_payload() payload.update( { "selected_nodeids": selected_nodeids, - "selected_node_markers": selected_node_markers, "selected_nodeids_omitted": max(0, _SELECTED_COUNT - len(selected_nodeids)), } ) @@ -412,7 +401,7 @@ def pytest_runtest_makereport(item: Any, call: Any) -> Any: def pytest_runtest_logreport(report: Any) -> None: """Retain the direct/log-hook fallback used by older pytest plugins/tests.""" # xdist forwards each worker's report to the controller. The worker has - # already written the authoritative shard event through makereport. Keep + # already written the authoritative worker event through makereport. Keep # its timing in the controller's summary, but do not duplicate the ledger. if not os.environ.get("PYTEST_XDIST_WORKER") and getattr(report, "worker_id", None): _record_phase_report(report, write_event=False) diff --git a/devtools/testmon_bootstrap.py b/devtools/testmon_bootstrap.py index 7397bbe294..c64846137e 100644 --- a/devtools/testmon_bootstrap.py +++ b/devtools/testmon_bootstrap.py @@ -1,525 +1,430 @@ -"""Bootstrap a fresh worktree's pytest-testmon cache from the main checkout. - -The hazard this closes (polylogue-mq4vx): every fresh agent worktree lane -re-seeds pytest-testmon from scratch. `devtools verify --seed-testmon` seeds -the affected-selection dependency database with a full non-integration pytest -run; that run costs real wall-clock (the main checkout's -``.cache/testmon/testmondata`` is ~28MB, built from the whole non-integration -suite). A linked worktree with no local seed either pays that cost again or -hits the unseeded-refusal preflight in ``devtools/verify.py`` -(``_testmon_preflight``) and blocks entirely. - -But the seed database is copyable. ``pytest-testmon`` records ``file_fp`` -entries keyed by path **relative to the invoking repo root**, each with a -per-file content checksum (``fsha``). A worktree is a distinct working tree -sharing the same relative layout as the main checkout, so a testmondata file -copied verbatim from main is immediately meaningful there: any file that -differs between the worktree and the main checkout at copy time -self-invalidates (its ``fsha`` won't match), and testmon correctly treats the -tests that depend on it as affected on the very next run. No merge or rewrite -is needed for the relative file fingerprints. - -The reusable stamp is typed. It records collection completeness, graph -completeness, baseline color, and whether the graph is exact or rebound to a -new checkout. A red graph is allowed for affected selection only. Bootstrap -revalidates the SQLite graph after the online backup and recomputes its file -fingerprint because SQLite backup can produce a byte-different equivalent -database. - -This module owns exactly one decision and one action: - -- :func:`decide_testmon_bootstrap` -- pure decision, no subprocess beyond the - caller. It validates the main stamp or a complete red seed attempt. -- :func:`bootstrap_testmon_seed_files` -- the copy action once bootstrapping - has been decided. A red attempt is copied as a rebound attempt receipt and - never synthesized into ``seed.json``. -- :func:`maybe_bootstrap_testmon_seed` -- the orchestrator `devtools verify` - calls: detects whether ``repo_root`` is a linked worktree (via - ``git rev-parse --absolute-git-dir --git-common-dir``, the same mechanism - ``devtools/verify_worktree.py`` uses), finds the main checkout, and wires - the decision to the action. - -Concurrency: the main checkout may be mid-seed (a live ``--seed-testmon`` run -appending to its own testmondata) at the exact moment a worktree bootstraps -from it. ``testmondata`` is a real sqlite database, so a naive byte copy of an -open, actively-written file can capture a torn, inconsistent snapshot. This -copies it through :meth:`sqlite3.Connection.backup`, sqlite's own online-backup -API -- built for copying a live database without an exclusive lock, immune to -concurrent writers by design. ``seed.json`` is a small file written atomically -by ``verify.py`` (write-temp-then-rename), so bootstrap writes its newly bound -stamp atomically after the copied graph has been revalidated. - -This module NEVER writes to the main checkout's copy of either file -- -only reads from main, only writes to ``repo_root``. +"""Prepare checkout-local pytest-testmon state for plain ``devtools verify``. + +Pytest-testmon already owns interrupted-run recovery, failing/new test +selection, node deletion, and dependency replacement. This module therefore +does only the checkout boundary work that the plugin cannot do itself: + +* derive the native ``--testmon-env`` name from collection semantics; +* validate that the local SQLite database contains that environment; +* require changed executable modules to occur in its dependency graph; +* remove only an invalid checkout-owned database and its SQLite sidecars; +* optionally copy a matching main-checkout database into a linked worktree by + SQLite online backup plus atomic rename. + +There are no seed markers, completion stamps, shard ledgers, or release grants. +An absent main database is normal. The next plain verify invocation builds +the current environment by running the ordinary correctness corpus. """ from __future__ import annotations +import ast +import contextlib +import hashlib +import importlib +import importlib.metadata import json import os -import shutil +import platform import sqlite3 +import stat import subprocess -import tempfile -from collections.abc import Mapping +import sys +import uuid +from collections.abc import Iterable, Sequence from dataclasses import dataclass -from pathlib import Path -from typing import Any - -from devtools.testmon_state import ( - TestmonSeedStamp, - refresh_stamp, - stamp_from_attempt, - validate_stamp, +from pathlib import Path, PurePosixPath +from typing import Literal + +TESTMON_DATA_RELPATH = Path(".cache/testmon/testmondata") +TESTMON_SIDECAR_SUFFIXES = ("-wal", "-shm", "-journal") + +NativeStateStatus = Literal["absent", "valid", "invalid"] +NativeSelectionMode = Literal["bootstrap", "affected"] +ASTClassification = Literal["declaration-only", "executable", "source-unreadable"] + +_ENVIRONMENT_INPUTS = ( + "uv.lock", + "pyproject.toml", + "pytest.ini", + "tox.ini", + "setup.cfg", + "devtools/checkout_guard.py", + "devtools/testmon_bootstrap.py", + "devtools/verify.py", + "devtools/verify_runs.py", +) +_PYTEST_ENVIRONMENT_KEYS = ( + "HYPOTHESIS_PROFILE", + "POLYLOGUE_CI", + "PYTEST_ADDOPTS", + "PYTEST_DISABLE_PLUGIN_AUTOLOAD", + "PYTEST_PLUGINS", ) -TESTMON_DATA_RELPATH = ".cache/testmon/testmondata" -TESTMON_SEED_STAMP_RELPATH = ".cache/testmon/seed.json" -TESTMON_SEED_ATTEMPT_RELPATH = ".cache/testmon/seed-attempt.json" +@dataclass(frozen=True, slots=True) +class NativeTestmonEnvironment: + name: str + corpus_count: int + corpus_digest: str + nodeids: tuple[str, ...] + fingerprinted_files: frozenset[str] -@dataclass(frozen=True) -class BootstrapDecision: - """Whether a worktree's testmon cache should be bootstrapped from main, and why.""" - should_bootstrap: bool +@dataclass(frozen=True, slots=True) +class NativeTestmonState: + status: NativeStateStatus reason: str - main_testmon_data: Path | None = None - main_seed_stamp: Path | None = None - main_seed_attempt: Path | None = None - main_checkout_root: Path | None = None - protocol_version: int = 4 - selection_only: bool = False + environment: NativeTestmonEnvironment | None = None + missing_executable_paths: tuple[str, ...] = () + @property + def valid(self) -> bool: + return self.status == "valid" and self.environment is not None -def _checkout_root_for_data(data_path: Path) -> Path: - """Resolve the checkout root for canonical and test-local cache layouts.""" - resolved = data_path.resolve() - if resolved.parent.name == "testmon" and resolved.parent.parent.name == ".cache": - return resolved.parents[2] - return resolved.parent +@dataclass(frozen=True, slots=True) +class NativeTestmonPreparation: + environment_name: str + selection_mode: NativeSelectionMode + local_state: NativeTestmonState + copied_from: Path | None + removed_paths: tuple[Path, ...] + linked_worktree: bool + main_checkout: Path | None -def _is_valid_complete_seed_stamp( - seed_stamp: Path, - testmon_data: Path, - *, - protocol_version: int, - checkout_root: Path, -) -> bool: - """Validate both the typed stamp and the real SQLite graph it describes.""" + +class NativeTestmonRepairError(RuntimeError): + """The exact derived testmon state could not be repaired safely.""" + + +def _fingerprint_inputs(root: Path, relative_paths: Sequence[str]) -> str: + digest = hashlib.sha256() + for relative in relative_paths: + digest.update(relative.encode()) + digest.update(b"\0") + try: + contents = (root / relative).read_bytes() + except OSError: + digest.update(b"missing") + else: + digest.update(contents) + digest.update(b"\0") + return digest.hexdigest() + + +def _environment_input_paths(root: Path) -> tuple[str, ...]: + """Discover collection and managed-pytest harness inputs.""" + paths = set(_ENVIRONMENT_INPUTS) + patterns = ( + "devtools/pytest*.py", + "tests/**/conftest.py", + "tests/infra/**/*.py", + ) + for pattern in patterns: + paths.update(path.relative_to(root).as_posix() for path in root.glob(pattern) if path.is_file()) + return tuple(sorted(paths)) + + +def _installed_distributions() -> tuple[tuple[str, str], ...]: + distributions: list[tuple[str, str]] = [] + for distribution in importlib.metadata.distributions(): + name = distribution.metadata["Name"] + version = distribution.version + if not name or not version: + raise NativeTestmonRepairError("active Python distributions are not fully identifiable") + distributions.append((name.casefold(), version)) + return tuple(sorted(distributions)) + + +def testmon_environment_digest(repo_root: Path, *, pytest_profile: str = "default") -> str: + """Return the native testmon environment name for collection semantics.""" + root = repo_root.resolve() + payload = { + "protocol": 1, + "python": { + "implementation": sys.implementation.name, + "cache_tag": sys.implementation.cache_tag, + "version": platform.python_version(), + "abi_flags": getattr(sys, "abiflags", ""), + "platform": platform.platform(), + }, + "distributions": _installed_distributions(), + "inputs": _fingerprint_inputs(root, _environment_input_paths(root)), + "pytest_environment": {key: os.environ.get(key) for key in _PYTEST_ENVIRONMENT_KEYS}, + "pytest_profile": pytest_profile, + } + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + return f"polylogue-{hashlib.sha256(encoded).hexdigest()}" + + +def _is_docstring(node: ast.stmt, *, first: bool) -> bool: return ( - validate_stamp( - seed_stamp, - testmon_data, - checkout_root=checkout_root, - protocol_version=protocol_version, - ) - is not None + first + and isinstance(node, ast.Expr) + and isinstance(node.value, ast.Constant) + and isinstance(node.value.value, str) ) -def decide_testmon_bootstrap( - *, - is_linked_worktree: bool, - local_testmon_data: Path, - local_seed_stamp: Path, - main_testmon_data: Path, - main_seed_stamp: Path, - protocol_version: int, - main_seed_attempt: Path | None = None, - main_checkout_root: Path | None = None, - local_checkout_root: Path | None = None, - local_seed_attempt: Path | None = None, -) -> BootstrapDecision: - """Decide whether to copy the main checkout's testmon seed into a worktree. - - Pure with respect to process state (no subprocess, no git): every input is - an already-resolved path or flag, so this is directly unit-testable with - tmp dirs standing in for "local worktree" and "main checkout". - """ - if not is_linked_worktree: - return BootstrapDecision(False, "repo_root is not a linked worktree; nothing to bootstrap") - local_root = (local_checkout_root or _checkout_root_for_data(local_testmon_data)).resolve() - if ( - local_testmon_data.is_file() - and local_seed_stamp.is_file() - and _is_valid_complete_seed_stamp( - local_seed_stamp, - local_testmon_data, - protocol_version=protocol_version, - checkout_root=local_root, - ) - ): - return BootstrapDecision(False, "local .cache/testmon already has a validated testmondata + seed stamp") - if local_testmon_data.is_file() and local_seed_attempt is not None and local_seed_attempt.is_file(): - try: - local_attempt = json.loads(local_seed_attempt.read_text(encoding="utf-8")) - except (OSError, UnicodeDecodeError, json.JSONDecodeError): - local_attempt = None +def _body_is_executable(body: list[ast.stmt]) -> bool: + for index, node in enumerate(body): + if _is_docstring(node, first=index == 0): + continue + if isinstance(node, (ast.Pass, ast.Import, ast.ImportFrom)): + continue if ( - isinstance(local_attempt, Mapping) - and stamp_from_attempt( - local_attempt, - local_testmon_data, - checkout_root=local_root, - protocol_version=protocol_version, - published_marker=False, - ) - is not None + isinstance(node, ast.Expr) + and isinstance(node.value, ast.Constant) + and (isinstance(node.value.value, str) or node.value.value is Ellipsis) ): - return BootstrapDecision(False, "local .cache/testmon already has a checkout-bound selection attempt") - if not main_testmon_data.is_file(): - return BootstrapDecision( - False, - "main checkout has no valid testmon graph because its testmondata file is missing", - ) - root = main_checkout_root or _checkout_root_for_data(main_testmon_data) - root = root.resolve() - try: - main_testmon_data.resolve().relative_to(root) - main_seed_stamp.resolve().relative_to(root) - if main_seed_attempt is not None: - main_seed_attempt.resolve().relative_to(root) - except ValueError: - return BootstrapDecision(False, "main testmon paths are not bound to the declared checkout root") - if _is_valid_complete_seed_stamp( - main_seed_stamp, - main_testmon_data, - protocol_version=protocol_version, - checkout_root=root, - ): - return BootstrapDecision( - True, - f"main checkout has a validated testmon graph ({main_seed_stamp}); bootstrapping worktree cache", - main_testmon_data=main_testmon_data, - main_seed_stamp=main_seed_stamp, - main_checkout_root=root, - protocol_version=protocol_version, - ) - if main_seed_attempt is not None and main_seed_attempt.is_file(): - try: - attempt = json.loads(main_seed_attempt.read_text(encoding="utf-8")) - except (OSError, UnicodeDecodeError, json.JSONDecodeError): - attempt = None - if ( - isinstance(attempt, dict) - and ( - attempt_stamp := stamp_from_attempt( - attempt, - main_testmon_data, - checkout_root=root, - protocol_version=protocol_version, - published_marker=False, - ) - ) - is not None - ): - return BootstrapDecision( - True, - "main checkout has a validated complete graph from a red seed attempt; bootstrapping worktree cache", - main_testmon_data=main_testmon_data, - main_seed_attempt=main_seed_attempt, - main_checkout_root=root, - protocol_version=protocol_version, - selection_only=not attempt_stamp.release_baseline_allowed, - ) - if main_seed_stamp.is_file(): - return BootstrapDecision(False, "main checkout seed stamp is stale, malformed, or graph-incomplete") - return BootstrapDecision( - False, - "main checkout has no validated reusable testmon state", - ) + continue + if isinstance(node, ast.AnnAssign) and node.value is None: + continue + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + if node.decorator_list or node.args.defaults or any(value is not None for value in node.args.kw_defaults): + return True + if _body_is_executable(node.body): + return True + continue + if isinstance(node, ast.ClassDef): + if node.decorator_list or node.bases or node.keywords or _body_is_executable(node.body): + return True + continue + if isinstance(node, ast.Assign): + return True + if isinstance(node, ast.AnnAssign): + if node.value is not None: + return True + continue + return True + return False -def _atomic_write_json(path: Path, payload: dict[str, object]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - tmp = path.with_name(f"{path.name}.{os.getpid()}.tmp") +def classify_source_ast(source_path: Path) -> ASTClassification: + """Classify whether a module contains executable runtime behavior.""" try: - tmp.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") - tmp.replace(path) - finally: - tmp.unlink(missing_ok=True) + tree = ast.parse(source_path.read_text(encoding="utf-8"), filename=str(source_path)) + except OSError: + return "source-unreadable" + except (SyntaxError, UnicodeDecodeError): + return "executable" + return "executable" if _body_is_executable(tree.body) else "declaration-only" + + +def _safe_relative_path(raw: str) -> str | None: + normalized = PurePosixPath(raw.replace("\\", "/")) + if normalized.is_absolute() or not normalized.parts or ".." in normalized.parts: + return None + return str(normalized) + + +def executable_python_paths(repo_root: Path, paths: Iterable[str]) -> tuple[str, ...]: + """Return changed Python paths whose runtime behavior needs graph edges.""" + root = repo_root.resolve() + executable: list[str] = [] + for raw in sorted(set(paths)): + relative = _safe_relative_path(raw) + if relative is None or not relative.endswith(".py"): + continue + source = root / relative + if not source.exists(): + continue + if not source.is_file() or classify_source_ast(source) != "declaration-only": + executable.append(relative) + return tuple(executable) + + +def _readonly_uri(path: Path) -> str: + return f"{path.resolve().as_uri()}?mode=ro" + +def _testmon_schema_version() -> int: + module = importlib.import_module("testmon.db") + value = getattr(module, "DATA_VERSION", None) + if not isinstance(value, int): + raise NativeTestmonRepairError("pytest-testmon does not expose an integer database version") + return value -def _atomic_write_stamp(seed_stamp: Path, stamp: TestmonSeedStamp) -> None: - _atomic_write_json(seed_stamp, stamp.as_dict()) +def _digest_nodeids(nodeids: Sequence[str]) -> str: + return hashlib.sha256("\n".join(nodeids).encode()).hexdigest() -def _rebind_run_receipt( - *, source: Path, destination: Path, checkout_root: Path, run_id: str, current_run_path: Path | None = None -) -> bool: - """Copy the run receipt while rebinding its checkout-local provenance.""" + +def inspect_native_testmon_environment( + data_path: Path, + *, + environment_name: str, + required_executable_paths: Sequence[str] = (), +) -> NativeTestmonState: + """Validate one native environment without interpreting plugin internals.""" + sidecars = tuple(Path(f"{data_path}{suffix}") for suffix in TESTMON_SIDECAR_SUFFIXES) + if not data_path.exists(): + if any(path.exists() or path.is_symlink() for path in sidecars): + return NativeTestmonState("invalid", "SQLite sidecars exist without the owned database") + return NativeTestmonState("absent", "native testmon database is absent") try: - payload = json.loads((source / "run.json").read_text(encoding="utf-8")) - if not isinstance(payload, Mapping) or payload.get("run_id") != run_id: - return False - source_root = payload.get("checkout_root") - if not isinstance(source_root, str) or Path(source_root).resolve() != source.parents[3].resolve(): - return False - payload_dict: dict[str, Any] = dict(payload) - payload_dict["checkout_root"] = str(checkout_root.resolve()) - payload_dict["artifact_dir"] = str(Path(".cache") / "verify" / "runs" / run_id) - environment = payload_dict.get("environment_fingerprint") - if isinstance(environment, dict): - environment["checkout_root"] = str(checkout_root.resolve()) - environment["verify_state_origin"] = str(checkout_root.resolve()) - _atomic_write_json(destination / "run.json", payload_dict) - _atomic_write_json( - current_run_path or checkout_root / ".cache" / "verify" / "current-run.json", - payload_dict, - ) - return True - except (OSError, UnicodeDecodeError, json.JSONDecodeError, TypeError, ValueError): - return False - - -def _atomic_copy_sqlite_db(src: Path, dst: Path) -> None: - """Copy a (possibly concurrently-written) sqlite db via the online backup API. - - `sqlite3.Connection.backup` is designed to snapshot a live database without - requiring an exclusive lock on the source, so this tolerates the main - checkout mid-write. The destination is built at a temp path and only - `rename`d into place once the backup completes, so a reader never observes - a partially-copied file at `dst`. - """ - dst.parent.mkdir(parents=True, exist_ok=True) - tmp = dst.with_name(f"{dst.name}.{os.getpid()}.tmp") - tmp.unlink(missing_ok=True) + mode = data_path.lstat().st_mode + except OSError as exc: + return NativeTestmonState("invalid", f"cannot inspect native testmon database: {exc}") + if not stat.S_ISREG(mode): + return NativeTestmonState("invalid", "native testmon database is not a regular file") try: - src_conn = sqlite3.connect(f"{src.resolve().as_uri()}?mode=ro", uri=True) + with sqlite3.connect(_readonly_uri(data_path), uri=True, timeout=10) as connection: + quick_check = connection.execute("PRAGMA quick_check").fetchone() + if quick_check is None or quick_check[0] != "ok": + return NativeTestmonState("invalid", "SQLite quick_check failed") + version_row = connection.execute("PRAGMA user_version").fetchone() + if version_row is None or version_row[0] != _testmon_schema_version(): + return NativeTestmonState("invalid", "pytest-testmon database schema version changed") + environment_rows = connection.execute( + "SELECT id FROM environment WHERE environment_name = ? ORDER BY id DESC", + (environment_name,), + ).fetchall() + if len(environment_rows) != 1: + reason = "native environment is absent" if not environment_rows else "native environment is ambiguous" + return NativeTestmonState("invalid", reason) + environment_id = int(environment_rows[0][0]) + nodeids = tuple( + row[0] + for row in connection.execute( + "SELECT test_name FROM test_execution WHERE environment_id = ? ORDER BY test_name", + (environment_id,), + ).fetchall() + if isinstance(row[0], str) and row[0] + ) + if not nodeids or len(nodeids) != len(set(nodeids)): + return NativeTestmonState("invalid", "native environment has no unique collected corpus") + uncovered = connection.execute( + """ + SELECT COUNT(*) + FROM test_execution AS execution + LEFT JOIN test_execution_file_fp AS edge ON edge.test_execution_id = execution.id + WHERE execution.environment_id = ? AND edge.test_execution_id IS NULL + """, + (environment_id,), + ).fetchone() + if uncovered is None or int(uncovered[0]) != 0: + return NativeTestmonState("invalid", "native environment has tests without dependency placeholders") + raw_files = connection.execute( + """ + SELECT DISTINCT fingerprint.filename + FROM test_execution AS execution + JOIN test_execution_file_fp AS edge ON edge.test_execution_id = execution.id + JOIN file_fp AS fingerprint ON fingerprint.id = edge.fingerprint_id + WHERE execution.environment_id = ? + """, + (environment_id,), + ).fetchall() + except (NativeTestmonRepairError, OSError, sqlite3.Error, TypeError, ValueError) as exc: + return NativeTestmonState("invalid", f"native testmon database is unreadable: {exc}") + fingerprinted = frozenset( + relative + for row in raw_files + if row and isinstance(row[0], str) + if (relative := _safe_relative_path(row[0])) is not None + ) + required = tuple(sorted(set(required_executable_paths))) + missing = tuple(path for path in required if path not in fingerprinted) + environment = NativeTestmonEnvironment( + name=environment_name, + corpus_count=len(nodeids), + corpus_digest=_digest_nodeids(nodeids), + nodeids=nodeids, + fingerprinted_files=fingerprinted, + ) + if missing: + return NativeTestmonState( + "invalid", + "changed executable modules are absent from the native dependency graph", + environment, + missing, + ) + return NativeTestmonState("valid", "native environment is current", environment) + + +def _owned_paths(repo_root: Path) -> tuple[Path, ...]: + data = repo_root.resolve() / TESTMON_DATA_RELPATH + return (data, *(Path(f"{data}{suffix}") for suffix in TESTMON_SIDECAR_SUFFIXES)) + + +def remove_invalid_native_testmon_state(repo_root: Path) -> tuple[Path, ...]: + """Remove only the exact checkout-owned SQLite file and known sidecars.""" + removed: list[Path] = [] + for path in _owned_paths(repo_root): try: - dst_conn = sqlite3.connect(tmp) - try: - src_conn.backup(dst_conn) - finally: - dst_conn.close() - finally: - src_conn.close() - tmp.replace(dst) - finally: - tmp.unlink(missing_ok=True) + mode = path.lstat().st_mode + except FileNotFoundError: + continue + except OSError as exc: + raise NativeTestmonRepairError(f"cannot inspect owned testmon path {path}: {exc}") from exc + if stat.S_ISDIR(mode): + raise NativeTestmonRepairError(f"refusing to remove directory at owned SQLite path {path}") + try: + path.unlink() + except OSError as exc: + raise NativeTestmonRepairError(f"cannot remove invalid owned testmon path {path}: {exc}") from exc + removed.append(path) + return tuple(removed) -def _publish_staged_bootstrap_files(*, staging_dir: Path, files: list[tuple[Path, Path | None]]) -> None: - """Publish a validated bootstrap as one rollback-capable file set.""" - backup_dir = staging_dir / "backups" - backup_dir.mkdir(parents=True, exist_ok=True) - backups: list[tuple[Path, Path]] = [] - published: list[Path] = [] +def _fsync_directory(path: Path) -> None: + descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) try: - for index, (destination, staged) in enumerate(files): - destination.parent.mkdir(parents=True, exist_ok=True) - backup = backup_dir / str(index) - if destination.exists(): - os.replace(destination, backup) - backups.append((destination, backup)) - if staged is not None: - os.replace(staged, destination) - published.append(destination) - except (OSError, ValueError): - for destination in reversed(published): - destination.unlink(missing_ok=True) - for destination, backup in reversed(backups): - if backup.exists(): - os.replace(backup, destination) - raise + os.fsync(descriptor) finally: - shutil.rmtree(backup_dir, ignore_errors=True) - - -def _copy_runtime_identity_inputs(*, source_root: Path, destination_root: Path) -> None: - """Mirror the inputs used to validate a staged testmon receipt.""" - for relative_path in ( - "uv.lock", - "pyproject.toml", - "pytest.ini", - "tox.ini", - "setup.cfg", - "tests/conftest.py", - ): - source = source_root / relative_path - if not source.is_file(): - continue - destination = destination_root / relative_path - destination.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(source, destination) + os.close(descriptor) -def bootstrap_testmon_seed_files( - decision: BootstrapDecision, +def _atomic_copy_sqlite_database( + source: Path, + destination: Path, *, - local_testmon_data: Path, - local_seed_stamp: Path, - local_seed_attempt: Path | None = None, - checkout_root: Path | None = None, - inherited_from: Path | None = None, -) -> bool: - """Perform the copy `decision` describes and report whether it was stamped.""" - if not decision.should_bootstrap: - return True - assert decision.main_testmon_data is not None - if decision.main_seed_stamp is None and decision.main_seed_attempt is None: - return False - if decision.main_seed_attempt is not None and local_seed_attempt is None: - return False - if checkout_root is None or inherited_from is None: - return False - stamp: TestmonSeedStamp | None = None - try: - source_root = (decision.main_checkout_root or inherited_from).resolve() - destination_root = checkout_root.resolve() - if source_root == destination_root: - return False - if inherited_from.resolve() != source_root: - return False - if decision.main_testmon_data.resolve() == local_testmon_data.resolve(): - return False - decision.main_testmon_data.resolve().relative_to(source_root) - local_testmon_data.resolve().relative_to(destination_root) - local_seed_stamp.resolve().relative_to(destination_root) - if local_seed_stamp.resolve() == local_testmon_data.resolve(): - return False - if local_seed_attempt is not None: - local_seed_attempt.resolve().relative_to(destination_root) - if local_seed_attempt.resolve() in {local_testmon_data.resolve(), local_seed_stamp.resolve()}: - return False - if decision.main_seed_stamp is not None: - decision.main_seed_stamp.resolve().relative_to(source_root) - if decision.main_seed_stamp.resolve() == decision.main_testmon_data.resolve(): - return False - if decision.main_seed_attempt is not None: - decision.main_seed_attempt.resolve().relative_to(source_root) - if decision.main_seed_attempt.resolve() == decision.main_testmon_data.resolve(): - return False - if decision.main_seed_stamp is not None: - stamp = validate_stamp( - decision.main_seed_stamp, - decision.main_testmon_data, - checkout_root=source_root, - protocol_version=decision.protocol_version, - ) - else: - assert decision.main_seed_attempt is not None - source = json.loads(decision.main_seed_attempt.read_text(encoding="utf-8")) - if not isinstance(source, dict): - return False - stamp = stamp_from_attempt( - source, - decision.main_testmon_data, - checkout_root=source_root, - protocol_version=decision.protocol_version, - published_marker=False, - ) - if stamp is None: - return False - except (OSError, UnicodeDecodeError, json.JSONDecodeError, ValueError, TypeError, sqlite3.Error): - return False - if stamp is None: - return False - staging_dir: Path | None = None + environment_name: str, + required_executable_paths: Sequence[str], +) -> None: + destination.parent.mkdir(parents=True, exist_ok=True) + temporary = destination.with_name(f".{destination.name}.copy-{os.getpid()}-{uuid.uuid4().hex}.tmp") try: - local_testmon_data.parent.mkdir(parents=True, exist_ok=True) - staging_dir = Path(tempfile.mkdtemp(prefix=".bootstrap-", dir=str(local_testmon_data.parent))) - staged_data = staging_dir / "testmondata" - staged_stamp = staging_dir / "seed.json" - staged_attempt = staging_dir / "seed-attempt.json" - staged_artifact = staging_dir / "artifact" - staged_current_run = staging_dir / "current-run.json" - - _atomic_copy_sqlite_db(decision.main_testmon_data, staged_data) - rebound = stamp.rebound(checkout_root=destination_root, inherited_from=source_root) - refreshed = refresh_stamp(rebound, staged_data) - if refreshed is None or refreshed.graph != rebound.graph: - return False - source_artifact = source_root / Path(stamp.artifact_dir) - destination_artifact = (destination_root / Path(refreshed.artifact_dir)).resolve() - destination_artifact.relative_to(destination_root) - if not _rebind_run_receipt( - source=source_artifact, - destination=staged_artifact, - checkout_root=destination_root, - run_id=refreshed.run_id, - current_run_path=staged_current_run, + with ( + sqlite3.connect(_readonly_uri(source), uri=True, timeout=60) as source_connection, + sqlite3.connect(temporary, timeout=60) as destination_connection, ): - return False - staged_attempt_path: Path | None = None - publishes_selection_attempt = decision.main_seed_attempt is not None and decision.selection_only - if publishes_selection_attempt: - assert decision.main_seed_attempt is not None - assert local_seed_attempt is not None - source_attempt = json.loads(decision.main_seed_attempt.read_text(encoding="utf-8")) - if not isinstance(source_attempt, dict): - return False - rebound_attempt = dict(source_attempt) - rebound_attempt["testmon_data"] = refreshed.testmon_data - rebound_attempt["artifact_dir"] = f".cache/verify/runs/{refreshed.run_id}" - rebound_attempt["binding"] = refreshed.binding.as_dict() - rebound_attempt["release_baseline_allowed"] = False - rebound_attempt["verification_scope"] = "affected" - validation_root = staging_dir / "validation" - validation_receipt = json.loads(staged_current_run.read_text(encoding="utf-8")) - if not isinstance(validation_receipt, dict): - return False - validation_receipt["checkout_root"] = str(validation_root.resolve()) - validation_receipt["artifact_dir"] = f".cache/verify/runs/{refreshed.run_id}" - _copy_runtime_identity_inputs(source_root=destination_root, destination_root=validation_root) - _atomic_write_json( - validation_root / ".cache" / "verify" / "runs" / refreshed.run_id / "run.json", - validation_receipt, - ) - validation_attempt = dict(rebound_attempt) - raw_binding = validation_attempt.get("binding") - if not isinstance(raw_binding, Mapping): - return False - validation_binding = dict(raw_binding) - validation_binding["checkout_root"] = str(validation_root.resolve()) - validation_attempt["binding"] = validation_binding - if ( - stamp_from_attempt( - validation_attempt, - staged_data, - checkout_root=validation_root, - protocol_version=decision.protocol_version, - ) - is None - ): - return False - _atomic_write_json(staged_attempt, rebound_attempt) - staged_attempt_path = staged_attempt - else: - _atomic_write_stamp(staged_stamp, refreshed) - publication_files: list[tuple[Path, Path | None]] = [ - (local_testmon_data, staged_data), - (destination_artifact / "run.json", staged_artifact / "run.json"), - (destination_root / ".cache" / "verify" / "current-run.json", staged_current_run), - (local_seed_stamp, None if publishes_selection_attempt else staged_stamp), - ] - if local_seed_attempt is not None: - publication_files.append((local_seed_attempt, staged_attempt_path)) - _publish_staged_bootstrap_files(staging_dir=staging_dir, files=publication_files) - return True - except (OSError, sqlite3.Error, TypeError, ValueError): - return False + source_connection.backup(destination_connection) + copied = inspect_native_testmon_environment( + temporary, + environment_name=environment_name, + required_executable_paths=required_executable_paths, + ) + if not copied.valid: + raise NativeTestmonRepairError(f"copied main-checkout database failed validation: {copied.reason}") + descriptor = os.open(temporary, os.O_RDONLY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + os.replace(temporary, destination) + _fsync_directory(destination.parent) + except (OSError, sqlite3.Error) as exc: + raise NativeTestmonRepairError(f"SQLite online backup failed: {exc}") from exc finally: - if staging_dir is not None: - shutil.rmtree(staging_dir, ignore_errors=True) + with contextlib.suppress(FileNotFoundError): + temporary.unlink() + for suffix in TESTMON_SIDECAR_SUFFIXES: + with contextlib.suppress(FileNotFoundError): + Path(f"{temporary}{suffix}").unlink() -def _git_worktree_info(repo_root: Path) -> tuple[bool, Path] | None: - """Return `(is_linked_worktree, main_checkout_path)`, or None if undeterminable. - - Same `git rev-parse --absolute-git-dir --git-common-dir` mechanism - `devtools/verify_worktree.py:inspect_worktree` uses: a linked worktree's - git-dir (`.git/worktrees/`) differs from the shared - git-common-dir; a main checkout's git-dir *is* the common-dir. - """ +def linked_worktree_info(repo_root: Path) -> tuple[bool, Path] | None: + """Return linked-worktree status and the main checkout path.""" try: result = subprocess.run( ["git", "-C", str(repo_root), "rev-parse", "--absolute-git-dir", "--git-common-dir"], capture_output=True, text=True, timeout=10, + env={**os.environ, "GIT_OPTIONAL_LOCKS": "0"}, ) except (OSError, subprocess.TimeoutExpired): return None @@ -531,83 +436,79 @@ def _git_worktree_info(repo_root: Path) -> tuple[bool, Path] | None: git_dir = Path(lines[0]).resolve() raw_common = Path(lines[1]) common_dir = raw_common.resolve() if raw_common.is_absolute() else (repo_root / raw_common).resolve() - is_linked = git_dir != common_dir - main_checkout = common_dir.parent - return is_linked, main_checkout + return git_dir != common_dir, common_dir.parent -def maybe_bootstrap_testmon_seed( +def prepare_native_testmon_environment( repo_root: Path, *, - testmon_data_relpath: str = TESTMON_DATA_RELPATH, - seed_stamp_relpath: str = TESTMON_SEED_STAMP_RELPATH, - seed_attempt_relpath: str = TESTMON_SEED_ATTEMPT_RELPATH, - protocol_version: int, -) -> str | None: - """Bootstrap `repo_root`'s testmon seed from its main checkout if warranted. - - Returns a one-line message to log on success, or ``None`` when no - bootstrap happened (not a linked worktree, already seeded locally, or the - main checkout has nothing valid to offer). Called from - `devtools/verify.py` before `_testmon_preflight`, so a freshly-bootstrapped - worktree passes that preflight instead of refusing. - """ - info = _git_worktree_info(repo_root) - if info is None: - return None - is_linked_worktree, main_checkout = info - if main_checkout == repo_root.resolve(): - return None - local_testmon_data = repo_root / testmon_data_relpath - local_seed_stamp = repo_root / seed_stamp_relpath - local_seed_attempt = repo_root / seed_attempt_relpath - main_testmon_data = main_checkout / testmon_data_relpath - main_seed_stamp = main_checkout / seed_stamp_relpath - main_seed_attempt = main_checkout / seed_attempt_relpath - decision = decide_testmon_bootstrap( - is_linked_worktree=is_linked_worktree, - local_testmon_data=local_testmon_data, - local_seed_stamp=local_seed_stamp, - main_testmon_data=main_testmon_data, - main_seed_stamp=main_seed_stamp, - protocol_version=protocol_version, - main_seed_attempt=main_seed_attempt, - main_checkout_root=main_checkout, - local_checkout_root=repo_root, - local_seed_attempt=local_seed_attempt, - ) - if not decision.should_bootstrap: - return None - stamped = bootstrap_testmon_seed_files( - decision, - local_testmon_data=local_testmon_data, - local_seed_stamp=local_seed_stamp, - local_seed_attempt=local_seed_attempt, - checkout_root=repo_root, - inherited_from=main_checkout, + required_executable_paths: Sequence[str] = (), + pytest_profile: str = "default", +) -> NativeTestmonPreparation: + """Repair derived local state and optionally reuse a matching main graph.""" + root = repo_root.resolve() + environment_name = testmon_environment_digest(root, pytest_profile=pytest_profile) + local_data = root / TESTMON_DATA_RELPATH + local = inspect_native_testmon_environment( + local_data, + environment_name=environment_name, + required_executable_paths=required_executable_paths, ) - if not stamped: - return ( - f"verify: refused pytest-testmon bootstrap into {local_testmon_data.parent}; " - "no local state was published because provenance validation failed" + info = linked_worktree_info(root) + linked = bool(info and info[0]) + main_checkout = info[1] if linked and info is not None else None + if local.valid: + return NativeTestmonPreparation(environment_name, "affected", local, None, (), linked, main_checkout) + + removed = remove_invalid_native_testmon_state(root) + copied_from: Path | None = None + if main_checkout is not None and main_checkout != root: + main_data = main_checkout / TESTMON_DATA_RELPATH + main = inspect_native_testmon_environment( + main_data, + environment_name=environment_name, + required_executable_paths=required_executable_paths, ) - if decision.main_seed_attempt is not None and decision.selection_only: - return ( - f"verify: bootstrapped pytest-testmon graph from main checkout {main_checkout} " - f"into {local_testmon_data.parent} as a selection-only attempt receipt (no seed.json)" - ) - return ( - f"verify: bootstrapped pytest-testmon seed from main checkout {main_checkout} " - f"into {local_testmon_data.parent} (worktree had no local seed)" - ) + if main.valid: + _atomic_copy_sqlite_database( + main_data, + local_data, + environment_name=environment_name, + required_executable_paths=required_executable_paths, + ) + copied_from = main_data + local = inspect_native_testmon_environment( + local_data, + environment_name=environment_name, + required_executable_paths=required_executable_paths, + ) + if not local.valid: + raise NativeTestmonRepairError(f"published native testmon copy is invalid: {local.reason}") + return NativeTestmonPreparation( + environment_name, + "affected", + local, + copied_from, + removed, + linked, + main_checkout, + ) + + return NativeTestmonPreparation(environment_name, "bootstrap", local, copied_from, removed, linked, main_checkout) __all__ = [ + "ASTClassification", + "NativeTestmonEnvironment", + "NativeTestmonPreparation", + "NativeTestmonRepairError", + "NativeTestmonState", "TESTMON_DATA_RELPATH", - "TESTMON_SEED_STAMP_RELPATH", - "TESTMON_SEED_ATTEMPT_RELPATH", - "BootstrapDecision", - "decide_testmon_bootstrap", - "bootstrap_testmon_seed_files", - "maybe_bootstrap_testmon_seed", + "classify_source_ast", + "executable_python_paths", + "inspect_native_testmon_environment", + "linked_worktree_info", + "prepare_native_testmon_environment", + "remove_invalid_native_testmon_state", + "testmon_environment_digest", ] diff --git a/devtools/testmon_mutation_proof.py b/devtools/testmon_mutation_proof.py deleted file mode 100644 index 3e80bc20ba..0000000000 --- a/devtools/testmon_mutation_proof.py +++ /dev/null @@ -1,288 +0,0 @@ -"""Executable anti-vacuity proof for the pytest-testmon affected gate. - -The proof deliberately operates on a disposable copy of an actual Polylogue -production module and its existing production-route test. It does not mock -pytest, testmon, changed paths, selection, or the verdict: pytest-testmon -records the dependency graph and the real test must fail after a semantic -source mutation. -""" - -from __future__ import annotations - -import argparse -import json -import os -import shutil -import signal -import sqlite3 -import subprocess -import sys -import tempfile -from collections.abc import Iterator -from contextlib import contextmanager -from dataclasses import asdict, dataclass, replace -from pathlib import Path -from typing import Any - -from devtools import repo_root - -_SOURCE = Path("polylogue/core/web_urls.py") -_UNRELATED_SOURCE = Path("polylogue/core/stats.py") -_TEST = Path("tests/unit/core/test_web_urls.py") -_UNRELATED_TEST = Path("tests/unit/core/test_percentile.py") -_TARGET_NODEID = "tests/unit/core/test_web_urls.py::test_chatgpt_url_bare" -_MUTATION_FROM = 'return f"https://chatgpt.com/c/{native_id}"' -_MUTATION_TO = 'return f"https://chatgpt.invalid/c/{native_id}"' -_TIMEOUT_S = 60.0 - - -@dataclass(frozen=True, slots=True) -class ProofResult: - ok: bool - target_nodeid: str - selected_nodeids: tuple[str, ...] - selected_count: int - total_seeded_nodes: int - mutation_exit_code: int - restored_exit_code: int - severed_edge_rejected: bool - unrelated_selected_count: int - cleanup_complete: bool - failure: str | None = None - - def to_dict(self) -> dict[str, object]: - return asdict(self) - - -def _copy_required_tree(source_root: Path, scratch: Path) -> None: - shutil.copytree(source_root / "polylogue" / "core", scratch / "polylogue" / "core") - for relative in ( - Path("polylogue/__init__.py"), - # polylogue/__init__.py unconditionally imports this before any - # sqlite3 import (must run first to swap in a modern bundled - # SQLite); omitting it breaks collection of every copied test. - Path("polylogue/_sqlite_compat.py"), - _TEST, - _UNRELATED_TEST, - Path("devtools/__init__.py"), - Path("devtools/pytest_progress_plugin.py"), - ): - destination = scratch / relative - destination.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(source_root / relative, destination) - - -def _pytest_env(scratch: Path, name: str) -> dict[str, str]: - env = os.environ.copy() - env["PYTHONPATH"] = str(scratch) - # The project devshell normally points testmon at .cache/testmon/testmondata. - # Keep this proof's graph inside its disposable copy instead. - env["TESTMON_DATAFILE"] = str(scratch / ".testmondata") - env["POLYLOGUE_PYTEST_SELECTION_PATH"] = str(scratch / ".artifacts" / f"{name}-selection.json") - env["POLYLOGUE_PYTEST_EVENTS_PATH"] = str(scratch / ".artifacts" / f"{name}-events.jsonl") - env["POLYLOGUE_PYTEST_SUMMARY_PATH"] = str(scratch / ".artifacts" / f"{name}-summary.json") - env["POLYLOGUE_PYTEST_SELECTION_NODEID_LIMIT"] = "100" - return env - - -def _run_pytest(scratch: Path, *, name: str, args: list[str]) -> subprocess.CompletedProcess[str]: - return subprocess.run( - [ - sys.executable, - "-m", - "pytest", - "-q", - "-p", - "devtools.pytest_progress_plugin", - # The scratch tree deliberately omits pyproject.toml (a disposable - # copy of only the files this proof needs), so pytest-randomly's - # default random ordering is uncontrolled here. That randomness - # is real: two trivial same-file tests whose production-module - # coverage happens to fingerprint identically can, depending on - # which runs first, leave one test's file_fp edge unrecorded -- - # not a testmon defect, just order nondeterminism this proof - # must not have. Force a fixed order so the seed graph (and the - # dependency-edge assertions that follow) are reproducible. - "-p", - "no:randomly", - *args, - ], - cwd=scratch, - env=_pytest_env(scratch, name), - text=True, - capture_output=True, - timeout=_TIMEOUT_S, - check=False, - ) - - -def _selection(scratch: Path, name: str) -> tuple[int, tuple[str, ...]]: - path = scratch / ".artifacts" / f"{name}-selection.json" - try: - payload = json.loads(path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): - return 0, () - selected = payload.get("selected_nodeids") if isinstance(payload, dict) else None - count = payload.get("selected_count") if isinstance(payload, dict) else None - if not isinstance(selected, list) or not all(isinstance(item, str) for item in selected): - selected = [] - return (count if isinstance(count, int) else len(selected), tuple(selected)) - - -def _mutate(path: Path, before: str, after: str) -> None: - source = path.read_text(encoding="utf-8") - if source.count(before) != 1: - raise RuntimeError(f"expected one mutation anchor in {path}") - path.write_text(source.replace(before, after), encoding="utf-8") - - -def _sever_target_edge(testmon_db: Path) -> None: - with sqlite3.connect(testmon_db) as connection: - cursor = connection.execute( - """ - DELETE FROM test_execution_file_fp - WHERE test_execution_id IN ( - SELECT id FROM test_execution WHERE test_name = ? - ) - AND fingerprint_id IN ( - SELECT id FROM file_fp WHERE filename LIKE ? - ) - """, - (_TARGET_NODEID, "%polylogue/core/web_urls.py"), - ) - if cursor.rowcount < 1: - raise RuntimeError("could not sever the named testmon dependency edge") - - -def _target_edge_present(testmon_db: Path) -> bool: - with sqlite3.connect(testmon_db) as connection: - row = connection.execute( - """ - SELECT 1 - FROM test_execution_file_fp AS edge - JOIN test_execution AS execution ON execution.id = edge.test_execution_id - JOIN file_fp AS fingerprint ON fingerprint.id = edge.fingerprint_id - WHERE execution.test_name = ? AND fingerprint.filename LIKE ? - LIMIT 1 - """, - (_TARGET_NODEID, "%polylogue/core/web_urls.py"), - ).fetchone() - return row is not None - - -@contextmanager -def _cleanup_on_interrupt() -> Iterator[None]: - # typeshed models the previous handler returned by ``signal.signal`` more - # narrowly than the handler value it accepts on restore. - previous: dict[int, Any] = {} - - def interrupt(signum: int, _frame: object) -> None: - raise InterruptedError(f"received signal {signum}") - - for signum in (int(signal.SIGINT), int(signal.SIGTERM)): - previous[signum] = signal.signal(signum, interrupt) - try: - yield - finally: - for restored_signal, handler in previous.items(): - signal.signal(restored_signal, handler) - - -def run_proof(*, source_root: Path | None = None) -> ProofResult: - """Run the real, bounded testmon dependency proof in a disposable copy.""" - source_root = (source_root or repo_root()).resolve() - scratch: Path | None = None - try: - with _cleanup_on_interrupt(), tempfile.TemporaryDirectory(prefix="polylogue-testmon-proof-") as temporary: - scratch = Path(temporary) - _copy_required_tree(source_root, scratch) - seed = _run_pytest(scratch, name="seed", args=["--testmon", "--testmon-noselect"]) - if seed.returncode != 0: - raise RuntimeError(f"real-route seed failed: {seed.stderr or seed.stdout}") - total_seeded_nodes, _ = _selection(scratch, "seed") - if total_seeded_nodes < 2: - raise RuntimeError("seed did not record a meaningful real-route test set") - if not _target_edge_present(scratch / ".testmondata"): - raise RuntimeError("seed did not record the named real-route dependency edge") - seed_graph = scratch / ".testmondata.seed" - shutil.copy2(scratch / ".testmondata", seed_graph) - - _mutate(scratch / _SOURCE, _MUTATION_FROM, _MUTATION_TO) - affected = _run_pytest(scratch, name="mutation", args=["--testmon", "--testmon-forceselect", "-n", "0"]) - selected_count, selected_nodeids = _selection(scratch, "mutation") - if ( - affected.returncode == 0 - or _TARGET_NODEID not in selected_nodeids - or _TARGET_NODEID not in (affected.stdout + affected.stderr) - ): - raise RuntimeError("semantic mutation was not selected and failed by the named real-route test") - - _mutate(scratch / _SOURCE, _MUTATION_TO, _MUTATION_FROM) - restored = _run_pytest(scratch, name="restored", args=[]) - if restored.returncode != 0: - raise RuntimeError("removing the semantic mutation did not return the real route to green") - - shutil.copy2(seed_graph, scratch / ".testmondata") - _sever_target_edge(scratch / ".testmondata") - if _target_edge_present(scratch / ".testmondata"): - raise RuntimeError("severed testmon dependency edge remained readable") - severed_edge_rejected = True - - shutil.copy2(seed_graph, scratch / ".testmondata") - # A byte-preserving rewrite is not a source change, so append a comment - # outside executable behavior solely to make testmon inspect an unrelated file. - unrelated = scratch / _UNRELATED_SOURCE - unrelated.write_text( - unrelated.read_text(encoding="utf-8") + "\n# testmon-proof unrelated change\n", encoding="utf-8" - ) - unrelated_run = _run_pytest( - scratch, name="unrelated", args=["--testmon", "--testmon-forceselect", "-n", "0"] - ) - del unrelated_run - unrelated_selected_count, _ = _selection(scratch, "unrelated") - if unrelated_selected_count >= total_seeded_nodes: - raise RuntimeError("unrelated production change fell back to the complete seeded route") - - proof = ProofResult( - ok=True, - target_nodeid=_TARGET_NODEID, - selected_nodeids=selected_nodeids, - selected_count=selected_count, - total_seeded_nodes=total_seeded_nodes, - mutation_exit_code=affected.returncode, - restored_exit_code=restored.returncode, - severed_edge_rejected=severed_edge_rejected, - unrelated_selected_count=unrelated_selected_count, - cleanup_complete=False, - ) - return replace(proof, cleanup_complete=not scratch.exists()) - except (InterruptedError, subprocess.TimeoutExpired, OSError, RuntimeError, sqlite3.Error) as exc: - return ProofResult( - ok=False, - target_nodeid=_TARGET_NODEID, - selected_nodeids=(), - selected_count=0, - total_seeded_nodes=0, - mutation_exit_code=-1, - restored_exit_code=-1, - severed_edge_rejected=False, - unrelated_selected_count=0, - cleanup_complete=scratch is not None and not scratch.exists(), - failure=str(exc), - ) - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser( - description="Prove real pytest-testmon affected selection with a semantic mutation." - ) - parser.add_argument("--json", action="store_true", help="Emit the complete proof receipt as JSON.") - args = parser.parse_args(argv) - result = run_proof() - if args.json: - print(json.dumps(result.to_dict(), indent=2)) - elif result.ok: - print(f"testmon mutation proof passed: {result.target_nodeid}") - else: - print(f"testmon mutation proof failed: {result.failure}", file=sys.stderr) - return 0 if result.ok else 1 diff --git a/devtools/testmon_state.py b/devtools/testmon_state.py deleted file mode 100644 index 8ac4bdaa14..0000000000 --- a/devtools/testmon_state.py +++ /dev/null @@ -1,1154 +0,0 @@ -"""Typed safety contract for reusable pytest-testmon state. - -The testmon database answers two different questions which must not share a -boolean marker: - -* did collection and dependency capture cover every promised node? -* did that run establish a green release baseline? - -A failed test can still have a complete dependency graph. Such a graph is -usable for affected-test selection, but it is never evidence that the suite -is releasable. This module is the single parser and SQLite validator used by -verification, worktree bootstrap, and the checkout guard. -""" - -from __future__ import annotations - -import contextlib -import hashlib -import importlib.metadata -import json -import os -import sqlite3 -from collections.abc import Mapping, Sequence -from dataclasses import dataclass, replace -from enum import StrEnum -from pathlib import Path -from typing import Any - - -class CollectionStatus(StrEnum): - COMPLETE = "complete" - INCOMPLETE = "incomplete" - - -class GraphStatus(StrEnum): - COMPLETE = "complete" - INCOMPLETE = "incomplete" - INVALID = "invalid" - - -class BaselineStatus(StrEnum): - GREEN = "green" - RED = "red" - - -class SeedAttemptOutcome(StrEnum): - """Terminal result of a seed attempt, separate from graph reusability.""" - - GREEN_RELEASE_BASELINE = "green-release-baseline" - RED_BASELINE = "red-baseline" - SELECTION_ONLY = "selection-only" - INCOMPLETE = "incomplete" - RESOURCE_TIMEOUT = "resource-timeout" - - -class SeedShardStatus(StrEnum): - """Durable state of one sequential pytest-testmon seed shard.""" - - PENDING = "pending" - RUNNING = "running" - COMPLETE = "complete" - INCOMPLETE = "incomplete" - - -class BindingMode(StrEnum): - EXACT = "exact" - RELATIVE_FILE_FINGERPRINTS = "relative-file-fingerprints" - - -class VerificationScope(StrEnum): - AFFECTED = "affected" - RELEASE_BASELINE = "release-baseline" - NARROW_TERMINAL = "narrow-terminal" - NON_TEST = "non-test" - - -class TerminalAuthorization(StrEnum): - NARROW_TERMINAL = "narrow-terminal" - - -# Pytest reports expected failures separately from ordinary skips/passes. They -# still finish the selected node and therefore make its dependency capture -# reusable. ``xpassed`` remains subject to pytest's configured strictness via -# the process exit code; it is not independently recast as a failure here. -TERMINAL_NODE_OUTCOMES = frozenset({"passed", "failed", "error", "skipped", "xfailed", "xpassed"}) -SUCCESSFUL_NODE_OUTCOMES = frozenset({"passed", "skipped", "xfailed", "xpassed"}) - - -def seed_shard_plan( - nodeids: Sequence[str], - *, - shard_size: int, - serial_nodeids: Sequence[str] = (), -) -> list[dict[str, Any]]: - """Partition a complete node set into stable, contiguous, serial shards.""" - if shard_size <= 0: - raise ValueError("testmon seed shard_size must be positive") - if not nodeids or any(not nodeid for nodeid in nodeids): - raise ValueError("testmon seed nodeids must be non-empty strings") - if len(set(nodeids)) != len(nodeids): - raise ValueError("testmon seed nodeids must be unique") - ordered = tuple(sorted(nodeids)) - serial = set(serial_nodeids) - if not serial.issubset(ordered): - raise ValueError("testmon serial shard nodes must belong to the seed corpus") - chunks: list[tuple[str, list[str]]] = [] - for offset in range(0, len(ordered), shard_size): - chunk = list(ordered[offset : offset + shard_size]) - parallel = [nodeid for nodeid in chunk if nodeid not in serial] - isolated = [nodeid for nodeid in chunk if nodeid in serial] - if parallel: - chunks.append(("parallel", parallel)) - if isolated: - chunks.append(("serial", isolated)) - return [ - { - "index": index, - "nodeids": chunk, - "nodeid_count": len(chunk), - "nodeid_digest": hashlib.sha256("\n".join(chunk).encode()).hexdigest(), - "execution_mode": mode, - "status": SeedShardStatus.PENDING.value, - "node_outcomes": [], - } - for index, (mode, chunk) in enumerate(chunks, start=1) - ] - - -def validate_seed_shard_ledger( - shards: object, - *, - expected_nodeids: Sequence[str], -) -> list[dict[str, Any]] | None: - """Validate the full shard ledger without granting release authority. - - Every shard owns a disjoint contiguous part of the sorted expected node - set. A completed shard carries an explicit terminal result for every node; - interrupted shards remain visible and are eligible for resume. - """ - if not isinstance(shards, list) or not shards: - return None - expected = tuple(sorted(expected_nodeids)) - if not expected or len(set(expected)) != len(expected): - return None - normalized: list[dict[str, Any]] = [] - observed: set[str] = set() - for index, raw in enumerate(shards, start=1): - if not isinstance(raw, Mapping) or raw.get("index") != index: - return None - nodeids = raw.get("nodeids") - if ( - not isinstance(nodeids, list) - or not nodeids - or any(not isinstance(nodeid, str) or not nodeid for nodeid in nodeids) - or nodeids != sorted(nodeids) - ): - return None - if raw.get("nodeid_count") != len(nodeids): - return None - if raw.get("nodeid_digest") != hashlib.sha256("\n".join(nodeids).encode()).hexdigest(): - return None - raw_status = raw.get("status") - if not isinstance(raw_status, str): - return None - try: - status = SeedShardStatus(raw_status) - except (TypeError, ValueError): - return None - outcomes = raw.get("node_outcomes") - if not isinstance(outcomes, list): - return None - outcome_by_node: dict[str, dict[str, Any]] = {} - for outcome in outcomes: - if not isinstance(outcome, Mapping): - return None - nodeid = outcome.get("nodeid") - state = outcome.get("outcome") - if not isinstance(nodeid, str) or nodeid not in nodeids or not isinstance(state, str): - return None - if nodeid in outcome_by_node: - return None - outcome_by_node[nodeid] = dict(outcome) - if status is SeedShardStatus.PENDING and outcomes: - return None - if status is SeedShardStatus.COMPLETE and ( - set(outcome_by_node) != set(nodeids) - or any(item.get("outcome") not in TERMINAL_NODE_OUTCOMES for item in outcome_by_node.values()) - ): - return None - if ( - status in {SeedShardStatus.RUNNING, SeedShardStatus.INCOMPLETE} - and outcomes - and set(outcome_by_node) != set(nodeids) - ): - return None - if observed.intersection(nodeids): - return None - normalized.append(dict(raw)) - observed.update(nodeids) - if observed != set(expected): - return None - return normalized - - -def seed_shard_ledger_is_terminal(shards: Sequence[Mapping[str, Any]]) -> bool: - """Return whether every planned shard completed with explicit node results.""" - return all(shard.get("status") == SeedShardStatus.COMPLETE.value for shard in shards) - - -@dataclass(frozen=True, slots=True) -class TestmonIdentity: - git_head: str | None - worktree_fingerprint: str - python: str - skip_slow: bool - lab: bool - git_tree: str | None = None - terminal_authorization: str | None = None - dependency_environment: str = "" - pytest_harness: str = "" - - @classmethod - def from_mapping(cls, value: Mapping[str, Any]) -> TestmonIdentity: - git_head = value.get("git_head") - if git_head is not None and (not isinstance(git_head, str) or not git_head): - raise ValueError("identity.git_head must be a non-empty string or null") - git_tree = value.get("git_tree") - if git_tree is not None and (not isinstance(git_tree, str) or not git_tree): - raise ValueError("identity.git_tree must be a non-empty string or null") - worktree = value.get("worktree_fingerprint") - python = value.get("python") - if not isinstance(worktree, str) or not worktree: - raise ValueError("identity.worktree_fingerprint must be a non-empty string") - if not isinstance(python, str) or not python: - raise ValueError("identity.python must be a non-empty string") - dependency_environment = value.get("dependency_environment") - pytest_harness = value.get("pytest_harness") - if dependency_environment is None: - dependency_environment = "" - if pytest_harness is None: - pytest_harness = "" - if not isinstance(dependency_environment, str): - raise ValueError("identity.dependency_environment must be a string") - if not isinstance(pytest_harness, str): - raise ValueError("identity.pytest_harness must be a string") - if not isinstance(value.get("skip_slow"), bool) or not isinstance(value.get("lab"), bool): - raise ValueError("identity selection flags must be booleans") - terminal_authorization = value.get("terminal_authorization") - if terminal_authorization is not None and terminal_authorization not in { - authorization.value for authorization in TerminalAuthorization - }: - raise ValueError("identity.terminal_authorization is invalid") - return cls( - git_head, - worktree, - python, - value["skip_slow"], - value["lab"], - git_tree, - terminal_authorization, - dependency_environment, - pytest_harness, - ) - - def as_dict(self) -> dict[str, Any]: - return { - "git_head": self.git_head, - "worktree_fingerprint": self.worktree_fingerprint, - "python": self.python, - "skip_slow": self.skip_slow, - "lab": self.lab, - "git_tree": self.git_tree, - "terminal_authorization": self.terminal_authorization, - "dependency_environment": self.dependency_environment, - "pytest_harness": self.pytest_harness, - } - - -def _fingerprint_files(checkout_root: Path, relative_paths: Sequence[str]) -> str: - """Hash named checkout inputs, preserving absent inputs as typed state.""" - digest = hashlib.sha256() - for relative_path in relative_paths: - digest.update(relative_path.encode()) - digest.update(b"\0") - try: - contents = (checkout_root / relative_path).read_bytes() - except OSError: - digest.update(b"missing") - else: - digest.update(contents) - digest.update(b"\0") - return digest.hexdigest() - - -def _installed_distributions() -> tuple[tuple[str, str], ...] | None: - """Return the active environment's normalized installed distributions.""" - try: - distributions = [] - for distribution in importlib.metadata.distributions(): - try: - name = distribution.metadata["Name"] - except KeyError: - return None - version = distribution.version - if not name or not version: - return None - distributions.append((name.casefold(), version)) - except (OSError, TypeError, ValueError, importlib.metadata.PackageNotFoundError): - return None - return tuple(sorted(distributions)) - - -def testmon_runtime_identity(checkout_root: Path) -> tuple[str, str] | None: - """Identify the lock, installed dependencies, and pytest execution harness. - - A testmon graph is reusable only under this exact dependency environment. - The application lock catches declared changes; installed distributions and - pytest-specific configuration catch a stale or differently provisioned - virtual environment even when ``sys.version`` is unchanged. - """ - distributions = _installed_distributions() - if distributions is None: - return None - normalized_root = checkout_root.resolve() - dependency_payload = { - "lock_inputs": _fingerprint_files(normalized_root, ("uv.lock", "pyproject.toml")), - "distributions": distributions, - } - harness_payload = { - "configuration": _fingerprint_files( - normalized_root, - ("pyproject.toml", "pytest.ini", "tox.ini", "setup.cfg", "tests/conftest.py"), - ), - "environment": { - key: os.environ.get(key) - for key in ( - "PYTEST_ADDOPTS", - "PYTEST_DISABLE_PLUGIN_AUTOLOAD", - "PYTEST_PLUGINS", - "HYPOTHESIS_PROFILE", - "POLYLOGUE_CI", - ) - }, - "pytest_distributions": tuple( - item for item in distributions if item[0] in {"pytest", "pytest-testmon", "pytest-xdist", "pluggy"} - ), - } - return ( - hashlib.sha256(json.dumps(dependency_payload, sort_keys=True, separators=(",", ":")).encode()).hexdigest(), - hashlib.sha256(json.dumps(harness_payload, sort_keys=True, separators=(",", ":")).encode()).hexdigest(), - ) - - -def _identity_matches_runtime(identity: TestmonIdentity, *, checkout_root: Path, protocol_version: int) -> bool: - """Keep pre-binding protocol receipts parseable but never reusable today.""" - if protocol_version < 5: - return True - runtime_identity = testmon_runtime_identity(checkout_root) - return ( - runtime_identity is not None - and ( - identity.dependency_environment, - identity.pytest_harness, - ) - == runtime_identity - ) - - -@dataclass(frozen=True, slots=True) -class TestmonBinding: - mode: BindingMode - checkout_root: str - source_checkout_root: str | None = None - - @classmethod - def from_mapping(cls, value: Mapping[str, Any]) -> TestmonBinding: - raw_mode = value.get("mode") - if not isinstance(raw_mode, str): - raise ValueError("binding.mode is invalid") - try: - mode = BindingMode(raw_mode) - except ValueError as exc: - raise ValueError("binding.mode is invalid") from exc - checkout_root = value.get("checkout_root") - source = value.get("source_checkout_root") - if not isinstance(checkout_root, str) or not checkout_root: - raise ValueError("binding.checkout_root must be a non-empty string") - if not Path(checkout_root).is_absolute(): - raise ValueError("binding.checkout_root must be absolute") - if source is not None and (not isinstance(source, str) or not source): - raise ValueError("binding.source_checkout_root must be a non-empty string or null") - if source is not None and not Path(source).is_absolute(): - raise ValueError("binding.source_checkout_root must be absolute") - if mode is BindingMode.EXACT and source is not None: - raise ValueError("exact bindings cannot have a source checkout") - if mode is BindingMode.RELATIVE_FILE_FINGERPRINTS: - if source is None: - raise ValueError("rebound bindings require a source checkout") - if Path(source).resolve() == Path(checkout_root).resolve(): - raise ValueError("rebound binding source and destination must differ") - return cls(mode, checkout_root, source) - - def as_dict(self) -> dict[str, Any]: - return { - "mode": self.mode.value, - "checkout_root": self.checkout_root, - "source_checkout_root": self.source_checkout_root, - } - - -@dataclass(frozen=True, slots=True) -class GraphInspection: - status: GraphStatus - recorded_count: int - dependency_edge_count: int - missing_nodeids: tuple[str, ...] - orphan_execution_edges: int - orphan_fingerprint_edges: int - error: str | None - failed_nodeids: tuple[str, ...] - - @property - def usable_for_selection(self) -> bool: - return self.status is GraphStatus.COMPLETE - - def as_dict(self) -> dict[str, Any]: - return { - "status": self.status.value, - "recorded_count": self.recorded_count, - "dependency_edge_count": self.dependency_edge_count, - "missing_nodeids": list(self.missing_nodeids), - "orphan_execution_edges": self.orphan_execution_edges, - "orphan_fingerprint_edges": self.orphan_fingerprint_edges, - "error": self.error, - "failed_nodeids": list(self.failed_nodeids), - } - - -@dataclass(frozen=True, slots=True) -class TestmonSeedStamp: - protocol_version: int - collection_status: CollectionStatus - expected_nodeids: tuple[str, ...] - selected_nodeids_omitted: int - baseline_status: BaselineStatus - release_baseline_allowed: bool - baseline_exit_code: int - graph: GraphInspection - identity: TestmonIdentity - binding: TestmonBinding - testmon_data: str - run_id: str - artifact_dir: str - - @property - def affected_selection_allowed(self) -> bool: - return ( - self.collection_status is CollectionStatus.COMPLETE - and self.selected_nodeids_omitted == 0 - and self.graph.usable_for_selection - ) - - @property - def expected_digest(self) -> str: - return hashlib.sha256("\n".join(sorted(self.expected_nodeids)).encode()).hexdigest() - - def as_dict(self) -> dict[str, Any]: - return { - "protocol_version": self.protocol_version, - "status": "usable", - "collection": { - "status": self.collection_status.value, - "expected_count": len(self.expected_nodeids), - "expected_digest": self.expected_digest, - "selected_nodeids": list(self.expected_nodeids), - "selected_nodeids_omitted": self.selected_nodeids_omitted, - }, - "baseline": { - "status": self.baseline_status.value, - "exit_code": self.baseline_exit_code, - "release_baseline_allowed": self.release_baseline_allowed, - }, - "graph": self.graph.as_dict(), - "identity": self.identity.as_dict(), - "binding": self.binding.as_dict(), - "testmon_data": self.testmon_data, - "run_id": self.run_id, - "artifact_dir": self.artifact_dir, - } - - @classmethod - def from_mapping(cls, value: Mapping[str, Any], *, protocol_version: int) -> TestmonSeedStamp: - if value.get("protocol_version") != protocol_version or value.get("status") != "usable": - raise ValueError("seed stamp is not a current usable testmon stamp") - collection = value.get("collection") - baseline = value.get("baseline") - graph = value.get("graph") - identity = value.get("identity") - binding = value.get("binding") - if not all(isinstance(item, Mapping) for item in (collection, baseline, graph, identity, binding)): - raise ValueError("seed stamp has incomplete typed state") - assert isinstance(collection, Mapping) - assert isinstance(baseline, Mapping) - assert isinstance(graph, Mapping) - assert isinstance(identity, Mapping) - assert isinstance(binding, Mapping) - if collection.get("status") != CollectionStatus.COMPLETE.value: - raise ValueError("seed stamp collection is not complete") - nodeids = collection.get("selected_nodeids") - if ( - not isinstance(nodeids, list) - or not nodeids - or any(not isinstance(item, str) or not item for item in nodeids) - ): - raise ValueError("seed stamp selected nodeids are missing or malformed") - if len(set(nodeids)) != len(nodeids): - raise ValueError("seed stamp selected nodeids are not unique") - omitted = collection.get("selected_nodeids_omitted") - if not isinstance(omitted, int) or isinstance(omitted, bool) or omitted != 0: - raise ValueError("seed stamp has controlled collection omissions") - if collection.get("expected_count") != len(nodeids): - raise ValueError("seed stamp expected count does not match selected nodeids") - expected_digest = hashlib.sha256("\n".join(sorted(nodeids)).encode()).hexdigest() - if collection.get("expected_digest") != expected_digest: - raise ValueError("seed stamp expected nodeid digest is stale") - raw_baseline_status = baseline.get("status") - if not isinstance(raw_baseline_status, str): - raise ValueError("seed stamp baseline status is invalid") - try: - baseline_status = BaselineStatus(raw_baseline_status) - except ValueError as exc: - raise ValueError("seed stamp baseline status is invalid") from exc - exit_code = baseline.get("exit_code") - release_allowed = baseline.get("release_baseline_allowed") - if not isinstance(exit_code, int) or isinstance(exit_code, bool) or not isinstance(release_allowed, bool): - raise ValueError("seed stamp baseline fields are malformed") - if release_allowed != (baseline_status is BaselineStatus.GREEN): - raise ValueError("release permission does not match baseline status") - if baseline_status is BaselineStatus.GREEN and exit_code != 0: - raise ValueError("green seed stamp must have a zero exit code") - graph_status = graph.get("status") - if not isinstance(graph_status, str): - raise ValueError("seed stamp graph status is invalid") - try: - status = GraphStatus(graph_status) - except ValueError as exc: - raise ValueError("seed stamp graph status is invalid") from exc - if status is not GraphStatus.COMPLETE: - raise ValueError("seed stamp graph is not complete") - graph_expected = [ - "recorded_count", - "dependency_edge_count", - "orphan_execution_edges", - "orphan_fingerprint_edges", - ] - graph_counts = {key: graph.get(key) for key in graph_expected} - if any(not isinstance(item, int) or isinstance(item, bool) or item < 0 for item in graph_counts.values()): - raise ValueError("seed stamp graph counts are malformed") - dependency_edge_count = graph.get("dependency_edge_count") - if not isinstance(dependency_edge_count, int) or isinstance(dependency_edge_count, bool): - raise ValueError("seed stamp dependency edge count is malformed") - if ( - graph.get("recorded_count") != len(nodeids) - or dependency_edge_count < len(nodeids) - or graph.get("orphan_execution_edges") != 0 - or graph.get("orphan_fingerprint_edges") != 0 - ): - raise ValueError("seed stamp graph coverage is incomplete") - missing_nodeids = graph.get("missing_nodeids") - if ( - not isinstance(missing_nodeids, list) - or any(not isinstance(item, str) or not item for item in missing_nodeids) - or not set(missing_nodeids).issubset(nodeids) - ): - raise ValueError("seed stamp missing-node ledger is malformed") - if graph.get("error") is not None or missing_nodeids: - raise ValueError("seed stamp graph has missing or erroneous nodes") - graph_nodeids = graph.get("failed_nodeids", []) - if ( - not isinstance(graph_nodeids, list) - or any(not isinstance(item, str) or not item for item in graph_nodeids) - or not set(graph_nodeids).issubset(nodeids) - or len(set(graph_nodeids)) != len(graph_nodeids) - ): - raise ValueError("seed stamp graph failure ledger is malformed") - if baseline_status is BaselineStatus.GREEN and graph_nodeids: - raise ValueError("green seed stamp cannot contain failed graph nodes") - testmon_data = value.get("testmon_data") - run_id = value.get("run_id") - artifact_dir = value.get("artifact_dir") - if not all(isinstance(item, str) and item for item in (testmon_data, run_id, artifact_dir)): - raise ValueError("seed stamp provenance is incomplete") - assert isinstance(testmon_data, str) - assert isinstance(run_id, str) - assert isinstance(artifact_dir, str) - typed_binding = TestmonBinding.from_mapping(binding) - if not _is_bound_run_artifact( - artifact_dir, - checkout_root=Path(typed_binding.checkout_root), - run_id=run_id, - ): - raise ValueError("seed stamp artifact directory is not checkout-bound") - return cls( - protocol_version, - CollectionStatus.COMPLETE, - tuple(nodeids), - 0, - baseline_status, - release_allowed, - exit_code, - GraphInspection( - status, - graph["recorded_count"], - dependency_edge_count, - tuple(graph.get("missing_nodeids", [])), - graph["orphan_execution_edges"], - graph["orphan_fingerprint_edges"], - graph.get("error"), - tuple(graph_nodeids), - ), - TestmonIdentity.from_mapping(identity), - typed_binding, - testmon_data, - run_id, - artifact_dir, - ) - - def rebound(self, *, checkout_root: Path, inherited_from: Path) -> TestmonSeedStamp: - return replace( - self, - binding=TestmonBinding( - BindingMode.RELATIVE_FILE_FINGERPRINTS, - str(checkout_root.resolve()), - str(inherited_from.resolve()), - ), - ) - - -def file_fingerprint(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as handle: - for chunk in iter(lambda: handle.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def _is_bound_run_artifact(raw: object, *, checkout_root: Path, run_id: str) -> bool: - if not isinstance(raw, str) or not raw or not run_id: - return False - path = Path(raw) - if path.is_absolute() or path.parts[:3] != (".cache", "verify", "runs"): - return False - if path.parts[3:] != (run_id,): - return False - try: - artifact_dir = (checkout_root / path).resolve() - artifact_dir.relative_to((checkout_root / ".cache" / "verify" / "runs" / run_id).resolve()) - receipt = json.loads((artifact_dir / "run.json").read_text(encoding="utf-8")) - if not isinstance(receipt, Mapping): - return False - return ( - receipt.get("run_id") == run_id - and isinstance(receipt.get("checkout_root"), str) - and Path(receipt["checkout_root"]).resolve() == checkout_root.resolve() - and receipt.get("artifact_dir") == str(Path(".cache") / "verify" / "runs" / run_id) - ) - except (OSError, UnicodeDecodeError, json.JSONDecodeError, TypeError, ValueError): - return False - - -def seed_marker_is_checkout_bound( - marker_path: Path, - *, - checkout_root: Path, - protocol_version: int, -) -> bool: - """Validate only the typed ownership envelope of a seed marker. - - This intentionally does not open or fingerprint SQLite. The checkout guard - uses this cheap predicate for every entrypoint; verify preflight performs - the exhaustive graph validation before authorizing selection. - """ - try: - payload = json.loads(marker_path.read_text(encoding="utf-8")) - if not isinstance(payload, Mapping): - return False - stamp = TestmonSeedStamp.from_mapping(payload, protocol_version=protocol_version) - return Path(stamp.binding.checkout_root).resolve() == checkout_root.resolve() - except (OSError, UnicodeDecodeError, json.JSONDecodeError, TypeError, ValueError): - return False - - -def attempt_is_checkout_bound( - attempt: Mapping[str, Any], - *, - checkout_root: Path, - protocol_version: int, - reusable_only: bool = True, -) -> bool: - """Check a seed-attempt receipt without inspecting its SQLite graph.""" - allowed_statuses = {"reusable", "complete"} if reusable_only else {"running", "incomplete", "reusable", "complete"} - if attempt.get("protocol_version") != protocol_version or attempt.get("status") not in allowed_statuses: - return False - identity = attempt.get("identity") - expected = attempt.get("expected_nodeids") - selection = attempt.get("selection") - if not isinstance(identity, Mapping) or not isinstance(expected, list) or not isinstance(selection, Mapping): - return False - if not expected or any(not isinstance(nodeid, str) or not nodeid for nodeid in expected): - return False - if len(set(expected)) != len(expected): - return False - if ( - not isinstance(attempt.get("expected_count"), int) - or isinstance(attempt.get("expected_count"), bool) - or attempt.get("expected_count") != len(expected) - ): - return False - expected_digest = hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest() - if attempt.get("expected_digest") != expected_digest: - return False - try: - typed_identity = TestmonIdentity.from_mapping(identity) - except ValueError: - return False - if not _identity_matches_runtime(typed_identity, checkout_root=checkout_root, protocol_version=protocol_version): - return False - omitted = selection.get("selected_nodeids_omitted") - selected_count = selection.get("selected_count") - if ( - not isinstance(omitted, int) - or isinstance(omitted, bool) - or omitted != 0 - or not isinstance(selected_count, int) - or isinstance(selected_count, bool) - or selected_count != len(expected) - ): - return False - recorded_data = attempt.get("testmon_data") - run_id = attempt.get("run_id") - artifact_dir = attempt.get("artifact_dir") - if ( - not isinstance(recorded_data, str) - or not recorded_data - or not isinstance(run_id, str) - or not run_id - or not isinstance(artifact_dir, str) - or not artifact_dir - or not _is_bound_run_artifact(artifact_dir, checkout_root=checkout_root, run_id=run_id) - ): - return False - raw_binding = attempt.get("binding") - if raw_binding is None: - binding = TestmonBinding(BindingMode.EXACT, str(checkout_root.resolve())) - elif isinstance(raw_binding, Mapping): - try: - binding = TestmonBinding.from_mapping(raw_binding) - except ValueError: - return False - else: - return False - if Path(binding.checkout_root).resolve() != checkout_root.resolve(): - return False - raw_permission = attempt.get("release_baseline_allowed") - if raw_permission is not None and not isinstance(raw_permission, bool): - return False - raw_scope = attempt.get("verification_scope") - if raw_scope is not None and raw_scope not in {scope.value for scope in VerificationScope}: - return False - if reusable_only and raw_permission is not False: - return False - if reusable_only: - outcomes = attempt.get("node_outcomes") - if not isinstance(outcomes, list) or len(outcomes) != len(expected): - return False - nodeids = [item.get("nodeid") for item in outcomes if isinstance(item, Mapping)] - if len(nodeids) != len(outcomes) or set(nodeids) != set(expected) or len(set(nodeids)) != len(nodeids): - return False - if any(item.get("outcome") not in TERMINAL_NODE_OUTCOMES for item in outcomes): - return False - return True - - -def inspect_testmon_database(path: Path, expected_nodeids: Sequence[str]) -> GraphInspection: - """Validate the real testmon schema and every expected dependency edge.""" - expected = tuple(expected_nodeids) - if not path.is_file() or not expected or len(set(expected)) != len(expected): - return GraphInspection( - GraphStatus.INCOMPLETE, 0, 0, expected, 0, 0, "missing or malformed expected nodeids", () - ) - try: - with contextlib.closing(sqlite3.connect(f"{path.resolve().as_uri()}?mode=ro", uri=True)) as connection: - if connection.execute("PRAGMA integrity_check").fetchone() != ("ok",): - return GraphInspection(GraphStatus.INVALID, 0, 0, expected, 0, 0, "sqlite integrity check failed", ()) - required = {"test_execution", "test_execution_file_fp", "file_fp"} - tables = {str(row[0]) for row in connection.execute("select name from sqlite_master where type='table'")} - if not required <= tables: - return GraphInspection(GraphStatus.INVALID, 0, 0, expected, 0, 0, "testmon schema is incomplete", ()) - required_columns = { - "test_execution": {"id", "test_name", "failed"}, - "test_execution_file_fp": {"test_execution_id", "fingerprint_id"}, - "file_fp": {"id", "filename", "fsha"}, - } - for table, columns in required_columns.items(): - actual = {str(row[1]) for row in connection.execute(f"pragma table_info({table})")} - if not columns <= actual: - return GraphInspection( - GraphStatus.INVALID, - 0, - 0, - expected, - 0, - 0, - f"testmon schema is missing columns from {table}", - (), - ) - executions = connection.execute( - "select id, test_name, failed from test_execution where test_name is not null" - ).fetchall() - latest: dict[str, tuple[int, bool]] = {} - execution_ids: set[int] = set() - for execution_id, test_name, failed in executions: - if ( - not isinstance(execution_id, int) - or isinstance(execution_id, bool) - or execution_id <= 0 - or not isinstance(test_name, str) - or not test_name - or not isinstance(failed, int) - or isinstance(failed, bool) - or failed not in (0, 1) - ): - return GraphInspection( - GraphStatus.INVALID, 0, 0, expected, 0, 0, "testmon execution row is malformed", () - ) - if execution_id in execution_ids: - return GraphInspection( - GraphStatus.INVALID, 0, 0, expected, 0, 0, "testmon execution ids are not unique", () - ) - execution_ids.add(execution_id) - name = test_name - if name not in expected: - grouped = [nodeid for nodeid in expected if name.startswith(nodeid + "@")] - name = max(grouped, key=len, default=name) - prior = latest.get(name) - if prior is None or execution_id > prior[0]: - latest[name] = (execution_id, failed == 1) - missing = tuple(sorted(set(expected) - latest.keys())) - expected_ids = {latest[nodeid][0] for nodeid in expected if nodeid in latest} - edge_rows = connection.execute( - "select test_execution_id, fingerprint_id from test_execution_file_fp" - ).fetchall() - fingerprints = connection.execute("select id, filename, fsha from file_fp").fetchall() - fingerprint_ids: set[int] = set() - for fingerprint_id, filename, fsha in fingerprints: - if ( - not isinstance(fingerprint_id, int) - or isinstance(fingerprint_id, bool) - or fingerprint_id <= 0 - or not isinstance(filename, str) - or not filename - or Path(filename).is_absolute() - or ".." in Path(filename).parts - or not isinstance(fsha, str) - or not fsha - ): - return GraphInspection( - GraphStatus.INVALID, 0, 0, expected, 0, 0, "testmon fingerprint row is malformed", () - ) - if fingerprint_id in fingerprint_ids: - return GraphInspection( - GraphStatus.INVALID, 0, 0, expected, 0, 0, "testmon fingerprint ids are not unique", () - ) - fingerprint_ids.add(fingerprint_id) - for execution_id, fingerprint_id in edge_rows: - if ( - not isinstance(execution_id, int) - or isinstance(execution_id, bool) - or execution_id <= 0 - or not isinstance(fingerprint_id, int) - or isinstance(fingerprint_id, bool) - or fingerprint_id <= 0 - ): - return GraphInspection( - GraphStatus.INVALID, 0, 0, expected, 0, 0, "testmon dependency edge is malformed", () - ) - orphan_execution_edges = sum(1 for row in edge_rows if row[0] not in execution_ids) - orphan_fingerprint_edges = sum(1 for row in edge_rows if row[1] not in fingerprint_ids) - edge_counts: dict[int, int] = {} - for execution_id, _fingerprint_id in edge_rows: - edge_counts[execution_id] = edge_counts.get(execution_id, 0) + 1 - uncovered = tuple( - sorted(nodeid for nodeid in expected if nodeid in latest and edge_counts.get(latest[nodeid][0], 0) == 0) - ) - missing = tuple(sorted(set(missing) | set(uncovered))) - failed = tuple(sorted(nodeid for nodeid in expected if nodeid in latest and latest[nodeid][1])) - edge_count = sum(edge_counts.get(execution_id, 0) for execution_id in expected_ids) - status = ( - GraphStatus.COMPLETE - if not missing and not orphan_execution_edges and not orphan_fingerprint_edges - else GraphStatus.INCOMPLETE - ) - return GraphInspection( - status, - len(expected) - len(missing), - edge_count, - missing, - orphan_execution_edges, - orphan_fingerprint_edges, - None, - failed, - ) - except (OSError, sqlite3.Error, UnicodeError, TypeError, ValueError, OverflowError) as exc: - return GraphInspection(GraphStatus.INVALID, 0, 0, expected, 0, 0, str(exc), ()) - - -def validate_stamp( - stamp_path: Path, - data_path: Path, - *, - checkout_root: Path, - protocol_version: int, -) -> TestmonSeedStamp | None: - """Parse and re-check a stamp against its current SQLite graph.""" - try: - payload = json.loads(stamp_path.read_text(encoding="utf-8")) - if not isinstance(payload, Mapping): - return None - stamp = TestmonSeedStamp.from_mapping(payload, protocol_version=protocol_version) - if not stamp.release_baseline_allowed: - return None - if ( - stamp.identity.skip_slow - and stamp.identity.terminal_authorization != TerminalAuthorization.NARROW_TERMINAL.value - ): - return None - if not _identity_matches_runtime( - stamp.identity, checkout_root=checkout_root, protocol_version=protocol_version - ): - return None - if Path(stamp.binding.checkout_root).resolve() != checkout_root.resolve(): - return None - if file_fingerprint(data_path) != stamp.testmon_data: - return None - graph = inspect_testmon_database(data_path, stamp.expected_nodeids) - if graph != stamp.graph: - return None - return stamp - except (OSError, TypeError, ValueError, json.JSONDecodeError): - return None - - -def refresh_stamp(stamp: TestmonSeedStamp, data_path: Path) -> TestmonSeedStamp | None: - """Refresh mutable SQLite provenance after a successful affected run.""" - graph = inspect_testmon_database(data_path, stamp.expected_nodeids) - if not graph.usable_for_selection: - return None - try: - return replace(stamp, graph=graph, testmon_data=file_fingerprint(data_path)) - except OSError: - return None - - -def stamp_from_attempt( - attempt: Mapping[str, Any], - data_path: Path, - *, - checkout_root: Path, - protocol_version: int, - published_marker: bool = True, -) -> TestmonSeedStamp | None: - """Parse a complete attempt, withholding release authority until publication.""" - if attempt.get("protocol_version") != protocol_version or attempt.get("status") not in {"reusable", "complete"}: - return None - if protocol_version >= 6: - raw_outcome = attempt.get("outcome") - if not isinstance(raw_outcome, str): - return None - try: - outcome = SeedAttemptOutcome(raw_outcome) - except ValueError: - return None - if attempt.get("status") == "complete" and outcome is not SeedAttemptOutcome.GREEN_RELEASE_BASELINE: - return None - if attempt.get("status") == "reusable" and outcome not in { - SeedAttemptOutcome.RED_BASELINE, - SeedAttemptOutcome.SELECTION_ONLY, - }: - return None - selection = attempt.get("selection") - expected = attempt.get("expected_nodeids") - identity = attempt.get("identity") - if not isinstance(selection, Mapping) or not isinstance(expected, list) or not isinstance(identity, Mapping): - return None - assert isinstance(selection, Mapping) - assert isinstance(identity, Mapping) - omitted = selection.get("selected_nodeids_omitted") - selected_count = selection.get("selected_count") - if ( - not isinstance(omitted, int) - or isinstance(omitted, bool) - or omitted != 0 - or not isinstance(selected_count, int) - or isinstance(selected_count, bool) - or selected_count != len(expected) - or not expected - or any(not isinstance(nodeid, str) or not nodeid for nodeid in expected) - or len(set(expected)) != len(expected) - ): - return None - expected_count = attempt.get("expected_count") - if not isinstance(expected_count, int) or isinstance(expected_count, bool) or expected_count != len(expected): - return None - if protocol_version >= 7: - shards = validate_seed_shard_ledger(attempt.get("shards"), expected_nodeids=expected) - if shards is None or not seed_shard_ledger_is_terminal(shards): - return None - expected_digest = attempt.get("expected_digest") - if ( - not isinstance(expected_digest, str) - or expected_digest != hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest() - ): - return None - recorded_data = attempt.get("testmon_data") - if not isinstance(recorded_data, str) or not recorded_data or not data_path.is_file(): - return None - try: - if file_fingerprint(data_path) != recorded_data: - return None - except OSError: - return None - run_id = attempt.get("run_id") - artifact_dir = attempt.get("artifact_dir") - if not isinstance(run_id, str) or not run_id or not isinstance(artifact_dir, str) or not artifact_dir: - return None - if not _is_bound_run_artifact(artifact_dir, checkout_root=checkout_root, run_id=run_id): - return None - outcomes = attempt.get("node_outcomes") - if not isinstance(outcomes, list) or len(outcomes) != len(expected): - return None - if any(not isinstance(item, Mapping) for item in outcomes): - return None - outcome_items = [item for item in outcomes if isinstance(item, Mapping)] - if any( - not isinstance(item.get("nodeid"), str) or not item.get("nodeid") or item.get("nodeid") not in expected - for item in outcome_items - ): - return None - outcome_by_node = {item["nodeid"]: item.get("outcome") for item in outcome_items} - if set(outcome_by_node) != set(expected): - return None - if len(outcome_by_node) != len(outcomes) or any( - not isinstance(nodeid, str) or not nodeid for nodeid in outcome_by_node - ): - return None - if any(outcome not in TERMINAL_NODE_OUTCOMES for outcome in outcome_by_node.values()): - return None - exit_code = attempt.get("exit_code") - if not isinstance(exit_code, int) or isinstance(exit_code, bool): - return None - graph = inspect_testmon_database(data_path, [str(nodeid) for nodeid in expected]) - if not graph.usable_for_selection: - return None - try: - typed_identity = TestmonIdentity.from_mapping(identity) - except ValueError: - return None - if not _identity_matches_runtime(typed_identity, checkout_root=checkout_root, protocol_version=protocol_version): - return None - baseline = ( - BaselineStatus.GREEN - if attempt.get("status") == "complete" - and exit_code == 0 - and all(outcome in SUCCESSFUL_NODE_OUTCOMES for outcome in outcome_by_node.values()) - and not graph.failed_nodeids - else BaselineStatus.RED - ) - raw_scope = attempt.get("verification_scope") - if raw_scope is not None and raw_scope not in {scope.value for scope in VerificationScope}: - return None - terminal_authorized = ( - typed_identity.skip_slow is True - and raw_scope == VerificationScope.NARROW_TERMINAL.value - and typed_identity.terminal_authorization == TerminalAuthorization.NARROW_TERMINAL.value - ) - if baseline is BaselineStatus.GREEN and typed_identity.skip_slow and not terminal_authorized: - baseline = BaselineStatus.RED - if not published_marker: - baseline = BaselineStatus.RED - raw_permission = attempt.get("release_baseline_allowed") - if baseline is BaselineStatus.GREEN and ( - raw_scope != VerificationScope.NARROW_TERMINAL.value - if typed_identity.skip_slow - else raw_scope != VerificationScope.RELEASE_BASELINE.value - ): - baseline = BaselineStatus.RED - if baseline is BaselineStatus.GREEN and raw_permission is not True: - baseline = BaselineStatus.RED - if raw_permission is not None and not isinstance(raw_permission, bool): - return None - if published_marker and raw_permission is not None and raw_permission != (baseline is BaselineStatus.GREEN): - return None - raw_binding = attempt.get("binding") - if raw_binding is None: - typed_binding = TestmonBinding(BindingMode.EXACT, str(checkout_root.resolve())) - elif isinstance(raw_binding, Mapping): - try: - typed_binding = TestmonBinding.from_mapping(raw_binding) - except ValueError: - return None - if Path(typed_binding.checkout_root).resolve() != checkout_root.resolve(): - return None - else: - return None - return TestmonSeedStamp( - protocol_version, - CollectionStatus.COMPLETE, - tuple(str(nodeid) for nodeid in expected), - 0, - baseline, - baseline is BaselineStatus.GREEN, - exit_code, - graph, - typed_identity, - typed_binding, - recorded_data, - run_id, - artifact_dir, - ) - - -__all__ = [ - "BaselineStatus", - "BindingMode", - "CollectionStatus", - "GraphInspection", - "GraphStatus", - "SeedAttemptOutcome", - "TestmonBinding", - "TestmonIdentity", - "SUCCESSFUL_NODE_OUTCOMES", - "TERMINAL_NODE_OUTCOMES", - "TestmonSeedStamp", - "TerminalAuthorization", - "VerificationScope", - "attempt_is_checkout_bound", - "file_fingerprint", - "inspect_testmon_database", - "refresh_stamp", - "seed_marker_is_checkout_bound", - "stamp_from_attempt", - "testmon_runtime_identity", - "validate_stamp", -] diff --git a/devtools/verification_contracts.py b/devtools/verification_contracts.py new file mode 100644 index 0000000000..a2407b0eac --- /dev/null +++ b/devtools/verification_contracts.py @@ -0,0 +1,19 @@ +"""Typed scope fields shared by verification and merge receipts.""" + +from __future__ import annotations + +from enum import StrEnum + + +class VerificationScope(StrEnum): + AFFECTED = "affected" + RELEASE_BASELINE = "release-baseline" + NARROW_TERMINAL = "narrow-terminal" + NON_TEST = "non-test" + + +class TerminalAuthorization(StrEnum): + NARROW_TERMINAL = "narrow-terminal" + + +__all__ = ["TerminalAuthorization", "VerificationScope"] diff --git a/devtools/verify.py b/devtools/verify.py index b7c4171d2d..fd01ea494b 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -7,10 +7,9 @@ --commit Pre-commit tier: ruff format + check + mypy (~3s warm). --quick Pre-push tier: all non-pytest gates (~15s warm). (default) Baseline with pytest-testmon affected tests. - --seed-testmon - Full non-integration pytest run that seeds/updates .cache/testmon/testmondata. --all/--full - Explicit full non-integration pytest diagnostic. + Explicit full non-integration pytest diagnostic in the current + native testmon environment. --lab Default testmon baseline plus lab smoke and SLO checks. Output formats: @@ -58,27 +57,16 @@ update_receipt, write_termination_request, ) -from devtools.testmon_bootstrap import maybe_bootstrap_testmon_seed -from devtools.testmon_state import ( - SUCCESSFUL_NODE_OUTCOMES, - TERMINAL_NODE_OUTCOMES, - BindingMode, - GraphStatus, - SeedAttemptOutcome, - SeedShardStatus, - TerminalAuthorization, - TestmonBinding, - TestmonSeedStamp, - VerificationScope, - inspect_testmon_database, - refresh_stamp, - seed_shard_ledger_is_terminal, - seed_shard_plan, - stamp_from_attempt, - testmon_runtime_identity, - validate_seed_shard_ledger, - validate_stamp, +from devtools.testmon_bootstrap import ( + NativeTestmonPreparation, + NativeTestmonRepairError, + NativeTestmonState, + executable_python_paths, + inspect_native_testmon_environment, + prepare_native_testmon_environment, + remove_invalid_native_testmon_state, ) +from devtools.verification_contracts import TerminalAuthorization, VerificationScope from devtools.verify_runs import ( CURRENT_CONTAINMENT_PATH, CURRENT_EVENTS_DIR, @@ -94,6 +82,7 @@ ResourceSampler, VerifyRun, adaptive_pytest_worker_count, + aggregate_native_testmon_run, append_verify_history, apply_managed_pytest_runtime_policy, classify_pytest_result, @@ -231,15 +220,6 @@ def _format_completion_notification( HISTORY_PATH = VERIFY_HISTORY_PATH TESTMON_DATA = Path(".cache/testmon/testmondata") -TESTMON_SEED_STAMP = Path(".cache/testmon/seed.json") -TESTMON_SEED_ATTEMPT = Path(".cache/testmon/seed-attempt.json") -TESTMON_AFFECTED_STAMP = Path(".cache/testmon/affected.json") -TESTMON_SEED_PROTOCOL_VERSION = 7 -# Keep resumable checkpoints coarse enough that controller startup and -# per-shard testmon initialization do not dominate the seed. The seed still -# records every node outcome, so a failed shard remains retryable at node -# resolution; this size yields six shards for the current correctness corpus. -TESTMON_SEED_SHARD_SIZE = 4096 PYTEST_REPORT_DIR = Path(".cache/verify") PYTEST_REPORT_PATH = PYTEST_REPORT_DIR / "last-pytest.json" PYTEST_JUNIT_REPORT_DIR = Path(".cache/test-reports") @@ -258,6 +238,7 @@ def _format_completion_notification( PYTEST_RESOURCE_INTERVAL_ENV = "POLYLOGUE_VERIFY_RESOURCE_INTERVAL_S" DEFAULT_PYTEST_HEARTBEAT_S = 30.0 DEFAULT_PYTEST_TIMEOUT_S = 45 * 60.0 +VERIFY_INVOCATION_BUDGET_S = 3600.0 DEFAULT_PYTEST_STALL_TIMEOUT_S = 10 * 60.0 DEFAULT_PYTEST_TERM_GRACE_S = 5.0 DEFAULT_PYTEST_RESOURCE_INTERVAL_S = 2.0 @@ -449,13 +430,14 @@ def _pytest_workload_receipt( tmpfs_budget_mb: float | None, basetemp_cleanup: Path | None, concurrency: int, + timeout_s: float, ) -> dict[str, Any]: """Adapt managed-pytest accounting to the shared workload receipt.""" input_digest = hashlib.sha256( json.dumps(cmd, ensure_ascii=False, separators=(",", ":")).encode("utf-8") ).hexdigest() budgets: list[WorkloadBudget] = [] - if (timeout_s := _pytest_timeout_s()) > 0: + if timeout_s > 0: budgets.append( WorkloadBudget( BudgetMeasure.WALL_MS, @@ -1027,9 +1009,10 @@ def _run_pytest_with_heartbeat( t0: float, run: VerifyRun | None = None, artifacts: PytestStepArtifacts | None = None, + timeout_override_s: float | None = None, ) -> subprocess.CompletedProcess[str]: heartbeat_s = _pytest_heartbeat_interval() - timeout_s = _pytest_timeout_s() + timeout_s = _pytest_timeout_s() if timeout_override_s is None else max(0.0, timeout_override_s) stall_timeout_s = _pytest_stall_timeout_s() term_grace_s = _pytest_term_grace_s() resource_interval_s = _pytest_resource_interval_s() @@ -1608,12 +1591,32 @@ def _refresh_progress_marker(at: float, latest: dict[str, Any] | None = None) -> return subprocess.CompletedProcess(cmd, returncode, stdout, stderr) +def _recover_worker_collection_facts(*, events_dir: Path, selection_path: Path) -> bool: + """Recover xdist collection evidence if interruption skips sessionfinish.""" + merged = merge_worker_collection_payloads(events_dir) + if merged is None: + return False + selection = { + **merged, + "updated_at": datetime.now(timezone.utc).isoformat(), + "worker_id": "runner", + "pid": os.getpid(), + "recovered_after_interruption": True, + } + try: + _atomic_write_json(selection_path, selection) + except OSError: + return False + return True + + def _run( label: str, cmd: list[str], *, cwd: str | None = None, run: VerifyRun | None = None, + timeout_s: float | None = None, ) -> tuple[int, float, dict[str, Any]]: t0 = time.monotonic() sys.stderr.write(f" {label} ... ") @@ -1676,10 +1679,10 @@ def _run( pytest_tmpfs = env.get("POLYLOGUE_PYTEST_TMPFS") == "1" budget_kb = pytest_tmpfs_budget_kb(env) pytest_tmpfs_budget_mb = budget_kb / 1024 if budget_kb is not None else None - if label.startswith("pytest seed-testmon"): - # A complete corpus is currently ~16K nodes. Preserve the whole - # selection in the attempt receipt so interrupted seeds can prove - # eventual coverage instead of relying on a 500-node sample. + if label.startswith("pytest native"): + # The invocation aggregate compares the exact two-lane collection + # with the native environment corpus before granting release + # authority. Keep the complete node set in these bounded artifacts. env["POLYLOGUE_PYTEST_SELECTION_NODEID_LIMIT"] = "50000" if run is not None and artifacts is not None: env = env_for_pytest_step(env, run=run, artifacts=artifacts) @@ -1689,7 +1692,15 @@ def _run( if is_pytest: try: try: - result = _run_pytest_with_heartbeat(cmd, cwd=cwd, env=env, t0=t0, run=run, artifacts=artifacts) + result = _run_pytest_with_heartbeat( + cmd, + cwd=cwd, + env=env, + t0=t0, + run=run, + artifacts=artifacts, + timeout_override_s=timeout_s, + ) except PytestContainmentError as exc: pytest_containment_quiescent = False containment_error = str(exc) @@ -1706,7 +1717,16 @@ def _run( ) else: try: - result = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True, env=env) + result = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True, env=env, timeout=timeout_s) + except subprocess.TimeoutExpired as exc: + captured_stdout = exc.stdout.decode(errors="replace") if isinstance(exc.stdout, bytes) else "" + captured_stderr = exc.stderr.decode(errors="replace") if isinstance(exc.stderr, bytes) else "" + result = subprocess.CompletedProcess( + args=cmd, + returncode=124, + stdout=captured_stdout, + stderr=captured_stderr + "\nverify: invocation deadline exhausted\n", + ) except KeyboardInterrupt: interrupted = True result = subprocess.CompletedProcess(args=cmd, returncode=130, stdout="", stderr="") @@ -1721,7 +1741,7 @@ def _run( metadata["termination_reason"] = f"pytest containment did not quiesce: {containment_error}" metadata.update(_pytest_command_metadata(cmd)) metadata["heartbeat_s"] = _pytest_heartbeat_interval() - metadata["timeout_s"] = _pytest_timeout_s() + metadata["timeout_s"] = _pytest_timeout_s() if timeout_s is None else timeout_s metadata["stall_timeout_s"] = _pytest_stall_timeout_s() metadata["term_grace_s"] = _pytest_term_grace_s() metadata["resource_interval_s"] = _pytest_resource_interval_s() @@ -1810,6 +1830,19 @@ def _run( slowest_reports = summary.get("slowest_reports") if isinstance(slowest_reports, list): metadata["slowest_report_count"] = len(slowest_reports) + if ( + label.startswith("pytest native") + and result.returncode == 5 + and metadata.get("selected_count") == 0 + and metadata.get("report_status") == "present" + and isinstance(summary, Mapping) + and summary.get("exitstatus") == 5 + ): + # Either semantic partition may legitimately be empty. Collection + # still synchronized the complete native corpus, so an empty lane + # is successful evidence rather than a pytest usage failure. + result.returncode = 0 + metadata["empty_semantic_lane"] = True containment_path = artifacts.containment_path if artifacts is not None else PYTEST_CONTAINMENT_PATH containment = _read_json_artifact(containment_path) if containment is not None: @@ -1944,6 +1977,7 @@ def _run( tmpfs_budget_mb=pytest_tmpfs_budget_mb, basetemp_cleanup=basetemp_cleanup, concurrency=max(1, pytest_concurrency), + timeout_s=_pytest_timeout_s() if timeout_s is None else timeout_s, ) metadata["workload_receipt"] = workload_receipt if artifacts is not None: @@ -2041,22 +2075,12 @@ def _subprocess_env() -> dict[str, str]: def _stop_after_failed_step(label: str) -> bool: - return label.startswith("pytest") or label in {"lab smoke", "bench slo"} + return label in {"lab smoke", "bench slo"} -def _seed_shard_failure_requires_stop(step: Mapping[str, Any], *, shard_complete: bool) -> bool: - """Stop shard admission after harness failure while retaining red-test evidence. - - A normal pytest exit 1 with a structured ``pytest_failed`` diagnosis is - useful seed evidence: later shards can still populate the resumable - dependency graph. Timeouts, resource refusals, worker/internal errors, - usage errors, and unclassified failures mean the harness is no longer - healthy enough to admit another expensive shard. - """ - exit_code = step.get("exit") - if exit_code == 0: - return False - return not (exit_code == 1 and step.get("diagnosis") == "pytest_failed" and shard_complete) +def _native_lane_failure_requires_stop(step: Mapping[str, Any]) -> bool: + """Continue the serial lane only after an ordinary test failure.""" + return not (step.get("exit") == 1 and step.get("diagnosis") == "pytest_failed") # ── step builder ──────────────────────────────────────────────────── @@ -2068,10 +2092,8 @@ def build_verify_steps( lab: bool, skip_slow: bool, commit: bool = False, - seed_testmon: bool = False, - resume_testmon_seed: bool = False, - full_pytest: bool = False, - broad_testmon: bool = False, + testmon_mode: str = "affected", + testmon_environment: str = "", ) -> list[tuple[str, list[str]]]: steps: list[tuple[str, list[str]]] = [ ("ruff format", ["ruff", "format", "--check", "polylogue/", "tests/", "devtools/"]), @@ -2143,54 +2165,52 @@ def build_verify_steps( "devtools.pytest_progress_plugin", ] # Benchmark cases are an explicit campaign surface, not part of the - # correctness/testmon seed. Keeping them out here is important: a + # correctness corpus. Keeping them out here is important: a # benchmark marker is not necessarily paired with ``slow`` or a scale - # marker, and a serial shard would otherwise spend minutes executing a + # marker, and the serial lane would otherwise spend minutes executing a # performance probe before it can checkpoint any correctness nodes. base_marker = f"not benchmark and {scale_marker_expr}" if skip_slow: base_marker = f"not slow and {base_marker}" - if seed_testmon: - # Collection produces the exact corpus contract before any testmon - # write. Shards below are generated from this ledger and run one - # at a time, so pytest-testmon has exactly one SQLite writer. - pytest_cmd.extend(["-m", base_marker, "--collect-only", "-n", "0"]) - label = "pytest seed-testmon collect (resume)" if resume_testmon_seed else "pytest seed-testmon collect" - steps.append((label, pytest_cmd)) - elif full_pytest: - # #1775: the full diagnostic runs as two lanes. The bulk lane keeps - # xdist parallelism but deselects wall-clock-bound tests; the - # isolated lane reruns those (``load_sensitive``/``tui`` — timing - # budgets, loopback-socket timeouts, TUI render timing) single- - # process with a stable order, so worker contention can no longer - # flake them. Both lanes are correctness blockers; the split only - # removes the scheduling jitter that made ``--all`` an unreliable - # completion gate. - bulk_cmd = [ - *pytest_cmd, + if testmon_mode not in {"affected", "bootstrap", "full"}: + raise ValueError(f"unknown native testmon mode: {testmon_mode}") + if not testmon_environment: + raise ValueError("native testmon environment is required for pytest verification") + native_args = ["--testmon", f"--testmon-env={testmon_environment}"] + if testmon_mode == "affected": + native_args.append("--testmon-forceselect") + else: + native_args.append("--testmon-noselect") + + parallel_cmd = [ + *pytest_cmd, + "-m", + f"({base_marker}) and not load_sensitive and not tui", + *native_args, + *_pytest_worker_args(), + ] + steps.append((f"pytest native parallel ({testmon_mode})", parallel_cmd)) + + def _serial_report_arg(arg: str) -> str: + if arg.startswith("--junitxml="): + return f"--junitxml={_report_dir}/verify-latest-serial.xml" + if arg.startswith("--json-report-file="): + return f"--json-report-file={PYTEST_REPORT_DIR / 'last-pytest-serial.json'}" + return arg + + serial_cmd = [_serial_report_arg(arg) for arg in pytest_cmd] + serial_cmd.extend( + [ "-m", - f"({base_marker}) and not load_sensitive and not tui", - *_pytest_worker_args(), + f"({base_marker}) and (load_sensitive or tui)", + *native_args, + "-p", + "no:randomly", + "-n", + "0", ] - steps.append((BROAD_PYTEST_STEP_LABELS["full_parallel"], bulk_cmd)) - - def _isolated_report_arg(arg: str) -> str: - # Keep the bulk lane's canonical report artifacts intact for - # _compare_against_last; the isolated lane writes its own files. - if arg.startswith("--junitxml="): - return f"--junitxml={_report_dir}/verify-latest-isolated.xml" - if arg.startswith("--json-report-file="): - return f"--json-report-file={PYTEST_REPORT_DIR / 'last-pytest-isolated.json'}" - return arg - - isolated_cmd = [_isolated_report_arg(arg) for arg in pytest_cmd] - isolated_cmd.extend(["-m", f"({base_marker}) and (load_sensitive or tui)", "-p", "no:randomly", "-n", "0"]) - steps.append((BROAD_PYTEST_STEP_LABELS["load_sensitive"], isolated_cmd)) - else: - pytest_cmd.extend(["-m", base_marker, "--testmon", *_pytest_worker_args()]) - pytest_cmd.append("--testmon-forceselect") - label = BROAD_PYTEST_STEP_LABELS["testmon_broad"] if broad_testmon else "pytest testmon" - steps.append((label, pytest_cmd)) + ) + steps.append((f"pytest native serial ({testmon_mode})", serial_cmd)) if lab: steps.append(("lab smoke", _devtools_cmd("lab smoke", "run", "archive-smoke", "--tier", "0"))) @@ -2292,17 +2312,11 @@ def _stamp_head() -> None: (stamp_dir / "last-verify-head").write_text(head + "\n") -def _file_fingerprint(path: Path) -> str: - if not path.exists() or not path.is_file(): - return "missing" - h = hashlib.sha256() - try: - with path.open("rb") as f: - for chunk in iter(lambda: f.read(1024 * 1024), b""): - h.update(chunk) - except OSError: - return "unreadable" - return h.hexdigest() +def _atomic_write_json(path: Path, payload: Mapping[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f"{path.name}.{os.getpid()}.tmp") + temporary.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + temporary.replace(path) def _pytest_worker_args(*, maximum: int | None = None) -> list[str]: @@ -2313,34 +2327,18 @@ def _pytest_worker_args(*, maximum: int | None = None) -> list[str]: return ["--dist=loadgroup", "-n", str(workers)] -BROAD_PYTEST_STEP_LABELS = { - "seed": "pytest seed-testmon", - "seed_resume": "pytest seed-testmon (resume)", - "full_parallel": "pytest full (parallel)", - "load_sensitive": "pytest load-sensitive (isolated)", - "testmon_broad": "pytest testmon (broad)", -} - - def _pytest_command_concurrency(cmd: Sequence[str], *, env: Mapping[str, str] | None = None) -> int: - """Return a fail-closed reservation for the final pytest command. - - ``-n auto`` can launch one worker per logical CPU. Reserve that maximum - instead of guessing one worker; an unrecognised xdist value is treated the - same way so malformed or future values cannot weaken admission. - """ + """Return a fail-closed reservation for the final pytest command.""" request = pytest_command_worker_request(cmd) if request is None: return 0 if request == "auto": auto_workers = (env if env is not None else os.environ).get("PYTEST_XDIST_AUTO_NUM_WORKERS", "").strip() if auto_workers: - try: + with contextlib.suppress(ValueError): configured = int(auto_workers) - except ValueError: - configured = 0 - if configured > 0: - return configured + if configured > 0: + return configured try: return max(0, int(request)) except ValueError: @@ -2348,14 +2346,8 @@ def _pytest_command_concurrency(cmd: Sequence[str], *, env: Mapping[str, str] | def _pytest_uses_full_suite_basetemp(label: str) -> bool: - """Whether this pytest step can materialize the measured full-suite tree.""" - return label in BROAD_PYTEST_STEP_LABELS.values() or label.startswith("pytest seed-testmon shard ") - - -_BROAD_TESTMON_CHANGED_PATHS = { - "pyproject.toml", - "tests/conftest.py", -} + """Whether this semantic lane may materialize the complete corpus tree.""" + return label.startswith("pytest native") and ("(bootstrap)" in label or "(full)" in label) def _changed_paths(base_commit: str, head_commit: str) -> set[str]: @@ -2383,1046 +2375,71 @@ def _changed_paths(base_commit: str, head_commit: str) -> set[str]: return changed -def _default_testmon_is_broad_change(base_commit: str, head_commit: str) -> bool: - """Return true when affected-test selection should be treated as broad.""" - return bool(_changed_paths(base_commit, head_commit) & _BROAD_TESTMON_CHANGED_PATHS) - - -def _changed_executable_paths(base_commit: str, head_commit: str) -> tuple[str, ...]: - """Return changed paths whose behavior should select at least one test.""" +def _changed_test_relevant_paths(base_commit: str, head_commit: str) -> tuple[str, ...]: roots = ("polylogue/", "devtools/", "tests/", "packaging/") - exact = {"pyproject.toml", "uv.lock"} + exact = {"pyproject.toml", "uv.lock", "pytest.ini", "tox.ini", "setup.cfg"} return tuple( sorted(path for path in _changed_paths(base_commit, head_commit) if path in exact or path.startswith(roots)) ) -def _testmon_coverage_identity(executable_paths: Sequence[str]) -> dict[str, Any]: - """Identify the exact worktree contents covered by an affected/full run.""" - return { - "worktree_fingerprint": worktree_fingerprint(), - "executable_paths": list(executable_paths), - } - - -def _matching_testmon_coverage(executable_paths: Sequence[str]) -> str | None: - """Return the receipt kind proving that zero new selection is legitimate.""" - identity = _testmon_coverage_identity(executable_paths) - affected = _read_json_artifact(TESTMON_AFFECTED_STAMP) - selected_count = affected.get("selected_count") if isinstance(affected, dict) else None - if ( - isinstance(affected, dict) - and affected.get("protocol_version") == 1 - and affected.get("status") == "complete" - and isinstance(affected.get("timestamp"), str) - and bool(affected.get("timestamp")) - and isinstance(affected.get("run_id"), str) - and bool(affected.get("run_id")) - and isinstance(selected_count, int) - and not isinstance(selected_count, bool) - and selected_count > 0 - and affected.get("identity") == identity - ): - return "successful_affected_run" - return None - - -def _record_testmon_affected_coverage(*, executable_paths: Sequence[str], selected_count: int, run_id: str) -> None: - """Persist proof that testmon exercised dependencies for these contents.""" - _atomic_write_json( - TESTMON_AFFECTED_STAMP, - { - "protocol_version": 1, - "status": "complete", - "timestamp": datetime.now(timezone.utc).isoformat(), - "run_id": run_id, - "selected_count": selected_count, - "identity": _testmon_coverage_identity(executable_paths), - }, - ) - - -def _testmon_preflight(*, seed_testmon: bool, full_pytest: bool, quick: bool, commit: bool) -> str | None: - if quick or commit or seed_testmon or full_pytest: - return None - seed_message = ( - "verify: pytest-testmon is not seeded; run `devtools verify --seed-testmon` " - "to create .cache/testmon/testmondata and .cache/testmon/seed.json " - "before using the default affected-test path.\n" - ) - if not TESTMON_DATA.exists(): - return seed_message - if not TESTMON_SEED_STAMP.exists(): - attempt = _read_testmon_seed_attempt() - if ( - attempt is not None - and stamp_from_attempt( - attempt, - TESTMON_DATA, - checkout_root=ROOT, - protocol_version=TESTMON_SEED_PROTOCOL_VERSION, - published_marker=False, - ) - is not None - ): - sys.stderr.write( - "verify: using a validated complete pytest-testmon graph from a red seed attempt; " - "the release baseline remains red.\n" - ) - return None - return seed_message - stamp = validate_stamp( - TESTMON_SEED_STAMP, - TESTMON_DATA, - checkout_root=ROOT, - protocol_version=TESTMON_SEED_PROTOCOL_VERSION, - ) - if stamp is None: - return ( - "verify: pytest-testmon seed state is unreadable, stale, malformed, or not graph-complete; run " - "`devtools verify --seed-testmon` to rebuild the dependency baseline.\n" - ) - return None - - -def _atomic_write_json(path: Path, payload: Mapping[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - temporary = path.with_name(f"{path.name}.{os.getpid()}.tmp") - temporary.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") - temporary.replace(path) - - -def _testmon_seed_identity( - *, - git_head: str | None, - git_tree: str | None = None, - skip_slow: bool, - lab: bool, - terminal_authorization: str | None = None, -) -> dict[str, Any]: - runtime_identity = testmon_runtime_identity(ROOT) - if runtime_identity is None: - raise RuntimeError("could not identify the active dependency environment and pytest harness") - dependency_environment, pytest_harness = runtime_identity - return { - "git_head": git_head, - "git_tree": git_tree, - "worktree_fingerprint": worktree_fingerprint(), - "python": sys.version, - "skip_slow": skip_slow, - "lab": lab, - "terminal_authorization": terminal_authorization, - "dependency_environment": dependency_environment, - "pytest_harness": pytest_harness, - } - - -def _read_testmon_seed_attempt() -> dict[str, Any] | None: - payload = _read_json_artifact(TESTMON_SEED_ATTEMPT) - return payload if isinstance(payload, dict) else None - - -def _recover_worker_collection_facts(*, events_dir: Path, selection_path: Path) -> bool: - """Publish xdist worker collection facts when its controller never finishes. - - The progress plugin normally merges these facts during controller - ``pytest_sessionfinish``. Interrupted containment bypasses that hook, so - the runner recovers the same canonical worker fact before it terminalizes - the durable step record. - """ - merged = merge_worker_collection_payloads(events_dir) - if merged is None: - return False - selection = dict(merged) - selection.update( - { - "updated_at": datetime.now(timezone.utc).isoformat(), - "worker_id": "runner", - "pid": os.getpid(), - "recovered_after_interruption": True, - } - ) - try: - selection_path.parent.mkdir(parents=True, exist_ok=True) - temporary = selection_path.with_name(f"{selection_path.name}.{os.getpid()}.tmp") - temporary.write_text(json.dumps(selection, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") - temporary.replace(selection_path) - except OSError: - return False - return True - - -def _flatten_seed_outcomes(attempt: Mapping[str, Any] | None) -> list[dict[str, Any]]: - """Flatten outcomes from every interrupted attempt, newest result winning.""" - if attempt is None: - return [] - flattened: dict[str, dict[str, Any]] = {} - for field in ("prior_node_outcomes", "node_outcomes"): - raw = attempt.get(field) - if not isinstance(raw, list): - continue - for item in raw: - if isinstance(item, Mapping) and isinstance(item.get("nodeid"), str) and item["nodeid"]: - flattened[item["nodeid"]] = dict(item) - return [flattened[nodeid] for nodeid in sorted(flattened)] - - -def _testmon_release_baseline_permission() -> bool | None: - """Return release permission for current testmon state, or ``None`` when not applicable.""" - if TESTMON_SEED_STAMP.exists(): - stamp = validate_stamp( - TESTMON_SEED_STAMP, - TESTMON_DATA, - checkout_root=ROOT, - protocol_version=TESTMON_SEED_PROTOCOL_VERSION, - ) - return stamp.release_baseline_allowed if stamp is not None else False - attempt = _read_testmon_seed_attempt() - if attempt is None: - return False - stamp = stamp_from_attempt( - attempt, - TESTMON_DATA, - checkout_root=ROOT, - protocol_version=TESTMON_SEED_PROTOCOL_VERSION, - published_marker=False, - ) - return stamp.release_baseline_allowed if stamp is not None else False - - -def _safe_testmon_artifact_dir(raw: object, *, require_run_root: bool = False) -> Path | None: - if not isinstance(raw, str) or not raw: - return None - path = Path(raw) - checkout_root = Path.cwd().resolve() - if require_run_root and path.is_absolute(): - return None - resolved = (path if path.is_absolute() else checkout_root / path).resolve() - try: - resolved.relative_to(checkout_root) - if require_run_root: - resolved.relative_to((checkout_root / ".cache" / "verify" / "runs").resolve()) - except ValueError: - return None - return resolved - - -def _testmon_seed_expected_nodeids(attempt: Mapping[str, Any]) -> list[str]: - """Recover the seed ledger, including after an abrupt outer-run exit.""" - expected = attempt.get("expected_nodeids") - if isinstance(expected, list) and expected: - if ( - any(not isinstance(nodeid, str) or not nodeid for nodeid in expected) - or len(set(expected)) != len(expected) - or not isinstance(attempt.get("expected_count"), int) - or isinstance(attempt.get("expected_count"), bool) - or attempt.get("expected_count") != len(expected) - or not isinstance(attempt.get("expected_digest"), str) - or attempt.get("expected_digest") != hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest() - ): - return [] - return list(expected) - - artifact_dir = _safe_testmon_artifact_dir(attempt.get("artifact_dir"), require_run_root=True) - if artifact_dir is None: - return [] - for selection_path in sorted(artifact_dir.glob("steps/*/selection.json")): - selection = _read_json_artifact(selection_path) - if not isinstance(selection, dict): - continue - omitted = selection.get("selected_nodeids_omitted") - selected_count = selection.get("selected_count") - if ( - not isinstance(omitted, int) - or isinstance(omitted, bool) - or omitted != 0 - or not isinstance(selected_count, int) - or isinstance(selected_count, bool) - ): - continue - selected = selection.get("selected_nodeids") - if ( - isinstance(selected, list) - and selected - and all(isinstance(nodeid, str) and nodeid for nodeid in selected) - and len(set(selected)) == len(selected) - and selected_count == len(selected) - ): - return list(selected) - return [] - - -def _testmon_seed_resume_contract(identity: Mapping[str, Any]) -> dict[str, Any]: - """Return inputs that change which corpus a seed promises to cover.""" - return { - key: identity.get(key) - for key in ( - "git_tree", - "worktree_fingerprint", - "python", - "skip_slow", - "lab", - "terminal_authorization", - "dependency_environment", - "pytest_harness", - ) - } - - -def _testmon_seed_can_resume(identity: Mapping[str, Any]) -> bool: - attempt = _read_testmon_seed_attempt() - if attempt is None or not TESTMON_DATA.exists(): - return False - prior_identity = attempt.get("identity") - contract = _testmon_seed_resume_contract(identity) - return ( - attempt.get("protocol_version") == TESTMON_SEED_PROTOCOL_VERSION - and attempt.get("status") in {"running", "incomplete"} - and isinstance(prior_identity, dict) - and isinstance(contract["git_tree"], str) - and bool(contract["git_tree"]) - and _testmon_seed_resume_contract(prior_identity) == contract - and bool(_testmon_seed_expected_nodeids(attempt)) - ) - - -def _prepare_testmon_seed_attempt( - *, - identity: Mapping[str, Any], - run: VerifyRun, - resume: bool, -) -> dict[str, Any]: - prior = _read_testmon_seed_attempt() if resume else None - expected = sorted(_testmon_seed_expected_nodeids(prior)) if prior is not None else [] - prior_outcomes = _flatten_seed_outcomes(prior) - payload = { - "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "running", - "identity": dict(identity), - "resume": resume, - "expected_nodeids": expected, - "expected_count": len(expected), - "expected_digest": hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest() if expected else None, - "prior_node_outcomes": prior_outcomes, - "shards": list(prior.get("shards", [])) if prior is not None and isinstance(prior.get("shards"), list) else [], - "started_at": datetime.now(timezone.utc).isoformat(), - "run_id": run.run_id, - "artifact_dir": str(run.relative_run_dir), - "testmon_data_before": _file_fingerprint(TESTMON_DATA), - "binding": TestmonBinding(BindingMode.EXACT, str(ROOT.resolve())).as_dict(), - } - TESTMON_SEED_STAMP.unlink(missing_ok=True) - _atomic_write_json(TESTMON_SEED_ATTEMPT, payload) - return payload - - -def _seed_selection_nodeids(selection: Mapping[str, Any]) -> list[str] | None: - """Accept only a complete, untruncated collection ledger.""" - nodeids = selection.get("selected_nodeids") - selected_count = selection.get("selected_count") - omitted = selection.get("selected_nodeids_omitted") - if ( - not isinstance(nodeids, list) - or not nodeids - or any(not isinstance(nodeid, str) or not nodeid for nodeid in nodeids) - or len(set(nodeids)) != len(nodeids) - or not isinstance(selected_count, int) - or isinstance(selected_count, bool) - or selected_count != len(nodeids) - or not isinstance(omitted, int) - or isinstance(omitted, bool) - or omitted != 0 - ): - return None - return sorted(nodeids) - - -def _prepare_testmon_seed_shards( - prepared: Mapping[str, Any], - *, - selection: Mapping[str, Any] | None, -) -> dict[str, Any]: - """Persist the full planned corpus before the first testmon DB mutation.""" - expected = sorted(_testmon_seed_expected_nodeids(prepared)) if prepared.get("resume") else [] - if not expected: - expected = _seed_selection_nodeids(selection or {}) or [] - prior_shards = validate_seed_shard_ledger(prepared.get("shards"), expected_nodeids=expected) - shards = ( - prior_shards - if prior_shards is not None - else ( - seed_shard_plan( - expected, - shard_size=TESTMON_SEED_SHARD_SIZE, - serial_nodeids=[ - nodeid - for nodeid, markers in (selection or {}).get("selected_node_markers", {}).items() - if "load_sensitive" in markers or "tui" in markers - ], - ) - if expected - else [] - ) - ) - payload = { - **dict(prepared), - "expected_nodeids": expected, - "expected_count": len(expected), - "expected_digest": hashlib.sha256("\n".join(expected).encode()).hexdigest() if expected else None, - "selection": dict(selection or {}), - "shard_size": TESTMON_SEED_SHARD_SIZE, - "shards": shards, - } - _atomic_write_json(TESTMON_SEED_ATTEMPT, payload) - return payload - - -def _seed_shard_command( - collection_command: Sequence[str], - shard: Mapping[str, Any], - *, - nodeids_file: Path, -) -> list[str]: - """Build a bounded-argv, dynamically balanced pytest-testmon invocation. - - A full shard's node IDs can exceed the host's ``execve`` argument budget - once ``systemd-run`` and the managed environment are included. Pytest's - response-file syntax keeps the authoritative node list in the run - artifact while making the child command size independent of shard size. - """ - nodeids = shard.get("nodeids") - if not isinstance(nodeids, list) or not nodeids: - raise ValueError("testmon seed shard is missing nodeids") - nodeids_file.parent.mkdir(parents=True, exist_ok=True) - nodeids_file.write_text("\n".join(nodeids) + "\n", encoding="utf-8") - command: list[str] = [] - skip_next = False - for argument in collection_command: - if skip_next: - skip_next = False - continue - if argument == "--collect-only": - continue - if argument in {"-n", "--numprocesses"}: - skip_next = True - continue - if argument.startswith("--numprocesses=") or (argument.startswith("-n") and len(argument) > 2): - continue - command.append(argument) - # Collection is deliberately serial, but execution is not. pytest-testmon - # has an xdist-aware controller database; retaining the managed worker pool - # here avoids turning a 20k-node seed into hours of serial fixture setup. - if shard.get("execution_mode") == "serial": - command.extend(["-n", "0", "--testmon", "--testmon-noselect", f"@{nodeids_file}"]) - else: - command.extend( - [ - *_pytest_worker_args(maximum=10), - "--testmon", - "--testmon-noselect", - f"@{nodeids_file}", - ] - ) - return command - - -def _canonical_seed_nodeid(nodeid: str, expected_nodeids: Sequence[str]) -> str: - """Map xdist's ``nodeid@group`` reports back to the collected node ID.""" - if nodeid in expected_nodeids: - return nodeid - candidates = [expected for expected in expected_nodeids if nodeid.startswith(expected + "@")] - return max(candidates, key=len, default=nodeid) - - -def _seed_shard_outcomes(shards: Sequence[Mapping[str, Any]]) -> list[dict[str, Any]]: - """Flatten the shard ledger in canonical node order for legacy readers.""" - outcomes: dict[str, dict[str, Any]] = {} - for shard in shards: - raw_outcomes = shard.get("node_outcomes") - if not isinstance(raw_outcomes, list): - continue - for item in raw_outcomes: - if isinstance(item, Mapping) and isinstance(item.get("nodeid"), str): - outcomes[str(item["nodeid"])] = dict(item) - return [outcomes[nodeid] for nodeid in sorted(outcomes)] +_ACTIVE_VERIFY_RUN: tuple[VerifyRun, float, VerificationScope] | None = None -def _checkpoint_testmon_seed_shard( +def _planned_verification_scope( + args: argparse.Namespace, *, - prepared: Mapping[str, Any], - shard_index: int, - step: Mapping[str, Any], -) -> dict[str, Any]: - """Record one shard's result atomically before another shard may start.""" - expected = sorted(_testmon_seed_expected_nodeids(prepared)) - shards = validate_seed_shard_ledger(prepared.get("shards"), expected_nodeids=expected) - if shards is None or shard_index < 1 or shard_index > len(shards): - raise ValueError("testmon seed shard ledger is malformed") - shard = dict(shards[shard_index - 1]) - nodeids = shard["nodeids"] - artifact_dir = _safe_testmon_artifact_dir(step.get("artifact_dir")) - selection = _read_json_artifact(artifact_dir / "selection.json") if artifact_dir is not None else None - selected_raw = _seed_selection_nodeids(selection) if isinstance(selection, Mapping) else None - selected = ( - sorted(_canonical_seed_nodeid(nodeid, nodeids) for nodeid in selected_raw) if selected_raw is not None else None - ) - database = _testmon_database_state(nodeids) - prior = { - str(item["nodeid"]): item - for item in shard.get("node_outcomes", []) - if isinstance(item, Mapping) and isinstance(item.get("nodeid"), str) - } - outcomes = _seed_node_outcomes_from_events( - artifact_dir / "events.jsonl" if artifact_dir is not None else Path(".missing-testmon-events"), - expected_nodeids=nodeids, - database=database, - pytest_step=step, - prior_node_outcomes=prior, - use_database_fallback=False, - ) - terminal = all(item.get("outcome") in TERMINAL_NODE_OUTCOMES for item in outcomes) - selection_matches = selected == nodeids - shard.update( - { - "status": SeedShardStatus.COMPLETE.value - if selection_matches and terminal - else SeedShardStatus.INCOMPLETE.value, - "started_at": shard.get("started_at") or datetime.now(timezone.utc).isoformat(), - "finished_at": datetime.now(timezone.utc).isoformat(), - "exit_code": step.get("exit"), - "artifact_dir": step.get("artifact_dir"), - "selection": dict(selection) if isinstance(selection, Mapping) else None, - "database": database, - "node_outcomes": outcomes, - "pytest_step": dict(step), - } - ) - shards[shard_index - 1] = shard - payload = { - **dict(prepared), - "status": "running", - "shards": shards, - "node_outcomes": _seed_shard_outcomes(shards), - "testmon_data": _file_fingerprint(TESTMON_DATA), - } - _atomic_write_json(TESTMON_SEED_ATTEMPT, payload) - return payload + testmon_mode: str | None, +) -> VerificationScope: + if args.quick or args.commit: + return VerificationScope.NON_TEST + if testmon_mode in {"bootstrap", "full"}: + return VerificationScope.NARROW_TERMINAL if args.skip_slow else VerificationScope.RELEASE_BASELINE + return VerificationScope.AFFECTED -def _testmon_seed_terminal_authorized(prepared: Mapping[str, Any]) -> bool: - identity = prepared.get("identity") - return ( - isinstance(identity, Mapping) - and identity.get("skip_slow") is True - and identity.get("terminal_authorization") == TerminalAuthorization.NARROW_TERMINAL.value - ) +def _pytest_profile(*, skip_slow: bool, lab: bool) -> str: + return f"slow={'exclude' if skip_slow else 'include'};scale={'medium' if lab else 'small'}" -def _testmon_database_state(expected_nodeids: Sequence[str]) -> dict[str, Any]: - graph = inspect_testmon_database(TESTMON_DATA, expected_nodeids) - expected = set(expected_nodeids) - failed = list(graph.failed_nodeids) - return { - "recorded_count": graph.recorded_count, - "failed_count": len(failed), - "dependency_edge_count": graph.dependency_edge_count, - "missing_nodeids": list(graph.missing_nodeids), - "failed_nodeids": failed, - "node_outcomes": { - nodeid: ("failed" if nodeid in failed else "passed" if nodeid not in graph.missing_nodeids else "missing") - for nodeid in sorted(expected) - }, - "error": graph.error, - "graph_status": graph.status.value, - "orphan_execution_edges": graph.orphan_execution_edges, - "orphan_fingerprint_edges": graph.orphan_fingerprint_edges, - } +def _remaining_invocation_budget(started_at: float) -> float: + return max(0.0, VERIFY_INVOCATION_BUDGET_S - (time.monotonic() - started_at)) -def _seed_node_outcomes_from_events( - path: Path, +def _native_environment_after_run( + preparation: NativeTestmonPreparation, *, - expected_nodeids: Sequence[str], - database: Mapping[str, Any], - pytest_step: Mapping[str, Any] | None, - use_database_fallback: bool = True, - prior_node_outcomes: Mapping[str, Mapping[str, Any]] | None = None, -) -> list[dict[str, Any]]: - """Classify every promised seed node into one explicit terminal state.""" - reports: dict[str, list[dict[str, Any]]] = {} - started: set[str] = set() - finished: set[str] = set() - try: - with path.open(encoding="utf-8") as handle: - for line in handle: - if not line.strip(): - continue - with contextlib.suppress(json.JSONDecodeError): - event = json.loads(line) - nodeid = event.get("nodeid") - if not isinstance(nodeid, str) or not nodeid: - continue - nodeid = _canonical_seed_nodeid(nodeid, expected_nodeids) - if event.get("event") == "test_started": - started.add(nodeid) - elif event.get("event") == "test_finished": - finished.add(nodeid) - elif event.get("event") == "test_report": - reports.setdefault(nodeid, []).append(event) - except OSError: - pass - - database_outcomes = database.get("node_outcomes") - recorded = database_outcomes if isinstance(database_outcomes, dict) else {} - diagnosis = str((pytest_step or {}).get("diagnosis") or "").lower() - results: list[dict[str, Any]] = [] - for nodeid in expected_nodeids: - node_reports = reports.get(nodeid, []) - failed_reports = [report for report in node_reports if report.get("outcome") == "failed"] - call_reports = [report for report in node_reports if report.get("when") == "call"] - longrepr = "\n".join(str(report.get("longrepr") or "") for report in failed_reports).lower() - outcome: str - reason: str - if "timeout" in longrepr: - outcome, reason = "timeout", "pytest-timeout report" - elif any(report.get("when") in {"setup", "teardown"} for report in failed_reports): - outcome, reason = "error", "fixture setup/teardown failed" - elif any(report.get("outcome") == "xfailed" for report in node_reports): - outcome, reason = "xfailed", "pytest expected failure" - elif any(report.get("outcome") == "xpassed" for report in node_reports): - outcome, reason = "xpassed", "pytest unexpected pass" - elif any(report.get("outcome") == "failed" for report in call_reports): - outcome, reason = "failed", "test call failed" - elif any(report.get("outcome") == "passed" for report in call_reports): - outcome, reason = "passed", "test call passed" - elif any(report.get("outcome") == "skipped" for report in call_reports): - outcome, reason = "skipped", "test call skipped" - elif any(report.get("outcome") == "skipped" for report in node_reports): - outcome, reason = "skipped", "test setup or teardown skipped" - elif nodeid in finished and any( - report.get("when") == "teardown" and report.get("outcome") == "passed" for report in node_reports - ): - # Teardown describes fixture cleanup, not the test body. It may - # corroborate a terminal testmon row, but it cannot replace a - # missing call report: a failed call can still end with a passing - # teardown, and an unrecorded call must remain resumable. - if recorded.get(nodeid) == "passed": - outcome, reason = "passed", "passing teardown corroborated by testmon success" - elif recorded.get(nodeid) == "failed": - outcome, reason = "failed", "passing teardown contradicted by testmon failure" - else: - outcome, reason = "missing", "passing teardown without call report or testmon result" - elif nodeid in started and nodeid not in finished and "timeout" in diagnosis: - outcome, reason = "timeout", "supervisor timed out while node was active" - elif nodeid in started and nodeid not in finished and "worker" in diagnosis: - outcome, reason = "worker_crash", "worker exited while node was active" - elif ( - nodeid in started - and nodeid not in finished - and any(marker in diagnosis for marker in ("interrupt", "signal", "terminated")) - ): - outcome, reason = "interrupted", "run ended while node was active" - elif use_database_fallback and recorded.get(nodeid) == "passed": - outcome, reason = "passed", "testmon database recorded success" - elif use_database_fallback and recorded.get(nodeid) == "failed": - outcome, reason = "failed", "testmon database recorded failure" - elif prior_node_outcomes is not None and nodeid in prior_node_outcomes: - prior = prior_node_outcomes[nodeid] - prior_outcome = prior.get("outcome") - if prior_outcome in TERMINAL_NODE_OUTCOMES: - outcome, reason = str(prior_outcome), "terminal outcome carried from the prior seed attempt" - else: - outcome, reason = "missing", "prior seed attempt has no terminal outcome" - else: - outcome, reason = "missing", "no terminal report or testmon execution row" - results.append( - { - "nodeid": nodeid, - "outcome": outcome, - "reason": reason, - "started": nodeid in started, - "finished": nodeid in finished, - "phases": [ - { - "when": report.get("when"), - "outcome": report.get("outcome"), - "duration_s": report.get("duration_s"), - } - for report in node_reports - ], - } - ) - return results - - -def _seed_attempt_outcome( - *, - release_eligible: bool, - terminal_graph: bool, - exit_code: int, - pytest_step: Mapping[str, Any] | None, -) -> SeedAttemptOutcome: - """Classify the terminal seed result without hiding a bounded resource stop.""" - if release_eligible: - return SeedAttemptOutcome.GREEN_RELEASE_BASELINE - if terminal_graph: - return SeedAttemptOutcome.RED_BASELINE if exit_code != 0 else SeedAttemptOutcome.SELECTION_ONLY - diagnosis = str((pytest_step or {}).get("diagnosis") or "").casefold() - termination_reason = str((pytest_step or {}).get("termination_reason") or "").casefold() - if diagnosis in {"pytest_timeout", "pytest_stall_timeout", "pytest_resource_preflight_failed"} or any( - marker in termination_reason - for marker in ("runtime exceeded", "tmpfs budget exceeded", "resource budget", "resource limit") - ): - return SeedAttemptOutcome.RESOURCE_TIMEOUT - return SeedAttemptOutcome.INCOMPLETE - - -def _finalize_testmon_seed_attempt( - *, - prepared: Mapping[str, Any], - step_results: Sequence[Mapping[str, Any]], - exit_code: int, -) -> dict[str, Any]: - pytest_step = next( - (step for step in step_results if str(step.get("name", "")).startswith("pytest seed-testmon")), None - ) - selection: dict[str, Any] = {} - events_path: Path | None = None - if pytest_step is not None: - artifact_dir = _safe_testmon_artifact_dir(pytest_step.get("artifact_dir")) - if artifact_dir is not None: - selection_payload = _read_json_artifact(artifact_dir / "selection.json") - if isinstance(selection_payload, dict): - selection = selection_payload - events_path = artifact_dir / "events.jsonl" - - raw_omitted = selection.get("selected_nodeids_omitted") - raw_selected_count = selection.get("selected_count") - selected_nodeids = selection.get("selected_nodeids") - selection_valid = ( - isinstance(raw_omitted, int) - and not isinstance(raw_omitted, bool) - and raw_omitted >= 0 - and isinstance(raw_selected_count, int) - and not isinstance(raw_selected_count, bool) - and isinstance(selected_nodeids, list) - and all(isinstance(nodeid, str) and nodeid for nodeid in selected_nodeids) - and len(set(selected_nodeids)) == len(selected_nodeids) - and raw_selected_count == len(selected_nodeids) - ) - prepared_expected = prepared.get("expected_nodeids") - expected_raw = prepared_expected if isinstance(prepared_expected, list) and prepared_expected else selected_nodeids - expected = list(expected_raw) if isinstance(expected_raw, list) else [] - shards = validate_seed_shard_ledger(prepared.get("shards"), expected_nodeids=expected) - sharded = shards is not None - if sharded: - assert shards is not None - selection_valid = seed_shard_ledger_is_terminal(shards) - omitted = 0 - database = _testmon_database_state(expected) - outcome_by_node = {item["nodeid"]: item for item in _seed_shard_outcomes(shards)} - node_outcomes = [ - outcome_by_node.get(nodeid, {"nodeid": nodeid, "outcome": "missing", "reason": "shard not completed"}) - for nodeid in expected - ] - shard_steps: list[Mapping[str, Any]] = [] - for shard in shards: - raw_step = shard.get("pytest_step") - if isinstance(raw_step, Mapping): - shard_steps.append(raw_step) - if shard_steps: - pytest_step = dict(shard_steps[-1]) - timed_out = next( - ( - step - for step in shard_steps - if str(step.get("diagnosis")) in {"pytest_timeout", "pytest_stall_timeout"} - ), - None, - ) - if timed_out is not None: - pytest_step = dict(timed_out) - selection = { - "selected_count": len(expected), - "selected_nodeids_omitted": 0, - "shard_count": len(shards), - "completed_shard_count": sum(shard.get("status") == SeedShardStatus.COMPLETE.value for shard in shards), - } - else: - omitted = raw_omitted if selection_valid and isinstance(raw_omitted, int) else 1 - database = _testmon_database_state(expected) - node_outcomes = _seed_node_outcomes_from_events( - events_path or Path(".missing-testmon-events"), - expected_nodeids=expected, - database=database, - pytest_step=pytest_step, - use_database_fallback=False, - prior_node_outcomes={ - str(item["nodeid"]): item - for item in prepared.get("prior_node_outcomes", []) - if isinstance(item, Mapping) and isinstance(item.get("nodeid"), str) - }, - ) - unsuccessful_nodeids = [ - str(item["nodeid"]) for item in node_outcomes if item.get("outcome") not in SUCCESSFUL_NODE_OUTCOMES - ] - green_complete = ( - exit_code == 0 - and bool(expected) - and selection_valid - and omitted == 0 - and database["error"] is None - and database["graph_status"] == "complete" - and not database["missing_nodeids"] - and not database["failed_nodeids"] - and database["orphan_execution_edges"] == 0 - and database["orphan_fingerprint_edges"] == 0 - and not unsuccessful_nodeids - ) - identity = prepared.get("identity") - narrow_terminal = isinstance(identity, Mapping) and identity.get("skip_slow") is True - terminal_authorized = _testmon_seed_terminal_authorized(prepared) - release_eligible = green_complete and (not narrow_terminal or terminal_authorized) - terminal_graph = ( - database["error"] is None - and database["graph_status"] == GraphStatus.COMPLETE.value - and not database["missing_nodeids"] - and database["orphan_execution_edges"] == 0 - and database["orphan_fingerprint_edges"] == 0 - and all(item.get("outcome") in TERMINAL_NODE_OUTCOMES for item in node_outcomes) - ) - outcome = _seed_attempt_outcome( - release_eligible=release_eligible, - terminal_graph=terminal_graph, - exit_code=exit_code, - pytest_step=pytest_step, - ) - shard_ledger = ( - shards if sharded else seed_shard_plan(expected, shard_size=max(1, len(expected))) if expected else [] - ) - if not sharded and shard_ledger: - shard_ledger[0].update( - { - "status": ( - SeedShardStatus.COMPLETE.value - if all(item.get("outcome") in TERMINAL_NODE_OUTCOMES for item in node_outcomes) - else SeedShardStatus.INCOMPLETE.value - ), - "node_outcomes": node_outcomes, - } - ) - seed_scope = ( - VerificationScope.NARROW_TERMINAL.value if narrow_terminal else VerificationScope.RELEASE_BASELINE.value - ) - attempt_candidate = { - **dict(prepared), - "status": "complete" if release_eligible else "reusable" if terminal_graph else "incomplete", - "outcome": outcome.value, - "exit_code": exit_code, - "expected_nodeids": expected, - "expected_count": len(expected), - "expected_digest": hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest() if expected else None, - "selection": { - **selection, - # A resumed run inherits the complete collection ledger from its - # original selection. The current pytest step may select only a - # subset while it repairs missing graph edges. - "selected_count": len(expected) - if prepared.get("resume") and selection_valid - else selection.get("selected_count"), - "selected_nodeids_omitted": 0 if prepared.get("resume") and selection_valid else omitted, - }, - "shards": shard_ledger, - "node_outcomes": node_outcomes, - "identity": prepared.get("identity"), - "run_id": prepared.get("run_id"), - "artifact_dir": prepared.get("artifact_dir"), - "testmon_data": _file_fingerprint(TESTMON_DATA), - "verification_scope": seed_scope, - "terminal_authorization": (TerminalAuthorization.NARROW_TERMINAL.value if terminal_authorized else None), - "release_baseline_allowed": release_eligible, - } - reusable_stamp = stamp_from_attempt( - attempt_candidate, + required_executable_paths: Sequence[str], +) -> NativeTestmonState: + return inspect_native_testmon_environment( TESTMON_DATA, - checkout_root=Path.cwd(), - protocol_version=TESTMON_SEED_PROTOCOL_VERSION, + environment_name=preparation.environment_name, + required_executable_paths=required_executable_paths, ) - reusable = reusable_stamp is not None - release_permission = bool( - reusable - and reusable_stamp is not None - and reusable_stamp.release_baseline_allowed - and (not narrow_terminal or terminal_authorized) - ) - attempt_status = "complete" if green_complete and release_permission else "reusable" if reusable else "incomplete" - payload = { - **dict(prepared), - "status": attempt_status, - "outcome": outcome.value, - "finished_at": datetime.now(timezone.utc).isoformat(), - "exit_code": exit_code, - "expected_nodeids": expected, - "expected_count": len(expected), - "expected_digest": hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest() if expected else None, - "selection": { - key: ( - len(expected) - if key == "selected_count" and prepared.get("resume") and selection_valid - else 0 - if key == "selected_nodeids_omitted" and prepared.get("resume") and selection_valid - else selection.get(key) - ) - for key in ( - "selected_count", - "deselected_count", - "selected_nodeids_omitted", - "deselected_nodeids_omitted", - "collection_duration_s", - ) - }, - "shards": shard_ledger, - "database": database, - "node_outcomes": node_outcomes, - "node_outcome_counts": dict( - sorted( - { - outcome: sum(1 for item in node_outcomes if item.get("outcome") == outcome) - for outcome in {str(item.get("outcome")) for item in node_outcomes} - }.items() - ) - ), - "unsuccessful_nodeids": unsuccessful_nodeids, - "testmon_data": _file_fingerprint(TESTMON_DATA), - "pytest_step": dict(pytest_step) if pytest_step is not None else None, - "binding": TestmonBinding(BindingMode.EXACT, str(ROOT.resolve())).as_dict(), - "verification_scope": seed_scope, - "terminal_authorization": (TerminalAuthorization.NARROW_TERMINAL.value if terminal_authorized else None), - } - payload["release_baseline_allowed"] = release_permission - _atomic_write_json(TESTMON_SEED_ATTEMPT, payload) - if release_permission and reusable_stamp is not None: - _atomic_write_json(TESTMON_SEED_STAMP, reusable_stamp.as_dict()) - else: - TESTMON_SEED_STAMP.unlink(missing_ok=True) - return payload -def _refresh_testmon_selection_attempt( +def _release_baseline_allowed( *, - step: Mapping[str, Any], - run: VerifyRun, + selection_mode: str | None, exit_code: int, -) -> None: - """Refresh a reusable red graph after every completed affected run.""" - attempt = _read_testmon_seed_attempt() - if attempt is None or attempt.get("release_baseline_allowed") is True: - return - expected = _testmon_seed_expected_nodeids(attempt) - if not expected: - return - database = _testmon_database_state(expected) - artifact_dir = _safe_testmon_artifact_dir(step.get("artifact_dir")) - events_path = artifact_dir / "events.jsonl" if artifact_dir is not None else Path(".missing-testmon-events") - prior = { - str(item["nodeid"]): item - for item in attempt.get("node_outcomes", []) - if isinstance(item, Mapping) and isinstance(item.get("nodeid"), str) - } - node_outcomes = _seed_node_outcomes_from_events( - events_path, - expected_nodeids=expected, - database=database, - pytest_step=step, - use_database_fallback=False, - prior_node_outcomes=prior, - ) - graph_complete = ( - database.get("graph_status") == GraphStatus.COMPLETE.value - and not database.get("missing_nodeids") - and database.get("error") is None - and database.get("orphan_execution_edges") == 0 - and database.get("orphan_fingerprint_edges") == 0 + checkout_stable: bool, + aggregate: Mapping[str, Any] | None, +) -> bool: + if selection_mode not in {"bootstrap", "full"} or exit_code != 0 or not checkout_stable or aggregate is None: + return False + cleanup = aggregate.get("cleanup") + containment = aggregate.get("containment") + deadline = aggregate.get("deadline") + return bool( + aggregate.get("complete_corpus_covered") is True + and aggregate.get("terminal_green") is True + and isinstance(cleanup, Mapping) + and cleanup.get("complete") is True + and isinstance(containment, Mapping) + and containment.get("complete") is True + and isinstance(deadline, Mapping) + and deadline.get("met") is True ) - terminal = all(item.get("outcome") in TERMINAL_NODE_OUTCOMES for item in node_outcomes) - prior_selection = attempt.get("selection") - payload = { - **attempt, - "status": "reusable" if graph_complete and terminal else "incomplete", - "outcome": ( - SeedAttemptOutcome.RED_BASELINE.value - if graph_complete and terminal - else SeedAttemptOutcome.INCOMPLETE.value - ), - "finished_at": datetime.now(timezone.utc).isoformat(), - "exit_code": exit_code, - "expected_nodeids": expected, - "expected_count": len(expected), - "expected_digest": hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest(), - "selection": { - **(dict(prior_selection) if isinstance(prior_selection, Mapping) else {}), - "selected_count": len(expected), - "selected_nodeids_omitted": 0, - }, - "database": database, - "node_outcomes": node_outcomes, - "node_outcome_counts": dict( - sorted( - { - outcome: sum(1 for item in node_outcomes if item.get("outcome") == outcome) - for outcome in {str(item.get("outcome")) for item in node_outcomes} - }.items() - ) - ), - "unsuccessful_nodeids": [ - str(item["nodeid"]) for item in node_outcomes if item.get("outcome") not in SUCCESSFUL_NODE_OUTCOMES - ], - "testmon_data": _file_fingerprint(TESTMON_DATA), - "run_id": run.run_id, - "artifact_dir": str(run.relative_run_dir), - "pytest_step": dict(step), - "release_baseline_allowed": False, - "verification_scope": VerificationScope.AFFECTED.value, - } - raw_binding = attempt.get("binding") - if not isinstance(raw_binding, Mapping): - payload["binding"] = TestmonBinding(BindingMode.EXACT, str(ROOT.resolve())).as_dict() - _atomic_write_json(TESTMON_SEED_ATTEMPT, payload) - - -def _discard_testmon_dependency_authority() -> None: - """Remove a dependency graph learned while checkout authority was unstable.""" - for path in ( - TESTMON_SEED_STAMP, - TESTMON_SEED_ATTEMPT, - TESTMON_DATA, - Path(f"{TESTMON_DATA}-wal"), - Path(f"{TESTMON_DATA}-shm"), - Path(f"{TESTMON_DATA}-journal"), - ): - path.unlink(missing_ok=True) - - -# ── main ──────────────────────────────────────────────────────────── - - -_ACTIVE_VERIFY_RUN: tuple[VerifyRun, float, VerificationScope] | None = None - - -def _planned_verification_scope(args: argparse.Namespace, *, full_pytest: bool) -> VerificationScope: - """Return the immutable scope requested before the runner starts.""" - if args.quick or args.commit: - return VerificationScope.NON_TEST - if full_pytest or args.seed_testmon: - return VerificationScope.NARROW_TERMINAL if args.skip_slow else VerificationScope.RELEASE_BASELINE - return VerificationScope.AFFECTED - - -def _changed_paths_from_testmon_authority(base_commit: str | None, head_commit: str | None) -> tuple[str, ...]: - """Require immutable refs before deriving affected executable paths.""" - if base_commit is None or head_commit is None: - raise PytestResourceError("testmon changed-path authority is unavailable") - return _changed_executable_paths(base_commit, head_commit) def _main(argv: list[str] | None = None) -> int: @@ -3430,16 +2447,11 @@ def _main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description="Run the local verification baseline.") parser.add_argument("--quick", action="store_true", help="Skip pytest and run only fast local gates.") parser.add_argument( - "--seed-testmon", + "--all", action="store_true", - help="Run full non-integration pytest with --testmon-noselect to seed/update .cache/testmon/testmondata.", - ) - parser.add_argument( - "--all", action="store_true", help="Force the full non-integration pytest diagnostic instead of testmon." - ) - parser.add_argument( - "--full", action="store_true", help="Alias for --all: run full non-integration pytest diagnostic." + help="Run the complete non-integration corpus in the current native testmon environment.", ) + parser.add_argument("--full", action="store_true", help="Alias for --all.") parser.add_argument("--commit", action="store_true", help="Pre-commit tier: format + lint + mypy only.") parser.add_argument( "--skip-slow", action="store_true", help="Exclude @pytest.mark.slow tests from the pytest step." @@ -3452,22 +2464,13 @@ def _main(argv: list[str] | None = None) -> int: parser.add_argument( "--lab", action="store_true", - help=( - "Run the default pytest-testmon baseline plus verification-lab " - "scenario and verify-slos checks; does not imply --all." - ), + help="Run the native pytest-testmon lifecycle plus verification-lab checks.", ) parser.add_argument("--history", action="store_true", help="Print last 10 verify runs and exit.") parser.add_argument("--json", action="store_true", default=None, help="Write structured JSON to stdout.") args = parser.parse_args(sys.argv[1:] if argv is None else argv) _anchor_verification_paths() - bootstrap_message = maybe_bootstrap_testmon_seed( - ROOT, - protocol_version=TESTMON_SEED_PROTOCOL_VERSION, - ) - if bootstrap_message is not None: - sys.stderr.write(bootstrap_message + "\n") try: fingerprint = assert_polylogue_matches_checkout(ROOT, context="devtools verify") except CheckoutImportMismatchError as exc: @@ -3481,45 +2484,61 @@ def _main(argv: list[str] | None = None) -> int: _print_history() return 0 - # Auto-detect JSON when stdout is not a TTY (agent/pipe context). - use_json = args.json if args.json is not None else not sys.stdout.isatty() - - tier = "full" - if args.commit: - tier = "commit" - elif args.quick: - tier = "quick" - elif args.seed_testmon: - tier = "seed-testmon" - elif args.all or args.full: - tier = "full" - elif args.lab: - tier = "lab" - else: - tier = "testmon" + full_requested = bool(args.all or args.full) + if args.terminal_authorization is not None and not (full_requested and args.skip_slow): + parser.error("--terminal-authorization requires --all or --full with --skip-slow") + use_json = args.json if args.json is not None else not sys.stdout.isatty() + tier = ( + "commit" + if args.commit + else "quick" + if args.quick + else "full" + if full_requested + else "lab" + if args.lab + else "testmon" + ) head = _git_head() - full_pytest = bool(args.all or args.full) - affected_testmon = not (args.quick or args.commit or args.seed_testmon or full_pytest) - planned_verification_scope = _planned_verification_scope(args, full_pytest=full_pytest) - testmon_base_commit = _git_commit("origin/master") if affected_testmon else None - testmon_head_commit = head if affected_testmon else None - if affected_testmon and (testmon_base_commit is None or testmon_head_commit is None): - sys.stderr.write("verify: cannot resolve immutable Git refs for affected-test authority.\n") + pytest_enabled = not (args.quick or args.commit) + base_commit = _git_commit("origin/master") if pytest_enabled else None + if pytest_enabled and (base_commit is None or head is None): + sys.stderr.write("verify: cannot resolve immutable Git refs for native affected-test authority.\n") return 125 - if args.terminal_authorization is not None and not ((full_pytest or args.seed_testmon) and args.skip_slow): - parser.error("--terminal-authorization requires --all, --full, or --seed-testmon with --skip-slow") - preflight_error = _testmon_preflight( - seed_testmon=bool(args.seed_testmon), - full_pytest=full_pytest, - quick=bool(args.quick), - commit=bool(args.commit), - ) - if preflight_error is not None: - sys.stderr.write(preflight_error) - return 2 - t0 = time.monotonic() + started_at = time.monotonic() + relevant_paths: tuple[str, ...] = () + required_executable_paths: tuple[str, ...] = () + preparation: NativeTestmonPreparation | None = None + testmon_mode: str | None = None + if pytest_enabled: + assert base_commit is not None + assert head is not None + try: + relevant_paths = _changed_test_relevant_paths(base_commit, head) + required_executable_paths = executable_python_paths(ROOT, relevant_paths) + preparation = prepare_native_testmon_environment( + ROOT, + required_executable_paths=required_executable_paths, + pytest_profile=_pytest_profile(skip_slow=bool(args.skip_slow), lab=bool(args.lab)), + ) + except (NativeTestmonRepairError, PytestResourceError) as exc: + sys.stderr.write(f"verify: native pytest-testmon preparation failed: {exc}\n") + return 125 + testmon_mode = "full" if full_requested else preparation.selection_mode + if preparation.removed_paths: + sys.stderr.write( + "verify: repaired invalid native pytest-testmon state by removing only " + + ", ".join(str(path) for path in preparation.removed_paths) + + "\n" + ) + if preparation.copied_from is not None: + sys.stderr.write(f"verify: copied matching native pytest-testmon DB from {preparation.copied_from}\n") + elif preparation.selection_mode == "bootstrap": + sys.stderr.write("verify: native pytest-testmon environment is empty; plain verify will build it\n") + + planned_scope = _planned_verification_scope(args, testmon_mode=testmon_mode) mutation_monitor = CheckoutMutationMonitor(ROOT) start_checkout_mutation_monitor(mutation_monitor) checkout_fingerprint = worktree_fingerprint(ROOT) @@ -3531,224 +2550,71 @@ def _main(argv: list[str] | None = None) -> int: environment_fingerprint=environment_fingerprint, worktree_fingerprint=checkout_fingerprint, ) - _ACTIVE_VERIFY_RUN = (verify_run, t0, planned_verification_scope) - seed_identity: dict[str, Any] | None = None - resume_testmon_seed = False - prepared_seed_attempt: dict[str, Any] | None = None - if args.seed_testmon: - try: - seed_identity = _testmon_seed_identity( - git_head=head, - git_tree=_git_committed_tree(), - skip_slow=bool(args.skip_slow), - lab=bool(args.lab), - terminal_authorization=args.terminal_authorization, - ) - except RuntimeError as exc: - finish_checkout_mutation_monitor(mutation_monitor) - sys.stderr.write(f"verify: {exc}\n") - early_payload = verify_run.finish( - exit_code=125, - duration_s=time.monotonic() - t0, - diagnosis="testmon_environment_identity_unavailable", - ) - _save_history(early_payload) - return 125 - resume_testmon_seed = _testmon_seed_can_resume(seed_identity) - prepared_seed_attempt = _prepare_testmon_seed_attempt( - identity=seed_identity, - run=verify_run, - resume=resume_testmon_seed, - ) - if resume_testmon_seed: - sys.stderr.write("verify: resuming the matching incomplete pytest-testmon seed\n") + _ACTIVE_VERIFY_RUN = (verify_run, started_at, planned_scope) if not use_json: sys.stderr.write("verify: running local verification baseline\n") - - # Resource preflight before heavy steps. - if not args.quick and not args.commit: + if pytest_enabled: _warn_low_memory() - exit_code = 0 try: steps = build_verify_steps( quick=bool(args.quick), commit=bool(args.commit), lab=bool(args.lab), skip_slow=bool(args.skip_slow), - seed_testmon=bool(args.seed_testmon), - resume_testmon_seed=resume_testmon_seed, - full_pytest=full_pytest, - broad_testmon=( - _default_testmon_is_broad_change(testmon_base_commit, testmon_head_commit) - if testmon_base_commit is not None and testmon_head_commit is not None - else False - ), + testmon_mode=testmon_mode or "affected", + testmon_environment=preparation.environment_name if preparation is not None else "", ) - except PytestResourceError as exc: + except (PytestResourceError, ValueError) as exc: finish_checkout_mutation_monitor(mutation_monitor) - sys.stderr.write(f"verify: {exc}\n") - early_payload = verify_run.finish( + payload = verify_run.finish( exit_code=125, - duration_s=time.monotonic() - t0, + duration_s=time.monotonic() - started_at, diagnosis="pytest_resource_preflight_failed", + verification_scope=planned_scope.value, + release_baseline_allowed=False, ) - _save_history(early_payload) + _save_history(payload) + sys.stderr.write(f"verify: {exc}\n") return 125 step_results: list[dict[str, Any]] = [] - pending_testmon_stamp: TestmonSeedStamp | None = None - pending_affected_coverage: tuple[tuple[str, ...], int] | None = None - pending_selection_refresh: tuple[dict[str, Any], int] | None = None - testmon_graph_touched = False - changed_path_authority_failed = False + exit_code = 0 + native_graph_touched = False for label, cmd in steps: + remaining = _remaining_invocation_budget(started_at) + if remaining <= 0: + deadline_step = { + "name": label, + "duration_s": 0.0, + "exit": 124, + "diagnosis": "verify_invocation_deadline_exceeded", + "timeout_s": VERIFY_INVOCATION_BUDGET_S, + } + step_results.append(deadline_step) + exit_code = 124 + break if label.startswith("pytest"): - _warn_low_memory() # check again right before the heavy step - rc, elapsed, metadata = _run(label, cmd, run=verify_run) - if label in {"pytest testmon", "pytest testmon (broad)"} or label.startswith("pytest seed-testmon"): - testmon_graph_touched = True - if rc == 0 and label in {"pytest testmon", "pytest testmon (broad)"}: - raw_stamp = _read_json_artifact(TESTMON_SEED_STAMP) - try: - current_stamp = ( - TestmonSeedStamp.from_mapping(raw_stamp, protocol_version=TESTMON_SEED_PROTOCOL_VERSION) - if isinstance(raw_stamp, Mapping) - else None - ) - except ValueError: - current_stamp = None - if current_stamp is not None: - refreshed_stamp = refresh_stamp(current_stamp, TESTMON_DATA) - if refreshed_stamp is not None: - pending_testmon_stamp = refreshed_stamp - try: - executable_paths = _changed_paths_from_testmon_authority(testmon_base_commit, testmon_head_commit) - except PytestResourceError as exc: - changed_path_authority_failed = True - executable_paths = () - rc = 125 - metadata["diagnosis"] = "testmon_changed_path_authority_unavailable" - metadata["error"] = str(exc) - pending_testmon_stamp = None - sys.stderr.write( - "verify: changed-path authority became unavailable after pytest; " - "discarding the affected dependency graph.\n" - ) - selected_count = metadata.get("selected_count") - if selected_count == 0 and executable_paths: - coverage = _matching_testmon_coverage(executable_paths) - if coverage is None: - rc = 5 - metadata["diagnosis"] = "zero_testmon_selection_for_executable_change" - metadata["zero_selection_changed_paths"] = list(executable_paths) - sys.stderr.write( - "verify: pytest-testmon selected zero tests for executable changes and no " - "matching successful coverage receipt exists; refresh the seed or repair " - "dependency capture: " + ", ".join(executable_paths) + "\n" - ) - else: - metadata["zero_selection_coverage"] = coverage - elif isinstance(selected_count, int) and selected_count > 0: - pending_affected_coverage = (tuple(executable_paths), selected_count) + _warn_low_memory() + rc, elapsed, metadata = _run(label, cmd, run=verify_run, timeout_s=remaining) step_result: dict[str, Any] = {"name": label, "duration_s": round(elapsed, 2), "exit": rc} step_result.update(metadata) + if label.startswith("pytest native parallel"): + step_result["semantic_lane"] = "parallel" + native_graph_touched = True + elif label.startswith("pytest native serial"): + step_result["semantic_lane"] = "serial" + native_graph_touched = True step_results.append(step_result) - if args.seed_testmon and label.startswith("pytest seed-testmon collect"): - if rc != 0: - exit_code = rc - break - artifact_dir = _safe_testmon_artifact_dir(metadata.get("artifact_dir")) - selection = _read_json_artifact(artifact_dir / "selection.json") if artifact_dir is not None else None - assert prepared_seed_attempt is not None - prepared_seed_attempt = _prepare_testmon_seed_shards( - prepared_seed_attempt, - selection=selection if isinstance(selection, Mapping) else None, - ) - expected = _testmon_seed_expected_nodeids(prepared_seed_attempt) - shards = validate_seed_shard_ledger(prepared_seed_attempt.get("shards"), expected_nodeids=expected) - if shards is None: - exit_code = 5 - step_result["exit"] = 5 - step_result["diagnosis"] = "testmon_seed_collection_incomplete" - sys.stderr.write("verify: pytest-testmon collection did not produce a complete shard plan.\n") - break - for shard in shards: - if shard.get("status") == SeedShardStatus.COMPLETE.value: - continue - shard_index = int(shard["index"]) - shard_label = f"pytest seed-testmon shard {shard_index}/{len(shards)}" - shard_args_path = verify_run.run_dir / "seed-shards" / f"{shard_index:04d}.args" - try: - shard_cmd = _seed_shard_command(cmd, shard, nodeids_file=shard_args_path) - except (OSError, PytestResourceError) as exc: - resource_failure_result = { - "name": shard_label, - "duration_s": 0.0, - "exit": 125, - "diagnosis": ( - "pytest_resource_refusal" - if isinstance(exc, PytestResourceError) - else "testmon_seed_args_file_write_failed" - ), - "error": str(exc), - "shard_index": shard_index, - "shard_count": len(shards), - "shard_nodeid_count": len(shard["nodeids"]), - } - step_results.append(resource_failure_result) - prepared_seed_attempt = _checkpoint_testmon_seed_shard( - prepared=prepared_seed_attempt, - shard_index=shard_index, - step=resource_failure_result, - ) - exit_code = 125 - break - _warn_low_memory() - shard_rc, shard_elapsed, shard_metadata = _run(shard_label, shard_cmd, run=verify_run) - shard_result: dict[str, Any] = { - "name": shard_label, - "duration_s": round(shard_elapsed, 2), - "exit": shard_rc, - "shard_index": shard_index, - "shard_count": len(shards), - "shard_nodeid_count": len(shard["nodeids"]), - } - shard_result.update(shard_metadata) - step_results.append(shard_result) - prepared_seed_attempt = _checkpoint_testmon_seed_shard( - prepared=prepared_seed_attempt, - shard_index=shard_index, - step=shard_result, - ) - checkpointed_shards = prepared_seed_attempt.get("shards") - if ( - not isinstance(checkpointed_shards, list) - or shard_index > len(checkpointed_shards) - or not isinstance(checkpointed_shards[shard_index - 1], Mapping) - ): - raise RuntimeError("testmon seed shard checkpoint is malformed") - shard_complete = checkpointed_shards[shard_index - 1].get("status") == SeedShardStatus.COMPLETE.value - if shard_rc != 0: - stop_seed = _seed_shard_failure_requires_stop( - shard_result, - shard_complete=shard_complete, - ) - if exit_code == 0 or stop_seed: - # A later infrastructure failure is the terminal - # condition even when an earlier shard recorded - # ordinary red-test evidence. - exit_code = shard_rc - if stop_seed: - break + if rc == 0: continue - if label in {"pytest testmon", "pytest testmon (broad)"} and not args.seed_testmon and not full_pytest: - pending_selection_refresh = (step_result, rc) - if rc != 0: + if exit_code == 0 or rc in {2, 3, 4, 124, 125, 130}: exit_code = rc - if rc == 130 or _stop_after_failed_step(label): - break + if label.startswith("pytest native parallel") and not _native_lane_failure_requires_stop(step_result): + continue + if label.startswith("pytest") or rc == 130 or _stop_after_failed_step(label): + break final_head = _git_head() final_checkout_fingerprint = worktree_fingerprint(ROOT) @@ -3763,96 +2629,109 @@ def _main(argv: list[str] | None = None) -> int: final_checkout_fingerprint, } ) - if changed_path_authority_failed or checkout_fingerprint_unavailable or mutation_observation.unavailable: + if checkout_fingerprint_unavailable or mutation_observation.unavailable: checkout_stable = False diagnosis = ( - "testmon_changed_path_authority_unavailable" - if changed_path_authority_failed - else ( - "checkout_fingerprint_unavailable" - if checkout_fingerprint_unavailable - else "checkout_mutation_monitor_unavailable" - ) + "checkout_fingerprint_unavailable" + if checkout_fingerprint_unavailable + else "checkout_mutation_monitor_unavailable" ) - step_results.append( - { - "name": "checkout stability", - "duration_s": 0.0, - "exit": 125, - "diagnosis": diagnosis, - "initial_git_head": head, - "final_git_head": final_head, - "initial_worktree_fingerprint": checkout_fingerprint, - "final_worktree_fingerprint": final_checkout_fingerprint, - } - ) - if exit_code == 0: - exit_code = 125 - sys.stderr.write(f"verify: {diagnosis.replace('_', ' ')}; evidence is not exact-head.\n") elif final_head != head or mutation_observation.changed or final_checkout_fingerprint != checkout_fingerprint: checkout_stable = False - step_results.append( - { - "name": "checkout stability", + diagnosis = "checkout_changed_during_verification" + else: + diagnosis = None + if diagnosis is not None: + stability_step = { + "name": "checkout stability", + "duration_s": 0.0, + "exit": 125, + "diagnosis": diagnosis, + "initial_git_head": head, + "final_git_head": final_head, + "initial_worktree_fingerprint": checkout_fingerprint, + "final_worktree_fingerprint": final_checkout_fingerprint, + "transient_checkout_mutation": mutation_observation.changed, + "checkout_mutation_path": mutation_observation.observed_path, + } + step_results.append(stability_step) + exit_code = 125 + sys.stderr.write("verify: checkout contents were not stable for exact-head evidence.\n") + if native_graph_touched: + try: + removed = remove_invalid_native_testmon_state(ROOT) + except NativeTestmonRepairError as exc: + stability_step["testmon_cleanup_error"] = str(exc) + else: + stability_step["testmon_cleanup_paths"] = [str(path) for path in removed] + + native_state = None + if preparation is not None: + native_state = _native_environment_after_run( + preparation, + required_executable_paths=required_executable_paths, + ) + if not native_state.valid: + graph_step = { + "name": "pytest native graph validation", "duration_s": 0.0, - "exit": 125, - "diagnosis": "checkout_changed_during_verification", - "initial_git_head": head, - "final_git_head": final_head, - "initial_worktree_fingerprint": checkout_fingerprint, - "final_worktree_fingerprint": final_checkout_fingerprint, - "transient_checkout_mutation": mutation_observation.changed, - "checkout_mutation_path": mutation_observation.observed_path, + "exit": 5, + "diagnosis": "native_testmon_graph_invalid", + "reason": native_state.reason, + "missing_executable_paths": list(native_state.missing_executable_paths), } - ) - if exit_code == 0: - exit_code = 125 - sys.stderr.write("verify: checkout contents changed during verification; evidence is not exact-head.\n") - - seed_receipt: dict[str, Any] | None = None - if checkout_stable: - if pending_testmon_stamp is not None: - _atomic_write_json(TESTMON_SEED_STAMP, pending_testmon_stamp.as_dict()) - if pending_affected_coverage is not None: - executable_paths, selected_count = pending_affected_coverage - _record_testmon_affected_coverage( - executable_paths=executable_paths, - selected_count=selected_count, - run_id=verify_run.run_id, - ) - if pending_selection_refresh is not None: - step_result, selection_exit_code = pending_selection_refresh - _refresh_testmon_selection_attempt( - step=step_result, - run=verify_run, - exit_code=selection_exit_code, - ) - if prepared_seed_attempt is not None: - seed_receipt = _finalize_testmon_seed_attempt( - prepared=prepared_seed_attempt, - step_results=step_results, - exit_code=exit_code, - ) - if exit_code == 0 and seed_receipt["status"] != "complete": + step_results.append(graph_step) + if exit_code == 0: exit_code = 5 + if native_state.missing_executable_paths: sys.stderr.write( - "verify: pytest passed but the testmon dependency baseline is incomplete; " - f"inspect {TESTMON_SEED_ATTEMPT}.\n" + "verify: changed executable modules have no runtime dependency edge: " + + ", ".join(native_state.missing_executable_paths) + + "\n" ) - elif testmon_graph_touched: - _discard_testmon_dependency_authority() - if prepared_seed_attempt is not None: - seed_receipt = { - "status": "discarded", - "outcome": SeedAttemptOutcome.INCOMPLETE.value, - "resume": False, - "expected_count": len(_testmon_seed_expected_nodeids(prepared_seed_attempt)), - "release_baseline_allowed": False, - } - total_duration = round(time.monotonic() - t0, 2) + total_duration = round(time.monotonic() - started_at, 2) + pytest_aggregate: dict[str, Any] | None = None + native_environment = native_state.environment if native_state is not None else None + if preparation is not None and native_environment is not None: + pytest_aggregate = aggregate_native_testmon_run( + ROOT, + steps=step_results, + environment_name=preparation.environment_name, + corpus_nodeids=native_environment.nodeids, + selection_mode=testmon_mode or "affected", + invocation_duration_s=total_duration, + budget_s=VERIFY_INVOCATION_BUDGET_S, + ) + + release_baseline_allowed = _release_baseline_allowed( + selection_mode=testmon_mode, + exit_code=exit_code, + checkout_stable=checkout_stable, + aggregate=pytest_aggregate, + ) + verification_scope = planned_scope + if testmon_mode == "affected": + release_baseline_allowed = False + + checkout_diagnosis = next( + ( + str(step["diagnosis"]) + for step in reversed(step_results) + if step.get("name") == "checkout stability" and isinstance(step.get("diagnosis"), str) + ), + None, + ) + pytest_diagnosis = next( + ( + str(step["diagnosis"]) + for step in reversed(step_results) + if str(step.get("name", "")).startswith("pytest") and step.get("exit") != 0 + ), + None, + ) + run_diagnosis = checkout_diagnosis or pytest_diagnosis - # Build history entry. history_entry: dict[str, Any] = { "timestamp": datetime.now(timezone.utc).isoformat(), "git_head": head, @@ -3866,91 +2745,22 @@ def _main(argv: list[str] | None = None) -> int: "steps": step_results, "total_duration_s": total_duration, "exit_code": exit_code, + "verification_scope": verification_scope.value, + "release_baseline_allowed": release_baseline_allowed, + "terminal_authorization": args.terminal_authorization, } - checkout_stability_diagnosis = next( - ( - str(step["diagnosis"]) - for step in reversed(step_results) - if step.get("name") == "checkout stability" and "diagnosis" in step - ), - None, - ) - fallback_pytest_diagnosis = next( - ( - str(step["diagnosis"]) - for step in reversed(step_results) - if str(step.get("name", "")).startswith("pytest") and "diagnosis" in step - ), - None, - ) - pytest_diagnosis = next( - ( - str(step["diagnosis"]) - for step in reversed(step_results) - if str(step.get("name", "")).startswith("pytest") and step.get("exit") == exit_code and "diagnosis" in step - ), - fallback_pytest_diagnosis, - ) - run_diagnosis = checkout_stability_diagnosis or pytest_diagnosis + if preparation is not None: + history_entry["testmon_environment"] = { + "name": preparation.environment_name, + "selection_mode": testmon_mode, + "copied_from": str(preparation.copied_from) if preparation.copied_from is not None else None, + "required_executable_paths": list(required_executable_paths), + } + if pytest_aggregate is not None: + history_entry["pytest_aggregate"] = pytest_aggregate if run_diagnosis is not None: history_entry["diagnosis"] = run_diagnosis - if seed_receipt is not None: - history_entry["testmon_seed"] = { - "status": seed_receipt["status"], - "outcome": seed_receipt["outcome"], - "resume": seed_receipt["resume"], - "expected_count": seed_receipt["expected_count"], - "attempt_path": str(TESTMON_SEED_ATTEMPT), - "stamp_path": str(TESTMON_SEED_STAMP) if seed_receipt["release_baseline_allowed"] else None, - "release_baseline_allowed": seed_receipt["release_baseline_allowed"], - } - - verification_scope = planned_verification_scope - if args.quick or args.commit: - # Non-test verification is intentionally not release authority, but it - # is still a typed verification receipt. ``None`` made merge-gate - # treat an explicit quick receipt as malformed instead of as a valid - # non-release gate. - release_baseline_allowed: bool | None = False - elif full_pytest or args.seed_testmon: - narrow_terminal = bool(args.skip_slow) - authorized_narrow_terminal = args.terminal_authorization == TerminalAuthorization.NARROW_TERMINAL.value - if full_pytest: - release_baseline_allowed = exit_code == 0 and (not narrow_terminal or authorized_narrow_terminal) - else: - release_baseline_allowed = _testmon_release_baseline_permission() and ( - not narrow_terminal or authorized_narrow_terminal - ) - else: - release_baseline_allowed = _testmon_release_baseline_permission() - history_entry["verification_scope"] = verification_scope.value - history_entry["release_baseline_allowed"] = release_baseline_allowed - history_entry["terminal_authorization"] = args.terminal_authorization - if release_baseline_allowed is False and tier in {"testmon", "lab", "seed-testmon"}: - sys.stderr.write( - "verify: affected-test selection is usable, but the current testmon state does not grant " - "release-baseline permission.\n" - ) - - if use_json: - _print_json(history_entry) - else: - if exit_code == 0: - # Compare against last run, flag regressions. - flags = _compare_against_last(step_results) - sys.stderr.write(f"\nverify: all checks passed ({total_duration:.1f}s total)") - if flags: - sys.stderr.write(" — " + "; ".join(flags) if len(flags) == 1 else "") - sys.stderr.write("\n") - for flag in flags: - sys.stderr.write(flag + "\n") - else: - sys.stderr.write("\n") - else: - sys.stderr.write(f"\nverify: FAILED ({total_duration:.1f}s) — fix before pushing\n") - # Persist history and stamp. - _save_history(history_entry) verify_run.finish( exit_code=exit_code, duration_s=total_duration, @@ -3960,14 +2770,28 @@ def _main(argv: list[str] | None = None) -> int: terminal_authorization=args.terminal_authorization, final_worktree_fingerprint=final_checkout_fingerprint, checkout_mutation_path=mutation_observation.observed_path, + checkout_diagnosis=checkout_diagnosis, + pytest_aggregate=pytest_aggregate, ) + if use_json: + _print_json(history_entry) + elif exit_code == 0: + flags = _compare_against_last(step_results) + sys.stderr.write(f"\nverify: all checks passed ({total_duration:.1f}s total)") + if flags: + sys.stderr.write("\n") + for flag in flags: + sys.stderr.write(flag + "\n") + else: + sys.stderr.write("\n") + else: + sys.stderr.write(f"\nverify: FAILED ({total_duration:.1f}s); fix before pushing\n") + + _save_history(history_entry) + if exit_code == 0: _stamp_head() - - # Notify only on failure. Passing runs stay silent — the terminal - # already shows the green summary and a desktop popup per run is - # spammy when verify is invoked on every push. - if exit_code != 0: + else: _notify( _format_completion_notification( exit_code=exit_code, @@ -3975,7 +2799,6 @@ def _main(argv: list[str] | None = None) -> int: step_results=step_results, ) ) - return exit_code diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index ec9d1bee52..85fa89bdf4 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -964,6 +964,30 @@ def aggregate_pytest_statistics( ), default=None, ), + "peak_tree_swap_pss_kb": max( + (int(row["tree_swap_pss_kb"]) for row in resources if isinstance(row.get("tree_swap_pss_kb"), int)), + default=None, + ), + "tree_read_bytes_delta": ( + max( + (int(row["tree_read_bytes"]) for row in resources if isinstance(row.get("tree_read_bytes"), int)), + default=0, + ) + - min( + (int(row["tree_read_bytes"]) for row in resources if isinstance(row.get("tree_read_bytes"), int)), + default=0, + ) + ), + "tree_write_bytes_delta": ( + max( + (int(row["tree_write_bytes"]) for row in resources if isinstance(row.get("tree_write_bytes"), int)), + default=0, + ) + - min( + (int(row["tree_write_bytes"]) for row in resources if isinstance(row.get("tree_write_bytes"), int)), + default=0, + ) + ), }, "cleanup": { "complete": True @@ -976,6 +1000,234 @@ def aggregate_pytest_statistics( } +_GREEN_TERMINAL_OUTCOMES = frozenset({"passed", "skipped", "xfailed", "xpassed"}) + + +def _safe_step_dir(root: Path, raw: object) -> Path | None: + if not isinstance(raw, str) or not raw: + return None + relative = Path(raw) + if relative.is_absolute() or ".." in relative.parts: + return None + candidate = (root / relative).resolve() + try: + candidate.relative_to(root.resolve()) + except ValueError: + return None + return candidate + + +def _read_json_object(path: Path) -> dict[str, Any] | None: + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError): + return None + return payload if isinstance(payload, dict) else None + + +def _terminal_outcomes_by_node(step_dir: Path) -> dict[str, str]: + canonical = _read_json_object(step_dir / PYTEST_CANONICAL_REPORT_NAME) + raw_tests = canonical.get("tests") if canonical is not None else None + if isinstance(raw_tests, list): + outcomes = { + str(test["nodeid"]): str(test["outcome"]) + for test in raw_tests + if isinstance(test, dict) and isinstance(test.get("nodeid"), str) and isinstance(test.get("outcome"), str) + } + if outcomes: + return outcomes + + reports: dict[str, dict[str, str]] = {} + events_path = step_dir / "events.jsonl" + if events_path.exists(): + for line in events_path.read_text(encoding="utf-8", errors="replace").splitlines(): + with contextlib.suppress(json.JSONDecodeError): + row = json.loads(line) + if not isinstance(row, dict) or row.get("event") != "test_report": + continue + nodeid = row.get("nodeid") + when = row.get("when") + outcome = row.get("outcome") + if isinstance(nodeid, str) and when in {"setup", "call", "teardown"} and isinstance(outcome, str): + reports.setdefault(nodeid, {})[str(when)] = outcome + fallback_outcomes: dict[str, str] = {} + for nodeid, phases in reports.items(): + setup = phases.get("setup") + call = phases.get("call") + teardown = phases.get("teardown") + if setup == "failed" or teardown == "failed": + fallback_outcomes[nodeid] = "error" + elif call is not None: + fallback_outcomes[nodeid] = call + elif setup in {"skipped", "xfailed", "xpassed"}: + fallback_outcomes[nodeid] = setup + elif teardown in {"skipped", "xfailed", "xpassed"}: + fallback_outcomes[nodeid] = teardown + else: + fallback_outcomes[nodeid] = "interrupted" + return fallback_outcomes + + +def aggregate_native_testmon_run( + root: Path, + *, + steps: Sequence[Mapping[str, Any]], + environment_name: str, + corpus_nodeids: Sequence[str], + selection_mode: str, + invocation_duration_s: float, + budget_s: float, +) -> dict[str, Any]: + """Build one compact, durable aggregate for the two semantic pytest lanes.""" + lanes: list[dict[str, Any]] = [] + selected_union: set[str] = set() + outcome_by_node: dict[str, str] = {} + duplicate_outcomes: set[str] = set() + outcomes: dict[str, int] = {} + collection_wall_s = 0.0 + peak_rss_kb: int | None = None + peak_pss_kb: int | None = None + peak_swap_pss_kb: int | None = None + peak_storage_bytes: int | None = None + read_bytes = 0 + write_bytes = 0 + cleanup_complete = True + containment_complete = True + selection_complete = True + for step in steps: + lane = step.get("semantic_lane") + if lane not in {"parallel", "serial"}: + continue + step_dir = _safe_step_dir(root, step.get("artifact_dir")) + selection = _read_json_object(step_dir / "selection.json") if step_dir is not None else None + containment_receipt = _read_json_object(step_dir / "containment.json") if step_dir is not None else None + selected = selection.get("selected_nodeids") if selection is not None else None + omitted = selection.get("selected_nodeids_omitted") if selection is not None else None + if not isinstance(selected, list) or not all(isinstance(nodeid, str) for nodeid in selected) or omitted != 0: + selection_complete = False + selected = [] + selected_union.update(selected) + lane_outcomes = _terminal_outcomes_by_node(step_dir) if step_dir is not None else {} + for nodeid, outcome in lane_outcomes.items(): + if nodeid in outcome_by_node: + duplicate_outcomes.add(nodeid) + outcome_by_node[nodeid] = outcome + statistics = step.get("statistics") + resources = statistics.get("resources") if isinstance(statistics, Mapping) else None + storage = statistics.get("storage") if isinstance(statistics, Mapping) else None + cleanup = statistics.get("cleanup") if isinstance(statistics, Mapping) else None + + def _peak(current: int | None, value: object) -> int | None: + return max(current or 0, value) if isinstance(value, int) else current + + if isinstance(resources, Mapping): + peak_rss_kb = _peak(peak_rss_kb, resources.get("peak_tree_rss_kb")) + peak_pss_kb = _peak(peak_pss_kb, resources.get("peak_tree_pss_kb")) + peak_swap_pss_kb = _peak(peak_swap_pss_kb, resources.get("peak_tree_swap_pss_kb")) + lane_read = resources.get("tree_read_bytes_delta") + lane_write = resources.get("tree_write_bytes_delta") + read_bytes += lane_read if isinstance(lane_read, int) else 0 + write_bytes += lane_write if isinstance(lane_write, int) else 0 + if isinstance(storage, Mapping): + peak_storage_bytes = _peak(peak_storage_bytes, storage.get("basetemp_allocated_bytes_max")) + lane_cleanup = cleanup.get("complete") if isinstance(cleanup, Mapping) else None + cleanup_complete = cleanup_complete and lane_cleanup is True + lane_containment_complete = bool( + containment_receipt is not None + and containment_receipt.get("status") == "finished" + and containment_receipt.get("controller_group_alive") is False + and containment_receipt.get("termination_reason") is None + and containment_receipt.get("escalated_to_sigkill") is False + and not bool(step.get("termination_reason")) + and step.get("containment_escalated_to_sigkill") is not True + ) + containment_complete = containment_complete and lane_containment_complete + collection_duration = step.get("collection_duration_s") + if isinstance(collection_duration, (int, float)): + collection_wall_s += float(collection_duration) + lanes.append( + { + "lane": lane, + "exit_code": step.get("exit"), + "duration_s": step.get("duration_s"), + "collection_duration_s": collection_duration, + "selected_count": step.get("selected_count"), + "terminal_count": len(lane_outcomes), + "cleanup_complete": lane_cleanup, + "containment_mode": step.get("containment_mode"), + "containment_complete": lane_containment_complete, + } + ) + + for outcome in outcome_by_node.values(): + outcomes[outcome] = outcomes.get(outcome, 0) + 1 + native_corpus = tuple(sorted(set(corpus_nodeids))) + complete_mode = selection_mode in {"bootstrap", "full"} + # testmon synchronizes every collected node before pytest applies markers. + # The correctness corpus for this invocation is therefore the complete + # selection union from the two complementary semantic lanes, while the + # native database corpus remains useful environment evidence of its own. + corpus = tuple(sorted(selected_union)) if complete_mode else native_corpus + corpus_set = set(corpus) + lane_names = [lane["lane"] for lane in lanes] + complete_corpus_covered = ( + complete_mode + and selection_complete + and bool(corpus) + and set(outcome_by_node) == corpus_set + and not duplicate_outcomes + and len(lane_names) == 2 + and lane_names.count("parallel") == 1 + and lane_names.count("serial") == 1 + ) + missing_terminal = tuple(sorted(corpus_set - set(outcome_by_node))) if complete_mode else () + non_green = tuple( + sorted(nodeid for nodeid in corpus if outcome_by_node.get(nodeid) not in _GREEN_TERMINAL_OUTCOMES) + ) + terminal_green = complete_corpus_covered and not missing_terminal and not non_green + return { + "schema_version": 1, + "environment": { + "name": environment_name, + "digest": environment_name.removeprefix("polylogue-"), + "native_corpus_count": len(native_corpus), + "native_corpus_digest": hashlib.sha256("\n".join(native_corpus).encode()).hexdigest(), + }, + "corpus": { + "count": len(corpus), + "digest": hashlib.sha256("\n".join(corpus).encode()).hexdigest(), + }, + "selection_mode": selection_mode, + "lanes": lanes, + "outcomes": outcomes, + "selected_union_count": len(selected_union), + "terminal_union_count": len(outcome_by_node), + "missing_terminal_count": len(missing_terminal), + "missing_terminal_sample": list(missing_terminal[:20]), + "non_green_count": len(non_green), + "non_green_sample": list(non_green[:20]), + "duplicate_outcome_count": len(duplicate_outcomes), + "complete_corpus_covered": complete_corpus_covered, + "terminal_green": terminal_green, + "wall_s": round(invocation_duration_s, 4), + "collection_wall_s": round(collection_wall_s, 4), + "resources": { + "peak_tree_rss_kb": peak_rss_kb, + "peak_tree_pss_kb": peak_pss_kb, + "peak_tree_swap_pss_kb": peak_swap_pss_kb, + "peak_storage_bytes": peak_storage_bytes, + "read_bytes": read_bytes, + "write_bytes": write_bytes, + }, + "cleanup": {"complete": cleanup_complete}, + "containment": {"complete": containment_complete}, + "deadline": { + "budget_s": budget_s, + "met": invocation_duration_s <= budget_s, + }, + } + + def git_dirty(cwd: Path | None = None) -> bool: try: result = subprocess.run(["git", "status", "--short"], capture_output=True, text=True, timeout=5, cwd=cwd) @@ -1130,7 +1382,7 @@ def finish_step(self, *, step_id: str, result: dict[str, Any]) -> dict[str, Any] if not str(step.get("name", "")).startswith("pytest"): break # An interrupted runner never returns through the normal - # post-subprocess merge. Fold shards here, before every + # post-subprocess merge. Fold worker evidence here, before every # aggregation path, so completed worker evidence survives. with contextlib.suppress(OSError): merge_worker_events(step_dir / "events", step_dir / "events.jsonl") @@ -1185,6 +1437,7 @@ def finish( final_worktree_fingerprint: str | None = None, checkout_mutation_path: str | None = None, checkout_diagnosis: str | None = None, + pytest_aggregate: Mapping[str, Any] | None = None, ) -> dict[str, Any]: self._payload["finished_at"] = utc_now() self._payload["duration_s"] = round(duration_s, 2) @@ -1198,6 +1451,8 @@ def finish( self._payload["checkout_mutation_path"] = checkout_mutation_path if checkout_diagnosis is not None: self._payload["checkout_diagnosis"] = checkout_diagnosis + if pytest_aggregate is not None: + self._payload["pytest_aggregate"] = dict(pytest_aggregate) if verification_scope is not None: self._payload["verification_scope"] = verification_scope self._payload["release_baseline_allowed"] = release_baseline_allowed diff --git a/docs/devtools.md b/docs/devtools.md index 47847f455a..6602040cab 100644 --- a/docs/devtools.md +++ b/docs/devtools.md @@ -56,7 +56,6 @@ They are not a proof ledger or end-user archive workflow. | Command | Role | | --- | --- | | `devtools lab provider completeness` | Inspect detector, parser, fixture, schema, docs, ImportExplain, and caveat coverage before claiming a provider/importer mode is product-ready. | -| `devtools lab testmon-proof` | Validate the affected-test harness itself: a disposable copy of a real Polylogue module and existing route test is seeded, semantically mutated, edge-severed, restored, and checked for bounded unrelated-change selection. | | `devtools lab snapshot read-surface` | Freeze archive read-surface behavior before archive work, then compare candidate archives against the captured envelope baseline. | | `devtools lab policy schema-versioning` | Enforce the policy boundary documented in docs/internals.md § 'Schema Versioning Model'. Durable tiers use explicit additive migrations with a backup gate; derived tiers are rebuilt or blue-green replaced from source evidence. | | `devtools lab policy bead-graph` | Run before shipping a bead-state delta. With no source option it checks live `bd` state; `--export .beads/issues.jsonl` validates the branch snapshot without importing it into the shared database. The gate reads dependency records only and does not make prose, labels, or campaign-specific edge lists machine authority. | @@ -148,7 +147,6 @@ These are the commands worth remembering during normal repo work: | `devtools lab schema roundtrip` | Verify committed provider schema packages reload and roundtrip cleanly. | | `devtools lab smoke` | Run direct archive and reader smoke sets. | | `devtools lab snapshot read-surface` | Capture and compare archive read-surface snapshots. | -| `devtools lab testmon-proof` | Prove real testmon affected selection against a semantic production mutation. | ### Verification diff --git a/tests/conftest.py b/tests/conftest.py index 211ed0d8f3..b5851dfcc5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -42,7 +42,7 @@ from devtools.verify_runs import pytest_basetemp_claim_path as _basetemp_claim_path # Resolve (but don't yet raise on) the polylogue-vs-checkout mismatch check -# before the first `from polylogue...` import below: a shared/editable venv's +# before test execution can import product modules: a shared/editable venv's # `.pth` entry can point at a different checkout than the one this pytest # process is actually running from (e.g. a linked git worktree reusing the # main checkout's `.venv`), and whichever tree `import polylogue` resolves to @@ -57,12 +57,6 @@ except CheckoutImportMismatchError as _checkout_exc: _CHECKOUT_GUARD_ERROR = _checkout_exc -from polylogue.archive.models import Session -from polylogue.scenarios import CorpusSpec, build_default_corpus_specs -from polylogue.storage.runtime import RawSessionRecord -from tests.infra.builders import make_conv, make_msg -from tests.infra.timeout_policy import timeout_marker_error - pytest_plugins = ( "tests.infra.corpus_fixtures", "tests.infra.scale_fixtures", @@ -84,8 +78,10 @@ if TYPE_CHECKING: from click.testing import CliRunner + from polylogue.archive.models import Session from polylogue.config import Source from polylogue.storage.repository import SessionRepository + from polylogue.storage.runtime import RawSessionRecord from polylogue.storage.sqlite import SQLiteBackend from tests.infra.storage_records import SessionBuilder @@ -257,6 +253,8 @@ def pytest_unconfigure(config: pytest.Config) -> None: def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: """Reject unbounded or effectively disabled per-test timeout markers.""" + from tests.infra.timeout_policy import timeout_marker_error + for item in items: marker = item.get_closest_marker("timeout") if marker is None: @@ -584,7 +582,7 @@ def _reclaim_test_tmp_path( # survive the host-configuration scrub below. They are test-run evidence # plumbing, not operator configuration; removing them after collection makes # setup/call reports disappear from the event ledger while teardown still gets -# recorded, which makes interrupted seed shards look falsely successful. +# recorded, which makes interrupted native runs look falsely successful. _MANAGED_VERIFY_ENV = frozenset( { "POLYLOGUE_VERIFY_RUN_ID", @@ -1283,6 +1281,8 @@ def sample_session() -> Session: Replaces duplicate fixtures in: test_projections.py """ + from tests.infra.builders import make_conv, make_msg + messages = [ make_msg(id="m1", role="user", text="User question", timestamp="2024-01-01T10:00:00"), make_msg(id="m2", role="assistant", text="Assistant response", timestamp="2024-01-01T10:01:00"), @@ -1335,6 +1335,7 @@ def raw_synthetic_samples() -> list[RawSessionRecord]: import hashlib from datetime import datetime, timezone + from polylogue.scenarios import build_default_corpus_specs from polylogue.schemas.synthetic import SyntheticCorpus from polylogue.storage.runtime import RawSessionRecord @@ -1390,6 +1391,7 @@ def test_something(synthetic_source): source = synthetic_source("claude-code", count=3) """ from polylogue.config import Source + from polylogue.scenarios import CorpusSpec from polylogue.schemas.synthetic import SyntheticCorpus from tests.infra.source_builders import SyntheticAntigravityLanguageServerClient diff --git a/tests/integration/devtools/test_native_testmon_lifecycle.py b/tests/integration/devtools/test_native_testmon_lifecycle.py new file mode 100644 index 0000000000..8801917bd7 --- /dev/null +++ b/tests/integration/devtools/test_native_testmon_lifecycle.py @@ -0,0 +1,497 @@ +from __future__ import annotations + +import json +import os +import signal +import sqlite3 +import subprocess +import sys +import time +import uuid +from dataclasses import dataclass +from pathlib import Path + +import pytest + +from devtools.testmon_bootstrap import ( + TESTMON_DATA_RELPATH, + inspect_native_testmon_environment, + prepare_native_testmon_environment, +) +from devtools.testmon_bootstrap import ( + testmon_environment_digest as _testmon_environment_digest, +) +from devtools.verify_runs import ( + PYTEST_CANONICAL_REPORT_NAME, + VerifyRun, + aggregate_native_testmon_run, + aggregate_pytest_statistics, + append_verify_history, +) + +PROJECT_ROOT = Path(__file__).resolve().parents[3] +pytestmark = pytest.mark.uses_real_clock("coordinates real pytest subprocesses and an interrupt deadline") + + +@dataclass(frozen=True) +class LaneResult: + completed: subprocess.CompletedProcess[str] + artifact_dir: Path + selection: dict[str, object] + + +def _git(repo: Path, *args: str) -> str: + completed = subprocess.run( + ["git", *args], + cwd=repo, + capture_output=True, + text=True, + check=True, + ) + return completed.stdout.strip() + + +def _init_repo(root: Path, *, conftest: str = "") -> None: + (root / "tests").mkdir(parents=True) + (root / "pyproject.toml").write_text( + """ +[tool.pytest.ini_options] +addopts = "-p no:randomly" +markers = [ + "load_sensitive: serial native-testmon lane", + "tui: serial native-testmon lane", + "scale_large: excluded from the ordinary correctness corpus", +] +""".lstrip(), + encoding="utf-8", + ) + (root / "tests" / "conftest.py").write_text(conftest, encoding="utf-8") + _git(root, "init", "-q") + _git(root, "config", "user.email", "tests@example.invalid") + _git(root, "config", "user.name", "Polylogue Tests") + + +def _commit_all(root: Path, message: str) -> str: + _git(root, "add", ".") + _git(root, "commit", "-qm", message) + return _git(root, "rev-parse", "HEAD") + + +def _pytest_environment(repo: Path) -> dict[str, str]: + (repo / TESTMON_DATA_RELPATH).parent.mkdir(parents=True, exist_ok=True) + env = os.environ.copy() + env["TESTMON_DATAFILE"] = str(repo / TESTMON_DATA_RELPATH) + env["PYTHONPATH"] = os.pathsep.join((str(repo), str(PROJECT_ROOT), env.get("PYTHONPATH", ""))) + env["POLYLOGUE_PYTEST_SELECTION_NODEID_LIMIT"] = "50000" + return env + + +def _run_lane( + repo: Path, + *, + environment_name: str, + mode: str, + lane: str, + workers: int = 0, + timeout: float = 30, + base_marker: str | None = None, +) -> LaneResult: + artifact_dir = repo / ".artifacts" / f"{mode}-{lane}-{uuid.uuid4().hex}" + artifact_dir.mkdir(parents=True) + env = _pytest_environment(repo) + env.update( + { + "POLYLOGUE_PYTEST_EVENTS_DIR": str(artifact_dir / "events"), + "POLYLOGUE_PYTEST_SELECTION_PATH": str(artifact_dir / "selection.json"), + "POLYLOGUE_PYTEST_SUMMARY_PATH": str(artifact_dir / "summary.json"), + } + ) + semantic_marker = "not load_sensitive and not tui" if lane == "parallel" else "load_sensitive or tui" + marker = semantic_marker if base_marker is None else f"({base_marker}) and ({semantic_marker})" + selection = "--testmon-forceselect" if mode == "affected" else "--testmon-noselect" + command = [ + sys.executable, + "-m", + "pytest", + "-q", + "--tb=short", + "--testmon", + f"--testmon-env={environment_name}", + selection, + "-m", + marker, + "-p", + "devtools.pytest_progress_plugin", + "--json-report", + "--json-report-omit=collectors,log,streams,warnings", + f"--json-report-file={artifact_dir / PYTEST_CANONICAL_REPORT_NAME}", + "-n", + str(workers), + ] + completed = subprocess.run(command, cwd=repo, env=env, capture_output=True, text=True, timeout=timeout) + selection_payload = json.loads((artifact_dir / "selection.json").read_text(encoding="utf-8")) + return LaneResult(completed, artifact_dir, selection_payload) + + +def _run_plain_verify_corpus( + repo: Path, + *, + mode: str, + environment_name: str, + base_marker: str | None = None, +) -> tuple[LaneResult, LaneResult]: + parallel = _run_lane( + repo, + environment_name=environment_name, + mode=mode, + lane="parallel", + workers=2, + base_marker=base_marker, + ) + serial = _run_lane( + repo, + environment_name=environment_name, + mode=mode, + lane="serial", + base_marker=base_marker, + ) + return parallel, serial + + +def _selected(*results: LaneResult) -> set[str]: + selected: set[str] = set() + for result in results: + raw_nodeids = result.selection.get("selected_nodeids") + assert isinstance(raw_nodeids, list) + assert all(isinstance(nodeid, str) for nodeid in raw_nodeids) + selected.update(raw_nodeids) + return selected + + +def test_empty_plain_verify_bootstraps_then_warm_verify_is_affected(tmp_path: Path) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _init_repo(repo) + (repo / "app.py").write_text("def answer() -> int:\n return 42\n", encoding="utf-8") + (repo / "tests" / "test_app.py").write_text( + """ +import pytest + +def test_parallel_owner(): + from app import answer + assert answer() == 42 + +@pytest.mark.load_sensitive +def test_serial_owner(): + from app import answer + assert answer() == 42 +""".lstrip(), + encoding="utf-8", + ) + _commit_all(repo, "fixture") + + preparation = prepare_native_testmon_environment(repo) + assert preparation.selection_mode == "bootstrap" + first = _run_plain_verify_corpus( + repo, + mode=preparation.selection_mode, + environment_name=preparation.environment_name, + ) + assert [result.completed.returncode for result in first] == [0, 0] + state = inspect_native_testmon_environment( + repo / TESTMON_DATA_RELPATH, + environment_name=preparation.environment_name, + required_executable_paths=("app.py",), + ) + assert state.valid + assert state.environment is not None + assert state.environment.corpus_count == 2 + + warm = prepare_native_testmon_environment(repo, required_executable_paths=("app.py",)) + assert warm.selection_mode == "affected" + second = _run_plain_verify_corpus(repo, mode="affected", environment_name=warm.environment_name) + assert [result.completed.returncode for result in second] == [0, 0] + assert _selected(*second) == set() + + +def test_empty_linked_worktree_with_empty_main_self_bootstraps(tmp_path: Path) -> None: + main = tmp_path / "main" + main.mkdir() + _init_repo(main) + (main / "app.py").write_text("VALUE = 1\n", encoding="utf-8") + (main / "tests" / "test_app.py").write_text("def test_value():\n from app import VALUE\n assert VALUE == 1\n") + _commit_all(main, "fixture") + lane = tmp_path / "lane" + _git(main, "worktree", "add", "-qb", "lane", str(lane)) + + preparation = prepare_native_testmon_environment(lane) + + assert preparation.linked_worktree + assert preparation.main_checkout == main + assert preparation.copied_from is None + assert preparation.selection_mode == "bootstrap" + results = _run_plain_verify_corpus(lane, mode="bootstrap", environment_name=preparation.environment_name) + assert [result.completed.returncode for result in results] == [0, 0] + + +def test_matching_main_copy_then_product_mutation_selects_and_fails_owner(tmp_path: Path) -> None: + main = tmp_path / "main" + main.mkdir() + _init_repo(main) + (main / "app.py").write_text("def answer() -> int:\n return 42\n", encoding="utf-8") + (main / "tests" / "test_app.py").write_text( + "def test_answer():\n from app import answer\n assert answer() == 42\n", + encoding="utf-8", + ) + _commit_all(main, "fixture") + main_preparation = prepare_native_testmon_environment(main) + _run_plain_verify_corpus(main, mode="bootstrap", environment_name=main_preparation.environment_name) + lane = tmp_path / "lane" + _git(main, "worktree", "add", "-qb", "lane", str(lane)) + (lane / "app.py").write_text("def answer() -> int:\n return 0\n", encoding="utf-8") + + preparation = prepare_native_testmon_environment(lane, required_executable_paths=("app.py",)) + + assert preparation.selection_mode == "affected" + assert preparation.copied_from == main / TESTMON_DATA_RELPATH + result = _run_lane( + lane, + environment_name=preparation.environment_name, + mode="affected", + lane="parallel", + ) + assert result.completed.returncode == 1 + assert result.selection["selected_nodeids"] == ["tests/test_app.py::test_answer"] + assert "assert 0 == 42" in result.completed.stdout + + +def test_interrupted_bootstrap_native_state_resumes_failed_and_unfinished_only(tmp_path: Path) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _init_repo(repo) + started = repo / "started" + (repo / "tests" / "test_a_completed.py").write_text( + "def test_completed():\n assert True\n", + encoding="utf-8", + ) + (repo / "tests" / "test_b_failed.py").write_text( + "def test_failed():\n assert False\n", + encoding="utf-8", + ) + (repo / "tests" / "test_c_unfinished.py").write_text( + f""" +import pathlib +import time + +def test_unfinished(): + marker = pathlib.Path({str(started)!r}) + if marker.exists(): + return + marker.write_text('started') + time.sleep(30) +""".lstrip(), + encoding="utf-8", + ) + _commit_all(repo, "fixture") + environment_name = _testmon_environment_digest(repo) + env = _pytest_environment(repo) + command = [ + sys.executable, + "-m", + "pytest", + "-q", + "-p", + "no:randomly", + "--testmon", + f"--testmon-env={environment_name}", + "--testmon-noselect", + "tests", + ] + process = subprocess.Popen(command, cwd=repo, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + deadline = time.monotonic() + 10 + while not started.exists() and time.monotonic() < deadline: + time.sleep(0.02) + assert started.exists(), process.communicate(timeout=1) + process.send_signal(signal.SIGINT) + process.communicate(timeout=10) + + state = inspect_native_testmon_environment( + repo / TESTMON_DATA_RELPATH, + environment_name=environment_name, + ) + assert state.valid + resumed = _run_lane(repo, environment_name=environment_name, mode="affected", lane="parallel") + assert resumed.completed.returncode == 1 + assert _selected(resumed) == { + "tests/test_b_failed.py::test_failed", + "tests/test_c_unfinished.py::test_unfinished", + } + assert "tests/test_a_completed.py::test_completed" not in _selected(resumed) + + +def test_node_add_delete_converges_without_a_custom_ledger(tmp_path: Path) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _init_repo(repo) + tests = repo / "tests" / "test_nodes.py" + tests.write_text("def test_existing():\n assert True\n", encoding="utf-8") + _commit_all(repo, "fixture") + preparation = prepare_native_testmon_environment(repo) + _run_plain_verify_corpus(repo, mode="bootstrap", environment_name=preparation.environment_name) + + tests.write_text("def test_existing():\n assert True\n\ndef test_new():\n assert True\n", encoding="utf-8") + added = _run_plain_verify_corpus(repo, mode="affected", environment_name=preparation.environment_name) + assert "tests/test_nodes.py::test_new" in _selected(*added) + tests.write_text("def test_new():\n assert True\n", encoding="utf-8") + _run_plain_verify_corpus(repo, mode="affected", environment_name=preparation.environment_name) + + state = inspect_native_testmon_environment( + repo / TESTMON_DATA_RELPATH, + environment_name=preparation.environment_name, + ) + assert state.valid + assert state.environment is not None + assert state.environment.nodeids == ("tests/test_nodes.py::test_new",) + + +def test_collection_only_executable_dependency_blocks_until_test_executes_it(tmp_path: Path) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _init_repo(repo, conftest="from app import answer\n") + (repo / "app.py").write_text("def answer() -> int:\n return 42\n", encoding="utf-8") + test_file = repo / "tests" / "test_app.py" + test_file.write_text("def test_unrelated():\n assert True\n", encoding="utf-8") + _commit_all(repo, "fixture") + preparation = prepare_native_testmon_environment(repo) + _run_plain_verify_corpus(repo, mode="bootstrap", environment_name=preparation.environment_name) + + blind = inspect_native_testmon_environment( + repo / TESTMON_DATA_RELPATH, + environment_name=preparation.environment_name, + required_executable_paths=("app.py",), + ) + assert not blind.valid + assert blind.missing_executable_paths == ("app.py",) + + (repo / "tests" / "conftest.py").write_text("", encoding="utf-8") + test_file.write_text( + "def test_owner():\n from app import answer\n assert answer() == 42\n", + encoding="utf-8", + ) + repaired = prepare_native_testmon_environment(repo, required_executable_paths=("app.py",)) + assert repaired.selection_mode == "bootstrap" + _run_plain_verify_corpus(repo, mode="bootstrap", environment_name=repaired.environment_name) + assert inspect_native_testmon_environment( + repo / TESTMON_DATA_RELPATH, + environment_name=repaired.environment_name, + required_executable_paths=("app.py",), + ).valid + + +def test_removed_environment_or_dependency_edge_invalidates_native_state(tmp_path: Path) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _init_repo(repo) + (repo / "app.py").write_text("VALUE = 1\n", encoding="utf-8") + (repo / "tests" / "test_app.py").write_text("def test_value():\n from app import VALUE\n assert VALUE == 1\n") + _commit_all(repo, "fixture") + preparation = prepare_native_testmon_environment(repo) + _run_plain_verify_corpus(repo, mode="bootstrap", environment_name=preparation.environment_name) + data = repo / TESTMON_DATA_RELPATH + + with sqlite3.connect(data) as connection: + connection.execute("DELETE FROM environment WHERE environment_name = ?", (preparation.environment_name,)) + assert not inspect_native_testmon_environment(data, environment_name=preparation.environment_name).valid + + rebuilt = prepare_native_testmon_environment(repo) + _run_plain_verify_corpus(repo, mode="bootstrap", environment_name=rebuilt.environment_name) + with sqlite3.connect(data) as connection: + connection.execute( + """ + DELETE FROM test_execution_file_fp + WHERE fingerprint_id IN (SELECT id FROM file_fp WHERE filename = 'app.py') + """ + ) + missing = inspect_native_testmon_environment( + data, + environment_name=rebuilt.environment_name, + required_executable_paths=("app.py",), + ) + assert not missing.valid + assert missing.missing_executable_paths == ("app.py",) + + +def test_two_real_lanes_form_one_run_aggregate_and_one_budget(tmp_path: Path) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _init_repo(repo) + (repo / "tests" / "test_lanes.py").write_text( + """ +import pytest + +def test_parallel(): + assert True + +@pytest.mark.load_sensitive +def test_serial(): + assert True + +@pytest.mark.scale_large +def test_excluded_from_correctness_corpus(): + assert True +""".lstrip(), + encoding="utf-8", + ) + _commit_all(repo, "fixture") + preparation = prepare_native_testmon_environment(repo) + started = time.monotonic() + results = _run_plain_verify_corpus( + repo, + mode="bootstrap", + environment_name=preparation.environment_name, + base_marker="not scale_large", + ) + elapsed = time.monotonic() - started + state = inspect_native_testmon_environment( + repo / TESTMON_DATA_RELPATH, environment_name=preparation.environment_name + ) + assert state.environment is not None + steps: list[dict[str, object]] = [] + for lane, result in zip(("parallel", "serial"), results, strict=True): + statistics = aggregate_pytest_statistics(result.artifact_dir) + steps.append( + { + "semantic_lane": lane, + "exit": result.completed.returncode, + "duration_s": elapsed, + "artifact_dir": str(result.artifact_dir.relative_to(repo)), + "selected_count": result.selection["selected_count"], + "collection_duration_s": result.selection.get("collection_duration_s"), + "statistics": statistics, + } + ) + aggregate = aggregate_native_testmon_run( + repo, + steps=steps, + environment_name=preparation.environment_name, + corpus_nodeids=state.environment.nodeids, + selection_mode="bootstrap", + invocation_duration_s=elapsed, + budget_s=3600, + ) + run = VerifyRun(tier="testmon", argv=[], git_head="head", root=repo) + run.finish(exit_code=0, duration_s=elapsed, pytest_aggregate=aggregate) + durable = json.loads((run.run_dir / "run.json").read_text(encoding="utf-8")) + history_path = repo / "xdg-state" / "polylogue" / "devtools" / "verify-history.jsonl" + append_verify_history({"run_id": run.run_id, "pytest_aggregate": aggregate}, path=history_path) + history = json.loads(history_path.read_text(encoding="utf-8")) + + assert aggregate["corpus"]["count"] == 2 + assert aggregate["environment"]["native_corpus_count"] == 3 + assert aggregate["terminal_union_count"] == 2 + assert [lane["lane"] for lane in aggregate["lanes"]] == ["parallel", "serial"] + assert aggregate["complete_corpus_covered"] is True + assert aggregate["terminal_green"] is True + assert aggregate["deadline"] == {"budget_s": 3600, "met": True} + assert durable["pytest_aggregate"] == aggregate + assert history["pytest_aggregate"] == aggregate diff --git a/tests/integration/devtools/test_testmon_seed_recovery.py b/tests/integration/devtools/test_testmon_seed_recovery.py deleted file mode 100644 index 5968d77279..0000000000 --- a/tests/integration/devtools/test_testmon_seed_recovery.py +++ /dev/null @@ -1,185 +0,0 @@ -from __future__ import annotations - -import hashlib -import json -import os -import sqlite3 -import subprocess -import sys -from pathlib import Path - -import pytest - -from devtools import testmon_bootstrap, testmon_state, verify -from devtools.testmon_state import file_fingerprint, inspect_testmon_database, seed_shard_plan -from devtools.verify_runs import CheckoutMutationObservation - - -def test_real_testmon_graph_copies_and_rebinds_in_a_temporary_lane( - tmp_path: Path, - capsys: pytest.CaptureFixture[str], - monkeypatch: pytest.MonkeyPatch, -) -> None: - class _StableMutationMonitor: - def __init__(self, _root: Path) -> None: - pass - - def start(self) -> None: - pass - - def finish(self) -> CheckoutMutationObservation: - return CheckoutMutationObservation(changed=False, unavailable=False) - - source = tmp_path / "source" - source.mkdir() - (source / "pyproject.toml").write_text('[project]\nname = "polylogue"\n', encoding="utf-8") - (source / "test_sample.py").write_text( - "def test_passed():\n assert 1 == 1\n\ndef test_failed():\n assert 1 == 2\n", - encoding="utf-8", - ) - data = source / ".cache" / "testmon" / "testmondata" - data.parent.mkdir(parents=True) - env = os.environ.copy() - env["TESTMON_DATAFILE"] = str(data) - run = subprocess.run( - [sys.executable, "-m", "pytest", "-q", "--testmon", "--testmon-noselect"], - cwd=source, - env=env, - capture_output=True, - text=True, - check=False, - ) - assert run.returncode != 0 - expected = ("test_sample.py::test_passed", "test_sample.py::test_failed") - assert inspect_testmon_database(data, expected).usable_for_selection - runtime_identity = testmon_state.testmon_runtime_identity(source) - assert runtime_identity is not None - dependency_environment, pytest_harness = runtime_identity - shards = seed_shard_plan(expected, shard_size=len(expected)) - shards[0]["status"] = "complete" - shards[0]["node_outcomes"] = [ - {"nodeid": expected[0], "outcome": "passed"}, - {"nodeid": expected[1], "outcome": "failed"}, - ] - attempt = { - "protocol_version": verify.TESTMON_SEED_PROTOCOL_VERSION, - "status": "reusable", - "outcome": "red-baseline", - "identity": { - "git_head": "head", - "worktree_fingerprint": "source-tree", - "python": sys.version, - "skip_slow": False, - "lab": False, - "dependency_environment": dependency_environment, - "pytest_harness": pytest_harness, - }, - "selection": {"selected_count": 2, "selected_nodeids_omitted": 0}, - "expected_nodeids": list(expected), - "expected_count": len(expected), - "expected_digest": hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest(), - "node_outcomes": [ - {"nodeid": expected[0], "outcome": "passed"}, - {"nodeid": expected[1], "outcome": "failed"}, - ], - "shards": shards, - "exit_code": 1, - "run_id": "real-testmon", - "artifact_dir": ".cache/verify/runs/real-testmon", - "testmon_data": file_fingerprint(data), - } - artifact_dir = source / ".cache" / "verify" / "runs" / "real-testmon" - artifact_dir.mkdir(parents=True, exist_ok=True) - (artifact_dir / "run.json").write_text( - json.dumps( - { - "run_id": "real-testmon", - "checkout_root": str(source.resolve()), - "artifact_dir": ".cache/verify/runs/real-testmon", - } - ), - encoding="utf-8", - ) - source_attempt = source / ".cache" / "testmon" / "seed-attempt.json" - source_attempt.parent.mkdir(parents=True, exist_ok=True) - source_attempt.write_text(json.dumps(attempt), encoding="utf-8") - - lane = tmp_path / "lane" - lane.mkdir() - (lane / "test_sample.py").write_text( - "def test_passed():\n assert 1 == 1\n\ndef test_failed():\n assert 1 == 1\n", - encoding="utf-8", - ) - monkeypatch.setattr(testmon_bootstrap, "_git_worktree_info", lambda _root: (True, source)) - copy_calls: list[tuple[Path, Path]] = [] - original_copy = testmon_bootstrap._atomic_copy_sqlite_db - - def counted_copy(src: Path, dst: Path) -> None: - copy_calls.append((src, dst)) - original_copy(src, dst) - - monkeypatch.setattr(testmon_bootstrap, "_atomic_copy_sqlite_db", counted_copy) - (lane / "pyproject.toml").write_text('[project]\nname = "polylogue"\n', encoding="utf-8") - (lane / "polylogue" / "cli").mkdir(parents=True) - (lane / "polylogue" / "__init__.py").write_text("", encoding="utf-8") - (lane / "polylogue" / "cli" / "click_app.py").write_text("", encoding="utf-8") - - local_data = lane / ".cache" / "testmon" / "testmondata" - local_stamp = lane / ".cache" / "testmon" / "seed.json" - local_attempt = lane / ".cache" / "testmon" / "seed-attempt.json" - - monkeypatch.chdir(lane) - monkeypatch.setattr(verify, "ROOT", lane) - monkeypatch.setattr("devtools.checkout_guard.resolved_polylogue_path", lambda: lane / "polylogue" / "__init__.py") - monkeypatch.setattr("devtools.checkout_guard._is_linked_worktree", lambda _root: True) - monkeypatch.setattr("devtools.checkout_guard._python_environment_root", lambda _executable: lane) - monkeypatch.setattr(verify, "build_verify_steps", lambda **_kwargs: [("pytest testmon", ["pytest"])]) - run_count = 0 - - def fake_run(*_args: object, **_kwargs: object) -> tuple[int, float, dict[str, object]]: - nonlocal run_count - run_count += 1 - if run_count == 1: - with sqlite3.connect(local_data) as connection: - connection.execute("update test_execution set failed = 0 where test_name = ?", (expected[1],)) - return 1, 0.01, {"selected_count": 1} - return 0, 0.01, {"selected_count": 1} - - monkeypatch.setattr(verify, "_run", fake_run) - monkeypatch.setattr(verify, "_git_head", lambda: "head") - monkeypatch.setattr(verify, "_git_commit", lambda _ref: "base") - monkeypatch.setattr(verify, "_default_testmon_is_broad_change", lambda _base_commit, _head_commit: False) - monkeypatch.setattr(verify, "_changed_executable_paths", lambda _base_commit, _head_commit: ()) - monkeypatch.setattr(verify, "_stamp_head", lambda: None) - monkeypatch.setattr(verify, "worktree_fingerprint", lambda *_args: "stable") - monkeypatch.setattr(verify, "CheckoutMutationMonitor", _StableMutationMonitor) - - assert verify.main([]) == 1 - result = json.loads(capsys.readouterr().out) - assert local_data.is_file() - assert local_attempt.is_file() - assert not local_stamp.exists() - assert result["steps"][0]["selected_count"] == 1 - assert result["release_baseline_allowed"] is False - assert verify._testmon_release_baseline_permission() is False - assert not local_stamp.exists() - assert len(copy_calls) == 1 - assert (lane / ".cache" / "verify" / "current-run.json").is_file() - refreshed_attempt = json.loads(local_attempt.read_text()) - assert refreshed_attempt["testmon_data"] == file_fingerprint(local_data) - current_run = json.loads((lane / ".cache" / "verify" / "current-run.json").read_text()) - assert refreshed_attempt["run_id"] == current_run["run_id"] - - assert verify.main([]) == 0 - second = json.loads(capsys.readouterr().out) - assert second["steps"][0]["selected_count"] == 1 - assert len(copy_calls) == 1 - - assert verify.main([]) == 0 - third = json.loads(capsys.readouterr().out) - assert third["steps"][0]["selected_count"] == 1 - assert len(copy_calls) == 1 - - with sqlite3.connect(local_data) as connection: - connection.execute("delete from test_execution_file_fp") - assert verify.main([]) == 2 diff --git a/tests/unit/devtools/test_checkout_guard.py b/tests/unit/devtools/test_checkout_guard.py index a5ed1b0916..18f15ae9ee 100644 --- a/tests/unit/devtools/test_checkout_guard.py +++ b/tests/unit/devtools/test_checkout_guard.py @@ -9,7 +9,6 @@ from __future__ import annotations -import hashlib import json from pathlib import Path @@ -17,7 +16,6 @@ import devtools.click_dispatch as click_dispatch import devtools.run_tests as run_tests -import devtools.testmon_state as testmon_state import devtools.verify as verify import polylogue from devtools.checkout_guard import ( @@ -166,85 +164,6 @@ def _fake_linked_checkout(tmp_path: Path) -> Path: return root -def _write_in_progress_seed_attempt(root: Path, *, status: str = "running", **overrides: object) -> Path: - payload: dict[str, object] = { - "protocol_version": verify.TESTMON_SEED_PROTOCOL_VERSION, - "status": status, - "identity": { - "git_head": "head", - "worktree_fingerprint": "fingerprint", - "python": "3.14", - "skip_slow": True, - "lab": False, - }, - "resume": False, - "expected_nodeids": [], - "expected_count": 0, - "started_at": "2026-08-05T12:00:00+00:00", - "run_id": "seed-testmon-20260805T120000Z", - "artifact_dir": ".cache/verify/runs/seed-testmon-20260805T120000Z", - "testmon_data_before": "missing", - } - if status == "reusable": - nodeid = "tests/test.py::test_one" - runtime_identity = testmon_state.testmon_runtime_identity(root) - assert runtime_identity is not None - dependency_environment, pytest_harness = runtime_identity - payload.update( - { - "identity": { - "git_head": "head", - "worktree_fingerprint": "fingerprint", - "python": "3.14", - "skip_slow": True, - "lab": False, - "dependency_environment": dependency_environment, - "pytest_harness": pytest_harness, - }, - "expected_nodeids": [nodeid], - "expected_count": 1, - "expected_digest": hashlib.sha256(nodeid.encode()).hexdigest(), - "selection": {"selected_count": 1, "selected_nodeids_omitted": 0}, - "node_outcomes": [{"nodeid": nodeid, "outcome": "failed"}], - "exit_code": 1, - "testmon_data": "fingerprint", - "release_baseline_allowed": False, - "verification_scope": "affected", - "binding": { - "mode": "exact", - "checkout_root": str(root.resolve()), - "source_checkout_root": None, - }, - } - ) - payload.update(overrides) - attempt = root / ".cache" / "testmon" / "seed-attempt.json" - attempt.parent.mkdir(parents=True, exist_ok=True) - attempt.write_text(json.dumps(payload)) - if status == "reusable": - run_dir = root / ".cache" / "verify" / "runs" / str(payload["run_id"]) - run_dir.mkdir(parents=True, exist_ok=True) - (run_dir / "run.json").write_text( - json.dumps( - { - "run_id": payload["run_id"], - "checkout_root": str(root.resolve()), - "artifact_dir": f".cache/verify/runs/{payload['run_id']}", - } - ) - ) - (root / ".cache" / "verify" / "current-run.json").write_text( - json.dumps( - { - "run_id": payload["run_id"], - "checkout_root": str(root.resolve()), - "artifact_dir": f".cache/verify/runs/{payload['run_id']}", - } - ) - ) - return attempt - - def test_checkout_environment_fingerprint_accepts_clean_linked_worktree(tmp_path: Path) -> None: root = _fake_linked_checkout(tmp_path) fingerprint = checkout_environment_fingerprint( @@ -258,114 +177,40 @@ def test_checkout_environment_fingerprint_accepts_clean_linked_worktree(tmp_path assert fingerprint.python_environment_root == root -def test_checkout_environment_fingerprint_accepts_current_in_progress_seed_attempt( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch +def test_checkout_guard_leaves_derived_native_testmon_state_for_verify_to_repair( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, ) -> None: root = _fake_linked_checkout(tmp_path) - attempt = _write_in_progress_seed_attempt(root) - (root / ".cache" / "testmon" / "testmondata").write_text("partial") + data = root / ".cache" / "testmon" / "testmondata" + data.parent.mkdir(parents=True) + data.write_bytes(b"interrupted or invalid derived SQLite state") package_path = root / "polylogue" / "__init__.py" monkeypatch.setattr("devtools.checkout_guard.resolved_polylogue_path", lambda: package_path) fingerprint = assert_polylogue_matches_checkout( root, - context="seed-testmon bootstrap", + context="plain verify repair", python_executable=root / ".venv" / "bin" / "python", ) assert fingerprint.clean - assert fingerprint.testmon_state_origin is None - assert attempt.is_file() + assert data.read_bytes().startswith(b"interrupted") -def test_checkout_environment_fingerprint_accepts_finalized_selection_attempt( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - root = _fake_linked_checkout(tmp_path) - attempt = _write_in_progress_seed_attempt(root, status="reusable") - (root / ".cache" / "testmon" / "testmondata").write_text("complete graph") - package_path = root / "polylogue" / "__init__.py" - monkeypatch.setattr("devtools.checkout_guard.resolved_polylogue_path", lambda: package_path) - - fingerprint = assert_polylogue_matches_checkout( - root, - context="affected selection", - python_executable=root / ".venv" / "bin" / "python", - ) - - assert fingerprint.clean - assert fingerprint.testmon_state_origin is None - assert attempt.is_file() - - -@pytest.mark.parametrize( - ("status", "overrides"), - [ - ("running", {"identity": {}}), - ("running", {"expected_count": 1}), - ("running", {"artifact_dir": "/foreign/.cache/verify/runs/seed"}), - ("complete", {}), - ], -) -def test_checkout_environment_fingerprint_rejects_invalid_or_completed_seed_attempt( +def test_checkout_preflight_reports_runtime_provenance_but_not_testmon_state( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, - status: str, - overrides: dict[str, object], -) -> None: - root = _fake_linked_checkout(tmp_path) - attempt = _write_in_progress_seed_attempt(root, status=status, **overrides) - (root / ".cache" / "testmon" / "testmondata").write_text("foreign or incomplete") - package_path = root / "polylogue" / "__init__.py" - monkeypatch.setattr("devtools.checkout_guard.resolved_polylogue_path", lambda: package_path) - - with pytest.raises(CheckoutEnvironmentMismatchError) as excinfo: - assert_polylogue_matches_checkout( - root, - context="invalid testmon state", - python_executable=root / ".venv" / "bin" / "python", - ) - - message = str(excinfo.value) - assert str(attempt.parent) in message - assert "no verifiable checkout-root marker" in message - - -def test_checkout_environment_fingerprint_requires_provenance_for_completed_seed( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - root = _fake_linked_checkout(tmp_path) - seed_dir = root / ".cache" / "testmon" - seed_dir.mkdir(parents=True) - (seed_dir / "testmondata").write_text("complete") - (seed_dir / "seed.json").write_text( - json.dumps({"protocol_version": verify.TESTMON_SEED_PROTOCOL_VERSION, "status": "complete"}) - ) - package_path = root / "polylogue" / "__init__.py" - monkeypatch.setattr("devtools.checkout_guard.resolved_polylogue_path", lambda: package_path) - - with pytest.raises(CheckoutEnvironmentMismatchError) as excinfo: - assert_polylogue_matches_checkout( - root, - context="completed testmon state", - python_executable=root / ".venv" / "bin" / "python", - ) - - assert str(seed_dir / "seed.json") in str(excinfo.value) - - -def test_checkout_preflight_reports_seeded_artifacts_and_main_interpreter( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: root = _fake_linked_checkout(tmp_path) main = tmp_path / "main-checkout" - main_venv = main / ".venv" / "bin" - main_venv.mkdir(parents=True) - main_python = main_venv / "python" + main_python = main / ".venv" / "bin" / "python" + main_python.parent.mkdir(parents=True) (root / ".venv").mkdir() (root / "node_modules").mkdir() - (root / ".cache" / "testmon").mkdir(parents=True) - (root / ".cache" / "testmon" / "seed.json").write_text(json.dumps({"status": "complete"})) + data = root / ".cache" / "testmon" / "testmondata" + data.parent.mkdir(parents=True) + data.write_bytes(b"repairable") (root / ".cache" / "verify").mkdir(parents=True) verify_marker = root / ".cache" / "verify" / "current-run.json" verify_marker.write_text(json.dumps({"checkout_root": str(main)})) @@ -373,16 +218,14 @@ def test_checkout_preflight_reports_seeded_artifacts_and_main_interpreter( monkeypatch.setattr("devtools.checkout_guard.resolved_polylogue_path", lambda: package_path) with pytest.raises(CheckoutEnvironmentMismatchError) as excinfo: - assert_polylogue_matches_checkout(root, context="seeded lane", python_executable=main_python) + assert_polylogue_matches_checkout(root, context="lane", python_executable=main_python) message = str(excinfo.value) assert str(main_python) in message assert str(root / ".venv") in message assert str(root / "node_modules") in message - assert str(root / ".cache" / "testmon" / "seed.json") in message assert str(verify_marker) in message - assert "direnv allow" in message - assert "remediation" in message + assert str(data) not in message def test_verify_run_persists_environment_fingerprint(tmp_path: Path) -> None: diff --git a/tests/unit/devtools/test_merge_gate.py b/tests/unit/devtools/test_merge_gate.py index 3d955674e9..687191127b 100644 --- a/tests/unit/devtools/test_merge_gate.py +++ b/tests/unit/devtools/test_merge_gate.py @@ -258,9 +258,7 @@ def test_check_rejects_successful_non_test_receipt(monkeypatch: pytest.MonkeyPat assert merge_gate.cmd_check(42, max_age_s=3600, poll_rounds=1, poll_interval_s=0, as_json=False) == 1 -@pytest.mark.parametrize( - "command", ["devtools verify --all", "devtools verify --full", "devtools verify --seed-testmon"] -) +@pytest.mark.parametrize("command", ["devtools verify --all", "devtools verify --full"]) def test_check_blocks_full_receipt_without_release_baseline_permission( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, command: str ) -> None: diff --git a/tests/unit/devtools/test_testmon_bootstrap.py b/tests/unit/devtools/test_testmon_bootstrap.py index 2fde1706a0..39116abe38 100644 --- a/tests/unit/devtools/test_testmon_bootstrap.py +++ b/tests/unit/devtools/test_testmon_bootstrap.py @@ -1,830 +1,99 @@ -"""Tests for the worktree testmon-seed bootstrap (devtools/testmon_bootstrap.py). - -Covers polylogue-mq4vx: a fresh agent worktree lane starts with no local -`.cache/testmon/testmondata`, either paying the full `--seed-testmon` cost -again or hitting the unseeded-refusal preflight in `devtools/verify.py`. The -main checkout's testmondata is copyable (file_fp entries are relative paths -with per-file checksums, so a stale copy self-invalidates changed files), so -`maybe_bootstrap_testmon_seed` copies it in before that preflight runs. - -These tests target `decide_testmon_bootstrap` (the pure decision) and -`bootstrap_testmon_seed_files` (the copy action) directly with tmp dirs -- -not the full `devtools verify` pipeline, per the bootstrap's own module -docstring contract. -""" - from __future__ import annotations -import hashlib -import json -import sqlite3 -from collections.abc import Callable from pathlib import Path -from typing import cast import pytest -import devtools.checkout_guard as checkout_guard -import devtools.testmon_bootstrap as testmon_bootstrap -import devtools.verify as verify from devtools.testmon_bootstrap import ( - BootstrapDecision, - bootstrap_testmon_seed_files, - decide_testmon_bootstrap, -) -from devtools.testmon_state import ( - BaselineStatus, - BindingMode, - CollectionStatus, - GraphInspection, - GraphStatus, - file_fingerprint, -) -from devtools.testmon_state import ( - TestmonBinding as _TestmonBinding, + NativeTestmonRepairError, + classify_source_ast, + executable_python_paths, + remove_invalid_native_testmon_state, ) -from devtools.testmon_state import ( - TestmonIdentity as _TestmonIdentity, -) -from devtools.testmon_state import ( - TestmonSeedStamp as _TestmonSeedStamp, +from devtools.testmon_bootstrap import ( + testmon_environment_digest as _testmon_environment_digest, ) -PROTOCOL_VERSION = 4 - - -def _write_valid_seed_stamp(path: Path, *, protocol_version: int = PROTOCOL_VERSION) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - data = path.parent / "testmondata" - if not data.exists(): - _write_sqlite_db(data) - with sqlite3.connect(data) as conn: - nodeids = tuple(row[0] for row in conn.execute("select test_name from test_execution")) - graph = GraphInspection(GraphStatus.COMPLETE, len(nodeids), len(nodeids), (), 0, 0, None, ()) - stamp = _TestmonSeedStamp( - protocol_version, - CollectionStatus.COMPLETE, - nodeids, - 0, - BaselineStatus.GREEN, - True, - 0, - graph, - _TestmonIdentity("head", "tree", "python", True, False, None, "narrow-terminal"), - _TestmonBinding(BindingMode.EXACT, str(path.parent.resolve())), - file_fingerprint(data), - "seed", - ".cache/verify/runs/seed", - ) - artifact_dir = path.parent / ".cache" / "verify" / "runs" / "seed" - artifact_dir.mkdir(parents=True, exist_ok=True) - (artifact_dir / "run.json").write_text( - json.dumps( - { - "run_id": "seed", - "checkout_root": str(path.parent.resolve()), - "artifact_dir": ".cache/verify/runs/seed", - } - ) - ) - path.write_text(json.dumps(stamp.as_dict())) - - -def _write_sqlite_db(path: Path, *, rows: tuple[str, ...] = ("a", "b")) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - conn = sqlite3.connect(path) - try: - conn.execute("CREATE TABLE environment (id INTEGER PRIMARY KEY, environment_name TEXT)") - conn.execute("CREATE TABLE file_fp (id INTEGER PRIMARY KEY, filename TEXT, fsha TEXT)") - conn.execute( - "CREATE TABLE test_execution (id INTEGER PRIMARY KEY, environment_id INTEGER, test_name TEXT, failed INTEGER)" - ) - conn.execute("CREATE TABLE test_execution_file_fp (test_execution_id INTEGER, fingerprint_id INTEGER)") - conn.executemany("INSERT INTO file_fp(filename, fsha) VALUES (?, ?)", [(row, f"sha-{row}") for row in rows]) - conn.executemany("INSERT INTO test_execution(test_name, failed) VALUES (?, 0)", [(row,) for row in rows]) - conn.executemany( - "INSERT INTO test_execution_file_fp VALUES (?, ?)", - [(index, index) for index, _row in enumerate(rows, start=1)], - ) - conn.commit() - finally: - conn.close() - - -def _red_attempt_decision(tmp_path: Path) -> tuple[BootstrapDecision, Path, Path, Path, Path]: - main_root = tmp_path / "main" - main_data = main_root / "testmondata" - _write_sqlite_db(main_data, rows=("tests/test.py::test_passed", "tests/test.py::test_failed")) - attempt = main_root / "seed-attempt.json" - attempt.write_text( - json.dumps( - { - "protocol_version": PROTOCOL_VERSION, - "status": "reusable", - "identity": { - "git_head": "head", - "worktree_fingerprint": "tree", - "python": "python", - "skip_slow": True, - "lab": False, - }, - "selection": {"selected_count": 2, "selected_nodeids_omitted": 0}, - "expected_nodeids": ["tests/test.py::test_passed", "tests/test.py::test_failed"], - "expected_count": 2, - "expected_digest": hashlib.sha256( - "\n".join(sorted(["tests/test.py::test_passed", "tests/test.py::test_failed"])).encode() - ).hexdigest(), - "testmon_data": file_fingerprint(main_data), - "node_outcomes": [ - {"nodeid": "tests/test.py::test_passed", "outcome": "passed"}, - {"nodeid": "tests/test.py::test_failed", "outcome": "failed"}, - ], - "exit_code": 1, - "run_id": "red-run", - "artifact_dir": ".cache/verify/runs/red-run", - } - ) - ) - artifact = main_root / ".cache" / "verify" / "runs" / "red-run" - artifact.mkdir(parents=True, exist_ok=True) - (artifact / "run.json").write_text( - json.dumps( - { - "run_id": "red-run", - "checkout_root": str(main_root.resolve()), - "artifact_dir": ".cache/verify/runs/red-run", - } - ) - ) - lane = tmp_path / "lane" - decision = decide_testmon_bootstrap( - is_linked_worktree=True, - local_testmon_data=lane / "testmondata", - local_seed_stamp=lane / "seed.json", - local_seed_attempt=lane / "seed-attempt.json", - main_testmon_data=main_data, - main_seed_stamp=main_root / "seed.json", - main_seed_attempt=attempt, - protocol_version=PROTOCOL_VERSION, - main_checkout_root=main_root, - local_checkout_root=lane, - ) - return decision, lane / "testmondata", lane / "seed.json", lane / "seed-attempt.json", lane - - -def test_not_a_linked_worktree_never_bootstraps(tmp_path: Path) -> None: - """The main checkout itself must never "bootstrap from itself".""" - decision = decide_testmon_bootstrap( - is_linked_worktree=False, - local_testmon_data=tmp_path / "local" / "testmondata", - local_seed_stamp=tmp_path / "local" / "seed.json", - main_testmon_data=tmp_path / "main" / "testmondata", - main_seed_stamp=tmp_path / "main" / "seed.json", - protocol_version=PROTOCOL_VERSION, - ) - assert decision == BootstrapDecision(False, decision.reason) - assert not decision.should_bootstrap - - -def test_local_seed_already_present_skips_bootstrap(tmp_path: Path) -> None: - """A worktree that already seeded itself must not be clobbered by main's copy.""" - local_data = tmp_path / "local" / "testmondata" - local_stamp = tmp_path / "local" / "seed.json" - _write_sqlite_db(local_data) - _write_valid_seed_stamp(local_stamp) - main_data = tmp_path / "main" / "testmondata" - main_stamp = tmp_path / "main" / "seed.json" - _write_sqlite_db(main_data) - _write_valid_seed_stamp(main_stamp) - - decision = decide_testmon_bootstrap( - is_linked_worktree=True, - local_testmon_data=local_data, - local_seed_stamp=local_stamp, - main_testmon_data=main_data, - main_seed_stamp=main_stamp, - protocol_version=PROTOCOL_VERSION, - ) - assert not decision.should_bootstrap - assert "already has" in decision.reason - - -def test_invalid_local_seed_does_not_block_valid_main_bootstrap(tmp_path: Path) -> None: - local_data = tmp_path / "local" / "testmondata" - local_stamp = tmp_path / "local" / "seed.json" - _write_sqlite_db(local_data) - _write_valid_seed_stamp(local_stamp) - local_data.write_bytes(local_data.read_bytes() + b"stale") - main_data = tmp_path / "main" / "testmondata" - main_stamp = tmp_path / "main" / "seed.json" - _write_sqlite_db(main_data) - _write_valid_seed_stamp(main_stamp) - - decision = decide_testmon_bootstrap( - is_linked_worktree=True, - local_testmon_data=local_data, - local_seed_stamp=local_stamp, - main_testmon_data=main_data, - main_seed_stamp=main_stamp, - protocol_version=PROTOCOL_VERSION, - ) - - assert decision.should_bootstrap - - -def test_main_seed_absent_skips_bootstrap(tmp_path: Path) -> None: - decision = decide_testmon_bootstrap( - is_linked_worktree=True, - local_testmon_data=tmp_path / "local" / "testmondata", - local_seed_stamp=tmp_path / "local" / "seed.json", - main_testmon_data=tmp_path / "main" / "testmondata", - main_seed_stamp=tmp_path / "main" / "seed.json", - protocol_version=PROTOCOL_VERSION, - ) - assert not decision.should_bootstrap - assert "testmondata file is missing" in decision.reason - - -def test_main_seed_stamp_wrong_protocol_version_skips_bootstrap(tmp_path: Path) -> None: - main_stamp = tmp_path / "main" / "seed.json" - _write_valid_seed_stamp(main_stamp, protocol_version=PROTOCOL_VERSION + 1) - - decision = decide_testmon_bootstrap( - is_linked_worktree=True, - local_testmon_data=tmp_path / "local" / "testmondata", - local_seed_stamp=tmp_path / "local" / "seed.json", - main_testmon_data=tmp_path / "main" / "testmondata", - main_seed_stamp=main_stamp, - protocol_version=PROTOCOL_VERSION, - ) - assert not decision.should_bootstrap - assert "stale" in decision.reason or "no validated" in decision.reason - - -def test_main_seed_stamp_incomplete_status_skips_bootstrap(tmp_path: Path) -> None: - main_stamp = tmp_path / "main" / "seed.json" - main_stamp.parent.mkdir(parents=True, exist_ok=True) - main_stamp.write_text(json.dumps({"protocol_version": PROTOCOL_VERSION, "status": "incomplete"})) - _write_sqlite_db(tmp_path / "main" / "testmondata") - - decision = decide_testmon_bootstrap( - is_linked_worktree=True, - local_testmon_data=tmp_path / "local" / "testmondata", - local_seed_stamp=tmp_path / "local" / "seed.json", - main_testmon_data=tmp_path / "main" / "testmondata", - main_seed_stamp=main_stamp, - protocol_version=PROTOCOL_VERSION, - ) - assert not decision.should_bootstrap - - -def test_main_seed_stamp_unreadable_json_skips_bootstrap(tmp_path: Path) -> None: - main_stamp = tmp_path / "main" / "seed.json" - main_stamp.parent.mkdir(parents=True, exist_ok=True) - main_stamp.write_text("{not valid json") - _write_sqlite_db(tmp_path / "main" / "testmondata") - - decision = decide_testmon_bootstrap( - is_linked_worktree=True, - local_testmon_data=tmp_path / "local" / "testmondata", - local_seed_stamp=tmp_path / "local" / "seed.json", - main_testmon_data=tmp_path / "main" / "testmondata", - main_seed_stamp=main_stamp, - protocol_version=PROTOCOL_VERSION, - ) - assert not decision.should_bootstrap - - -def test_valid_seed_stamp_but_missing_testmondata_skips_bootstrap(tmp_path: Path) -> None: - """A seed stamp claims completeness but the db file itself vanished -- don't copy nothing.""" - main_stamp = tmp_path / "main" / "seed.json" - main_stamp.parent.mkdir(parents=True, exist_ok=True) - main_stamp.write_text(json.dumps({"protocol_version": PROTOCOL_VERSION, "status": "usable"})) - - decision = decide_testmon_bootstrap( - is_linked_worktree=True, - local_testmon_data=tmp_path / "local" / "testmondata", - local_seed_stamp=tmp_path / "local" / "seed.json", - main_testmon_data=tmp_path / "main" / "testmondata", - main_seed_stamp=main_stamp, - protocol_version=PROTOCOL_VERSION, - ) - assert not decision.should_bootstrap - assert "testmondata file is missing" in decision.reason - - -def test_valid_main_seed_and_empty_local_bootstraps(tmp_path: Path) -> None: - main_data = tmp_path / "main" / "testmondata" - main_stamp = tmp_path / "main" / "seed.json" - _write_sqlite_db(main_data) - _write_valid_seed_stamp(main_stamp) - - decision = decide_testmon_bootstrap( - is_linked_worktree=True, - local_testmon_data=tmp_path / "local" / "testmondata", - local_seed_stamp=tmp_path / "local" / "seed.json", - main_testmon_data=main_data, - main_seed_stamp=main_stamp, - protocol_version=PROTOCOL_VERSION, - ) - assert decision.should_bootstrap - assert decision.main_testmon_data == main_data - assert decision.main_seed_stamp == main_stamp - - -def test_complete_red_attempt_bootstraps_as_selection_only_state(tmp_path: Path) -> None: - main_data = tmp_path / "main" / "testmondata" - _write_sqlite_db(main_data, rows=("tests/test.py::test_passed", "tests/test.py::test_failed")) - attempt = tmp_path / "main" / "seed-attempt.json" - attempt.write_text( - json.dumps( - { - "protocol_version": PROTOCOL_VERSION, - "status": "reusable", - "identity": { - "git_head": "head", - "worktree_fingerprint": "tree", - "python": "python", - "skip_slow": True, - "lab": False, - }, - "selection": {"selected_count": 2, "selected_nodeids_omitted": 0}, - "expected_nodeids": ["tests/test.py::test_passed", "tests/test.py::test_failed"], - "expected_count": 2, - "expected_digest": hashlib.sha256( - "\n".join(sorted(["tests/test.py::test_passed", "tests/test.py::test_failed"])).encode() - ).hexdigest(), - "testmon_data": file_fingerprint(main_data), - "node_outcomes": [ - {"nodeid": "tests/test.py::test_passed", "outcome": "passed"}, - {"nodeid": "tests/test.py::test_failed", "outcome": "failed"}, - ], - "exit_code": 1, - "run_id": "red-run", - "artifact_dir": ".cache/verify/runs/red-run", - } - ) - ) - red_artifact = tmp_path / "main" / ".cache" / "verify" / "runs" / "red-run" - red_artifact.mkdir(parents=True, exist_ok=True) - (red_artifact / "run.json").write_text( - json.dumps( - { - "run_id": "red-run", - "checkout_root": str((tmp_path / "main").resolve()), - "artifact_dir": ".cache/verify/runs/red-run", - } - ) - ) - decision = decide_testmon_bootstrap( - is_linked_worktree=True, - local_testmon_data=tmp_path / "lane" / "testmondata", - local_seed_stamp=tmp_path / "lane" / "seed.json", - main_testmon_data=main_data, - main_seed_stamp=tmp_path / "main" / "seed.json", - main_seed_attempt=attempt, - protocol_version=PROTOCOL_VERSION, - ) - - assert decision.should_bootstrap - assert decision.main_seed_attempt == attempt - local_data = tmp_path / "lane" / "testmondata" - local_stamp = tmp_path / "lane" / "seed.json" - local_attempt = tmp_path / "lane" / "seed-attempt.json" - assert bootstrap_testmon_seed_files( - decision, - local_testmon_data=local_data, - local_seed_stamp=local_stamp, - local_seed_attempt=local_attempt, - checkout_root=tmp_path / "lane", - inherited_from=tmp_path / "main", - ) - assert not local_stamp.exists() - rebound_attempt = json.loads(local_attempt.read_text()) - assert rebound_attempt["artifact_dir"] == ".cache/verify/runs/red-run" - assert rebound_attempt["testmon_data"] == file_fingerprint(local_data) - rebound_receipt = json.loads( - (tmp_path / "lane" / ".cache" / "verify" / "runs" / "red-run" / "run.json").read_text() - ) - assert rebound_receipt["run_id"] == "red-run" - assert rebound_receipt["checkout_root"] == str((tmp_path / "lane").resolve()) - current_run = json.loads((tmp_path / "lane" / ".cache" / "verify" / "current-run.json").read_text()) - assert current_run["run_id"] == "red-run" - assert current_run["checkout_root"] == str((tmp_path / "lane").resolve()) - rebound_decision = decide_testmon_bootstrap( - is_linked_worktree=True, - local_testmon_data=local_data, - local_seed_stamp=local_stamp, - local_seed_attempt=local_attempt, - main_testmon_data=main_data, - main_seed_stamp=tmp_path / "main" / "seed.json", - main_seed_attempt=attempt, - protocol_version=PROTOCOL_VERSION, - main_checkout_root=tmp_path / "main", - local_checkout_root=tmp_path / "lane", +def test_ast_classification_distinguishes_declarations_from_execution(tmp_path: Path) -> None: + declarations = tmp_path / "types.py" + declarations.write_text( + '"""Types only."""\nname: str\n\nclass Record:\n identifier: int\n\n def label(self) -> str: ...\n', + encoding="utf-8", ) - assert not rebound_decision.should_bootstrap - assert "checkout-bound selection attempt" in rebound_decision.reason + executable = tmp_path / "runtime.py" + executable.write_text("VALUE: int = 3\n", encoding="utf-8") + assert classify_source_ast(declarations) == "declaration-only" + assert classify_source_ast(executable) == "executable" -def test_complete_typed_markerless_green_attempt_bootstraps_only_as_selection_state(tmp_path: Path) -> None: - main_root = tmp_path / "main" - main_data = main_root / "testmondata" - _write_sqlite_db(main_data, rows=("tests/test.py::test_passed",)) - attempt = main_root / "seed-attempt.json" - attempt.write_text( - json.dumps( - { - "protocol_version": PROTOCOL_VERSION, - "status": "complete", - "identity": { - "git_head": "head", - "worktree_fingerprint": "tree", - "python": "python", - "skip_slow": False, - "lab": False, - "terminal_authorization": None, - }, - "selection": {"selected_count": 1, "selected_nodeids_omitted": 0}, - "expected_nodeids": ["tests/test.py::test_passed"], - "expected_count": 1, - "expected_digest": hashlib.sha256(b"tests/test.py::test_passed").hexdigest(), - "node_outcomes": [{"nodeid": "tests/test.py::test_passed", "outcome": "passed"}], - "exit_code": 0, - "verification_scope": "release-baseline", - "release_baseline_allowed": True, - "run_id": "green-run", - "artifact_dir": ".cache/verify/runs/green-run", - "testmon_data": file_fingerprint(main_data), - } - ) - ) - artifact = main_root / ".cache" / "verify" / "runs" / "green-run" - artifact.mkdir(parents=True) - (artifact / "run.json").write_text( - json.dumps( - { - "run_id": "green-run", - "checkout_root": str(main_root.resolve()), - "artifact_dir": ".cache/verify/runs/green-run", - } - ) - ) - decision = decide_testmon_bootstrap( - is_linked_worktree=True, - local_testmon_data=tmp_path / "lane" / "testmondata", - local_seed_stamp=tmp_path / "lane" / "seed.json", - main_testmon_data=main_data, - main_seed_stamp=main_root / "seed.json", - main_seed_attempt=attempt, - protocol_version=PROTOCOL_VERSION, - ) +def test_executable_paths_require_current_runtime_modules_but_allow_deletion(tmp_path: Path) -> None: + module = tmp_path / "polylogue" / "runtime.py" + module.parent.mkdir() + module.write_text("VALUE = factory()\n", encoding="utf-8") + malformed = module.with_name("malformed.py") + malformed.write_text("def broken(:\n", encoding="utf-8") - assert decision.should_bootstrap - assert decision.selection_only - assert bootstrap_testmon_seed_files( - decision, - local_testmon_data=tmp_path / "lane" / "testmondata", - local_seed_stamp=tmp_path / "lane" / "seed.json", - local_seed_attempt=tmp_path / "lane" / "seed-attempt.json", - checkout_root=tmp_path / "lane", - inherited_from=main_root, + assert executable_python_paths( + tmp_path, + ("polylogue/runtime.py", "polylogue/malformed.py", "polylogue/deleted.py"), + ) == ( + "polylogue/malformed.py", + "polylogue/runtime.py", ) - assert not (tmp_path / "lane" / "seed.json").exists() -def test_markerless_complete_bootstrap_passes_guard_and_verify_preflight( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch +def test_environment_digest_changes_with_collection_semantics( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, ) -> None: - main_root = tmp_path / "main" - main_data = main_root / "testmondata" - nodeid = "tests/test.py::test_passed" - _write_sqlite_db(main_data, rows=(nodeid,)) - attempt = main_root / "seed-attempt.json" - attempt.write_text( - json.dumps( - { - "protocol_version": PROTOCOL_VERSION, - "status": "complete", - "identity": { - "git_head": "head", - "worktree_fingerprint": "tree", - "python": "python", - "skip_slow": False, - "lab": False, - "terminal_authorization": None, - }, - "selection": {"selected_count": 1, "selected_nodeids_omitted": 0}, - "expected_nodeids": [nodeid], - "expected_count": 1, - "expected_digest": hashlib.sha256(nodeid.encode()).hexdigest(), - "node_outcomes": [{"nodeid": nodeid, "outcome": "passed"}], - "exit_code": 0, - "verification_scope": "release-baseline", - "release_baseline_allowed": True, - "run_id": "green-run", - "artifact_dir": ".cache/verify/runs/green-run", - "testmon_data": file_fingerprint(main_data), - } - ) - ) - artifact = main_root / ".cache" / "verify" / "runs" / "green-run" - artifact.mkdir(parents=True) - (artifact / "run.json").write_text( - json.dumps( + (tmp_path / "pyproject.toml").write_text("[tool.pytest.ini_options]\naddopts = '-q'\n", encoding="utf-8") + initial = _testmon_environment_digest(tmp_path, pytest_profile="slow=include") + + (tmp_path / "pyproject.toml").write_text("[tool.pytest.ini_options]\naddopts = '-ra'\n", encoding="utf-8") + config_changed = _testmon_environment_digest(tmp_path, pytest_profile="slow=include") + plugin = tmp_path / "tests" / "infra" / "new_plugin.py" + plugin.parent.mkdir(parents=True) + plugin.write_text("pytest_plugins = ()\n", encoding="utf-8") + harness_changed = _testmon_environment_digest(tmp_path, pytest_profile="slow=include") + monkeypatch.setattr("devtools.testmon_bootstrap._installed_distributions", lambda: (("pytest", "changed"),)) + distributions_changed = _testmon_environment_digest(tmp_path, pytest_profile="slow=include") + monkeypatch.setenv("PYTEST_ADDOPTS", "--strict-markers") + environment_changed = _testmon_environment_digest(tmp_path, pytest_profile="slow=include") + profile_changed = _testmon_environment_digest(tmp_path, pytest_profile="slow=exclude") + + assert ( + len( { - "run_id": "green-run", - "checkout_root": str(main_root.resolve()), - "artifact_dir": ".cache/verify/runs/green-run", + initial, + config_changed, + harness_changed, + distributions_changed, + environment_changed, + profile_changed, } ) + == 6 ) - lane = tmp_path / "lane" - lane.mkdir() - (lane / ".git").write_text("gitdir: /main/.git/worktrees/lane\n") - (lane / ".venv" / "bin").mkdir(parents=True) - package = lane / "polylogue" - package.mkdir() - (package / "__init__.py").write_text("") - local_data = lane / ".cache" / "testmon" / "testmondata" - local_stamp = lane / ".cache" / "testmon" / "seed.json" - local_attempt = lane / ".cache" / "testmon" / "seed-attempt.json" - decision = decide_testmon_bootstrap( - is_linked_worktree=True, - local_testmon_data=local_data, - local_seed_stamp=local_stamp, - main_testmon_data=main_data, - main_seed_stamp=main_root / "seed.json", - main_seed_attempt=attempt, - protocol_version=PROTOCOL_VERSION, - ) - assert decision.selection_only - assert bootstrap_testmon_seed_files( - decision, - local_testmon_data=local_data, - local_seed_stamp=local_stamp, - local_seed_attempt=local_attempt, - checkout_root=lane, - inherited_from=main_root, - ) - - monkeypatch.setattr(checkout_guard, "_is_linked_worktree", lambda _root: True) - fingerprint = checkout_guard.checkout_environment_fingerprint( - lane, - polylogue_import_path=package / "__init__.py", - python_executable=lane / ".venv" / "bin" / "python", - ) - assert not fingerprint.clean - monkeypatch.setattr(verify, "ROOT", lane) - monkeypatch.setattr(verify, "TESTMON_DATA", local_data) - monkeypatch.setattr(verify, "TESTMON_SEED_STAMP", local_stamp) - monkeypatch.setattr(verify, "TESTMON_SEED_ATTEMPT", local_attempt) - assert verify._testmon_preflight(seed_testmon=False, full_pytest=False, quick=False, commit=False) is not None - assert json.loads(local_attempt.read_text())["release_baseline_allowed"] is False - - -def test_local_seed_missing_only_stamp_still_bootstraps(tmp_path: Path) -> None: - """Partial local state (e.g. a stale stamp with no db, or vice versa) still needs a fresh copy.""" - local_stamp = tmp_path / "local" / "seed.json" - local_stamp.parent.mkdir(parents=True, exist_ok=True) - local_stamp.write_text(json.dumps({"protocol_version": PROTOCOL_VERSION, "status": "usable"})) - main_data = tmp_path / "main" / "testmondata" - main_stamp = tmp_path / "main" / "seed.json" - _write_sqlite_db(main_data) - _write_valid_seed_stamp(main_stamp) - - decision = decide_testmon_bootstrap( - is_linked_worktree=True, - local_testmon_data=tmp_path / "local" / "testmondata", - local_seed_stamp=local_stamp, - main_testmon_data=main_data, - main_seed_stamp=main_stamp, - protocol_version=PROTOCOL_VERSION, - ) - assert decision.should_bootstrap - - -def test_bootstrap_seed_files_copies_db_and_stamp(tmp_path: Path) -> None: - main_data = tmp_path / "main?fragment#1" / "testmondata" - main_stamp = tmp_path / "main?fragment#1" / "seed.json" - _write_sqlite_db(main_data, rows=("x", "y", "z")) - _write_valid_seed_stamp(main_stamp) - local_data = tmp_path / "local" / "testmondata" - local_stamp = tmp_path / "local" / "seed.json" - - decision = BootstrapDecision( - True, - "test", - main_testmon_data=main_data, - main_seed_stamp=main_stamp, - ) - assert bootstrap_testmon_seed_files( - decision, - local_testmon_data=local_data, - local_seed_stamp=local_stamp, - checkout_root=tmp_path / "local", - inherited_from=tmp_path / "main?fragment#1", - ) - - local_payload = json.loads(local_stamp.read_text()) - source_payload = json.loads(main_stamp.read_text()) - comparable_keys = set(source_payload) - {"binding", "testmon_data"} - assert {key: local_payload[key] for key in comparable_keys} == {key: source_payload[key] for key in comparable_keys} - assert local_payload["binding"]["checkout_root"] == str(tmp_path / "local") - assert local_payload["binding"]["source_checkout_root"] == str(tmp_path / "main?fragment#1") - conn = sqlite3.connect(local_data) - try: - rows = conn.execute("SELECT filename, fsha FROM file_fp ORDER BY filename").fetchall() - finally: - conn.close() - assert rows == [("x", "sha-x"), ("y", "sha-y"), ("z", "sha-z")] - # No temp files left behind. - assert sorted(p.name for p in local_data.parent.iterdir()) == [".cache", "seed.json", "testmondata"] - - -def test_bootstrap_seed_files_marks_destination_and_source_checkout(tmp_path: Path) -> None: - main_data = tmp_path / "main" / "testmondata" - main_stamp = tmp_path / "main" / "seed.json" - _write_sqlite_db(main_data) - _write_valid_seed_stamp(main_stamp) - local_data = tmp_path / "lane" / "testmondata" - local_stamp = tmp_path / "lane" / "seed.json" - bootstrap_testmon_seed_files( - BootstrapDecision(True, "test", main_testmon_data=main_data, main_seed_stamp=main_stamp), - local_testmon_data=local_data, - local_seed_stamp=local_stamp, - checkout_root=tmp_path / "lane", - inherited_from=tmp_path / "main", - ) - - payload = json.loads(local_stamp.read_text()) - assert payload["binding"]["checkout_root"] == str((tmp_path / "lane").resolve()) - assert payload["binding"]["source_checkout_root"] == str((tmp_path / "main").resolve()) - source = json.loads(main_stamp.read_text()) - assert {key: payload[key] for key in source if key not in {"binding", "testmon_data"}} == { - key: source[key] for key in source if key not in {"binding", "testmon_data"} - } - - -def test_bootstrap_seed_files_rejects_paths_outside_or_colliding_with_destination(tmp_path: Path) -> None: - main_data = tmp_path / "main" / "testmondata" - main_stamp = tmp_path / "main" / "seed.json" - _write_sqlite_db(main_data) - _write_valid_seed_stamp(main_stamp) - decision = BootstrapDecision(True, "test", main_testmon_data=main_data, main_seed_stamp=main_stamp) - local_data = tmp_path / "lane" / "testmondata" - - assert not bootstrap_testmon_seed_files( - decision, - local_testmon_data=local_data, - local_seed_stamp=tmp_path / "outside" / "seed.json", - checkout_root=tmp_path / "lane", - inherited_from=tmp_path / "main", - ) - assert not (tmp_path / "outside" / "seed.json").exists() - assert not local_data.exists() - - assert not bootstrap_testmon_seed_files( - decision, - local_testmon_data=local_data, - local_seed_stamp=local_data, - checkout_root=tmp_path / "lane", - inherited_from=tmp_path / "main", - ) - assert not local_data.exists() - - -def test_bootstrap_seed_files_keeps_copied_state_when_stamp_turns_invalid(tmp_path: Path) -> None: - main_data = tmp_path / "main" / "testmondata" - main_stamp = tmp_path / "main" / "seed.json" - _write_sqlite_db(main_data) - main_stamp.parent.mkdir(parents=True, exist_ok=True) - main_stamp.write_text("{concurrent rewrite") - local_data = tmp_path / "lane" / "testmondata" - local_stamp = tmp_path / "lane" / "seed.json" - - stamped = bootstrap_testmon_seed_files( - BootstrapDecision(True, "test", main_testmon_data=main_data, main_seed_stamp=main_stamp), - local_testmon_data=local_data, - local_seed_stamp=local_stamp, - checkout_root=tmp_path / "lane", - inherited_from=tmp_path / "main", - ) - - assert stamped is False - assert not local_data.exists() - assert not local_stamp.exists() - - -def test_bootstrap_graph_mismatch_publishes_no_destination_state( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - main_data = tmp_path / "main" / "testmondata" - main_stamp = tmp_path / "main" / "seed.json" - _write_sqlite_db(main_data) - _write_valid_seed_stamp(main_stamp) - local_data = tmp_path / "lane" / "testmondata" - local_stamp = tmp_path / "lane" / "seed.json" - monkeypatch.setattr(testmon_bootstrap, "refresh_stamp", lambda *_args, **_kwargs: None) - - assert not bootstrap_testmon_seed_files( - BootstrapDecision(True, "test", main_testmon_data=main_data, main_seed_stamp=main_stamp), - local_testmon_data=local_data, - local_seed_stamp=local_stamp, - checkout_root=tmp_path / "lane", - inherited_from=tmp_path / "main", - ) - assert not local_data.exists() - assert not local_stamp.exists() - assert not (tmp_path / "lane" / ".cache" / "verify" / "current-run.json").exists() - - -def test_bootstrap_receipt_rebind_failure_publishes_no_destination_state( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - main_data = tmp_path / "main" / "testmondata" - main_stamp = tmp_path / "main" / "seed.json" - _write_sqlite_db(main_data) - _write_valid_seed_stamp(main_stamp) - local_data = tmp_path / "lane" / "testmondata" - local_stamp = tmp_path / "lane" / "seed.json" - monkeypatch.setattr(testmon_bootstrap, "_rebind_run_receipt", lambda **_kwargs: False) - - assert not bootstrap_testmon_seed_files( - BootstrapDecision(True, "test", main_testmon_data=main_data, main_seed_stamp=main_stamp), - local_testmon_data=local_data, - local_seed_stamp=local_stamp, - checkout_root=tmp_path / "lane", - inherited_from=tmp_path / "main", - ) - assert not local_data.exists() - assert not local_stamp.exists() - assert not (tmp_path / "lane" / ".cache" / "verify" / "current-run.json").exists() - - -def test_bootstrap_rebound_attempt_failure_publishes_no_destination_state( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - decision, local_data, local_stamp, local_attempt, lane = _red_attempt_decision(tmp_path) - original_stamp_from_attempt = cast(Callable[..., object], testmon_bootstrap.__dict__["stamp_from_attempt"]) - calls = 0 - - def fail_rebound_attempt(*args: object, **kwargs: object) -> object: - nonlocal calls - calls += 1 - if calls == 2: - return None - return original_stamp_from_attempt(*args, **kwargs) - - monkeypatch.setattr(testmon_bootstrap, "stamp_from_attempt", fail_rebound_attempt) - - assert not bootstrap_testmon_seed_files( - decision, - local_testmon_data=local_data, - local_seed_stamp=local_stamp, - local_seed_attempt=local_attempt, - checkout_root=lane, - inherited_from=tmp_path / "main", - ) - assert not local_data.exists() - assert not local_stamp.exists() - assert not local_attempt.exists() - assert not (lane / ".cache" / "verify" / "current-run.json").exists() - - -def test_maybe_bootstrap_does_not_migrate_an_untyped_legacy_local_stamp( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - lane = tmp_path / "lane" - main = tmp_path / "main" - local_data = lane / "cache" / "testmondata" - local_stamp = lane / "cache" / "seed.json" - _write_sqlite_db(local_data) - local_stamp.parent.mkdir(parents=True, exist_ok=True) - local_stamp.write_text(json.dumps({"protocol_version": PROTOCOL_VERSION, "status": "complete"})) - monkeypatch.setattr(testmon_bootstrap, "_git_worktree_info", lambda _root: (True, main)) - - message = testmon_bootstrap.maybe_bootstrap_testmon_seed( - lane, - testmon_data_relpath="cache/testmondata", - seed_stamp_relpath="cache/seed.json", - protocol_version=PROTOCOL_VERSION, - ) +def test_invalid_cleanup_removes_only_owned_sqlite_and_sidecars(tmp_path: Path) -> None: + state_dir = tmp_path / ".cache" / "testmon" + state_dir.mkdir(parents=True) + owned = [state_dir / "testmondata", state_dir / "testmondata-wal", state_dir / "testmondata-shm"] + unrelated = state_dir / "keep.txt" + for path in (*owned, unrelated): + path.write_text(path.name, encoding="utf-8") - assert message is None - assert json.loads(local_stamp.read_text())["status"] == "complete" + removed = remove_invalid_native_testmon_state(tmp_path) + assert set(removed) == set(owned) + assert unrelated.read_text(encoding="utf-8") == "keep.txt" -def test_bootstrap_seed_files_noop_when_decision_says_no(tmp_path: Path) -> None: - local_data = tmp_path / "local" / "testmondata" - local_stamp = tmp_path / "local" / "seed.json" - decision = BootstrapDecision(False, "not needed") - bootstrap_testmon_seed_files(decision, local_testmon_data=local_data, local_seed_stamp=local_stamp) +def test_invalid_cleanup_refuses_directory_at_database_path(tmp_path: Path) -> None: + (tmp_path / ".cache" / "testmon" / "testmondata").mkdir(parents=True) - assert not local_data.exists() - assert not local_stamp.exists() + with pytest.raises(NativeTestmonRepairError, match="refusing to remove directory"): + remove_invalid_native_testmon_state(tmp_path) diff --git a/tests/unit/devtools/test_testmon_mutation_proof.py b/tests/unit/devtools/test_testmon_mutation_proof.py deleted file mode 100644 index 3d1217701b..0000000000 --- a/tests/unit/devtools/test_testmon_mutation_proof.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Real-route tests for the bounded testmon anti-vacuity proof.""" - -from __future__ import annotations - -from devtools.testmon_mutation_proof import _TARGET_NODEID, run_proof - - -def test_real_testmon_mutation_proof() -> None: - result = run_proof() - - assert result.ok, result.failure - assert result.target_nodeid == _TARGET_NODEID - assert _TARGET_NODEID in result.selected_nodeids - assert result.mutation_exit_code != 0 - assert result.restored_exit_code == 0 - assert result.severed_edge_rejected - assert result.unrelated_selected_count < result.total_seeded_nodes - assert result.cleanup_complete diff --git a/tests/unit/devtools/test_testmon_state.py b/tests/unit/devtools/test_testmon_state.py deleted file mode 100644 index 3df6ce4440..0000000000 --- a/tests/unit/devtools/test_testmon_state.py +++ /dev/null @@ -1,455 +0,0 @@ -from __future__ import annotations - -import hashlib -import json -import sqlite3 -from pathlib import Path -from unittest.mock import patch - -import pytest - -import devtools.testmon_state as testmon_state -from devtools.testmon_state import ( - BaselineStatus, - GraphStatus, - file_fingerprint, - inspect_testmon_database, - stamp_from_attempt, - validate_stamp, -) -from devtools.testmon_state import ( - TestmonSeedStamp as _TestmonSeedStamp, -) - -PROTOCOL = 4 -NODEIDS = ("tests/test_seed.py::test_passed", "tests/test_seed.py::test_failed") - - -def _write_graph(path: Path, *, failed: bool = False, with_edges: bool = True) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - with sqlite3.connect(path) as connection: - connection.execute("CREATE TABLE environment (id INTEGER PRIMARY KEY, environment_name TEXT)") - connection.execute("CREATE TABLE file_fp (id INTEGER PRIMARY KEY, filename TEXT, fsha TEXT)") - connection.execute("CREATE TABLE test_execution (id INTEGER PRIMARY KEY, test_name TEXT, failed INTEGER)") - connection.execute("CREATE TABLE test_execution_file_fp (test_execution_id INTEGER, fingerprint_id INTEGER)") - for index, nodeid in enumerate(NODEIDS, start=1): - connection.execute("INSERT INTO file_fp VALUES (?, ?, ?)", (index, f"file-{index}.py", f"sha-{index}")) - connection.execute( - "INSERT INTO test_execution VALUES (?, ?, ?)", - (index, nodeid, int(failed and index == 2)), - ) - if with_edges: - connection.execute("INSERT INTO test_execution_file_fp VALUES (?, ?)", (index, index)) - - -def _attempt(data: Path, *, outcomes: tuple[str, str] = ("passed", "failed")) -> dict[str, object]: - artifact_dir = data.parent / ".cache" / "verify" / "runs" / "run-red" - artifact_dir.mkdir(parents=True, exist_ok=True) - (artifact_dir / "run.json").write_text( - json.dumps( - { - "run_id": "run-red", - "checkout_root": str(data.parent.resolve()), - "artifact_dir": ".cache/verify/runs/run-red", - } - ) - ) - return { - "protocol_version": PROTOCOL, - "status": "reusable", - "identity": { - "git_head": "head", - "git_tree": "tree-hash", - "worktree_fingerprint": "tree", - "python": "python", - "skip_slow": True, - "lab": False, - "terminal_authorization": "narrow-terminal", - }, - "selection": { - "selected_count": len(NODEIDS), - "selected_nodeids_omitted": 0, - }, - "expected_nodeids": list(NODEIDS), - "expected_count": len(NODEIDS), - "expected_digest": hashlib.sha256("\n".join(sorted(NODEIDS)).encode()).hexdigest(), - "verification_scope": "narrow-terminal", - "release_baseline_allowed": False, - "node_outcomes": [ - {"nodeid": nodeid, "outcome": outcome} for nodeid, outcome in zip(NODEIDS, outcomes, strict=True) - ], - "exit_code": 1, - "run_id": "run-red", - "artifact_dir": ".cache/verify/runs/run-red", - "testmon_data": file_fingerprint(data), - } - - -def test_failed_complete_graph_is_selection_only_and_rebindable(tmp_path: Path) -> None: - data = tmp_path / "testmondata" - _write_graph(data, failed=True) - - stamp = stamp_from_attempt(_attempt(data), data, checkout_root=tmp_path, protocol_version=PROTOCOL) - - assert stamp is not None - assert stamp.baseline_status is BaselineStatus.RED - assert stamp.affected_selection_allowed - assert not stamp.release_baseline_allowed - - passed_outcomes = stamp_from_attempt( - _attempt(data, outcomes=("passed", "passed")), data, checkout_root=tmp_path, protocol_version=PROTOCOL - ) - assert passed_outcomes is not None - assert passed_outcomes.baseline_status is BaselineStatus.RED - assert not passed_outcomes.release_baseline_allowed - - stamp_path = tmp_path / "seed.json" - stamp_path.write_text(json.dumps(stamp.as_dict())) - assert validate_stamp(stamp_path, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None - - -def test_seed_shard_ledger_rejects_duplicate_nodes_across_shards() -> None: - shard = { - "index": 1, - "nodeids": [NODEIDS[0]], - "nodeid_count": 1, - "nodeid_digest": hashlib.sha256(NODEIDS[0].encode()).hexdigest(), - "status": "complete", - "node_outcomes": [{"nodeid": NODEIDS[0], "outcome": "passed"}], - } - duplicate = {**shard, "index": 2} - - assert testmon_state.validate_seed_shard_ledger([shard, duplicate], expected_nodeids=[NODEIDS[0]]) is None - - -def test_seed_shard_ledger_accepts_expected_and_unexpected_xfail_outcomes() -> None: - nodes = sorted(NODEIDS) - shard = { - "index": 1, - "nodeids": nodes, - "nodeid_count": len(nodes), - "nodeid_digest": hashlib.sha256("\n".join(nodes).encode()).hexdigest(), - "status": "complete", - "node_outcomes": [ - {"nodeid": nodes[0], "outcome": "xfailed"}, - {"nodeid": nodes[1], "outcome": "xpassed"}, - ], - } - - assert testmon_state.validate_seed_shard_ledger([shard], expected_nodeids=nodes) == [shard] - - -def test_expected_failure_and_non_strict_xpass_preserve_green_baseline(tmp_path: Path) -> None: - data = tmp_path / "testmondata" - _write_graph(data) - attempt = _attempt(data, outcomes=("xfailed", "xpassed")) - attempt.update(status="complete", exit_code=0, release_baseline_allowed=True, verification_scope="release-baseline") - identity = attempt["identity"] - assert isinstance(identity, dict) - identity.update(skip_slow=False, terminal_authorization=None) - - green = stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) - - assert green is not None - assert green.baseline_status is BaselineStatus.GREEN - serialized = green.as_dict() - assert _TestmonSeedStamp.from_mapping(serialized, protocol_version=PROTOCOL).baseline_status is BaselineStatus.GREEN - - # pytest controls strict-xpass behavior through its process exit code. A - # strict xpass therefore remains reusable graph evidence but is red. - attempt.update(status="reusable", exit_code=1, release_baseline_allowed=False) - strict = stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) - - assert strict is not None - assert strict.baseline_status is BaselineStatus.RED - assert strict.affected_selection_allowed - - -def test_testmon_database_canonicalizes_xdist_group_names(tmp_path: Path) -> None: - data = tmp_path / "testmondata" - _write_graph(data) - with sqlite3.connect(data) as connection: - connection.execute( - "UPDATE test_execution SET test_name = ? WHERE test_name = ?", - (f"{NODEIDS[0]}@web-reader", NODEIDS[0]), - ) - - graph = inspect_testmon_database(data, NODEIDS) - - assert graph.missing_nodeids == () - assert graph.recorded_count == len(NODEIDS) - - -def test_omitted_interrupted_and_uncovered_nodes_fail_closed(tmp_path: Path) -> None: - data = tmp_path / "testmondata" - _write_graph(data) - omitted = _attempt(data) - omitted["selection"] = {"selected_count": 1, "selected_nodeids_omitted": 1} - assert stamp_from_attempt(omitted, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None - interrupted = _attempt(data, outcomes=("passed", "interrupted")) - assert stamp_from_attempt(interrupted, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None - data.unlink() - _write_graph(data, with_edges=False) - assert stamp_from_attempt(_attempt(data), data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None - - -def test_current_protocol_rejects_a_reusable_attempt_with_a_nonreusable_outcome( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - data = tmp_path / "testmondata" - _write_graph(data, failed=True) - attempt = _attempt(data) - attempt["protocol_version"] = 6 - identity = attempt["identity"] - assert isinstance(identity, dict) - identity["dependency_environment"] = "dependencies" - identity["pytest_harness"] = "harness" - monkeypatch.setattr(testmon_state, "testmon_runtime_identity", lambda _root: ("dependencies", "harness")) - - attempt["outcome"] = "resource-timeout" - assert stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=6) is None - - attempt["outcome"] = "red-baseline" - assert stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=6) is not None - - -def test_stamp_from_attempt_does_not_reopen_the_validated_database(tmp_path: Path) -> None: - data = tmp_path / "testmondata" - _write_graph(data) - attempt = _attempt(data) - - with patch("devtools.testmon_state.file_fingerprint", return_value=attempt["testmon_data"]) as fingerprint: - stamp = stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) - - assert stamp is not None - fingerprint.assert_called_once_with(data) - - -def test_incomplete_attempt_fails_closed(tmp_path: Path) -> None: - data = tmp_path / "testmondata" - _write_graph(data) - attempt = _attempt(data, outcomes=("passed", "passed")) - attempt["exit_code"] = 0 - attempt["release_baseline_allowed"] = True - attempt["status"] = "incomplete" - - stamp = stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) - - assert stamp is None - - attempt["status"] = "complete" - completed = stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) - - assert completed is not None - assert completed.baseline_status is BaselineStatus.GREEN - assert completed.release_baseline_allowed - - -def test_reusable_attempt_rejects_a_changed_dependency_or_pytest_harness( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - """Reusable graphs belong to the environment that captured them.""" - data = tmp_path / "testmondata" - _write_graph(data) - attempt = _attempt(data) - identity = attempt["identity"] - assert isinstance(identity, dict) - identity["dependency_environment"] = "dependency-environment" - identity["pytest_harness"] = "pytest-harness" - attempt["protocol_version"] = 5 - monkeypatch.setattr( - testmon_state, - "testmon_runtime_identity", - lambda _root: ("dependency-environment", "pytest-harness"), - raising=False, - ) - - assert stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=5) is not None - - identity["dependency_environment"] = "different-environment" - assert stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=5) is None - - -def test_runtime_identity_includes_test_behavior_environment(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - monkeypatch.setattr(testmon_state, "_installed_distributions", lambda: (("pytest", "9"),)) - monkeypatch.setenv("HYPOTHESIS_PROFILE", "ci") - monkeypatch.setenv("POLYLOGUE_CI", "1") - first = testmon_state.testmon_runtime_identity(tmp_path) - - monkeypatch.setenv("HYPOTHESIS_PROFILE", "default") - second = testmon_state.testmon_runtime_identity(tmp_path) - - assert first is not None - assert second is not None - assert first[0] == second[0] - assert first[1] != second[1] - - -def test_green_skipped_slow_attempt_without_typed_terminal_authority_is_selection_only(tmp_path: Path) -> None: - data = tmp_path / "testmondata" - _write_graph(data) - attempt = _attempt(data, outcomes=("passed", "passed")) - attempt["status"] = "complete" - attempt["exit_code"] = 0 - raw_identity = attempt["identity"] - assert isinstance(raw_identity, dict) - identity = dict(raw_identity) - identity["terminal_authorization"] = None - attempt["identity"] = identity - attempt["verification_scope"] = "narrow-terminal" - attempt["release_baseline_allowed"] = False - - stamp = stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) - - assert stamp is not None - assert stamp.baseline_status is BaselineStatus.RED - assert stamp.affected_selection_allowed - assert not stamp.release_baseline_allowed - - -def test_typed_complete_markerless_attempt_is_selection_only(tmp_path: Path) -> None: - data = tmp_path / "testmondata" - _write_graph(data, failed=False) - attempt = _attempt(data, outcomes=("passed", "passed")) - attempt["status"] = "complete" - attempt["exit_code"] = 0 - raw_identity = attempt["identity"] - assert isinstance(raw_identity, dict) - identity = dict(raw_identity) - identity["skip_slow"] = False - identity["terminal_authorization"] = None - attempt["identity"] = identity - attempt["verification_scope"] = "release-baseline" - attempt["release_baseline_allowed"] = True - - published = stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) - markerless = stamp_from_attempt( - attempt, - data, - checkout_root=tmp_path, - protocol_version=PROTOCOL, - published_marker=False, - ) - - assert published is not None - assert published.release_baseline_allowed - assert markerless is not None - assert markerless.baseline_status is BaselineStatus.RED - assert markerless.affected_selection_allowed - assert not markerless.release_baseline_allowed - - -def test_malformed_sqlite_and_stale_stamp_fail_closed(tmp_path: Path) -> None: - malformed = tmp_path / "malformed" - malformed.write_bytes(b"not sqlite") - inspection = inspect_testmon_database(malformed, NODEIDS) - assert inspection.status is GraphStatus.INVALID - - data = tmp_path / "testmondata" - _write_graph(data) - stamp = stamp_from_attempt(_attempt(data), data, checkout_root=tmp_path, protocol_version=PROTOCOL) - assert stamp is not None - stamp_path = tmp_path / "seed.json" - stamp_path.write_text(json.dumps(stamp.as_dict())) - data.write_bytes(data.read_bytes() + b"stale") - assert validate_stamp(stamp_path, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None - - -def test_attempt_and_green_stamp_artifacts_fail_closed_when_malformed(tmp_path: Path) -> None: - data = tmp_path / "testmondata" - _write_graph(data) - attempt = _attempt(data, outcomes=("passed", "passed")) - attempt["exit_code"] = 0 - attempt["release_baseline_allowed"] = True - attempt["artifact_dir"] = "/tmp/outside-testmon-run" - assert stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None - - attempt["artifact_dir"] = ".cache/verify/runs/run-red" - attempt["status"] = "complete" - stamp = stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) - assert stamp is not None - receipt = tmp_path / ".cache" / "verify" / "runs" / "run-red" / "run.json" - receipt.unlink() - assert stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None - receipt.write_text( - json.dumps( - { - "run_id": "wrong-run", - "checkout_root": str(tmp_path.resolve()), - "artifact_dir": ".cache/verify/runs/run-red", - } - ) - ) - assert stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None - receipt.write_text( - json.dumps( - { - "run_id": "run-red", - "checkout_root": str(tmp_path.resolve()), - "artifact_dir": ".cache/verify/runs/run-red", - } - ) - ) - stamp_path = tmp_path / ".cache" / "testmon" / "seed.json" - stamp_path.parent.mkdir(parents=True) - payload = stamp.as_dict() - payload["baseline"]["exit_code"] = 1 - stamp_path.write_text(json.dumps(payload)) - assert validate_stamp(stamp_path, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None - - payload = stamp.as_dict() - payload["graph"]["failed_nodeids"] = [NODEIDS[0]] - stamp_path.write_text(json.dumps(payload)) - assert validate_stamp(stamp_path, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None - - -def test_malformed_sqlite_values_fail_closed(tmp_path: Path) -> None: - data = tmp_path / "testmondata" - _write_graph(data) - with sqlite3.connect(data) as connection: - connection.execute("update test_execution set failed = 'bad' where id = 1") - - inspection = inspect_testmon_database(data, NODEIDS) - - assert inspection.status is GraphStatus.INVALID - - -def test_sqlite_paths_with_uri_characters_are_inspected_safely(tmp_path: Path) -> None: - data = tmp_path / "checkout?fragment#1" / "testmondata" - _write_graph(data) - - inspection = inspect_testmon_database(data, NODEIDS) - - assert inspection.status is GraphStatus.COMPLETE - - -@pytest.mark.parametrize("filename", ["../outside.py", "/tmp/outside.py"]) -def test_unsafe_testmon_fingerprint_paths_fail_closed(tmp_path: Path, filename: str) -> None: - data = tmp_path / "testmondata" - _write_graph(data) - with sqlite3.connect(data) as connection: - connection.execute("update file_fp set filename = ? where id = 1", (filename,)) - - assert inspect_testmon_database(data, NODEIDS).status is GraphStatus.INVALID - - -def test_attempt_status_must_be_promotable(tmp_path: Path) -> None: - data = tmp_path / "testmondata" - _write_graph(data) - attempt = _attempt(data) - attempt["status"] = "running" - - assert stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None - - attempt = _attempt(data) - attempt["run_id"] = None - assert stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None - - -def test_stamp_parser_rejects_untyped_or_non_graph_state() -> None: - try: - _TestmonSeedStamp.from_mapping({"protocol_version": PROTOCOL, "status": "complete"}, protocol_version=PROTOCOL) - except ValueError: - pass - else: - raise AssertionError("legacy green-looking stamp must not be accepted") diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 9e3ab287df..4361e558c2 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -1,12 +1,10 @@ from __future__ import annotations import fcntl -import hashlib import json import os import platform import shutil -import sqlite3 import subprocess import sys import threading @@ -21,26 +19,7 @@ import watchfiles from devtools import run_tests, verify, verify_runs -from devtools.testmon_state import ( - BaselineStatus, - BindingMode, - CollectionStatus, - GraphInspection, - GraphStatus, - file_fingerprint, -) -from devtools.testmon_state import ( - TestmonBinding as _TestmonBinding, -) -from devtools.testmon_state import ( - TestmonIdentity as _TestmonIdentity, -) -from devtools.testmon_state import ( - TestmonSeedStamp as _TestmonSeedStamp, -) -from devtools.testmon_state import ( - testmon_runtime_identity as _testmon_runtime_identity, -) +from devtools.testmon_bootstrap import executable_python_paths from devtools.verify import ( PYTEST_CONTAINMENT_PATH, PYTEST_EVENTS_PATH, @@ -49,34 +28,18 @@ PYTEST_PROGRESS_PATH, PYTEST_REPORT_PATH, ROOT, - TESTMON_AFFECTED_STAMP, - TESTMON_DATA, - TESTMON_SEED_ATTEMPT, - TESTMON_SEED_PROTOCOL_VERSION, - TESTMON_SEED_SHARD_SIZE, - TESTMON_SEED_STAMP, _anchor_verification_paths, - _checkpoint_testmon_seed_shard, - _finalize_testmon_seed_attempt, - _flatten_seed_outcomes, _format_completion_notification, - _matching_testmon_coverage, + _native_lane_failure_requires_stop, _parse_pytest_test_count, - _prepare_testmon_seed_attempt, - _prepare_testmon_seed_shards, _pytest_command_metadata, _pytest_metadata_from_report, _pytest_stall_timeout_s, _pytest_timeout_s, _read_pytest_report, - _record_testmon_affected_coverage, + _release_baseline_allowed, _run, - _seed_node_outcomes_from_events, - _seed_shard_command, _stop_after_failed_step, - _testmon_database_state, - _testmon_preflight, - _testmon_seed_can_resume, build_verify_steps, main, ) @@ -108,17 +71,10 @@ @pytest.fixture(autouse=True) def _isolate_verify_artifacts(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """Keep supervisor and testmon receipts private to each test.""" + """Keep supervisor and native testmon state private to each test.""" monkeypatch.chdir(tmp_path) - for name in ( - "TESTMON_DATA", - "TESTMON_SEED_STAMP", - "TESTMON_SEED_ATTEMPT", - "TESTMON_AFFECTED_STAMP", - ): - isolated = tmp_path / ".cache" / "testmon" / getattr(verify, name).name - monkeypatch.setattr(verify, name, isolated) - monkeypatch.setattr(sys.modules[__name__], name, isolated) + isolated = tmp_path / ".cache" / "testmon" / "testmondata" + monkeypatch.setattr(verify, "TESTMON_DATA", isolated) def _pytest_marker_expr(command: list[str]) -> str: @@ -128,78 +84,6 @@ def _pytest_marker_expr(command: list[str]) -> str: return command[marker_indexes[-1] + 1] -def _testmon_runtime_identity_fields(checkout_root: Path = ROOT) -> dict[str, str]: - runtime_identity = _testmon_runtime_identity(checkout_root) - assert runtime_identity is not None - dependency_environment, pytest_harness = runtime_identity - return {"dependency_environment": dependency_environment, "pytest_harness": pytest_harness} - - -def _write_real_testmon_state(nodeids: tuple[str, ...] = ("tests/test_a.py::test_one",)) -> Path: - TESTMON_DATA.parent.mkdir(parents=True, exist_ok=True) - with sqlite3.connect(TESTMON_DATA) as conn: - conn.execute("CREATE TABLE environment (id INTEGER PRIMARY KEY, environment_name TEXT)") - conn.execute("CREATE TABLE file_fp (id INTEGER PRIMARY KEY, filename TEXT, fsha TEXT)") - conn.execute("CREATE TABLE test_execution (id INTEGER PRIMARY KEY, test_name TEXT, failed INTEGER)") - conn.execute("CREATE TABLE test_execution_file_fp (test_execution_id INTEGER, fingerprint_id INTEGER)") - for index, nodeid in enumerate(nodeids, start=1): - conn.execute("INSERT INTO file_fp(id, filename, fsha) VALUES (?, ?, ?)", (index, nodeid, f"sha-{index}")) - conn.execute("INSERT INTO test_execution(id, test_name, failed) VALUES (?, ?, 0)", (index, nodeid)) - conn.execute("INSERT INTO test_execution_file_fp VALUES (?, ?)", (index, index)) - stamp = _TestmonSeedStamp( - TESTMON_SEED_PROTOCOL_VERSION, - CollectionStatus.COMPLETE, - nodeids, - 0, - BaselineStatus.GREEN, - True, - 0, - GraphInspection(GraphStatus.COMPLETE, len(nodeids), len(nodeids), (), 0, 0, None, ()), - _TestmonIdentity( - "current-head", - "covered", - "python", - True, - False, - None, - "narrow-terminal", - **_testmon_runtime_identity_fields(), - ), - _TestmonBinding(BindingMode.EXACT, str(ROOT.resolve())), - file_fingerprint(TESTMON_DATA), - "seed", - ".cache/verify/runs/seed", - ) - TESTMON_SEED_STAMP.parent.mkdir(parents=True, exist_ok=True) - artifact_dir = ROOT / ".cache" / "verify" / "runs" / "seed" - artifact_dir.mkdir(parents=True, exist_ok=True) - (artifact_dir / "run.json").write_text( - json.dumps( - { - "run_id": "seed", - "checkout_root": str(ROOT.resolve()), - "artifact_dir": ".cache/verify/runs/seed", - } - ) - ) - TESTMON_SEED_STAMP.write_text(json.dumps(stamp.as_dict())) - return TESTMON_DATA - - -def _write_run_receipt(root: Path, run_id: str) -> None: - artifact_dir = root / ".cache" / "verify" / "runs" / run_id - artifact_dir.mkdir(parents=True, exist_ok=True) - (artifact_dir / "run.json").write_text( - json.dumps( - { - "run_id": run_id, - "checkout_root": str(root.resolve()), - "artifact_dir": f".cache/verify/runs/{run_id}", - } - ) - ) - - def test_quick_verify_omits_pytest() -> None: steps = build_verify_steps(quick=True, lab=False, skip_slow=False) @@ -218,490 +102,63 @@ def test_quick_verify_omits_pytest() -> None: ] -def test_default_verify_uses_adaptive_pytest_testmon(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr("devtools.verify.adaptive_pytest_worker_count", lambda _env: 8) - steps = build_verify_steps(quick=False, lab=False, skip_slow=False) - - label, command = steps[-1] - assert label == "pytest testmon" - assert "--testmon" in command - assert "--testmon-noselect" not in command - assert "--testmon-forceselect" in command - assert "-n" in command - assert command[command.index("-n") + 1] == "8" - assert "--dist=loadgroup" in command - - -def test_broad_default_verify_uses_parallel_testmon(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr("devtools.verify.adaptive_pytest_worker_count", lambda _env: 8) - steps = build_verify_steps(quick=False, lab=False, skip_slow=False, broad_testmon=True) - - label, command = steps[-1] - assert label == "pytest testmon (broad)" - assert "--testmon" in command - assert "--testmon-forceselect" in command - assert command[command.index("-n") + 1] == "8" - - -def test_pytest_step_requests_structured_json_report() -> None: - """Every pytest invocation must emit the report consumed by verify and dashboards (#1026).""" - for kwargs in ( - {"seed_testmon": True}, - {"full_pytest": True}, - {}, # default testmon - ): - steps = build_verify_steps(quick=False, lab=False, skip_slow=False, **kwargs) - pytest_steps = [(label, command) for label, command in steps if label.startswith("pytest")] - assert pytest_steps, kwargs - # Every pytest lane emits a structured JSON report. - for label, command in pytest_steps: - assert "--json-report" in command, f"{label}: {command}" - assert any(arg.startswith("--json-report-file=") for arg in command), label - assert command[command.index("-p") + 1] == "devtools.pytest_progress_plugin" - # The canonical report path consumed by verify/dashboards is emitted by - # the primary lane; the #1775 isolated lane writes its own file. - expected_target = f"--json-report-file={PYTEST_REPORT_PATH}" - assert any(expected_target in command for _label, command in pytest_steps), kwargs - - -def test_seed_testmon_runs_full_collection_without_selection(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.delenv("POLYLOGUE_PYTEST_WORKERS", raising=False) +@pytest.mark.parametrize( + ("mode", "selection_flag"), + [("affected", "--testmon-forceselect"), ("bootstrap", "--testmon-noselect"), ("full", "--testmon-noselect")], +) +def test_native_testmon_uses_exactly_two_semantic_lanes( + monkeypatch: pytest.MonkeyPatch, + mode: str, + selection_flag: str, +) -> None: monkeypatch.setattr("devtools.verify.adaptive_pytest_worker_count", lambda _env: 8) - steps = build_verify_steps(quick=False, lab=False, skip_slow=False, seed_testmon=True) - - label, command = steps[-1] - assert label == "pytest seed-testmon collect" - assert "--collect-only" in command - assert command[command.index("--ignore=tests/benchmarks")] == "--ignore=tests/benchmarks" - assert "--testmon" not in command - assert "-n" in command - assert command[command.index("-n") + 1] == "0" - - -def test_seed_testmon_caps_adaptive_workers(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.delenv("POLYLOGUE_PYTEST_WORKERS", raising=False) - monkeypatch.setattr("devtools.verify.adaptive_pytest_worker_count", lambda _env: 12) - - steps = build_verify_steps(quick=False, lab=False, skip_slow=False, seed_testmon=True) - - label, command = steps[-1] - assert label == "pytest seed-testmon collect" - assert command[command.index("-n") + 1] == "0" - - -def test_seed_shards_are_deterministic_and_use_managed_xdist(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.chdir(tmp_path) - monkeypatch.setattr("devtools.verify.adaptive_pytest_worker_count", lambda _environment: 64) - expected = sorted(f"tests/test_seed.py::test_{index:03d}" for index in range(TESTMON_SEED_SHARD_SIZE + 2)) - prepared = _prepare_testmon_seed_shards( - {"resume": False, "expected_nodeids": []}, - selection={ - "selected_count": len(expected), - "selected_nodeids": list(reversed(expected)), - "selected_nodeids_omitted": 0, - }, - ) - - shards = prepared["shards"] - assert [shard["nodeid_count"] for shard in shards] == [TESTMON_SEED_SHARD_SIZE, 2] - assert shards[0]["nodeids"] == expected[:TESTMON_SEED_SHARD_SIZE] - assert shards[1]["nodeids"] == expected[TESTMON_SEED_SHARD_SIZE:] - nodeids_file = tmp_path / "seed-shard.args" - command = _seed_shard_command(["pytest", "--collect-only", "-n", "0"], shards[0], nodeids_file=nodeids_file) - assert "--collect-only" not in command - assert command[command.index("-n") + 1] == "10" - assert "--testmon" in command - assert "--testmon-noselect" in command - assert "--dist=loadgroup" in command - assert command.count("-n") == 1 - assert command[command.index("-n") + 1] == "10" - assert command[-1] == f"@{nodeids_file}" - assert nodeids_file.read_text().splitlines() == expected[:TESTMON_SEED_SHARD_SIZE] - - -def test_seed_outcomes_normalize_xdist_group_suffix(tmp_path: Path) -> None: - expected = ["tests/test_seed.py::test_grouped"] - events = tmp_path / "events.jsonl" - events.write_text( - json.dumps( - { - "event": "test_report", - "nodeid": f"{expected[0]}@web-reader", - "when": "call", - "outcome": "passed", - } - ) - + "\n" - ) - outcomes = _seed_node_outcomes_from_events( - events, - expected_nodeids=expected, - database={"node_outcomes": {}}, - pytest_step=None, + steps = build_verify_steps( + quick=False, + lab=False, + skip_slow=False, + testmon_mode=mode, + testmon_environment="env-digest", ) - assert outcomes == [ - { - "nodeid": expected[0], - "outcome": "passed", - "reason": "test call passed", - "started": False, - "finished": False, - "phases": [{"when": "call", "outcome": "passed", "duration_s": None}], - } - ] - - -def test_seed_shard_checkpoint_preserves_completed_shards_for_resume( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.chdir(tmp_path) - expected = ["tests/test_seed.py::test_b", "tests/test_seed.py::test_a"] - ordered = sorted(expected) - prepared = _prepare_testmon_seed_shards( - {"resume": False, "expected_nodeids": []}, - selection={"selected_count": 2, "selected_nodeids": expected, "selected_nodeids_omitted": 0}, - ) - prepared["shards"] = [ - { - **prepared["shards"][0], - "nodeids": [ordered[0]], - "nodeid_count": 1, - "nodeid_digest": hashlib.sha256(ordered[0].encode()).hexdigest(), - }, - { - **prepared["shards"][0], - "index": 2, - "nodeids": [ordered[1]], - "nodeid_count": 1, - "nodeid_digest": hashlib.sha256(ordered[1].encode()).hexdigest(), - "status": "pending", - "node_outcomes": [], - }, + pytest_steps = [(label, command) for label, command in steps if label.startswith("pytest")] + assert [label for label, _command in pytest_steps] == [ + f"pytest native parallel ({mode})", + f"pytest native serial ({mode})", ] - _atomic_payload = { - **prepared, - "expected_nodeids": ordered, - "expected_count": 2, - "expected_digest": hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest(), - } - artifact_dir = tmp_path / "shard-1" - artifact_dir.mkdir() - (artifact_dir / "selection.json").write_text( - json.dumps( - { - "selected_count": 1, - "selected_nodeids": [f"{ordered[0]}@web-reader"], - "selected_nodeids_omitted": 0, - } - ) - ) - (artifact_dir / "events.jsonl").write_text( - json.dumps( - { - "event": "test_report", - "nodeid": f"{ordered[0]}@web-reader", - "when": "call", - "outcome": "xfailed", - } - ) - + "\n" - ) - - checkpointed = _checkpoint_testmon_seed_shard( - prepared=_atomic_payload, - shard_index=1, - step={"name": "pytest seed-testmon shard 1/2", "exit": 0, "artifact_dir": str(artifact_dir)}, - ) - - assert checkpointed["shards"][0]["status"] == "complete" - assert checkpointed["shards"][1]["status"] == "pending" - assert json.loads(TESTMON_SEED_ATTEMPT.read_text())["shards"][0]["node_outcomes"][0]["outcome"] == "xfailed" - resumed = _prepare_testmon_seed_attempt( - identity={ - "git_head": "head", - "git_tree": "tree", - "worktree_fingerprint": "fingerprint", - "python": "python", - "skip_slow": False, - "lab": False, - **_testmon_runtime_identity_fields(Path.cwd()), - }, - run=VerifyRun(tier="seed-testmon", argv=[], git_head="head", polylogue_import_path="polylogue"), - resume=True, - ) - assert resumed["shards"][0]["status"] == "complete" - assert resumed["shards"][1]["status"] == "pending" - - -def test_seed_shard_checkpoint_does_not_trust_preexisting_testmon_rows( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.chdir(tmp_path) - expected = ["tests/test_seed.py::test_only_database_row"] - prepared = _prepare_testmon_seed_shards( - {"resume": False, "expected_nodeids": []}, - selection={"selected_count": 1, "selected_nodeids": expected, "selected_nodeids_omitted": 0}, - ) - artifact_dir = tmp_path / "shard-1" - artifact_dir.mkdir() - (artifact_dir / "selection.json").write_text( - json.dumps({"selected_count": 1, "selected_nodeids": expected, "selected_nodeids_omitted": 0}) - ) - (artifact_dir / "events.jsonl").write_text("") - monkeypatch.setattr( - "devtools.verify._testmon_database_state", - lambda _nodeids: { - "recorded_count": 1, - "failed_count": 0, - "dependency_edge_count": 0, - "missing_nodeids": [], - "failed_nodeids": [], - "node_outcomes": {expected[0]: "passed"}, - "error": None, - "graph_status": "complete", - "orphan_execution_edges": 0, - "orphan_fingerprint_edges": 0, - }, - ) - - checkpointed = _checkpoint_testmon_seed_shard( - prepared=prepared, - shard_index=1, - step={"name": "pytest seed-testmon shard 1/1", "exit": 0, "artifact_dir": str(artifact_dir)}, - ) - - shard = checkpointed["shards"][0] - assert shard["status"] == "incomplete" - assert shard["node_outcomes"][0]["outcome"] == "missing" - - -def test_seed_outcome_does_not_infer_call_success_from_teardown( - tmp_path: Path, -) -> None: - nodeid = "tests/test_seed.py::test_call_missing" - events = tmp_path / "events.jsonl" - events.write_text( - json.dumps({"event": "test_started", "nodeid": nodeid}) - + "\n" - + json.dumps({"event": "test_report", "nodeid": nodeid, "when": "teardown", "outcome": "passed"}) - + "\n" - + json.dumps({"event": "test_finished", "nodeid": nodeid}) - + "\n" - ) - - without_database = _seed_node_outcomes_from_events( - events, - expected_nodeids=[nodeid], - database={"node_outcomes": {}}, - pytest_step={"exit": 0}, - ) - assert without_database[0]["outcome"] == "missing" - - with_failed_database = _seed_node_outcomes_from_events( - events, - expected_nodeids=[nodeid], - database={"node_outcomes": {nodeid: "failed"}}, - pytest_step={"exit": 1}, - ) - assert with_failed_database[0]["outcome"] == "failed" - - -def test_seed_shard_failure_remains_visible_and_blocks_release(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.chdir(tmp_path) - expected = ["tests/test_seed.py::test_failed"] - prepared = _prepare_testmon_seed_shards( - { - "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "running", - "identity": { - "git_head": "head", - "worktree_fingerprint": "tree", - "python": "python", - "skip_slow": False, - "lab": False, - **_testmon_runtime_identity_fields(Path.cwd()), - }, - "resume": False, - "run_id": "sharded-failure", - "artifact_dir": ".cache/verify/runs/sharded-failure", - }, - selection={"selected_count": 1, "selected_nodeids": expected, "selected_nodeids_omitted": 0}, - ) - artifact_dir = tmp_path / "shard-failure" - artifact_dir.mkdir() - (artifact_dir / "selection.json").write_text( - json.dumps({"selected_count": 1, "selected_nodeids": expected, "selected_nodeids_omitted": 0}) - ) - (artifact_dir / "events.jsonl").write_text( - json.dumps({"event": "test_report", "nodeid": expected[0], "when": "call", "outcome": "failed"}) + "\n" - ) - TESTMON_DATA.parent.mkdir(parents=True, exist_ok=True) - with sqlite3.connect(TESTMON_DATA) as connection: - connection.execute("create table environment (id integer primary key, environment_name text)") - connection.execute("create table file_fp (id integer primary key, filename text, fsha text)") - connection.execute("create table test_execution (id integer primary key, test_name text, failed integer)") - connection.execute("create table test_execution_file_fp (test_execution_id integer, fingerprint_id integer)") - connection.execute("insert into test_execution values (1, ?, 1)", (expected[0],)) - connection.execute("insert into file_fp values (1, 'test_seed.py', 'sha')") - connection.execute("insert into test_execution_file_fp values (1, 1)") - _write_run_receipt(tmp_path, "sharded-failure") - - checkpointed = _checkpoint_testmon_seed_shard( - prepared=prepared, - shard_index=1, - step={"name": "pytest seed-testmon shard 1/1", "exit": 1, "artifact_dir": str(artifact_dir)}, - ) - receipt = _finalize_testmon_seed_attempt( - prepared=checkpointed, - step_results=[{"name": "pytest seed-testmon shard 1/1", "exit": 1, "artifact_dir": str(artifact_dir)}], - exit_code=1, - ) - - assert receipt["shards"][0]["status"] == "complete" - assert receipt["unsuccessful_nodeids"] == expected - assert receipt["release_baseline_allowed"] is False - - -def test_resumed_seed_uses_affected_selection_for_remaining_tests() -> None: - steps = build_verify_steps( + parallel = pytest_steps[0][1] + serial = pytest_steps[1][1] + assert "not load_sensitive and not tui" in _pytest_marker_expr(parallel) + assert "load_sensitive or tui" in _pytest_marker_expr(serial) + assert parallel[parallel.index("-n") + 1] == "8" + assert serial[serial.index("-n") + 1] == "0" + for _label, command in pytest_steps: + assert "--testmon" in command + assert "--testmon-env=env-digest" in command + assert selection_flag in command + assert "--json-report" in command + assert command[command.index("-p") + 1] == "devtools.pytest_progress_plugin" + + +def test_native_marker_policy_composes_slow_and_scale_tiers() -> None: + default_steps = build_verify_steps( quick=False, lab=False, + skip_slow=True, + testmon_environment="env-digest", + ) + lab_steps = build_verify_steps( + quick=False, + lab=True, skip_slow=False, - seed_testmon=True, - resume_testmon_seed=True, + testmon_environment="env-digest", ) - label, command = steps[-1] - assert label == "pytest seed-testmon collect (resume)" - assert "--collect-only" in command - assert "--testmon" not in command - - -def test_full_verify_includes_full_pytest_without_testmon(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.delenv("POLYLOGUE_PYTEST_WORKERS", raising=False) - monkeypatch.setattr("devtools.verify.adaptive_pytest_worker_count", lambda _env: 8) - steps = build_verify_steps(quick=False, lab=False, skip_slow=False, full_pytest=True) - - # #1775: the full diagnostic runs as two lanes — a parallel bulk lane plus a - # single-process isolated lane for load-sensitive/tui tests. Neither uses - # testmon; the bulk lane keeps xdist parallelism, the isolated lane forces -n 0. - labels = [label for label, _command in steps] - assert labels[-2:] == ["pytest full (parallel)", "pytest load-sensitive (isolated)"] - - bulk_label, bulk_command = steps[-2] - assert bulk_label == "pytest full (parallel)" - assert "--testmon" not in bulk_command - assert "-n" in bulk_command - assert bulk_command[bulk_command.index("-n") + 1] == "8" - assert "--dist=loadgroup" in bulk_command - - isolated_label, isolated_command = steps[-1] - assert isolated_label == "pytest load-sensitive (isolated)" - assert "--testmon" not in isolated_command - assert isolated_command[isolated_command.index("-n") + 1] == "0" - - -def test_seed_collection_refuses_parallel_worker_overrides(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("POLYLOGUE_PYTEST_WORKERS", "4") - - steps = build_verify_steps(quick=False, lab=False, skip_slow=False, seed_testmon=True) - - label, command = steps[-1] - assert label == "pytest seed-testmon collect" - assert command[command.index("-n") + 1] == "0" - - -def test_seed_defaults_to_managed_scratch(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - shm, scratch = _patch_basetemp_roots(monkeypatch, tmp_path, realm_mounted=True) - _patch_resource_capacity(monkeypatch, shm=shm, scratch=scratch, available_mb=8192) - for name in ( - "POLYLOGUE_PYTEST_BASETEMP_ROOT", - "POLYLOGUE_PYTEST_TMPFS", - "POLYLOGUE_PYTEST_TMPFS_MAX_MB", - "POLYLOGUE_PYTEST_BASETEMP_REQUIRED_MB", - "POLYLOGUE_PYTEST_BASETEMP_MIN_FREE_MB", - ): - monkeypatch.delenv(name, raising=False) - completed = subprocess.CompletedProcess(args=["pytest"], returncode=0, stdout="1 passed in 0.1s\n", stderr="") - - with ( - patch("devtools.verify._run_pytest_with_heartbeat", return_value=completed) as run, - patch("devtools.verify._read_pytest_report", return_value=None), - ): - rc, _elapsed, metadata = _run("pytest seed-testmon", ["pytest", "--testmon", "--testmon-noselect"]) - - assert rc == 0 - assert metadata["pytest_tmpfs"] is False - assert run.call_args.kwargs["env"]["POLYLOGUE_PYTEST_TMPFS"] == "0" - assert run.call_args.kwargs["env"]["POLYLOGUE_PYTEST_BASETEMP_ROOT"] == str(scratch) - assert run.call_args.kwargs["env"]["POLYLOGUE_PYTEST_SELECTION_NODEID_LIMIT"] == "50000" - - -def test_default_testmon_worker_count_can_be_overridden(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("POLYLOGUE_PYTEST_WORKERS", "3") - - steps = build_verify_steps(quick=False, lab=False, skip_slow=False) - - label, command = steps[-1] - assert label == "pytest testmon" - assert command[command.index("-n") + 1] == "3" - - -def test_marker_filters_keep_testmon_selection_forced() -> None: - steps = build_verify_steps(quick=False, lab=False, skip_slow=False) - - label, command = steps[-1] - assert label == "pytest testmon" - marker_expr = _pytest_marker_expr(command) - assert "not benchmark" in marker_expr - assert "not scale_medium" in marker_expr - assert "not scale_large" in marker_expr - assert "--testmon-forceselect" in command - - -def test_skip_slow_composes_with_forced_testmon_selection() -> None: - steps = build_verify_steps(quick=False, lab=False, skip_slow=True) - - label, command = steps[-1] - assert label == "pytest testmon" - # Scale-tier policy (#1183): the default verify gate filters out - # ``scale_medium``/``scale_large``; ``--skip-slow`` composes with that - # filter via ``and`` rather than replacing it. - marker_expr = _pytest_marker_expr(command) - assert "not benchmark" in marker_expr - assert "not slow" in marker_expr - assert "not scale_medium" in marker_expr - assert "not scale_large" in marker_expr - assert "--testmon-forceselect" in command - - -def test_default_verify_excludes_medium_and_large_scale_markers() -> None: - """Default verify pytest step deselects the medium/large scale tiers (#1183).""" - steps = build_verify_steps(quick=False, lab=False, skip_slow=False) - - label, command = steps[-1] - assert label == "pytest testmon" - marker_expr = _pytest_marker_expr(command) - assert "not benchmark" in marker_expr - assert "not scale_medium" in marker_expr - assert "not scale_large" in marker_expr - # ``scale_small`` is *not* excluded — it runs in the default gate. - assert "scale_small" not in marker_expr - - -def test_lab_verify_includes_medium_scale_marker() -> None: - """``--lab`` lets ``scale_medium`` into the pytest step but still gates ``scale_large`` (#1183).""" - steps = build_verify_steps(quick=False, lab=True, skip_slow=False) - - pytest_step = next((label, command) for label, command in steps if label.startswith("pytest")) - label, command = pytest_step - marker_expr = _pytest_marker_expr(command) - assert "not benchmark" in marker_expr - assert "not scale_large" in marker_expr - assert "not scale_medium" not in marker_expr - assert "scale_small" not in marker_expr + default_expr = _pytest_marker_expr(next(command for label, command in default_steps if "parallel" in label)) + lab_expr = _pytest_marker_expr(next(command for label, command in lab_steps if "parallel" in label)) + assert all(term in default_expr for term in ("not benchmark", "not slow", "not scale_medium", "not scale_large")) + assert "not scale_medium" not in lab_expr + assert "not scale_large" in lab_expr def test_lab_verify_delegates_to_lab_smoke() -> None: @@ -717,175 +174,6 @@ def test_lab_verify_delegates_to_lab_smoke() -> None: ) -def test_testmon_preflight_requires_seed_when_database_missing(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.chdir(tmp_path) - - message = _testmon_preflight(seed_testmon=False, full_pytest=False, quick=False, commit=False) - - assert message is not None - assert "devtools verify --seed-testmon" in message - - -def test_testmon_preflight_requires_seed_stamp(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.chdir(tmp_path) - TESTMON_DATA.parent.mkdir(parents=True) - TESTMON_DATA.write_text("partial") - - message = _testmon_preflight(seed_testmon=False, full_pytest=False, quick=False, commit=False) - - assert message is not None - assert ".cache/testmon/seed.json" in message - - -def test_testmon_preflight_accepts_seeded_database(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.chdir(tmp_path) - _write_real_testmon_state() - - assert _testmon_preflight(seed_testmon=False, full_pytest=False, quick=False, commit=False) is None - - -def test_testmon_preflight_rejects_stale_database_fingerprint( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] -) -> None: - monkeypatch.chdir(tmp_path) - _write_real_testmon_state() - TESTMON_DATA.write_bytes(TESTMON_DATA.read_bytes() + b"stale") - - message = _testmon_preflight(seed_testmon=False, full_pytest=False, quick=False, commit=False) - - assert message is not None - assert "stale" in message - assert capsys.readouterr().err == "" - - -def test_testmon_preflight_rejects_malformed_sqlite_state( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] -) -> None: - monkeypatch.chdir(tmp_path) - TESTMON_DATA.parent.mkdir(parents=True) - TESTMON_DATA.write_text("mutated") - seed_stamp = tmp_path / ".cache" / "testmon" / "seed.json" - seed_stamp.parent.mkdir(parents=True, exist_ok=True) - seed_stamp.write_text( - json.dumps( - { - "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "usable", - } - ) - ) - - message = _testmon_preflight(seed_testmon=False, full_pytest=False, quick=False, commit=False) - - assert message is not None - assert "stale" in message or "malformed" in message - assert capsys.readouterr().err == "" - - -def test_testmon_preflight_rejects_incomplete_seed_receipt(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.chdir(tmp_path) - TESTMON_DATA.parent.mkdir(parents=True) - TESTMON_DATA.write_text("partial") - seed_stamp = tmp_path / ".cache" / "testmon" / "seed.json" - seed_stamp.write_text( - json.dumps( - { - "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "incomplete", - "git_head": "current-head", - "testmon_data": hashlib.sha256(b"partial").hexdigest(), - } - ) - ) - - message = _testmon_preflight(seed_testmon=False, full_pytest=False, quick=False, commit=False) - - assert message is not None - assert "stale" in message or "malformed" in message - - -def test_matching_incomplete_seed_is_resumable(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.chdir(tmp_path) - TESTMON_DATA.parent.mkdir(parents=True) - TESTMON_DATA.write_text("partial") - identity = { - "git_head": "head", - "git_tree": "tree-hash", - "worktree_fingerprint": "tree", - "python": "3.13", - "skip_slow": True, - "lab": False, - "terminal_authorization": None, - } - TESTMON_SEED_ATTEMPT.write_text( - json.dumps( - { - "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "incomplete", - "identity": identity, - "expected_nodeids": ["tests/unit/test_example.py::test_one"], - "expected_count": 1, - "expected_digest": hashlib.sha256(b"tests/unit/test_example.py::test_one").hexdigest(), - "run_id": "interrupted", - "started_at": "2026-08-05T12:00:00+00:00", - "testmon_data_before": "partial", - } - ) - ) - - assert _testmon_seed_can_resume(identity) is True - assert _testmon_seed_can_resume({**identity, "git_head": "other", "git_tree": "tree-hash"}) is True - assert _testmon_seed_can_resume({**identity, "git_tree": "different-tree"}) is False - assert _testmon_seed_can_resume({**identity, "worktree_fingerprint": "changed"}) is False - assert _testmon_seed_can_resume({**identity, "skip_slow": False}) is False - - -def test_two_interrupted_resumes_flatten_all_carried_outcomes(tmp_path: Path) -> None: - monkeypatch = pytest.MonkeyPatch() - monkeypatch.chdir(tmp_path) - try: - expected = ["tests/test_a.py::test_one", "tests/test_b.py::test_two"] - TESTMON_DATA.parent.mkdir(parents=True) - TESTMON_DATA.write_text("partial") - identity = { - "git_head": "head", - "git_tree": "tree-hash", - "worktree_fingerprint": "tree", - "python": "3.13", - "skip_slow": False, - "lab": False, - "terminal_authorization": None, - } - TESTMON_SEED_ATTEMPT.write_text( - json.dumps( - { - "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "incomplete", - "identity": identity, - "expected_nodeids": expected, - "expected_count": len(expected), - "expected_digest": hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest(), - "node_outcomes": [{"nodeid": expected[0], "outcome": "passed"}], - } - ) - ) - first = VerifyRun(tier="seed-testmon", argv=["--seed-testmon"], git_head="head", root=tmp_path) - _prepare_testmon_seed_attempt(identity=identity, run=first, resume=True) - first_payload = json.loads(TESTMON_SEED_ATTEMPT.read_text()) - first_payload["status"] = "incomplete" - first_payload["node_outcomes"] = [{"nodeid": expected[1], "outcome": "passed"}] - TESTMON_SEED_ATTEMPT.write_text(json.dumps(first_payload)) - - second = VerifyRun(tier="seed-testmon", argv=["--seed-testmon"], git_head="head", root=tmp_path) - prepared = _prepare_testmon_seed_attempt(identity=identity, run=second, resume=True) - - assert {item["nodeid"] for item in prepared["prior_node_outcomes"]} == set(expected) - assert {item["outcome"] for item in prepared["prior_node_outcomes"]} == {"passed"} - assert _flatten_seed_outcomes(prepared) == prepared["prior_node_outcomes"] - finally: - monkeypatch.undo() - - def test_focused_run_can_record_typed_affected_scope(tmp_path: Path) -> None: run = VerifyRun(tier="focused-test", argv=["tests/unit/example.py"], git_head="head", root=tmp_path) @@ -1384,45 +672,6 @@ def _write_worker_facts(*_args: object, artifacts: PytestStepArtifacts, **_kwarg assert selection["worker_id"] == "runner" -def test_verify_main_records_containment_failure_as_terminal_history( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - history_path = tmp_path / "verify-history.jsonl" - monkeypatch.setattr(verify, "HISTORY_PATH", history_path) - - with ( - patch("devtools.verify._anchor_verification_paths"), - patch("devtools.verify._git_head", return_value="head"), - patch("devtools.verify._git_commit", return_value="base"), - patch("devtools.verify._default_testmon_is_broad_change", return_value=False), - patch("devtools.verify._testmon_preflight", return_value=None), - patch("devtools.verify.build_verify_steps", return_value=[("pytest containment", ["pytest", "-n", "0"])]), - patch("devtools.verify.apply_managed_pytest_runtime_policy", return_value=({}, None)), - patch( - "devtools.verify._run_pytest_with_heartbeat", - side_effect=verify.PytestContainmentError("owned child still running"), - ), - patch("devtools.verify.cleanup_managed_pytest_basetemp") as cleanup, - patch("devtools.verify._notify"), - ): - rc = main(["--json"]) - - history = json.loads(history_path.read_text(encoding="utf-8")) - run_json = next((tmp_path / ".cache" / "verify" / "runs").glob("*/run.json")) - run_payload = json.loads(run_json.read_text(encoding="utf-8")) - payload = json.loads(capsys.readouterr().out) - - assert rc == 125 - cleanup.assert_not_called() - assert payload["diagnosis"] == "pytest_containment_unproven" - assert history["exit_code"] == 125 - assert history["diagnosis"] == "pytest_containment_unproven" - assert run_payload["status"] == "failed" - assert run_payload["steps"][0]["status"] == "failed" - - def test_print_history_accepts_verify_and_focused_run_records( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], @@ -1557,200 +806,6 @@ def test_compare_against_last_selects_prior_run_independently_per_step(monkeypat assert "pytest testmon" in flags[0] -def test_running_seed_recovers_ledger_from_selection_artifact(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.chdir(tmp_path) - TESTMON_DATA.parent.mkdir(parents=True) - TESTMON_DATA.write_text("partial") - artifact_dir = tmp_path / ".cache" / "verify" / "runs" / "interrupted" - step_dir = artifact_dir / "steps" / "17-pytest-seed-testmon" - step_dir.mkdir(parents=True) - expected = ["tests/unit/test_example.py::test_one"] - (step_dir / "selection.json").write_text( - json.dumps({"selected_nodeids": expected, "selected_nodeids_omitted": 0, "selected_count": 1}) - ) - identity = { - "git_head": "head", - "git_tree": "tree-hash", - "worktree_fingerprint": "tree", - "python": "3.13", - "skip_slow": True, - "lab": False, - "terminal_authorization": None, - } - TESTMON_SEED_ATTEMPT.write_text( - json.dumps( - { - "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "running", - "identity": identity, - "expected_nodeids": [], - "artifact_dir": str(artifact_dir.relative_to(tmp_path)), - } - ) - ) - - assert _testmon_seed_can_resume({**identity, "git_head": "fixed", "git_tree": "tree-hash"}) is True - - run = VerifyRun(tier="seed-testmon", argv=["--seed-testmon"], git_head="fixed") - prepared = _prepare_testmon_seed_attempt( - identity={**identity, "git_head": "fixed", "git_tree": "tree-hash"}, run=run, resume=True - ) - - assert prepared["expected_nodeids"] == expected - assert prepared["expected_count"] == 1 - assert prepared["expected_digest"] == hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest() - persisted = json.loads(TESTMON_SEED_ATTEMPT.read_text()) - assert persisted["expected_digest"] == prepared["expected_digest"] - - -def test_seed_resume_rejects_selection_artifact_outside_checkout( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.chdir(tmp_path) - TESTMON_DATA.parent.mkdir(parents=True) - TESTMON_DATA.write_text("partial") - outside = tmp_path.parent / "outside-testmon-artifacts" - step_dir = outside / "steps" / "17-pytest-seed-testmon" - step_dir.mkdir(parents=True) - (step_dir / "selection.json").write_text( - json.dumps( - { - "selected_nodeids": ["tests/unit/test_example.py::test_one"], - "selected_nodeids_omitted": 0, - "selected_count": 1, - } - ) - ) - identity = { - "git_head": "head", - "worktree_fingerprint": "tree", - "python": "3.13", - "skip_slow": True, - "lab": False, - } - TESTMON_SEED_ATTEMPT.write_text( - json.dumps( - { - "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "running", - "identity": identity, - "expected_nodeids": [], - "artifact_dir": str(outside), - } - ) - ) - - assert _testmon_seed_can_resume(identity) is False - - -def test_resumed_seed_does_not_reuse_an_unexecuted_database_row(tmp_path: Path) -> None: - monkeypatch = pytest.MonkeyPatch() - monkeypatch.chdir(tmp_path) - try: - expected = ["tests/test_a.py::test_one", "tests/test_b.py::test_two"] - artifact_dir = tmp_path / "artifacts" - artifact_dir.mkdir() - (artifact_dir / "selection.json").write_text( - json.dumps({"selected_count": 1, "selected_nodeids": [expected[0]], "selected_nodeids_omitted": 0}) - ) - (artifact_dir / "events.jsonl").write_text( - json.dumps({"event": "test_report", "nodeid": expected[0], "when": "call", "outcome": "passed"}) + "\n" - ) - TESTMON_DATA.parent.mkdir(parents=True) - with sqlite3.connect(TESTMON_DATA) as connection: - connection.execute("create table environment (id integer primary key, environment_name text)") - connection.execute("create table file_fp (id integer primary key, filename text, fsha text)") - connection.execute("create table test_execution (id integer primary key, test_name text, failed integer)") - connection.execute( - "create table test_execution_file_fp (test_execution_id integer, fingerprint_id integer)" - ) - connection.executemany("insert into test_execution values (?, ?, 0)", [(1, expected[0]), (2, expected[1])]) - connection.executemany("insert into file_fp values (?, ?, ?)", [(1, "a.py", "a"), (2, "b.py", "b")]) - connection.executemany("insert into test_execution_file_fp values (?, ?)", [(1, 1), (2, 2)]) - prepared = { - "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "running", - "identity": { - "git_head": "head", - "worktree_fingerprint": "tree", - "python": "python", - "skip_slow": False, - "lab": False, - **_testmon_runtime_identity_fields(Path.cwd()), - }, - "resume": True, - "expected_nodeids": expected, - "run_id": "resume", - "artifact_dir": ".cache/verify/runs/resume", - } - _write_run_receipt(tmp_path, "resume") - - receipt = _finalize_testmon_seed_attempt( - prepared=prepared, - step_results=[{"name": "pytest seed-testmon (resume)", "artifact_dir": str(artifact_dir)}], - exit_code=0, - ) - - assert receipt["status"] == "incomplete" - assert {item["nodeid"]: item["outcome"] for item in receipt["node_outcomes"]} == { - expected[0]: "passed", - expected[1]: "missing", - } - - (artifact_dir / "selection.json").write_text(json.dumps({})) - (artifact_dir / "events.jsonl").write_text( - "\n".join( - json.dumps({"event": "test_report", "nodeid": nodeid, "when": "call", "outcome": "passed"}) - for nodeid in expected - ) - + "\n" - ) - missing_selection = _finalize_testmon_seed_attempt( - prepared=prepared, - step_results=[{"name": "pytest seed-testmon (resume)", "artifact_dir": str(artifact_dir)}], - exit_code=0, - ) - assert missing_selection["status"] == "incomplete" - finally: - monkeypatch.undo() - - -def test_testmon_database_state_reports_missing_and_failed_nodes( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.chdir(tmp_path) - TESTMON_DATA.parent.mkdir(parents=True) - with sqlite3.connect(TESTMON_DATA) as conn: - conn.execute("CREATE TABLE environment (id INTEGER PRIMARY KEY, environment_name TEXT)") - conn.execute("CREATE TABLE file_fp (id INTEGER PRIMARY KEY, filename TEXT, fsha TEXT)") - conn.execute( - "CREATE TABLE test_execution (id INTEGER PRIMARY KEY, test_name TEXT NOT NULL, failed INTEGER NOT NULL)" - ) - conn.execute("CREATE TABLE test_execution_file_fp (test_execution_id INTEGER, fingerprint_id INTEGER)") - conn.executemany( - "INSERT INTO test_execution(test_name, failed) VALUES (?, ?)", - [("tests/test_a.py::test_ok", 0), ("tests/test_b.py::test_failed", 1)], - ) - conn.executemany( - "INSERT INTO file_fp(id, filename, fsha) VALUES (?, ?, ?)", - [(1, "a.py", "a"), (2, "b.py", "b")], - ) - conn.executemany("INSERT INTO test_execution_file_fp VALUES (?, ?)", [(1, 1), (2, 2)]) - - state = _testmon_database_state( - ["tests/test_a.py::test_ok", "tests/test_b.py::test_failed", "tests/test_c.py::test_missing"] - ) - - assert state["recorded_count"] == 2 - assert state["failed_nodeids"] == ["tests/test_b.py::test_failed"] - assert state["missing_nodeids"] == ["tests/test_c.py::test_missing"] - assert state["node_outcomes"] == { - "tests/test_a.py::test_ok": "passed", - "tests/test_b.py::test_failed": "failed", - "tests/test_c.py::test_missing": "missing", - } - - def test_worktree_fingerprint_hashes_untracked_file_contents(tmp_path: Path) -> None: subprocess.run(["git", "init", "-q"], check=True) subprocess.run(["git", "config", "user.email", "tests@example.invalid"], check=True) @@ -1816,7 +871,8 @@ def test_changed_paths_keep_start_time_base_when_remote_ref_advances( assert pinned_base == base subprocess.run(["git", "update-ref", "refs/remotes/origin/master", "HEAD"], cwd=tmp_path, check=True) - assert verify._changed_executable_paths(pinned_base, feature_head) == ("polylogue/example.py",) + changed = verify._changed_test_relevant_paths(pinned_base, feature_head) + assert executable_python_paths(tmp_path, changed) == ("polylogue/example.py",) def test_changed_paths_include_untracked_executable_files( @@ -1838,7 +894,8 @@ def test_changed_paths_include_untracked_executable_files( untracked.write_text("value = 1\n", encoding="utf-8") monkeypatch.setattr(verify, "ROOT", tmp_path) - assert verify._changed_executable_paths(head, head) == ("devtools/new_command.py",) + changed = verify._changed_test_relevant_paths(head, head) + assert executable_python_paths(tmp_path, changed) == ("devtools/new_command.py",) def test_changed_paths_include_executable_rename_sources( @@ -1865,7 +922,8 @@ def test_changed_paths_include_executable_rename_sources( ).stdout.strip() monkeypatch.setattr(verify, "ROOT", tmp_path) - assert verify._changed_executable_paths(base, head) == ("polylogue/example.py",) + changed = verify._changed_test_relevant_paths(base, head) + assert executable_python_paths(tmp_path, changed) == ("polylogue/example.py",) def test_changed_paths_parse_non_ascii_names_without_git_quoting( @@ -1886,7 +944,8 @@ def test_changed_paths_parse_non_ascii_names_without_git_quoting( source.write_text("value = 2\n", encoding="utf-8") monkeypatch.setattr(verify, "ROOT", tmp_path) - assert verify._changed_executable_paths(base, base) == ("polylogue/café.py",) + changed = verify._changed_test_relevant_paths(base, base) + assert executable_python_paths(tmp_path, changed) == ("polylogue/café.py",) def test_git_head_uses_bounded_authoritative_probe() -> None: @@ -2461,587 +1520,20 @@ def failing_walk(top: Path, *, onerror: object = None) -> object: assert observation == CheckoutMutationObservation(changed=False, unavailable=True) -def test_checkout_mutation_monitor_fails_closed_when_portable_watcher_fails( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - def broken_watch(*_paths: Path, **_kwargs: object) -> object: - raise OSError("watcher unavailable") - yield set() - - monkeypatch.setattr(watchfiles, "watch", broken_watch) - monitor = CheckoutMutationMonitor(tmp_path) - monitor.start() - observation = monitor.finish() - - assert observation == CheckoutMutationObservation(changed=False, unavailable=True) - - -def test_seed_receipt_classifies_every_node_terminal_outcome( - tmp_path: Path, -) -> None: - artifact_dir = tmp_path / "artifacts" - artifact_dir.mkdir() - expected = [ - "tests/test_seed.py::test_passed", - "tests/test_seed.py::test_failed", - "tests/test_seed.py::test_error", - "tests/test_seed.py::test_timeout", - "tests/test_seed.py::test_worker_crash", - "tests/test_seed.py::test_missing", - ] - (artifact_dir / "selection.json").write_text( - json.dumps( - { - "selected_count": len(expected), - "deselected_count": 0, - "selected_nodeids": expected, - "selected_nodeids_omitted": 0, - } - ) - ) - events = [ - {"event": "test_report", "nodeid": expected[0], "when": "call", "outcome": "passed"}, - { - "event": "test_report", - "nodeid": expected[1], - "when": "call", - "outcome": "failed", - "longrepr": "assert false", - }, - { - "event": "test_report", - "nodeid": expected[2], - "when": "setup", - "outcome": "failed", - "longrepr": "fixture exploded", - }, - { - "event": "test_report", - "nodeid": expected[3], - "when": "call", - "outcome": "failed", - "longrepr": "Failed: Timeout > 10s", - }, - {"event": "test_started", "nodeid": expected[4]}, - ] - (artifact_dir / "events.jsonl").write_text("".join(json.dumps(event) + "\n" for event in events)) - TESTMON_DATA.parent.mkdir(parents=True) - with sqlite3.connect(TESTMON_DATA) as conn: - conn.execute("CREATE TABLE environment (id INTEGER PRIMARY KEY, environment_name TEXT)") - conn.execute("CREATE TABLE file_fp (id INTEGER PRIMARY KEY, filename TEXT, fsha TEXT)") - conn.execute( - "CREATE TABLE test_execution (id INTEGER PRIMARY KEY, test_name TEXT NOT NULL, failed INTEGER NOT NULL)" - ) - conn.execute("CREATE TABLE test_execution_file_fp (test_execution_id INTEGER, fingerprint_id INTEGER)") - conn.executemany( - "INSERT INTO test_execution(test_name, failed) VALUES (?, ?)", - [(nodeid, int(nodeid != expected[0])) for nodeid in expected[:-1]], - ) - conn.executemany( - "INSERT INTO file_fp(id, filename, fsha) VALUES (?, ?, ?)", - [(index, f"file-{index}.py", f"sha-{index}") for index, _nodeid in enumerate(expected[:-1], start=1)], - ) - conn.executemany( - "INSERT INTO test_execution_file_fp VALUES (?, ?)", - [(index, index) for index, _nodeid in enumerate(expected[:-1], start=1)], - ) - - _write_run_receipt(tmp_path, "run-mixed") - receipt = _finalize_testmon_seed_attempt( - prepared={ - "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "running", - "identity": { - "git_head": "head", - "worktree_fingerprint": "tree", - "python": "python", - "skip_slow": False, - "lab": False, - **_testmon_runtime_identity_fields(Path.cwd()), - }, - "resume": False, - "expected_nodeids": [], - "run_id": "run-mixed", - "artifact_dir": ".cache/verify/runs/run-mixed", - }, - step_results=[ - { - "name": "pytest seed-testmon", - "artifact_dir": str(artifact_dir), - "exit": 1, - "diagnosis": "xdist_worker_crash", - } - ], - exit_code=1, - ) - - assert receipt["status"] == "incomplete" - assert {item["nodeid"]: item["outcome"] for item in receipt["node_outcomes"]} == { - expected[0]: "passed", - expected[1]: "failed", - expected[2]: "error", - expected[3]: "timeout", - expected[4]: "worker_crash", - expected[5]: "missing", - } - assert receipt["node_outcome_counts"] == { - "error": 1, - "failed": 1, - "missing": 1, - "passed": 1, - "timeout": 1, - "worker_crash": 1, - } - - -def test_seed_node_outcomes_preserve_interrupted_active_node(tmp_path: Path) -> None: - events = tmp_path / "events.jsonl" - events.write_text(json.dumps({"event": "test_started", "nodeid": "tests/test_a.py::test_active"}) + "\n") - - outcomes = _seed_node_outcomes_from_events( - events, - expected_nodeids=["tests/test_a.py::test_active"], - database={"node_outcomes": {"tests/test_a.py::test_active": "missing"}}, - pytest_step={"diagnosis": "terminated by signal"}, - ) - - assert outcomes[0]["outcome"] == "interrupted" - - -def test_seed_node_outcomes_keep_unconfirmed_teardown_incomplete(tmp_path: Path) -> None: - """A terminal teardown does not prove that the missing call phase passed.""" - events = tmp_path / "events.jsonl" - events.write_text( - "\n".join( - [ - json.dumps({"event": "test_started", "nodeid": "tests/test_a.py::test_finished"}), - json.dumps( - { - "event": "test_report", - "nodeid": "tests/test_a.py::test_finished", - "when": "teardown", - "outcome": "passed", - } - ), - json.dumps({"event": "test_finished", "nodeid": "tests/test_a.py::test_finished"}), - ] - ) - + "\n" - ) - - outcomes = _seed_node_outcomes_from_events( - events, - expected_nodeids=["tests/test_a.py::test_finished"], - database={"node_outcomes": {"tests/test_a.py::test_finished": "missing"}}, - pytest_step={"diagnosis": "pytest_failed"}, - ) - - assert outcomes[0]["outcome"] == "missing" - assert outcomes[0]["reason"] == "passing teardown without call report or testmon result" - - -def test_seed_resource_timeout_has_a_distinct_typed_terminal_outcome( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """A supervisor resource stop must not collapse into generic incompleteness.""" - monkeypatch.chdir(tmp_path) - expected = ["tests/test_seed.py::test_active"] - artifact_dir = tmp_path / "artifacts" - artifact_dir.mkdir() - (artifact_dir / "selection.json").write_text( - json.dumps( - { - "selected_count": len(expected), - "deselected_count": 0, - "selected_nodeids": expected, - "selected_nodeids_omitted": 0, - } - ) - ) - (artifact_dir / "events.jsonl").write_text(json.dumps({"event": "test_started", "nodeid": expected[0]}) + "\n") - TESTMON_DATA.parent.mkdir(parents=True) - with sqlite3.connect(TESTMON_DATA) as connection: - connection.execute("create table environment (id integer primary key, environment_name text)") - connection.execute("create table file_fp (id integer primary key, filename text, fsha text)") - connection.execute("create table test_execution (id integer primary key, test_name text, failed integer)") - connection.execute("create table test_execution_file_fp (test_execution_id integer, fingerprint_id integer)") - _write_run_receipt(tmp_path, "run-resource-timeout") - - receipt = _finalize_testmon_seed_attempt( - prepared={ - "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "running", - "identity": { - "git_head": "head", - "worktree_fingerprint": "tree", - "python": "python", - "skip_slow": False, - "lab": False, - **_testmon_runtime_identity_fields(Path.cwd()), - }, - "resume": False, - "expected_nodeids": [], - "run_id": "run-resource-timeout", - "artifact_dir": ".cache/verify/runs/run-resource-timeout", - }, - step_results=[ - { - "name": "pytest seed-testmon", - "artifact_dir": str(artifact_dir), - "exit": 124, - "diagnosis": "pytest_terminated", - "termination_reason": "pytest tmpfs budget exceeded: 512.0 MiB > 500 MiB", - } - ], - exit_code=124, - ) - - assert receipt["status"] == "incomplete" - assert receipt["outcome"] == "resource-timeout" - assert receipt["release_baseline_allowed"] is False - - -def test_seed_node_outcomes_accept_setup_skip_as_terminal_skip(tmp_path: Path) -> None: - events = tmp_path / "events.jsonl" - events.write_text( - json.dumps( - { - "event": "test_report", - "nodeid": "tests/test_a.py::test_setup_skip", - "when": "setup", - "outcome": "skipped", - } - ) - + "\n" - ) - - outcomes = _seed_node_outcomes_from_events( - events, - expected_nodeids=["tests/test_a.py::test_setup_skip"], - database={"node_outcomes": {"tests/test_a.py::test_setup_skip": "missing"}}, - pytest_step={}, - use_database_fallback=False, - ) - - assert outcomes == [ - { - "nodeid": "tests/test_a.py::test_setup_skip", - "outcome": "skipped", - "reason": "test setup or teardown skipped", - "started": False, - "finished": False, - "phases": [{"when": "setup", "outcome": "skipped", "duration_s": None}], - } - ] - - -def test_seed_node_outcomes_preserve_call_and_fixture_xfail_xpass(tmp_path: Path) -> None: - """Durable pytest reports, including fixture ``pytest.xfail()``, finish seed nodes.""" - events = tmp_path / "events.jsonl" - nodes = [ - "tests/test_a.py::test_call_xfailed", - "tests/test_a.py::test_call_xpassed", - "tests/test_a.py::test_setup_xfailed", - ] - events.write_text( - "\n".join( - json.dumps(event) - for event in ( - {"event": "test_report", "nodeid": nodes[0], "when": "call", "outcome": "xfailed"}, - {"event": "test_report", "nodeid": nodes[1], "when": "call", "outcome": "xpassed"}, - {"event": "test_report", "nodeid": nodes[2], "when": "setup", "outcome": "xfailed"}, - ) - ) - + "\n" - ) - - outcomes = _seed_node_outcomes_from_events( - events, - expected_nodeids=nodes, - database={"node_outcomes": {}}, - pytest_step={}, - use_database_fallback=False, - ) - - assert {item["nodeid"]: item["outcome"] for item in outcomes} == dict( - zip(nodes, ("xfailed", "xpassed", "xfailed"), strict=True) - ) - - -def test_resumed_seed_carries_forward_prior_terminal_outcome(tmp_path: Path) -> None: - events = tmp_path / "events.jsonl" - events.write_text( - json.dumps( - {"event": "test_report", "nodeid": "tests/test_a.py::test_repaired", "when": "call", "outcome": "passed"} - ) - + "\n" - ) - outcomes = _seed_node_outcomes_from_events( - events, - expected_nodeids=[ - "tests/test_a.py::test_repaired", - "tests/test_b.py::test_prior", - "tests/test_c.py::test_expected_failure", - ], - database={"node_outcomes": {"tests/test_b.py::test_prior": "passed"}}, - pytest_step={}, - use_database_fallback=False, - prior_node_outcomes={ - "tests/test_b.py::test_prior": {"nodeid": "tests/test_b.py::test_prior", "outcome": "passed"}, - "tests/test_c.py::test_expected_failure": { - "nodeid": "tests/test_c.py::test_expected_failure", - "outcome": "xfailed", - }, - }, - ) - - assert {item["nodeid"]: item["outcome"] for item in outcomes} == { - "tests/test_a.py::test_repaired": "passed", - "tests/test_b.py::test_prior": "passed", - "tests/test_c.py::test_expected_failure": "xfailed", - } - - -def test_seed_completion_requires_full_failure_free_database(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.chdir(tmp_path) - artifact_dir = tmp_path / "artifacts" - artifact_dir.mkdir() - expected = ["tests/test_a.py::test_one", "tests/test_b.py::test_two"] - (artifact_dir / "selection.json").write_text( - json.dumps( - { - "selected_count": 2, - "deselected_count": 0, - "selected_nodeids": expected, - "selected_nodeids_omitted": 0, - } - ) - ) - (artifact_dir / "events.jsonl").write_text( - "".join( - json.dumps({"event": "test_report", "nodeid": nodeid, "when": "call", "outcome": "passed"}) + "\n" - for nodeid in expected - ) - ) - TESTMON_DATA.parent.mkdir(parents=True) - with sqlite3.connect(TESTMON_DATA) as conn: - conn.execute("CREATE TABLE environment (id INTEGER PRIMARY KEY, environment_name TEXT)") - conn.execute("CREATE TABLE file_fp (id INTEGER PRIMARY KEY, filename TEXT, fsha TEXT)") - conn.execute( - "CREATE TABLE test_execution (id INTEGER PRIMARY KEY, test_name TEXT NOT NULL, failed INTEGER NOT NULL)" - ) - conn.execute("CREATE TABLE test_execution_file_fp (test_execution_id INTEGER, fingerprint_id INTEGER)") - conn.executemany( - "INSERT INTO test_execution(test_name, failed) VALUES (?, 0)", - [(nodeid,) for nodeid in expected], - ) - conn.executemany( - "INSERT INTO file_fp(id, filename, fsha) VALUES (?, ?, ?)", - [(index, f"file-{index}.py", f"sha-{index}") for index, _nodeid in enumerate(expected, start=1)], - ) - conn.executemany( - "INSERT INTO test_execution_file_fp VALUES (?, ?)", - [(index, index) for index, _nodeid in enumerate(expected, start=1)], - ) - - _write_run_receipt(tmp_path, "run-1") - _write_run_receipt(tmp_path, "run-stale-db") - _write_run_receipt(tmp_path, "run-orphaned") - receipt = _finalize_testmon_seed_attempt( - prepared={ - "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "running", - "identity": { - "git_head": "head", - "worktree_fingerprint": "tree", - "python": "python", - "skip_slow": False, - "lab": False, - **_testmon_runtime_identity_fields(Path.cwd()), - }, - "resume": False, - "expected_nodeids": [], - "run_id": "run-1", - "artifact_dir": ".cache/verify/runs/run-1", - }, - step_results=[{"name": "pytest seed-testmon", "artifact_dir": str(artifact_dir), "exit": 0}], - exit_code=0, - ) - - assert receipt["status"] == "complete" - assert receipt["expected_count"] == 2 - stamp = json.loads((tmp_path / ".cache" / "testmon" / "seed.json").read_text()) - assert stamp["status"] == "usable" - assert stamp["collection"]["expected_count"] == 2 - - _write_run_receipt(tmp_path, "run-authorized") - authorized_receipt = _finalize_testmon_seed_attempt( - prepared={ - "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "running", - "identity": { - "git_head": "head", - "git_tree": "tree-hash", - "worktree_fingerprint": "tree", - "python": "python", - "skip_slow": True, - "lab": False, - "terminal_authorization": "narrow-terminal", - **_testmon_runtime_identity_fields(Path.cwd()), - }, - "resume": False, - "expected_nodeids": [], - "run_id": "run-authorized", - "artifact_dir": ".cache/verify/runs/run-authorized", - }, - step_results=[{"name": "pytest seed-testmon", "artifact_dir": str(artifact_dir), "exit": 0}], - exit_code=0, - ) - assert authorized_receipt["status"] == "complete" - assert authorized_receipt["release_baseline_allowed"] is True - - _write_run_receipt(tmp_path, "run-red") - with sqlite3.connect(TESTMON_DATA) as connection: - connection.execute("update test_execution set failed = 1 where test_name = ?", (expected[0],)) - red_receipt = _finalize_testmon_seed_attempt( - prepared={ - "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "running", - "identity": { - "git_head": "head", - "worktree_fingerprint": "tree", - "python": "python", - "skip_slow": False, - "lab": False, - **_testmon_runtime_identity_fields(Path.cwd()), - }, - "resume": False, - "expected_nodeids": [], - "run_id": "run-red", - "artifact_dir": ".cache/verify/runs/run-red", - }, - step_results=[{"name": "pytest seed-testmon", "artifact_dir": str(artifact_dir), "exit": 1}], - exit_code=1, - ) - assert red_receipt["status"] == "reusable" - assert red_receipt["release_baseline_allowed"] is False - persisted_attempt = json.loads((tmp_path / ".cache" / "testmon" / "seed-attempt.json").read_text()) - assert persisted_attempt["release_baseline_allowed"] is False - assert not (tmp_path / ".cache" / "testmon" / "seed.json").exists() - - (artifact_dir / "events.jsonl").write_text("") - stale_database = _finalize_testmon_seed_attempt( - prepared={ - "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "running", - "identity": { - "git_head": "head", - "worktree_fingerprint": "tree", - "python": "python", - "skip_slow": True, - "lab": False, - }, - "resume": False, - "expected_nodeids": [], - "run_id": "run-stale-db", - "artifact_dir": ".cache/verify/runs/run-stale-db", - }, - step_results=[{"name": "pytest seed-testmon", "artifact_dir": str(artifact_dir)}], - exit_code=0, - ) - assert stale_database["status"] == "incomplete" - - with sqlite3.connect(TESTMON_DATA) as connection: - connection.execute("insert into test_execution_file_fp values (999, 1)") - orphaned = _finalize_testmon_seed_attempt( - prepared={ - "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "running", - "identity": { - "git_head": "head", - "worktree_fingerprint": "tree", - "python": "python", - "skip_slow": True, - "lab": False, - }, - "resume": False, - "expected_nodeids": [], - "run_id": "run-orphaned", - "artifact_dir": ".cache/verify/runs/run-orphaned", - }, - step_results=[{"name": "pytest seed-testmon", "artifact_dir": str(artifact_dir)}], - exit_code=0, - ) - assert orphaned["status"] == "incomplete" - - -def test_resumed_seed_persists_full_selection_before_stamp_publication( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch +def test_checkout_mutation_monitor_fails_closed_when_portable_watcher_fails( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, ) -> None: - monkeypatch.chdir(tmp_path) - expected = ["tests/test_a.py::test_one", "tests/test_b.py::test_two"] - artifact_dir = tmp_path / "artifacts" - artifact_dir.mkdir() - (artifact_dir / "selection.json").write_text( - json.dumps({"selected_count": 1, "selected_nodeids": [expected[0]], "selected_nodeids_omitted": 0}) - ) - (artifact_dir / "events.jsonl").write_text( - json.dumps({"event": "test_report", "nodeid": expected[0], "when": "call", "outcome": "passed"}) + "\n" - ) - TESTMON_DATA.parent.mkdir(parents=True) - with sqlite3.connect(TESTMON_DATA) as connection: - connection.execute("create table environment (id integer primary key, environment_name text)") - connection.execute("create table file_fp (id integer primary key, filename text, fsha text)") - connection.execute("create table test_execution (id integer primary key, test_name text, failed integer)") - connection.execute("create table test_execution_file_fp (test_execution_id integer, fingerprint_id integer)") - connection.executemany("insert into test_execution values (?, ?, 0)", [(1, expected[0]), (2, expected[1])]) - connection.executemany("insert into file_fp values (?, ?, ?)", [(1, "a.py", "a"), (2, "b.py", "b")]) - connection.executemany("insert into test_execution_file_fp values (?, ?)", [(1, 1), (2, 2)]) - _write_run_receipt(tmp_path, "resumed") - prepared = { - "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "running", - "identity": { - "git_head": "head", - "git_tree": "tree-hash", - "worktree_fingerprint": "tree", - "python": "python", - "skip_slow": False, - "lab": False, - "terminal_authorization": None, - **_testmon_runtime_identity_fields(Path.cwd()), - }, - "resume": True, - "expected_nodeids": expected, - "expected_count": len(expected), - "expected_digest": hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest(), - "prior_node_outcomes": [{"nodeid": expected[1], "outcome": "passed"}], - "run_id": "resumed", - "artifact_dir": ".cache/verify/runs/resumed", - } - original_write = verify._atomic_write_json - - def crash_before_stamp(path: Path, payload: object) -> None: - if path == TESTMON_SEED_STAMP: - raise RuntimeError("simulated crash before seed publication") - assert isinstance(payload, dict) - original_write(path, payload) - - with patch("devtools.verify._atomic_write_json", side_effect=crash_before_stamp): - with pytest.raises(RuntimeError, match="before seed publication"): - _finalize_testmon_seed_attempt( - prepared=prepared, - step_results=[{"name": "pytest seed-testmon (resume)", "artifact_dir": str(artifact_dir)}], - exit_code=0, - ) + def broken_watch(*_paths: Path, **_kwargs: object) -> object: + raise OSError("watcher unavailable") + yield set() + + monkeypatch.setattr(watchfiles, "watch", broken_watch) + monitor = CheckoutMutationMonitor(tmp_path) + monitor.start() + observation = monitor.finish() - persisted = json.loads(TESTMON_SEED_ATTEMPT.read_text()) - assert persisted["status"] == "complete" - assert persisted["expected_count"] == len(expected) - assert persisted["selection"]["selected_count"] == len(expected) - assert persisted["selection"]["selected_nodeids_omitted"] == 0 - assert not TESTMON_SEED_STAMP.exists() + assert observation == CheckoutMutationObservation(changed=False, unavailable=True) def test_classify_late_sigterm_after_pytest_success_summary() -> None: @@ -4094,6 +2586,7 @@ def test_pytest_workload_receipt_uses_allocated_basetemp_peak() -> None: tmpfs_budget_mb=1, basetemp_cleanup=None, concurrency=1, + timeout_s=3600, ) execute = next(phase for phase in receipt["phases"] if phase["name"] == "execute") @@ -4137,15 +2630,6 @@ def test_cleanup_managed_pytest_basetemp_keeps_seed_cache(tmp_path: Path) -> Non assert seeded.exists() -def test_testmon_preflight_allows_seed_and_full_without_database( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.chdir(tmp_path) - - assert _testmon_preflight(seed_testmon=True, full_pytest=False, quick=False, commit=False) is None - assert _testmon_preflight(seed_testmon=False, full_pytest=True, quick=False, commit=False) is None - - def test_parse_pytest_test_count_from_summary() -> None: output = "bringing up nodes...\n\n6 passed, 2 skipped, 1 xfailed in 8.49s\n" @@ -4459,7 +2943,7 @@ def to_dict(self) -> dict[str, int]: patch("devtools.verify._run_pytest_with_heartbeat", return_value=completed), patch("devtools.verify._read_pytest_report", return_value=None), ): - rc, _elapsed, metadata = _run("pytest seed-testmon", ["pytest", "--testmon", "-n", "4"]) + rc, _elapsed, metadata = _run("pytest native parallel (bootstrap)", ["pytest", "--testmon", "-n", "4"]) assert rc == 0 assert apply_policy.call_args.kwargs["worker_count"] == 4 @@ -4472,12 +2956,10 @@ def to_dict(self) -> dict[str, int]: ("label", "full_suite"), [ ("pytest focused", False), - ("pytest testmon", False), - ("pytest testmon (broad)", True), - ("pytest seed-testmon", True), - ("pytest seed-testmon shard 1/4", True), - ("pytest full (parallel)", True), - ("pytest load-sensitive (isolated)", True), + ("pytest native parallel (affected)", False), + ("pytest native serial (affected)", False), + ("pytest native parallel (bootstrap)", True), + ("pytest native serial (full)", True), ], ) def test_run_scopes_measured_full_suite_basetemp_demand(tmp_path: Path, label: str, full_suite: bool) -> None: @@ -5180,309 +3662,6 @@ def finish(self) -> CheckoutMutationObservation: assert durable_payload["final_worktree_fingerprint"] == "stable" -def test_transient_checkout_mutation_discards_testmon_graph_before_publication( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - class _ChangedMonitor: - def __init__(self, _root: Path) -> None: - pass - - def start(self) -> None: - pass - - def finish(self) -> CheckoutMutationObservation: - return CheckoutMutationObservation(changed=True, unavailable=False, observed_path="polylogue/example.py") - - monkeypatch.chdir(tmp_path) - monkeypatch.setattr(verify, "ROOT", tmp_path) - monkeypatch.setattr(verify, "CheckoutMutationMonitor", _ChangedMonitor) - monkeypatch.setattr( - verify, - "assert_polylogue_matches_checkout", - lambda *_args, **_kwargs: SimpleNamespace(polylogue_import_path=tmp_path / "polylogue", as_dict=lambda: {}), - ) - TESTMON_DATA.parent.mkdir(parents=True) - TESTMON_DATA.write_bytes(b"transient dependency graph") - TESTMON_SEED_STAMP.write_text("{}", encoding="utf-8") - affected_publish = MagicMock() - selection_publish = MagicMock() - - with ( - patch("devtools.verify._anchor_verification_paths"), - patch("devtools.verify.maybe_bootstrap_testmon_seed", return_value=None), - patch("devtools.verify._testmon_preflight", return_value=None), - patch("devtools.verify.build_verify_steps", return_value=[("pytest testmon", ["pytest", "--testmon"])]), - patch("devtools.verify._run", return_value=(0, 0.01, {"selected_count": 1})), - patch("devtools.verify._changed_executable_paths", return_value=("polylogue/example.py",)), - patch("devtools.verify._record_testmon_affected_coverage", affected_publish), - patch("devtools.verify._refresh_testmon_selection_attempt", selection_publish), - patch("devtools.verify._git_head", return_value="head"), - patch("devtools.verify._git_commit", return_value="base"), - patch("devtools.verify._default_testmon_is_broad_change", return_value=False), - patch("devtools.verify._testmon_release_baseline_permission", return_value=False), - patch("devtools.verify._warn_low_memory"), - patch("devtools.verify._save_history"), - patch("devtools.verify._stamp_head"), - patch("devtools.verify._notify"), - patch("devtools.verify.worktree_fingerprint", return_value="stable"), - ): - assert main(["--json"]) == 125 - - assert not TESTMON_DATA.exists() - assert not TESTMON_SEED_STAMP.exists() - affected_publish.assert_not_called() - selection_publish.assert_not_called() - assert json.loads(capsys.readouterr().out)["diagnosis"] == "checkout_changed_during_verification" - - -def test_verify_stops_after_failed_heavy_step(capsys: pytest.CaptureFixture[str]) -> None: - calls: list[str] = [] - - def fake_run(label: str, command: list[str], **kwargs: object) -> tuple[int, float, dict[str, object]]: - calls.append(label) - return (1 if label.startswith("pytest") else 0), 0.01, {} - - with ( - patch("devtools.verify._run", side_effect=fake_run), - patch("devtools.verify.build_verify_steps", return_value=[("pytest testmon", ["pytest"])]), - patch("devtools.verify._git_head", return_value="head"), - patch("devtools.verify._git_commit", return_value="base"), - patch("devtools.verify._default_testmon_is_broad_change", return_value=False), - patch("devtools.verify._save_history"), - patch("devtools.verify._stamp_head"), - patch("devtools.verify._notify"), - patch("devtools.verify._testmon_preflight", return_value=None), - ): - rc = main(["--json"]) - - assert rc == 1 - assert calls[-1].startswith("pytest") - payload = capsys.readouterr().out - assert '"exit_code": 1' in payload - - -@pytest.mark.parametrize( - ("shard_results", "expected_exit", "expected_diagnosis", "expected_statuses"), - [ - ( - [(124, "pytest_timeout"), (0, "pytest_passed")], - 124, - "pytest_timeout", - ["incomplete", "pending"], - ), - ( - [(1, "pytest_failed"), (0, "pytest_passed")], - 1, - "pytest_failed", - ["complete", "complete"], - ), - ( - [(1, "pytest_failed"), (124, "pytest_timeout")], - 124, - "pytest_timeout", - ["complete", "incomplete"], - ), - ( - [(1, "pytest_failed"), (0, "pytest_passed")], - 1, - "pytest_failed", - ["incomplete", "pending"], - ), - ], -) -def test_seed_testmon_stops_only_after_infrastructure_failed_shard( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], - shard_results: list[tuple[int, str]], - expected_exit: int, - expected_diagnosis: str, - expected_statuses: list[str], -) -> None: - nodeids = ["tests/test_seed.py::test_one", "tests/test_seed.py::test_two"] - collection_dir = tmp_path / "collection" - collection_dir.mkdir() - (collection_dir / "selection.json").write_text( - json.dumps( - { - "selected_count": len(nodeids), - "selected_nodeids": nodeids, - "selected_nodeids_omitted": 0, - } - ) - ) - calls: list[str] = [] - checkpointed: list[int] = [] - finalized_shard_statuses: list[str] = [] - - def fake_run(label: str, command: list[str], **kwargs: object) -> tuple[int, float, dict[str, object]]: - del command, kwargs - calls.append(label) - if label == "pytest seed-testmon collect": - return 0, 0.01, {"artifact_dir": str(collection_dir)} - if label.startswith("pytest seed-testmon shard "): - shard_index = int(label.rsplit(" ", 1)[1].split("/", 1)[0]) - shard_exit, diagnosis = shard_results[shard_index - 1] - return shard_exit, 0.01, {"diagnosis": diagnosis} - pytest.fail(f"unexpected seed step: {label}") - - def fake_checkpoint(*, prepared: dict[str, object], shard_index: int, step: dict[str, object]) -> dict[str, object]: - del step - checkpointed.append(shard_index) - raw_shards = prepared["shards"] - assert isinstance(raw_shards, list) - assert all(isinstance(shard, dict) for shard in raw_shards) - shards = [dict(shard) for shard in raw_shards] - shards[shard_index - 1]["status"] = expected_statuses[shard_index - 1] - return {**prepared, "shards": shards} - - def fake_finalize( - *, prepared: dict[str, object], step_results: list[dict[str, object]], exit_code: int - ) -> dict[str, object]: - del step_results - assert exit_code == expected_exit - raw_shards = prepared["shards"] - assert isinstance(raw_shards, list) - assert all(isinstance(shard, dict) for shard in raw_shards) - finalized_shard_statuses.extend(str(shard["status"]) for shard in raw_shards) - return { - "status": "incomplete" if "incomplete" in expected_statuses else "complete", - "outcome": "resource_timeout" if expected_exit == 124 else "red-baseline", - "resume": False, - "expected_count": len(nodeids), - "release_baseline_allowed": False, - } - - monkeypatch.setattr(verify, "TESTMON_SEED_SHARD_SIZE", 1) - with ( - patch("devtools.verify._anchor_verification_paths"), - patch("devtools.verify.maybe_bootstrap_testmon_seed", return_value=None), - patch("devtools.verify._run", side_effect=fake_run), - patch( - "devtools.verify.build_verify_steps", - return_value=[("pytest seed-testmon collect", ["pytest", "--collect-only"])], - ), - patch("devtools.verify._git_head", return_value="head"), - patch("devtools.verify._git_committed_tree", return_value="tree"), - patch( - "devtools.verify._testmon_seed_identity", - return_value={"git_head": "head", "git_tree": "tree", "skip_slow": False, "lab": False}, - ), - patch("devtools.verify._testmon_seed_can_resume", return_value=False), - patch("devtools.verify._checkpoint_testmon_seed_shard", side_effect=fake_checkpoint), - patch("devtools.verify._finalize_testmon_seed_attempt", side_effect=fake_finalize), - patch("devtools.verify._testmon_release_baseline_permission", return_value=False), - patch("devtools.verify._warn_low_memory"), - patch("devtools.verify._save_history"), - patch("devtools.verify._stamp_head"), - patch("devtools.verify._notify"), - ): - rc = main(["--seed-testmon", "--json"]) - - assert rc == expected_exit - executed_shards = sum(status != "pending" for status in expected_statuses) - assert calls == [ - "pytest seed-testmon collect", - *(f"pytest seed-testmon shard {index}/2" for index in range(1, executed_shards + 1)), - ] - assert checkpointed == list(range(1, executed_shards + 1)) - assert finalized_shard_statuses == expected_statuses - output = json.loads(capsys.readouterr().out) - assert output["exit_code"] == expected_exit - assert output["diagnosis"] == expected_diagnosis - - -@pytest.mark.parametrize( - ("argv", "expected_scope", "expected_permission"), - [ - (["--all", "--skip-slow"], "narrow-terminal", False), - (["--all", "--skip-slow", "--terminal-authorization", "narrow-terminal"], "narrow-terminal", True), - ], -) -def test_verify_main_types_skip_slow_terminal_authority( - capsys: pytest.CaptureFixture[str], argv: list[str], expected_scope: str, expected_permission: bool -) -> None: - def fake_run(label: str, command: list[str], **kwargs: object) -> tuple[int, float, dict[str, object]]: - del label, command, kwargs - return 0, 0.01, {} - - with ( - patch("devtools.verify._run", side_effect=fake_run), - patch("devtools.verify.build_verify_steps", return_value=[("pytest full", ["pytest"])]), - patch("devtools.verify._git_head", return_value="head"), - patch("devtools.verify._save_history") as save_history, - patch("devtools.verify._stamp_head"), - patch("devtools.verify._notify"), - ): - assert main([*argv, "--json"]) == 0 - - payload = json.loads(capsys.readouterr().out) - assert payload["verification_scope"] == expected_scope - assert payload["release_baseline_allowed"] is expected_permission - assert payload["terminal_authorization"] == ("narrow-terminal" if expected_permission else None) - assert save_history.call_args.args[0]["checkout_root"] == str(ROOT.resolve()) - - -def test_verify_refuses_unbudgeted_pytest_before_running_steps(capsys: pytest.CaptureFixture[str]) -> None: - with ( - patch("devtools.verify.build_verify_steps", side_effect=PytestResourceError("only 0.50 GiB available")), - patch("devtools.verify._git_head", return_value="head"), - patch("devtools.verify._git_commit", return_value="base"), - patch("devtools.verify._default_testmon_is_broad_change", return_value=False), - patch("devtools.verify._testmon_preflight", return_value=None), - patch("devtools.verify._run") as run, - patch("devtools.verify._save_history") as save_history, - ): - rc = main(["--json"]) - - assert rc == 125 - run.assert_not_called() - assert save_history.call_args.args[0]["diagnosis"] == "pytest_resource_preflight_failed" - assert "only 0.50 GiB available" in capsys.readouterr().err - - -def test_verify_starts_checkout_monitor_before_broad_change_classification( - capsys: pytest.CaptureFixture[str], -) -> None: - events: list[str] = [] - - class _OrderingMonitor: - def __init__(self, _root: Path) -> None: - pass - - def start(self) -> None: - events.append("monitor-started") - - def finish(self) -> CheckoutMutationObservation: - events.append("monitor-finished") - return CheckoutMutationObservation(changed=False, unavailable=False) - - def classify(_base: str, _head: str) -> bool: - assert events == ["monitor-started"] - events.append("classified") - return False - - with ( - patch("devtools.verify.CheckoutMutationMonitor", _OrderingMonitor), - patch("devtools.verify._git_head", return_value="head"), - patch("devtools.verify._git_commit", return_value="base"), - patch("devtools.verify.worktree_fingerprint", return_value="stable"), - patch("devtools.verify._default_testmon_is_broad_change", side_effect=classify), - patch("devtools.verify.build_verify_steps", return_value=[]), - patch("devtools.verify._testmon_preflight", return_value=None), - patch("devtools.verify._testmon_release_baseline_permission", return_value=False), - patch("devtools.verify._save_history"), - patch("devtools.verify._stamp_head"), - patch("devtools.verify._notify"), - ): - assert main(["--json"]) == 0 - - assert events == ["monitor-started", "classified", "monitor-finished"] - assert json.loads(capsys.readouterr().out)["exit_code"] == 0 - - def test_verify_finalizes_checkout_monitor_when_startup_fingerprint_raises() -> None: events: list[str] = [] @@ -5508,45 +3687,6 @@ def finish(self) -> CheckoutMutationObservation: assert events == ["monitor-started", "monitor-finished"] -def test_verify_finalizes_runner_exception_after_open_step( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - history: dict[str, Any] = {} - monkeypatch.setattr(verify, "ROOT", tmp_path) - monkeypatch.chdir(tmp_path) - monkeypatch.setattr( - verify, - "assert_polylogue_matches_checkout", - lambda *_args, **_kwargs: SimpleNamespace(polylogue_import_path=tmp_path / "polylogue", as_dict=lambda: {}), - ) - monkeypatch.setattr(verify, "maybe_bootstrap_testmon_seed", lambda *_args, **_kwargs: None) - monkeypatch.setattr(verify, "_git_head", lambda: "head") - monkeypatch.setattr(verify, "worktree_fingerprint", lambda _root: "stable") - monitor = MagicMock() - monitor.finish.return_value = CheckoutMutationObservation(changed=False, unavailable=False) - monkeypatch.setattr(verify, "CheckoutMutationMonitor", lambda _root: monitor) - monkeypatch.setattr(verify, "_save_history", lambda payload: history.update(payload)) - monkeypatch.setattr(verify, "build_verify_steps", lambda **_kwargs: [("ruff check", ["ruff", "check"])]) - - def explode(_label: str, command: list[str], **kwargs: Any) -> tuple[int, float, dict[str, Any]]: - run = kwargs["run"] - run.start_step(label="ruff check", cmd=command) - raise RuntimeError("verification runner exploded") - - monkeypatch.setattr(verify, "_run", explode) - monotonic_values = iter((100.0, 107.5)) - monkeypatch.setattr("devtools.verify.time.monotonic", lambda: next(monotonic_values)) - - assert verify.main(["--quick", "--json"]) == 125 - assert history["exit_code"] == 125 - assert history["diagnosis"] == "verify_runner_exception" - assert history["duration_s"] == 7.5 - assert history["verification_scope"] == "non-test" - assert history["steps"][0]["status"] == "failed" - assert history["steps"][0]["exit"] == 125 - - def test_verify_anchors_relative_state_to_checkout_when_invoked_from_subdirectory( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -5557,136 +3697,114 @@ def test_verify_anchors_relative_state_to_checkout_when_invoked_from_subdirector assert Path.cwd() == ROOT.resolve() -def test_verify_rejects_zero_testmon_selection_for_executable_change( - capsys: pytest.CaptureFixture[str], -) -> None: - changed_executable_paths = MagicMock(return_value=("polylogue/example.py",)) - - def fake_run(label: str, command: list[str], **kwargs: object) -> tuple[int, float, dict[str, object]]: - del command, kwargs - return 0, 0.01, ({"selected_count": 0} if label.startswith("pytest") else {}) - - with ( - patch("devtools.verify._run", side_effect=fake_run), - patch("devtools.verify._git_head", return_value="head"), - patch("devtools.verify._git_commit", return_value="pinned-base"), - patch("devtools.verify._default_testmon_is_broad_change", return_value=False), - patch("devtools.verify._save_history"), - patch("devtools.verify._stamp_head"), - patch("devtools.verify._notify"), - patch("devtools.verify._testmon_preflight", return_value=None), - patch("devtools.verify._changed_executable_paths", changed_executable_paths), - patch("devtools.verify._matching_testmon_coverage", return_value=None), - ): - rc = main(["--json"]) - - assert rc == 5 - payload = json.loads(capsys.readouterr().out) - pytest_step = next(step for step in payload["steps"] if step["name"].startswith("pytest")) - assert pytest_step["diagnosis"] == "zero_testmon_selection_for_executable_change" - assert pytest_step["zero_selection_changed_paths"] == ["polylogue/example.py"] - changed_executable_paths.assert_called_once_with("pinned-base", "head") +def test_failed_step_stop_policy_distinguishes_cheap_and_heavy_steps() -> None: + assert _stop_after_failed_step("ruff check") is False + assert _stop_after_failed_step("verify layering") is False + assert _stop_after_failed_step("pytest native serial (affected)") is False + assert _stop_after_failed_step("lab smoke") is True + assert _stop_after_failed_step("bench slo") is True + assert _native_lane_failure_requires_stop({"exit": 1, "diagnosis": "pytest_failed"}) is False + assert _native_lane_failure_requires_stop({"exit": 2, "diagnosis": "pytest_collection_failed"}) is True -def test_verify_finalizes_and_discards_graph_when_post_pytest_path_authority_fails( +def test_verify_continues_serial_lane_after_parallel_test_failure( capsys: pytest.CaptureFixture[str], ) -> None: - monitor = MagicMock() - monitor.finish.return_value = CheckoutMutationObservation(changed=False, unavailable=False) - discard = MagicMock() - - with ( - patch("devtools.verify._run", return_value=(0, 0.01, {"selected_count": 1})), - patch("devtools.verify.build_verify_steps", return_value=[("pytest testmon", ["pytest"])]), - patch("devtools.verify._git_head", return_value="head"), - patch("devtools.verify._git_commit", return_value="base"), - patch("devtools.verify._default_testmon_is_broad_change", return_value=False), - patch("devtools.verify._changed_executable_paths", side_effect=PytestResourceError("git unavailable")), - patch("devtools.verify._discard_testmon_dependency_authority", discard), - patch("devtools.verify.CheckoutMutationMonitor", return_value=monitor), - patch("devtools.verify.worktree_fingerprint", return_value="stable"), - patch("devtools.verify._save_history"), - patch("devtools.verify._stamp_head"), - patch("devtools.verify._notify"), - patch("devtools.verify._testmon_preflight", return_value=None), - ): - rc = main(["--json"]) - - assert rc == 125 - monitor.finish.assert_called_once_with() - discard.assert_called_once_with() - payload = json.loads(capsys.readouterr().out) - assert payload["diagnosis"] == "testmon_changed_path_authority_unavailable" - + calls: list[str] = [] + lane_timeouts: list[float] = [] -def test_testmon_changed_path_authority_refuses_missing_commit_binding() -> None: - changed_paths = MagicMock() + class _StableMonitor: + def __init__(self, _root: Path) -> None: + pass - with patch("devtools.verify._changed_executable_paths", changed_paths): - with pytest.raises(PytestResourceError, match="changed-path authority is unavailable"): - verify._changed_paths_from_testmon_authority(None, "head") + def start(self) -> None: + pass - changed_paths.assert_not_called() + def finish(self) -> CheckoutMutationObservation: + return CheckoutMutationObservation(changed=False, unavailable=False) + preparation = SimpleNamespace( + environment_name="env", + selection_mode="affected", + removed_paths=(), + copied_from=None, + ) + native_state = SimpleNamespace( + valid=True, + environment=SimpleNamespace(nodeids=("tests/test_parallel.py::test_owner",)), + missing_executable_paths=(), + reason="current", + ) -def test_verify_accepts_zero_testmon_selection_after_matching_coverage( - capsys: pytest.CaptureFixture[str], -) -> None: - def fake_run(label: str, command: list[str], **kwargs: object) -> tuple[int, float, dict[str, object]]: - del command, kwargs - return 0, 0.01, ({"selected_count": 0} if label.startswith("pytest") else {}) + def fake_run(label: str, _command: list[str], **_kwargs: object) -> tuple[int, float, dict[str, object]]: + calls.append(label) + lane_timeouts.append(cast(float, _kwargs["timeout_s"])) + if "parallel" in label: + return 1, 0.01, {"diagnosis": "pytest_failed"} + return 0, 0.01, {"diagnosis": "pytest_passed"} with ( - patch("devtools.verify._run", side_effect=fake_run), - patch("devtools.verify.build_verify_steps", return_value=[("pytest testmon", ["pytest"])]), patch("devtools.verify._git_head", return_value="head"), patch("devtools.verify._git_commit", return_value="base"), - patch("devtools.verify._default_testmon_is_broad_change", return_value=False), + patch("devtools.verify._changed_test_relevant_paths", return_value=()), + patch("devtools.verify.prepare_native_testmon_environment", return_value=preparation), + patch("devtools.verify._native_environment_after_run", return_value=native_state), + patch( + "devtools.verify.build_verify_steps", + return_value=[ + ("pytest native parallel (affected)", ["pytest"]), + ("pytest native serial (affected)", ["pytest"]), + ], + ), + patch("devtools.verify._run", side_effect=fake_run), + patch("devtools.verify._remaining_invocation_budget", side_effect=(3500.0, 3200.0)), + patch("devtools.verify.aggregate_native_testmon_run", return_value={"terminal_green": False}), + patch("devtools.verify.CheckoutMutationMonitor", _StableMonitor), + patch("devtools.verify.worktree_fingerprint", return_value="stable"), patch("devtools.verify._save_history"), - patch("devtools.verify._stamp_head"), patch("devtools.verify._notify"), - patch("devtools.verify._testmon_preflight", return_value=None), - patch("devtools.verify._changed_executable_paths", return_value=("polylogue/example.py",)), - patch("devtools.verify._matching_testmon_coverage", return_value="successful_affected_run"), ): - rc = main(["--json"]) + assert main(["--json"]) == 1 - assert rc == 0 - payload = json.loads(capsys.readouterr().out) - pytest_step = next(step for step in payload["steps"] if step["name"].startswith("pytest")) - assert pytest_step["zero_selection_coverage"] == "successful_affected_run" + assert calls == ["pytest native parallel (affected)", "pytest native serial (affected)"] + assert lane_timeouts == [3500.0, 3200.0] + assert json.loads(capsys.readouterr().out)["release_baseline_allowed"] is False -def test_testmon_coverage_receipts_are_content_exact() -> None: - paths = ("polylogue/example.py",) - _write_real_testmon_state() - assert _matching_testmon_coverage(paths) is None +def test_release_authority_requires_current_complete_green_invocation() -> None: + aggregate = { + "complete_corpus_covered": True, + "terminal_green": True, + "cleanup": {"complete": True}, + "containment": {"complete": True}, + "deadline": {"met": True}, + } - TESTMON_SEED_STAMP.unlink() - with patch("devtools.verify.worktree_fingerprint", return_value="affected"): - _record_testmon_affected_coverage( - executable_paths=paths, - selected_count=3, - run_id="run-1", + assert _release_baseline_allowed( + selection_mode="bootstrap", + exit_code=0, + checkout_stable=True, + aggregate=aggregate, + ) + assert not _release_baseline_allowed( + selection_mode="affected", + exit_code=0, + checkout_stable=True, + aggregate=aggregate, + ) + for broken in ( + {**aggregate, "complete_corpus_covered": False}, + {**aggregate, "terminal_green": False}, + {**aggregate, "cleanup": {"complete": False}}, + {**aggregate, "containment": {"complete": False}}, + {**aggregate, "deadline": {"met": False}}, + ): + assert not _release_baseline_allowed( + selection_mode="full", + exit_code=0, + checkout_stable=True, + aggregate=broken, ) - assert TESTMON_AFFECTED_STAMP.exists() - assert _matching_testmon_coverage(paths) == "successful_affected_run" - assert _matching_testmon_coverage(("polylogue/other.py",)) is None - - with patch("devtools.verify.worktree_fingerprint", return_value="changed"): - assert _matching_testmon_coverage(paths) is None - - TESTMON_AFFECTED_STAMP.write_text(json.dumps({"identity": {"worktree_fingerprint": "affected"}})) - with patch("devtools.verify.worktree_fingerprint", return_value="affected"): - assert _matching_testmon_coverage(paths) is None - - -def test_failed_step_stop_policy_distinguishes_cheap_and_heavy_steps() -> None: - assert _stop_after_failed_step("ruff check") is False - assert _stop_after_failed_step("verify layering") is False - assert _stop_after_failed_step("pytest testmon") is True - assert _stop_after_failed_step("lab smoke") is True - assert _stop_after_failed_step("bench slo") is True def test_completion_notification_uses_pytest_count() -> None: @@ -5712,34 +3830,6 @@ def test_completion_notification_omits_unknown_pytest_count() -> None: assert summary == "PASS (118s)" -def test_default_testmon_step_pairs_marker_filter_with_forceselect() -> None: - """#1632: any pytest -m marker filter in the default lane MUST be paired with --testmon-forceselect. - - Without ``--testmon-forceselect``, a marker selector deactivates - pytest-testmon's affected-test selection and the run silently - expands to the whole suite — PR #1550 fixed exactly this regression - after a full week of every default verify running 9.5K tests - instead of the affected subset. This invariant is the regression - guard so the footgun cannot re-land silently again. - """ - steps = build_verify_steps(quick=False, lab=False, skip_slow=False) - label, command = steps[-1] - assert label == "pytest testmon" - if "-m" in command: - assert "--testmon-forceselect" in command, ( - f"marker filter without --testmon-forceselect re-introduces the #1550 silent-deselection footgun: {command}" - ) - - -def test_skip_slow_testmon_step_keeps_forceselect_with_compound_marker() -> None: - """``--skip-slow`` composes the marker; the pairing invariant must still hold.""" - steps = build_verify_steps(quick=False, lab=False, skip_slow=True) - label, command = steps[-1] - assert label == "pytest testmon" - assert "-m" in command - assert "--testmon-forceselect" in command - - def test_verify_does_not_notify_on_pass() -> None: """Passing verify runs stay silent — only failures send a desktop popup.""" From 32ed7b960d801c09b7571b5e02eadda0ca31d42d Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 19:22:46 +0200 Subject: [PATCH 03/31] fix(test): harden native lifecycle authority --- devtools/verify.py | 57 +++- devtools/verify_runs.py | 187 ++++++++++++- .../devtools/test_native_testmon_lifecycle.py | 157 +++++++++++ tests/unit/devtools/test_verify.py | 252 +++++++++++++++++- 4 files changed, 639 insertions(+), 14 deletions(-) diff --git a/devtools/verify.py b/devtools/verify.py index fd01ea494b..d7b6e64da6 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -2421,12 +2421,20 @@ def _native_environment_after_run( def _release_baseline_allowed( *, selection_mode: str | None, + verification_scope: VerificationScope, + terminal_authorization: str | None, exit_code: int, checkout_stable: bool, aggregate: Mapping[str, Any] | None, ) -> bool: if selection_mode not in {"bootstrap", "full"} or exit_code != 0 or not checkout_stable or aggregate is None: return False + scope_authorized = verification_scope == VerificationScope.RELEASE_BASELINE or ( + verification_scope == VerificationScope.NARROW_TERMINAL + and terminal_authorization == TerminalAuthorization.NARROW_TERMINAL.value + ) + if not scope_authorized: + return False cleanup = aggregate.get("cleanup") containment = aggregate.get("containment") deadline = aggregate.get("deadline") @@ -2444,6 +2452,7 @@ def _release_baseline_allowed( def _main(argv: list[str] | None = None) -> int: global _ACTIVE_VERIFY_RUN + started_at = time.monotonic() parser = argparse.ArgumentParser(description="Run the local verification baseline.") parser.add_argument("--quick", action="store_true", help="Skip pytest and run only fast local gates.") parser.add_argument( @@ -2507,7 +2516,6 @@ def _main(argv: list[str] | None = None) -> int: sys.stderr.write("verify: cannot resolve immutable Git refs for native affected-test authority.\n") return 125 - started_at = time.monotonic() relevant_paths: tuple[str, ...] = () required_executable_paths: tuple[str, ...] = () preparation: NativeTestmonPreparation | None = None @@ -2691,21 +2699,64 @@ def _main(argv: list[str] | None = None) -> int: ) total_duration = round(time.monotonic() - started_at, 2) + deadline_recorded = any(step.get("diagnosis") == "verify_invocation_deadline_exceeded" for step in step_results) + if total_duration > VERIFY_INVOCATION_BUDGET_S and not deadline_recorded: + step_results.append( + { + "name": "verify invocation deadline", + "duration_s": 0.0, + "exit": 124, + "diagnosis": "verify_invocation_deadline_exceeded", + "timeout_s": VERIFY_INVOCATION_BUDGET_S, + } + ) + exit_code = 124 + deadline_recorded = True pytest_aggregate: dict[str, Any] | None = None native_environment = native_state.environment if native_state is not None else None - if preparation is not None and native_environment is not None: + if preparation is not None: pytest_aggregate = aggregate_native_testmon_run( ROOT, steps=step_results, environment_name=preparation.environment_name, - corpus_nodeids=native_environment.nodeids, + corpus_nodeids=native_environment.nodeids if native_environment is not None else (), + environment_status=native_state.status if native_state is not None else "unavailable", + environment_reason=native_state.reason if native_state is not None else "post-run inspection unavailable", selection_mode=testmon_mode or "affected", invocation_duration_s=total_duration, budget_s=VERIFY_INVOCATION_BUDGET_S, ) + # Aggregation and final graph inspection are part of the same invocation + # deadline as collection and execution. Recompute once after aggregation + # so a run cannot gain release authority by crossing the budget during + # finalization rather than during a pytest lane. + finalized_duration = round(time.monotonic() - started_at, 2) + if finalized_duration > total_duration: + total_duration = finalized_duration + if total_duration > VERIFY_INVOCATION_BUDGET_S and not deadline_recorded: + step_results.append( + { + "name": "verify invocation deadline", + "duration_s": 0.0, + "exit": 124, + "diagnosis": "verify_invocation_deadline_exceeded", + "timeout_s": VERIFY_INVOCATION_BUDGET_S, + } + ) + exit_code = 124 + deadline_recorded = True + if pytest_aggregate is not None: + pytest_aggregate["wall_s"] = total_duration + pytest_aggregate["deadline"] = { + "budget_s": VERIFY_INVOCATION_BUDGET_S, + "met": total_duration <= VERIFY_INVOCATION_BUDGET_S, + } + release_baseline_allowed = _release_baseline_allowed( selection_mode=testmon_mode, + verification_scope=planned_scope, + terminal_authorization=args.terminal_authorization, exit_code=exit_code, checkout_stable=checkout_stable, aggregate=pytest_aggregate, diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index 85fa89bdf4..6adc093efb 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -100,6 +100,144 @@ def _trailing_history_record(descriptor: int, *, end: int) -> tuple[int, bytes]: return 0, b"".join(reversed(suffix)) +def _history_pytest_aggregate(entry: Mapping[str, Any]) -> dict[str, Any]: + existing = entry.get("pytest_aggregate") + if isinstance(existing, Mapping): + return dict(existing) + + def optional_int(value: object) -> int | None: + return value if isinstance(value, int) and not isinstance(value, bool) else None + + def max_optional(current: int | None, value: object) -> int | None: + candidate = optional_int(value) + if candidate is None: + return current + return candidate if current is None else max(current, candidate) + + pytest_steps = [ + step + for step in entry.get("steps", ()) + if isinstance(step, Mapping) and str(step.get("name", "")).startswith("pytest") + ] + outcomes: dict[str, int] = {} + lanes: list[dict[str, Any]] = [] + selected_count = 0 + terminal_count = 0 + collection_wall_s = 0.0 + cleanup_complete = True + containment_complete = True + peak_rss: int | None = None + peak_pss: int | None = None + peak_swap: int | None = None + peak_storage: int | None = None + read_bytes = 0 + write_bytes = 0 + for step in pytest_steps: + statistics = step.get("statistics") + stats = statistics if isinstance(statistics, Mapping) else {} + raw_outcomes = stats.get("outcomes") + if isinstance(raw_outcomes, Mapping): + for outcome, count in raw_outcomes.items(): + if isinstance(count, int): + outcomes[str(outcome)] = outcomes.get(str(outcome), 0) + count + raw_node_count = stats.get("node_count") + if not isinstance(raw_node_count, int): + raw_node_count = step.get("count") + node_count = raw_node_count if isinstance(raw_node_count, int) else 0 + raw_selected = step.get("selected_count") + selected = raw_selected if isinstance(raw_selected, int) else node_count + selected_count += selected + terminal_count += node_count + collection_duration = step.get("collection_duration_s") + if isinstance(collection_duration, int | float): + collection_wall_s += float(collection_duration) + cleanup = stats.get("cleanup") + step_cleanup = cleanup.get("complete") if isinstance(cleanup, Mapping) else None + cleanup_complete = cleanup_complete and step_cleanup is True + containment_complete = containment_complete and isinstance(step.get("containment_mode"), str) + resources = stats.get("resources") + resource_values = resources if isinstance(resources, Mapping) else {} + peak_rss = max_optional(peak_rss, resource_values.get("peak_tree_rss_kb")) + peak_pss = max_optional(peak_pss, resource_values.get("peak_tree_pss_kb")) + peak_swap = max_optional(peak_swap, resource_values.get("peak_tree_swap_pss_kb")) + storage = stats.get("storage") + storage_values = storage if isinstance(storage, Mapping) else {} + peak_storage = max_optional(peak_storage, storage_values.get("basetemp_allocated_bytes_max")) + read_bytes += optional_int(resource_values.get("tree_read_bytes_delta")) or 0 + write_bytes += optional_int(resource_values.get("tree_write_bytes_delta")) or 0 + lanes.append( + { + "lane": "focused" if entry.get("tier") == "focused-test" else "pytest", + "exit_code": step.get("exit"), + "duration_s": step.get("duration_s"), + "selected_count": selected, + "terminal_count": node_count, + "collection_duration_s": collection_duration, + "cleanup_complete": step_cleanup, + "containment_complete": isinstance(step.get("containment_mode"), str), + "containment_mode": step.get("containment_mode"), + } + ) + + no_pytest = not pytest_steps + corpus_digest = hashlib.sha256(b"").hexdigest() + exit_code = entry.get("exit_code") + return { + "schema_version": 1, + "environment": { + "name": None, + "digest": None, + "status": "not-applicable", + "reason": "run did not use the native pytest-testmon lifecycle", + "native_corpus_count": 0, + "native_corpus_digest": corpus_digest, + }, + "corpus": {"count": terminal_count, "digest": corpus_digest}, + "selection_mode": "focused" if entry.get("tier") == "focused-test" else "none", + "lanes": lanes, + "selected_union_count": selected_count, + "terminal_union_count": terminal_count, + "duplicate_outcome_count": 0, + "outcomes": outcomes, + "missing_terminal_count": 0, + "missing_terminal_sample": [], + "non_green_count": sum(count for outcome, count in outcomes.items() if outcome not in {"passed", "skipped"}), + "non_green_sample": [], + "complete_corpus_covered": False, + "terminal_green": bool(pytest_steps) and exit_code == 0, + "collection_wall_s": round(collection_wall_s, 4), + "resources": { + "peak_tree_rss_kb": peak_rss, + "peak_tree_pss_kb": peak_pss, + "peak_tree_swap_pss_kb": peak_swap, + "peak_storage_bytes": peak_storage, + "read_bytes": read_bytes, + "write_bytes": write_bytes, + }, + "cleanup": {"complete": True if no_pytest else cleanup_complete}, + "containment": {"complete": True if no_pytest else containment_complete}, + "deadline": {"budget_s": None, "met": True}, + "wall_s": entry.get("total_duration_s", entry.get("duration_s", 0.0)), + } + + +def normalize_verify_history_entry(entry: Mapping[str, Any]) -> dict[str, Any]: + """Return the single timestamped history schema shared by every run kind.""" + normalized = dict(entry) + normalized["history_schema_version"] = 1 + normalized["timestamp"] = next( + ( + value + for key in ("timestamp", "finished_at", "started_at") + if isinstance((value := normalized.get(key)), str) and value + ), + datetime.now(UTC).isoformat(), + ) + normalized.setdefault("total_duration_s", normalized.get("duration_s", 0.0)) + normalized["pytest_aggregate"] = _history_pytest_aggregate(normalized) + return normalized + + def append_verify_history(entry: Mapping[str, Any], *, path: Path = VERIFY_HISTORY_PATH) -> None: """Append one complete invocation to the cross-worktree run history. @@ -109,7 +247,7 @@ def append_verify_history(entry: Mapping[str, Any], *, path: Path = VERIFY_HISTO durable index used to find and compare them. """ path.parent.mkdir(parents=True, exist_ok=True) - payload = (json.dumps(dict(entry), ensure_ascii=False) + "\n").encode() + payload = (json.dumps(normalize_verify_history_entry(entry), ensure_ascii=False) + "\n").encode() descriptor = os.open(path, os.O_RDWR | os.O_CREAT | os.O_APPEND, 0o600) try: fcntl.flock(descriptor, fcntl.LOCK_EX) @@ -287,6 +425,7 @@ def __init__(self, root: Path) -> None: self._git_current_ref_path: Path | None = None self._git_current_ref_was_loose: bool | None = None self._git_authority_paths: dict[Path, str] = {} + self._git_authority_signatures: dict[Path, str | None] = {} self._directory_topology_fingerprint: frozenset[str] | None = None def start(self) -> None: @@ -425,6 +564,7 @@ def walk_error(_error: OSError) -> None: if self._git_index_path is not None: self._git_authority_paths[self._git_index_path] = ".git/index" self._git_authority_paths.update(self._resolve_git_head_paths()) + self._git_authority_signatures = {path: self._authority_signature(path) for path in self._git_authority_paths} for authority_path in self._git_authority_paths: watched_parent = authority_path.parent while not watched_parent.exists() and watched_parent != watched_parent.parent: @@ -434,6 +574,14 @@ def walk_error(_error: OSError) -> None: self._directory_topology_fingerprint = self._directory_topology(directories) return directories + @staticmethod + def _authority_signature(path: Path) -> str | None: + """Identify authority semantics while ignoring byte-identical rewrites.""" + try: + return hashlib.sha256(path.read_bytes()).hexdigest() + except OSError: + return None + def _directory_topology(self, directories: Sequence[Path]) -> frozenset[str]: """Fingerprint source directory membership without trusting pre-watch state.""" return frozenset( @@ -561,6 +709,20 @@ def _is_within_ignored_root(relative: Path, ignored_roots: frozenset[Path]) -> b def _record_change(self, candidate: Path) -> None: if not candidate.is_absolute(): candidate = self.root / candidate + nested_authorities = [ + (authority_path, label) + for authority_path, label in self._git_authority_paths.items() + if candidate != authority_path and authority_path.is_relative_to(candidate) + and not (label == ".git/packed-refs" and self._git_current_ref_was_loose is True) + ] + if nested_authorities: + for authority_path, label in nested_authorities: + if self._authority_signature(authority_path) != self._git_authority_signatures.get(authority_path): + with self._state_lock: + self._changed = True + self._observed_path = label + return + return for authority_path, label in self._git_authority_paths.items(): if label == ".git/packed-refs" and self._git_current_ref_was_loose is True: # packed-refs is shared by linked worktrees. When this @@ -570,11 +732,6 @@ def _record_change(self, candidate: Path) -> None: # is removed or replaced. Preserve the startup state so a # packed-to-loose transition cannot hide its own first event. continue - if candidate != authority_path and authority_path.is_relative_to(candidate): - with self._state_lock: - self._changed = True - self._observed_path = label - return if candidate.parent != authority_path.parent: continue if candidate.name == f"{authority_path.name}.lock": @@ -583,6 +740,8 @@ def _record_change(self, candidate: Path) -> None: # its authority file. return if candidate.name == authority_path.name: + if self._authority_signature(authority_path) == self._git_authority_signatures.get(authority_path): + return with self._state_lock: self._changed = True self._observed_path = label @@ -1074,6 +1233,8 @@ def aggregate_native_testmon_run( steps: Sequence[Mapping[str, Any]], environment_name: str, corpus_nodeids: Sequence[str], + environment_status: str = "valid", + environment_reason: str | None = None, selection_mode: str, invocation_duration_s: float, budget_s: float, @@ -1185,11 +1346,15 @@ def _peak(current: int | None, value: object) -> int | None: sorted(nodeid for nodeid in corpus if outcome_by_node.get(nodeid) not in _GREEN_TERMINAL_OUTCOMES) ) terminal_green = complete_corpus_covered and not missing_terminal and not non_green + cleanup_complete = bool(lanes) and cleanup_complete + containment_complete = bool(lanes) and containment_complete return { "schema_version": 1, "environment": { "name": environment_name, "digest": environment_name.removeprefix("polylogue-"), + "status": environment_status, + "reason": environment_reason, "native_corpus_count": len(native_corpus), "native_corpus_digest": hashlib.sha256("\n".join(native_corpus).encode()).hexdigest(), }, @@ -1230,7 +1395,14 @@ def _peak(current: int | None, value: object) -> int | None: def git_dirty(cwd: Path | None = None) -> bool: try: - result = subprocess.run(["git", "status", "--short"], capture_output=True, text=True, timeout=5, cwd=cwd) + result = subprocess.run( + ["git", "status", "--short"], + capture_output=True, + text=True, + timeout=5, + cwd=cwd, + env=_read_only_git_env(), + ) except (OSError, subprocess.TimeoutExpired): return True return bool(result.stdout.strip()) @@ -1250,6 +1422,7 @@ def git_head(cwd: Path | None = None) -> str | None: text=True, timeout=5, cwd=cwd, + env=_read_only_git_env(), ) except (OSError, subprocess.TimeoutExpired): return None diff --git a/tests/integration/devtools/test_native_testmon_lifecycle.py b/tests/integration/devtools/test_native_testmon_lifecycle.py index 8801917bd7..2a89f2d45e 100644 --- a/tests/integration/devtools/test_native_testmon_lifecycle.py +++ b/tests/integration/devtools/test_native_testmon_lifecycle.py @@ -10,6 +10,7 @@ import uuid from dataclasses import dataclass from pathlib import Path +from typing import Any import pytest @@ -53,6 +54,10 @@ def _git(repo: Path, *args: str) -> str: def _init_repo(root: Path, *, conftest: str = "") -> None: (root / "tests").mkdir(parents=True) + (root / ".gitignore").write_text( + ".artifacts/\n.benchmarks/\n.cache/\n.coverage*\n.pytest_cache/\n__pycache__/\n", + encoding="utf-8", + ) (root / "pyproject.toml").write_text( """ [tool.pytest.ini_options] @@ -158,6 +163,89 @@ def _run_plain_verify_corpus( return parallel, serial +def _run_production_verify(repo: Path, *args: str) -> tuple[subprocess.CompletedProcess[str], dict[str, Any]]: + """Run the production verifier orchestration against a tiny fixture corpus. + + The subprocess keeps the real native preparation, two-lane runner, + containment, deadline, aggregate, invocation receipt, and XDG history. + Only unrelated static gates are filtered so this fixture need not copy the + entire Polylogue source tree. + """ + state_root = repo.parent / f"{repo.name}-verify-state" + receipt = state_root / "receipts" / f"{uuid.uuid4().hex}.json" + invocation_id = uuid.uuid4().hex + driver = """ +import os +import sys +from pathlib import Path +from types import SimpleNamespace + +import devtools.verify as verify + +root = Path(sys.argv[1]).resolve() +real_build = verify.build_verify_steps + +def native_steps_only(**kwargs): + return [step for step in real_build(**kwargs) if step[0].startswith("pytest native")] + +verify.ROOT = root +verify.build_verify_steps = native_steps_only +verify.assert_polylogue_matches_checkout = lambda *_args, **_kwargs: SimpleNamespace( + polylogue_import_path=root / "polylogue" / "__init__.py", + as_dict=lambda: {"checkout_root": str(root), "test_fixture": True}, +) +os.chdir(root) +raise SystemExit(verify.main(sys.argv[2:])) +""" + env = os.environ.copy() + env.update( + { + "PYTHONPATH": str(PROJECT_ROOT), + "XDG_STATE_HOME": str(state_root / "xdg-state"), + "POLYLOGUE_PYTEST_WORKERS": "1", + "POLYLOGUE_VERIFICATION_INVOCATION_ID": invocation_id, + "POLYLOGUE_VERIFICATION_RECEIPT_PATH": str(receipt), + "GIT_OPTIONAL_LOCKS": "0", + } + ) + try: + completed = subprocess.run( + [sys.executable, "-c", driver, str(repo), *args, "--json"], + cwd=PROJECT_ROOT, + env=env, + capture_output=True, + text=True, + timeout=30, + ) + except subprocess.TimeoutExpired as exc: + stdout_value: object = exc.stdout + stderr_value: object = exc.stderr + stdout = ( + stdout_value.decode(errors="replace") + if isinstance(stdout_value, bytes) + else stdout_value + if isinstance(stdout_value, str) + else "" + ) + stderr = ( + stderr_value.decode(errors="replace") + if isinstance(stderr_value, bytes) + else stderr_value + if isinstance(stderr_value, str) + else "" + ) + pytest.fail(f"production verify fixture timed out\nstdout:\n{stdout}\nstderr:\n{stderr}") + if not receipt.exists(): + pytest.fail( + f"production verify wrote no invocation receipt\nstdout:\n{completed.stdout}\nstderr:\n{completed.stderr}" + ) + payload = json.loads(completed.stdout) + persisted = json.loads(receipt.read_text(encoding="utf-8")) + assert persisted["invocation_id"] == invocation_id + assert persisted["pytest_aggregate"] == payload["pytest_aggregate"] + return completed, payload + + def _selected(*results: LaneResult) -> set[str]: selected: set[str] = set() for result in results: @@ -214,6 +302,75 @@ def test_serial_owner(): assert _selected(*second) == set() +def test_production_plain_verify_owns_bootstrap_warm_selection_deadline_and_history(tmp_path: Path) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _init_repo(repo) + package = repo / "polylogue" + package.mkdir() + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "app.py").write_text("def answer() -> int:\n return 42\n", encoding="utf-8") + (repo / "tests" / "test_app.py").write_text( + """ +import pytest + +def test_parallel_owner(): + from polylogue.app import answer + assert answer() == 42 + +@pytest.mark.load_sensitive +def test_serial_owner(): + from polylogue.app import answer + assert answer() == 42 +""".lstrip(), + encoding="utf-8", + ) + _commit_all(repo, "fixture") + origin = tmp_path / "origin.git" + _git(origin.parent, "init", "--bare", "-q", str(origin)) + _git(repo, "remote", "add", "origin", str(origin)) + _git(repo, "branch", "-M", "master") + _git(repo, "push", "-qu", "origin", "master") + + first, bootstrap = _run_production_verify(repo) + + assert first.returncode == 0, f"{first.stderr}\n{json.dumps(bootstrap, indent=2, sort_keys=True)}" + assert bootstrap["testmon_environment"]["selection_mode"] == "bootstrap" + aggregate = bootstrap["pytest_aggregate"] + assert aggregate["complete_corpus_covered"] is True + assert aggregate["terminal_green"] is True + assert aggregate["cleanup"] == {"complete": True} + assert aggregate["containment"] == {"complete": True} + assert aggregate["deadline"] == {"budget_s": 3600.0, "met": True} + lane_steps = [step for step in bootstrap["steps"] if step.get("semantic_lane")] + assert [step["semantic_lane"] for step in lane_steps] == ["parallel", "serial"] + environments = { + arg for step in lane_steps for arg in step["cmd"] if isinstance(arg, str) and arg.startswith("--testmon-env=") + } + assert environments == {f"--testmon-env={bootstrap['testmon_environment']['name']}"} + lane_timeouts = [step["timeout_s"] for step in lane_steps] + assert 0 < lane_timeouts[1] < lane_timeouts[0] <= 3600 + history_path = repo.parent / "repo-verify-state" / "xdg-state" / "polylogue" / "devtools" / "verify-history.jsonl" + history = [json.loads(line) for line in history_path.read_text(encoding="utf-8").splitlines()] + assert history[-1]["pytest_aggregate"] == aggregate + + second, warm = _run_production_verify(repo) + + assert second.returncode == 0, second.stderr + assert warm["testmon_environment"]["selection_mode"] == "affected" + assert warm["release_baseline_allowed"] is False + assert warm["pytest_aggregate"]["selected_union_count"] == 0 + + (package / "app.py").write_text("def answer() -> int:\n return 0\n", encoding="utf-8") + third, mutated = _run_production_verify(repo) + + assert third.returncode == 1 + assert mutated["testmon_environment"]["selection_mode"] == "affected" + assert mutated["pytest_aggregate"]["terminal_union_count"] == 2 + assert mutated["release_baseline_allowed"] is False + assert "assert 0 == 42" in third.stderr + + def test_empty_linked_worktree_with_empty_main_self_bootstraps(tmp_path: Path) -> None: main = tmp_path / "main" main.mkdir() diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 4361e558c2..9fd629e17a 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -20,6 +20,7 @@ from devtools import run_tests, verify, verify_runs from devtools.testmon_bootstrap import executable_python_paths +from devtools.verification_contracts import TerminalAuthorization, VerificationScope from devtools.verify import ( PYTEST_CONTAINMENT_PATH, PYTEST_EVENTS_PATH, @@ -724,6 +725,49 @@ def test_verify_history_appends_concurrent_records_without_interleaving(tmp_path rows = [json.loads(line) for line in history.read_text(encoding="utf-8").splitlines()] assert sorted(row["sequence"] for row in rows) == list(range(64)) + assert all(row["history_schema_version"] == 1 for row in rows) + assert all(isinstance(row["timestamp"], str) for row in rows) + assert all(row["pytest_aggregate"]["selection_mode"] == "none" for row in rows) + + +def test_verify_history_normalizes_focused_and_quick_runs_to_one_aggregate_schema(tmp_path: Path) -> None: + history = tmp_path / "state" / "verify-history.jsonl" + finished_at = "2026-08-13T17:30:00+00:00" + focused = { + "tier": "focused-test", + "finished_at": finished_at, + "duration_s": 1.25, + "exit_code": 0, + "steps": [ + { + "name": "pytest focused", + "exit": 0, + "duration_s": 1.0, + "selected_count": 2, + "collection_duration_s": 0.1, + "containment_mode": "systemd-scope", + "statistics": { + "node_count": 2, + "outcomes": {"passed": 2}, + "cleanup": {"complete": True}, + "resources": {"peak_tree_rss_kb": 512, "tree_read_bytes_delta": 64}, + }, + } + ], + } + quick = {"tier": "quick", "timestamp": "2026-08-13T17:31:00+00:00", "exit_code": 0, "steps": []} + + append_verify_history(focused, path=history) + append_verify_history(quick, path=history) + + focused_row, quick_row = [json.loads(line) for line in history.read_text(encoding="utf-8").splitlines()] + assert focused_row["timestamp"] == finished_at + assert focused_row["total_duration_s"] == 1.25 + assert focused_row["pytest_aggregate"]["selection_mode"] == "focused" + assert focused_row["pytest_aggregate"]["outcomes"] == {"passed": 2} + assert focused_row["pytest_aggregate"]["resources"]["peak_tree_rss_kb"] == 512 + assert quick_row["pytest_aggregate"]["selection_mode"] == "none" + assert quick_row["pytest_aggregate"]["lanes"] == [] def test_verify_history_repairs_or_frames_an_incomplete_trailing_record(tmp_path: Path) -> None: @@ -737,7 +781,9 @@ def test_verify_history_repairs_or_frames_an_incomplete_trailing_record(tmp_path append_verify_history({"sequence": 2}, path=history) rows = [json.loads(line) for line in history.read_text(encoding="utf-8").splitlines()] - assert rows == [{"sequence": 0}, {"sequence": 1}, {"sequence": 2}] + assert rows[0] == {"sequence": 0} + assert [row["sequence"] for row in rows[1:]] == [1, 2] + assert all(row["history_schema_version"] == 1 for row in rows[1:]) def test_verify_history_append_reads_only_the_trailing_record( @@ -761,7 +807,9 @@ def measured_read(descriptor: int, count: int) -> bytes: append_verify_history({"sequence": 1}, path=history) assert bytes_read < 128 * 1024 - assert json.loads(history.read_text(encoding="utf-8").splitlines()[-1]) == {"sequence": 1} + row = json.loads(history.read_text(encoding="utf-8").splitlines()[-1]) + assert row["sequence"] == 1 + assert row["history_schema_version"] == 1 def test_compare_against_last_skips_intervening_focused_history(monkeypatch: pytest.MonkeyPatch) -> None: @@ -1454,7 +1502,7 @@ def test_checkout_mutation_monitor_ignores_uncommitted_git_index_lock(tmp_path: assert observation == CheckoutMutationObservation(changed=False, unavailable=False) -def test_checkout_mutation_monitor_treats_every_ready_index_event_as_authority_change( +def test_checkout_mutation_monitor_ignores_semantically_unchanged_index_replacement( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -1465,6 +1513,9 @@ def test_checkout_mutation_monitor_treats_every_ready_index_event_as_authority_c def portable_watch(*_paths: Path, **_kwargs: object) -> object: yield set() + replacement = index.with_suffix(".replacement") + replacement.write_bytes(index.read_bytes()) + replacement.replace(index) yield {(watchfiles.Change.modified, str(index))} stop_event = _kwargs["stop_event"] assert isinstance(stop_event, threading.Event) @@ -1475,7 +1526,7 @@ def portable_watch(*_paths: Path, **_kwargs: object) -> object: monitor.start() observation = monitor.finish() - assert observation == CheckoutMutationObservation(changed=True, unavailable=False, observed_path=".git/index") + assert observation == CheckoutMutationObservation(changed=False, unavailable=False) def test_checkout_mutation_monitor_rejects_partial_git_enumeration( @@ -3782,12 +3833,32 @@ def test_release_authority_requires_current_complete_green_invocation() -> None: assert _release_baseline_allowed( selection_mode="bootstrap", + verification_scope=VerificationScope.RELEASE_BASELINE, + terminal_authorization=None, exit_code=0, checkout_stable=True, aggregate=aggregate, ) assert not _release_baseline_allowed( selection_mode="affected", + verification_scope=VerificationScope.AFFECTED, + terminal_authorization=None, + exit_code=0, + checkout_stable=True, + aggregate=aggregate, + ) + assert not _release_baseline_allowed( + selection_mode="full", + verification_scope=VerificationScope.NARROW_TERMINAL, + terminal_authorization=None, + exit_code=0, + checkout_stable=True, + aggregate=aggregate, + ) + assert _release_baseline_allowed( + selection_mode="full", + verification_scope=VerificationScope.NARROW_TERMINAL, + terminal_authorization=TerminalAuthorization.NARROW_TERMINAL.value, exit_code=0, checkout_stable=True, aggregate=aggregate, @@ -3801,12 +3872,185 @@ def test_release_authority_requires_current_complete_green_invocation() -> None: ): assert not _release_baseline_allowed( selection_mode="full", + verification_scope=VerificationScope.RELEASE_BASELINE, + terminal_authorization=None, exit_code=0, checkout_stable=True, aggregate=broken, ) +@pytest.mark.parametrize( + ("authorization", "expected_release"), + [(None, False), ("narrow-terminal", True)], +) +def test_skip_slow_command_requires_typed_narrow_terminal_authorization( + authorization: str | None, + expected_release: bool, + capsys: pytest.CaptureFixture[str], +) -> None: + class _StableMonitor: + def __init__(self, _root: Path) -> None: + pass + + def start(self) -> None: + pass + + def finish(self) -> CheckoutMutationObservation: + return CheckoutMutationObservation(changed=False, unavailable=False) + + preparation = SimpleNamespace( + environment_name="env", + selection_mode="bootstrap", + removed_paths=(), + copied_from=None, + ) + native_state = SimpleNamespace( + valid=True, + status="valid", + reason="current", + environment=SimpleNamespace(nodeids=("tests/test_owner.py::test_owner",)), + missing_executable_paths=(), + ) + aggregate = { + "complete_corpus_covered": True, + "terminal_green": True, + "cleanup": {"complete": True}, + "containment": {"complete": True}, + "deadline": {"met": True}, + } + argv = ["--all", "--skip-slow", "--json"] + if authorization is not None: + argv.extend(("--terminal-authorization", authorization)) + + with ( + patch("devtools.verify._git_head", return_value="head"), + patch("devtools.verify._git_commit", return_value="base"), + patch("devtools.verify._changed_test_relevant_paths", return_value=()), + patch("devtools.verify.prepare_native_testmon_environment", return_value=preparation), + patch("devtools.verify._native_environment_after_run", return_value=native_state), + patch("devtools.verify.build_verify_steps", return_value=[]), + patch("devtools.verify.aggregate_native_testmon_run", return_value=aggregate), + patch("devtools.verify.CheckoutMutationMonitor", _StableMonitor), + patch("devtools.verify.worktree_fingerprint", return_value="stable"), + patch("devtools.verify._save_history"), + patch("devtools.verify._stamp_head"), + patch("devtools.verify._notify"), + ): + assert main(argv) == 0 + + assert json.loads(capsys.readouterr().out)["release_baseline_allowed"] is expected_release + + +def test_collection_failure_still_persists_native_run_aggregate( + capsys: pytest.CaptureFixture[str], +) -> None: + class _StableMonitor: + def __init__(self, _root: Path) -> None: + pass + + def start(self) -> None: + pass + + def finish(self) -> CheckoutMutationObservation: + return CheckoutMutationObservation(changed=False, unavailable=False) + + preparation = SimpleNamespace( + environment_name="env", + selection_mode="bootstrap", + removed_paths=(), + copied_from=None, + ) + invalid_state = SimpleNamespace( + valid=False, + status="invalid", + reason="native environment has no unique collected corpus", + environment=None, + missing_executable_paths=(), + ) + with ( + patch("devtools.verify._git_head", return_value="head"), + patch("devtools.verify._git_commit", return_value="base"), + patch("devtools.verify._changed_test_relevant_paths", return_value=()), + patch("devtools.verify.prepare_native_testmon_environment", return_value=preparation), + patch("devtools.verify._native_environment_after_run", return_value=invalid_state), + patch( + "devtools.verify.build_verify_steps", + return_value=[("pytest native parallel (bootstrap)", ["pytest"])], + ), + patch( + "devtools.verify._run", + return_value=(2, 0.01, {"diagnosis": "pytest_collection_failed"}), + ), + patch("devtools.verify.CheckoutMutationMonitor", _StableMonitor), + patch("devtools.verify.worktree_fingerprint", return_value="stable"), + patch("devtools.verify._save_history"), + patch("devtools.verify._notify"), + ): + assert main(["--json"]) == 2 + + payload = json.loads(capsys.readouterr().out) + aggregate = payload["pytest_aggregate"] + assert aggregate["environment"]["status"] == "invalid" + assert aggregate["environment"]["native_corpus_count"] == 0 + assert aggregate["lanes"][0]["lane"] == "parallel" + assert aggregate["cleanup"]["complete"] is False + assert aggregate["containment"]["complete"] is False + + +def test_deadline_starts_before_native_preparation_and_fails_closed_after_steps( + capsys: pytest.CaptureFixture[str], +) -> None: + class _StableMonitor: + def __init__(self, _root: Path) -> None: + pass + + def start(self) -> None: + pass + + def finish(self) -> CheckoutMutationObservation: + return CheckoutMutationObservation(changed=False, unavailable=False) + + preparation = SimpleNamespace( + environment_name="env", + selection_mode="bootstrap", + removed_paths=(), + copied_from=None, + ) + native_state = SimpleNamespace( + valid=True, + status="valid", + reason="current", + environment=SimpleNamespace(nodeids=("tests/test_owner.py::test_owner",)), + missing_executable_paths=(), + ) + clock = iter((100.0, 3701.0, 3701.0, 3701.0)) + + def prepare(*_args: object, **_kwargs: object) -> object: + assert time.monotonic() == 3701.0 + return preparation + + with ( + patch("devtools.verify.time.monotonic", side_effect=lambda: next(clock)), + patch("devtools.verify._git_head", return_value="head"), + patch("devtools.verify._git_commit", return_value="base"), + patch("devtools.verify._changed_test_relevant_paths", return_value=()), + patch("devtools.verify.prepare_native_testmon_environment", side_effect=prepare), + patch("devtools.verify._native_environment_after_run", return_value=native_state), + patch("devtools.verify.build_verify_steps", return_value=[]), + patch("devtools.verify.CheckoutMutationMonitor", _StableMonitor), + patch("devtools.verify.worktree_fingerprint", return_value="stable"), + patch("devtools.verify._save_history"), + patch("devtools.verify._notify"), + ): + assert main(["--json"]) == 124 + + payload = json.loads(capsys.readouterr().out) + assert payload["total_duration_s"] == 3601.0 + assert payload["pytest_aggregate"]["deadline"] == {"budget_s": 3600.0, "met": False} + assert any(step["diagnosis"] == "verify_invocation_deadline_exceeded" for step in payload["steps"]) + + def test_completion_notification_uses_pytest_count() -> None: summary = _format_completion_notification( exit_code=0, From 2a7ac04472bd1dc4aedb21a8748ee8a3fffd5561 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 19:38:39 +0200 Subject: [PATCH 04/31] test: stabilize native lifecycle evidence --- .../devtools/test_native_testmon_lifecycle.py | 6 +++++- tests/unit/devtools/test_verify.py | 11 ++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/integration/devtools/test_native_testmon_lifecycle.py b/tests/integration/devtools/test_native_testmon_lifecycle.py index 2a89f2d45e..b4643f8287 100644 --- a/tests/integration/devtools/test_native_testmon_lifecycle.py +++ b/tests/integration/devtools/test_native_testmon_lifecycle.py @@ -62,6 +62,7 @@ def _init_repo(root: Path, *, conftest: str = "") -> None: """ [tool.pytest.ini_options] addopts = "-p no:randomly" +cache_dir = ".cache/pytest" markers = [ "load_sensitive: serial native-testmon lane", "tui: serial native-testmon lane", @@ -345,7 +346,10 @@ def test_serial_owner(): lane_steps = [step for step in bootstrap["steps"] if step.get("semantic_lane")] assert [step["semantic_lane"] for step in lane_steps] == ["parallel", "serial"] environments = { - arg for step in lane_steps for arg in step["cmd"] if isinstance(arg, str) and arg.startswith("--testmon-env=") + arg + for step in lane_steps + for arg in step["statistics"]["command"] + if isinstance(arg, str) and arg.startswith("--testmon-env=") } assert environments == {f"--testmon-env={bootstrap['testmon_environment']['name']}"} lane_timeouts = [step["timeout_s"] for step in lane_steps] diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 9fd629e17a..3a4bf9187e 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -971,7 +971,7 @@ def test_changed_paths_include_executable_rename_sources( monkeypatch.setattr(verify, "ROOT", tmp_path) changed = verify._changed_test_relevant_paths(base, head) - assert executable_python_paths(tmp_path, changed) == ("polylogue/example.py",) + assert "polylogue/example.py" in changed def test_changed_paths_parse_non_ascii_names_without_git_quoting( @@ -1321,6 +1321,10 @@ def test_checkout_mutation_monitor_observes_transient_index_authority_change(tmp hidden.parent.mkdir() hidden.write_text("secret authority\n", encoding="utf-8") subprocess.run(["git", "add", "-f", "ignored/hidden.py"], cwd=tmp_path, check=True) + deadline = time.monotonic() + 1 + while not monitor._changed and time.monotonic() < deadline: + time.sleep(0.001) + assert monitor._changed, "monitor did not witness the changed index authority before it was restored" subprocess.run(["git", "reset", "-q", "--", "ignored/hidden.py"], cwd=tmp_path, check=True) observation = monitor.finish() @@ -1351,6 +1355,10 @@ def test_checkout_mutation_monitor_observes_transient_head_ref_change(tmp_path: monitor = CheckoutMutationMonitor(tmp_path) monitor.start() subprocess.run(["git", "update-ref", branch, first], cwd=tmp_path, check=True) + deadline = time.monotonic() + 1 + while not monitor._changed and time.monotonic() < deadline: + time.sleep(0.001) + assert monitor._changed, "monitor did not witness the changed ref authority before it was restored" subprocess.run(["git", "update-ref", branch, second], cwd=tmp_path, check=True) observation = monitor.finish() @@ -3782,6 +3790,7 @@ def finish(self) -> CheckoutMutationObservation: ) native_state = SimpleNamespace( valid=True, + status="valid", environment=SimpleNamespace(nodeids=("tests/test_parallel.py::test_owner",)), missing_executable_paths=(), reason="current", From 8b94a0cf25c96645ef5de59093766ccfe8ecb606 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 20:17:16 +0200 Subject: [PATCH 05/31] refactor(test): make native correctness corpus complete --- CLAUDE.md | 5 +- CONTRIBUTING.md | 15 +- TESTING.md | 17 +-- devtools/merge_boundary.py | 22 +-- devtools/merge_gate.py | 17 +-- devtools/project_motd.py | 2 +- devtools/testmon_bootstrap.py | 1 + devtools/verification_contracts.py | 7 +- devtools/verify.py | 56 ++----- devtools/verify_runs.py | 2 - docs/visual-evidence.md | 2 +- pyproject.toml | 4 - tests/benchmarks/test_daemon_convergence.py | 20 +-- tests/benchmarks/test_scale_tiers.py | 57 +++---- .../benchmarks/test_schema_linear_scaling.py | 4 - tests/conftest.py | 46 +----- tests/infra/scale_fixtures.py | 142 ------------------ .../devtools/test_native_testmon_lifecycle.py | 132 +++++++--------- .../test_durable_schema_policy_gate.py | 3 +- tests/unit/devtools/test_merge_boundary.py | 44 ------ tests/unit/devtools/test_merge_gate.py | 11 +- tests/unit/devtools/test_project_motd.py | 2 +- tests/unit/devtools/test_verify.py | 112 ++------------ tests/unit/infra/test_archive_templates.py | 41 +++++ tests/unit/storage/test_perf_rescue_1314.py | 25 ++- tests/unit/storage/test_store_ops.py | 4 +- tests/unit/test_pytest_temp_policy.py | 32 +--- 27 files changed, 193 insertions(+), 632 deletions(-) delete mode 100644 tests/infra/scale_fixtures.py diff --git a/CLAUDE.md b/CLAUDE.md index 76c1d9662f..51ec00ca16 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -340,8 +340,9 @@ testmon-affected set. invalid local state, optionally copies a matching main-checkout database, and automatically runs the complete correctness corpus when no valid native environment exists. Never ask an operator or agent to seed or repair it. -- Reserve `devtools verify --all` (full non-integration run) for - harness/dependency changes or a final pre-PR diagnostic. +- Reserve `devtools verify --all` (complete unit/property/fuzz/integration + correctness corpus; performance benchmarks excluded) for harness/dependency + changes or a final pre-PR diagnostic. - `devtools verify --quick` = format + lint + mypy + `render all --check` (no tests); it runs on `git push` via the pre-push hook. It is a fast gate, not a substitute for the default baseline before a PR. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5c2fa05f01..da50acd38e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -294,7 +294,7 @@ local pytest selection is accelerated by pytest-testmon. ```bash devtools verify # static/generated gates + pytest-testmon affected tests -devtools verify --all # explicit full non-integration pytest diagnostic +devtools verify --all # complete correctness corpus; benchmarks excluded devtools verify --quick # format + lint + mypy + render all --check (skip tests) devtools verify --lab # explicit lab checks beyond the quick/default loop ``` @@ -306,12 +306,13 @@ environment when collection semantics change. `devtools verify` does not replay a prior verify result. It always runs the static gates and then invokes pytest-testmon for affected-test selection from -the current source, dependency, and Python-version state. The default pytest -step combines marker filters with `--testmon-forceselect` so scale-tier -deselection does not silently expand the run back to the whole suite. It uses -one parallel lane and one serial `load_sensitive`/`tui` lane over the same -native environment. Use `devtools verify --all` for an explicit full -diagnostic; there is no manual seed or repair command. +the current source, dependency, and Python-version state. The pytest step +covers unit, property, fuzz, and integration tests while excluding the +separately operated `tests/benchmarks` performance surface. It uses +`--testmon-forceselect` for affected selection, with one parallel lane and one +serial `load_sensitive`/`tui` lane over the same native environment. Use +`devtools verify --all` for the complete correctness corpus; there is no +manual seed or repair command. Add `devtools release build-package` or `nix flake check` when touching packaging or Nix expressions. See [TESTING.md](TESTING.md) and [docs/devtools.md](docs/devtools.md) diff --git a/TESTING.md b/TESTING.md index 686adb71a5..2013955fc5 100644 --- a/TESTING.md +++ b/TESTING.md @@ -20,10 +20,9 @@ devtools test tests/unit/pipeline -x POLYLOGUE_PYTEST_WORKERS=8 devtools test tests/unit/storage # override workers # Raw pytest still works for ad-hoc needs the wrapper does not cover: -pytest -x --ignore=tests/integration -pytest tests/unit/storage/test_hybrid_laws.py +pytest -x tests/unit/storage/test_hybrid_laws.py -# Explicit full non-integration pytest diagnostic +# Complete correctness corpus (unit/property/fuzz/integration; benchmarks excluded) devtools verify --all # Full Nix/CI parity @@ -67,12 +66,12 @@ checks do not spawn a worker pool. `devtools verify` keeps pytest-testmon as the affected-test selector and runs the selected default lane with an adaptive worker pool (up to 12, override with `POLYLOGUE_PYTEST_WORKERS`) so a stale or genuinely broad affected set cannot spend the full timeout in one -multi-GiB Python process. Because the default gate also applies marker filters -for scale tiers, it passes `--testmon-forceselect` so pytest-testmon still -selects affected tests instead of letting pytest marker selection expand the -run. Bootstrap and full diagnostic runs use the same policy, which budgets roughly -768 MiB per worker, reserves host and tmpfs headroom, and reduces concurrency -when memory pressure is elevated. +multi-GiB Python process. It passes `--testmon-forceselect` so pytest-testmon +selects affected tests within the two semantic lanes. Bootstrap and full runs +cover unit, property, fuzz, and integration correctness tests while excluding +the separately operated `tests/benchmarks` performance surface. They budget +roughly 768 MiB per worker, reserve host and tmpfs headroom, and reduce +concurrency when memory pressure is elevated. Every native run has exactly two semantic lanes over one environment and one database: a parallel lane for tests that are neither `load_sensitive` nor diff --git a/devtools/merge_boundary.py b/devtools/merge_boundary.py index cfe0f2c6a0..d4022210e4 100644 --- a/devtools/merge_boundary.py +++ b/devtools/merge_boundary.py @@ -79,7 +79,7 @@ from typing import Any from devtools import merge_gate, pr_scope -from devtools.verification_contracts import TerminalAuthorization, VerificationScope +from devtools.verification_contracts import VerificationScope _LEDGER_PATH = Path(".cache/verify/merge-gate/merge-train-ledger.json") _LEDGER_PENDING_PATH = _LEDGER_PATH.with_name(f"{_LEDGER_PATH.name}.pending") @@ -212,7 +212,7 @@ def _validate_ledger(data: object) -> dict[str, Any]: permission = receipt.get("release_baseline_allowed") if permission is not None and not isinstance(permission, bool): raise LedgerStateError("merge-train terminal receipt has malformed release permission") - for key in ("terminal_authorization", "verified_head_sha", "target_sha", "merged_master_sha"): + for key in ("verified_head_sha", "target_sha", "merged_master_sha"): value = receipt.get(key) if value is not None and (not isinstance(value, str) or not value): raise LedgerStateError(f"merge-train terminal receipt has malformed {key!r}") @@ -396,13 +396,7 @@ def _pending_prs_since_last_full_verify(ledger: dict[str, Any]) -> list[dict[str if last_verify.get("accepted") is True and last_verify.get("exit_code") == 0 and last_verify.get("release_baseline_allowed") is True - and ( - scope == VerificationScope.RELEASE_BASELINE.value - or ( - scope == VerificationScope.NARROW_TERMINAL.value - and last_verify.get("terminal_authorization") == TerminalAuthorization.NARROW_TERMINAL.value - ) - ) + and scope == VerificationScope.RELEASE_BASELINE.value else 0.0 ) snapshot_sequence = last_verify.get("merge_sequence") @@ -834,18 +828,11 @@ def cmd_record_full_verify( duration_s = round(time.time() - started, 2) release_allowed = merge_gate._release_baseline_permission(receipt) verification_scope = merge_gate._verification_scope(receipt) - terminal_authorization = merge_gate._terminal_authorization(receipt) verified_head = receipt.get("git_head") if isinstance(receipt, dict) else None accepted = ( result.returncode == 0 and release_allowed is True - and ( - verification_scope == VerificationScope.RELEASE_BASELINE.value - or ( - verification_scope == VerificationScope.NARROW_TERMINAL.value - and terminal_authorization == TerminalAuthorization.NARROW_TERMINAL.value - ) - ) + and verification_scope == VerificationScope.RELEASE_BASELINE.value and verified_head == target_sha ) @@ -860,7 +847,6 @@ def cmd_record_full_verify( "verification_started_at": verification_started_at, "verification_scope": verification_scope, "release_baseline_allowed": release_allowed, - "terminal_authorization": terminal_authorization, "verified_head_sha": verified_head, "target_sha": target_sha, "merged_master_sha": target_sha, diff --git a/devtools/merge_gate.py b/devtools/merge_gate.py index f4295d8647..c8c30ee255 100644 --- a/devtools/merge_gate.py +++ b/devtools/merge_gate.py @@ -75,7 +75,7 @@ from typing import Any from devtools import pr_scope -from devtools.verification_contracts import TerminalAuthorization, VerificationScope +from devtools.verification_contracts import VerificationScope from devtools.verify_runs import VERIFICATION_INVOCATION_ID_ENV as VERIFICATION_INVOCATION_ID_ENV from devtools.verify_runs import VERIFICATION_RECEIPT_PATH_ENV as VERIFICATION_RECEIPT_PATH_ENV @@ -86,7 +86,6 @@ _MERGE_AUTHORIZING_VERIFICATION_SCOPES = frozenset( { VerificationScope.AFFECTED.value, - VerificationScope.NARROW_TERMINAL.value, VerificationScope.RELEASE_BASELINE.value, } ) @@ -227,11 +226,6 @@ def _invocation_receipt( return None if _verification_scope(payload) is None or _release_baseline_permission(payload) is None: return None - terminal_authorization = payload.get("terminal_authorization") - if terminal_authorization is not None and terminal_authorization not in { - authorization.value for authorization in TerminalAuthorization - }: - return None return payload @@ -251,14 +245,6 @@ def _verification_scope(payload: Mapping[str, Any] | None) -> str | None: return value if value in {scope.value for scope in VerificationScope} else None -def _terminal_authorization(payload: Mapping[str, Any] | None) -> str | None: - """Read terminal authorization from an invocation-bound receipt.""" - if payload is None: - return None - value = payload.get("terminal_authorization") - return value if value in {authorization.value for authorization in TerminalAuthorization} else None - - def _base_sha(info: dict[str, Any]) -> str | None: """Read the PR base commit SHA when GitHub reported one.""" value = info.get("baseRefOid") @@ -370,7 +356,6 @@ def cmd_record(pr: int, command: str) -> int: "command": command, "verification_scope": _verification_scope(verification_receipt), "release_baseline_allowed": _release_baseline_permission(verification_receipt), - "terminal_authorization": _terminal_authorization(verification_receipt), "exit_code": result.returncode, "duration_s": duration_s, "recorded_at": time.time(), diff --git a/devtools/project_motd.py b/devtools/project_motd.py index bf51e5a982..5916ca5385 100644 --- a/devtools/project_motd.py +++ b/devtools/project_motd.py @@ -158,7 +158,7 @@ def status_snapshot(cwd: Path, *, verify_generated: bool = False) -> StatusSnaps "render_all_check": control_plane_command("render all", "--check"), "verify_quick": control_plane_command("verify", "--quick"), "build_package": control_plane_command("release build-package"), - "test_baseline": "pytest -q --ignore=tests/integration", + "test_baseline": control_plane_command("verify"), }, "local_state": { "cache": ".cache/", diff --git a/devtools/testmon_bootstrap.py b/devtools/testmon_bootstrap.py index c64846137e..5878cc1fa4 100644 --- a/devtools/testmon_bootstrap.py +++ b/devtools/testmon_bootstrap.py @@ -119,6 +119,7 @@ def _environment_input_paths(root: Path) -> tuple[str, ...]: paths = set(_ENVIRONMENT_INPUTS) patterns = ( "devtools/pytest*.py", + "pytest*.py", "tests/**/conftest.py", "tests/infra/**/*.py", ) diff --git a/devtools/verification_contracts.py b/devtools/verification_contracts.py index a2407b0eac..757f0f190a 100644 --- a/devtools/verification_contracts.py +++ b/devtools/verification_contracts.py @@ -8,12 +8,7 @@ class VerificationScope(StrEnum): AFFECTED = "affected" RELEASE_BASELINE = "release-baseline" - NARROW_TERMINAL = "narrow-terminal" NON_TEST = "non-test" -class TerminalAuthorization(StrEnum): - NARROW_TERMINAL = "narrow-terminal" - - -__all__ = ["TerminalAuthorization", "VerificationScope"] +__all__ = ["VerificationScope"] diff --git a/devtools/verify.py b/devtools/verify.py index d7b6e64da6..c10b9dc39d 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -8,8 +8,8 @@ --quick Pre-push tier: all non-pytest gates (~15s warm). (default) Baseline with pytest-testmon affected tests. --all/--full - Explicit full non-integration pytest diagnostic in the current - native testmon environment. + Complete pytest correctness corpus in the current native + testmon environment (performance benchmarks excluded). --lab Default testmon baseline plus lab smoke and SLO checks. Output formats: @@ -66,7 +66,7 @@ prepare_native_testmon_environment, remove_invalid_native_testmon_state, ) -from devtools.verification_contracts import TerminalAuthorization, VerificationScope +from devtools.verification_contracts import VerificationScope from devtools.verify_runs import ( CURRENT_CONTAINMENT_PATH, CURRENT_EVENTS_DIR, @@ -1663,7 +1663,6 @@ def _run( "diagnosis": "pytest_resource_preflight_failed", "error": str(exc), "termination_reason": "pytest resource preflight refused basetemp admission", - "verification_scope": "narrow-terminal", "release_baseline_allowed": False, } if run is not None and artifacts is not None: @@ -2090,7 +2089,6 @@ def build_verify_steps( *, quick: bool, lab: bool, - skip_slow: bool, commit: bool = False, testmon_mode: str = "affected", testmon_environment: str = "", @@ -2139,18 +2137,12 @@ def build_verify_steps( _report_dir = PYTEST_JUNIT_REPORT_DIR _report_dir.mkdir(parents=True, exist_ok=True) PYTEST_REPORT_DIR.mkdir(parents=True, exist_ok=True) - # Scale-tier policy (issue #1183): default verify includes - # ``scale_small`` but excludes ``scale_medium`` / ``scale_large``. - # ``--lab`` lets the medium tier in; the large tier is reserved - # for nightly CI's direct pytest-benchmark execution. - scale_marker_expr = "not scale_large" if lab else "not scale_medium and not scale_large" pytest_cmd = [ sys.executable, "-m", "pytest", "-q", "--tb=short", - "--ignore=tests/integration", # Benchmark files are an explicit campaign surface. A number of # them are correctness-shaped and lack the benchmark marker, so a # marker expression alone cannot keep performance probes out of @@ -2166,12 +2158,10 @@ def build_verify_steps( ] # Benchmark cases are an explicit campaign surface, not part of the # correctness corpus. Keeping them out here is important: a - # benchmark marker is not necessarily paired with ``slow`` or a scale - # marker, and the serial lane would otherwise spend minutes executing a - # performance probe before it can checkpoint any correctness nodes. - base_marker = f"not benchmark and {scale_marker_expr}" - if skip_slow: - base_marker = f"not slow and {base_marker}" + # benchmark marker is not necessarily present, and the serial lane + # would otherwise spend minutes executing a performance probe before + # it can checkpoint any correctness nodes. + base_marker = "not benchmark" if testmon_mode not in {"affected", "bootstrap", "full"}: raise ValueError(f"unknown native testmon mode: {testmon_mode}") if not testmon_environment: @@ -2394,12 +2384,12 @@ def _planned_verification_scope( if args.quick or args.commit: return VerificationScope.NON_TEST if testmon_mode in {"bootstrap", "full"}: - return VerificationScope.NARROW_TERMINAL if args.skip_slow else VerificationScope.RELEASE_BASELINE + return VerificationScope.RELEASE_BASELINE return VerificationScope.AFFECTED -def _pytest_profile(*, skip_slow: bool, lab: bool) -> str: - return f"slow={'exclude' if skip_slow else 'include'};scale={'medium' if lab else 'small'}" +def _pytest_profile() -> str: + return "correctness=complete" def _remaining_invocation_budget(started_at: float) -> float: @@ -2422,18 +2412,13 @@ def _release_baseline_allowed( *, selection_mode: str | None, verification_scope: VerificationScope, - terminal_authorization: str | None, exit_code: int, checkout_stable: bool, aggregate: Mapping[str, Any] | None, ) -> bool: if selection_mode not in {"bootstrap", "full"} or exit_code != 0 or not checkout_stable or aggregate is None: return False - scope_authorized = verification_scope == VerificationScope.RELEASE_BASELINE or ( - verification_scope == VerificationScope.NARROW_TERMINAL - and terminal_authorization == TerminalAuthorization.NARROW_TERMINAL.value - ) - if not scope_authorized: + if verification_scope != VerificationScope.RELEASE_BASELINE: return False cleanup = aggregate.get("cleanup") containment = aggregate.get("containment") @@ -2458,18 +2443,10 @@ def _main(argv: list[str] | None = None) -> int: parser.add_argument( "--all", action="store_true", - help="Run the complete non-integration corpus in the current native testmon environment.", + help="Run the complete pytest correctness corpus (excluding performance benchmarks).", ) parser.add_argument("--full", action="store_true", help="Alias for --all.") parser.add_argument("--commit", action="store_true", help="Pre-commit tier: format + lint + mypy only.") - parser.add_argument( - "--skip-slow", action="store_true", help="Exclude @pytest.mark.slow tests from the pytest step." - ) - parser.add_argument( - "--terminal-authorization", - choices=[TerminalAuthorization.NARROW_TERMINAL.value], - help="Typed authorization for a narrow terminal verification that skips slow tests.", - ) parser.add_argument( "--lab", action="store_true", @@ -2494,9 +2471,6 @@ def _main(argv: list[str] | None = None) -> int: return 0 full_requested = bool(args.all or args.full) - if args.terminal_authorization is not None and not (full_requested and args.skip_slow): - parser.error("--terminal-authorization requires --all or --full with --skip-slow") - use_json = args.json if args.json is not None else not sys.stdout.isatty() tier = ( "commit" @@ -2529,7 +2503,7 @@ def _main(argv: list[str] | None = None) -> int: preparation = prepare_native_testmon_environment( ROOT, required_executable_paths=required_executable_paths, - pytest_profile=_pytest_profile(skip_slow=bool(args.skip_slow), lab=bool(args.lab)), + pytest_profile=_pytest_profile(), ) except (NativeTestmonRepairError, PytestResourceError) as exc: sys.stderr.write(f"verify: native pytest-testmon preparation failed: {exc}\n") @@ -2570,7 +2544,6 @@ def _main(argv: list[str] | None = None) -> int: quick=bool(args.quick), commit=bool(args.commit), lab=bool(args.lab), - skip_slow=bool(args.skip_slow), testmon_mode=testmon_mode or "affected", testmon_environment=preparation.environment_name if preparation is not None else "", ) @@ -2756,7 +2729,6 @@ def _main(argv: list[str] | None = None) -> int: release_baseline_allowed = _release_baseline_allowed( selection_mode=testmon_mode, verification_scope=planned_scope, - terminal_authorization=args.terminal_authorization, exit_code=exit_code, checkout_stable=checkout_stable, aggregate=pytest_aggregate, @@ -2798,7 +2770,6 @@ def _main(argv: list[str] | None = None) -> int: "exit_code": exit_code, "verification_scope": verification_scope.value, "release_baseline_allowed": release_baseline_allowed, - "terminal_authorization": args.terminal_authorization, } if preparation is not None: history_entry["testmon_environment"] = { @@ -2818,7 +2789,6 @@ def _main(argv: list[str] | None = None) -> int: diagnosis=run_diagnosis, verification_scope=verification_scope.value, release_baseline_allowed=release_baseline_allowed, - terminal_authorization=args.terminal_authorization, final_worktree_fingerprint=final_checkout_fingerprint, checkout_mutation_path=mutation_observation.observed_path, checkout_diagnosis=checkout_diagnosis, diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index 6adc093efb..990936cc45 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -1606,7 +1606,6 @@ def finish( diagnosis: str | None = None, verification_scope: str | None = None, release_baseline_allowed: bool | None = None, - terminal_authorization: str | None = None, final_worktree_fingerprint: str | None = None, checkout_mutation_path: str | None = None, checkout_diagnosis: str | None = None, @@ -1629,7 +1628,6 @@ def finish( if verification_scope is not None: self._payload["verification_scope"] = verification_scope self._payload["release_baseline_allowed"] = release_baseline_allowed - self._payload["terminal_authorization"] = terminal_authorization self.write() return dict(self._payload) diff --git a/docs/visual-evidence.md b/docs/visual-evidence.md index 56b9bdf69e..638f01cd0d 100644 --- a/docs/visual-evidence.md +++ b/docs/visual-evidence.md @@ -84,7 +84,7 @@ wrapper reads those executed artifacts into its report. The manifests use `schema_version: 1`, `evidence_kind: browserless-dom`, the command, artifact id, fixture id, route, and structural checks asserted by that test. -Both suites are part of the standard non-integration test run. There is no +Both suites are part of the standard correctness corpus. There is no browser binary or Playwright dependency in these fast lanes: they use Python's standard `http.server`, `urllib.request`, and `html.parser` against the real `DaemonAPIHTTPServer`. The `devtools lab smoke` command is the diff --git a/pyproject.toml b/pyproject.toml index 74da3f7584..fc97d3bfda 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -180,10 +180,6 @@ markers = [ "integration: marks integration tests (pipeline, CLI end-to-end)", "benchmark: marks benchmark tests (run with --benchmark-enable -p no:xdist -p no:randomly)", "contract: marks public contract tests that may emit bounded evidence artifacts", - "scale(level): parametric scale marker (small/medium/large/stretch)", - "scale_small: small-tier scale fixture (~100 convs / ~1k msgs); default verify gate (#1183)", - "scale_medium: medium-tier scale fixture (~1k convs / ~10k msgs); devtools verify --lab gate (#1183)", - "scale_large: large-tier scale fixture (~10k convs / ~100k msgs); nightly CI / explicit campaigns only (#1183)", "machine_contract: marks root CLI JSON success/failure contract tests", "query_routing: marks query-first CLI routing and read-surface tests", "evidence_law: marks generated-surface completeness, anti-dead-code, and evidence-quality laws", diff --git a/tests/benchmarks/test_daemon_convergence.py b/tests/benchmarks/test_daemon_convergence.py index e821f56520..d9675e6783 100644 --- a/tests/benchmarks/test_daemon_convergence.py +++ b/tests/benchmarks/test_daemon_convergence.py @@ -167,25 +167,8 @@ def __init__(self, archive_root: Path, db_path: Path) -> None: # ── Benchmark tests ───────────────────────────────────────────────── -# Tiers whose per-iteration runtime exceeds the default benchmark budget -# (multiple minutes per repeat in CI) are routed to the ``scale_large`` -# nightly marker. The xxl mega-session tier ingests 100k messages from a -# single file and is the canonical regression probe for #1244 / #845-A. -_NIGHTLY_TIERS = {"xxl-mega-session"} - - -def _tier_params() -> list[Any]: - params: list[Any] = [] - for tier in _SCALE_TIERS: - if tier in _NIGHTLY_TIERS: - params.append(pytest.param(tier, marks=[pytest.mark.scale_large])) - else: - params.append(pytest.param(tier)) - return params - - @pytest.mark.benchmark -@pytest.mark.parametrize("tier", _tier_params()) +@pytest.mark.parametrize("tier", _SCALE_TIERS) def test_convergence_scale_tier(benchmark, tier: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: # type: ignore[no-untyped-def] """Measure convergence throughput at each scale tier.""" corpus_root = _generate_corpus(tmp_path, tier) @@ -339,7 +322,6 @@ def test_convergence_large_session_memory( @pytest.mark.benchmark -@pytest.mark.scale_large def test_convergence_huge_session_memory_bounded( benchmark: BenchmarkFixture, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/tests/benchmarks/test_scale_tiers.py b/tests/benchmarks/test_scale_tiers.py index fdba53bec9..d2579f5470 100644 --- a/tests/benchmarks/test_scale_tiers.py +++ b/tests/benchmarks/test_scale_tiers.py @@ -1,28 +1,15 @@ -"""Tiered scale tests demonstrating the small/medium/large fixture contract. +"""Query probes over the benchmark runner's shared database sizes. Each test exercises a measured surface (FTS5 search, hybrid RRF search, -list_sessions) against the corresponding scale fixture and asserts -basic shape so a regression in the seeder or query path surfaces even -when ``--benchmark-disable`` is in effect. - -Marker plumbing (issue #1183): - - ``@pytest.mark.scale_small`` — default ``devtools verify`` gate. - ``@pytest.mark.scale_medium`` — ``devtools verify --lab`` only. - ``@pytest.mark.scale_large`` — nightly CI / explicit campaigns only. - -The default ``devtools verify`` pytest step passes ``-m "not scale_medium -and not scale_large"`` so only the small tier runs in the inner loop. -Tests that want measured timings should also carry ``@pytest.mark.benchmark`` -and run under the direct pytest-benchmark nightly workflow. +list_sessions) against the benchmark fixtures and asserts basic shape. The +entire file is an explicit performance-plugin surface; native correctness +verification excludes ``tests/benchmarks`` by path instead of tier markers. """ from __future__ import annotations from pathlib import Path -import pytest - from tests.benchmarks.helpers import open_bench_store @@ -40,43 +27,39 @@ def _list_query(db_path: Path, *, limit: int = 20) -> int: # --------------------------------------------------------------------------- -# Small tier — runs in default `devtools verify` baseline. +# Small benchmark database. # --------------------------------------------------------------------------- -@pytest.mark.scale_small -def test_scale_small_fts_search_returns_results(tier_small_db: Path) -> None: +def test_bench_1k_fts_search_returns_results(bench_db_1k: Path) -> None: """FTS5 search over the small tier returns at most ``limit`` rows.""" - hits = _fts_query(tier_small_db, "analysis", limit=10) + hits = _fts_query(bench_db_1k, "analysis", limit=10) assert 0 <= hits <= 10 -@pytest.mark.scale_small -def test_scale_small_list_sessions(tier_small_db: Path) -> None: +def test_bench_1k_list_sessions(bench_db_1k: Path) -> None: """``list_summaries`` returns up to ``limit`` entries from the small tier.""" - rows = _list_query(tier_small_db, limit=10) + rows = _list_query(bench_db_1k, limit=10) assert 0 < rows <= 10 # --------------------------------------------------------------------------- -# Medium tier — runs in `devtools verify --lab`. +# Medium benchmark database. # --------------------------------------------------------------------------- -@pytest.mark.scale_medium -def test_scale_medium_fts_search_returns_results(tier_medium_db: Path) -> None: - hits = _fts_query(tier_medium_db, "analysis", limit=20) +def test_bench_10k_fts_search_returns_results(bench_db_10k: Path) -> None: + hits = _fts_query(bench_db_10k, "analysis", limit=20) assert 0 <= hits <= 20 -@pytest.mark.scale_medium -def test_scale_medium_list_sessions(tier_medium_db: Path) -> None: - rows = _list_query(tier_medium_db, limit=20) +def test_bench_10k_list_sessions(bench_db_10k: Path) -> None: + rows = _list_query(bench_db_10k, limit=20) assert 0 < rows <= 20 # --------------------------------------------------------------------------- -# Large tier — runs in nightly CI / explicit campaigns only. +# Large benchmark database — direct file/node campaigns only. # # The fixture creation cost dominates wall-clock time for this tier # (~tens of seconds to minutes depending on host). The tests below are @@ -86,13 +69,11 @@ def test_scale_medium_list_sessions(tier_medium_db: Path) -> None: # --------------------------------------------------------------------------- -@pytest.mark.scale_large -def test_scale_large_fts_search_returns_results(tier_large_db: Path) -> None: - hits = _fts_query(tier_large_db, "analysis", limit=50) +def test_bench_50k_fts_search_returns_results(bench_db_50k: Path) -> None: + hits = _fts_query(bench_db_50k, "analysis", limit=50) assert 0 <= hits <= 50 -@pytest.mark.scale_large -def test_scale_large_list_sessions(tier_large_db: Path) -> None: - rows = _list_query(tier_large_db, limit=50) +def test_bench_50k_list_sessions(bench_db_50k: Path) -> None: + rows = _list_query(bench_db_50k, limit=50) assert 0 < rows <= 50 diff --git a/tests/benchmarks/test_schema_linear_scaling.py b/tests/benchmarks/test_schema_linear_scaling.py index 59784ad5f9..656dcf4ce6 100644 --- a/tests/benchmarks/test_schema_linear_scaling.py +++ b/tests/benchmarks/test_schema_linear_scaling.py @@ -12,8 +12,6 @@ from collections.abc import Callable from pathlib import Path -import pytest - from polylogue.schemas.validation.corpus import verify_raw_corpus from polylogue.schemas.validation.requests import SchemaVerificationRequest @@ -26,7 +24,6 @@ def _measure(db_path: Path, record_limit: int | None) -> float: return (time.perf_counter() - start) * 1000 -@pytest.mark.scale_small def test_schema_check_completes_quickly(named_seeded_archive: Callable[[str], Path]) -> None: """Smoke: verify_raw_corpus finishes and returns a valid report.""" db = named_seeded_archive("schema-small") @@ -34,7 +31,6 @@ def test_schema_check_completes_quickly(named_seeded_archive: Callable[[str], Pa assert ms < 30_000, f"10-record corpus took {ms:.0f} ms; expected <30s" -@pytest.mark.scale_medium def test_schema_check_linear_scaling(named_seeded_archive: Callable[[str], Path]) -> None: """Wall time must grow sub-quadratically across record limits. diff --git a/tests/conftest.py b/tests/conftest.py index b5851dfcc5..08bc87bfef 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -40,6 +40,7 @@ resolve_pytest_basetemp_root, ) from devtools.verify_runs import pytest_basetemp_claim_path as _basetemp_claim_path +from tests.infra.archive_templates import clone_archive_template # Resolve (but don't yet raise on) the polylogue-vs-checkout mismatch check # before test execution can import product modules: a shared/editable venv's @@ -59,7 +60,6 @@ pytest_plugins = ( "tests.infra.corpus_fixtures", - "tests.infra.scale_fixtures", "tests.infra.frozen_clock", "tests.infra.clock_guard", ) @@ -85,10 +85,6 @@ from polylogue.storage.sqlite import SQLiteBackend from tests.infra.storage_records import SessionBuilder -# --------------------------------------------------------------------------- -# Scale markers for data-gravity and long-haul validation (Workstream H) -# --------------------------------------------------------------------------- - def _set_managed_pytest_identity(identity: tuple[str, str] | None) -> None: """Expose only the managed identity owned by the active invocation.""" @@ -146,20 +142,6 @@ def pytest_configure(config: pytest.Config) -> None: # devtools/checkout_guard.py for the full hazard writeup). raise pytest.UsageError(f"pytest: {_CHECKOUT_GUARD_ERROR}") from _CHECKOUT_GUARD_ERROR sys.stderr.write(f"pytest: polylogue package → {resolved_polylogue_path()} (checkout: {_TESTS_REPO_ROOT})\n") - config.addinivalue_line("markers", "scale(level): parametric scale marker (small/medium/large/stretch)") - # Tiered scale markers (issue #1183); definitions also live in - # pyproject.toml `markers` so xfail_strict + filterwarnings agree. - config.addinivalue_line( - "markers", "scale_small: small-tier scale fixture (~100 convs / ~1k msgs); default verify gate (#1183)" - ) - config.addinivalue_line( - "markers", "scale_medium: medium-tier scale fixture (~1k convs / ~10k msgs); verify --lab gate (#1183)" - ) - config.addinivalue_line( - "markers", - "scale_large: large-tier scale fixture (~10k convs / ~100k msgs); nightly CI / campaigns only (#1183)", - ) - if config.option.basetemp is not None: configured_basetemp = str(config.option.basetemp) run_id = os.environ.get("POLYLOGUE_PYTEST_RUN_ID") @@ -874,7 +856,7 @@ def workspace_env( # contract strictness. Keep validation deterministic and opt-in per test. monkeypatch.setenv("POLYLOGUE_SCHEMA_VALIDATION", "off") - _clone_archive_template(empty_archive_template, archive_root) + clone_archive_template(empty_archive_template, archive_root) return { "archive_root": archive_root, @@ -956,7 +938,7 @@ def cli_workspace( monkeypatch.setenv("POLYLOGUE_FORCE_PLAIN", "1") # Plain output for tests monkeypatch.setenv("POLYLOGUE_SCHEMA_VALIDATION", "off") - _clone_archive_template(empty_archive_template, archive_root) + clone_archive_template(empty_archive_template, archive_root) return { "archive_root": archive_root, @@ -968,28 +950,6 @@ def cli_workspace( } -def _clone_archive_template(source: Path, destination: Path) -> None: - """Clone one immutable empty archive into a test-private workspace.""" - destination.mkdir(parents=True, exist_ok=True) - try: - subprocess.run( - ["cp", "-a", "--reflink=auto", f"{source}/.", str(destination)], - check=True, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - timeout=10, - ) - except (OSError, subprocess.CalledProcessError, subprocess.TimeoutExpired): - shutil.copytree(source, destination, dirs_exist_ok=True) - - bootstrap_marker = destination / ".maintenance-state" / "durable-change-trains" / ".bootstrap" - if bootstrap_marker.is_file(): - from polylogue.storage.sqlite.durable_change_train import _record_fresh_durable_bootstrap - - bootstrap_marker.unlink() - _record_fresh_durable_bootstrap(destination) - - @pytest.fixture(scope="session") def empty_archive_template( tmp_path_factory: pytest.TempPathFactory, diff --git a/tests/infra/scale_fixtures.py b/tests/infra/scale_fixtures.py deleted file mode 100644 index e1d262cbd7..0000000000 --- a/tests/infra/scale_fixtures.py +++ /dev/null @@ -1,142 +0,0 @@ -"""Tiered scale fixtures for performance/regression tests (issue #1183). - -Provides three explicit tiers with documented session/message counts -and pytest markers so each tier runs in the right gate: - - ``tier_small_db`` — ~100 sessions / ~1,000 messages - Marker: ``@pytest.mark.scale_small`` - Default ``devtools verify`` includes these. - ``tier_medium_db`` — ~1,000 sessions / ~10,000 messages - Marker: ``@pytest.mark.scale_medium`` - ``devtools verify --lab`` includes these. - ``tier_large_db`` — ~10,000 sessions / ~100,000 messages - Marker: ``@pytest.mark.scale_large`` - Nightly CI / explicit campaigns only. - -The factories return cached SQLite paths per test session — generating -the large tier is expensive (~minutes), so tests within a session share -one fixture instance. Each tier uses the same realistic distribution -helpers used by ``tests/benchmarks/conftest.py``. - -Growth-shape rule: tests that assert latency across tiers must compare -ratios (large/medium, medium/small) rather than absolute milliseconds. -Absolute budgets bake in host-machine assumptions; ratio budgets stay -portable. -""" - -from __future__ import annotations - -from collections.abc import Callable -from dataclasses import dataclass -from pathlib import Path - -import pytest - - -@dataclass(frozen=True) -class ScaleTier: - """Declarative scale-tier definition. - - Attributes: - name: tier label (``small``/``medium``/``large``) - target_messages: approximate number of messages to seed; the seeder - stops once the running total reaches this floor. - target_sessions: approximate number of sessions expected - for this tier (informational; the seeder is driven by message - count and the realistic-distribution profile). - marker: pytest marker name registered in ``pyproject.toml``. - """ - - name: str - target_messages: int - target_sessions: int - marker: str - - -SCALE_SMALL = ScaleTier( - name="small", - target_messages=1_000, - target_sessions=100, - marker="scale_small", -) - -SCALE_MEDIUM = ScaleTier( - name="medium", - target_messages=10_000, - target_sessions=1_000, - marker="scale_medium", -) - -SCALE_LARGE = ScaleTier( - name="large", - target_messages=100_000, - target_sessions=10_000, - marker="scale_large", -) - -ALL_TIERS: tuple[ScaleTier, ...] = (SCALE_SMALL, SCALE_MEDIUM, SCALE_LARGE) -ALL_MARKERS: tuple[str, ...] = tuple(t.marker for t in ALL_TIERS) - - -def _seed_tier_db(db_path: Path, tier: ScaleTier, *, seed: int = 1183) -> dict[str, int]: - """Seed ``db_path`` with realistic data for the requested tier. - - Defers the heavy import to fixture invocation so unit collection stays - fast — ``tests.benchmarks.conftest`` pulls in storage backends, the - rebuild_index path, and the synthetic corpus generator. - """ - from tests.benchmarks.conftest import _seed_realistic_db # local import: heavy - - return _seed_realistic_db(db_path, target_messages=tier.target_messages, seed=seed) - - -# --------------------------------------------------------------------------- -# Session-scoped tier factories -# -# Each fixture is session-scoped so the (expensive) seeding only happens -# once per test session. The factory returns a ``Path`` to the seeded -# SQLite database; tests open it read-only via the standard -# ``open_bench_store`` / repository helpers. -# --------------------------------------------------------------------------- - - -@pytest.fixture(scope="session") -def tier_small_db(tmp_path_factory: pytest.TempPathFactory) -> Path: - """Small-tier SQLite DB: ~100 convs / ~1k messages. Default verify gate.""" - db_path = tmp_path_factory.mktemp("scale_small") / "small.db" - _seed_tier_db(db_path, SCALE_SMALL) - return db_path - - -@pytest.fixture(scope="session") -def tier_medium_db(tmp_path_factory: pytest.TempPathFactory) -> Path: - """Medium-tier SQLite DB: ~1k convs / ~10k messages. Lab gate.""" - db_path = tmp_path_factory.mktemp("scale_medium") / "medium.db" - _seed_tier_db(db_path, SCALE_MEDIUM) - return db_path - - -@pytest.fixture(scope="session") -def tier_large_db(tmp_path_factory: pytest.TempPathFactory) -> Path: - """Large-tier SQLite DB: ~10k convs / ~100k messages. Nightly gate.""" - db_path = tmp_path_factory.mktemp("scale_large") / "large.db" - _seed_tier_db(db_path, SCALE_LARGE) - return db_path - - -# Public re-exports for documentation/registry consumers. -__all__ = [ - "ALL_MARKERS", - "ALL_TIERS", - "SCALE_LARGE", - "SCALE_MEDIUM", - "SCALE_SMALL", - "ScaleTier", - "tier_large_db", - "tier_medium_db", - "tier_small_db", -] - - -# Factory protocol re-exports for `Callable[..., Path]`-typed call sites. -_FactoryReExports: tuple[Callable[..., Path], ...] = () diff --git a/tests/integration/devtools/test_native_testmon_lifecycle.py b/tests/integration/devtools/test_native_testmon_lifecycle.py index b4643f8287..be3e87bc2f 100644 --- a/tests/integration/devtools/test_native_testmon_lifecycle.py +++ b/tests/integration/devtools/test_native_testmon_lifecycle.py @@ -22,16 +22,13 @@ from devtools.testmon_bootstrap import ( testmon_environment_digest as _testmon_environment_digest, ) -from devtools.verify_runs import ( - PYTEST_CANONICAL_REPORT_NAME, - VerifyRun, - aggregate_native_testmon_run, - aggregate_pytest_statistics, - append_verify_history, -) +from devtools.verify_runs import PYTEST_CANONICAL_REPORT_NAME PROJECT_ROOT = Path(__file__).resolve().parents[3] -pytestmark = pytest.mark.uses_real_clock("coordinates real pytest subprocesses and an interrupt deadline") +pytestmark = [ + pytest.mark.uses_real_clock("coordinates real pytest subprocesses and an interrupt deadline"), + pytest.mark.timeout(90), +] @dataclass(frozen=True) @@ -66,7 +63,6 @@ def _init_repo(root: Path, *, conftest: str = "") -> None: markers = [ "load_sensitive: serial native-testmon lane", "tui: serial native-testmon lane", - "scale_large: excluded from the ordinary correctness corpus", ] """.lstrip(), encoding="utf-8", @@ -311,7 +307,9 @@ def test_production_plain_verify_owns_bootstrap_warm_selection_deadline_and_hist package.mkdir() (package / "__init__.py").write_text("", encoding="utf-8") (package / "app.py").write_text("def answer() -> int:\n return 42\n", encoding="utf-8") - (repo / "tests" / "test_app.py").write_text( + integration_dir = repo / "tests" / "integration" + integration_dir.mkdir() + (integration_dir / "test_app.py").write_text( """ import pytest @@ -582,77 +580,61 @@ def test_removed_environment_or_dependency_edge_invalidates_native_state(tmp_pat assert missing.missing_executable_paths == ("app.py",) -def test_two_real_lanes_form_one_run_aggregate_and_one_budget(tmp_path: Path) -> None: +def test_environment_and_plugin_identity_changes_start_real_fresh_bootstraps( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: repo = tmp_path / "repo" repo.mkdir() _init_repo(repo) - (repo / "tests" / "test_lanes.py").write_text( - """ -import pytest - -def test_parallel(): - assert True + test_file = repo / "tests" / "test_identity.py" + test_file.write_text("def test_initial():\n assert True\n", encoding="utf-8") + _commit_all(repo, "fixture") -@pytest.mark.load_sensitive -def test_serial(): - assert True + initial = prepare_native_testmon_environment(repo) + assert initial.selection_mode == "bootstrap" + assert [ + result.completed.returncode + for result in _run_plain_verify_corpus(repo, mode="bootstrap", environment_name=initial.environment_name) + ] == [0, 0] + + monkeypatch.setenv("PYTEST_ADDOPTS", "-ra") + environment_changed = prepare_native_testmon_environment(repo) + assert environment_changed.selection_mode == "bootstrap" + assert environment_changed.environment_name != initial.environment_name + assert [ + result.completed.returncode + for result in _run_plain_verify_corpus( + repo, mode="bootstrap", environment_name=environment_changed.environment_name + ) + ] == [0, 0] -@pytest.mark.scale_large -def test_excluded_from_correctness_corpus(): - assert True -""".lstrip(), + plugin = repo / "pytest_native_identity.py" + plugin.write_text( + "import pytest\n\n@pytest.fixture\ndef native_identity():\n return 'v1'\n", encoding="utf-8", ) - _commit_all(repo, "fixture") - preparation = prepare_native_testmon_environment(repo) - started = time.monotonic() - results = _run_plain_verify_corpus( - repo, - mode="bootstrap", - environment_name=preparation.environment_name, - base_marker="not scale_large", - ) - elapsed = time.monotonic() - started - state = inspect_native_testmon_environment( - repo / TESTMON_DATA_RELPATH, environment_name=preparation.environment_name + test_file.write_text( + "def test_initial():\n assert True\n\ndef test_plugin(native_identity):\n assert native_identity == 'v1'\n", + encoding="utf-8", ) - assert state.environment is not None - steps: list[dict[str, object]] = [] - for lane, result in zip(("parallel", "serial"), results, strict=True): - statistics = aggregate_pytest_statistics(result.artifact_dir) - steps.append( - { - "semantic_lane": lane, - "exit": result.completed.returncode, - "duration_s": elapsed, - "artifact_dir": str(result.artifact_dir.relative_to(repo)), - "selected_count": result.selection["selected_count"], - "collection_duration_s": result.selection.get("collection_duration_s"), - "statistics": statistics, - } - ) - aggregate = aggregate_native_testmon_run( - repo, - steps=steps, - environment_name=preparation.environment_name, - corpus_nodeids=state.environment.nodeids, - selection_mode="bootstrap", - invocation_duration_s=elapsed, - budget_s=3600, + monkeypatch.setenv("PYTEST_PLUGINS", "pytest_native_identity") + plugin_changed = prepare_native_testmon_environment(repo) + assert plugin_changed.selection_mode == "bootstrap" + assert plugin_changed.environment_name != environment_changed.environment_name + plugin_results = _run_plain_verify_corpus(repo, mode="bootstrap", environment_name=plugin_changed.environment_name) + assert [result.completed.returncode for result in plugin_results] == [0, 0] + assert "tests/test_identity.py::test_plugin" in _selected(*plugin_results) + + plugin.write_text( + "import pytest\n\n@pytest.fixture\ndef native_identity():\n return 'v2'\n", + encoding="utf-8", ) - run = VerifyRun(tier="testmon", argv=[], git_head="head", root=repo) - run.finish(exit_code=0, duration_s=elapsed, pytest_aggregate=aggregate) - durable = json.loads((run.run_dir / "run.json").read_text(encoding="utf-8")) - history_path = repo / "xdg-state" / "polylogue" / "devtools" / "verify-history.jsonl" - append_verify_history({"run_id": run.run_id, "pytest_aggregate": aggregate}, path=history_path) - history = json.loads(history_path.read_text(encoding="utf-8")) - - assert aggregate["corpus"]["count"] == 2 - assert aggregate["environment"]["native_corpus_count"] == 3 - assert aggregate["terminal_union_count"] == 2 - assert [lane["lane"] for lane in aggregate["lanes"]] == ["parallel", "serial"] - assert aggregate["complete_corpus_covered"] is True - assert aggregate["terminal_green"] is True - assert aggregate["deadline"] == {"budget_s": 3600, "met": True} - assert durable["pytest_aggregate"] == aggregate - assert history["pytest_aggregate"] == aggregate + test_file.write_text(test_file.read_text(encoding="utf-8").replace("'v1'", "'v2'"), encoding="utf-8") + plugin_mutated = prepare_native_testmon_environment(repo) + assert plugin_mutated.selection_mode == "bootstrap" + assert plugin_mutated.environment_name != plugin_changed.environment_name + assert [ + result.completed.returncode + for result in _run_plain_verify_corpus(repo, mode="bootstrap", environment_name=plugin_mutated.environment_name) + ] == [0, 0] diff --git a/tests/unit/devtools/test_durable_schema_policy_gate.py b/tests/unit/devtools/test_durable_schema_policy_gate.py index 8e1388cbca..1415ceb58b 100644 --- a/tests/unit/devtools/test_durable_schema_policy_gate.py +++ b/tests/unit/devtools/test_durable_schema_policy_gate.py @@ -94,7 +94,7 @@ def test_schema_versioning_policy_runs_exactly_once_in_every_noncommit_fast_gate for label, _command in verify.build_verify_steps( quick=quick, lab=lab, - skip_slow=True, + testmon_environment="env-digest" if not quick else "", ) ] assert labels.count("lab policy schema-versioning") == 1 @@ -104,7 +104,6 @@ def test_schema_versioning_policy_runs_exactly_once_in_every_noncommit_fast_gate for label, _command in verify.build_verify_steps( quick=True, lab=False, - skip_slow=True, commit=True, ) ] diff --git a/tests/unit/devtools/test_merge_boundary.py b/tests/unit/devtools/test_merge_boundary.py index a906cc16db..f303c69f2e 100644 --- a/tests/unit/devtools/test_merge_boundary.py +++ b/tests/unit/devtools/test_merge_boundary.py @@ -137,7 +137,6 @@ def _run(cmd: list[str], **kwargs: Any) -> MagicMock: "exit_code": local_exit, "verification_scope": "affected", "release_baseline_allowed": False, - "terminal_authorization": None, } ) ) @@ -156,7 +155,6 @@ def _write_terminal_receipt( head: str = "merged-master", scope: str = "release-baseline", release_allowed: bool = True, - terminal_authorization: str | None = None, ) -> None: env = kwargs["env"] assert isinstance(env, dict) @@ -170,7 +168,6 @@ def _write_terminal_receipt( "exit_code": 0, "verification_scope": scope, "release_baseline_allowed": release_allowed, - "terminal_authorization": terminal_authorization, } ) ) @@ -1011,47 +1008,6 @@ def test_record_full_verify_rejects_success_without_structured_release_permissio assert merge_boundary.cmd_train_status(as_json=False) == 1 -def test_record_full_verify_rejects_skip_slow_without_typed_authorization( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - monkeypatch.chdir(tmp_path) - merge_boundary._append_merge_entry(1, "sha1", "some title") - monkeypatch.setattr( - subprocess, - "run", - lambda _cmd, **_kwargs: MagicMock( - returncode=0, - stdout=json.dumps({"verification_scope": "narrow-terminal", "release_baseline_allowed": False}), - stderr="", - ), - ) - - assert merge_boundary.cmd_record_full_verify("devtools verify --all --skip-slow", target_sha="merged-master") == 1 - assert merge_boundary._read_ledger()["last_full_verify"]["accepted"] is False - assert merge_boundary.cmd_train_status(as_json=False) == 1 - - -def test_record_full_verify_accepts_explicit_typed_narrow_terminal_authorization( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - monkeypatch.chdir(tmp_path) - merge_boundary._append_merge_entry(1, "sha1", "some title") - - def run(_cmd: list[str], **kwargs: Any) -> MagicMock: - _write_terminal_receipt( - kwargs, - scope="narrow-terminal", - terminal_authorization="narrow-terminal", - ) - return MagicMock(returncode=0, stdout="", stderr="") - - monkeypatch.setattr(subprocess, "run", run) - - assert merge_boundary.cmd_record_full_verify("devtools verify --all --skip-slow", target_sha="merged-master") == 0 - assert merge_boundary._read_ledger()["last_full_verify"]["accepted"] is True - assert merge_boundary.cmd_train_status(as_json=False) == 0 - - def test_record_full_verify_rejects_untyped_scope_even_when_permission_is_true( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: diff --git a/tests/unit/devtools/test_merge_gate.py b/tests/unit/devtools/test_merge_gate.py index 687191127b..86c29526a1 100644 --- a/tests/unit/devtools/test_merge_gate.py +++ b/tests/unit/devtools/test_merge_gate.py @@ -126,7 +126,6 @@ def _run(cmd: list[str], **kwargs: object) -> MagicMock: "exit_code": local_exit, "verification_scope": "affected", "release_baseline_allowed": False, - "terminal_authorization": None, } ) ) @@ -227,7 +226,7 @@ def test_record_persists_receipt_keyed_to_current_head_sha(monkeypatch: pytest.M assert receipt["exit_code"] == 0 -@pytest.mark.parametrize("command", ["devtools verify", "devtools verify --lab", "devtools verify --json --skip-slow"]) +@pytest.mark.parametrize("command", ["devtools verify", "devtools verify --lab", "devtools verify --json"]) def test_check_accepts_affected_receipt_without_release_baseline_permission( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, command: str ) -> None: @@ -297,7 +296,6 @@ def _run(cmd: list[str], **kwargs: object) -> MagicMock: "exit_code": 0, "verification_scope": "release-baseline", "release_baseline_allowed": True, - "terminal_authorization": None, } ), stderr="", @@ -317,9 +315,8 @@ def test_record_consumes_receipt_after_streamed_verifier_progress( pr_view: dict[str, object] = {"headRefOid": "abc123", "headRefName": "feature/x"} base = cast(Callable[..., MagicMock], _fake_run(pr_view, [], local_head_sha="abc123")) payload = { - "verification_scope": "narrow-terminal", + "verification_scope": "affected", "release_baseline_allowed": False, - "terminal_authorization": "narrow-terminal", } def _run(cmd: list[str], **kwargs: object) -> MagicMock: @@ -344,9 +341,8 @@ def _run(cmd: list[str], **kwargs: object) -> MagicMock: monkeypatch.setattr(subprocess, "run", _run) assert merge_gate.cmd_record(42, "devtools test tests/unit/foo.py") == 0 receipt = json.loads(merge_gate._receipt_path(42).read_text()) - assert receipt["verification_scope"] == "narrow-terminal" + assert receipt["verification_scope"] == "affected" assert receipt["release_baseline_allowed"] is False - assert receipt["terminal_authorization"] == "narrow-terminal" def test_record_consumes_exact_invocation_receipt_when_verifier_writes_only_stderr( @@ -373,7 +369,6 @@ def _run(cmd: list[str], **kwargs: object) -> MagicMock: "exit_code": 0, "verification_scope": "affected", "release_baseline_allowed": False, - "terminal_authorization": None, } ) ) diff --git a/tests/unit/devtools/test_project_motd.py b/tests/unit/devtools/test_project_motd.py index 44b821b051..d602c88843 100644 --- a/tests/unit/devtools/test_project_motd.py +++ b/tests/unit/devtools/test_project_motd.py @@ -37,7 +37,7 @@ def test_render_motd_contains_expected_sections( assert ( "ready devtools render all --check · devtools verify --quick · devtools release build-package" in rendered ) - assert "test pytest -q --ignore=tests/integration" in rendered + assert "test devtools verify" in rendered assert "roots keep .venv/ .direnv/ · cache .cache/ · outputs .local/ · build .local/result" in rendered assert "dirty · 1 staged · 2 modified · 3 untracked" in rendered diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 3a4bf9187e..2fc36186be 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -20,7 +20,7 @@ from devtools import run_tests, verify, verify_runs from devtools.testmon_bootstrap import executable_python_paths -from devtools.verification_contracts import TerminalAuthorization, VerificationScope +from devtools.verification_contracts import VerificationScope from devtools.verify import ( PYTEST_CONTAINMENT_PATH, PYTEST_EVENTS_PATH, @@ -86,7 +86,7 @@ def _pytest_marker_expr(command: list[str]) -> str: def test_quick_verify_omits_pytest() -> None: - steps = build_verify_steps(quick=True, lab=False, skip_slow=False) + steps = build_verify_steps(quick=True, lab=False) labels = [label for label, _command in steps] assert labels == [ @@ -117,7 +117,6 @@ def test_native_testmon_uses_exactly_two_semantic_lanes( steps = build_verify_steps( quick=False, lab=False, - skip_slow=False, testmon_mode=mode, testmon_environment="env-digest", ) @@ -141,29 +140,23 @@ def test_native_testmon_uses_exactly_two_semantic_lanes( assert command[command.index("-p") + 1] == "devtools.pytest_progress_plugin" -def test_native_marker_policy_composes_slow_and_scale_tiers() -> None: - default_steps = build_verify_steps( - quick=False, - lab=False, - skip_slow=True, - testmon_environment="env-digest", - ) - lab_steps = build_verify_steps( +def test_native_marker_policy_only_excludes_benchmarks() -> None: + complete_steps = build_verify_steps( quick=False, lab=True, - skip_slow=False, testmon_environment="env-digest", ) - default_expr = _pytest_marker_expr(next(command for label, command in default_steps if "parallel" in label)) - lab_expr = _pytest_marker_expr(next(command for label, command in lab_steps if "parallel" in label)) - assert all(term in default_expr for term in ("not benchmark", "not slow", "not scale_medium", "not scale_large")) - assert "not scale_medium" not in lab_expr - assert "not scale_large" in lab_expr + complete_command = next(command for label, command in complete_steps if "parallel" in label) + complete_expr = _pytest_marker_expr(complete_command) + assert "not benchmark" in complete_expr + assert "not slow" not in complete_expr + assert "--ignore=tests/benchmarks" in complete_command + assert "--ignore=tests/integration" not in complete_command def test_lab_verify_delegates_to_lab_smoke() -> None: - steps = build_verify_steps(quick=True, lab=True, skip_slow=False) + steps = build_verify_steps(quick=True, lab=True) labels = [label for label, _command in steps] assert "lab smoke" in labels @@ -3527,7 +3520,7 @@ def fake_run(label: str, command: list[str], **kwargs: object) -> tuple[int, flo rc = main(["--quick", "--json"]) assert rc == 1 - assert calls == [label for label, _command in build_verify_steps(quick=True, lab=False, skip_slow=False)] + assert calls == [label for label, _command in build_verify_steps(quick=True, lab=False)] payload = json.loads(capsys.readouterr().out) assert payload["exit_code"] == 1 assert payload["verification_scope"] == "non-test" @@ -3843,7 +3836,6 @@ def test_release_authority_requires_current_complete_green_invocation() -> None: assert _release_baseline_allowed( selection_mode="bootstrap", verification_scope=VerificationScope.RELEASE_BASELINE, - terminal_authorization=None, exit_code=0, checkout_stable=True, aggregate=aggregate, @@ -3851,23 +3843,6 @@ def test_release_authority_requires_current_complete_green_invocation() -> None: assert not _release_baseline_allowed( selection_mode="affected", verification_scope=VerificationScope.AFFECTED, - terminal_authorization=None, - exit_code=0, - checkout_stable=True, - aggregate=aggregate, - ) - assert not _release_baseline_allowed( - selection_mode="full", - verification_scope=VerificationScope.NARROW_TERMINAL, - terminal_authorization=None, - exit_code=0, - checkout_stable=True, - aggregate=aggregate, - ) - assert _release_baseline_allowed( - selection_mode="full", - verification_scope=VerificationScope.NARROW_TERMINAL, - terminal_authorization=TerminalAuthorization.NARROW_TERMINAL.value, exit_code=0, checkout_stable=True, aggregate=aggregate, @@ -3882,75 +3857,12 @@ def test_release_authority_requires_current_complete_green_invocation() -> None: assert not _release_baseline_allowed( selection_mode="full", verification_scope=VerificationScope.RELEASE_BASELINE, - terminal_authorization=None, exit_code=0, checkout_stable=True, aggregate=broken, ) -@pytest.mark.parametrize( - ("authorization", "expected_release"), - [(None, False), ("narrow-terminal", True)], -) -def test_skip_slow_command_requires_typed_narrow_terminal_authorization( - authorization: str | None, - expected_release: bool, - capsys: pytest.CaptureFixture[str], -) -> None: - class _StableMonitor: - def __init__(self, _root: Path) -> None: - pass - - def start(self) -> None: - pass - - def finish(self) -> CheckoutMutationObservation: - return CheckoutMutationObservation(changed=False, unavailable=False) - - preparation = SimpleNamespace( - environment_name="env", - selection_mode="bootstrap", - removed_paths=(), - copied_from=None, - ) - native_state = SimpleNamespace( - valid=True, - status="valid", - reason="current", - environment=SimpleNamespace(nodeids=("tests/test_owner.py::test_owner",)), - missing_executable_paths=(), - ) - aggregate = { - "complete_corpus_covered": True, - "terminal_green": True, - "cleanup": {"complete": True}, - "containment": {"complete": True}, - "deadline": {"met": True}, - } - argv = ["--all", "--skip-slow", "--json"] - if authorization is not None: - argv.extend(("--terminal-authorization", authorization)) - - with ( - patch("devtools.verify._git_head", return_value="head"), - patch("devtools.verify._git_commit", return_value="base"), - patch("devtools.verify._changed_test_relevant_paths", return_value=()), - patch("devtools.verify.prepare_native_testmon_environment", return_value=preparation), - patch("devtools.verify._native_environment_after_run", return_value=native_state), - patch("devtools.verify.build_verify_steps", return_value=[]), - patch("devtools.verify.aggregate_native_testmon_run", return_value=aggregate), - patch("devtools.verify.CheckoutMutationMonitor", _StableMonitor), - patch("devtools.verify.worktree_fingerprint", return_value="stable"), - patch("devtools.verify._save_history"), - patch("devtools.verify._stamp_head"), - patch("devtools.verify._notify"), - ): - assert main(argv) == 0 - - assert json.loads(capsys.readouterr().out)["release_baseline_allowed"] is expected_release - - def test_collection_failure_still_persists_native_run_aggregate( capsys: pytest.CaptureFixture[str], ) -> None: diff --git a/tests/unit/infra/test_archive_templates.py b/tests/unit/infra/test_archive_templates.py index 60091c87ad..521aab7985 100644 --- a/tests/unit/infra/test_archive_templates.py +++ b/tests/unit/infra/test_archive_templates.py @@ -9,10 +9,51 @@ import subprocess import sys from pathlib import Path +from unittest.mock import patch from tests.infra.archive_templates import clone_archive_template, finalize_archive_template +def test_clone_fallback_is_private_writable_and_symlink_safe(tmp_path: Path) -> None: + """The portable fallback preserves links and thaws only the private clone.""" + template = tmp_path / "template" + template.mkdir() + source_file = template / "source.db" + with contextlib.closing(sqlite3.connect(source_file)) as conn, conn: + conn.execute("CREATE TABLE entries (value TEXT)") + conn.execute("INSERT INTO entries VALUES ('immutable-template')") + (template / "source-link.db").symlink_to(source_file.name) + finalize_archive_template(template) + source_bytes = source_file.read_bytes() + + clone = tmp_path / "clone" + with patch("tests.infra.archive_templates.subprocess.run", side_effect=OSError("cp unavailable")): + clone_archive_template(template, clone) + + assert clone.joinpath("source-link.db").is_symlink() + assert clone.joinpath("source.db").stat().st_mode & stat.S_IWUSR + clone.joinpath("source.db").write_bytes(b"private-mutation") + assert source_file.read_bytes() == source_bytes + + +def test_clone_rebinds_durable_bootstrap_identity(tmp_path: Path) -> None: + from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore + + template = tmp_path / "template" + clone = tmp_path / "clone" + marker = Path(".maintenance-state/durable-change-trains/.bootstrap") + with ArchiveStore(template): + pass + source_identity = template.joinpath(marker).read_bytes() + + clone_archive_template(template, clone) + + assert clone.joinpath(marker).read_bytes() != source_identity + with ArchiveStore(clone): + pass + assert template.joinpath(marker).read_bytes() == source_identity + + def _leave_crash_recovered_wal(database: Path) -> None: writer = subprocess.Popen( [ diff --git a/tests/unit/storage/test_perf_rescue_1314.py b/tests/unit/storage/test_perf_rescue_1314.py index efef2e6900..dc208d8e6f 100644 --- a/tests/unit/storage/test_perf_rescue_1314.py +++ b/tests/unit/storage/test_perf_rescue_1314.py @@ -25,13 +25,14 @@ from typing import Any import aiosqlite -import pytest from polylogue.storage.insights.session.rebuild import _SESSION_INSIGHT_REBUILD_PAGE_SIZE from polylogue.storage.sqlite.queries.sessions_search import search_session_hits from polylogue.storage.sqlite.queries.stats import get_origin_metrics_rows from tests.benchmarks.helpers import open_bench_store +pytest_plugins = ("tests.benchmarks.conftest",) + @contextmanager def _capture_aiosqlite_sql() -> Iterator[list[str]]: @@ -73,12 +74,11 @@ def test_session_insight_rebuild_page_size_is_at_least_50() -> None: # --------------------------------------------------------------------------- -@pytest.mark.scale_small -def test_search_session_hits_uses_freshness_ledger_before_match(tier_small_db: Path) -> None: +def test_search_session_hits_uses_freshness_ledger_before_match(bench_db_1k: Path) -> None: """Search should not pay archive-scale COUNT(*) probes after daemon readiness.""" from polylogue.storage.fts.freshness import READY, record_fts_surface_state_async - with open_bench_store(tier_small_db) as store: + with open_bench_store(bench_db_1k) as store: backend = store.backend async def _run(statements: list[str]) -> None: @@ -102,10 +102,9 @@ async def _run(statements: list[str]) -> None: assert all("count(*) from messages where text is not null" not in sql for sql in lowered[:match_index]) -@pytest.mark.scale_small -def test_search_session_hits_falls_back_to_exact_freshness(tier_small_db: Path) -> None: +def test_search_session_hits_falls_back_to_exact_freshness(bench_db_1k: Path) -> None: """Absent ledger rows fall back to exact FTS verification before MATCH.""" - with open_bench_store(tier_small_db) as store: + with open_bench_store(bench_db_1k) as store: backend = store.backend async def _run(statements: list[str]) -> None: @@ -132,12 +131,11 @@ async def _run(statements: list[str]) -> None: # --------------------------------------------------------------------------- -@pytest.mark.scale_small -def test_origin_metrics_reads_sessions_aggregates(tier_small_db: Path) -> None: +def test_origin_metrics_reads_sessions_aggregates(bench_db_1k: Path) -> None: """Origin metrics must source the per-session pre-aggregates from ``sessions`` rather than scanning ``messages``. """ - with open_bench_store(tier_small_db) as store: + with open_bench_store(bench_db_1k) as store: backend = store.backend async def _run() -> list[dict[str, object]]: @@ -148,7 +146,7 @@ async def _run() -> list[dict[str, object]]: with _capture_aiosqlite_sql() as statements: rows = store.run(_run()) - assert rows, "scale_small fixture should produce at least one origin row" + assert rows, "1k-message benchmark fixture should produce at least one origin row" joined = "\n".join(statements).lower() assert "from sessions" in joined assert "session_stats" not in joined @@ -177,8 +175,7 @@ async def _run() -> list[dict[str, object]]: # --------------------------------------------------------------------------- -@pytest.mark.scale_small -def test_get_messages_hydration_does_not_call_model_copy(tier_small_db: Path) -> None: +def test_get_messages_hydration_does_not_call_model_copy(bench_db_1k: Path) -> None: """``get_messages`` must mutate the freshly-constructed MessageRecord instances in place rather than calling pydantic's ``model_copy``. @@ -197,7 +194,7 @@ def _spy(self: Any, *args: Any, **kwargs: Any) -> Any: MessageRecord.model_copy = _spy # type: ignore[method-assign] try: - with open_bench_store(tier_small_db) as store: + with open_bench_store(bench_db_1k) as store: async def _run() -> int: summaries = await store.repository.list_summaries(limit=5) diff --git a/tests/unit/storage/test_store_ops.py b/tests/unit/storage/test_store_ops.py index 6d905b2875..2715f69f34 100644 --- a/tests/unit/storage/test_store_ops.py +++ b/tests/unit/storage/test_store_ops.py @@ -1790,12 +1790,12 @@ async def test_tag_assignment_roundtrip_and_counts( empty_archive_template: Path, ) -> None: """Strategy-generated tags are retrievable and counted consistently.""" - from tests.conftest import _clone_archive_template from tests.infra.archive_scenarios import archive_for_scenario_db, native_session_id_for + from tests.infra.archive_templates import clone_archive_template with tempfile.TemporaryDirectory() as tmp_dir: archive_root = Path(tmp_dir) / "archive" - _clone_archive_template(empty_archive_template, archive_root) + clone_archive_template(empty_archive_template, archive_root) db_path = archive_root / "index.db" seed_session_graph(db_path, spec.sessions) diff --git a/tests/unit/test_pytest_temp_policy.py b/tests/unit/test_pytest_temp_policy.py index 9b4242851e..261b1679e0 100644 --- a/tests/unit/test_pytest_temp_policy.py +++ b/tests/unit/test_pytest_temp_policy.py @@ -556,7 +556,7 @@ def test_explicit_basetemp_claim_survives_real_pytest_basetemp_replacement(tmp_p "pytest", "--basetemp", str(explicit), - "tests/unit/test_pytest_temp_policy.py::test_archive_template_clone_is_private", + "tests/unit/infra/test_archive_templates.py::test_clone_fallback_is_private_writable_and_symlink_safe", ], cwd=repo_root, env=env, @@ -921,33 +921,3 @@ def test_sessionfinish_retains_explicit_diagnostic_basetemp( conftest.pytest_sessionfinish(cast("pytest.Session", session), 1) assert explicit.exists() - - -def test_archive_template_clone_is_private(tmp_path: Path) -> None: - source = tmp_path / "source" - destination = tmp_path / "destination" - source.mkdir() - (source / "index.db").write_bytes(b"immutable-template") - - conftest._clone_archive_template(source, destination) - (destination / "index.db").write_bytes(b"private-mutation") - - assert (source / "index.db").read_bytes() == b"immutable-template" - - -def test_archive_template_clone_rebinds_durable_identity(tmp_path: Path) -> None: - from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore - - source = tmp_path / "source" - destination = tmp_path / "destination" - marker_relative = Path(".maintenance-state/durable-change-trains/.bootstrap") - with ArchiveStore(source): - pass - source_marker = (source / marker_relative).read_bytes() - - conftest._clone_archive_template(source, destination) - - assert (destination / marker_relative).read_bytes() != source_marker - with ArchiveStore(destination): - pass - assert (source / marker_relative).read_bytes() == source_marker From 906dc6508194ae69c5e173df27c156027acc8111 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 20:40:10 +0200 Subject: [PATCH 06/31] fix(test): close native lifecycle authority gaps --- devtools/testmon_bootstrap.py | 133 ++++++++-- devtools/verify.py | 232 ++++++++++++++---- devtools/verify_runs.py | 27 +- .../devtools/test_native_testmon_lifecycle.py | 4 +- tests/unit/devtools/test_testmon_bootstrap.py | 20 ++ tests/unit/devtools/test_verify.py | 130 +++++++++- 6 files changed, 478 insertions(+), 68 deletions(-) diff --git a/devtools/testmon_bootstrap.py b/devtools/testmon_bootstrap.py index 5878cc1fa4..5ef4eda6da 100644 --- a/devtools/testmon_bootstrap.py +++ b/devtools/testmon_bootstrap.py @@ -30,6 +30,7 @@ import stat import subprocess import sys +import time import uuid from collections.abc import Iterable, Sequence from dataclasses import dataclass @@ -99,32 +100,72 @@ class NativeTestmonRepairError(RuntimeError): """The exact derived testmon state could not be repaired safely.""" -def _fingerprint_inputs(root: Path, relative_paths: Sequence[str]) -> str: +class NativeTestmonDeadlineError(NativeTestmonRepairError): + """The verify invocation deadline expired during native-state preparation.""" + + +def _ensure_deadline(deadline_monotonic: float | None) -> None: + if deadline_monotonic is not None and time.monotonic() >= deadline_monotonic: + raise NativeTestmonDeadlineError("verify invocation deadline expired during native testmon preparation") + + +def _remaining_timeout(deadline_monotonic: float | None, maximum: float) -> float: + _ensure_deadline(deadline_monotonic) + if deadline_monotonic is None: + return maximum + return max(0.001, min(maximum, deadline_monotonic - time.monotonic())) + + +def _fingerprint_inputs( + root: Path, + relative_paths: Sequence[str], + *, + deadline_monotonic: float | None = None, +) -> str: digest = hashlib.sha256() for relative in relative_paths: + _ensure_deadline(deadline_monotonic) digest.update(relative.encode()) digest.update(b"\0") try: - contents = (root / relative).read_bytes() + with (root / relative).open("rb") as handle: + while chunk := handle.read(1024 * 1024): + digest.update(chunk) + _ensure_deadline(deadline_monotonic) except OSError: digest.update(b"missing") - else: - digest.update(contents) digest.update(b"\0") return digest.hexdigest() +def _active_local_pytest_plugin_paths(root: Path) -> set[str]: + """Resolve explicitly active local pytest plugins regardless of filename.""" + paths: set[str] = set() + for raw_name in os.environ.get("PYTEST_PLUGINS", "").split(","): + module_name = raw_name.strip() + if not module_name or any(part in {"", ".", ".."} for part in module_name.split(".")): + continue + module_path = Path(*module_name.split(".")) + module_file = root / module_path.with_suffix(".py") + if module_file.is_file(): + paths.add(module_file.relative_to(root).as_posix()) + package = root / module_path + if (package / "__init__.py").is_file(): + paths.update(path.relative_to(root).as_posix() for path in package.rglob("*.py") if path.is_file()) + return paths + + def _environment_input_paths(root: Path) -> tuple[str, ...]: """Discover collection and managed-pytest harness inputs.""" paths = set(_ENVIRONMENT_INPUTS) patterns = ( "devtools/pytest*.py", - "pytest*.py", "tests/**/conftest.py", "tests/infra/**/*.py", ) for pattern in patterns: paths.update(path.relative_to(root).as_posix() for path in root.glob(pattern) if path.is_file()) + paths.update(_active_local_pytest_plugin_paths(root)) return tuple(sorted(paths)) @@ -139,9 +180,15 @@ def _installed_distributions() -> tuple[tuple[str, str], ...]: return tuple(sorted(distributions)) -def testmon_environment_digest(repo_root: Path, *, pytest_profile: str = "default") -> str: +def testmon_environment_digest( + repo_root: Path, + *, + pytest_profile: str = "default", + deadline_monotonic: float | None = None, +) -> str: """Return the native testmon environment name for collection semantics.""" root = repo_root.resolve() + _ensure_deadline(deadline_monotonic) payload = { "protocol": 1, "python": { @@ -152,7 +199,11 @@ def testmon_environment_digest(repo_root: Path, *, pytest_profile: str = "defaul "platform": platform.platform(), }, "distributions": _installed_distributions(), - "inputs": _fingerprint_inputs(root, _environment_input_paths(root)), + "inputs": _fingerprint_inputs( + root, + _environment_input_paths(root), + deadline_monotonic=deadline_monotonic, + ), "pytest_environment": {key: os.environ.get(key) for key in _PYTEST_ENVIRONMENT_KEYS}, "pytest_profile": pytest_profile, } @@ -258,8 +309,10 @@ def inspect_native_testmon_environment( *, environment_name: str, required_executable_paths: Sequence[str] = (), + deadline_monotonic: float | None = None, ) -> NativeTestmonState: """Validate one native environment without interpreting plugin internals.""" + _ensure_deadline(deadline_monotonic) sidecars = tuple(Path(f"{data_path}{suffix}") for suffix in TESTMON_SIDECAR_SUFFIXES) if not data_path.exists(): if any(path.exists() or path.is_symlink() for path in sidecars): @@ -272,8 +325,15 @@ def inspect_native_testmon_environment( if not stat.S_ISREG(mode): return NativeTestmonState("invalid", "native testmon database is not a regular file") try: - with sqlite3.connect(_readonly_uri(data_path), uri=True, timeout=10) as connection: + with sqlite3.connect( + _readonly_uri(data_path), + uri=True, + timeout=_remaining_timeout(deadline_monotonic, 10), + ) as connection: + if deadline_monotonic is not None: + connection.set_progress_handler(lambda: int(time.monotonic() >= deadline_monotonic), 1_000) quick_check = connection.execute("PRAGMA quick_check").fetchone() + _ensure_deadline(deadline_monotonic) if quick_check is None or quick_check[0] != "ok": return NativeTestmonState("invalid", "SQLite quick_check failed") version_row = connection.execute("PRAGMA user_version").fetchone() @@ -283,6 +343,7 @@ def inspect_native_testmon_environment( "SELECT id FROM environment WHERE environment_name = ? ORDER BY id DESC", (environment_name,), ).fetchall() + _ensure_deadline(deadline_monotonic) if len(environment_rows) != 1: reason = "native environment is absent" if not environment_rows else "native environment is ambiguous" return NativeTestmonState("invalid", reason) @@ -295,6 +356,7 @@ def inspect_native_testmon_environment( ).fetchall() if isinstance(row[0], str) and row[0] ) + _ensure_deadline(deadline_monotonic) if not nodeids or len(nodeids) != len(set(nodeids)): return NativeTestmonState("invalid", "native environment has no unique collected corpus") uncovered = connection.execute( @@ -306,6 +368,7 @@ def inspect_native_testmon_environment( """, (environment_id,), ).fetchone() + _ensure_deadline(deadline_monotonic) if uncovered is None or int(uncovered[0]) != 0: return NativeTestmonState("invalid", "native environment has tests without dependency placeholders") raw_files = connection.execute( @@ -318,7 +381,11 @@ def inspect_native_testmon_environment( """, (environment_id,), ).fetchall() + _ensure_deadline(deadline_monotonic) + except NativeTestmonDeadlineError: + raise except (NativeTestmonRepairError, OSError, sqlite3.Error, TypeError, ValueError) as exc: + _ensure_deadline(deadline_monotonic) return NativeTestmonState("invalid", f"native testmon database is unreadable: {exc}") fingerprinted = frozenset( relative @@ -384,19 +451,32 @@ def _atomic_copy_sqlite_database( *, environment_name: str, required_executable_paths: Sequence[str], + deadline_monotonic: float | None, ) -> None: + _ensure_deadline(deadline_monotonic) destination.parent.mkdir(parents=True, exist_ok=True) temporary = destination.with_name(f".{destination.name}.copy-{os.getpid()}-{uuid.uuid4().hex}.tmp") try: with ( - sqlite3.connect(_readonly_uri(source), uri=True, timeout=60) as source_connection, - sqlite3.connect(temporary, timeout=60) as destination_connection, + sqlite3.connect( + _readonly_uri(source), + uri=True, + timeout=_remaining_timeout(deadline_monotonic, 60), + ) as source_connection, + sqlite3.connect(temporary, timeout=_remaining_timeout(deadline_monotonic, 60)) as destination_connection, ): - source_connection.backup(destination_connection) + source_connection.backup( + destination_connection, + pages=256, + progress=lambda _status, _remaining, _total: _ensure_deadline(deadline_monotonic), + sleep=0.05, + ) + _ensure_deadline(deadline_monotonic) copied = inspect_native_testmon_environment( temporary, environment_name=environment_name, required_executable_paths=required_executable_paths, + deadline_monotonic=deadline_monotonic, ) if not copied.valid: raise NativeTestmonRepairError(f"copied main-checkout database failed validation: {copied.reason}") @@ -407,6 +487,9 @@ def _atomic_copy_sqlite_database( os.close(descriptor) os.replace(temporary, destination) _fsync_directory(destination.parent) + _ensure_deadline(deadline_monotonic) + except NativeTestmonDeadlineError: + raise except (OSError, sqlite3.Error) as exc: raise NativeTestmonRepairError(f"SQLite online backup failed: {exc}") from exc finally: @@ -417,17 +500,24 @@ def _atomic_copy_sqlite_database( Path(f"{temporary}{suffix}").unlink() -def linked_worktree_info(repo_root: Path) -> tuple[bool, Path] | None: +def linked_worktree_info( + repo_root: Path, + *, + deadline_monotonic: float | None = None, +) -> tuple[bool, Path] | None: """Return linked-worktree status and the main checkout path.""" try: result = subprocess.run( ["git", "-C", str(repo_root), "rev-parse", "--absolute-git-dir", "--git-common-dir"], capture_output=True, text=True, - timeout=10, + timeout=_remaining_timeout(deadline_monotonic, 10), env={**os.environ, "GIT_OPTIONAL_LOCKS": "0"}, ) - except (OSError, subprocess.TimeoutExpired): + except subprocess.TimeoutExpired: + _ensure_deadline(deadline_monotonic) + return None + except OSError: return None if result.returncode != 0: return None @@ -445,23 +535,30 @@ def prepare_native_testmon_environment( *, required_executable_paths: Sequence[str] = (), pytest_profile: str = "default", + deadline_monotonic: float | None = None, ) -> NativeTestmonPreparation: """Repair derived local state and optionally reuse a matching main graph.""" root = repo_root.resolve() - environment_name = testmon_environment_digest(root, pytest_profile=pytest_profile) + environment_name = testmon_environment_digest( + root, + pytest_profile=pytest_profile, + deadline_monotonic=deadline_monotonic, + ) local_data = root / TESTMON_DATA_RELPATH local = inspect_native_testmon_environment( local_data, environment_name=environment_name, required_executable_paths=required_executable_paths, + deadline_monotonic=deadline_monotonic, ) - info = linked_worktree_info(root) + info = linked_worktree_info(root, deadline_monotonic=deadline_monotonic) linked = bool(info and info[0]) main_checkout = info[1] if linked and info is not None else None if local.valid: return NativeTestmonPreparation(environment_name, "affected", local, None, (), linked, main_checkout) removed = remove_invalid_native_testmon_state(root) + _ensure_deadline(deadline_monotonic) copied_from: Path | None = None if main_checkout is not None and main_checkout != root: main_data = main_checkout / TESTMON_DATA_RELPATH @@ -469,6 +566,7 @@ def prepare_native_testmon_environment( main_data, environment_name=environment_name, required_executable_paths=required_executable_paths, + deadline_monotonic=deadline_monotonic, ) if main.valid: _atomic_copy_sqlite_database( @@ -476,12 +574,14 @@ def prepare_native_testmon_environment( local_data, environment_name=environment_name, required_executable_paths=required_executable_paths, + deadline_monotonic=deadline_monotonic, ) copied_from = main_data local = inspect_native_testmon_environment( local_data, environment_name=environment_name, required_executable_paths=required_executable_paths, + deadline_monotonic=deadline_monotonic, ) if not local.valid: raise NativeTestmonRepairError(f"published native testmon copy is invalid: {local.reason}") @@ -501,6 +601,7 @@ def prepare_native_testmon_environment( __all__ = [ "ASTClassification", "NativeTestmonEnvironment", + "NativeTestmonDeadlineError", "NativeTestmonPreparation", "NativeTestmonRepairError", "NativeTestmonState", diff --git a/devtools/verify.py b/devtools/verify.py index c10b9dc39d..bde90cecb2 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -58,6 +58,7 @@ write_termination_request, ) from devtools.testmon_bootstrap import ( + NativeTestmonDeadlineError, NativeTestmonPreparation, NativeTestmonRepairError, NativeTestmonState, @@ -2156,12 +2157,6 @@ def build_verify_steps( "-p", "devtools.pytest_progress_plugin", ] - # Benchmark cases are an explicit campaign surface, not part of the - # correctness corpus. Keeping them out here is important: a - # benchmark marker is not necessarily present, and the serial lane - # would otherwise spend minutes executing a performance probe before - # it can checkpoint any correctness nodes. - base_marker = "not benchmark" if testmon_mode not in {"affected", "bootstrap", "full"}: raise ValueError(f"unknown native testmon mode: {testmon_mode}") if not testmon_environment: @@ -2175,7 +2170,7 @@ def build_verify_steps( parallel_cmd = [ *pytest_cmd, "-m", - f"({base_marker}) and not load_sensitive and not tui", + "not load_sensitive and not tui", *native_args, *_pytest_worker_args(), ] @@ -2192,7 +2187,7 @@ def _serial_report_arg(arg: str) -> str: serial_cmd.extend( [ "-m", - f"({base_marker}) and (load_sensitive or tui)", + "load_sensitive or tui", *native_args, "-p", "no:randomly", @@ -2435,6 +2430,97 @@ def _release_baseline_allowed( ) +def _finalize_preflight_failure( + run: VerifyRun, + *, + started_at: float, + tier: str, + head: str | None, + verification_scope: VerificationScope, + diagnosis: str, + exit_code: int, + message: str, + use_json: bool, + mutation_monitor: CheckoutMutationMonitor | None = None, + initial_worktree_fingerprint: str | None = None, +) -> int: + """Persist one normalized failed invocation before pytest can start.""" + final_head = _git_head() + try: + final_worktree_fingerprint = worktree_fingerprint(ROOT) if mutation_monitor is not None else "unavailable" + except Exception: + final_worktree_fingerprint = "unavailable" + mutation_observation = finish_checkout_mutation_monitor(mutation_monitor) if mutation_monitor is not None else None + checkout_diagnosis: str | None = None + if mutation_monitor is None: + checkout_diagnosis = "preflight_failed_before_checkout_monitor" + elif ( + head is None + or final_head is None + or initial_worktree_fingerprint in {None, "unavailable"} + or final_worktree_fingerprint == "unavailable" + or mutation_observation is None + or mutation_observation.unavailable + ): + checkout_diagnosis = "checkout_fingerprint_unavailable" + elif ( + final_head != head or mutation_observation.changed or final_worktree_fingerprint != initial_worktree_fingerprint + ): + checkout_diagnosis = "checkout_changed_during_verification" + + duration_s = round(time.monotonic() - started_at, 2) + artifacts = run.start_step(label="verify preflight", cmd=[]) + step = run.finish_step( + step_id=artifacts.step_id, + result={ + "duration_s": duration_s, + "exit": exit_code, + "diagnosis": diagnosis, + "error": message, + "checkout_diagnosis": checkout_diagnosis, + }, + ) + payload = run.finish( + exit_code=exit_code, + duration_s=duration_s, + diagnosis=diagnosis, + verification_scope=verification_scope.value, + release_baseline_allowed=False, + final_worktree_fingerprint=final_worktree_fingerprint, + checkout_mutation_path=(mutation_observation.observed_path if mutation_observation is not None else None), + checkout_diagnosis=checkout_diagnosis, + ) + history_entry = { + **payload, + "timestamp": datetime.now(timezone.utc).isoformat(), + "git_head": head, + "final_git_head": final_head, + "tier": tier, + "checkout_root": str(ROOT.resolve()), + "worktree_fingerprint": initial_worktree_fingerprint, + "final_worktree_fingerprint": final_worktree_fingerprint, + "steps": [step] if step is not None else [], + "total_duration_s": duration_s, + "invocation_budget_s": VERIFY_INVOCATION_BUDGET_S, + "exit_code": exit_code, + "verification_scope": verification_scope.value, + "release_baseline_allowed": False, + "diagnosis": diagnosis, + } + _save_history(history_entry) + if use_json: + _print_json(history_entry) + sys.stderr.write(f"verify: {message}\n") + _notify( + _format_completion_notification( + exit_code=exit_code, + total_duration=duration_s, + step_results=history_entry["steps"], + ) + ) + return exit_code + + def _main(argv: list[str] | None = None) -> int: global _ACTIVE_VERIFY_RUN started_at = time.monotonic() @@ -2457,16 +2543,12 @@ def _main(argv: list[str] | None = None) -> int: args = parser.parse_args(sys.argv[1:] if argv is None else argv) _anchor_verification_paths() - try: - fingerprint = assert_polylogue_matches_checkout(ROOT, context="devtools verify") - except CheckoutImportMismatchError as exc: - sys.stderr.write(f"verify: {exc}\n") - return 125 - polylogue_import_path = fingerprint.polylogue_import_path - environment_fingerprint = fingerprint.as_dict() - sys.stderr.write(f"verify: polylogue package → {polylogue_import_path}\n") - if args.history: + try: + assert_polylogue_matches_checkout(ROOT, context="devtools verify") + except CheckoutImportMismatchError as exc: + sys.stderr.write(f"verify: {exc}\n") + return 125 _print_history() return 0 @@ -2485,10 +2567,59 @@ def _main(argv: list[str] | None = None) -> int: ) head = _git_head() pytest_enabled = not (args.quick or args.commit) + planned_scope = _planned_verification_scope( + args, + testmon_mode="full" if full_requested else None, + ) + verify_run = VerifyRun( + tier=tier, + argv=list(sys.argv[1:] if argv is None else argv), + git_head=head, + ) + _ACTIVE_VERIFY_RUN = (verify_run, started_at, planned_scope) + + try: + fingerprint = assert_polylogue_matches_checkout(ROOT, context="devtools verify") + except CheckoutImportMismatchError as exc: + return _finalize_preflight_failure( + verify_run, + started_at=started_at, + tier=tier, + head=head, + verification_scope=planned_scope, + diagnosis="checkout_import_mismatch", + exit_code=125, + message=str(exc), + use_json=bool(use_json), + ) + polylogue_import_path = fingerprint.polylogue_import_path + environment_fingerprint = fingerprint.as_dict() + verify_run.update_checkout_provenance( + polylogue_import_path=str(polylogue_import_path), + environment_fingerprint=environment_fingerprint, + ) + sys.stderr.write(f"verify: polylogue package → {polylogue_import_path}\n") + + mutation_monitor = CheckoutMutationMonitor(ROOT) + start_checkout_mutation_monitor(mutation_monitor) + checkout_fingerprint = worktree_fingerprint(ROOT) + verify_run.update_checkout_provenance(worktree_fingerprint=checkout_fingerprint) + base_commit = _git_commit("origin/master") if pytest_enabled else None if pytest_enabled and (base_commit is None or head is None): - sys.stderr.write("verify: cannot resolve immutable Git refs for native affected-test authority.\n") - return 125 + return _finalize_preflight_failure( + verify_run, + started_at=started_at, + tier=tier, + head=head, + verification_scope=planned_scope, + diagnosis="native_git_authority_unavailable", + exit_code=125, + message="cannot resolve immutable Git refs for native affected-test authority.", + use_json=bool(use_json), + mutation_monitor=mutation_monitor, + initial_worktree_fingerprint=checkout_fingerprint, + ) relevant_paths: tuple[str, ...] = () required_executable_paths: tuple[str, ...] = () @@ -2504,10 +2635,36 @@ def _main(argv: list[str] | None = None) -> int: ROOT, required_executable_paths=required_executable_paths, pytest_profile=_pytest_profile(), + deadline_monotonic=started_at + VERIFY_INVOCATION_BUDGET_S, + ) + except NativeTestmonDeadlineError as exc: + return _finalize_preflight_failure( + verify_run, + started_at=started_at, + tier=tier, + head=head, + verification_scope=planned_scope, + diagnosis="verify_invocation_deadline_exceeded", + exit_code=124, + message=str(exc), + use_json=bool(use_json), + mutation_monitor=mutation_monitor, + initial_worktree_fingerprint=checkout_fingerprint, ) except (NativeTestmonRepairError, PytestResourceError) as exc: - sys.stderr.write(f"verify: native pytest-testmon preparation failed: {exc}\n") - return 125 + return _finalize_preflight_failure( + verify_run, + started_at=started_at, + tier=tier, + head=head, + verification_scope=planned_scope, + diagnosis="native_testmon_preparation_failed", + exit_code=125, + message=f"native pytest-testmon preparation failed: {exc}", + use_json=bool(use_json), + mutation_monitor=mutation_monitor, + initial_worktree_fingerprint=checkout_fingerprint, + ) testmon_mode = "full" if full_requested else preparation.selection_mode if preparation.removed_paths: sys.stderr.write( @@ -2521,17 +2678,6 @@ def _main(argv: list[str] | None = None) -> int: sys.stderr.write("verify: native pytest-testmon environment is empty; plain verify will build it\n") planned_scope = _planned_verification_scope(args, testmon_mode=testmon_mode) - mutation_monitor = CheckoutMutationMonitor(ROOT) - start_checkout_mutation_monitor(mutation_monitor) - checkout_fingerprint = worktree_fingerprint(ROOT) - verify_run = VerifyRun( - tier=tier, - argv=list(sys.argv[1:] if argv is None else argv), - git_head=head, - polylogue_import_path=str(polylogue_import_path), - environment_fingerprint=environment_fingerprint, - worktree_fingerprint=checkout_fingerprint, - ) _ACTIVE_VERIFY_RUN = (verify_run, started_at, planned_scope) if not use_json: @@ -2548,17 +2694,19 @@ def _main(argv: list[str] | None = None) -> int: testmon_environment=preparation.environment_name if preparation is not None else "", ) except (PytestResourceError, ValueError) as exc: - finish_checkout_mutation_monitor(mutation_monitor) - payload = verify_run.finish( - exit_code=125, - duration_s=time.monotonic() - started_at, + return _finalize_preflight_failure( + verify_run, + started_at=started_at, + tier=tier, + head=head, + verification_scope=planned_scope, diagnosis="pytest_resource_preflight_failed", - verification_scope=planned_scope.value, - release_baseline_allowed=False, + exit_code=125, + message=str(exc), + use_json=bool(use_json), + mutation_monitor=mutation_monitor, + initial_worktree_fingerprint=checkout_fingerprint, ) - _save_history(payload) - sys.stderr.write(f"verify: {exc}\n") - return 125 step_results: list[dict[str, Any]] = [] exit_code = 0 @@ -2767,6 +2915,7 @@ def _main(argv: list[str] | None = None) -> int: "artifact_dir": str(verify_run.relative_run_dir), "steps": step_results, "total_duration_s": total_duration, + "invocation_budget_s": VERIFY_INVOCATION_BUDGET_S, "exit_code": exit_code, "verification_scope": verification_scope.value, "release_baseline_allowed": release_baseline_allowed, @@ -2852,6 +3001,7 @@ def _finalize_verify_runner_exception( ) payload["exception_type"] = type(exc).__name__ payload["error"] = str(exc) + payload["invocation_budget_s"] = VERIFY_INVOCATION_BUDGET_S _save_history(payload) if use_json: _print_json(payload) diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index 990936cc45..451bd6a4ef 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -182,6 +182,13 @@ def max_optional(current: int | None, value: object) -> int | None: no_pytest = not pytest_steps corpus_digest = hashlib.sha256(b"").hexdigest() exit_code = entry.get("exit_code") + raw_budget = entry.get("invocation_budget_s") + invocation_budget = float(raw_budget) if isinstance(raw_budget, int | float) else None + raw_wall = entry.get("total_duration_s", entry.get("duration_s", 0.0)) + wall_s = float(raw_wall) if isinstance(raw_wall, int | float) else 0.0 + deadline_met = entry.get("diagnosis") != "verify_invocation_deadline_exceeded" + if invocation_budget is not None: + deadline_met = deadline_met and wall_s <= invocation_budget return { "schema_version": 1, "environment": { @@ -216,8 +223,8 @@ def max_optional(current: int | None, value: object) -> int | None: }, "cleanup": {"complete": True if no_pytest else cleanup_complete}, "containment": {"complete": True if no_pytest else containment_complete}, - "deadline": {"budget_s": None, "met": True}, - "wall_s": entry.get("total_duration_s", entry.get("duration_s", 0.0)), + "deadline": {"budget_s": invocation_budget, "met": deadline_met}, + "wall_s": wall_s, } @@ -1510,6 +1517,22 @@ def write(self) -> None: if not _current_owner_is_other_live_run(current_path): _write_json(current_path, self._payload) + def update_checkout_provenance( + self, + *, + polylogue_import_path: str | None = None, + environment_fingerprint: Mapping[str, Any] | None = None, + worktree_fingerprint: str | None = None, + ) -> None: + """Persist provenance as each preflight authority becomes available.""" + if polylogue_import_path is not None: + self._payload["polylogue_import_path"] = polylogue_import_path + if environment_fingerprint is not None: + self._payload["environment_fingerprint"] = dict(environment_fingerprint) + if worktree_fingerprint is not None: + self._payload["worktree_fingerprint"] = worktree_fingerprint + self.write() + def start_step(self, *, label: str, cmd: list[str]) -> PytestStepArtifacts: index = len(self._payload["steps"]) + 1 step_id = f"{index:02d}-{_slug(label)}" diff --git a/tests/integration/devtools/test_native_testmon_lifecycle.py b/tests/integration/devtools/test_native_testmon_lifecycle.py index be3e87bc2f..a9dc55fb70 100644 --- a/tests/integration/devtools/test_native_testmon_lifecycle.py +++ b/tests/integration/devtools/test_native_testmon_lifecycle.py @@ -609,7 +609,7 @@ def test_environment_and_plugin_identity_changes_start_real_fresh_bootstraps( ) ] == [0, 0] - plugin = repo / "pytest_native_identity.py" + plugin = repo / "local_plugin.py" plugin.write_text( "import pytest\n\n@pytest.fixture\ndef native_identity():\n return 'v1'\n", encoding="utf-8", @@ -618,7 +618,7 @@ def test_environment_and_plugin_identity_changes_start_real_fresh_bootstraps( "def test_initial():\n assert True\n\ndef test_plugin(native_identity):\n assert native_identity == 'v1'\n", encoding="utf-8", ) - monkeypatch.setenv("PYTEST_PLUGINS", "pytest_native_identity") + monkeypatch.setenv("PYTEST_PLUGINS", "local_plugin") plugin_changed = prepare_native_testmon_environment(repo) assert plugin_changed.selection_mode == "bootstrap" assert plugin_changed.environment_name != environment_changed.environment_name diff --git a/tests/unit/devtools/test_testmon_bootstrap.py b/tests/unit/devtools/test_testmon_bootstrap.py index 39116abe38..7dd989456c 100644 --- a/tests/unit/devtools/test_testmon_bootstrap.py +++ b/tests/unit/devtools/test_testmon_bootstrap.py @@ -5,6 +5,7 @@ import pytest from devtools.testmon_bootstrap import ( + NativeTestmonDeadlineError, NativeTestmonRepairError, classify_source_ast, executable_python_paths, @@ -78,6 +79,25 @@ def test_environment_digest_changes_with_collection_semantics( ) +def test_environment_digest_hashes_explicit_local_plugin_regardless_of_name( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + plugin = tmp_path / "local_plugin.py" + plugin.write_text("VALUE = 'v1'\n", encoding="utf-8") + monkeypatch.setenv("PYTEST_PLUGINS", "local_plugin") + + initial = _testmon_environment_digest(tmp_path) + plugin.write_text("VALUE = 'v2'\n", encoding="utf-8") + + assert _testmon_environment_digest(tmp_path) != initial + + +def test_environment_digest_stops_at_invocation_deadline(tmp_path: Path) -> None: + with pytest.raises(NativeTestmonDeadlineError, match="invocation deadline"): + _testmon_environment_digest(tmp_path, deadline_monotonic=0.0) + + def test_invalid_cleanup_removes_only_owned_sqlite_and_sidecars(tmp_path: Path) -> None: state_dir = tmp_path / ".cache" / "testmon" state_dir.mkdir(parents=True) diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 2fc36186be..ad00e4d2dd 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -19,7 +19,8 @@ import watchfiles from devtools import run_tests, verify, verify_runs -from devtools.testmon_bootstrap import executable_python_paths +from devtools.checkout_guard import CheckoutImportMismatchError +from devtools.testmon_bootstrap import NativeTestmonDeadlineError, executable_python_paths from devtools.verification_contracts import VerificationScope from devtools.verify import ( PYTEST_CONTAINMENT_PATH, @@ -140,7 +141,7 @@ def test_native_testmon_uses_exactly_two_semantic_lanes( assert command[command.index("-p") + 1] == "devtools.pytest_progress_plugin" -def test_native_marker_policy_only_excludes_benchmarks() -> None: +def test_native_corpus_excludes_only_benchmark_directory() -> None: complete_steps = build_verify_steps( quick=False, lab=True, @@ -149,8 +150,7 @@ def test_native_marker_policy_only_excludes_benchmarks() -> None: complete_command = next(command for label, command in complete_steps if "parallel" in label) complete_expr = _pytest_marker_expr(complete_command) - assert "not benchmark" in complete_expr - assert "not slow" not in complete_expr + assert complete_expr == "not load_sensitive and not tui" assert "--ignore=tests/benchmarks" in complete_command assert "--ignore=tests/integration" not in complete_command @@ -3714,8 +3714,11 @@ def finish(self) -> CheckoutMutationObservation: assert durable_payload["final_worktree_fingerprint"] == "stable" -def test_verify_finalizes_checkout_monitor_when_startup_fingerprint_raises() -> None: +def test_verify_finalizes_checkout_monitor_when_startup_fingerprint_raises( + capsys: pytest.CaptureFixture[str], +) -> None: events: list[str] = [] + history: dict[str, Any] = {} class _ExceptionalExitMonitor: def __init__(self, _root: Path) -> None: @@ -3732,11 +3735,124 @@ def finish(self) -> CheckoutMutationObservation: patch("devtools.verify.CheckoutMutationMonitor", _ExceptionalExitMonitor), patch("devtools.verify._git_head", return_value="head"), patch("devtools.verify.worktree_fingerprint", side_effect=RuntimeError("fingerprint failed")), + patch("devtools.verify._save_history", side_effect=lambda entry: history.update(entry)), + patch("devtools.verify._notify"), ): - with pytest.raises(RuntimeError, match="fingerprint failed"): - main(["--quick", "--json"]) + assert main(["--quick", "--json"]) == 125 assert events == ["monitor-started", "monitor-finished"] + assert history["diagnosis"] == "verify_runner_exception" + assert json.loads(capsys.readouterr().out)["diagnosis"] == "verify_runner_exception" + + +def test_import_guard_failure_writes_normalized_history_and_invocation_receipt( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + history: dict[str, Any] = {} + receipt = tmp_path / "invocation-receipt.json" + monkeypatch.setenv(verify_runs.VERIFICATION_INVOCATION_ID_ENV, "import-mismatch") + monkeypatch.setenv(verify_runs.VERIFICATION_RECEIPT_PATH_ENV, str(receipt)) + + with ( + patch("devtools.verify._git_head", return_value="head"), + patch( + "devtools.verify.assert_polylogue_matches_checkout", + side_effect=CheckoutImportMismatchError("wrong checkout import"), + ), + patch("devtools.verify._save_history", side_effect=lambda entry: history.update(entry)), + patch("devtools.verify._notify"), + ): + assert main(["--quick", "--json"]) == 125 + + normalized = verify_runs.normalize_verify_history_entry(history) + assert normalized["timestamp"] + assert normalized["pytest_aggregate"]["selection_mode"] == "none" + assert normalized["diagnosis"] == "checkout_import_mismatch" + assert json.loads(receipt.read_text())["diagnosis"] == "checkout_import_mismatch" + assert json.loads(capsys.readouterr().out)["diagnosis"] == "checkout_import_mismatch" + + +def test_git_authority_failure_writes_history_and_invocation_receipt( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + class _StableMonitor: + def __init__(self, _root: Path) -> None: + pass + + def start(self) -> None: + pass + + def finish(self) -> CheckoutMutationObservation: + return CheckoutMutationObservation(changed=False, unavailable=False) + + history: dict[str, Any] = {} + receipt = tmp_path / "invocation-receipt.json" + monkeypatch.setenv(verify_runs.VERIFICATION_INVOCATION_ID_ENV, "git-authority") + monkeypatch.setenv(verify_runs.VERIFICATION_RECEIPT_PATH_ENV, str(receipt)) + + with ( + patch("devtools.verify._git_head", return_value="head"), + patch("devtools.verify._git_commit", return_value=None), + patch("devtools.verify.CheckoutMutationMonitor", _StableMonitor), + patch("devtools.verify.worktree_fingerprint", return_value="stable"), + patch("devtools.verify._save_history", side_effect=lambda entry: history.update(entry)), + patch("devtools.verify._notify"), + ): + assert main(["--json"]) == 125 + + assert history["diagnosis"] == "native_git_authority_unavailable" + assert history["final_worktree_fingerprint"] == "stable" + assert json.loads(receipt.read_text())["diagnosis"] == "native_git_authority_unavailable" + assert json.loads(capsys.readouterr().out)["diagnosis"] == "native_git_authority_unavailable" + + +def test_native_preparation_uses_invocation_deadline_and_records_expiry( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + class _StableMonitor: + def __init__(self, _root: Path) -> None: + pass + + def start(self) -> None: + pass + + def finish(self) -> CheckoutMutationObservation: + return CheckoutMutationObservation(changed=False, unavailable=False) + + history: dict[str, Any] = {} + receipt = tmp_path / "invocation-receipt.json" + monkeypatch.setenv(verify_runs.VERIFICATION_INVOCATION_ID_ENV, "preparation-deadline") + monkeypatch.setenv(verify_runs.VERIFICATION_RECEIPT_PATH_ENV, str(receipt)) + monkeypatch.setattr(verify, "VERIFY_INVOCATION_BUDGET_S", 42.0) + + def expire_preparation(*_args: object, **kwargs: object) -> object: + assert kwargs["deadline_monotonic"] == 142.0 + raise NativeTestmonDeadlineError("verify invocation deadline expired during native testmon preparation") + + with ( + patch("devtools.verify.time.monotonic", return_value=100.0), + patch("devtools.verify._git_head", return_value="head"), + patch("devtools.verify._git_commit", return_value="base"), + patch("devtools.verify._changed_test_relevant_paths", return_value=()), + patch("devtools.verify.prepare_native_testmon_environment", side_effect=expire_preparation), + patch("devtools.verify.CheckoutMutationMonitor", _StableMonitor), + patch("devtools.verify.worktree_fingerprint", return_value="stable"), + patch("devtools.verify._save_history", side_effect=lambda entry: history.update(entry)), + patch("devtools.verify._notify"), + ): + assert main(["--json"]) == 124 + + normalized = verify_runs.normalize_verify_history_entry(history) + assert normalized["diagnosis"] == "verify_invocation_deadline_exceeded" + assert normalized["pytest_aggregate"]["deadline"] == {"budget_s": 42.0, "met": False} + assert json.loads(receipt.read_text())["exit_code"] == 124 + assert json.loads(capsys.readouterr().out)["exit_code"] == 124 def test_verify_anchors_relative_state_to_checkout_when_invoked_from_subdirectory( From df68f13785a0fdc1ce680d26f697fe2ed793a74e Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 21:06:14 +0200 Subject: [PATCH 07/31] refactor(test): narrow native lifecycle boundaries --- TESTING.md | 35 ++++++++++--- devtools/testmon_bootstrap.py | 49 ++++++++++++++++-- devtools/verify.py | 4 +- pyproject.toml | 2 +- tests/conftest.py | 5 +- .../devtools/test_native_testmon_lifecycle.py | 51 ++++++++++++++++++- tests/unit/devtools/test_merge_boundary.py | 40 ++++++++++++++- tests/unit/devtools/test_testmon_bootstrap.py | 31 ++++++++--- tests/unit/devtools/test_verify.py | 6 +-- 9 files changed, 197 insertions(+), 26 deletions(-) diff --git a/TESTING.md b/TESTING.md index 2013955fc5..181f969cc9 100644 --- a/TESTING.md +++ b/TESTING.md @@ -57,9 +57,17 @@ receipt or resume command: pytest-testmon keeps failed, unfinished, and new tests selected on the next plain invocation. The native environment key includes Python, active distributions, lock and -project metadata, pytest configuration, `tests/conftest.py`, and the managed -pytest plugins and harness. Product source changes stay incremental. A change -to collection or dependency semantics starts a fresh native environment. +project metadata, pytest configuration, every `conftest.py`, and collection-active +local pytest plugins. Ordinary `tests/infra` helpers stay incremental: import +them from an executing fixture or test when their module initialization is +dependency-bearing, so pytest-testmon observes that initialization and helper +execution instead of forcing a complete-corpus bootstrap for every helper edit. +Product source changes likewise stay incremental. A genuine collection or +dependency-semantics change starts a fresh native environment. +The 2026-08-13 audit reduced this key from all 83 `tests/infra` Python files to +23 actual inputs (configuration, conftests, and active plugins); discovery took +0.95s. A real isolated mutation of a runtime-imported helper retained the same +environment and selected both owning tests through native dependency edges. Plain focused `pytest` runs are single-process by default so small inner-loop checks do not spawn a worker pool. `devtools verify` keeps pytest-testmon as @@ -74,10 +82,23 @@ roughly 768 MiB per worker, reserve host and tmpfs headroom, and reduce concurrency when memory pressure is elevated. Every native run has exactly two semantic lanes over one environment and one -database: a parallel lane for tests that are neither `load_sensitive` nor -`tui`, followed by a serial lane for either marker. Ordinary test failures in -the parallel lane do not suppress the serial lane. Typed collection, -containment, resource, or timeout failures do. +database: a parallel lane for tests not marked `load_sensitive`, followed by a +serial lane for the load-sensitive set. Ordinary test failures in the parallel +lane do not suppress the serial lane. Typed collection, containment, resource, +or timeout failures do. + +The lane boundary is evidence-based. On 2026-08-13 the complete correctness +corpus collected 20,447 nodes in 35.06s; only 17 were `load_sensitive`, while +16 were tagged `tui` with no overlap. A managed serial run retained 93.09s of +call time for the load-sensitive set (real PTYs, loopback servers, timing SLAs, +and process/cgroup teardown), versus 7.10s for TUI (0.74s maximum). The same 16 +TUI nodes passed under two xdist workers in 45.22s including duplicate worker +collection, so `tui` remains a useful category but is not a serial-execution +boundary. The captured serial run passed 32 nodes; one process-owner SIGKILL +probe missed its five-second readiness poll twice, then passed a diagnostic +live-output rerun in 6.28s. That host-timing sensitivity supports retaining the +probe in the serial lane. Performance benchmarks remain outside this +correctness corpus. Every collected test has a 120-second `pytest-timeout` budget. A test that genuinely needs longer must declare the exception at the test site with diff --git a/devtools/testmon_bootstrap.py b/devtools/testmon_bootstrap.py index 5ef4eda6da..a3cc40aca2 100644 --- a/devtools/testmon_bootstrap.py +++ b/devtools/testmon_bootstrap.py @@ -138,10 +138,54 @@ def _fingerprint_inputs( return digest.hexdigest() +def _pytest_plugins_assignment(node: ast.stmt) -> ast.expr | None: + match node: + case ast.Assign(targets=targets, value=value) if any( + isinstance(target, ast.Name) and target.id == "pytest_plugins" for target in targets + ): + return value + case ast.AnnAssign(target=ast.Name(id="pytest_plugins"), value=value): + return value + case _: + return None + + +def _declared_pytest_plugin_names(root: Path) -> set[str]: + """Read static local plugin declarations that pytest loads at collection.""" + names: set[str] = set() + candidates = set(root.glob("tests/**/conftest.py")) + for path in root.glob("tests/**/*.py"): + try: + source = path.read_text(encoding="utf-8") + except OSError: + continue + if "pytest_plugins" not in source: + continue + candidates.add(path) + for path in candidates: + try: + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + except (OSError, SyntaxError, UnicodeDecodeError): + continue + for node in tree.body: + value = _pytest_plugins_assignment(node) + if value is None: + continue + with contextlib.suppress(ValueError, TypeError): + declared = ast.literal_eval(value) + if isinstance(declared, str): + names.add(declared) + elif isinstance(declared, tuple | list): + names.update(name for name in declared if isinstance(name, str)) + return names + + def _active_local_pytest_plugin_paths(root: Path) -> set[str]: - """Resolve explicitly active local pytest plugins regardless of filename.""" + """Resolve collection-active local pytest plugins regardless of filename.""" paths: set[str] = set() - for raw_name in os.environ.get("PYTEST_PLUGINS", "").split(","): + plugin_names = _declared_pytest_plugin_names(root) + plugin_names.update(os.environ.get("PYTEST_PLUGINS", "").split(",")) + for raw_name in plugin_names: module_name = raw_name.strip() if not module_name or any(part in {"", ".", ".."} for part in module_name.split(".")): continue @@ -161,7 +205,6 @@ def _environment_input_paths(root: Path) -> tuple[str, ...]: patterns = ( "devtools/pytest*.py", "tests/**/conftest.py", - "tests/infra/**/*.py", ) for pattern in patterns: paths.update(path.relative_to(root).as_posix() for path in root.glob(pattern) if path.is_file()) diff --git a/devtools/verify.py b/devtools/verify.py index bde90cecb2..cf5f3c147b 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -2170,7 +2170,7 @@ def build_verify_steps( parallel_cmd = [ *pytest_cmd, "-m", - "not load_sensitive and not tui", + "not load_sensitive", *native_args, *_pytest_worker_args(), ] @@ -2187,7 +2187,7 @@ def _serial_report_arg(arg: str) -> str: serial_cmd.extend( [ "-m", - "load_sensitive or tui", + "load_sensitive", *native_args, "-p", "no:randomly", diff --git a/pyproject.toml b/pyproject.toml index fc97d3bfda..9f51f28fab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -183,7 +183,7 @@ markers = [ "machine_contract: marks root CLI JSON success/failure contract tests", "query_routing: marks query-first CLI routing and read-surface tests", "evidence_law: marks generated-surface completeness, anti-dead-code, and evidence-quality laws", - "tui: marks Textual dashboard interaction tests", + "tui: marks Textual dashboard interaction tests (category only; these run in the parallel correctness lane)", "chaos: marks ingestion hostility, interruption, and chronology tests", "live: marks operator-run live archive validation lanes", "xdist_group(group_name): distribute parametrized tests within a file across workers (#1026)", diff --git a/tests/conftest.py b/tests/conftest.py index 08bc87bfef..7bcc93383d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -40,7 +40,6 @@ resolve_pytest_basetemp_root, ) from devtools.verify_runs import pytest_basetemp_claim_path as _basetemp_claim_path -from tests.infra.archive_templates import clone_archive_template # Resolve (but don't yet raise on) the polylogue-vs-checkout mismatch check # before test execution can import product modules: a shared/editable venv's @@ -845,6 +844,8 @@ def workspace_env( monkeypatch: pytest.MonkeyPatch, empty_archive_template: Path, ) -> dict[str, Path]: + from tests.infra.archive_templates import clone_archive_template + data_dir = tmp_path / "data" state_dir = tmp_path / "state" archive_root = tmp_path / "archive" @@ -917,6 +918,8 @@ def cli_workspace( Returns: dict with paths: archive_root, data_root, inbox_dir, db_path """ + from tests.infra.archive_templates import clone_archive_template + # Create directory structure data_dir = tmp_path / "data" state_dir = tmp_path / "state" diff --git a/tests/integration/devtools/test_native_testmon_lifecycle.py b/tests/integration/devtools/test_native_testmon_lifecycle.py index a9dc55fb70..839de719ba 100644 --- a/tests/integration/devtools/test_native_testmon_lifecycle.py +++ b/tests/integration/devtools/test_native_testmon_lifecycle.py @@ -62,7 +62,7 @@ def _init_repo(root: Path, *, conftest: str = "") -> None: cache_dir = ".cache/pytest" markers = [ "load_sensitive: serial native-testmon lane", - "tui: serial native-testmon lane", + "tui: Textual interaction category", ] """.lstrip(), encoding="utf-8", @@ -108,7 +108,7 @@ def _run_lane( "POLYLOGUE_PYTEST_SUMMARY_PATH": str(artifact_dir / "summary.json"), } ) - semantic_marker = "not load_sensitive and not tui" if lane == "parallel" else "load_sensitive or tui" + semantic_marker = "not load_sensitive" if lane == "parallel" else "load_sensitive" marker = semantic_marker if base_marker is None else f"({base_marker}) and ({semantic_marker})" selection = "--testmon-forceselect" if mode == "affected" else "--testmon-noselect" command = [ @@ -638,3 +638,50 @@ def test_environment_and_plugin_identity_changes_start_real_fresh_bootstraps( result.completed.returncode for result in _run_plain_verify_corpus(repo, mode="bootstrap", environment_name=plugin_mutated.environment_name) ] == [0, 0] + + +def test_runtime_helper_mutation_stays_incremental_and_selects_owner(tmp_path: Path) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _init_repo(repo) + infra = repo / "tests" / "infra" + infra.mkdir() + (repo / "tests" / "__init__.py").write_text("", encoding="utf-8") + (infra / "__init__.py").write_text("", encoding="utf-8") + helper = infra / "runtime_helper.py" + helper.write_text("def answer() -> int:\n return 42\n", encoding="utf-8") + (repo / "tests" / "test_runtime_helper.py").write_text( + """ +import pytest + +def test_runtime_helper_owner(): + from tests.infra.runtime_helper import answer + assert answer() == 42 + +@pytest.mark.load_sensitive +def test_runtime_helper_serial_owner(): + from tests.infra.runtime_helper import answer + assert answer() == 42 +""".lstrip(), + encoding="utf-8", + ) + _commit_all(repo, "fixture") + + preparation = prepare_native_testmon_environment(repo) + bootstrap = _run_plain_verify_corpus( + repo, + mode="bootstrap", + environment_name=preparation.environment_name, + ) + assert [result.completed.returncode for result in bootstrap] == [0, 0] + + helper.write_text("def answer() -> int:\n return 0\n", encoding="utf-8") + affected = prepare_native_testmon_environment(repo) + assert affected.selection_mode == "affected" + results = _run_plain_verify_corpus(repo, mode="affected", environment_name=affected.environment_name) + + assert [result.completed.returncode for result in results] == [1, 1] + assert _selected(*results) == { + "tests/test_runtime_helper.py::test_runtime_helper_owner", + "tests/test_runtime_helper.py::test_runtime_helper_serial_owner", + } diff --git a/tests/unit/devtools/test_merge_boundary.py b/tests/unit/devtools/test_merge_boundary.py index f303c69f2e..527bc99c15 100644 --- a/tests/unit/devtools/test_merge_boundary.py +++ b/tests/unit/devtools/test_merge_boundary.py @@ -11,7 +11,7 @@ import pytest -from devtools import merge_boundary, merge_gate, pr_scope +from devtools import click_dispatch, merge_boundary, merge_gate, pr_scope from devtools.checkout_guard import checkout_environment_fingerprint from tests.infra.frozen_clock import FrozenClock @@ -203,6 +203,44 @@ def merge(pr: int, **_kwargs: object) -> int: assert captured == [3948] +def test_workspace_merge_dispatches_documented_direct_pr_form(monkeypatch: pytest.MonkeyPatch) -> None: + captured: list[tuple[int, bool]] = [] + + def merge(pr: int, **kwargs: object) -> int: + captured.append((pr, bool(kwargs["dry_run"]))) + return 0 + + monkeypatch.setattr(merge_boundary, "cmd_merge", merge) + + assert click_dispatch._dispatch(["workspace", "merge", "3952", "--dry-run"]) == 0 + assert captured == [(3952, True)] + + +def test_workspace_merge_preserves_train_status_dispatch(monkeypatch: pytest.MonkeyPatch) -> None: + calls: list[bool] = [] + monkeypatch.setattr(merge_boundary, "cmd_train_status", lambda as_json: calls.append(as_json) or 0) + + assert click_dispatch._dispatch(["workspace", "merge", "train-status", "--json"]) == 0 + assert calls == [True] + + +def test_workspace_merge_preserves_record_full_verify_dispatch(monkeypatch: pytest.MonkeyPatch) -> None: + captured: list[tuple[str, str]] = [] + monkeypatch.setattr(merge_boundary, "_reconciled_terminal_verify_snapshot", lambda: {}) + monkeypatch.setattr(merge_boundary, "_fetched_current_default_branch_sha", lambda: "master-sha") + monkeypatch.setattr( + merge_boundary, + "_run_post_merge_terminal_verify", + lambda command, target_sha, **_kwargs: captured.append((command, target_sha)) or 0, + ) + + assert ( + click_dispatch._dispatch(["workspace", "merge", "record-full-verify", "--command", "devtools verify --all"]) + == 0 + ) + assert captured == [("devtools verify --all", "master-sha")] + + # --------------------------------------------------------------------------- # cmd_merge # --------------------------------------------------------------------------- diff --git a/tests/unit/devtools/test_testmon_bootstrap.py b/tests/unit/devtools/test_testmon_bootstrap.py index 7dd989456c..e9ca0dde8d 100644 --- a/tests/unit/devtools/test_testmon_bootstrap.py +++ b/tests/unit/devtools/test_testmon_bootstrap.py @@ -54,10 +54,6 @@ def test_environment_digest_changes_with_collection_semantics( (tmp_path / "pyproject.toml").write_text("[tool.pytest.ini_options]\naddopts = '-ra'\n", encoding="utf-8") config_changed = _testmon_environment_digest(tmp_path, pytest_profile="slow=include") - plugin = tmp_path / "tests" / "infra" / "new_plugin.py" - plugin.parent.mkdir(parents=True) - plugin.write_text("pytest_plugins = ()\n", encoding="utf-8") - harness_changed = _testmon_environment_digest(tmp_path, pytest_profile="slow=include") monkeypatch.setattr("devtools.testmon_bootstrap._installed_distributions", lambda: (("pytest", "changed"),)) distributions_changed = _testmon_environment_digest(tmp_path, pytest_profile="slow=include") monkeypatch.setenv("PYTEST_ADDOPTS", "--strict-markers") @@ -69,16 +65,39 @@ def test_environment_digest_changes_with_collection_semantics( { initial, config_changed, - harness_changed, distributions_changed, environment_changed, profile_changed, } ) - == 6 + == 5 ) +def test_inactive_runtime_helper_does_not_force_fresh_environment(tmp_path: Path) -> None: + helper = tmp_path / "tests" / "infra" / "runtime_helper.py" + helper.parent.mkdir(parents=True) + helper.write_text("def answer() -> int:\n return 41\n", encoding="utf-8") + initial = _testmon_environment_digest(tmp_path) + + helper.write_text("def answer() -> int:\n return 42\n", encoding="utf-8") + + assert _testmon_environment_digest(tmp_path) == initial + + +def test_declared_local_fixture_plugin_changes_environment(tmp_path: Path) -> None: + conftest = tmp_path / "tests" / "conftest.py" + plugin = tmp_path / "tests" / "infra" / "fixture_plugin.py" + plugin.parent.mkdir(parents=True) + conftest.write_text('pytest_plugins = ("tests.infra.fixture_plugin",)\n', encoding="utf-8") + plugin.write_text("import pytest\n\n@pytest.fixture\ndef value():\n return 1\n", encoding="utf-8") + initial = _testmon_environment_digest(tmp_path) + + plugin.write_text("import pytest\n\n@pytest.fixture\ndef value():\n return 2\n", encoding="utf-8") + + assert _testmon_environment_digest(tmp_path) != initial + + def test_environment_digest_hashes_explicit_local_plugin_regardless_of_name( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index ad00e4d2dd..768e074820 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -129,8 +129,8 @@ def test_native_testmon_uses_exactly_two_semantic_lanes( ] parallel = pytest_steps[0][1] serial = pytest_steps[1][1] - assert "not load_sensitive and not tui" in _pytest_marker_expr(parallel) - assert "load_sensitive or tui" in _pytest_marker_expr(serial) + assert _pytest_marker_expr(parallel) == "not load_sensitive" + assert _pytest_marker_expr(serial) == "load_sensitive" assert parallel[parallel.index("-n") + 1] == "8" assert serial[serial.index("-n") + 1] == "0" for _label, command in pytest_steps: @@ -150,7 +150,7 @@ def test_native_corpus_excludes_only_benchmark_directory() -> None: complete_command = next(command for label, command in complete_steps if "parallel" in label) complete_expr = _pytest_marker_expr(complete_command) - assert complete_expr == "not load_sensitive and not tui" + assert complete_expr == "not load_sensitive" assert "--ignore=tests/benchmarks" in complete_command assert "--ignore=tests/integration" not in complete_command From fe083e916cab7d15dcbb261785c87e55a3fd7f9f Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 21:09:33 +0200 Subject: [PATCH 08/31] test(devtools): type merge dispatch probes --- tests/unit/devtools/test_merge_boundary.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/tests/unit/devtools/test_merge_boundary.py b/tests/unit/devtools/test_merge_boundary.py index 527bc99c15..1ccc3a1358 100644 --- a/tests/unit/devtools/test_merge_boundary.py +++ b/tests/unit/devtools/test_merge_boundary.py @@ -218,7 +218,12 @@ def merge(pr: int, **kwargs: object) -> int: def test_workspace_merge_preserves_train_status_dispatch(monkeypatch: pytest.MonkeyPatch) -> None: calls: list[bool] = [] - monkeypatch.setattr(merge_boundary, "cmd_train_status", lambda as_json: calls.append(as_json) or 0) + + def train_status(as_json: bool) -> int: + calls.append(as_json) + return 0 + + monkeypatch.setattr(merge_boundary, "cmd_train_status", train_status) assert click_dispatch._dispatch(["workspace", "merge", "train-status", "--json"]) == 0 assert calls == [True] @@ -226,13 +231,14 @@ def test_workspace_merge_preserves_train_status_dispatch(monkeypatch: pytest.Mon def test_workspace_merge_preserves_record_full_verify_dispatch(monkeypatch: pytest.MonkeyPatch) -> None: captured: list[tuple[str, str]] = [] + + def record(command: str, target_sha: str, **_kwargs: object) -> int: + captured.append((command, target_sha)) + return 0 + monkeypatch.setattr(merge_boundary, "_reconciled_terminal_verify_snapshot", lambda: {}) monkeypatch.setattr(merge_boundary, "_fetched_current_default_branch_sha", lambda: "master-sha") - monkeypatch.setattr( - merge_boundary, - "_run_post_merge_terminal_verify", - lambda command, target_sha, **_kwargs: captured.append((command, target_sha)) or 0, - ) + monkeypatch.setattr(merge_boundary, "_run_post_merge_terminal_verify", record) assert ( click_dispatch._dispatch(["workspace", "merge", "record-full-verify", "--command", "devtools verify --all"]) From d7d85fe9019e5aee68df845452df466c9ab3cfb2 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 21:37:17 +0200 Subject: [PATCH 09/31] fix(test): close final native lifecycle gaps --- .github/workflows/nightly-scale.yml | 12 ++++---- CONTRIBUTING.md | 6 ++-- devtools/testmon_bootstrap.py | 24 +++++++++++++++ tests/unit/devtools/test_testmon_bootstrap.py | 30 +++++++++++++++++++ 4 files changed, 65 insertions(+), 7 deletions(-) diff --git a/.github/workflows/nightly-scale.yml b/.github/workflows/nightly-scale.yml index ff0ea0f2f2..e9f0815b87 100644 --- a/.github/workflows/nightly-scale.yml +++ b/.github/workflows/nightly-scale.yml @@ -1,6 +1,6 @@ name: Nightly Scale -# Runs the large-tier scale fixtures (issue #1183) outside the normal +# Runs the explicit large benchmark nodes outside the normal # CI critical path, captures benchmark results as structured JSON, and # compares against the committed baseline to detect regressions (#1220). # @@ -41,16 +41,18 @@ jobs: python-version: "3.14" - run: uv sync --extra dev --frozen - - name: Run large-tier scale tests + - name: Run large benchmark campaign env: POLYLOGUE_FORCE_PLAIN: "1" HYPOTHESIS_PROFILE: ci run: | - uv run pytest -q --tb=short --ignore=tests/integration \ - -m "scale_large" -p no:randomly -n 0 \ + uv run pytest -q --tb=short -p no:randomly -n 0 \ --benchmark-json=nightly-results.json \ --benchmark-group-by=group \ - tests/benchmarks + 'tests/benchmarks/test_daemon_convergence.py::test_convergence_scale_tier[xxl-mega-session]' \ + tests/benchmarks/test_daemon_convergence.py::test_convergence_huge_session_memory_bounded \ + tests/benchmarks/test_scale_tiers.py::test_bench_50k_fts_search_returns_results \ + tests/benchmarks/test_scale_tiers.py::test_bench_50k_list_sessions - name: Upload benchmark results uses: actions/upload-artifact@v7 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index da50acd38e..0379fb0653 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -309,8 +309,10 @@ static gates and then invokes pytest-testmon for affected-test selection from the current source, dependency, and Python-version state. The pytest step covers unit, property, fuzz, and integration tests while excluding the separately operated `tests/benchmarks` performance surface. It uses -`--testmon-forceselect` for affected selection, with one parallel lane and one -serial `load_sensitive`/`tui` lane over the same native environment. Use +`--testmon-forceselect` for affected selection, with one parallel `not +load_sensitive` lane and one serial `load_sensitive` lane over the same native +environment. `tui` is a category marker and remains parallel unless a test is +also explicitly `load_sensitive`. Use `devtools verify --all` for the complete correctness corpus; there is no manual seed or repair command. diff --git a/devtools/testmon_bootstrap.py b/devtools/testmon_bootstrap.py index a3cc40aca2..be46dd4f04 100644 --- a/devtools/testmon_bootstrap.py +++ b/devtools/testmon_bootstrap.py @@ -50,6 +50,10 @@ "pytest.ini", "tox.ini", "setup.cfg", + # Keep the conventional repository-root hook as an absent-path sentinel: + # creating it changes collection even though it was not present when the + # previous environment was named. + "conftest.py", "devtools/checkout_guard.py", "devtools/testmon_bootstrap.py", "devtools/verify.py", @@ -154,6 +158,7 @@ def _declared_pytest_plugin_names(root: Path) -> set[str]: """Read static local plugin declarations that pytest loads at collection.""" names: set[str] = set() candidates = set(root.glob("tests/**/conftest.py")) + candidates.add(root / "conftest.py") for path in root.glob("tests/**/*.py"): try: source = path.read_text(encoding="utf-8") @@ -263,6 +268,19 @@ def _is_docstring(node: ast.stmt, *, first: bool) -> bool: ) +def _is_type_checking_guard(node: ast.expr) -> bool: + return ( + isinstance(node, ast.Name) + and node.id == "TYPE_CHECKING" + or ( + isinstance(node, ast.Attribute) + and isinstance(node.value, ast.Name) + and node.value.id == "typing" + and node.attr == "TYPE_CHECKING" + ) + ) + + def _body_is_executable(body: list[ast.stmt]) -> bool: for index, node in enumerate(body): if _is_docstring(node, first=index == 0): @@ -277,6 +295,12 @@ def _body_is_executable(body: list[ast.stmt]) -> bool: continue if isinstance(node, ast.AnnAssign) and node.value is None: continue + if isinstance(node, ast.If) and _is_type_checking_guard(node.test): + # The guarded body is deliberately invisible at runtime. An else + # branch does execute and therefore retains ordinary classification. + if _body_is_executable(node.orelse): + return True + continue if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): if node.decorator_list or node.args.defaults or any(value is not None for value in node.args.kw_defaults): return True diff --git a/tests/unit/devtools/test_testmon_bootstrap.py b/tests/unit/devtools/test_testmon_bootstrap.py index e9ca0dde8d..ba0d3df9c5 100644 --- a/tests/unit/devtools/test_testmon_bootstrap.py +++ b/tests/unit/devtools/test_testmon_bootstrap.py @@ -29,6 +29,25 @@ def test_ast_classification_distinguishes_declarations_from_execution(tmp_path: assert classify_source_ast(executable) == "executable" +def test_ast_classification_treats_type_checking_guards_as_declarations(tmp_path: Path) -> None: + declarations = tmp_path / "protocols.py" + declarations.write_text( + "from typing import TYPE_CHECKING\n\n" + "if TYPE_CHECKING:\n" + " from polylogue.archive.models import Session\n\n" + "class SessionReader:\n" + " session: 'Session'\n" + " def read(self) -> 'Session': ...\n", + encoding="utf-8", + ) + + assert classify_source_ast(declarations) == "declaration-only" + + declarations.write_text(declarations.read_text(encoding="utf-8") + "\nVALUE = build_runtime_value()\n") + + assert classify_source_ast(declarations) == "executable" + + def test_executable_paths_require_current_runtime_modules_but_allow_deletion(tmp_path: Path) -> None: module = tmp_path / "polylogue" / "runtime.py" module.parent.mkdir() @@ -74,6 +93,17 @@ def test_environment_digest_changes_with_collection_semantics( ) +def test_environment_digest_changes_when_root_conftest_is_added(tmp_path: Path) -> None: + initial = _testmon_environment_digest(tmp_path) + + (tmp_path / "conftest.py").write_text( + "def pytest_collection_modifyitems(items):\n items.reverse()\n", + encoding="utf-8", + ) + + assert _testmon_environment_digest(tmp_path) != initial + + def test_inactive_runtime_helper_does_not_force_fresh_environment(tmp_path: Path) -> None: helper = tmp_path / "tests" / "infra" / "runtime_helper.py" helper.parent.mkdir(parents=True) From e98ac04eb9b12ea50f29f8131932d3611b76d205 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 21:54:07 +0200 Subject: [PATCH 10/31] fix(test): finalize native lifecycle receipts --- devtools/testmon_bootstrap.py | 12 ++++++++++++ devtools/verify.py | 7 +++++-- devtools/verify_runs.py | 4 ++++ .../devtools/test_native_testmon_lifecycle.py | 2 +- tests/unit/devtools/test_run_tests.py | 2 ++ tests/unit/devtools/test_testmon_bootstrap.py | 16 ++++++++++++++++ tests/unit/devtools/test_verify.py | 17 +++++++++++++++-- 7 files changed, 55 insertions(+), 5 deletions(-) diff --git a/devtools/testmon_bootstrap.py b/devtools/testmon_bootstrap.py index be46dd4f04..9952e4967e 100644 --- a/devtools/testmon_bootstrap.py +++ b/devtools/testmon_bootstrap.py @@ -26,6 +26,7 @@ import json import os import platform +import shlex import sqlite3 import stat import subprocess @@ -190,6 +191,17 @@ def _active_local_pytest_plugin_paths(root: Path) -> set[str]: paths: set[str] = set() plugin_names = _declared_pytest_plugin_names(root) plugin_names.update(os.environ.get("PYTEST_PLUGINS", "").split(",")) + try: + addopts = shlex.split(os.environ.get("PYTEST_ADDOPTS", "")) + except ValueError as exc: + raise NativeTestmonRepairError(f"cannot parse PYTEST_ADDOPTS for native environment: {exc}") from exc + for index, option in enumerate(addopts): + if option == "-p" and index + 1 < len(addopts): + plugin_names.add(addopts[index + 1]) + elif option.startswith("-p="): + plugin_names.add(option.removeprefix("-p=")) + elif option.startswith("-p") and len(option) > 2: + plugin_names.add(option.removeprefix("-p")) for raw_name in plugin_names: module_name = raw_name.strip() if not module_name or any(part in {"", ".", ".."} for part in module_name.split(".")): diff --git a/devtools/verify.py b/devtools/verify.py index cf5f3c147b..d8dfca2983 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -2489,6 +2489,7 @@ def _finalize_preflight_failure( final_worktree_fingerprint=final_worktree_fingerprint, checkout_mutation_path=(mutation_observation.observed_path if mutation_observation is not None else None), checkout_diagnosis=checkout_diagnosis, + invocation_budget_s=VERIFY_INVOCATION_BUDGET_S, ) history_entry = { **payload, @@ -2932,7 +2933,7 @@ def _main(argv: list[str] | None = None) -> int: if run_diagnosis is not None: history_entry["diagnosis"] = run_diagnosis - verify_run.finish( + finalized_payload = verify_run.finish( exit_code=exit_code, duration_s=total_duration, diagnosis=run_diagnosis, @@ -2942,7 +2943,9 @@ def _main(argv: list[str] | None = None) -> int: checkout_mutation_path=mutation_observation.observed_path, checkout_diagnosis=checkout_diagnosis, pytest_aggregate=pytest_aggregate, + invocation_budget_s=VERIFY_INVOCATION_BUDGET_S, ) + history_entry["pytest_aggregate"] = finalized_payload["pytest_aggregate"] if use_json: _print_json(history_entry) elif exit_code == 0: @@ -2998,10 +3001,10 @@ def _finalize_verify_runner_exception( verification_scope=verification_scope.value, release_baseline_allowed=False, final_worktree_fingerprint=final_worktree_fingerprint, + invocation_budget_s=VERIFY_INVOCATION_BUDGET_S, ) payload["exception_type"] = type(exc).__name__ payload["error"] = str(exc) - payload["invocation_budget_s"] = VERIFY_INVOCATION_BUDGET_S _save_history(payload) if use_json: _print_json(payload) diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index 451bd6a4ef..a6b1a11046 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -1633,6 +1633,7 @@ def finish( checkout_mutation_path: str | None = None, checkout_diagnosis: str | None = None, pytest_aggregate: Mapping[str, Any] | None = None, + invocation_budget_s: float | None = None, ) -> dict[str, Any]: self._payload["finished_at"] = utc_now() self._payload["duration_s"] = round(duration_s, 2) @@ -1646,11 +1647,14 @@ def finish( self._payload["checkout_mutation_path"] = checkout_mutation_path if checkout_diagnosis is not None: self._payload["checkout_diagnosis"] = checkout_diagnosis + if invocation_budget_s is not None: + self._payload["invocation_budget_s"] = invocation_budget_s if pytest_aggregate is not None: self._payload["pytest_aggregate"] = dict(pytest_aggregate) if verification_scope is not None: self._payload["verification_scope"] = verification_scope self._payload["release_baseline_allowed"] = release_baseline_allowed + self._payload.setdefault("pytest_aggregate", _history_pytest_aggregate(self._payload)) self.write() return dict(self._payload) diff --git a/tests/integration/devtools/test_native_testmon_lifecycle.py b/tests/integration/devtools/test_native_testmon_lifecycle.py index 839de719ba..fbd8a5ba38 100644 --- a/tests/integration/devtools/test_native_testmon_lifecycle.py +++ b/tests/integration/devtools/test_native_testmon_lifecycle.py @@ -618,7 +618,7 @@ def test_environment_and_plugin_identity_changes_start_real_fresh_bootstraps( "def test_initial():\n assert True\n\ndef test_plugin(native_identity):\n assert native_identity == 'v1'\n", encoding="utf-8", ) - monkeypatch.setenv("PYTEST_PLUGINS", "local_plugin") + monkeypatch.setenv("PYTEST_ADDOPTS", "-p local_plugin") plugin_changed = prepare_native_testmon_environment(repo) assert plugin_changed.selection_mode == "bootstrap" assert plugin_changed.environment_name != environment_changed.environment_name diff --git a/tests/unit/devtools/test_run_tests.py b/tests/unit/devtools/test_run_tests.py index c57c78538a..ee4eccdbc8 100644 --- a/tests/unit/devtools/test_run_tests.py +++ b/tests/unit/devtools/test_run_tests.py @@ -267,6 +267,8 @@ def interrupt(*_args: Any, **_kwargs: Any) -> tuple[int, float, dict[str, Any]]: for payload in (history, run_payload, current_payload, receipt_payload): assert payload["diagnosis"] == "pytest_interrupted" assert payload["checkout_diagnosis"] == "checkout_changed_during_focused_test" + assert payload["pytest_aggregate"]["selection_mode"] == "focused" + assert history["pytest_aggregate"] == receipt_payload["pytest_aggregate"] def test_normalize_selection_paths_preserves_pytest_path_option_semantics( diff --git a/tests/unit/devtools/test_testmon_bootstrap.py b/tests/unit/devtools/test_testmon_bootstrap.py index ba0d3df9c5..ff9a5ae755 100644 --- a/tests/unit/devtools/test_testmon_bootstrap.py +++ b/tests/unit/devtools/test_testmon_bootstrap.py @@ -142,6 +142,22 @@ def test_environment_digest_hashes_explicit_local_plugin_regardless_of_name( assert _testmon_environment_digest(tmp_path) != initial +@pytest.mark.parametrize("addopts", ["-p local_plugin", "-p=local_plugin"]) +def test_environment_digest_hashes_local_plugin_from_pytest_addopts( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + addopts: str, +) -> None: + plugin = tmp_path / "local_plugin.py" + plugin.write_text("VALUE = 'v1'\n", encoding="utf-8") + monkeypatch.setenv("PYTEST_ADDOPTS", addopts) + + initial = _testmon_environment_digest(tmp_path) + plugin.write_text("VALUE = 'v2'\n", encoding="utf-8") + + assert _testmon_environment_digest(tmp_path) != initial + + def test_environment_digest_stops_at_invocation_deadline(tmp_path: Path) -> None: with pytest.raises(NativeTestmonDeadlineError, match="invocation deadline"): _testmon_environment_digest(tmp_path, deadline_monotonic=0.0) diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 768e074820..080bb54217 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -3503,8 +3503,15 @@ def test_read_pytest_report_parses_valid_payload(tmp_path: Path) -> None: assert parsed == {"summary": {"passed": 3}, "duration": 1.0} -def test_verify_continues_after_failed_cheap_step(capsys: pytest.CaptureFixture[str]) -> None: +def test_verify_continues_after_failed_cheap_step( + capsys: pytest.CaptureFixture[str], + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: calls: list[str] = [] + receipt = tmp_path / "quick-receipt.json" + monkeypatch.setenv(verify_runs.VERIFICATION_INVOCATION_ID_ENV, "quick-failure") + monkeypatch.setenv(verify_runs.VERIFICATION_RECEIPT_PATH_ENV, str(receipt)) def fake_run(label: str, command: list[str], **kwargs: object) -> tuple[int, float, dict[str, object]]: calls.append(label) @@ -3525,6 +3532,9 @@ def fake_run(label: str, command: list[str], **kwargs: object) -> tuple[int, flo assert payload["exit_code"] == 1 assert payload["verification_scope"] == "non-test" assert payload["release_baseline_allowed"] is False + assert payload["pytest_aggregate"]["selection_mode"] == "none" + assert payload["pytest_aggregate"]["deadline"] == {"budget_s": 3600.0, "met": True} + assert json.loads(receipt.read_text())["pytest_aggregate"] == payload["pytest_aggregate"] @pytest.mark.parametrize("fingerprints", [("unavailable", "stable"), ("stable", "unavailable")]) @@ -3770,7 +3780,10 @@ def test_import_guard_failure_writes_normalized_history_and_invocation_receipt( assert normalized["timestamp"] assert normalized["pytest_aggregate"]["selection_mode"] == "none" assert normalized["diagnosis"] == "checkout_import_mismatch" - assert json.loads(receipt.read_text())["diagnosis"] == "checkout_import_mismatch" + receipt_payload = json.loads(receipt.read_text()) + assert receipt_payload["diagnosis"] == "checkout_import_mismatch" + assert receipt_payload["pytest_aggregate"] == history["pytest_aggregate"] + assert receipt_payload["pytest_aggregate"]["deadline"] == {"budget_s": 3600.0, "met": True} assert json.loads(capsys.readouterr().out)["diagnosis"] == "checkout_import_mismatch" From 52632668c60e7d7a0f959924229f59facb29c337 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 23:30:13 +0200 Subject: [PATCH 11/31] fix(test): fail closed on runtime data changes Pytest-testmon traces Python execution but cannot observe package data reads. Classify changed non-Python files inside the shipped polylogue package as runtime data and route plain verify through the existing complete native corpus. Cover the reviewed pricing failure with seeded native state and a JSON-only mutation that executes both semantic owner lanes instead of succeeding with an empty affected selection. --- devtools/testmon_bootstrap.py | 31 ++++++++++ devtools/verify.py | 16 ++++- .../devtools/test_native_testmon_lifecycle.py | 59 +++++++++++++++++++ tests/unit/devtools/test_testmon_bootstrap.py | 26 ++++++++ 4 files changed, 129 insertions(+), 3 deletions(-) diff --git a/devtools/testmon_bootstrap.py b/devtools/testmon_bootstrap.py index 9952e4967e..cd8fccf0ac 100644 --- a/devtools/testmon_bootstrap.py +++ b/devtools/testmon_bootstrap.py @@ -101,6 +101,14 @@ class NativeTestmonPreparation: main_checkout: Path | None +@dataclass(frozen=True, slots=True) +class NativeTestmonChangeImpact: + """Changed inputs that native Python tracing can and cannot select.""" + + executable_paths: tuple[str, ...] + runtime_data_paths: tuple[str, ...] + + class NativeTestmonRepairError(RuntimeError): """The exact derived testmon state could not be repaired safely.""" @@ -367,6 +375,27 @@ def executable_python_paths(repo_root: Path, paths: Iterable[str]) -> tuple[str, return tuple(executable) +def classify_native_testmon_changes(repo_root: Path, paths: Iterable[str]) -> NativeTestmonChangeImpact: + """Classify changed product inputs against native testmon's trace boundary. + + Pytest-testmon records Python execution. Every non-Python file inside the + shipped ``polylogue`` package is therefore package-owned runtime data and + cannot safely use affected selection. The caller must run the complete + native corpus for those changes. This convention covers additions, + deletions, and all package-data formats without a filename registry. + """ + normalized = tuple(relative for raw in sorted(set(paths)) if (relative := _safe_relative_path(raw)) is not None) + runtime_data = tuple( + relative + for relative in normalized + if PurePosixPath(relative).parts[0] == "polylogue" and not relative.endswith(".py") + ) + return NativeTestmonChangeImpact( + executable_paths=executable_python_paths(repo_root, normalized), + runtime_data_paths=runtime_data, + ) + + def _readonly_uri(path: Path) -> str: return f"{path.resolve().as_uri()}?mode=ro" @@ -681,11 +710,13 @@ def prepare_native_testmon_environment( "ASTClassification", "NativeTestmonEnvironment", "NativeTestmonDeadlineError", + "NativeTestmonChangeImpact", "NativeTestmonPreparation", "NativeTestmonRepairError", "NativeTestmonState", "TESTMON_DATA_RELPATH", "classify_source_ast", + "classify_native_testmon_changes", "executable_python_paths", "inspect_native_testmon_environment", "linked_worktree_info", diff --git a/devtools/verify.py b/devtools/verify.py index d8dfca2983..d1aa154ad3 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -62,7 +62,7 @@ NativeTestmonPreparation, NativeTestmonRepairError, NativeTestmonState, - executable_python_paths, + classify_native_testmon_changes, inspect_native_testmon_environment, prepare_native_testmon_environment, remove_invalid_native_testmon_state, @@ -2624,6 +2624,7 @@ def _main(argv: list[str] | None = None) -> int: relevant_paths: tuple[str, ...] = () required_executable_paths: tuple[str, ...] = () + runtime_data_paths: tuple[str, ...] = () preparation: NativeTestmonPreparation | None = None testmon_mode: str | None = None if pytest_enabled: @@ -2631,7 +2632,9 @@ def _main(argv: list[str] | None = None) -> int: assert head is not None try: relevant_paths = _changed_test_relevant_paths(base_commit, head) - required_executable_paths = executable_python_paths(ROOT, relevant_paths) + change_impact = classify_native_testmon_changes(ROOT, relevant_paths) + required_executable_paths = change_impact.executable_paths + runtime_data_paths = change_impact.runtime_data_paths preparation = prepare_native_testmon_environment( ROOT, required_executable_paths=required_executable_paths, @@ -2666,7 +2669,7 @@ def _main(argv: list[str] | None = None) -> int: mutation_monitor=mutation_monitor, initial_worktree_fingerprint=checkout_fingerprint, ) - testmon_mode = "full" if full_requested else preparation.selection_mode + testmon_mode = "full" if full_requested or runtime_data_paths else preparation.selection_mode if preparation.removed_paths: sys.stderr.write( "verify: repaired invalid native pytest-testmon state by removing only " @@ -2677,6 +2680,12 @@ def _main(argv: list[str] | None = None) -> int: sys.stderr.write(f"verify: copied matching native pytest-testmon DB from {preparation.copied_from}\n") elif preparation.selection_mode == "bootstrap": sys.stderr.write("verify: native pytest-testmon environment is empty; plain verify will build it\n") + if runtime_data_paths and not full_requested: + sys.stderr.write( + "verify: changed package runtime data is outside Python tracing; running the complete corpus: " + + ", ".join(runtime_data_paths) + + "\n" + ) planned_scope = _planned_verification_scope(args, testmon_mode=testmon_mode) _ACTIVE_VERIFY_RUN = (verify_run, started_at, planned_scope) @@ -2927,6 +2936,7 @@ def _main(argv: list[str] | None = None) -> int: "selection_mode": testmon_mode, "copied_from": str(preparation.copied_from) if preparation.copied_from is not None else None, "required_executable_paths": list(required_executable_paths), + "runtime_data_paths": list(runtime_data_paths), } if pytest_aggregate is not None: history_entry["pytest_aggregate"] = pytest_aggregate diff --git a/tests/integration/devtools/test_native_testmon_lifecycle.py b/tests/integration/devtools/test_native_testmon_lifecycle.py index fbd8a5ba38..cfe723beab 100644 --- a/tests/integration/devtools/test_native_testmon_lifecycle.py +++ b/tests/integration/devtools/test_native_testmon_lifecycle.py @@ -373,6 +373,65 @@ def test_serial_owner(): assert "assert 0 == 42" in third.stderr +def test_runtime_json_only_mutation_forces_complete_native_selection(tmp_path: Path) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _init_repo(repo) + semantic = repo / "polylogue" / "archive" / "semantic" + data = semantic / "data" + data.mkdir(parents=True) + for package in (repo / "polylogue", repo / "polylogue" / "archive", semantic): + (package / "__init__.py").write_text("", encoding="utf-8") + pricing_data = data / "litellm_model_prices.json" + pricing_data.write_text('{"test-model": {"input_cost_per_token": 42}}\n', encoding="utf-8") + (semantic / "pricing.py").write_text( + "import json\n" + "from pathlib import Path\n\n" + "def input_price() -> int:\n" + " path = Path(__file__).parent / 'data' / 'litellm_model_prices.json'\n" + " return int(json.loads(path.read_text(encoding='utf-8'))['test-model']['input_cost_per_token'])\n", + encoding="utf-8", + ) + (repo / "tests" / "test_pricing.py").write_text( + "import pytest\n\n" + "def test_parallel_pricing_owner():\n" + " from polylogue.archive.semantic.pricing import input_price\n" + " assert input_price() == 42\n\n" + "@pytest.mark.load_sensitive\n" + "def test_serial_pricing_owner():\n" + " from polylogue.archive.semantic.pricing import input_price\n" + " assert input_price() == 42\n", + encoding="utf-8", + ) + _commit_all(repo, "fixture") + origin = tmp_path / "origin.git" + _git(origin.parent, "init", "--bare", "-q", str(origin)) + _git(repo, "remote", "add", "origin", str(origin)) + _git(repo, "branch", "-M", "master") + _git(repo, "push", "-qu", "origin", "master") + + seeded, bootstrap = _run_production_verify(repo) + assert seeded.returncode == 0, seeded.stderr + assert bootstrap["testmon_environment"]["selection_mode"] == "bootstrap" + + pricing_data.write_text('{"test-model": {"input_cost_per_token": 0}}\n', encoding="utf-8") + completed, mutated = _run_production_verify(repo) + + assert completed.returncode == 1 + assert mutated["testmon_environment"]["selection_mode"] == "full" + assert mutated["testmon_environment"]["runtime_data_paths"] == [ + "polylogue/archive/semantic/data/litellm_model_prices.json" + ] + assert mutated["pytest_aggregate"]["selected_union_count"] == 2 + assert mutated["pytest_aggregate"]["terminal_union_count"] == 2 + assert mutated["release_baseline_allowed"] is False + assert [step["semantic_lane"] for step in mutated["steps"] if step.get("semantic_lane")] == [ + "parallel", + "serial", + ] + assert "assert 0 == 42" in completed.stderr + + def test_empty_linked_worktree_with_empty_main_self_bootstraps(tmp_path: Path) -> None: main = tmp_path / "main" main.mkdir() diff --git a/tests/unit/devtools/test_testmon_bootstrap.py b/tests/unit/devtools/test_testmon_bootstrap.py index ff9a5ae755..ed336ed8b8 100644 --- a/tests/unit/devtools/test_testmon_bootstrap.py +++ b/tests/unit/devtools/test_testmon_bootstrap.py @@ -7,6 +7,7 @@ from devtools.testmon_bootstrap import ( NativeTestmonDeadlineError, NativeTestmonRepairError, + classify_native_testmon_changes, classify_source_ast, executable_python_paths, remove_invalid_native_testmon_state, @@ -64,6 +65,31 @@ def test_executable_paths_require_current_runtime_modules_but_allow_deletion(tmp ) +def test_package_runtime_data_changes_force_full_native_selection(tmp_path: Path) -> None: + runtime = tmp_path / "polylogue" / "archive" / "semantic" / "data" / "prices.json" + runtime.parent.mkdir(parents=True) + runtime.write_text("{}\n", encoding="utf-8") + declaration = runtime.with_name("types.pyi") + declaration.write_text("VALUE: int\n", encoding="utf-8") + + impact = classify_native_testmon_changes( + tmp_path, + ( + "polylogue/archive/semantic/data/prices.json", + "polylogue/archive/semantic/data/deleted.json", + "polylogue/archive/semantic/data/types.pyi", + "docs/prices.json", + ), + ) + + assert impact.executable_paths == () + assert impact.runtime_data_paths == ( + "polylogue/archive/semantic/data/deleted.json", + "polylogue/archive/semantic/data/prices.json", + "polylogue/archive/semantic/data/types.pyi", + ) + + def test_environment_digest_changes_with_collection_semantics( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, From 7e01d937d0ffe856da3ff711ac1a79359991a6ed Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 23:32:11 +0200 Subject: [PATCH 12/31] test(devtools): prove full release authority path Exercise production devtools verify --all in a tiny repository and require both semantic lanes, full native selection, complete terminal coverage, stable checkout evidence, cleanup, containment, deadline compliance, and release-baseline authority. --- .../devtools/test_native_testmon_lifecycle.py | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/tests/integration/devtools/test_native_testmon_lifecycle.py b/tests/integration/devtools/test_native_testmon_lifecycle.py index cfe723beab..538ad0a9e3 100644 --- a/tests/integration/devtools/test_native_testmon_lifecycle.py +++ b/tests/integration/devtools/test_native_testmon_lifecycle.py @@ -373,6 +373,60 @@ def test_serial_owner(): assert "assert 0 == 42" in third.stderr +def test_production_verify_all_grants_release_authority_after_complete_two_lane_run(tmp_path: Path) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _init_repo(repo) + package = repo / "polylogue" + package.mkdir() + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "app.py").write_text("def answer() -> int:\n return 42\n", encoding="utf-8") + (repo / "tests" / "test_release.py").write_text( + "import pytest\n\n" + "def test_parallel_release_owner():\n" + " from polylogue.app import answer\n" + " assert answer() == 42\n\n" + "@pytest.mark.load_sensitive\n" + "def test_serial_release_owner():\n" + " from polylogue.app import answer\n" + " assert answer() == 42\n", + encoding="utf-8", + ) + _commit_all(repo, "fixture") + origin = tmp_path / "origin.git" + _git(origin.parent, "init", "--bare", "-q", str(origin)) + _git(repo, "remote", "add", "origin", str(origin)) + _git(repo, "branch", "-M", "master") + _git(repo, "push", "-qu", "origin", "master") + + completed, payload = _run_production_verify(repo, "--all") + + assert completed.returncode == 0, completed.stderr + assert payload["tier"] == "full" + assert payload["testmon_environment"]["selection_mode"] == "full" + assert payload["verification_scope"] == "release-baseline" + assert payload["release_baseline_allowed"] is True + assert payload["worktree_fingerprint"] == payload["final_worktree_fingerprint"] + lanes = [step for step in payload["steps"] if step.get("semantic_lane")] + assert [step["semantic_lane"] for step in lanes] == ["parallel", "serial"] + assert [step["name"] for step in lanes] == ["pytest native parallel (full)", "pytest native serial (full)"] + for step in lanes: + assert "--testmon-noselect" in step["statistics"]["command"] + assert "--testmon-forceselect" not in step["statistics"]["command"] + aggregate = payload["pytest_aggregate"] + assert aggregate["selection_mode"] == "full" + assert aggregate["environment"]["native_corpus_count"] == 2 + assert aggregate["corpus"]["count"] == 2 + assert aggregate["selected_union_count"] == 2 + assert aggregate["terminal_union_count"] == 2 + assert aggregate["missing_terminal_count"] == 0 + assert aggregate["complete_corpus_covered"] is True + assert aggregate["terminal_green"] is True + assert aggregate["cleanup"] == {"complete": True} + assert aggregate["containment"] == {"complete": True} + assert aggregate["deadline"] == {"budget_s": 3600.0, "met": True} + + def test_runtime_json_only_mutation_forces_complete_native_selection(tmp_path: Path) -> None: repo = tmp_path / "repo" repo.mkdir() From c64115b31a4bb3c77ccb0273a9e973b6f4c29e66 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 14 Aug 2026 00:51:17 +0200 Subject: [PATCH 13/31] fix(testmon): fail closed on untraceable runtime changes Problem: native Python tracing cannot observe test runtime data, and a deleted module can remain present only in the stale dependency graph. What changed: classify non-Python data under polylogue and tests as complete-corpus inputs, retain deleted Python paths, and reject stale linked-worktree reuse when required modules are absent. Verification: focused unit and native lifecycle tests plus devtools verify --quick pass. --- devtools/testmon_bootstrap.py | 37 ++++-- .../devtools/test_native_testmon_lifecycle.py | 117 ++++++++++++++++++ tests/unit/devtools/test_testmon_bootstrap.py | 24 +++- 3 files changed, 166 insertions(+), 12 deletions(-) diff --git a/devtools/testmon_bootstrap.py b/devtools/testmon_bootstrap.py index cd8fccf0ac..4bc923b12d 100644 --- a/devtools/testmon_bootstrap.py +++ b/devtools/testmon_bootstrap.py @@ -360,7 +360,12 @@ def _safe_relative_path(raw: str) -> str | None: def executable_python_paths(repo_root: Path, paths: Iterable[str]) -> tuple[str, ...]: - """Return changed Python paths whose runtime behavior needs graph edges.""" + """Return changed Python paths whose runtime behavior needs graph edges. + + A deleted module cannot be parsed, but it still invalidates any graph edge + that pointed at it. Keep that path in the required set so inspection fails + closed and the verifier rebuilds the complete native corpus. + """ root = repo_root.resolve() executable: list[str] = [] for raw in sorted(set(paths)): @@ -368,27 +373,27 @@ def executable_python_paths(repo_root: Path, paths: Iterable[str]) -> tuple[str, if relative is None or not relative.endswith(".py"): continue source = root / relative - if not source.exists(): + if source.exists() and source.is_file() and classify_source_ast(source) == "declaration-only": continue - if not source.is_file() or classify_source_ast(source) != "declaration-only": - executable.append(relative) + executable.append(relative) return tuple(executable) def classify_native_testmon_changes(repo_root: Path, paths: Iterable[str]) -> NativeTestmonChangeImpact: """Classify changed product inputs against native testmon's trace boundary. - Pytest-testmon records Python execution. Every non-Python file inside the - shipped ``polylogue`` package is therefore package-owned runtime data and - cannot safely use affected selection. The caller must run the complete - native corpus for those changes. This convention covers additions, - deletions, and all package-data formats without a filename registry. + Python tracing does not observe non-Python runtime data. Non-Python files + under the shipped ``polylogue`` package or the test runtime tree are + therefore outside the native graph and cannot safely use affected + selection. The caller must run the complete native corpus for those + changes. This convention covers additions, deletions, and all data formats + without a filename registry. """ normalized = tuple(relative for raw in sorted(set(paths)) if (relative := _safe_relative_path(raw)) is not None) runtime_data = tuple( relative for relative in normalized - if PurePosixPath(relative).parts[0] == "polylogue" and not relative.endswith(".py") + if not relative.endswith(".py") and relative.startswith(("polylogue/", "tests/")) ) return NativeTestmonChangeImpact( executable_paths=executable_python_paths(repo_root, normalized), @@ -659,6 +664,16 @@ def prepare_native_testmon_environment( required_executable_paths=required_executable_paths, deadline_monotonic=deadline_monotonic, ) + missing_checkout_paths = tuple( + path for path in sorted(set(required_executable_paths)) if not (root / path).is_file() + ) + if local.valid and missing_checkout_paths: + local = NativeTestmonState( + "invalid", + "changed executable modules are absent from the current checkout", + local.environment, + missing_checkout_paths, + ) info = linked_worktree_info(root, deadline_monotonic=deadline_monotonic) linked = bool(info and info[0]) main_checkout = info[1] if linked and info is not None else None @@ -668,7 +683,7 @@ def prepare_native_testmon_environment( removed = remove_invalid_native_testmon_state(root) _ensure_deadline(deadline_monotonic) copied_from: Path | None = None - if main_checkout is not None and main_checkout != root: + if main_checkout is not None and main_checkout != root and not missing_checkout_paths: main_data = main_checkout / TESTMON_DATA_RELPATH main = inspect_native_testmon_environment( main_data, diff --git a/tests/integration/devtools/test_native_testmon_lifecycle.py b/tests/integration/devtools/test_native_testmon_lifecycle.py index 538ad0a9e3..5823323c7a 100644 --- a/tests/integration/devtools/test_native_testmon_lifecycle.py +++ b/tests/integration/devtools/test_native_testmon_lifecycle.py @@ -486,6 +486,123 @@ def test_runtime_json_only_mutation_forces_complete_native_selection(tmp_path: P assert "assert 0 == 42" in completed.stderr +def test_production_verify_test_runtime_data_mutation_executes_and_fails(tmp_path: Path) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _init_repo(repo) + package = repo / "polylogue" + package.mkdir() + (package / "__init__.py").write_text("", encoding="utf-8") + data = repo / "tests" / "data" / "expected.txt" + data.parent.mkdir() + data.write_text("42\n", encoding="utf-8") + (repo / "tests" / "test_data.py").write_text( + "import pytest\n" + "from pathlib import Path\n\n" + "def expected() -> int:\n" + " return int((Path(__file__).parent / 'data' / 'expected.txt').read_text())\n\n" + "def test_parallel_data_owner():\n" + " assert expected() == 42\n\n" + "@pytest.mark.load_sensitive\n" + "def test_serial_data_owner():\n" + " assert expected() == 42\n", + encoding="utf-8", + ) + _commit_all(repo, "fixture") + origin = tmp_path / "origin.git" + _git(origin.parent, "init", "--bare", "-q", str(origin)) + _git(repo, "remote", "add", "origin", str(origin)) + _git(repo, "branch", "-M", "master") + _git(repo, "push", "-qu", "origin", "master") + + seeded, bootstrap = _run_production_verify(repo) + assert seeded.returncode == 0, seeded.stderr + assert bootstrap["testmon_environment"]["selection_mode"] == "bootstrap" + + data.write_text("0\n", encoding="utf-8") + completed, mutated = _run_production_verify(repo) + + assert completed.returncode == 1 + assert mutated["testmon_environment"]["selection_mode"] == "full" + assert mutated["testmon_environment"]["runtime_data_paths"] == ["tests/data/expected.txt"] + assert mutated["pytest_aggregate"]["selected_union_count"] == 2 + assert mutated["pytest_aggregate"]["terminal_union_count"] == 2 + assert "assert 0 == 42" in completed.stderr + + +def test_production_verify_deleted_module_rebuilds_and_fails_dependents(tmp_path: Path) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _init_repo(repo) + package = repo / "polylogue" + package.mkdir() + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "app.py").write_text("def answer() -> int:\n return 42\n", encoding="utf-8") + (repo / "tests" / "test_app.py").write_text( + "def test_answer():\n from polylogue.app import answer\n assert answer() == 42\n", + encoding="utf-8", + ) + _commit_all(repo, "fixture") + origin = tmp_path / "origin.git" + _git(origin.parent, "init", "--bare", "-q", str(origin)) + _git(repo, "remote", "add", "origin", str(origin)) + _git(repo, "branch", "-M", "master") + _git(repo, "push", "-qu", "origin", "master") + + seeded, bootstrap = _run_production_verify(repo) + assert seeded.returncode == 0, seeded.stderr + assert bootstrap["pytest_aggregate"]["selected_union_count"] == 1 + + _git(repo, "rm", "polylogue/app.py") + _commit_all(repo, "delete production module") + completed, mutated = _run_production_verify(repo) + + assert completed.returncode == 1 + assert mutated["testmon_environment"]["selection_mode"] == "bootstrap" + assert mutated["testmon_environment"]["required_executable_paths"] == ["polylogue/app.py"] + assert mutated["pytest_aggregate"]["selected_union_count"] == 1 + assert mutated["pytest_aggregate"]["terminal_union_count"] == 1 + assert "ModuleNotFoundError" in completed.stderr + + +def test_production_verify_moved_module_rebuilds_and_fails_dependents(tmp_path: Path) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _init_repo(repo) + package = repo / "polylogue" + package.mkdir() + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "app.py").write_text("def answer() -> int:\n return 42\n", encoding="utf-8") + (repo / "tests" / "test_app.py").write_text( + "def test_answer():\n from polylogue.app import answer\n assert answer() == 42\n", + encoding="utf-8", + ) + _commit_all(repo, "fixture") + origin = tmp_path / "origin.git" + _git(origin.parent, "init", "--bare", "-q", str(origin)) + _git(repo, "remote", "add", "origin", str(origin)) + _git(repo, "branch", "-M", "master") + _git(repo, "push", "-qu", "origin", "master") + + seeded, bootstrap = _run_production_verify(repo) + assert seeded.returncode == 0, seeded.stderr + assert bootstrap["pytest_aggregate"]["selected_union_count"] == 1 + + _git(repo, "mv", "polylogue/app.py", "polylogue/renamed.py") + _commit_all(repo, "move production module") + completed, mutated = _run_production_verify(repo) + + assert completed.returncode == 1 + assert mutated["testmon_environment"]["selection_mode"] == "bootstrap" + assert mutated["testmon_environment"]["required_executable_paths"] == [ + "polylogue/app.py", + "polylogue/renamed.py", + ] + assert mutated["pytest_aggregate"]["selected_union_count"] == 1 + assert mutated["pytest_aggregate"]["terminal_union_count"] == 1 + assert "ModuleNotFoundError" in completed.stderr + + def test_empty_linked_worktree_with_empty_main_self_bootstraps(tmp_path: Path) -> None: main = tmp_path / "main" main.mkdir() diff --git a/tests/unit/devtools/test_testmon_bootstrap.py b/tests/unit/devtools/test_testmon_bootstrap.py index ed336ed8b8..26b538b13d 100644 --- a/tests/unit/devtools/test_testmon_bootstrap.py +++ b/tests/unit/devtools/test_testmon_bootstrap.py @@ -49,7 +49,7 @@ def test_ast_classification_treats_type_checking_guards_as_declarations(tmp_path assert classify_source_ast(declarations) == "executable" -def test_executable_paths_require_current_runtime_modules_but_allow_deletion(tmp_path: Path) -> None: +def test_executable_paths_require_current_runtime_modules_and_deleted_modules(tmp_path: Path) -> None: module = tmp_path / "polylogue" / "runtime.py" module.parent.mkdir() module.write_text("VALUE = factory()\n", encoding="utf-8") @@ -60,6 +60,7 @@ def test_executable_paths_require_current_runtime_modules_but_allow_deletion(tmp tmp_path, ("polylogue/runtime.py", "polylogue/malformed.py", "polylogue/deleted.py"), ) == ( + "polylogue/deleted.py", "polylogue/malformed.py", "polylogue/runtime.py", ) @@ -90,6 +91,27 @@ def test_package_runtime_data_changes_force_full_native_selection(tmp_path: Path ) +def test_test_runtime_data_changes_force_full_native_selection(tmp_path: Path) -> None: + runtime = tmp_path / "tests" / "data" / "payload.json" + runtime.parent.mkdir(parents=True) + runtime.write_text("{}\n", encoding="utf-8") + + impact = classify_native_testmon_changes( + tmp_path, + ( + "tests/data/payload.json", + "tests/data/deleted.json", + "docs/payload.json", + ), + ) + + assert impact.executable_paths == () + assert impact.runtime_data_paths == ( + "tests/data/deleted.json", + "tests/data/payload.json", + ) + + def test_environment_digest_changes_with_collection_semantics( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, From 715b325d7bc94fb16634ab9840cc669a7ac196bc Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 14 Aug 2026 02:02:35 +0200 Subject: [PATCH 14/31] fix: harden native testmon verification lifecycle --- .github/workflows/nightly-scale.yml | 1 + CONTRIBUTING.md | 10 +- TESTING.md | 5 +- devtools/merge_boundary.py | 4 +- devtools/testmon_bootstrap.py | 54 +++++-- devtools/verify.py | 40 ++++- devtools/verify_runs.py | 135 ++++++++-------- pyproject.toml | 2 +- tests/benchmarks/test_scale_tiers.py | 10 +- .../devtools/test_native_testmon_lifecycle.py | 115 +++++++++++++- tests/unit/devtools/test_merge_boundary.py | 28 ++++ tests/unit/devtools/test_testmon_bootstrap.py | 50 ++++++ tests/unit/devtools/test_verify.py | 145 +++++++++++++++++- tests/unit/infra/test_archive_templates.py | 8 +- tests/unit/storage/test_perf_rescue_1314.py | 26 +++- uv.lock | 2 +- 16 files changed, 515 insertions(+), 120 deletions(-) diff --git a/.github/workflows/nightly-scale.yml b/.github/workflows/nightly-scale.yml index e9f0815b87..d38f5e3daf 100644 --- a/.github/workflows/nightly-scale.yml +++ b/.github/workflows/nightly-scale.yml @@ -47,6 +47,7 @@ jobs: HYPOTHESIS_PROFILE: ci run: | uv run pytest -q --tb=short -p no:randomly -n 0 \ + --benchmark-enable \ --benchmark-json=nightly-results.json \ --benchmark-group-by=group \ 'tests/benchmarks/test_daemon_convergence.py::test_convergence_scale_tier[xxl-mega-session]' \ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0379fb0653..e3b9dffc71 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -305,10 +305,12 @@ repairs missing or invalid native testmon state and automatically builds a new environment when collection semantics change. `devtools verify` does not replay a prior verify result. It always runs the -static gates and then invokes pytest-testmon for affected-test selection from -the current source, dependency, and Python-version state. The pytest step -covers unit, property, fuzz, and integration tests while excluding the -separately operated `tests/benchmarks` performance surface. It uses +static gates. With a valid native environment, it invokes pytest-testmon for +affected-test selection from the current source, dependency, and Python-version +state. When it bootstraps a missing or invalid native environment, it runs the +complete correctness corpus. The pytest step covers unit, property, fuzz, and +integration tests while excluding the separately operated `tests/benchmarks` +performance surface. It uses `--testmon-forceselect` for affected selection, with one parallel `not load_sensitive` lane and one serial `load_sensitive` lane over the same native environment. `tui` is a category marker and remains parallel unless a test is diff --git a/TESTING.md b/TESTING.md index 181f969cc9..ab6b6ace45 100644 --- a/TESTING.md +++ b/TESTING.md @@ -57,8 +57,9 @@ receipt or resume command: pytest-testmon keeps failed, unfinished, and new tests selected on the next plain invocation. The native environment key includes Python, active distributions, lock and -project metadata, pytest configuration, every `conftest.py`, and collection-active -local pytest plugins. Ordinary `tests/infra` helpers stay incremental: import +project metadata, pytest configuration, the repository-root and `tests/**` +`conftest.py` files, and collection-active local pytest plugins. Ordinary +`tests/infra` helpers stay incremental: import them from an executing fixture or test when their module initialization is dependency-bearing, so pytest-testmon observes that initialization and helper execution instead of forcing a complete-corpus bootstrap for every helper edit. diff --git a/devtools/merge_boundary.py b/devtools/merge_boundary.py index d4022210e4..b6c10385dd 100644 --- a/devtools/merge_boundary.py +++ b/devtools/merge_boundary.py @@ -207,8 +207,8 @@ def _validate_ledger(data: object) -> dict[str, Any]: ): raise LedgerStateError("merge-train terminal receipt has malformed status fields") scope = receipt.get("verification_scope") - if scope is not None and scope not in {item.value for item in VerificationScope}: - raise LedgerStateError("merge-train terminal receipt has an invalid verification scope") + if scope is not None and not isinstance(scope, str): + raise LedgerStateError("merge-train terminal receipt has a malformed verification scope") permission = receipt.get("release_baseline_allowed") if permission is not None and not isinstance(permission, bool): raise LedgerStateError("merge-train terminal receipt has malformed release permission") diff --git a/devtools/testmon_bootstrap.py b/devtools/testmon_bootstrap.py index 4bc923b12d..3b00cf946b 100644 --- a/devtools/testmon_bootstrap.py +++ b/devtools/testmon_bootstrap.py @@ -305,7 +305,16 @@ def _body_is_executable(body: list[ast.stmt]) -> bool: for index, node in enumerate(body): if _is_docstring(node, first=index == 0): continue - if isinstance(node, (ast.Pass, ast.Import, ast.ImportFrom)): + if isinstance(node, ast.Pass): + continue + if ( + isinstance(node, ast.ImportFrom) + and node.module == "typing" + and all(alias.name == "TYPE_CHECKING" for alias in node.names) + ): + # Importing the sentinel only enables a declaration-only guard. + # Imports elsewhere execute at module import time and therefore + # remain executable graph inputs. continue if ( isinstance(node, ast.Expr) @@ -390,13 +399,19 @@ def classify_native_testmon_changes(repo_root: Path, paths: Iterable[str]) -> Na without a filename registry. """ normalized = tuple(relative for raw in sorted(set(paths)) if (relative := _safe_relative_path(raw)) is not None) - runtime_data = tuple( + native_paths = tuple( relative for relative in normalized - if not relative.endswith(".py") and relative.startswith(("polylogue/", "tests/")) + if relative != "tests/benchmarks" and not relative.startswith("tests/benchmarks/") + ) + runtime_data = tuple( + relative + for relative in native_paths + if (not relative.endswith(".py") and relative.startswith(("polylogue/", "tests/"))) + or relative.startswith("packaging/") ) return NativeTestmonChangeImpact( - executable_paths=executable_python_paths(repo_root, normalized), + executable_paths=executable_python_paths(repo_root, native_paths), runtime_data_paths=runtime_data, ) @@ -438,11 +453,16 @@ def inspect_native_testmon_environment( if not stat.S_ISREG(mode): return NativeTestmonState("invalid", "native testmon database is not a regular file") try: - with sqlite3.connect( - _readonly_uri(data_path), - uri=True, - timeout=_remaining_timeout(deadline_monotonic, 10), - ) as connection: + with ( + contextlib.closing( + sqlite3.connect( + _readonly_uri(data_path), + uri=True, + timeout=_remaining_timeout(deadline_monotonic, 10), + ) + ) as connection, + connection, + ): if deadline_monotonic is not None: connection.set_progress_handler(lambda: int(time.monotonic() >= deadline_monotonic), 1_000) quick_check = connection.execute("PRAGMA quick_check").fetchone() @@ -571,12 +591,18 @@ def _atomic_copy_sqlite_database( temporary = destination.with_name(f".{destination.name}.copy-{os.getpid()}-{uuid.uuid4().hex}.tmp") try: with ( - sqlite3.connect( - _readonly_uri(source), - uri=True, - timeout=_remaining_timeout(deadline_monotonic, 60), + contextlib.closing( + sqlite3.connect( + _readonly_uri(source), + uri=True, + timeout=_remaining_timeout(deadline_monotonic, 60), + ) ) as source_connection, - sqlite3.connect(temporary, timeout=_remaining_timeout(deadline_monotonic, 60)) as destination_connection, + contextlib.closing( + sqlite3.connect(temporary, timeout=_remaining_timeout(deadline_monotonic, 60)) + ) as destination_connection, + source_connection, + destination_connection, ): source_connection.backup( destination_connection, diff --git a/devtools/verify.py b/devtools/verify.py index d1aa154ad3..4c3b16b7da 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -2368,6 +2368,22 @@ def _changed_test_relevant_paths(base_commit: str, head_commit: str) -> tuple[st ) +def _has_inherited_collection_selector() -> bool: + """Return whether ambient pytest options narrow the collected corpus.""" + raw = os.environ.get("PYTEST_ADDOPTS", "") + if not raw.strip(): + return False + try: + tokens = shlex.split(raw) + except ValueError: + return True + selectors = {"-k", "-m", "--keyword", "--markexpr", "--deselect", "--ignore", "--ignore-glob"} + return any( + token in selectors or token.startswith(("--keyword=", "--markexpr=", "--deselect=", "--ignore=")) + for token in tokens + ) + + _ACTIVE_VERIFY_RUN: tuple[VerifyRun, float, VerificationScope] | None = None @@ -2446,11 +2462,11 @@ def _finalize_preflight_failure( ) -> int: """Persist one normalized failed invocation before pytest can start.""" final_head = _git_head() + mutation_observation = finish_checkout_mutation_monitor(mutation_monitor) if mutation_monitor is not None else None try: final_worktree_fingerprint = worktree_fingerprint(ROOT) if mutation_monitor is not None else "unavailable" except Exception: final_worktree_fingerprint = "unavailable" - mutation_observation = finish_checkout_mutation_monitor(mutation_monitor) if mutation_monitor is not None else None checkout_diagnosis: str | None = None if mutation_monitor is None: checkout_diagnosis = "preflight_failed_before_checkout_monitor" @@ -2604,6 +2620,9 @@ def _main(argv: list[str] | None = None) -> int: mutation_monitor = CheckoutMutationMonitor(ROOT) start_checkout_mutation_monitor(mutation_monitor) checkout_fingerprint = worktree_fingerprint(ROOT) + finish_checkout_mutation_monitor(mutation_monitor) + mutation_monitor = CheckoutMutationMonitor(ROOT) + start_checkout_mutation_monitor(mutation_monitor) verify_run.update_checkout_provenance(worktree_fingerprint=checkout_fingerprint) base_commit = _git_commit("origin/master") if pytest_enabled else None @@ -2624,6 +2643,7 @@ def _main(argv: list[str] | None = None) -> int: relevant_paths: tuple[str, ...] = () required_executable_paths: tuple[str, ...] = () + preparation_required_executable_paths: tuple[str, ...] = () runtime_data_paths: tuple[str, ...] = () preparation: NativeTestmonPreparation | None = None testmon_mode: str | None = None @@ -2633,11 +2653,14 @@ def _main(argv: list[str] | None = None) -> int: try: relevant_paths = _changed_test_relevant_paths(base_commit, head) change_impact = classify_native_testmon_changes(ROOT, relevant_paths) - required_executable_paths = change_impact.executable_paths + preparation_required_executable_paths = change_impact.executable_paths + required_executable_paths = tuple( + path for path in preparation_required_executable_paths if (ROOT / path).is_file() + ) runtime_data_paths = change_impact.runtime_data_paths preparation = prepare_native_testmon_environment( ROOT, - required_executable_paths=required_executable_paths, + required_executable_paths=preparation_required_executable_paths, pytest_profile=_pytest_profile(), deadline_monotonic=started_at + VERIFY_INVOCATION_BUDGET_S, ) @@ -2718,6 +2741,12 @@ def _main(argv: list[str] | None = None) -> int: initial_worktree_fingerprint=checkout_fingerprint, ) + # Git probes and native testmon preparation can refresh the index as part + # of their own read path. Discard that preflight interval and begin the + # authority interval immediately before the verification steps. + finish_checkout_mutation_monitor(mutation_monitor) + mutation_monitor = CheckoutMutationMonitor(ROOT) + start_checkout_mutation_monitor(mutation_monitor) step_results: list[dict[str, Any]] = [] exit_code = 0 native_graph_touched = False @@ -2755,9 +2784,10 @@ def _main(argv: list[str] | None = None) -> int: if label.startswith("pytest") or rc == 130 or _stop_after_failed_step(label): break + assert mutation_monitor is not None + mutation_observation = finish_checkout_mutation_monitor(mutation_monitor) final_head = _git_head() final_checkout_fingerprint = worktree_fingerprint(ROOT) - mutation_observation = finish_checkout_mutation_monitor(mutation_monitor) checkout_stable = True checkout_fingerprint_unavailable = ( head is None @@ -2856,6 +2886,7 @@ def _main(argv: list[str] | None = None) -> int: selection_mode=testmon_mode or "affected", invocation_duration_s=total_duration, budget_s=VERIFY_INVOCATION_BUDGET_S, + external_collection_selector=_has_inherited_collection_selector(), ) # Aggregation and final graph inspection are part of the same invocation @@ -2936,6 +2967,7 @@ def _main(argv: list[str] | None = None) -> int: "selection_mode": testmon_mode, "copied_from": str(preparation.copied_from) if preparation.copied_from is not None else None, "required_executable_paths": list(required_executable_paths), + "bootstrap_trigger_paths": list(preparation_required_executable_paths), "runtime_data_paths": list(runtime_data_paths), } if pytest_aggregate is not None: diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index a6b1a11046..b61a99a886 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -180,7 +180,7 @@ def max_optional(current: int | None, value: object) -> int | None: ) no_pytest = not pytest_steps - corpus_digest = hashlib.sha256(b"").hexdigest() + corpus_digest: str | None = None exit_code = entry.get("exit_code") raw_budget = entry.get("invocation_budget_s") invocation_budget = float(raw_budget) if isinstance(raw_budget, int | float) else None @@ -208,7 +208,7 @@ def max_optional(current: int | None, value: object) -> int | None: "outcomes": outcomes, "missing_terminal_count": 0, "missing_terminal_sample": [], - "non_green_count": sum(count for outcome, count in outcomes.items() if outcome not in {"passed", "skipped"}), + "non_green_count": sum(count for outcome, count in outcomes.items() if outcome not in _GREEN_TERMINAL_OUTCOMES), "non_green_sample": [], "complete_corpus_covered": False, "terminal_green": bool(pytest_steps) and exit_code == 0, @@ -747,8 +747,10 @@ def _record_change(self, candidate: Path) -> None: # its authority file. return if candidate.name == authority_path.name: - if self._authority_signature(authority_path) == self._git_authority_signatures.get(authority_path): - return + # The watch event is itself evidence that Git replaced the + # authority file. The bytes may already have been restored by + # the time the coalesced event reaches this thread, so a + # signature comparison here would discard a real mutation. with self._state_lock: self._changed = True self._observed_path = label @@ -945,6 +947,50 @@ def _distribution(values: list[float]) -> dict[str, float | int | None]: } +def _counter_delta(resources: list[dict[str, Any]], key: str) -> int: + """Return the observed delta for one monotonically increasing resource counter.""" + values = [int(row[key]) for row in resources if isinstance(row.get(key), int)] + return max(values) - min(values) if values else 0 + + +def _phase_outcome(phases: Mapping[str, object], name: str) -> str | None: + value = phases.get(name) + if isinstance(value, Mapping): + outcome = value.get("outcome") + return outcome if isinstance(outcome, str) else None + return value if isinstance(value, str) else None + + +def _collapse_terminal_outcome(phases: Mapping[str, object]) -> str: + """Collapse setup, call, and teardown evidence to one terminal outcome.""" + setup = _phase_outcome(phases, "setup") + call = _phase_outcome(phases, "call") + teardown = _phase_outcome(phases, "teardown") + if setup == "failed" or teardown == "failed": + return "error" + if call is not None: + return call + if setup in {"skipped", "xfailed", "xpassed"}: + return setup + if teardown in {"skipped", "xfailed", "xpassed"}: + return teardown + return "interrupted" + + +def _merge_terminal_outcomes( + canonical: Mapping[str, str], + phase_reports: Mapping[str, Mapping[str, object]], + *, + nodeids: Sequence[str] = (), +) -> dict[str, str]: + """Prefer canonical outcomes and fill omitted nodes from phase evidence.""" + all_nodeids = set(nodeids) | set(canonical) | set(phase_reports) + return { + nodeid: canonical[nodeid] if nodeid in canonical else _collapse_terminal_outcome(phase_reports.get(nodeid, {})) + for nodeid in all_nodeids + } + + def aggregate_pytest_statistics( step_dir: Path, *, @@ -1036,27 +1082,8 @@ def aggregate_pytest_statistics( bucket = phase_outcomes[when] bucket[outcome] = bucket.get(outcome, 0) + 1 - for nodeid in nodes: - node_reports = reports_by_node.get(nodeid, {}) - setup = node_reports.get("setup", {}).get("outcome") - call = node_reports.get("call", {}).get("outcome") - teardown = node_reports.get("teardown", {}).get("outcome") - canonical_outcome = canonical_outcomes.get(nodeid) - if canonical_outcome is not None: - terminal = canonical_outcome - elif setup == "failed" or teardown == "failed": - terminal = "error" - elif isinstance(call, str): - terminal = call - elif setup in {"skipped", "xfailed", "xpassed"}: - terminal = str(setup) - elif teardown in {"skipped", "xfailed", "xpassed"}: - terminal = str(teardown) - else: - # A test may have emitted its start event just before an interrupt - # or forced containment cleanup. Keep that missing terminal phase - # visible so outcome totals still account for every started node. - terminal = "interrupted" + terminal_outcomes = _merge_terminal_outcomes(canonical_outcomes, reports_by_node, nodeids=tuple(nodes)) + for terminal in terminal_outcomes.values(): outcomes[terminal] = outcomes.get(terminal, 0) + 1 resources: list[dict[str, Any]] = [] @@ -1134,26 +1161,8 @@ def aggregate_pytest_statistics( (int(row["tree_swap_pss_kb"]) for row in resources if isinstance(row.get("tree_swap_pss_kb"), int)), default=None, ), - "tree_read_bytes_delta": ( - max( - (int(row["tree_read_bytes"]) for row in resources if isinstance(row.get("tree_read_bytes"), int)), - default=0, - ) - - min( - (int(row["tree_read_bytes"]) for row in resources if isinstance(row.get("tree_read_bytes"), int)), - default=0, - ) - ), - "tree_write_bytes_delta": ( - max( - (int(row["tree_write_bytes"]) for row in resources if isinstance(row.get("tree_write_bytes"), int)), - default=0, - ) - - min( - (int(row["tree_write_bytes"]) for row in resources if isinstance(row.get("tree_write_bytes"), int)), - default=0, - ) - ), + "tree_read_bytes_delta": _counter_delta(resources, "tree_read_bytes"), + "tree_write_bytes_delta": _counter_delta(resources, "tree_write_bytes"), }, "cleanup": { "complete": True @@ -1194,16 +1203,15 @@ def _read_json_object(path: Path) -> dict[str, Any] | None: def _terminal_outcomes_by_node(step_dir: Path) -> dict[str, str]: canonical = _read_json_object(step_dir / PYTEST_CANONICAL_REPORT_NAME) raw_tests = canonical.get("tests") if canonical is not None else None + canonical_outcomes: dict[str, str] = {} if isinstance(raw_tests, list): - outcomes = { + canonical_outcomes = { str(test["nodeid"]): str(test["outcome"]) for test in raw_tests if isinstance(test, dict) and isinstance(test.get("nodeid"), str) and isinstance(test.get("outcome"), str) } - if outcomes: - return outcomes - reports: dict[str, dict[str, str]] = {} + reports: dict[str, dict[str, object]] = {} events_path = step_dir / "events.jsonl" if events_path.exists(): for line in events_path.read_text(encoding="utf-8", errors="replace").splitlines(): @@ -1216,22 +1224,7 @@ def _terminal_outcomes_by_node(step_dir: Path) -> dict[str, str]: outcome = row.get("outcome") if isinstance(nodeid, str) and when in {"setup", "call", "teardown"} and isinstance(outcome, str): reports.setdefault(nodeid, {})[str(when)] = outcome - fallback_outcomes: dict[str, str] = {} - for nodeid, phases in reports.items(): - setup = phases.get("setup") - call = phases.get("call") - teardown = phases.get("teardown") - if setup == "failed" or teardown == "failed": - fallback_outcomes[nodeid] = "error" - elif call is not None: - fallback_outcomes[nodeid] = call - elif setup in {"skipped", "xfailed", "xpassed"}: - fallback_outcomes[nodeid] = setup - elif teardown in {"skipped", "xfailed", "xpassed"}: - fallback_outcomes[nodeid] = teardown - else: - fallback_outcomes[nodeid] = "interrupted" - return fallback_outcomes + return _merge_terminal_outcomes(canonical_outcomes, reports) def aggregate_native_testmon_run( @@ -1245,6 +1238,7 @@ def aggregate_native_testmon_run( selection_mode: str, invocation_duration_s: float, budget_s: float, + external_collection_selector: bool = False, ) -> dict[str, Any]: """Build one compact, durable aggregate for the two semantic pytest lanes.""" lanes: list[dict[str, Any]] = [] @@ -1331,17 +1325,15 @@ def _peak(current: int | None, value: object) -> int | None: outcomes[outcome] = outcomes.get(outcome, 0) + 1 native_corpus = tuple(sorted(set(corpus_nodeids))) complete_mode = selection_mode in {"bootstrap", "full"} - # testmon synchronizes every collected node before pytest applies markers. - # The correctness corpus for this invocation is therefore the complete - # selection union from the two complementary semantic lanes, while the - # native database corpus remains useful environment evidence of its own. - corpus = tuple(sorted(selected_union)) if complete_mode else native_corpus + corpus = native_corpus corpus_set = set(corpus) lane_names = [lane["lane"] for lane in lanes] complete_corpus_covered = ( complete_mode and selection_complete and bool(corpus) + and not external_collection_selector + and selected_union == corpus_set and set(outcome_by_node) == corpus_set and not duplicate_outcomes and len(lane_names) == 2 @@ -1350,7 +1342,7 @@ def _peak(current: int | None, value: object) -> int | None: ) missing_terminal = tuple(sorted(corpus_set - set(outcome_by_node))) if complete_mode else () non_green = tuple( - sorted(nodeid for nodeid in corpus if outcome_by_node.get(nodeid) not in _GREEN_TERMINAL_OUTCOMES) + sorted(nodeid for nodeid, outcome in outcome_by_node.items() if outcome not in _GREEN_TERMINAL_OUTCOMES) ) terminal_green = complete_corpus_covered and not missing_terminal and not non_green cleanup_complete = bool(lanes) and cleanup_complete @@ -1370,6 +1362,7 @@ def _peak(current: int | None, value: object) -> int | None: "digest": hashlib.sha256("\n".join(corpus).encode()).hexdigest(), }, "selection_mode": selection_mode, + "external_collection_selector": external_collection_selector, "lanes": lanes, "outcomes": outcomes, "selected_union_count": len(selected_union), diff --git a/pyproject.toml b/pyproject.toml index 9f51f28fab..1f4c970b48 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,7 +70,7 @@ dev-common = [ "pytest-asyncio>=1.4.0", # Async test support "pytest-xdist>=3.5.0", # Parallel test execution "pytest-randomly>=4.1.0", # Detect order-dependent tests; adds --randomly-seed - "pytest-testmon>=2.1.3", # Per-test affected selection via dependency database + "pytest-testmon>=2.2.0,<2.3.0", # Bootstrap reads the tested DATA_VERSION-14 schema "pytest-json-report>=1.5.0", # Structured JSON report for verify pipeline + dashboard (#1026, #998) "pytest-benchmark>=5.0", # Microbenchmark suite (run with --benchmark-enable -p no:xdist) "hypothesis>=6.165.2", diff --git a/tests/benchmarks/test_scale_tiers.py b/tests/benchmarks/test_scale_tiers.py index d2579f5470..b362014670 100644 --- a/tests/benchmarks/test_scale_tiers.py +++ b/tests/benchmarks/test_scale_tiers.py @@ -10,7 +10,7 @@ from pathlib import Path -from tests.benchmarks.helpers import open_bench_store +from tests.benchmarks.helpers import BenchmarkFixture, open_bench_store def _fts_query(db_path: Path, term: str, *, limit: int = 20) -> int: @@ -69,11 +69,11 @@ def test_bench_10k_list_sessions(bench_db_10k: Path) -> None: # --------------------------------------------------------------------------- -def test_bench_50k_fts_search_returns_results(bench_db_50k: Path) -> None: - hits = _fts_query(bench_db_50k, "analysis", limit=50) +def test_bench_50k_fts_search_returns_results(benchmark: BenchmarkFixture, bench_db_50k: Path) -> None: + hits = benchmark(lambda: _fts_query(bench_db_50k, "analysis", limit=50)) assert 0 <= hits <= 50 -def test_bench_50k_list_sessions(bench_db_50k: Path) -> None: - rows = _list_query(bench_db_50k, limit=50) +def test_bench_50k_list_sessions(benchmark: BenchmarkFixture, bench_db_50k: Path) -> None: + rows = benchmark(lambda: _list_query(bench_db_50k, limit=50)) assert 0 < rows <= 50 diff --git a/tests/integration/devtools/test_native_testmon_lifecycle.py b/tests/integration/devtools/test_native_testmon_lifecycle.py index 5823323c7a..3ada9b6a46 100644 --- a/tests/integration/devtools/test_native_testmon_lifecycle.py +++ b/tests/integration/devtools/test_native_testmon_lifecycle.py @@ -27,7 +27,7 @@ PROJECT_ROOT = Path(__file__).resolve().parents[3] pytestmark = [ pytest.mark.uses_real_clock("coordinates real pytest subprocesses and an interrupt deadline"), - pytest.mark.timeout(90), + pytest.mark.timeout(300), ] @@ -131,7 +131,13 @@ def _run_lane( str(workers), ] completed = subprocess.run(command, cwd=repo, env=env, capture_output=True, text=True, timeout=timeout) - selection_payload = json.loads((artifact_dir / "selection.json").read_text(encoding="utf-8")) + selection_path = artifact_dir / "selection.json" + if not selection_path.is_file(): + raise AssertionError( + f"native pytest lane produced no selection artifact (returncode={completed.returncode})\n" + f"stdout:\n{completed.stdout}\nstderr:\n{completed.stderr}" + ) + selection_payload = json.loads(selection_path.read_text(encoding="utf-8")) return LaneResult(completed, artifact_dir, selection_payload) @@ -173,6 +179,7 @@ def _run_production_verify(repo: Path, *args: str) -> tuple[subprocess.Completed invocation_id = uuid.uuid4().hex driver = """ import os +import shutil import sys from pathlib import Path from types import SimpleNamespace @@ -181,12 +188,21 @@ def _run_production_verify(repo: Path, *args: str) -> tuple[subprocess.Completed root = Path(sys.argv[1]).resolve() real_build = verify.build_verify_steps +real_env_for_pytest_step = verify.env_for_pytest_step +pytest_index = root / ".git" / "pytest-index" +shutil.copy2(root / ".git" / "index", pytest_index) def native_steps_only(**kwargs): return [step for step in real_build(**kwargs) if step[0].startswith("pytest native")] +def fixture_env_for_pytest_step(env, **kwargs): + child_env = real_env_for_pytest_step(env, **kwargs) + child_env["GIT_INDEX_FILE"] = str(pytest_index) + return child_env + verify.ROOT = root verify.build_verify_steps = native_steps_only +verify.env_for_pytest_step = fixture_env_for_pytest_step verify.assert_polylogue_matches_checkout = lambda *_args, **_kwargs: SimpleNamespace( polylogue_import_path=root / "polylogue" / "__init__.py", as_dict=lambda: {"checkout_root": str(root), "test_fixture": True}, @@ -237,6 +253,10 @@ def native_steps_only(**kwargs): f"production verify wrote no invocation receipt\nstdout:\n{completed.stdout}\nstderr:\n{completed.stderr}" ) payload = json.loads(completed.stdout) + if completed.returncode == 125: + pytest.fail( + f"production verify rejected the fixture checkout\npayload:\n{completed.stdout}\nstderr:\n{completed.stderr}" + ) persisted = json.loads(receipt.read_text(encoding="utf-8")) assert persisted["invocation_id"] == invocation_id assert persisted["pytest_aggregate"] == payload["pytest_aggregate"] @@ -559,7 +579,8 @@ def test_production_verify_deleted_module_rebuilds_and_fails_dependents(tmp_path assert completed.returncode == 1 assert mutated["testmon_environment"]["selection_mode"] == "bootstrap" - assert mutated["testmon_environment"]["required_executable_paths"] == ["polylogue/app.py"] + assert mutated["testmon_environment"]["required_executable_paths"] == [] + assert mutated["testmon_environment"]["bootstrap_trigger_paths"] == ["polylogue/app.py"] assert mutated["pytest_aggregate"]["selected_union_count"] == 1 assert mutated["pytest_aggregate"]["terminal_union_count"] == 1 assert "ModuleNotFoundError" in completed.stderr @@ -594,7 +615,8 @@ def test_production_verify_moved_module_rebuilds_and_fails_dependents(tmp_path: assert completed.returncode == 1 assert mutated["testmon_environment"]["selection_mode"] == "bootstrap" - assert mutated["testmon_environment"]["required_executable_paths"] == [ + assert mutated["testmon_environment"]["required_executable_paths"] == ["polylogue/renamed.py"] + assert mutated["testmon_environment"]["bootstrap_trigger_paths"] == [ "polylogue/app.py", "polylogue/renamed.py", ] @@ -603,6 +625,86 @@ def test_production_verify_moved_module_rebuilds_and_fails_dependents(tmp_path: assert "ModuleNotFoundError" in completed.stderr +def test_production_verify_deleted_module_with_updated_imports_rebuilds_successfully(tmp_path: Path) -> None: + """Keeping the deleted path in post-run requirements makes the updated import fail.""" + repo = tmp_path / "repo" + repo.mkdir() + _init_repo(repo) + package = repo / "polylogue" + package.mkdir() + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "old.py").write_text("def answer() -> int:\n return 42\n", encoding="utf-8") + (package / "app.py").write_text("from polylogue.old import answer\n", encoding="utf-8") + (repo / "tests" / "test_app.py").write_text( + "def test_answer():\n from polylogue.app import answer\n assert answer() == 42\n", + encoding="utf-8", + ) + _commit_all(repo, "fixture") + origin = tmp_path / "origin.git" + _git(origin.parent, "init", "--bare", "-q", str(origin)) + _git(repo, "remote", "add", "origin", str(origin)) + _git(repo, "branch", "-M", "master") + _git(repo, "push", "-qu", "origin", "master") + + seeded, _bootstrap = _run_production_verify(repo) + assert seeded.returncode == 0, seeded.stderr + + _git(repo, "rm", "polylogue/old.py") + (package / "app.py").write_text("def answer() -> int:\n return 42\n", encoding="utf-8") + _commit_all(repo, "delete module and update imports") + completed, rebuilt = _run_production_verify(repo) + + assert completed.returncode == 0, completed.stderr + environment = rebuilt["testmon_environment"] + assert environment["selection_mode"] == "bootstrap" + assert environment["required_executable_paths"] == ["polylogue/app.py"] + assert environment["bootstrap_trigger_paths"] == ["polylogue/app.py", "polylogue/old.py"] + assert rebuilt["pytest_aggregate"]["terminal_green"] is True + + +def test_production_verify_moved_module_with_updated_imports_rebuilds_successfully(tmp_path: Path) -> None: + """Keeping the old path as a graph requirement makes the updated move fail.""" + repo = tmp_path / "repo" + repo.mkdir() + _init_repo(repo) + package = repo / "polylogue" + package.mkdir() + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "old.py").write_text("def answer() -> int:\n return 42\n", encoding="utf-8") + (repo / "tests" / "test_app.py").write_text( + "def test_answer():\n from polylogue.old import answer\n assert answer() == 42\n", + encoding="utf-8", + ) + _commit_all(repo, "fixture") + origin = tmp_path / "origin.git" + _git(origin.parent, "init", "--bare", "-q", str(origin)) + _git(repo, "remote", "add", "origin", str(origin)) + _git(repo, "branch", "-M", "master") + _git(repo, "push", "-qu", "origin", "master") + + seeded, _bootstrap = _run_production_verify(repo) + assert seeded.returncode == 0, seeded.stderr + + _git(repo, "mv", "polylogue/old.py", "polylogue/renamed.py") + (repo / "tests" / "test_app.py").write_text( + "def test_answer():\n from polylogue.renamed import answer\n assert answer() == 42\n", + encoding="utf-8", + ) + _commit_all(repo, "move module and update imports") + completed, rebuilt = _run_production_verify(repo) + + assert completed.returncode == 0, completed.stderr + environment = rebuilt["testmon_environment"] + assert environment["selection_mode"] == "bootstrap" + assert environment["required_executable_paths"] == ["polylogue/renamed.py", "tests/test_app.py"] + assert environment["bootstrap_trigger_paths"] == [ + "polylogue/old.py", + "polylogue/renamed.py", + "tests/test_app.py", + ] + assert rebuilt["pytest_aggregate"]["terminal_green"] is True + + def test_empty_linked_worktree_with_empty_main_self_bootstraps(tmp_path: Path) -> None: main = tmp_path / "main" main.mkdir() @@ -700,7 +802,10 @@ def test_unfinished(): deadline = time.monotonic() + 10 while not started.exists() and time.monotonic() < deadline: time.sleep(0.02) - assert started.exists(), process.communicate(timeout=1) + if not started.exists(): + process.send_signal(signal.SIGINT) + stdout, stderr = process.communicate(timeout=10) + raise AssertionError(f"interrupted fixture never started\nstdout:\n{stdout}\nstderr:\n{stderr}") process.send_signal(signal.SIGINT) process.communicate(timeout=10) diff --git a/tests/unit/devtools/test_merge_boundary.py b/tests/unit/devtools/test_merge_boundary.py index 1ccc3a1358..f85af6f427 100644 --- a/tests/unit/devtools/test_merge_boundary.py +++ b/tests/unit/devtools/test_merge_boundary.py @@ -944,6 +944,34 @@ def test_train_status_blocks_when_pr_merged_after_last_full_verify( assert merge_boundary.cmd_train_status(as_json=False) == 1 +def test_train_status_reads_historical_scope_without_granting_release_authority( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """Rejecting unknown historical scopes would make old ledgers unreadable.""" + monkeypatch.chdir(tmp_path) + merge_boundary._write_ledger( + { + "merges": [], + "last_full_verify": { + "at": 1000.0, + "verification_started_at": 1000.0, + "duration_s": 1.0, + "command": "devtools verify --affected", + "exit_code": 0, + "verification_scope": "narrow-terminal", + "release_baseline_allowed": True, + "merge_sequence": 0, + "accepted": True, + }, + } + ) + merge_boundary._append_merge_entry(1, "sha1", "some title") + + assert merge_boundary._read_ledger()["last_full_verify"]["verification_scope"] == "narrow-terminal" + assert merge_boundary._pending_prs_since_last_full_verify(merge_boundary._read_ledger()) + assert merge_boundary.cmd_train_status(as_json=False) == 1 + + def test_train_status_rejects_untyped_accepted_terminal_ledger(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: monkeypatch.chdir(tmp_path) merge_boundary._write_ledger( diff --git a/tests/unit/devtools/test_testmon_bootstrap.py b/tests/unit/devtools/test_testmon_bootstrap.py index 26b538b13d..3c1616adde 100644 --- a/tests/unit/devtools/test_testmon_bootstrap.py +++ b/tests/unit/devtools/test_testmon_bootstrap.py @@ -7,6 +7,7 @@ from devtools.testmon_bootstrap import ( NativeTestmonDeadlineError, NativeTestmonRepairError, + _testmon_schema_version, classify_native_testmon_changes, classify_source_ast, executable_python_paths, @@ -49,6 +50,14 @@ def test_ast_classification_treats_type_checking_guards_as_declarations(tmp_path assert classify_source_ast(declarations) == "executable" +def test_ast_classification_treats_ordinary_imports_as_executable(tmp_path: Path) -> None: + """Removing ordinary-import execution from the classifier makes this fail.""" + module = tmp_path / "runtime.py" + module.write_text("from package.runtime import value\n", encoding="utf-8") + + assert classify_source_ast(module) == "executable" + + def test_executable_paths_require_current_runtime_modules_and_deleted_modules(tmp_path: Path) -> None: module = tmp_path / "polylogue" / "runtime.py" module.parent.mkdir() @@ -112,6 +121,47 @@ def test_test_runtime_data_changes_force_full_native_selection(tmp_path: Path) - ) +def test_benchmarks_are_not_required_graph_paths_and_packaging_is_untraceable(tmp_path: Path) -> None: + """Restoring benchmark graph edges or dropping packaging inputs makes this fail.""" + benchmark = tmp_path / "tests" / "benchmarks" / "test_scale.py" + benchmark.parent.mkdir(parents=True) + benchmark.write_text("def test_scale(): pass\n", encoding="utf-8") + + impact = classify_native_testmon_changes( + tmp_path, + ( + "tests/benchmarks/test_scale.py", + "tests/benchmarks/deleted.py", + "packaging/polylogue.nix", + "docs/release.md", + ), + ) + + assert impact.executable_paths == () + assert impact.runtime_data_paths == ("packaging/polylogue.nix",) + + +def test_testmon_schema_matches_the_tested_dependency_contract(tmp_path: Path) -> None: + """Changing the pinned testmon schema or required columns makes this fail.""" + import testmon.db + + database = tmp_path / "testmondata" + db = testmon.db.DB(str(database)) + try: + assert _testmon_schema_version() == 14 + assert tuple(db.con.execute("PRAGMA user_version").fetchone()) == (14,) + for table, expected in { + "environment": {"id", "environment_name", "system_packages", "python_version"}, + "file_fp": {"id", "filename", "method_checksums", "mtime", "fsha"}, + "test_execution": {"id", "environment_id", "test_name", "duration", "failed", "forced"}, + "test_execution_file_fp": {"test_execution_id", "fingerprint_id"}, + }.items(): + columns = {row[1] for row in db.con.execute(f"PRAGMA table_info({table})")} + assert expected <= columns + finally: + db.con.close() + + def test_environment_digest_changes_with_collection_semantics( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 080bb54217..0024f8f7a3 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -54,6 +54,7 @@ VerifyRun, adaptive_pytest_runtime_policy, adaptive_pytest_worker_count, + aggregate_native_testmon_run, aggregate_pytest_statistics, append_verify_history, apply_managed_pytest_runtime_policy, @@ -155,6 +156,24 @@ def test_native_corpus_excludes_only_benchmark_directory() -> None: assert "--ignore=tests/integration" not in complete_command +@pytest.mark.parametrize("selector", ["-k smoke", "--keyword=smoke", "--ignore=tests/benchmarks"]) +def test_inherited_collection_selectors_are_not_release_authority( + monkeypatch: pytest.MonkeyPatch, + selector: str, +) -> None: + """Removing selector detection would incorrectly authorize the narrowed corpus.""" + monkeypatch.setenv("PYTEST_ADDOPTS", selector) + + assert verify._has_inherited_collection_selector() is True + + +def test_unrelated_inherited_pytest_options_do_not_disable_release_authority(monkeypatch: pytest.MonkeyPatch) -> None: + """Treating every ambient option as a selector would reject harmless options.""" + monkeypatch.setenv("PYTEST_ADDOPTS", "-ra --strict-markers") + + assert verify._has_inherited_collection_selector() is False + + def test_lab_verify_delegates_to_lab_smoke() -> None: steps = build_verify_steps(quick=True, lab=True) @@ -402,6 +421,94 @@ def test_aggregate_pytest_statistics_recognizes_completed_empty_report(tmp_path: assert result["outcomes"] == {} +def test_aggregate_pytest_statistics_merges_partial_canonical_and_event_outcomes(tmp_path: Path) -> None: + """Returning early on a partial canonical report drops the event-only node.""" + step = tmp_path / "step" + step.mkdir() + (step / "events.jsonl").write_text( + json.dumps( + { + "event": "test_report", + "nodeid": "event-only", + "when": "call", + "outcome": "passed", + "duration_s": 0.1, + } + ) + + "\n" + ) + (step / "pytest-report.json").write_text( + json.dumps( + { + "tests": [ + { + "nodeid": "canonical-only", + "outcome": "xfailed", + "call": {"outcome": "skipped", "duration": 0.1}, + } + ] + } + ) + ) + + result = aggregate_pytest_statistics(step) + + assert result["node_count"] == 2 + assert result["outcomes"] == {"passed": 1, "xfailed": 1} + + +def test_native_aggregate_rejects_inherited_selector_without_counting_missing_nodes(tmp_path: Path) -> None: + """Using the selected union as corpus authority would mark this narrowed run green.""" + for lane, nodeid, outcome in (("parallel", "a", "passed"), ("serial", "b", "skipped")): + step = tmp_path / lane + step.mkdir() + (step / "selection.json").write_text(json.dumps({"selected_nodeids": [nodeid], "selected_nodeids_omitted": 0})) + (step / "events.jsonl").write_text( + json.dumps({"event": "test_report", "nodeid": nodeid, "when": "call", "outcome": outcome}) + "\n" + ) + (step / "containment.json").write_text( + json.dumps( + { + "status": "finished", + "controller_group_alive": False, + "termination_reason": None, + "escalated_to_sigkill": False, + } + ) + ) + + result = aggregate_native_testmon_run( + tmp_path, + steps=[ + { + "semantic_lane": "parallel", + "artifact_dir": "parallel", + "exit": 0, + "statistics": {"cleanup": {"complete": True}}, + }, + { + "semantic_lane": "serial", + "artifact_dir": "serial", + "exit": 0, + "statistics": {"cleanup": {"complete": True}}, + }, + ], + environment_name="polylogue-test", + corpus_nodeids=("a", "b"), + selection_mode="bootstrap", + invocation_duration_s=0.1, + budget_s=10.0, + external_collection_selector=True, + ) + + assert result["external_collection_selector"] is True + assert result["selected_union_count"] == 2 + assert result["terminal_union_count"] == 2 + assert result["non_green_count"] == 0 + assert result["complete_corpus_covered"] is False + assert result["terminal_green"] is False + + def test_verify_run_statistics_only_cover_pytest_steps(tmp_path: Path) -> None: run = VerifyRun(tier="quick", argv=["--quick"], git_head="head", root=tmp_path) artifacts = run.start_step(label="ruff check", cmd=["ruff", "check"]) @@ -1324,6 +1431,40 @@ def test_checkout_mutation_monitor_observes_transient_index_authority_change(tmp assert observation == CheckoutMutationObservation(changed=True, unavailable=False, observed_path=".git/index") +@pytest.mark.uses_real_clock("coordinates a deliberately coalesced Git index watcher event") +def test_checkout_mutation_monitor_keeps_coalesced_index_authority_event( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Restoring signature comparison would miss the delayed authority event.""" + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + tracked = tmp_path / "tracked.py" + tracked.write_text("value = 1\n", encoding="utf-8") + subprocess.run(["git", "add", "tracked.py"], cwd=tmp_path, check=True) + index = tmp_path / ".git" / "index" + baseline = index.read_bytes() + release_event = threading.Event() + + def delayed_watch(*_paths: Path, **kwargs: object) -> object: + yield set() + assert release_event.wait(timeout=1) + yield {(watchfiles.Change.modified, str(index))} + stop_event = kwargs["stop_event"] + assert isinstance(stop_event, threading.Event) + stop_event.wait(timeout=1) + + monkeypatch.setattr(watchfiles, "watch", delayed_watch) + monitor = CheckoutMutationMonitor(tmp_path) + monitor.start() + tracked.write_text("value = 2\n", encoding="utf-8") + subprocess.run(["git", "add", "tracked.py"], cwd=tmp_path, check=True) + index.write_bytes(baseline) + release_event.set() + observation = monitor.finish() + + assert observation == CheckoutMutationObservation(changed=True, unavailable=False, observed_path=".git/index") + + @pytest.mark.uses_real_clock("waits for the filesystem watcher to witness a branch-ref replacement") def test_checkout_mutation_monitor_observes_transient_head_ref_change(tmp_path: Path) -> None: subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) @@ -1503,7 +1644,7 @@ def test_checkout_mutation_monitor_ignores_uncommitted_git_index_lock(tmp_path: assert observation == CheckoutMutationObservation(changed=False, unavailable=False) -def test_checkout_mutation_monitor_ignores_semantically_unchanged_index_replacement( +def test_checkout_mutation_monitor_records_semantically_unchanged_index_replacement( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -1527,7 +1668,7 @@ def portable_watch(*_paths: Path, **_kwargs: object) -> object: monitor.start() observation = monitor.finish() - assert observation == CheckoutMutationObservation(changed=False, unavailable=False) + assert observation == CheckoutMutationObservation(changed=True, unavailable=False, observed_path=".git/index") def test_checkout_mutation_monitor_rejects_partial_git_enumeration( diff --git a/tests/unit/infra/test_archive_templates.py b/tests/unit/infra/test_archive_templates.py index 521aab7985..19ab0c1145 100644 --- a/tests/unit/infra/test_archive_templates.py +++ b/tests/unit/infra/test_archive_templates.py @@ -11,6 +11,8 @@ from pathlib import Path from unittest.mock import patch +import pytest + from tests.infra.archive_templates import clone_archive_template, finalize_archive_template @@ -36,12 +38,14 @@ def test_clone_fallback_is_private_writable_and_symlink_safe(tmp_path: Path) -> assert source_file.read_bytes() == source_bytes -def test_clone_rebinds_durable_bootstrap_identity(tmp_path: Path) -> None: +def test_clone_rebinds_durable_bootstrap_identity(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Reusing the source store or global identity would make this reopen unsafe.""" from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore template = tmp_path / "template" clone = tmp_path / "clone" marker = Path(".maintenance-state/durable-change-trains/.bootstrap") + monkeypatch.setattr("polylogue.paths.archive_root", lambda: tmp_path / "configured") with ArchiveStore(template): pass source_identity = template.joinpath(marker).read_bytes() @@ -49,6 +53,8 @@ def test_clone_rebinds_durable_bootstrap_identity(tmp_path: Path) -> None: clone_archive_template(template, clone) assert clone.joinpath(marker).read_bytes() != source_identity + with ArchiveStore(template): + pass with ArchiveStore(clone): pass assert template.joinpath(marker).read_bytes() == source_identity diff --git a/tests/unit/storage/test_perf_rescue_1314.py b/tests/unit/storage/test_perf_rescue_1314.py index dc208d8e6f..a85ac58e75 100644 --- a/tests/unit/storage/test_perf_rescue_1314.py +++ b/tests/unit/storage/test_perf_rescue_1314.py @@ -19,12 +19,14 @@ from __future__ import annotations +import shutil from collections.abc import Iterator from contextlib import contextmanager from pathlib import Path from typing import Any import aiosqlite +import pytest from polylogue.storage.insights.session.rebuild import _SESSION_INSIGHT_REBUILD_PAGE_SIZE from polylogue.storage.sqlite.queries.sessions_search import search_session_hits @@ -34,6 +36,14 @@ pytest_plugins = ("tests.benchmarks.conftest",) +@pytest.fixture +def isolated_bench_db_1k(tmp_path: Path, bench_db_1k: Path) -> Path: + """Give each test a private copy of the session-scoped benchmark input.""" + index_db = tmp_path / "index.db" + shutil.copy2(bench_db_1k.parent / "index.db", index_db) + return index_db + + @contextmanager def _capture_aiosqlite_sql() -> Iterator[list[str]]: """Capture every SQL string passed to ``aiosqlite.Connection.execute``.""" @@ -74,11 +84,11 @@ def test_session_insight_rebuild_page_size_is_at_least_50() -> None: # --------------------------------------------------------------------------- -def test_search_session_hits_uses_freshness_ledger_before_match(bench_db_1k: Path) -> None: +def test_search_session_hits_uses_freshness_ledger_before_match(isolated_bench_db_1k: Path) -> None: """Search should not pay archive-scale COUNT(*) probes after daemon readiness.""" from polylogue.storage.fts.freshness import READY, record_fts_surface_state_async - with open_bench_store(bench_db_1k) as store: + with open_bench_store(isolated_bench_db_1k) as store: backend = store.backend async def _run(statements: list[str]) -> None: @@ -102,9 +112,9 @@ async def _run(statements: list[str]) -> None: assert all("count(*) from messages where text is not null" not in sql for sql in lowered[:match_index]) -def test_search_session_hits_falls_back_to_exact_freshness(bench_db_1k: Path) -> None: +def test_search_session_hits_falls_back_to_exact_freshness(isolated_bench_db_1k: Path) -> None: """Absent ledger rows fall back to exact FTS verification before MATCH.""" - with open_bench_store(bench_db_1k) as store: + with open_bench_store(isolated_bench_db_1k) as store: backend = store.backend async def _run(statements: list[str]) -> None: @@ -131,11 +141,11 @@ async def _run(statements: list[str]) -> None: # --------------------------------------------------------------------------- -def test_origin_metrics_reads_sessions_aggregates(bench_db_1k: Path) -> None: +def test_origin_metrics_reads_sessions_aggregates(isolated_bench_db_1k: Path) -> None: """Origin metrics must source the per-session pre-aggregates from ``sessions`` rather than scanning ``messages``. """ - with open_bench_store(bench_db_1k) as store: + with open_bench_store(isolated_bench_db_1k) as store: backend = store.backend async def _run() -> list[dict[str, object]]: @@ -175,7 +185,7 @@ async def _run() -> list[dict[str, object]]: # --------------------------------------------------------------------------- -def test_get_messages_hydration_does_not_call_model_copy(bench_db_1k: Path) -> None: +def test_get_messages_hydration_does_not_call_model_copy(isolated_bench_db_1k: Path) -> None: """``get_messages`` must mutate the freshly-constructed MessageRecord instances in place rather than calling pydantic's ``model_copy``. @@ -194,7 +204,7 @@ def _spy(self: Any, *args: Any, **kwargs: Any) -> Any: MessageRecord.model_copy = _spy # type: ignore[method-assign] try: - with open_bench_store(bench_db_1k) as store: + with open_bench_store(isolated_bench_db_1k) as store: async def _run() -> int: summaries = await store.repository.list_summaries(limit=5) diff --git a/uv.lock b/uv.lock index b7077ff4ae..4f0eb1f92e 100644 --- a/uv.lock +++ b/uv.lock @@ -1374,7 +1374,7 @@ requires-dist = [ { name = "pytest-cov", marker = "extra == 'dev-common'", specifier = ">=4" }, { name = "pytest-json-report", marker = "extra == 'dev-common'", specifier = ">=1.5.0" }, { name = "pytest-randomly", marker = "extra == 'dev-common'", specifier = ">=4.1.0" }, - { name = "pytest-testmon", marker = "extra == 'dev-common'", specifier = ">=2.1.3" }, + { name = "pytest-testmon", marker = "extra == 'dev-common'", specifier = ">=2.2.0,<2.3.0" }, { name = "pytest-timeout", marker = "extra == 'dev-common'", specifier = ">=2.4.0" }, { name = "pytest-xdist", marker = "extra == 'dev-common'", specifier = ">=3.5.0" }, { name = "python-multipart", specifier = ">=0.0.32" }, From 1f530456b1801c64363e93e4e955e2f912e687c4 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 14 Aug 2026 02:32:07 +0200 Subject: [PATCH 15/31] fix(verify): deny narrowed pytest baselines Problem: release evidence only inspected PYTEST_ADDOPTS tokens, so pytest\nconfiguration and some selector spellings could narrow collection while\nstill appearing complete.\n\nWhat changed: ask pytest to parse its effective inherited configuration,\nfail closed on parser errors, and require the aggregate to explicitly report\nno external collection selector before granting release authority. --- devtools/verify.py | 25 +++++++------ tests/unit/devtools/test_verify.py | 57 ++++++++++++++++++++++++++++-- 2 files changed, 70 insertions(+), 12 deletions(-) diff --git a/devtools/verify.py b/devtools/verify.py index 4c3b16b7da..a6b93aaf3e 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -39,6 +39,8 @@ from pathlib import Path from typing import Any +from _pytest.config import get_config + from devtools.checkout_guard import ( CheckoutImportMismatchError, assert_polylogue_matches_checkout, @@ -2369,18 +2371,20 @@ def _changed_test_relevant_paths(base_commit: str, head_commit: str) -> tuple[st def _has_inherited_collection_selector() -> bool: - """Return whether ambient pytest options narrow the collected corpus.""" - raw = os.environ.get("PYTEST_ADDOPTS", "") - if not raw.strip(): - return False + """Return whether pytest's inherited configuration narrows collection.""" try: - tokens = shlex.split(raw) - except ValueError: + config = get_config([]) + config.parse([]) + except BaseException: return True - selectors = {"-k", "-m", "--keyword", "--markexpr", "--deselect", "--ignore", "--ignore-glob"} - return any( - token in selectors or token.startswith(("--keyword=", "--markexpr=", "--deselect=", "--ignore=")) - for token in tokens + options = config.known_args_namespace + return bool( + options.keyword + or options.markexpr + or options.deselect + or options.ignore + or options.ignore_glob + or options.file_or_dir ) @@ -2437,6 +2441,7 @@ def _release_baseline_allowed( return bool( aggregate.get("complete_corpus_covered") is True and aggregate.get("terminal_green") is True + and aggregate.get("external_collection_selector") is False and isinstance(cleanup, Mapping) and cleanup.get("complete") is True and isinstance(containment, Mapping) diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 0024f8f7a3..305901b0cc 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -156,15 +156,53 @@ def test_native_corpus_excludes_only_benchmark_directory() -> None: assert "--ignore=tests/integration" not in complete_command -@pytest.mark.parametrize("selector", ["-k smoke", "--keyword=smoke", "--ignore=tests/benchmarks"]) +@pytest.mark.parametrize( + "selector", + [ + "-k smoke", + "--keyword=smoke", + "--ignore=tests/benchmarks", + "--ignore-glob=tests/integration/**", + "--ignore-glob tests/integration/**", + "tests/unit/devtools/test_verify.py::test_release_authority_requires_current_complete_green_invocation", + ], +) def test_inherited_collection_selectors_are_not_release_authority( monkeypatch: pytest.MonkeyPatch, selector: str, ) -> None: - """Removing selector detection would incorrectly authorize the narrowed corpus.""" + """The production authority path must reject every inherited narrowing form.""" monkeypatch.setenv("PYTEST_ADDOPTS", selector) assert verify._has_inherited_collection_selector() is True + assert not _release_baseline_allowed( + selection_mode="bootstrap", + verification_scope=VerificationScope.RELEASE_BASELINE, + exit_code=0, + checkout_stable=True, + aggregate={ + "complete_corpus_covered": True, + "terminal_green": True, + "external_collection_selector": verify._has_inherited_collection_selector(), + "cleanup": {"complete": True}, + "containment": {"complete": True}, + "deadline": {"met": True}, + }, + ) + + +def test_configured_collection_selector_is_not_release_authority( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Pytest config addopts are inherited by the production pytest process too.""" + (tmp_path / "pyproject.toml").write_text( + "[tool.pytest.ini_options]\naddopts = '--ignore-glob=tests/integration/**'\n", + encoding="utf-8", + ) + monkeypatch.chdir(tmp_path) + + assert verify._has_inherited_collection_selector() is True def test_unrelated_inherited_pytest_options_do_not_disable_release_authority(monkeypatch: pytest.MonkeyPatch) -> None: @@ -172,6 +210,20 @@ def test_unrelated_inherited_pytest_options_do_not_disable_release_authority(mon monkeypatch.setenv("PYTEST_ADDOPTS", "-ra --strict-markers") assert verify._has_inherited_collection_selector() is False + assert _release_baseline_allowed( + selection_mode="bootstrap", + verification_scope=VerificationScope.RELEASE_BASELINE, + exit_code=0, + checkout_stable=True, + aggregate={ + "complete_corpus_covered": True, + "terminal_green": True, + "external_collection_selector": verify._has_inherited_collection_selector(), + "cleanup": {"complete": True}, + "containment": {"complete": True}, + "deadline": {"met": True}, + }, + ) def test_lab_verify_delegates_to_lab_smoke() -> None: @@ -4098,6 +4150,7 @@ def test_release_authority_requires_current_complete_green_invocation() -> None: aggregate = { "complete_corpus_covered": True, "terminal_green": True, + "external_collection_selector": False, "cleanup": {"complete": True}, "containment": {"complete": True}, "deadline": {"met": True}, From f5fe9a3c381e8e5f07c72a84a92a5e52b7a28e24 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 14 Aug 2026 03:10:46 +0200 Subject: [PATCH 16/31] fix(verify): neutralize external pytest addopts Remove PYTEST_ADDOPTS and override configured addopts for bootstrap and full native pytest lanes. Record the invariant per lane and require it before granting release-baseline authority. --- devtools/verify.py | 33 ++--- devtools/verify_runs.py | 12 +- .../devtools/test_native_testmon_lifecycle.py | 81 +++++++++++++ tests/unit/devtools/test_verify.py | 114 ++++++------------ 4 files changed, 140 insertions(+), 100 deletions(-) diff --git a/devtools/verify.py b/devtools/verify.py index a6b93aaf3e..2a4eb55691 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -39,8 +39,6 @@ from pathlib import Path from typing import Any -from _pytest.config import get_config - from devtools.checkout_guard import ( CheckoutImportMismatchError, assert_polylogue_matches_checkout, @@ -119,6 +117,7 @@ ) ROOT = Path(__file__).resolve().parents[1] +_PYTEST_CLEAR_CONFIGURED_ADDOPTS = "--override-ini=addopts=" def _anchor_verification_paths() -> None: @@ -1635,6 +1634,9 @@ def _run( _clear_pytest_report(cmd) artifacts = run.start_step(label=label, cmd=cmd) if run is not None else None env = _subprocess_env() + release_addopts_neutralized = _pytest_uses_full_suite_basetemp(label) + if release_addopts_neutralized: + env.pop("PYTEST_ADDOPTS", None) explicit_basetemp = _pytest_command_basetemp(cmd, cwd=cwd, env=env) if explicit_basetemp is not None: env[PYTEST_EXPLICIT_BASETEMP_ENV] = str(explicit_basetemp) @@ -1688,6 +1690,9 @@ def _run( env["POLYLOGUE_PYTEST_SELECTION_NODEID_LIMIT"] = "50000" if run is not None and artifacts is not None: env = env_for_pytest_step(env, run=run, artifacts=artifacts) + if release_addopts_neutralized: + env.pop("PYTEST_ADDOPTS", None) + release_addopts_neutralized = _PYTEST_CLEAR_CONFIGURED_ADDOPTS in cmd interrupted = False pytest_containment_quiescent = True containment_error: str | None = None @@ -1742,6 +1747,7 @@ def _run( metadata["diagnosis"] = "pytest_containment_unproven" metadata["termination_reason"] = f"pytest containment did not quiesce: {containment_error}" metadata.update(_pytest_command_metadata(cmd)) + metadata["external_addopts_neutralized"] = release_addopts_neutralized metadata["heartbeat_s"] = _pytest_heartbeat_interval() metadata["timeout_s"] = _pytest_timeout_s() if timeout_s is None else timeout_s metadata["stall_timeout_s"] = _pytest_stall_timeout_s() @@ -2163,6 +2169,8 @@ def build_verify_steps( raise ValueError(f"unknown native testmon mode: {testmon_mode}") if not testmon_environment: raise ValueError("native testmon environment is required for pytest verification") + if testmon_mode in {"bootstrap", "full"}: + pytest_cmd.append(_PYTEST_CLEAR_CONFIGURED_ADDOPTS) native_args = ["--testmon", f"--testmon-env={testmon_environment}"] if testmon_mode == "affected": native_args.append("--testmon-forceselect") @@ -2370,24 +2378,6 @@ def _changed_test_relevant_paths(base_commit: str, head_commit: str) -> tuple[st ) -def _has_inherited_collection_selector() -> bool: - """Return whether pytest's inherited configuration narrows collection.""" - try: - config = get_config([]) - config.parse([]) - except BaseException: - return True - options = config.known_args_namespace - return bool( - options.keyword - or options.markexpr - or options.deselect - or options.ignore - or options.ignore_glob - or options.file_or_dir - ) - - _ACTIVE_VERIFY_RUN: tuple[VerifyRun, float, VerificationScope] | None = None @@ -2441,7 +2431,7 @@ def _release_baseline_allowed( return bool( aggregate.get("complete_corpus_covered") is True and aggregate.get("terminal_green") is True - and aggregate.get("external_collection_selector") is False + and aggregate.get("external_addopts_neutralized") is True and isinstance(cleanup, Mapping) and cleanup.get("complete") is True and isinstance(containment, Mapping) @@ -2891,7 +2881,6 @@ def _main(argv: list[str] | None = None) -> int: selection_mode=testmon_mode or "affected", invocation_duration_s=total_duration, budget_s=VERIFY_INVOCATION_BUDGET_S, - external_collection_selector=_has_inherited_collection_selector(), ) # Aggregation and final graph inspection are part of the same invocation diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index b61a99a886..d8566f2329 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -176,6 +176,7 @@ def max_optional(current: int | None, value: object) -> int | None: "cleanup_complete": step_cleanup, "containment_complete": isinstance(step.get("containment_mode"), str), "containment_mode": step.get("containment_mode"), + "external_addopts_neutralized": False, } ) @@ -201,6 +202,7 @@ def max_optional(current: int | None, value: object) -> int | None: }, "corpus": {"count": terminal_count, "digest": corpus_digest}, "selection_mode": "focused" if entry.get("tier") == "focused-test" else "none", + "external_addopts_neutralized": False, "lanes": lanes, "selected_union_count": selected_count, "terminal_union_count": terminal_count, @@ -1238,7 +1240,6 @@ def aggregate_native_testmon_run( selection_mode: str, invocation_duration_s: float, budget_s: float, - external_collection_selector: bool = False, ) -> dict[str, Any]: """Build one compact, durable aggregate for the two semantic pytest lanes.""" lanes: list[dict[str, Any]] = [] @@ -1256,6 +1257,7 @@ def aggregate_native_testmon_run( cleanup_complete = True containment_complete = True selection_complete = True + external_addopts_neutralized = True for step in steps: lane = step.get("semantic_lane") if lane not in {"parallel", "serial"}: @@ -1294,6 +1296,8 @@ def _peak(current: int | None, value: object) -> int | None: peak_storage_bytes = _peak(peak_storage_bytes, storage.get("basetemp_allocated_bytes_max")) lane_cleanup = cleanup.get("complete") if isinstance(cleanup, Mapping) else None cleanup_complete = cleanup_complete and lane_cleanup is True + lane_addopts_neutralized = step.get("external_addopts_neutralized") is True + external_addopts_neutralized = external_addopts_neutralized and lane_addopts_neutralized lane_containment_complete = bool( containment_receipt is not None and containment_receipt.get("status") == "finished" @@ -1318,6 +1322,7 @@ def _peak(current: int | None, value: object) -> int | None: "cleanup_complete": lane_cleanup, "containment_mode": step.get("containment_mode"), "containment_complete": lane_containment_complete, + "external_addopts_neutralized": lane_addopts_neutralized, } ) @@ -1332,7 +1337,7 @@ def _peak(current: int | None, value: object) -> int | None: complete_mode and selection_complete and bool(corpus) - and not external_collection_selector + and external_addopts_neutralized and selected_union == corpus_set and set(outcome_by_node) == corpus_set and not duplicate_outcomes @@ -1347,6 +1352,7 @@ def _peak(current: int | None, value: object) -> int | None: terminal_green = complete_corpus_covered and not missing_terminal and not non_green cleanup_complete = bool(lanes) and cleanup_complete containment_complete = bool(lanes) and containment_complete + external_addopts_neutralized = bool(lanes) and external_addopts_neutralized return { "schema_version": 1, "environment": { @@ -1362,7 +1368,7 @@ def _peak(current: int | None, value: object) -> int | None: "digest": hashlib.sha256("\n".join(corpus).encode()).hexdigest(), }, "selection_mode": selection_mode, - "external_collection_selector": external_collection_selector, + "external_addopts_neutralized": external_addopts_neutralized, "lanes": lanes, "outcomes": outcomes, "selected_union_count": len(selected_union), diff --git a/tests/integration/devtools/test_native_testmon_lifecycle.py b/tests/integration/devtools/test_native_testmon_lifecycle.py index 3ada9b6a46..637a40e7f3 100644 --- a/tests/integration/devtools/test_native_testmon_lifecycle.py +++ b/tests/integration/devtools/test_native_testmon_lifecycle.py @@ -433,7 +433,10 @@ def test_production_verify_all_grants_release_authority_after_complete_two_lane_ for step in lanes: assert "--testmon-noselect" in step["statistics"]["command"] assert "--testmon-forceselect" not in step["statistics"]["command"] + assert "--override-ini=addopts=" in step["statistics"]["command"] + assert step["external_addopts_neutralized"] is True aggregate = payload["pytest_aggregate"] + assert aggregate["external_addopts_neutralized"] is True assert aggregate["selection_mode"] == "full" assert aggregate["environment"]["native_corpus_count"] == 2 assert aggregate["corpus"]["count"] == 2 @@ -447,6 +450,84 @@ def test_production_verify_all_grants_release_authority_after_complete_two_lane_ assert aggregate["deadline"] == {"budget_s": 3600.0, "met": True} +@pytest.mark.parametrize( + ("environment_addopts", "configured_addopts"), + [ + ("--setup-only", None), + ("--collect-only", None), + ("tests/test_release.py::test_parallel_body_must_run", None), + ("--ignore-glob=tests/**", None), + ("--ignore-glob tests/**", None), + (None, "--setup-only --ignore-glob=tests/**"), + ("-ra --strict-markers", None), + ], + ids=( + "setup-only", + "collect-only", + "positional-node", + "ignore-glob-equal", + "ignore-glob-split", + "configured", + "harmless", + ), +) +def test_production_verify_all_neutralizes_external_pytest_addopts( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + environment_addopts: str | None, + configured_addopts: str | None, +) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _init_repo(repo) + package = repo / "polylogue" + package.mkdir() + (package / "__init__.py").write_text("", encoding="utf-8") + (repo / "tests" / "test_release.py").write_text( + "import pytest\n\n" + "def test_parallel_body_must_run():\n" + " assert False, 'parallel body executed'\n\n" + "@pytest.mark.load_sensitive\n" + "def test_serial_body_must_run():\n" + " assert False, 'serial body executed'\n", + encoding="utf-8", + ) + if configured_addopts is not None: + config = repo / "pyproject.toml" + config.write_text( + config.read_text(encoding="utf-8").replace( + 'addopts = "-p no:randomly"', + f'addopts = "{configured_addopts}"', + ), + encoding="utf-8", + ) + _commit_all(repo, "fixture") + origin = tmp_path / "origin.git" + _git(origin.parent, "init", "--bare", "-q", str(origin)) + _git(repo, "remote", "add", "origin", str(origin)) + _git(repo, "branch", "-M", "master") + _git(repo, "push", "-qu", "origin", "master") + if environment_addopts is None: + monkeypatch.delenv("PYTEST_ADDOPTS", raising=False) + else: + monkeypatch.setenv("PYTEST_ADDOPTS", environment_addopts) + + completed, payload = _run_production_verify(repo, "--all") + + assert completed.returncode == 1 + assert payload["release_baseline_allowed"] is False + aggregate = payload["pytest_aggregate"] + assert aggregate["external_addopts_neutralized"] is True + assert aggregate["selected_union_count"] == 2 + assert aggregate["terminal_union_count"] == 2 + assert aggregate["outcomes"] == {"failed": 2} + lanes = [step for step in payload["steps"] if step.get("semantic_lane")] + assert [step["external_addopts_neutralized"] for step in lanes] == [True, True] + assert all("--override-ini=addopts=" in step["statistics"]["command"] for step in lanes) + assert "parallel body executed" in completed.stderr + assert "serial body executed" in completed.stderr + + def test_runtime_json_only_mutation_forces_complete_native_selection(tmp_path: Path) -> None: repo = tmp_path / "repo" repo.mkdir() diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 305901b0cc..226270bfe7 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -140,6 +140,7 @@ def test_native_testmon_uses_exactly_two_semantic_lanes( assert selection_flag in command assert "--json-report" in command assert command[command.index("-p") + 1] == "devtools.pytest_progress_plugin" + assert (verify._PYTEST_CLEAR_CONFIGURED_ADDOPTS in command) is (mode in {"bootstrap", "full"}) def test_native_corpus_excludes_only_benchmark_directory() -> None: @@ -156,76 +157,6 @@ def test_native_corpus_excludes_only_benchmark_directory() -> None: assert "--ignore=tests/integration" not in complete_command -@pytest.mark.parametrize( - "selector", - [ - "-k smoke", - "--keyword=smoke", - "--ignore=tests/benchmarks", - "--ignore-glob=tests/integration/**", - "--ignore-glob tests/integration/**", - "tests/unit/devtools/test_verify.py::test_release_authority_requires_current_complete_green_invocation", - ], -) -def test_inherited_collection_selectors_are_not_release_authority( - monkeypatch: pytest.MonkeyPatch, - selector: str, -) -> None: - """The production authority path must reject every inherited narrowing form.""" - monkeypatch.setenv("PYTEST_ADDOPTS", selector) - - assert verify._has_inherited_collection_selector() is True - assert not _release_baseline_allowed( - selection_mode="bootstrap", - verification_scope=VerificationScope.RELEASE_BASELINE, - exit_code=0, - checkout_stable=True, - aggregate={ - "complete_corpus_covered": True, - "terminal_green": True, - "external_collection_selector": verify._has_inherited_collection_selector(), - "cleanup": {"complete": True}, - "containment": {"complete": True}, - "deadline": {"met": True}, - }, - ) - - -def test_configured_collection_selector_is_not_release_authority( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Pytest config addopts are inherited by the production pytest process too.""" - (tmp_path / "pyproject.toml").write_text( - "[tool.pytest.ini_options]\naddopts = '--ignore-glob=tests/integration/**'\n", - encoding="utf-8", - ) - monkeypatch.chdir(tmp_path) - - assert verify._has_inherited_collection_selector() is True - - -def test_unrelated_inherited_pytest_options_do_not_disable_release_authority(monkeypatch: pytest.MonkeyPatch) -> None: - """Treating every ambient option as a selector would reject harmless options.""" - monkeypatch.setenv("PYTEST_ADDOPTS", "-ra --strict-markers") - - assert verify._has_inherited_collection_selector() is False - assert _release_baseline_allowed( - selection_mode="bootstrap", - verification_scope=VerificationScope.RELEASE_BASELINE, - exit_code=0, - checkout_stable=True, - aggregate={ - "complete_corpus_covered": True, - "terminal_green": True, - "external_collection_selector": verify._has_inherited_collection_selector(), - "cleanup": {"complete": True}, - "containment": {"complete": True}, - "deadline": {"met": True}, - }, - ) - - def test_lab_verify_delegates_to_lab_smoke() -> None: steps = build_verify_steps(quick=True, lab=True) @@ -509,8 +440,8 @@ def test_aggregate_pytest_statistics_merges_partial_canonical_and_event_outcomes assert result["outcomes"] == {"passed": 1, "xfailed": 1} -def test_native_aggregate_rejects_inherited_selector_without_counting_missing_nodes(tmp_path: Path) -> None: - """Using the selected union as corpus authority would mark this narrowed run green.""" +def test_native_aggregate_requires_both_lanes_to_neutralize_external_addopts(tmp_path: Path) -> None: + """A missing lane invariant cannot become release authority through green outcomes.""" for lane, nodeid, outcome in (("parallel", "a", "passed"), ("serial", "b", "skipped")): step = tmp_path / lane step.mkdir() @@ -536,6 +467,7 @@ def test_native_aggregate_rejects_inherited_selector_without_counting_missing_no "semantic_lane": "parallel", "artifact_dir": "parallel", "exit": 0, + "external_addopts_neutralized": True, "statistics": {"cleanup": {"complete": True}}, }, { @@ -550,10 +482,11 @@ def test_native_aggregate_rejects_inherited_selector_without_counting_missing_no selection_mode="bootstrap", invocation_duration_s=0.1, budget_s=10.0, - external_collection_selector=True, ) - assert result["external_collection_selector"] is True + assert result["external_addopts_neutralized"] is False + assert result["lanes"][0]["external_addopts_neutralized"] is True + assert result["lanes"][1]["external_addopts_neutralized"] is False assert result["selected_union_count"] == 2 assert result["terminal_union_count"] == 2 assert result["non_green_count"] == 0 @@ -3077,6 +3010,36 @@ def apply_policy(env: dict[str, str], **_kwargs: object) -> tuple[dict[str, str] assert captured["POLYLOGUE_PYTEST_EXPLICIT_BASETEMP"] == str(explicit) +def test_release_lane_removes_environment_addopts_before_pytest( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured_command: list[str] = [] + captured_env: dict[str, str] = {} + completed = subprocess.CompletedProcess(args=["pytest"], returncode=0, stdout="1 passed in 0.1s\n", stderr="") + monkeypatch.setenv("PYTEST_ADDOPTS", "--setup-only") + + def run_pytest(command: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]: + captured_command.extend(command) + captured_env.update(cast("dict[str, str]", kwargs["env"])) + return completed + + with ( + patch("devtools.verify.apply_managed_pytest_runtime_policy", side_effect=lambda env, **_kwargs: (env, None)), + patch("devtools.verify._run_pytest_with_heartbeat", side_effect=run_pytest), + patch("devtools.verify.cleanup_managed_pytest_basetemp", return_value=None), + patch("devtools.verify._read_pytest_report", return_value=None), + ): + rc, _elapsed, metadata = _run( + "pytest native parallel (full)", + ["pytest", verify._PYTEST_CLEAR_CONFIGURED_ADDOPTS], + ) + + assert rc == 0 + assert verify._PYTEST_CLEAR_CONFIGURED_ADDOPTS in captured_command + assert "PYTEST_ADDOPTS" not in captured_env + assert metadata["external_addopts_neutralized"] is True + + def test_run_clears_stale_current_statistics_before_an_interrupted_pytest_step(tmp_path: Path) -> None: stale_statistics = tmp_path / verify_runs.CURRENT_STATISTICS_PATH stale_statistics.parent.mkdir(parents=True) @@ -4150,7 +4113,7 @@ def test_release_authority_requires_current_complete_green_invocation() -> None: aggregate = { "complete_corpus_covered": True, "terminal_green": True, - "external_collection_selector": False, + "external_addopts_neutralized": True, "cleanup": {"complete": True}, "containment": {"complete": True}, "deadline": {"met": True}, @@ -4173,6 +4136,7 @@ def test_release_authority_requires_current_complete_green_invocation() -> None: for broken in ( {**aggregate, "complete_corpus_covered": False}, {**aggregate, "terminal_green": False}, + {**aggregate, "external_addopts_neutralized": False}, {**aggregate, "cleanup": {"complete": False}}, {**aggregate, "containment": {"complete": False}}, {**aggregate, "deadline": {"met": False}}, From 5bd4b04b75b82648725fcd2d9efbf95eece25e9c Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 14 Aug 2026 03:58:00 +0200 Subject: [PATCH 17/31] fix(verify): own release pytest discovery Invoke release lanes with an exact owned plugin and discovery contract plus the explicit test root. Fail closed unless both lane receipts prove external pytest configuration was neutralized. --- devtools/verify.py | 194 ++++++++++++------ devtools/verify_runs.py | 18 ++ .../devtools/test_native_testmon_lifecycle.py | 81 ++++++++ tests/unit/devtools/test_verify.py | 62 +++++- 4 files changed, 294 insertions(+), 61 deletions(-) diff --git a/devtools/verify.py b/devtools/verify.py index 2a4eb55691..c0d37bdbd7 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -118,6 +118,35 @@ ROOT = Path(__file__).resolve().parents[1] _PYTEST_CLEAR_CONFIGURED_ADDOPTS = "--override-ini=addopts=" +_PYTEST_RELEASE_PLUGIN_NAMES = ( + "anyio", + "asyncio", + "hypothesispytest", + "benchmark", + "pytest_cov", + "pytest_jsonreport", + "randomly", + "syrupy", + "timeout", + "xdist", + "pytest-testmon", +) +_PYTEST_RELEASE_PLUGIN_ARGS = tuple(argument for name in _PYTEST_RELEASE_PLUGIN_NAMES for argument in ("-p", name)) +_PYTEST_CLOSED_WORLD_COLLECTION_ARGS = ( + _PYTEST_CLEAR_CONFIGURED_ADDOPTS, + "--override-ini=python_files=test_*.py *_test.py fuzz_*.py", + "--override-ini=python_classes=Test", + "--override-ini=python_functions=test", + "--override-ini=norecursedirs=", + "tests", +) + + +def _normalize_release_pytest_environment(env: dict[str, str]) -> None: + """Remove ambient pytest extensions from an authoritative child.""" + env.pop("PYTEST_ADDOPTS", None) + env.pop("PYTEST_PLUGINS", None) + env["PYTEST_DISABLE_PLUGIN_AUTOLOAD"] = "1" def _anchor_verification_paths() -> None: @@ -1624,6 +1653,8 @@ def _run( sys.stderr.write(f" {label} ... ") sys.stderr.flush() is_pytest = label.startswith("pytest") + release_lane = _pytest_uses_full_suite_basetemp(label) + closed_world_command = _release_pytest_command_is_closed_world(label, cmd) # ``bench slo`` starts pytest-benchmark itself, so it needs the same # bounded temp policy and run marker as a direct pytest step. has_managed_pytest_child = label == "bench slo" @@ -1634,9 +1665,10 @@ def _run( _clear_pytest_report(cmd) artifacts = run.start_step(label=label, cmd=cmd) if run is not None else None env = _subprocess_env() - release_addopts_neutralized = _pytest_uses_full_suite_basetemp(label) - if release_addopts_neutralized: - env.pop("PYTEST_ADDOPTS", None) + release_addopts_neutralized = release_lane + external_plugins_neutralized = release_lane + if release_lane: + _normalize_release_pytest_environment(env) explicit_basetemp = _pytest_command_basetemp(cmd, cwd=cwd, env=env) if explicit_basetemp is not None: env[PYTEST_EXPLICIT_BASETEMP_ENV] = str(explicit_basetemp) @@ -1690,9 +1722,13 @@ def _run( env["POLYLOGUE_PYTEST_SELECTION_NODEID_LIMIT"] = "50000" if run is not None and artifacts is not None: env = env_for_pytest_step(env, run=run, artifacts=artifacts) - if release_addopts_neutralized: - env.pop("PYTEST_ADDOPTS", None) + if release_lane: + _normalize_release_pytest_environment(env) release_addopts_neutralized = _PYTEST_CLEAR_CONFIGURED_ADDOPTS in cmd + external_plugins_neutralized = ( + "PYTEST_PLUGINS" not in env and env.get("PYTEST_DISABLE_PLUGIN_AUTOLOAD") == "1" + ) + closed_world_collection = closed_world_command and release_addopts_neutralized and external_plugins_neutralized interrupted = False pytest_containment_quiescent = True containment_error: str | None = None @@ -1748,6 +1784,8 @@ def _run( metadata["termination_reason"] = f"pytest containment did not quiesce: {containment_error}" metadata.update(_pytest_command_metadata(cmd)) metadata["external_addopts_neutralized"] = release_addopts_neutralized + metadata["external_plugins_neutralized"] = external_plugins_neutralized + metadata["closed_world_collection"] = closed_world_collection metadata["heartbeat_s"] = _pytest_heartbeat_interval() metadata["timeout_s"] = _pytest_timeout_s() if timeout_s is None else timeout_s metadata["stall_timeout_s"] = _pytest_stall_timeout_s() @@ -2094,6 +2132,87 @@ def _native_lane_failure_requires_stop(step: Mapping[str, Any]) -> bool: # ── step builder ──────────────────────────────────────────────────── +def _native_pytest_steps( + *, + testmon_mode: str, + testmon_environment: str, + parallel_worker_args: Sequence[str], +) -> list[tuple[str, list[str]]]: + pytest_cmd = [ + sys.executable, + "-m", + "pytest", + "-q", + "--tb=short", + "--ignore=tests/benchmarks", + "--durations=10", + f"--junitxml={PYTEST_JUNIT_REPORT_DIR}/verify-latest.xml", + "--json-report", + "--json-report-omit=collectors,log,streams,warnings", + f"--json-report-file={PYTEST_REPORT_PATH}", + "-p", + "devtools.pytest_progress_plugin", + ] + if testmon_mode in {"bootstrap", "full"}: + pytest_cmd.extend(_PYTEST_RELEASE_PLUGIN_ARGS) + pytest_cmd.extend(_PYTEST_CLOSED_WORLD_COLLECTION_ARGS) + native_args = ["--testmon", f"--testmon-env={testmon_environment}"] + if testmon_mode == "affected": + native_args.append("--testmon-forceselect") + else: + native_args.append("--testmon-noselect") + + parallel_cmd = [ + *pytest_cmd, + "-m", + "not load_sensitive", + *native_args, + *parallel_worker_args, + ] + + def _serial_report_arg(arg: str) -> str: + if arg.startswith("--junitxml="): + return f"--junitxml={PYTEST_JUNIT_REPORT_DIR}/verify-latest-serial.xml" + if arg.startswith("--json-report-file="): + return f"--json-report-file={PYTEST_REPORT_DIR / 'last-pytest-serial.json'}" + return arg + + serial_cmd = [_serial_report_arg(arg) for arg in pytest_cmd] + serial_cmd.extend( + [ + "-m", + "load_sensitive", + *native_args, + "-p", + "no:randomly", + "-n", + "0", + ] + ) + return [ + (f"pytest native parallel ({testmon_mode})", parallel_cmd), + (f"pytest native serial ({testmon_mode})", serial_cmd), + ] + + +def _release_pytest_command_is_closed_world(label: str, cmd: Sequence[str]) -> bool: + """Accept only a command produced by the owned release-lane builder.""" + match = re.fullmatch(r"pytest native (parallel|serial) \((bootstrap|full)\)", label) + if match is None: + return False + environment_args = [arg for arg in cmd if arg.startswith("--testmon-env=")] + worker_request = pytest_command_worker_request(cmd) + if len(environment_args) != 1 or worker_request is None or not worker_request.isdigit(): + return False + expected_steps = _native_pytest_steps( + testmon_mode=match.group(2), + testmon_environment=environment_args[0].removeprefix("--testmon-env="), + parallel_worker_args=("--dist=loadgroup", "-n", worker_request), + ) + expected = dict(expected_steps).get(label) + return expected is not None and list(cmd) == expected + + def build_verify_steps( *, quick: bool, @@ -2146,66 +2265,19 @@ def build_verify_steps( _report_dir = PYTEST_JUNIT_REPORT_DIR _report_dir.mkdir(parents=True, exist_ok=True) PYTEST_REPORT_DIR.mkdir(parents=True, exist_ok=True) - pytest_cmd = [ - sys.executable, - "-m", - "pytest", - "-q", - "--tb=short", - # Benchmark files are an explicit campaign surface. A number of - # them are correctness-shaped and lack the benchmark marker, so a - # marker expression alone cannot keep performance probes out of - # the correctness/testmon corpus. - "--ignore=tests/benchmarks", - "--durations=10", - f"--junitxml={_report_dir}/verify-latest.xml", - "--json-report", - "--json-report-omit=collectors,log,streams,warnings", - f"--json-report-file={PYTEST_REPORT_PATH}", - "-p", - "devtools.pytest_progress_plugin", - ] if testmon_mode not in {"affected", "bootstrap", "full"}: raise ValueError(f"unknown native testmon mode: {testmon_mode}") if not testmon_environment: raise ValueError("native testmon environment is required for pytest verification") - if testmon_mode in {"bootstrap", "full"}: - pytest_cmd.append(_PYTEST_CLEAR_CONFIGURED_ADDOPTS) - native_args = ["--testmon", f"--testmon-env={testmon_environment}"] - if testmon_mode == "affected": - native_args.append("--testmon-forceselect") - else: - native_args.append("--testmon-noselect") - - parallel_cmd = [ - *pytest_cmd, - "-m", - "not load_sensitive", - *native_args, - *_pytest_worker_args(), - ] - steps.append((f"pytest native parallel ({testmon_mode})", parallel_cmd)) - - def _serial_report_arg(arg: str) -> str: - if arg.startswith("--junitxml="): - return f"--junitxml={_report_dir}/verify-latest-serial.xml" - if arg.startswith("--json-report-file="): - return f"--json-report-file={PYTEST_REPORT_DIR / 'last-pytest-serial.json'}" - return arg - - serial_cmd = [_serial_report_arg(arg) for arg in pytest_cmd] - serial_cmd.extend( - [ - "-m", - "load_sensitive", - *native_args, - "-p", - "no:randomly", - "-n", - "0", - ] + # The release command owns its complete collection and plugin surface; + # the benchmark root remains the one explicit non-correctness corpus. + steps.extend( + _native_pytest_steps( + testmon_mode=testmon_mode, + testmon_environment=testmon_environment, + parallel_worker_args=_pytest_worker_args(), + ) ) - steps.append((f"pytest native serial ({testmon_mode})", serial_cmd)) if lab: steps.append(("lab smoke", _devtools_cmd("lab smoke", "run", "archive-smoke", "--tier", "0"))) @@ -2432,6 +2504,8 @@ def _release_baseline_allowed( aggregate.get("complete_corpus_covered") is True and aggregate.get("terminal_green") is True and aggregate.get("external_addopts_neutralized") is True + and aggregate.get("external_plugins_neutralized") is True + and aggregate.get("closed_world_collection") is True and isinstance(cleanup, Mapping) and cleanup.get("complete") is True and isinstance(containment, Mapping) diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index d8566f2329..b33f432bc3 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -177,6 +177,8 @@ def max_optional(current: int | None, value: object) -> int | None: "containment_complete": isinstance(step.get("containment_mode"), str), "containment_mode": step.get("containment_mode"), "external_addopts_neutralized": False, + "external_plugins_neutralized": False, + "closed_world_collection": False, } ) @@ -203,6 +205,8 @@ def max_optional(current: int | None, value: object) -> int | None: "corpus": {"count": terminal_count, "digest": corpus_digest}, "selection_mode": "focused" if entry.get("tier") == "focused-test" else "none", "external_addopts_neutralized": False, + "external_plugins_neutralized": False, + "closed_world_collection": False, "lanes": lanes, "selected_union_count": selected_count, "terminal_union_count": terminal_count, @@ -1258,6 +1262,8 @@ def aggregate_native_testmon_run( containment_complete = True selection_complete = True external_addopts_neutralized = True + external_plugins_neutralized = True + closed_world_collection = True for step in steps: lane = step.get("semantic_lane") if lane not in {"parallel", "serial"}: @@ -1298,6 +1304,10 @@ def _peak(current: int | None, value: object) -> int | None: cleanup_complete = cleanup_complete and lane_cleanup is True lane_addopts_neutralized = step.get("external_addopts_neutralized") is True external_addopts_neutralized = external_addopts_neutralized and lane_addopts_neutralized + lane_plugins_neutralized = step.get("external_plugins_neutralized") is True + external_plugins_neutralized = external_plugins_neutralized and lane_plugins_neutralized + lane_closed_world_collection = step.get("closed_world_collection") is True + closed_world_collection = closed_world_collection and lane_closed_world_collection lane_containment_complete = bool( containment_receipt is not None and containment_receipt.get("status") == "finished" @@ -1323,6 +1333,8 @@ def _peak(current: int | None, value: object) -> int | None: "containment_mode": step.get("containment_mode"), "containment_complete": lane_containment_complete, "external_addopts_neutralized": lane_addopts_neutralized, + "external_plugins_neutralized": lane_plugins_neutralized, + "closed_world_collection": lane_closed_world_collection, } ) @@ -1338,6 +1350,8 @@ def _peak(current: int | None, value: object) -> int | None: and selection_complete and bool(corpus) and external_addopts_neutralized + and external_plugins_neutralized + and closed_world_collection and selected_union == corpus_set and set(outcome_by_node) == corpus_set and not duplicate_outcomes @@ -1353,6 +1367,8 @@ def _peak(current: int | None, value: object) -> int | None: cleanup_complete = bool(lanes) and cleanup_complete containment_complete = bool(lanes) and containment_complete external_addopts_neutralized = bool(lanes) and external_addopts_neutralized + external_plugins_neutralized = bool(lanes) and external_plugins_neutralized + closed_world_collection = bool(lanes) and closed_world_collection return { "schema_version": 1, "environment": { @@ -1369,6 +1385,8 @@ def _peak(current: int | None, value: object) -> int | None: }, "selection_mode": selection_mode, "external_addopts_neutralized": external_addopts_neutralized, + "external_plugins_neutralized": external_plugins_neutralized, + "closed_world_collection": closed_world_collection, "lanes": lanes, "outcomes": outcomes, "selected_union_count": len(selected_union), diff --git a/tests/integration/devtools/test_native_testmon_lifecycle.py b/tests/integration/devtools/test_native_testmon_lifecycle.py index 637a40e7f3..a4f865cd52 100644 --- a/tests/integration/devtools/test_native_testmon_lifecycle.py +++ b/tests/integration/devtools/test_native_testmon_lifecycle.py @@ -435,8 +435,12 @@ def test_production_verify_all_grants_release_authority_after_complete_two_lane_ assert "--testmon-forceselect" not in step["statistics"]["command"] assert "--override-ini=addopts=" in step["statistics"]["command"] assert step["external_addopts_neutralized"] is True + assert step["external_plugins_neutralized"] is True + assert step["closed_world_collection"] is True aggregate = payload["pytest_aggregate"] assert aggregate["external_addopts_neutralized"] is True + assert aggregate["external_plugins_neutralized"] is True + assert aggregate["closed_world_collection"] is True assert aggregate["selection_mode"] == "full" assert aggregate["environment"]["native_corpus_count"] == 2 assert aggregate["corpus"]["count"] == 2 @@ -518,16 +522,93 @@ def test_production_verify_all_neutralizes_external_pytest_addopts( assert payload["release_baseline_allowed"] is False aggregate = payload["pytest_aggregate"] assert aggregate["external_addopts_neutralized"] is True + assert aggregate["external_plugins_neutralized"] is True assert aggregate["selected_union_count"] == 2 assert aggregate["terminal_union_count"] == 2 assert aggregate["outcomes"] == {"failed": 2} lanes = [step for step in payload["steps"] if step.get("semantic_lane")] assert [step["external_addopts_neutralized"] for step in lanes] == [True, True] + assert [step["external_plugins_neutralized"] for step in lanes] == [True, True] assert all("--override-ini=addopts=" in step["statistics"]["command"] for step in lanes) assert "parallel body executed" in completed.stderr assert "serial body executed" in completed.stderr +def test_production_verify_all_owns_complete_test_root_over_configured_testpaths( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _init_repo(repo) + package = repo / "polylogue" + package.mkdir() + (package / "__init__.py").write_text("", encoding="utf-8") + narrowed = repo / "tests" / "narrowed" + narrowed.mkdir() + (narrowed / "test_owned.py").write_text( + "import pytest\n\n" + "def test_parallel_owned():\n" + " assert True\n\n" + "@pytest.mark.load_sensitive\n" + "def test_serial_owned():\n" + " assert True\n", + encoding="utf-8", + ) + outside = repo / "tests" / "outside" + outside.mkdir() + (outside / "test_omitted_failure.py").write_text( + "class TestOmitted:\n" + " def test_must_not_be_omitted(self):\n" + " assert False, 'outside configured discovery executed'\n", + encoding="utf-8", + ) + (repo / "ambient_narrow.py").write_text( + "def pytest_ignore_collect(collection_path, config):\n return 'outside' in collection_path.parts\n", + encoding="utf-8", + ) + config = repo / "pyproject.toml" + config.write_text( + config.read_text(encoding="utf-8").replace( + 'cache_dir = ".cache/pytest"', + 'cache_dir = ".cache/pytest"\n' + 'testpaths = ["tests/narrowed"]\n' + 'python_files = ["test_owned.py"]\n' + 'python_classes = ["Owned"]\n' + 'python_functions = ["test_*_owned"]\n' + 'norecursedirs = ["outside"]', + ), + encoding="utf-8", + ) + _commit_all(repo, "narrow discovery fixture") + origin = tmp_path / "origin.git" + _git(origin.parent, "init", "--bare", "-q", str(origin)) + _git(repo, "remote", "add", "origin", str(origin)) + _git(repo, "branch", "-M", "master") + _git(repo, "push", "-qu", "origin", "master") + monkeypatch.setenv("PYTEST_PLUGINS", "ambient_narrow") + monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", raising=False) + + completed, payload = _run_production_verify(repo, "--all") + + assert completed.returncode == 1 + assert payload["release_baseline_allowed"] is False + aggregate = payload["pytest_aggregate"] + assert aggregate["external_plugins_neutralized"] is True + assert aggregate["closed_world_collection"] is True + assert aggregate["corpus"]["count"] == 3 + assert aggregate["selected_union_count"] == 3 + assert aggregate["terminal_union_count"] == 3 + assert aggregate["outcomes"] == {"failed": 1, "passed": 2} + assert aggregate["complete_corpus_covered"] is True + assert aggregate["terminal_green"] is False + lanes = [step for step in payload["steps"] if step.get("semantic_lane")] + assert [step["external_plugins_neutralized"] for step in lanes] == [True, True] + assert [step["closed_world_collection"] for step in lanes] == [True, True] + assert all(step["statistics"]["command"].count("tests") == 1 for step in lanes) + assert "outside configured discovery executed" in completed.stderr + + def test_runtime_json_only_mutation_forces_complete_native_selection(tmp_path: Path) -> None: repo = tmp_path / "repo" repo.mkdir() diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 226270bfe7..24e67c3286 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -141,6 +141,38 @@ def test_native_testmon_uses_exactly_two_semantic_lanes( assert "--json-report" in command assert command[command.index("-p") + 1] == "devtools.pytest_progress_plugin" assert (verify._PYTEST_CLEAR_CONFIGURED_ADDOPTS in command) is (mode in {"bootstrap", "full"}) + assert all(arg in command for arg in verify._PYTEST_CLOSED_WORLD_COLLECTION_ARGS) is ( + mode in {"bootstrap", "full"} + ) + assert all(arg in command for arg in verify._PYTEST_RELEASE_PLUGIN_ARGS) is (mode in {"bootstrap", "full"}) + if mode in {"bootstrap", "full"}: + assert command.count("tests") == 1 + assert "--override-ini=python_files=test_*.py *_test.py fuzz_*.py" in command + assert "--override-ini=python_classes=Test" in command + assert "--override-ini=python_functions=test" in command + assert "--override-ini=norecursedirs=" in command + + +def test_release_lane_command_contract_rejects_unowned_selectors( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr("devtools.verify.adaptive_pytest_worker_count", lambda _env: 2) + steps = build_verify_steps( + quick=False, + lab=False, + testmon_mode="full", + testmon_environment="env-digest", + ) + label, command = next(step for step in steps if "parallel" in step[0]) + + assert verify._release_pytest_command_is_closed_world(label, command) + for narrowed in ( + [*command, "--ignore=tests/unit"], + [*command, "tests/unit"], + [*command, "--deselect=tests/test_failure.py::test_failure"], + [*command, "--override-ini=python_files=test_owned.py"], + ): + assert not verify._release_pytest_command_is_closed_world(label, narrowed) def test_native_corpus_excludes_only_benchmark_directory() -> None: @@ -468,6 +500,8 @@ def test_native_aggregate_requires_both_lanes_to_neutralize_external_addopts(tmp "artifact_dir": "parallel", "exit": 0, "external_addopts_neutralized": True, + "external_plugins_neutralized": True, + "closed_world_collection": True, "statistics": {"cleanup": {"complete": True}}, }, { @@ -487,6 +521,12 @@ def test_native_aggregate_requires_both_lanes_to_neutralize_external_addopts(tmp assert result["external_addopts_neutralized"] is False assert result["lanes"][0]["external_addopts_neutralized"] is True assert result["lanes"][1]["external_addopts_neutralized"] is False + assert result["external_plugins_neutralized"] is False + assert result["lanes"][0]["external_plugins_neutralized"] is True + assert result["lanes"][1]["external_plugins_neutralized"] is False + assert result["closed_world_collection"] is False + assert result["lanes"][0]["closed_world_collection"] is True + assert result["lanes"][1]["closed_world_collection"] is False assert result["selected_union_count"] == 2 assert result["terminal_union_count"] == 2 assert result["non_green_count"] == 0 @@ -3017,6 +3057,18 @@ def test_release_lane_removes_environment_addopts_before_pytest( captured_env: dict[str, str] = {} completed = subprocess.CompletedProcess(args=["pytest"], returncode=0, stdout="1 passed in 0.1s\n", stderr="") monkeypatch.setenv("PYTEST_ADDOPTS", "--setup-only") + monkeypatch.setenv("PYTEST_PLUGINS", "ambient_narrow") + monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", raising=False) + command = next( + command + for label, command in build_verify_steps( + quick=False, + lab=False, + testmon_mode="full", + testmon_environment="env-digest", + ) + if "parallel" in label + ) def run_pytest(command: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]: captured_command.extend(command) @@ -3031,13 +3083,17 @@ def run_pytest(command: list[str], **kwargs: object) -> subprocess.CompletedProc ): rc, _elapsed, metadata = _run( "pytest native parallel (full)", - ["pytest", verify._PYTEST_CLEAR_CONFIGURED_ADDOPTS], + command, ) assert rc == 0 assert verify._PYTEST_CLEAR_CONFIGURED_ADDOPTS in captured_command assert "PYTEST_ADDOPTS" not in captured_env + assert "PYTEST_PLUGINS" not in captured_env + assert captured_env["PYTEST_DISABLE_PLUGIN_AUTOLOAD"] == "1" assert metadata["external_addopts_neutralized"] is True + assert metadata["external_plugins_neutralized"] is True + assert metadata["closed_world_collection"] is True def test_run_clears_stale_current_statistics_before_an_interrupted_pytest_step(tmp_path: Path) -> None: @@ -4114,6 +4170,8 @@ def test_release_authority_requires_current_complete_green_invocation() -> None: "complete_corpus_covered": True, "terminal_green": True, "external_addopts_neutralized": True, + "external_plugins_neutralized": True, + "closed_world_collection": True, "cleanup": {"complete": True}, "containment": {"complete": True}, "deadline": {"met": True}, @@ -4137,6 +4195,8 @@ def test_release_authority_requires_current_complete_green_invocation() -> None: {**aggregate, "complete_corpus_covered": False}, {**aggregate, "terminal_green": False}, {**aggregate, "external_addopts_neutralized": False}, + {**aggregate, "external_plugins_neutralized": False}, + {**aggregate, "closed_world_collection": False}, {**aggregate, "cleanup": {"complete": False}}, {**aggregate, "containment": {"complete": False}}, {**aggregate, "deadline": {"met": False}}, From 55c4cac8aec39673d5327eddfb56c9d4132e3c4a Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 14 Aug 2026 04:46:02 +0200 Subject: [PATCH 18/31] fix: close native pytest execution contract --- devtools/testmon_bootstrap.py | 39 +++--- devtools/verify.py | 118 ++++++++++++------ .../devtools/test_native_testmon_lifecycle.py | 60 +++++++++ tests/unit/devtools/test_testmon_bootstrap.py | 57 +++++++-- tests/unit/devtools/test_verify.py | 107 +++++++++++++--- 5 files changed, 297 insertions(+), 84 deletions(-) diff --git a/devtools/testmon_bootstrap.py b/devtools/testmon_bootstrap.py index 3b00cf946b..1175e14aa9 100644 --- a/devtools/testmon_bootstrap.py +++ b/devtools/testmon_bootstrap.py @@ -26,7 +26,6 @@ import json import os import platform -import shlex import sqlite3 import stat import subprocess @@ -63,9 +62,6 @@ _PYTEST_ENVIRONMENT_KEYS = ( "HYPOTHESIS_PROFILE", "POLYLOGUE_CI", - "PYTEST_ADDOPTS", - "PYTEST_DISABLE_PLUGIN_AUTOLOAD", - "PYTEST_PLUGINS", ) @@ -198,18 +194,6 @@ def _active_local_pytest_plugin_paths(root: Path) -> set[str]: """Resolve collection-active local pytest plugins regardless of filename.""" paths: set[str] = set() plugin_names = _declared_pytest_plugin_names(root) - plugin_names.update(os.environ.get("PYTEST_PLUGINS", "").split(",")) - try: - addopts = shlex.split(os.environ.get("PYTEST_ADDOPTS", "")) - except ValueError as exc: - raise NativeTestmonRepairError(f"cannot parse PYTEST_ADDOPTS for native environment: {exc}") from exc - for index, option in enumerate(addopts): - if option == "-p" and index + 1 < len(addopts): - plugin_names.add(addopts[index + 1]) - elif option.startswith("-p="): - plugin_names.add(option.removeprefix("-p=")) - elif option.startswith("-p") and len(option) > 2: - plugin_names.add(option.removeprefix("-p")) for raw_name in plugin_names: module_name = raw_name.strip() if not module_name or any(part in {"", ".", ".."} for part in module_name.split(".")): @@ -546,10 +530,29 @@ def inspect_native_testmon_environment( def _owned_paths(repo_root: Path) -> tuple[Path, ...]: - data = repo_root.resolve() / TESTMON_DATA_RELPATH + root = repo_root.resolve() + _validate_owned_state_parents(root) + data = root / TESTMON_DATA_RELPATH return (data, *(Path(f"{data}{suffix}") for suffix in TESTMON_SIDECAR_SUFFIXES)) +def _validate_owned_state_parents(repo_root: Path) -> None: + """Reject state paths that escape the checkout through a symlink parent.""" + parent = repo_root.resolve() + for part in TESTMON_DATA_RELPATH.parent.parts: + parent /= part + try: + mode = parent.lstat().st_mode + except FileNotFoundError: + return + except OSError as exc: + raise NativeTestmonRepairError(f"cannot inspect owned testmon parent {parent}: {exc}") from exc + if stat.S_ISLNK(mode): + raise NativeTestmonRepairError(f"refusing symlinked owned testmon parent {parent}") + if not stat.S_ISDIR(mode): + raise NativeTestmonRepairError(f"owned testmon parent is not a directory: {parent}") + + def remove_invalid_native_testmon_state(repo_root: Path) -> tuple[Path, ...]: """Remove only the exact checkout-owned SQLite file and known sidecars.""" removed: list[Path] = [] @@ -678,6 +681,7 @@ def prepare_native_testmon_environment( ) -> NativeTestmonPreparation: """Repair derived local state and optionally reuse a matching main graph.""" root = repo_root.resolve() + _validate_owned_state_parents(root) environment_name = testmon_environment_digest( root, pytest_profile=pytest_profile, @@ -710,6 +714,7 @@ def prepare_native_testmon_environment( _ensure_deadline(deadline_monotonic) copied_from: Path | None = None if main_checkout is not None and main_checkout != root and not missing_checkout_paths: + _validate_owned_state_parents(main_checkout) main_data = main_checkout / TESTMON_DATA_RELPATH main = inspect_native_testmon_environment( main_data, diff --git a/devtools/verify.py b/devtools/verify.py index c0d37bdbd7..01dcd24834 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -21,6 +21,7 @@ import argparse import contextlib +import fcntl import hashlib import json import math @@ -30,11 +31,12 @@ import shlex import shutil import signal +import stat import subprocess import sys import time import uuid -from collections.abc import Mapping, Sequence +from collections.abc import Iterator, Mapping, Sequence from datetime import datetime, timezone from pathlib import Path from typing import Any @@ -118,7 +120,7 @@ ROOT = Path(__file__).resolve().parents[1] _PYTEST_CLEAR_CONFIGURED_ADDOPTS = "--override-ini=addopts=" -_PYTEST_RELEASE_PLUGIN_NAMES = ( +_PYTEST_MANAGED_PLUGIN_NAMES = ( "anyio", "asyncio", "hypothesispytest", @@ -131,7 +133,7 @@ "xdist", "pytest-testmon", ) -_PYTEST_RELEASE_PLUGIN_ARGS = tuple(argument for name in _PYTEST_RELEASE_PLUGIN_NAMES for argument in ("-p", name)) +_PYTEST_MANAGED_PLUGIN_ARGS = tuple(argument for name in _PYTEST_MANAGED_PLUGIN_NAMES for argument in ("-p", name)) _PYTEST_CLOSED_WORLD_COLLECTION_ARGS = ( _PYTEST_CLEAR_CONFIGURED_ADDOPTS, "--override-ini=python_files=test_*.py *_test.py fuzz_*.py", @@ -142,13 +144,47 @@ ) -def _normalize_release_pytest_environment(env: dict[str, str]) -> None: - """Remove ambient pytest extensions from an authoritative child.""" +def _normalize_managed_pytest_environment(env: dict[str, str]) -> None: + """Remove ambient pytest options and extensions from a managed child.""" env.pop("PYTEST_ADDOPTS", None) env.pop("PYTEST_PLUGINS", None) env["PYTEST_DISABLE_PLUGIN_AUTOLOAD"] = "1" +@contextlib.contextmanager +def _native_testmon_lifecycle_lock(repo_root: Path) -> Iterator[None]: + """Serialize one checkout's native testmon preparation, lanes, and inspection.""" + cache = repo_root.resolve() / ".cache" + try: + mode = cache.lstat().st_mode + except FileNotFoundError: + cache.mkdir(exist_ok=True) + mode = cache.lstat().st_mode + except OSError as exc: + raise NativeTestmonRepairError(f"cannot inspect native testmon lock parent {cache}: {exc}") from exc + if not stat.S_ISDIR(mode): + raise NativeTestmonRepairError(f"native testmon lock parent is not an owned directory: {cache}") + lock_path = cache / "native-testmon-lifecycle.lock" + with lock_path.open("a+", encoding="utf-8") as handle: + try: + fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + handle.seek(0) + holder = handle.read().strip() or "another verify invocation" + sys.stderr.write(f"verify: waiting for native testmon lifecycle lock ({holder})\n") + sys.stderr.flush() + fcntl.flock(handle.fileno(), fcntl.LOCK_EX) + handle.seek(0) + handle.truncate() + handle.write(f"pid={os.getpid()}") + handle.flush() + try: + yield + finally: + handle.seek(0) + handle.truncate() + + def _anchor_verification_paths() -> None: """Use the checkout root for relative verification state when invoked inside it.""" current = Path.cwd().resolve() @@ -1653,8 +1689,8 @@ def _run( sys.stderr.write(f" {label} ... ") sys.stderr.flush() is_pytest = label.startswith("pytest") - release_lane = _pytest_uses_full_suite_basetemp(label) - closed_world_command = _release_pytest_command_is_closed_world(label, cmd) + managed_native_lane = label.startswith("pytest native") + closed_world_command = _native_pytest_command_is_closed_world(label, cmd) # ``bench slo`` starts pytest-benchmark itself, so it needs the same # bounded temp policy and run marker as a direct pytest step. has_managed_pytest_child = label == "bench slo" @@ -1665,10 +1701,10 @@ def _run( _clear_pytest_report(cmd) artifacts = run.start_step(label=label, cmd=cmd) if run is not None else None env = _subprocess_env() - release_addopts_neutralized = release_lane - external_plugins_neutralized = release_lane - if release_lane: - _normalize_release_pytest_environment(env) + external_addopts_neutralized = False + external_plugins_neutralized = False + if managed_native_lane: + _normalize_managed_pytest_environment(env) explicit_basetemp = _pytest_command_basetemp(cmd, cwd=cwd, env=env) if explicit_basetemp is not None: env[PYTEST_EXPLICIT_BASETEMP_ENV] = str(explicit_basetemp) @@ -1722,13 +1758,13 @@ def _run( env["POLYLOGUE_PYTEST_SELECTION_NODEID_LIMIT"] = "50000" if run is not None and artifacts is not None: env = env_for_pytest_step(env, run=run, artifacts=artifacts) - if release_lane: - _normalize_release_pytest_environment(env) - release_addopts_neutralized = _PYTEST_CLEAR_CONFIGURED_ADDOPTS in cmd + if managed_native_lane: + _normalize_managed_pytest_environment(env) + external_addopts_neutralized = _PYTEST_CLEAR_CONFIGURED_ADDOPTS in cmd external_plugins_neutralized = ( "PYTEST_PLUGINS" not in env and env.get("PYTEST_DISABLE_PLUGIN_AUTOLOAD") == "1" ) - closed_world_collection = closed_world_command and release_addopts_neutralized and external_plugins_neutralized + closed_world_collection = closed_world_command and external_addopts_neutralized and external_plugins_neutralized interrupted = False pytest_containment_quiescent = True containment_error: str | None = None @@ -1783,7 +1819,7 @@ def _run( metadata["diagnosis"] = "pytest_containment_unproven" metadata["termination_reason"] = f"pytest containment did not quiesce: {containment_error}" metadata.update(_pytest_command_metadata(cmd)) - metadata["external_addopts_neutralized"] = release_addopts_neutralized + metadata["external_addopts_neutralized"] = external_addopts_neutralized metadata["external_plugins_neutralized"] = external_plugins_neutralized metadata["closed_world_collection"] = closed_world_collection metadata["heartbeat_s"] = _pytest_heartbeat_interval() @@ -2153,9 +2189,8 @@ def _native_pytest_steps( "-p", "devtools.pytest_progress_plugin", ] - if testmon_mode in {"bootstrap", "full"}: - pytest_cmd.extend(_PYTEST_RELEASE_PLUGIN_ARGS) - pytest_cmd.extend(_PYTEST_CLOSED_WORLD_COLLECTION_ARGS) + pytest_cmd.extend(_PYTEST_MANAGED_PLUGIN_ARGS) + pytest_cmd.extend(_PYTEST_CLOSED_WORLD_COLLECTION_ARGS) native_args = ["--testmon", f"--testmon-env={testmon_environment}"] if testmon_mode == "affected": native_args.append("--testmon-forceselect") @@ -2195,9 +2230,9 @@ def _serial_report_arg(arg: str) -> str: ] -def _release_pytest_command_is_closed_world(label: str, cmd: Sequence[str]) -> bool: - """Accept only a command produced by the owned release-lane builder.""" - match = re.fullmatch(r"pytest native (parallel|serial) \((bootstrap|full)\)", label) +def _native_pytest_command_is_closed_world(label: str, cmd: Sequence[str]) -> bool: + """Accept only a command produced by the managed native-lane builder.""" + match = re.fullmatch(r"pytest native (parallel|serial) \((affected|bootstrap|full)\)", label) if match is None: return False environment_args = [arg for arg in cmd if arg.startswith("--testmon-env=")] @@ -2269,7 +2304,7 @@ def build_verify_steps( raise ValueError(f"unknown native testmon mode: {testmon_mode}") if not testmon_environment: raise ValueError("native testmon environment is required for pytest verification") - # The release command owns its complete collection and plugin surface; + # Every native command owns its collection, option, and plugin surface; # the benchmark root remains the one explicit non-correctness corpus. steps.extend( _native_pytest_steps( @@ -2854,9 +2889,9 @@ def _main(argv: list[str] | None = None) -> int: break assert mutation_monitor is not None - mutation_observation = finish_checkout_mutation_monitor(mutation_monitor) final_head = _git_head() final_checkout_fingerprint = worktree_fingerprint(ROOT) + mutation_observation = finish_checkout_mutation_monitor(mutation_monitor) checkout_stable = True checkout_fingerprint_unavailable = ( head is None @@ -3126,19 +3161,22 @@ def _finalize_verify_runner_exception( def main(argv: list[str] | None = None) -> int: global _ACTIVE_VERIFY_RUN _ACTIVE_VERIFY_RUN = None - try: - return _main(argv) - except Exception as exc: - if _ACTIVE_VERIFY_RUN is None: - raise - raw_argv = sys.argv[1:] if argv is None else argv - run, run_started, verification_scope = _ACTIVE_VERIFY_RUN - return _finalize_verify_runner_exception( - run, - exc, - run_started=run_started, - verification_scope=verification_scope, - use_json="--json" in raw_argv, - ) - finally: - _ACTIVE_VERIFY_RUN = None + raw_argv = list(sys.argv[1:] if argv is None else argv) + native_pytest_enabled = not any(flag in raw_argv for flag in ("--quick", "--commit", "--history")) + lock = _native_testmon_lifecycle_lock(ROOT) if native_pytest_enabled else contextlib.nullcontext() + with lock: + try: + return _main(argv) + except Exception as exc: + if _ACTIVE_VERIFY_RUN is None: + raise + run, run_started, verification_scope = _ACTIVE_VERIFY_RUN + return _finalize_verify_runner_exception( + run, + exc, + run_started=run_started, + verification_scope=verification_scope, + use_json="--json" in raw_argv, + ) + finally: + _ACTIVE_VERIFY_RUN = None diff --git a/tests/integration/devtools/test_native_testmon_lifecycle.py b/tests/integration/devtools/test_native_testmon_lifecycle.py index a4f865cd52..ec633b5452 100644 --- a/tests/integration/devtools/test_native_testmon_lifecycle.py +++ b/tests/integration/devtools/test_native_testmon_lifecycle.py @@ -534,6 +534,66 @@ def test_production_verify_all_neutralizes_external_pytest_addopts( assert "serial body executed" in completed.stderr +@pytest.mark.parametrize("ambient_addopts", ["--collect-only", "--setup-only"]) +def test_production_affected_verify_neutralizes_execution_suppressing_addopts( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ambient_addopts: str, +) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _init_repo(repo) + package = repo / "polylogue" + package.mkdir() + (package / "__init__.py").write_text("", encoding="utf-8") + tests = repo / "tests" / "test_affected.py" + tests.write_text( + "import pytest\n\n" + "def test_parallel_body():\n" + " assert True\n\n" + "@pytest.mark.load_sensitive\n" + "def test_serial_body():\n" + " assert True\n", + encoding="utf-8", + ) + _commit_all(repo, "fixture") + origin = tmp_path / "origin.git" + _git(origin.parent, "init", "--bare", "-q", str(origin)) + _git(repo, "remote", "add", "origin", str(origin)) + _git(repo, "branch", "-M", "master") + _git(repo, "push", "-qu", "origin", "master") + monkeypatch.setenv("PYTEST_ADDOPTS", ambient_addopts) + + seeded, bootstrap = _run_production_verify(repo) + + assert seeded.returncode == 0, seeded.stderr + assert bootstrap["testmon_environment"]["selection_mode"] == "bootstrap" + tests.write_text( + "import pytest\n\n" + "def test_parallel_body():\n" + " assert False, 'affected parallel body executed'\n\n" + "@pytest.mark.load_sensitive\n" + "def test_serial_body():\n" + " assert False, 'affected serial body executed'\n", + encoding="utf-8", + ) + + completed, payload = _run_production_verify(repo) + + assert completed.returncode == 1 + assert payload["testmon_environment"]["selection_mode"] == "affected" + assert payload["release_baseline_allowed"] is False + aggregate = payload["pytest_aggregate"] + assert aggregate["selected_union_count"] == 2 + assert aggregate["terminal_union_count"] == 2 + assert aggregate["outcomes"] == {"failed": 2} + lanes = [step for step in payload["steps"] if step.get("semantic_lane")] + assert [step["external_addopts_neutralized"] for step in lanes] == [True, True] + assert [step["closed_world_collection"] for step in lanes] == [True, True] + assert "affected parallel body executed" in completed.stderr + assert "affected serial body executed" in completed.stderr + + def test_production_verify_all_owns_complete_test_root_over_configured_testpaths( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/devtools/test_testmon_bootstrap.py b/tests/unit/devtools/test_testmon_bootstrap.py index 3c1616adde..3e0166b66f 100644 --- a/tests/unit/devtools/test_testmon_bootstrap.py +++ b/tests/unit/devtools/test_testmon_bootstrap.py @@ -11,6 +11,7 @@ classify_native_testmon_changes, classify_source_ast, executable_python_paths, + prepare_native_testmon_environment, remove_invalid_native_testmon_state, ) from devtools.testmon_bootstrap import ( @@ -173,8 +174,8 @@ def test_environment_digest_changes_with_collection_semantics( config_changed = _testmon_environment_digest(tmp_path, pytest_profile="slow=include") monkeypatch.setattr("devtools.testmon_bootstrap._installed_distributions", lambda: (("pytest", "changed"),)) distributions_changed = _testmon_environment_digest(tmp_path, pytest_profile="slow=include") - monkeypatch.setenv("PYTEST_ADDOPTS", "--strict-markers") - environment_changed = _testmon_environment_digest(tmp_path, pytest_profile="slow=include") + monkeypatch.setenv("POLYLOGUE_CI", "testmon-digest-contract") + managed_environment_changed = _testmon_environment_digest(tmp_path, pytest_profile="slow=include") profile_changed = _testmon_environment_digest(tmp_path, pytest_profile="slow=exclude") assert ( @@ -183,7 +184,7 @@ def test_environment_digest_changes_with_collection_semantics( initial, config_changed, distributions_changed, - environment_changed, + managed_environment_changed, profile_changed, } ) @@ -226,7 +227,7 @@ def test_declared_local_fixture_plugin_changes_environment(tmp_path: Path) -> No assert _testmon_environment_digest(tmp_path) != initial -def test_environment_digest_hashes_explicit_local_plugin_regardless_of_name( +def test_environment_digest_ignores_neutralized_pytest_plugins( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -237,11 +238,11 @@ def test_environment_digest_hashes_explicit_local_plugin_regardless_of_name( initial = _testmon_environment_digest(tmp_path) plugin.write_text("VALUE = 'v2'\n", encoding="utf-8") - assert _testmon_environment_digest(tmp_path) != initial + assert _testmon_environment_digest(tmp_path) == initial @pytest.mark.parametrize("addopts", ["-p local_plugin", "-p=local_plugin"]) -def test_environment_digest_hashes_local_plugin_from_pytest_addopts( +def test_environment_digest_ignores_plugins_from_neutralized_pytest_addopts( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, addopts: str, @@ -253,7 +254,18 @@ def test_environment_digest_hashes_local_plugin_from_pytest_addopts( initial = _testmon_environment_digest(tmp_path) plugin.write_text("VALUE = 'v2'\n", encoding="utf-8") - assert _testmon_environment_digest(tmp_path) != initial + assert _testmon_environment_digest(tmp_path) == initial + + +def test_environment_digest_ignores_neutralized_pytest_addopts( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + initial = _testmon_environment_digest(tmp_path) + + monkeypatch.setenv("PYTEST_ADDOPTS", "--setup-only --ignore-glob=tests/**") + + assert _testmon_environment_digest(tmp_path) == initial def test_environment_digest_stops_at_invocation_deadline(tmp_path: Path) -> None: @@ -280,3 +292,34 @@ def test_invalid_cleanup_refuses_directory_at_database_path(tmp_path: Path) -> N with pytest.raises(NativeTestmonRepairError, match="refusing to remove directory"): remove_invalid_native_testmon_state(tmp_path) + + +@pytest.mark.parametrize("symlinked_parent", [".cache", ".cache/testmon"]) +def test_invalid_cleanup_refuses_symlinked_state_parents( + tmp_path: Path, + symlinked_parent: str, +) -> None: + outside = tmp_path / "outside" + outside.mkdir() + sentinel = outside / "testmon" / "testmondata" if symlinked_parent == ".cache" else outside / "testmondata" + sentinel.parent.mkdir(parents=True, exist_ok=True) + sentinel.write_text("external state", encoding="utf-8") + parent = tmp_path / symlinked_parent + parent.parent.mkdir(parents=True, exist_ok=True) + parent.symlink_to(outside, target_is_directory=True) + + with pytest.raises(NativeTestmonRepairError, match="symlinked owned testmon parent"): + remove_invalid_native_testmon_state(tmp_path) + + assert sentinel.read_text(encoding="utf-8") == "external state" + + +def test_native_preparation_rejects_symlinked_state_parent_before_inspection(tmp_path: Path) -> None: + outside = tmp_path / "outside" + outside.mkdir() + cache = tmp_path / ".cache" + cache.mkdir() + (cache / "testmon").symlink_to(outside, target_is_directory=True) + + with pytest.raises(NativeTestmonRepairError, match="symlinked owned testmon parent"): + prepare_native_testmon_environment(tmp_path) diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 24e67c3286..e72019d543 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -140,39 +140,38 @@ def test_native_testmon_uses_exactly_two_semantic_lanes( assert selection_flag in command assert "--json-report" in command assert command[command.index("-p") + 1] == "devtools.pytest_progress_plugin" - assert (verify._PYTEST_CLEAR_CONFIGURED_ADDOPTS in command) is (mode in {"bootstrap", "full"}) - assert all(arg in command for arg in verify._PYTEST_CLOSED_WORLD_COLLECTION_ARGS) is ( - mode in {"bootstrap", "full"} - ) - assert all(arg in command for arg in verify._PYTEST_RELEASE_PLUGIN_ARGS) is (mode in {"bootstrap", "full"}) - if mode in {"bootstrap", "full"}: - assert command.count("tests") == 1 - assert "--override-ini=python_files=test_*.py *_test.py fuzz_*.py" in command - assert "--override-ini=python_classes=Test" in command - assert "--override-ini=python_functions=test" in command - assert "--override-ini=norecursedirs=" in command - - -def test_release_lane_command_contract_rejects_unowned_selectors( + assert verify._PYTEST_CLEAR_CONFIGURED_ADDOPTS in command + assert all(arg in command for arg in verify._PYTEST_CLOSED_WORLD_COLLECTION_ARGS) + assert all(arg in command for arg in verify._PYTEST_MANAGED_PLUGIN_ARGS) + assert command.count("tests") == 1 + assert "--override-ini=python_files=test_*.py *_test.py fuzz_*.py" in command + assert "--override-ini=python_classes=Test" in command + assert "--override-ini=python_functions=test" in command + assert "--override-ini=norecursedirs=" in command + + +@pytest.mark.parametrize("mode", ["affected", "bootstrap", "full"]) +def test_native_lane_command_contract_rejects_unowned_selectors( monkeypatch: pytest.MonkeyPatch, + mode: str, ) -> None: monkeypatch.setattr("devtools.verify.adaptive_pytest_worker_count", lambda _env: 2) steps = build_verify_steps( quick=False, lab=False, - testmon_mode="full", + testmon_mode=mode, testmon_environment="env-digest", ) label, command = next(step for step in steps if "parallel" in step[0]) - assert verify._release_pytest_command_is_closed_world(label, command) + assert verify._native_pytest_command_is_closed_world(label, command) for narrowed in ( [*command, "--ignore=tests/unit"], [*command, "tests/unit"], [*command, "--deselect=tests/test_failure.py::test_failure"], [*command, "--override-ini=python_files=test_owned.py"], ): - assert not verify._release_pytest_command_is_closed_world(label, narrowed) + assert not verify._native_pytest_command_is_closed_world(label, narrowed) def test_native_corpus_excludes_only_benchmark_directory() -> None: @@ -3050,8 +3049,10 @@ def apply_policy(env: dict[str, str], **_kwargs: object) -> tuple[dict[str, str] assert captured["POLYLOGUE_PYTEST_EXPLICIT_BASETEMP"] == str(explicit) -def test_release_lane_removes_environment_addopts_before_pytest( +@pytest.mark.parametrize("mode", ["affected", "bootstrap", "full"]) +def test_managed_native_lane_removes_environment_addopts_before_pytest( monkeypatch: pytest.MonkeyPatch, + mode: str, ) -> None: captured_command: list[str] = [] captured_env: dict[str, str] = {} @@ -3064,7 +3065,7 @@ def test_release_lane_removes_environment_addopts_before_pytest( for label, command in build_verify_steps( quick=False, lab=False, - testmon_mode="full", + testmon_mode=mode, testmon_environment="env-digest", ) if "parallel" in label @@ -3082,7 +3083,7 @@ def run_pytest(command: list[str], **kwargs: object) -> subprocess.CompletedProc patch("devtools.verify._read_pytest_report", return_value=None), ): rc, _elapsed, metadata = _run( - "pytest native parallel (full)", + f"pytest native parallel ({mode})", command, ) @@ -3096,6 +3097,33 @@ def run_pytest(command: list[str], **kwargs: object) -> subprocess.CompletedProc assert metadata["closed_world_collection"] is True +@pytest.mark.uses_real_clock("coordinates two contenders for the checkout lifecycle lock") +def test_native_testmon_lifecycle_lock_serializes_checkout_state(tmp_path: Path) -> None: + holder_entered = threading.Event() + release_holder = threading.Event() + contender_entered = threading.Event() + + def hold_lock() -> None: + with verify._native_testmon_lifecycle_lock(tmp_path): + holder_entered.set() + assert release_holder.wait(timeout=2) + + def contend_for_lock() -> None: + with verify._native_testmon_lifecycle_lock(tmp_path): + contender_entered.set() + + with ThreadPoolExecutor(max_workers=2) as pool: + holder = pool.submit(hold_lock) + assert holder_entered.wait(timeout=2) + contender = pool.submit(contend_for_lock) + assert not contender_entered.wait(timeout=0.05) + release_holder.set() + holder.result(timeout=2) + contender.result(timeout=2) + + assert contender_entered.is_set() + + def test_run_clears_stale_current_statistics_before_an_interrupted_pytest_step(tmp_path: Path) -> None: stale_statistics = tmp_path / verify_runs.CURRENT_STATISTICS_PATH stale_statistics.parent.mkdir(parents=True) @@ -3834,6 +3862,45 @@ def finish(self) -> CheckoutMutationObservation: assert checkout_step["final_git_head"] == "different-head" +def test_verify_keeps_checkout_monitor_active_through_final_authority_samples( + capsys: pytest.CaptureFixture[str], +) -> None: + events: list[str] = [] + + class _OrderedMonitor: + def __init__(self, _root: Path) -> None: + pass + + def start(self) -> None: + events.append("start") + + def finish(self) -> CheckoutMutationObservation: + events.append("finish") + return CheckoutMutationObservation(changed=False, unavailable=False) + + def git_head() -> str: + events.append("head") + return "stable-head" + + def fingerprint(_root: Path) -> str: + events.append("fingerprint") + return "stable-fingerprint" + + with ( + patch("devtools.verify._run", return_value=(0, 0.01, {})), + patch("devtools.verify._git_head", side_effect=git_head), + patch("devtools.verify.CheckoutMutationMonitor", _OrderedMonitor), + patch("devtools.verify.worktree_fingerprint", side_effect=fingerprint), + patch("devtools.verify._save_history"), + patch("devtools.verify._stamp_head"), + patch("devtools.verify._notify"), + ): + assert main(["--quick", "--json"]) == 0 + + assert events[-3:] == ["head", "fingerprint", "finish"] + assert json.loads(capsys.readouterr().out)["exit_code"] == 0 + + @pytest.mark.parametrize( ("fingerprints", "expected_diagnosis"), [ From 928caa55e33336659fc6b7a9d3ef69bc6c24e239 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 14 Aug 2026 04:54:08 +0200 Subject: [PATCH 19/31] fix: harden native verification finalization --- devtools/testmon_bootstrap.py | 9 ++++++ devtools/verify.py | 2 +- tests/unit/devtools/test_testmon_bootstrap.py | 29 +++++++++++++++++++ tests/unit/devtools/test_verify.py | 16 ++++++++-- 4 files changed, 53 insertions(+), 3 deletions(-) diff --git a/devtools/testmon_bootstrap.py b/devtools/testmon_bootstrap.py index 1175e14aa9..3010c4b31d 100644 --- a/devtools/testmon_bootstrap.py +++ b/devtools/testmon_bootstrap.py @@ -436,6 +436,15 @@ def inspect_native_testmon_environment( return NativeTestmonState("invalid", f"cannot inspect native testmon database: {exc}") if not stat.S_ISREG(mode): return NativeTestmonState("invalid", "native testmon database is not a regular file") + for sidecar in sidecars: + try: + sidecar_mode = sidecar.lstat().st_mode + except FileNotFoundError: + continue + except OSError as exc: + return NativeTestmonState("invalid", f"cannot inspect native testmon sidecar {sidecar}: {exc}") + if not stat.S_ISREG(sidecar_mode): + return NativeTestmonState("invalid", f"native testmon sidecar is not a regular file: {sidecar}") try: with ( contextlib.closing( diff --git a/devtools/verify.py b/devtools/verify.py index 01dcd24834..027387f729 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -2566,11 +2566,11 @@ def _finalize_preflight_failure( ) -> int: """Persist one normalized failed invocation before pytest can start.""" final_head = _git_head() - mutation_observation = finish_checkout_mutation_monitor(mutation_monitor) if mutation_monitor is not None else None try: final_worktree_fingerprint = worktree_fingerprint(ROOT) if mutation_monitor is not None else "unavailable" except Exception: final_worktree_fingerprint = "unavailable" + mutation_observation = finish_checkout_mutation_monitor(mutation_monitor) if mutation_monitor is not None else None checkout_diagnosis: str | None = None if mutation_monitor is None: checkout_diagnosis = "preflight_failed_before_checkout_monitor" diff --git a/tests/unit/devtools/test_testmon_bootstrap.py b/tests/unit/devtools/test_testmon_bootstrap.py index 3e0166b66f..31b62820ee 100644 --- a/tests/unit/devtools/test_testmon_bootstrap.py +++ b/tests/unit/devtools/test_testmon_bootstrap.py @@ -11,6 +11,7 @@ classify_native_testmon_changes, classify_source_ast, executable_python_paths, + inspect_native_testmon_environment, prepare_native_testmon_environment, remove_invalid_native_testmon_state, ) @@ -323,3 +324,31 @@ def test_native_preparation_rejects_symlinked_state_parent_before_inspection(tmp with pytest.raises(NativeTestmonRepairError, match="symlinked owned testmon parent"): prepare_native_testmon_environment(tmp_path) + + +def test_native_inspection_rejects_symlinked_database_before_sqlite_open(tmp_path: Path) -> None: + outside = tmp_path / "outside.db" + outside.write_text("external state", encoding="utf-8") + data = tmp_path / "testmondata" + data.symlink_to(outside) + + state = inspect_native_testmon_environment(data, environment_name="owned-environment") + + assert state.status == "invalid" + assert state.reason == "native testmon database is not a regular file" + assert outside.read_text(encoding="utf-8") == "external state" + + +def test_native_inspection_rejects_symlinked_sidecar_before_sqlite_open(tmp_path: Path) -> None: + data = tmp_path / "testmondata" + data.write_text("not opened", encoding="utf-8") + outside = tmp_path / "outside-wal" + outside.write_text("external sidecar", encoding="utf-8") + sidecar = Path(f"{data}-wal") + sidecar.symlink_to(outside) + + state = inspect_native_testmon_environment(data, environment_name="owned-environment") + + assert state.status == "invalid" + assert state.reason == f"native testmon sidecar is not a regular file: {sidecar}" + assert outside.read_text(encoding="utf-8") == "external sidecar" diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index e72019d543..7231d03dd6 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -4071,6 +4071,8 @@ def test_git_authority_failure_writes_history_and_invocation_receipt( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], ) -> None: + events: list[str] = [] + class _StableMonitor: def __init__(self, _root: Path) -> None: pass @@ -4079,18 +4081,27 @@ def start(self) -> None: pass def finish(self) -> CheckoutMutationObservation: + events.append("finish") return CheckoutMutationObservation(changed=False, unavailable=False) + def git_head() -> str: + events.append("head") + return "head" + + def fingerprint(_root: Path) -> str: + events.append("fingerprint") + return "stable" + history: dict[str, Any] = {} receipt = tmp_path / "invocation-receipt.json" monkeypatch.setenv(verify_runs.VERIFICATION_INVOCATION_ID_ENV, "git-authority") monkeypatch.setenv(verify_runs.VERIFICATION_RECEIPT_PATH_ENV, str(receipt)) with ( - patch("devtools.verify._git_head", return_value="head"), + patch("devtools.verify._git_head", side_effect=git_head), patch("devtools.verify._git_commit", return_value=None), patch("devtools.verify.CheckoutMutationMonitor", _StableMonitor), - patch("devtools.verify.worktree_fingerprint", return_value="stable"), + patch("devtools.verify.worktree_fingerprint", side_effect=fingerprint), patch("devtools.verify._save_history", side_effect=lambda entry: history.update(entry)), patch("devtools.verify._notify"), ): @@ -4098,6 +4109,7 @@ def finish(self) -> CheckoutMutationObservation: assert history["diagnosis"] == "native_git_authority_unavailable" assert history["final_worktree_fingerprint"] == "stable" + assert events[-3:] == ["head", "fingerprint", "finish"] assert json.loads(receipt.read_text())["diagnosis"] == "native_git_authority_unavailable" assert json.loads(capsys.readouterr().out)["diagnosis"] == "native_git_authority_unavailable" From d7aa3a9b827d2119239e83e432a172e62b24dcb7 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 14 Aug 2026 05:03:27 +0200 Subject: [PATCH 20/31] test: align native environment identity contract --- .../devtools/test_native_testmon_lifecycle.py | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/tests/integration/devtools/test_native_testmon_lifecycle.py b/tests/integration/devtools/test_native_testmon_lifecycle.py index ec633b5452..9e92d1ec6a 100644 --- a/tests/integration/devtools/test_native_testmon_lifecycle.py +++ b/tests/integration/devtools/test_native_testmon_lifecycle.py @@ -1137,7 +1137,7 @@ def test_removed_environment_or_dependency_edge_invalidates_native_state(tmp_pat assert missing.missing_executable_paths == ("app.py",) -def test_environment_and_plugin_identity_changes_start_real_fresh_bootstraps( +def test_neutralized_environment_and_declared_plugin_identity_are_owned( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -1155,30 +1155,27 @@ def test_environment_and_plugin_identity_changes_start_real_fresh_bootstraps( for result in _run_plain_verify_corpus(repo, mode="bootstrap", environment_name=initial.environment_name) ] == [0, 0] - monkeypatch.setenv("PYTEST_ADDOPTS", "-ra") - environment_changed = prepare_native_testmon_environment(repo) - assert environment_changed.selection_mode == "bootstrap" - assert environment_changed.environment_name != initial.environment_name - assert [ - result.completed.returncode - for result in _run_plain_verify_corpus( - repo, mode="bootstrap", environment_name=environment_changed.environment_name - ) - ] == [0, 0] + monkeypatch.setenv("PYTEST_ADDOPTS", "--setup-only") + monkeypatch.setenv("PYTEST_PLUGINS", "ambient_plugin") + environment_unchanged = prepare_native_testmon_environment(repo) + assert environment_unchanged.selection_mode == "affected" + assert environment_unchanged.environment_name == initial.environment_name + monkeypatch.delenv("PYTEST_ADDOPTS") + monkeypatch.delenv("PYTEST_PLUGINS") plugin = repo / "local_plugin.py" plugin.write_text( "import pytest\n\n@pytest.fixture\ndef native_identity():\n return 'v1'\n", encoding="utf-8", ) + (repo / "tests" / "conftest.py").write_text('pytest_plugins = ("local_plugin",)\n', encoding="utf-8") test_file.write_text( "def test_initial():\n assert True\n\ndef test_plugin(native_identity):\n assert native_identity == 'v1'\n", encoding="utf-8", ) - monkeypatch.setenv("PYTEST_ADDOPTS", "-p local_plugin") plugin_changed = prepare_native_testmon_environment(repo) assert plugin_changed.selection_mode == "bootstrap" - assert plugin_changed.environment_name != environment_changed.environment_name + assert plugin_changed.environment_name != environment_unchanged.environment_name plugin_results = _run_plain_verify_corpus(repo, mode="bootstrap", environment_name=plugin_changed.environment_name) assert [result.completed.returncode for result in plugin_results] == [0, 0] assert "tests/test_identity.py::test_plugin" in _selected(*plugin_results) From d342f689283d479c84de73bf87914d86396f3e18 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 14 Aug 2026 05:11:44 +0200 Subject: [PATCH 21/31] fix: preserve authority on verifier exceptions --- devtools/verify.py | 110 +++++++++++++++++++++++------ devtools/verify_runs.py | 2 + tests/unit/devtools/test_verify.py | 36 ++++++++-- 3 files changed, 121 insertions(+), 27 deletions(-) diff --git a/devtools/verify.py b/devtools/verify.py index 027387f729..f3637b2a7e 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -37,6 +37,7 @@ import time import uuid from collections.abc import Iterator, Mapping, Sequence +from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path from typing import Any @@ -80,6 +81,7 @@ PYTEST_EXPLICIT_BASETEMP_ENV, VERIFY_HISTORY_PATH, CheckoutMutationMonitor, + CheckoutMutationObservation, PytestResourceError, PytestStepArtifacts, ResourceSampler, @@ -2485,7 +2487,31 @@ def _changed_test_relevant_paths(base_commit: str, head_commit: str) -> tuple[st ) -_ACTIVE_VERIFY_RUN: tuple[VerifyRun, float, VerificationScope] | None = None +@dataclass(slots=True) +class _ActiveVerifyRun: + run: VerifyRun + started_at: float + verification_scope: VerificationScope + head: str | None + mutation_monitor: CheckoutMutationMonitor | None = None + initial_worktree_fingerprint: str | None = None + + +_ACTIVE_VERIFY_RUN: _ActiveVerifyRun | None = None + + +def _start_active_checkout_mutation_monitor(monitor: CheckoutMutationMonitor) -> None: + start_checkout_mutation_monitor(monitor) + if _ACTIVE_VERIFY_RUN is not None: + _ACTIVE_VERIFY_RUN.mutation_monitor = monitor + + +def _finish_active_checkout_mutation_monitor(monitor: CheckoutMutationMonitor) -> CheckoutMutationObservation: + try: + return finish_checkout_mutation_monitor(monitor) + finally: + if _ACTIVE_VERIFY_RUN is not None and _ACTIVE_VERIFY_RUN.mutation_monitor is monitor: + _ACTIVE_VERIFY_RUN.mutation_monitor = None def _planned_verification_scope( @@ -2570,7 +2596,9 @@ def _finalize_preflight_failure( final_worktree_fingerprint = worktree_fingerprint(ROOT) if mutation_monitor is not None else "unavailable" except Exception: final_worktree_fingerprint = "unavailable" - mutation_observation = finish_checkout_mutation_monitor(mutation_monitor) if mutation_monitor is not None else None + mutation_observation = ( + _finish_active_checkout_mutation_monitor(mutation_monitor) if mutation_monitor is not None else None + ) checkout_diagnosis: str | None = None if mutation_monitor is None: checkout_diagnosis = "preflight_failed_before_checkout_monitor" @@ -2606,6 +2634,7 @@ def _finalize_preflight_failure( diagnosis=diagnosis, verification_scope=verification_scope.value, release_baseline_allowed=False, + final_git_head=final_head, final_worktree_fingerprint=final_worktree_fingerprint, checkout_mutation_path=(mutation_observation.observed_path if mutation_observation is not None else None), checkout_diagnosis=checkout_diagnosis, @@ -2697,7 +2726,12 @@ def _main(argv: list[str] | None = None) -> int: argv=list(sys.argv[1:] if argv is None else argv), git_head=head, ) - _ACTIVE_VERIFY_RUN = (verify_run, started_at, planned_scope) + _ACTIVE_VERIFY_RUN = _ActiveVerifyRun( + run=verify_run, + started_at=started_at, + verification_scope=planned_scope, + head=head, + ) try: fingerprint = assert_polylogue_matches_checkout(ROOT, context="devtools verify") @@ -2722,11 +2756,13 @@ def _main(argv: list[str] | None = None) -> int: sys.stderr.write(f"verify: polylogue package → {polylogue_import_path}\n") mutation_monitor = CheckoutMutationMonitor(ROOT) - start_checkout_mutation_monitor(mutation_monitor) + _start_active_checkout_mutation_monitor(mutation_monitor) checkout_fingerprint = worktree_fingerprint(ROOT) - finish_checkout_mutation_monitor(mutation_monitor) + _finish_active_checkout_mutation_monitor(mutation_monitor) mutation_monitor = CheckoutMutationMonitor(ROOT) - start_checkout_mutation_monitor(mutation_monitor) + _start_active_checkout_mutation_monitor(mutation_monitor) + assert _ACTIVE_VERIFY_RUN is not None + _ACTIVE_VERIFY_RUN.initial_worktree_fingerprint = checkout_fingerprint verify_run.update_checkout_provenance(worktree_fingerprint=checkout_fingerprint) base_commit = _git_commit("origin/master") if pytest_enabled else None @@ -2815,7 +2851,8 @@ def _main(argv: list[str] | None = None) -> int: ) planned_scope = _planned_verification_scope(args, testmon_mode=testmon_mode) - _ACTIVE_VERIFY_RUN = (verify_run, started_at, planned_scope) + assert _ACTIVE_VERIFY_RUN is not None + _ACTIVE_VERIFY_RUN.verification_scope = planned_scope if not use_json: sys.stderr.write("verify: running local verification baseline\n") @@ -2848,9 +2885,9 @@ def _main(argv: list[str] | None = None) -> int: # Git probes and native testmon preparation can refresh the index as part # of their own read path. Discard that preflight interval and begin the # authority interval immediately before the verification steps. - finish_checkout_mutation_monitor(mutation_monitor) + _finish_active_checkout_mutation_monitor(mutation_monitor) mutation_monitor = CheckoutMutationMonitor(ROOT) - start_checkout_mutation_monitor(mutation_monitor) + _start_active_checkout_mutation_monitor(mutation_monitor) step_results: list[dict[str, Any]] = [] exit_code = 0 native_graph_touched = False @@ -2891,7 +2928,7 @@ def _main(argv: list[str] | None = None) -> int: assert mutation_monitor is not None final_head = _git_head() final_checkout_fingerprint = worktree_fingerprint(ROOT) - mutation_observation = finish_checkout_mutation_monitor(mutation_monitor) + mutation_observation = _finish_active_checkout_mutation_monitor(mutation_monitor) checkout_stable = True checkout_fingerprint_unavailable = ( head is None @@ -3084,6 +3121,7 @@ def _main(argv: list[str] | None = None) -> int: diagnosis=run_diagnosis, verification_scope=verification_scope.value, release_baseline_allowed=release_baseline_allowed, + final_git_head=final_head, final_worktree_fingerprint=final_checkout_fingerprint, checkout_mutation_path=mutation_observation.observed_path, checkout_diagnosis=checkout_diagnosis, @@ -3121,31 +3159,60 @@ def _main(argv: list[str] | None = None) -> int: def _finalize_verify_runner_exception( - run: VerifyRun, + active: _ActiveVerifyRun, exc: Exception, *, - run_started: float, - verification_scope: VerificationScope, use_json: bool, ) -> int: """Leave typed, durable failed evidence when verification orchestration raises.""" diagnosis = "verify_runner_exception" + run = active.run + try: + final_head = _git_head() + except Exception: + final_head = None + try: + final_worktree_fingerprint = worktree_fingerprint(ROOT) + except Exception: + final_worktree_fingerprint = "unavailable" + mutation_observation = None + if active.mutation_monitor is not None: + try: + mutation_observation = _finish_active_checkout_mutation_monitor(active.mutation_monitor) + except Exception: + mutation_observation = None run.finish_interrupted_steps( exit_code=125, diagnosis=diagnosis, termination_reason="runner_exception", ) - try: - final_worktree_fingerprint = worktree_fingerprint(ROOT) - except Exception: - final_worktree_fingerprint = "unavailable" + if ( + active.head is None + or final_head is None + or active.initial_worktree_fingerprint in {None, "unavailable"} + or final_worktree_fingerprint == "unavailable" + or mutation_observation is None + or mutation_observation.unavailable + ): + checkout_diagnosis = "checkout_fingerprint_unavailable" + elif ( + final_head != active.head + or mutation_observation.changed + or final_worktree_fingerprint != active.initial_worktree_fingerprint + ): + checkout_diagnosis = "checkout_changed_during_verification" + else: + checkout_diagnosis = None payload = run.finish( exit_code=125, - duration_s=time.monotonic() - run_started, + duration_s=time.monotonic() - active.started_at, diagnosis=diagnosis, - verification_scope=verification_scope.value, + verification_scope=active.verification_scope.value, release_baseline_allowed=False, + final_git_head=final_head, final_worktree_fingerprint=final_worktree_fingerprint, + checkout_mutation_path=(mutation_observation.observed_path if mutation_observation is not None else None), + checkout_diagnosis=checkout_diagnosis, invocation_budget_s=VERIFY_INVOCATION_BUDGET_S, ) payload["exception_type"] = type(exc).__name__ @@ -3170,12 +3237,9 @@ def main(argv: list[str] | None = None) -> int: except Exception as exc: if _ACTIVE_VERIFY_RUN is None: raise - run, run_started, verification_scope = _ACTIVE_VERIFY_RUN return _finalize_verify_runner_exception( - run, + _ACTIVE_VERIFY_RUN, exc, - run_started=run_started, - verification_scope=verification_scope, use_json="--json" in raw_argv, ) finally: diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index b33f432bc3..f386b97bae 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -1646,6 +1646,7 @@ def finish( diagnosis: str | None = None, verification_scope: str | None = None, release_baseline_allowed: bool | None = None, + final_git_head: str | None = None, final_worktree_fingerprint: str | None = None, checkout_mutation_path: str | None = None, checkout_diagnosis: str | None = None, @@ -1658,6 +1659,7 @@ def finish( self._payload["status"] = "success" if exit_code == 0 else "failed" if diagnosis: self._payload["diagnosis"] = diagnosis + self._payload["final_git_head"] = final_git_head if final_worktree_fingerprint is not None: self._payload["final_worktree_fingerprint"] = final_worktree_fingerprint if checkout_mutation_path is not None: diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 7231d03dd6..ae1d2a2e8c 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -4008,6 +4008,22 @@ def test_verify_finalizes_checkout_monitor_when_startup_fingerprint_raises( ) -> None: events: list[str] = [] history: dict[str, Any] = {} + head_calls = iter(("initial-head", "final-head")) + fingerprint_calls = 0 + + def _git_head() -> str: + head = next(head_calls) + events.append(head) + return head + + def _worktree_fingerprint(_root: Path) -> str: + nonlocal fingerprint_calls + fingerprint_calls += 1 + if fingerprint_calls == 1: + events.append("startup-fingerprint") + raise RuntimeError("fingerprint failed") + events.append("final-fingerprint") + return "final-fingerprint" class _ExceptionalExitMonitor: def __init__(self, _root: Path) -> None: @@ -4022,16 +4038,28 @@ def finish(self) -> CheckoutMutationObservation: with ( patch("devtools.verify.CheckoutMutationMonitor", _ExceptionalExitMonitor), - patch("devtools.verify._git_head", return_value="head"), - patch("devtools.verify.worktree_fingerprint", side_effect=RuntimeError("fingerprint failed")), + patch("devtools.verify._git_head", side_effect=_git_head), + patch("devtools.verify.worktree_fingerprint", side_effect=_worktree_fingerprint), patch("devtools.verify._save_history", side_effect=lambda entry: history.update(entry)), patch("devtools.verify._notify"), ): assert main(["--quick", "--json"]) == 125 - assert events == ["monitor-started", "monitor-finished"] + assert events == [ + "initial-head", + "monitor-started", + "startup-fingerprint", + "final-head", + "final-fingerprint", + "monitor-finished", + ] assert history["diagnosis"] == "verify_runner_exception" - assert json.loads(capsys.readouterr().out)["diagnosis"] == "verify_runner_exception" + assert history["final_git_head"] == "final-head" + assert history["final_worktree_fingerprint"] == "final-fingerprint" + payload = json.loads(capsys.readouterr().out) + assert payload["diagnosis"] == "verify_runner_exception" + assert payload["final_git_head"] == "final-head" + assert payload["final_worktree_fingerprint"] == "final-fingerprint" def test_import_guard_failure_writes_normalized_history_and_invocation_receipt( From a2d014dca3ad3cedcec8a66c4ecf0df98d3e7588 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 14 Aug 2026 05:24:29 +0200 Subject: [PATCH 22/31] fix: close native verifier edge paths --- devtools/testmon_bootstrap.py | 52 +++++++++++++-- devtools/verify.py | 27 ++++++-- .../devtools/test_native_testmon_lifecycle.py | 65 ++++++++++++++++++- tests/unit/devtools/test_testmon_bootstrap.py | 19 ++++++ tests/unit/devtools/test_verify.py | 18 +++-- 5 files changed, 163 insertions(+), 18 deletions(-) diff --git a/devtools/testmon_bootstrap.py b/devtools/testmon_bootstrap.py index 3010c4b31d..71a981ed5d 100644 --- a/devtools/testmon_bootstrap.py +++ b/devtools/testmon_bootstrap.py @@ -177,16 +177,42 @@ def _declared_pytest_plugin_names(root: Path) -> set[str]: tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) except (OSError, SyntaxError, UnicodeDecodeError): continue + declaration_count = 0 for node in tree.body: value = _pytest_plugins_assignment(node) if value is None: + dynamic_reference = any( + isinstance(child, ast.Name) and child.id == "pytest_plugins" for child in ast.walk(node) + ) or any( + alias.name == "pytest_plugins" or alias.asname == "pytest_plugins" + for child in ast.walk(node) + if isinstance(child, ast.Import | ast.ImportFrom) + for alias in child.names + ) + if dynamic_reference: + raise NativeTestmonRepairError( + f"repository pytest_plugins declaration must be one literal assignment: {path}" + ) continue - with contextlib.suppress(ValueError, TypeError): + declaration_count += 1 + if declaration_count != 1: + raise NativeTestmonRepairError( + f"repository pytest_plugins declaration must be one literal assignment: {path}" + ) + try: declared = ast.literal_eval(value) - if isinstance(declared, str): - names.add(declared) - elif isinstance(declared, tuple | list): - names.update(name for name in declared if isinstance(name, str)) + except (ValueError, TypeError) as exc: + raise NativeTestmonRepairError( + f"repository pytest_plugins declaration must be a literal string/list/tuple: {path}" + ) from exc + if isinstance(declared, str): + names.add(declared) + elif isinstance(declared, tuple | list) and all(isinstance(name, str) for name in declared): + names.update(declared) + else: + raise NativeTestmonRepairError( + f"repository pytest_plugins declaration must contain only literal plugin names: {path}" + ) return names @@ -562,6 +588,19 @@ def _validate_owned_state_parents(repo_root: Path) -> None: raise NativeTestmonRepairError(f"owned testmon parent is not a directory: {parent}") +def validate_native_testmon_state_ownership(repo_root: Path) -> None: + """Reject parent or file replacement before managed SQLite access.""" + for path in _owned_paths(repo_root): + try: + mode = path.lstat().st_mode + except FileNotFoundError: + continue + except OSError as exc: + raise NativeTestmonRepairError(f"cannot inspect owned testmon path {path}: {exc}") from exc + if not stat.S_ISREG(mode): + raise NativeTestmonRepairError(f"owned testmon path is not a regular file: {path}") + + def remove_invalid_native_testmon_state(repo_root: Path) -> tuple[Path, ...]: """Remove only the exact checkout-owned SQLite file and known sidecars.""" removed: list[Path] = [] @@ -697,6 +736,8 @@ def prepare_native_testmon_environment( deadline_monotonic=deadline_monotonic, ) local_data = root / TESTMON_DATA_RELPATH + local_data.parent.mkdir(parents=True, exist_ok=True) + _validate_owned_state_parents(root) local = inspect_native_testmon_environment( local_data, environment_name=environment_name, @@ -778,4 +819,5 @@ def prepare_native_testmon_environment( "prepare_native_testmon_environment", "remove_invalid_native_testmon_state", "testmon_environment_digest", + "validate_native_testmon_state_ownership", ] diff --git a/devtools/verify.py b/devtools/verify.py index f3637b2a7e..2ec001866e 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -69,6 +69,7 @@ inspect_native_testmon_environment, prepare_native_testmon_environment, remove_invalid_native_testmon_state, + validate_native_testmon_state_ownership, ) from devtools.verification_contracts import VerificationScope from devtools.verify_runs import ( @@ -1701,6 +1702,8 @@ def _run( cmd = [f"--json-report-file={isolated_report}" if arg.startswith("--json-report-file=") else arg for arg in cmd] if is_pytest: _clear_pytest_report(cmd) + if managed_native_lane: + validate_native_testmon_state_ownership(ROOT) artifacts = run.start_step(label=label, cmd=cmd) if run is not None else None env = _subprocess_env() external_addopts_neutralized = False @@ -2149,7 +2152,6 @@ def _subprocess_env() -> dict[str, str]: inherited_pythonpath = env.get("PYTHONPATH", "") env["PYTHONPATH"] = str(ROOT) if not inherited_pythonpath else f"{ROOT}{os.pathsep}{inherited_pythonpath}" env["PYTHONPYCACHEPREFIX"] = str(ROOT / ".cache" / "pycache") - TESTMON_DATA.parent.mkdir(parents=True, exist_ok=True) env["TESTMON_DATAFILE"] = str(TESTMON_DATA) env["POLYLOGUE_PYTEST_EVENTS_DIR"] = str(ROOT / PYTEST_EVENTS_DIR) env["POLYLOGUE_PYTEST_EVENTS_PATH"] = str(ROOT / PYTEST_EVENTS_PATH) @@ -2539,6 +2541,7 @@ def _native_environment_after_run( *, required_executable_paths: Sequence[str], ) -> NativeTestmonState: + validate_native_testmon_state_ownership(ROOT) return inspect_native_testmon_environment( TESTMON_DATA, environment_name=preparation.environment_name, @@ -3160,12 +3163,14 @@ def _main(argv: list[str] | None = None) -> int: def _finalize_verify_runner_exception( active: _ActiveVerifyRun, - exc: Exception, + exc: BaseException, *, use_json: bool, ) -> int: """Leave typed, durable failed evidence when verification orchestration raises.""" - diagnosis = "verify_runner_exception" + interrupted = isinstance(exc, KeyboardInterrupt) + diagnosis = "verify_interrupted" if interrupted else "verify_runner_exception" + exit_code = 130 if interrupted else 125 run = active.run try: final_head = _git_head() @@ -3182,9 +3187,9 @@ def _finalize_verify_runner_exception( except Exception: mutation_observation = None run.finish_interrupted_steps( - exit_code=125, + exit_code=exit_code, diagnosis=diagnosis, - termination_reason="runner_exception", + termination_reason="operator_interrupt" if interrupted else "runner_exception", ) if ( active.head is None @@ -3204,7 +3209,7 @@ def _finalize_verify_runner_exception( else: checkout_diagnosis = None payload = run.finish( - exit_code=125, + exit_code=exit_code, duration_s=time.monotonic() - active.started_at, diagnosis=diagnosis, verification_scope=active.verification_scope.value, @@ -3221,7 +3226,7 @@ def _finalize_verify_runner_exception( if use_json: _print_json(payload) sys.stderr.write(f"verify: unexpected runner exception: {exc}\n") - return 125 + return exit_code @finalize_checkout_mutation_monitors @@ -3234,6 +3239,14 @@ def main(argv: list[str] | None = None) -> int: with lock: try: return _main(argv) + except KeyboardInterrupt as exc: + if _ACTIVE_VERIFY_RUN is None: + raise + return _finalize_verify_runner_exception( + _ACTIVE_VERIFY_RUN, + exc, + use_json="--json" in raw_argv, + ) except Exception as exc: if _ACTIVE_VERIFY_RUN is None: raise diff --git a/tests/integration/devtools/test_native_testmon_lifecycle.py b/tests/integration/devtools/test_native_testmon_lifecycle.py index 9e92d1ec6a..5de91e85d3 100644 --- a/tests/integration/devtools/test_native_testmon_lifecycle.py +++ b/tests/integration/devtools/test_native_testmon_lifecycle.py @@ -14,8 +14,10 @@ import pytest +import devtools.verify as verify from devtools.testmon_bootstrap import ( TESTMON_DATA_RELPATH, + NativeTestmonRepairError, inspect_native_testmon_environment, prepare_native_testmon_environment, ) @@ -166,7 +168,11 @@ def _run_plain_verify_corpus( return parallel, serial -def _run_production_verify(repo: Path, *args: str) -> tuple[subprocess.CompletedProcess[str], dict[str, Any]]: +def _run_production_verify( + repo: Path, + *args: str, + allow_rejection: bool = False, +) -> tuple[subprocess.CompletedProcess[str], dict[str, Any]]: """Run the production verifier orchestration against a tiny fixture corpus. The subprocess keeps the real native preparation, two-lane runner, @@ -253,7 +259,7 @@ def fixture_env_for_pytest_step(env, **kwargs): f"production verify wrote no invocation receipt\nstdout:\n{completed.stdout}\nstderr:\n{completed.stderr}" ) payload = json.loads(completed.stdout) - if completed.returncode == 125: + if completed.returncode == 125 and not allow_rejection: pytest.fail( f"production verify rejected the fixture checkout\npayload:\n{completed.stdout}\nstderr:\n{completed.stderr}" ) @@ -1194,6 +1200,61 @@ def test_neutralized_environment_and_declared_plugin_identity_are_owned( ] == [0, 0] +def test_production_verify_fails_closed_on_dynamic_pytest_plugins(tmp_path: Path) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _init_repo(repo) + (repo / "plugin_config.py").write_text('plugin_names = ("local_plugin",)\n', encoding="utf-8") + (repo / "local_plugin.py").write_text("VALUE = 1\n", encoding="utf-8") + (repo / "tests" / "conftest.py").write_text( + "from plugin_config import plugin_names\n\npytest_plugins = plugin_names\n", + encoding="utf-8", + ) + (repo / "tests" / "test_body.py").write_text("def test_body():\n assert True\n", encoding="utf-8") + _commit_all(repo, "fixture") + origin = tmp_path / "origin.git" + _git(origin.parent, "init", "--bare", "-q", str(origin)) + _git(repo, "remote", "add", "origin", str(origin)) + _git(repo, "branch", "-M", "master") + _git(repo, "push", "-qu", "origin", "master") + + completed, payload = _run_production_verify(repo, allow_rejection=True) + + assert completed.returncode == 125 + assert payload["diagnosis"] == "native_testmon_preparation_failed" + assert payload["release_baseline_allowed"] is False + assert "pytest_plugins declaration must be a literal" in completed.stderr + + +def test_managed_native_routes_reject_replaced_cache_parent( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _init_repo(repo) + (repo / "tests" / "test_body.py").write_text("def test_body():\n assert True\n", encoding="utf-8") + _commit_all(repo, "fixture") + preparation = prepare_native_testmon_environment(repo) + owned_cache = repo / ".cache" + owned_cache.rename(repo / ".cache-owned") + external_cache = tmp_path / "external-cache" + external_cache.mkdir() + sentinel = external_cache / "sentinel" + sentinel.write_text("external", encoding="utf-8") + owned_cache.symlink_to(external_cache, target_is_directory=True) + monkeypatch.setattr(verify, "ROOT", repo) + monkeypatch.setattr(verify, "TESTMON_DATA", repo / TESTMON_DATA_RELPATH) + + with pytest.raises(NativeTestmonRepairError, match="refusing symlinked owned testmon parent"): + verify._run("pytest native parallel (affected)", ["pytest"]) + with pytest.raises(NativeTestmonRepairError, match="refusing symlinked owned testmon parent"): + verify._native_environment_after_run(preparation, required_executable_paths=()) + + assert sentinel.read_text(encoding="utf-8") == "external" + assert list(external_cache.iterdir()) == [sentinel] + + def test_runtime_helper_mutation_stays_incremental_and_selects_owner(tmp_path: Path) -> None: repo = tmp_path / "repo" repo.mkdir() diff --git a/tests/unit/devtools/test_testmon_bootstrap.py b/tests/unit/devtools/test_testmon_bootstrap.py index 31b62820ee..6d37878e49 100644 --- a/tests/unit/devtools/test_testmon_bootstrap.py +++ b/tests/unit/devtools/test_testmon_bootstrap.py @@ -228,6 +228,25 @@ def test_declared_local_fixture_plugin_changes_environment(tmp_path: Path) -> No assert _testmon_environment_digest(tmp_path) != initial +def test_dynamic_pytest_plugin_declaration_fails_closed(tmp_path: Path) -> None: + conftest = tmp_path / "tests" / "conftest.py" + conftest.parent.mkdir(parents=True) + conftest.write_text( + "from plugin_config import plugin_names\n\npytest_plugins = plugin_names\n", + encoding="utf-8", + ) + + with pytest.raises(NativeTestmonRepairError, match="must be a literal string/list/tuple"): + _testmon_environment_digest(tmp_path) + + conftest.write_text( + 'pytest_plugins = []\npytest_plugins.append("tests.infra.fixture_plugin")\n', + encoding="utf-8", + ) + with pytest.raises(NativeTestmonRepairError, match="must be one literal assignment"): + _testmon_environment_digest(tmp_path) + + def test_environment_digest_ignores_neutralized_pytest_plugins( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index ae1d2a2e8c..3ba1547735 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -4003,8 +4003,18 @@ def finish(self) -> CheckoutMutationObservation: assert durable_payload["final_worktree_fingerprint"] == "stable" +@pytest.mark.parametrize( + ("raised", "expected_exit", "expected_diagnosis"), + [ + (RuntimeError("fingerprint failed"), 125, "verify_runner_exception"), + (KeyboardInterrupt(), 130, "verify_interrupted"), + ], +) def test_verify_finalizes_checkout_monitor_when_startup_fingerprint_raises( capsys: pytest.CaptureFixture[str], + raised: BaseException, + expected_exit: int, + expected_diagnosis: str, ) -> None: events: list[str] = [] history: dict[str, Any] = {} @@ -4021,7 +4031,7 @@ def _worktree_fingerprint(_root: Path) -> str: fingerprint_calls += 1 if fingerprint_calls == 1: events.append("startup-fingerprint") - raise RuntimeError("fingerprint failed") + raise raised events.append("final-fingerprint") return "final-fingerprint" @@ -4043,7 +4053,7 @@ def finish(self) -> CheckoutMutationObservation: patch("devtools.verify._save_history", side_effect=lambda entry: history.update(entry)), patch("devtools.verify._notify"), ): - assert main(["--quick", "--json"]) == 125 + assert main(["--quick", "--json"]) == expected_exit assert events == [ "initial-head", @@ -4053,11 +4063,11 @@ def finish(self) -> CheckoutMutationObservation: "final-fingerprint", "monitor-finished", ] - assert history["diagnosis"] == "verify_runner_exception" + assert history["diagnosis"] == expected_diagnosis assert history["final_git_head"] == "final-head" assert history["final_worktree_fingerprint"] == "final-fingerprint" payload = json.loads(capsys.readouterr().out) - assert payload["diagnosis"] == "verify_runner_exception" + assert payload["diagnosis"] == expected_diagnosis assert payload["final_git_head"] == "final-head" assert payload["final_worktree_fingerprint"] == "final-fingerprint" From faadc881b93319fb7907f62e42bda435f950e5ee Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 14 Aug 2026 05:37:34 +0200 Subject: [PATCH 23/31] fix: bind native testmon state ownership --- devtools/testmon_bootstrap.py | 40 +++++-- devtools/verify.py | 105 ++++++++++++++++-- .../devtools/test_native_testmon_lifecycle.py | 51 ++++++++- tests/unit/devtools/test_testmon_bootstrap.py | 7 ++ 4 files changed, 178 insertions(+), 25 deletions(-) diff --git a/devtools/testmon_bootstrap.py b/devtools/testmon_bootstrap.py index 71a981ed5d..27b1173e21 100644 --- a/devtools/testmon_bootstrap.py +++ b/devtools/testmon_bootstrap.py @@ -159,6 +159,28 @@ def _pytest_plugins_assignment(node: ast.stmt) -> ast.expr | None: return None +def _indirect_pytest_plugins_declaration(node: ast.stmt) -> bool: + for child in ast.walk(node): + if ( + isinstance(child, ast.Subscript) + and isinstance(child.slice, ast.Constant) + and child.slice.value == "pytest_plugins" + ): + return True + if isinstance(child, ast.Call): + if ( + isinstance(child.func, ast.Name) + and child.func.id == "setattr" + and len(child.args) >= 2 + and isinstance(child.args[1], ast.Constant) + and child.args[1].value == "pytest_plugins" + ): + return True + if any(keyword.arg == "pytest_plugins" for keyword in child.keywords): + return True + return False + + def _declared_pytest_plugin_names(root: Path) -> set[str]: """Read static local plugin declarations that pytest loads at collection.""" names: set[str] = set() @@ -181,13 +203,15 @@ def _declared_pytest_plugin_names(root: Path) -> set[str]: for node in tree.body: value = _pytest_plugins_assignment(node) if value is None: - dynamic_reference = any( - isinstance(child, ast.Name) and child.id == "pytest_plugins" for child in ast.walk(node) - ) or any( - alias.name == "pytest_plugins" or alias.asname == "pytest_plugins" - for child in ast.walk(node) - if isinstance(child, ast.Import | ast.ImportFrom) - for alias in child.names + dynamic_reference = ( + any(isinstance(child, ast.Name) and child.id == "pytest_plugins" for child in ast.walk(node)) + or any( + alias.name == "pytest_plugins" or alias.asname == "pytest_plugins" + for child in ast.walk(node) + if isinstance(child, ast.Import | ast.ImportFrom) + for alias in child.names + ) + or _indirect_pytest_plugins_declaration(node) ) if dynamic_reference: raise NativeTestmonRepairError( @@ -427,7 +451,7 @@ def classify_native_testmon_changes(repo_root: Path, paths: Iterable[str]) -> Na def _readonly_uri(path: Path) -> str: - return f"{path.resolve().as_uri()}?mode=ro" + return f"{path.absolute().as_uri()}?mode=ro" def _testmon_schema_version() -> int: diff --git a/devtools/verify.py b/devtools/verify.py index 2ec001866e..d94cc34320 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -154,6 +154,41 @@ def _normalize_managed_pytest_environment(env: dict[str, str]) -> None: env["PYTEST_DISABLE_PLUGIN_AUTOLOAD"] = "1" +@dataclass(slots=True) +class _OwnedNativeTestmonState: + descriptor: int + data_path: Path + + def close(self) -> None: + os.close(self.descriptor) + + +def _open_owned_native_testmon_state(repo_root: Path) -> _OwnedNativeTestmonState: + """Bind managed SQLite access to one no-follow checkout directory.""" + validate_native_testmon_state_ownership(repo_root) + raw_data = TESTMON_DATA if TESTMON_DATA.is_absolute() else repo_root.resolve() / TESTMON_DATA + parent = raw_data.parent + parent.mkdir(parents=True, exist_ok=True) + validate_native_testmon_state_ownership(repo_root) + flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0) + descriptor: int | None = None + try: + descriptor = os.open(parent, flags) + opened = os.fstat(descriptor) + current = parent.lstat() + except OSError as exc: + if descriptor is not None: + with contextlib.suppress(OSError): + os.close(descriptor) + raise NativeTestmonRepairError(f"cannot bind owned testmon directory {parent}: {exc}") from exc + assert descriptor is not None + if not stat.S_ISDIR(opened.st_mode) or (opened.st_dev, opened.st_ino) != (current.st_dev, current.st_ino): + os.close(descriptor) + raise NativeTestmonRepairError(f"owned testmon directory changed while binding: {parent}") + bound = Path(f"/proc/{os.getpid()}/fd/{descriptor}") / raw_data.name + return _OwnedNativeTestmonState(descriptor=descriptor, data_path=bound) + + @contextlib.contextmanager def _native_testmon_lifecycle_lock(repo_root: Path) -> Iterator[None]: """Serialize one checkout's native testmon preparation, lanes, and inspection.""" @@ -1687,6 +1722,35 @@ def _run( cwd: str | None = None, run: VerifyRun | None = None, timeout_s: float | None = None, +) -> tuple[int, float, dict[str, Any]]: + if not label.startswith("pytest native"): + return _run_step(label, cmd, cwd=cwd, run=run, timeout_s=timeout_s) + state = _ACTIVE_VERIFY_RUN.owned_native_testmon_state if _ACTIVE_VERIFY_RUN is not None else None + temporary_state = state is None + if state is None: + state = _open_owned_native_testmon_state(ROOT) + try: + return _run_step( + label, + cmd, + cwd=cwd, + run=run, + timeout_s=timeout_s, + native_testmon_data=state.data_path, + ) + finally: + if temporary_state: + state.close() + + +def _run_step( + label: str, + cmd: list[str], + *, + cwd: str | None = None, + run: VerifyRun | None = None, + timeout_s: float | None = None, + native_testmon_data: Path | None = None, ) -> tuple[int, float, dict[str, Any]]: t0 = time.monotonic() sys.stderr.write(f" {label} ... ") @@ -1702,10 +1766,8 @@ def _run( cmd = [f"--json-report-file={isolated_report}" if arg.startswith("--json-report-file=") else arg for arg in cmd] if is_pytest: _clear_pytest_report(cmd) - if managed_native_lane: - validate_native_testmon_state_ownership(ROOT) artifacts = run.start_step(label=label, cmd=cmd) if run is not None else None - env = _subprocess_env() + env = _subprocess_env(native_testmon_data=native_testmon_data) external_addopts_neutralized = False external_plugins_neutralized = False if managed_native_lane: @@ -2142,7 +2204,7 @@ def _pytest_command_basetemp( return (Path(cwd) if cwd is not None else Path.cwd()) / path -def _subprocess_env() -> dict[str, str]: +def _subprocess_env(*, native_testmon_data: Path | None = None) -> dict[str, str]: env = normalize_pytest_basetemp_env(os.environ) # Tests and verification helpers may inspect Git, but observational reads # must not refresh the index and invalidate the exact-head mutation watch. @@ -2152,7 +2214,7 @@ def _subprocess_env() -> dict[str, str]: inherited_pythonpath = env.get("PYTHONPATH", "") env["PYTHONPATH"] = str(ROOT) if not inherited_pythonpath else f"{ROOT}{os.pathsep}{inherited_pythonpath}" env["PYTHONPYCACHEPREFIX"] = str(ROOT / ".cache" / "pycache") - env["TESTMON_DATAFILE"] = str(TESTMON_DATA) + env["TESTMON_DATAFILE"] = str(native_testmon_data or TESTMON_DATA) env["POLYLOGUE_PYTEST_EVENTS_DIR"] = str(ROOT / PYTEST_EVENTS_DIR) env["POLYLOGUE_PYTEST_EVENTS_PATH"] = str(ROOT / PYTEST_EVENTS_PATH) env["POLYLOGUE_PYTEST_SELECTION_PATH"] = str(ROOT / PYTEST_SELECTION_PATH) @@ -2497,6 +2559,7 @@ class _ActiveVerifyRun: head: str | None mutation_monitor: CheckoutMutationMonitor | None = None initial_worktree_fingerprint: str | None = None + owned_native_testmon_state: _OwnedNativeTestmonState | None = None _ACTIVE_VERIFY_RUN: _ActiveVerifyRun | None = None @@ -2516,6 +2579,14 @@ def _finish_active_checkout_mutation_monitor(monitor: CheckoutMutationMonitor) - _ACTIVE_VERIFY_RUN.mutation_monitor = None +def _close_active_native_testmon_state() -> None: + if _ACTIVE_VERIFY_RUN is None or _ACTIVE_VERIFY_RUN.owned_native_testmon_state is None: + return + state = _ACTIVE_VERIFY_RUN.owned_native_testmon_state + _ACTIVE_VERIFY_RUN.owned_native_testmon_state = None + state.close() + + def _planned_verification_scope( args: argparse.Namespace, *, @@ -2541,12 +2612,19 @@ def _native_environment_after_run( *, required_executable_paths: Sequence[str], ) -> NativeTestmonState: - validate_native_testmon_state_ownership(ROOT) - return inspect_native_testmon_environment( - TESTMON_DATA, - environment_name=preparation.environment_name, - required_executable_paths=required_executable_paths, - ) + state = _ACTIVE_VERIFY_RUN.owned_native_testmon_state if _ACTIVE_VERIFY_RUN is not None else None + temporary_state = state is None + if state is None: + state = _open_owned_native_testmon_state(ROOT) + try: + return inspect_native_testmon_environment( + state.data_path, + environment_name=preparation.environment_name, + required_executable_paths=required_executable_paths, + ) + finally: + if temporary_state: + state.close() def _release_baseline_allowed( @@ -2807,6 +2885,8 @@ def _main(argv: list[str] | None = None) -> int: pytest_profile=_pytest_profile(), deadline_monotonic=started_at + VERIFY_INVOCATION_BUDGET_S, ) + assert _ACTIVE_VERIFY_RUN is not None + _ACTIVE_VERIFY_RUN.owned_native_testmon_state = _open_owned_native_testmon_state(ROOT) except NativeTestmonDeadlineError as exc: return _finalize_preflight_failure( verify_run, @@ -3002,6 +3082,7 @@ def _main(argv: list[str] | None = None) -> int: + ", ".join(native_state.missing_executable_paths) + "\n" ) + _close_active_native_testmon_state() total_duration = round(time.monotonic() - started_at, 2) deadline_recorded = any(step.get("diagnosis") == "verify_invocation_deadline_exceeded" for step in step_results) @@ -3186,6 +3267,7 @@ def _finalize_verify_runner_exception( mutation_observation = _finish_active_checkout_mutation_monitor(active.mutation_monitor) except Exception: mutation_observation = None + _close_active_native_testmon_state() run.finish_interrupted_steps( exit_code=exit_code, diagnosis=diagnosis, @@ -3256,4 +3338,5 @@ def main(argv: list[str] | None = None) -> int: use_json="--json" in raw_argv, ) finally: + _close_active_native_testmon_state() _ACTIVE_VERIFY_RUN = None diff --git a/tests/integration/devtools/test_native_testmon_lifecycle.py b/tests/integration/devtools/test_native_testmon_lifecycle.py index 5de91e85d3..23d40c618b 100644 --- a/tests/integration/devtools/test_native_testmon_lifecycle.py +++ b/tests/integration/devtools/test_native_testmon_lifecycle.py @@ -1200,16 +1200,20 @@ def test_neutralized_environment_and_declared_plugin_identity_are_owned( ] == [0, 0] -def test_production_verify_fails_closed_on_dynamic_pytest_plugins(tmp_path: Path) -> None: +@pytest.mark.parametrize( + "declaration", + [ + "from plugin_config import plugin_names\n\npytest_plugins = plugin_names\n", + 'from plugin_config import plugin_names\n\nglobals()["pytest_plugins"] = plugin_names\n', + ], +) +def test_production_verify_fails_closed_on_dynamic_pytest_plugins(tmp_path: Path, declaration: str) -> None: repo = tmp_path / "repo" repo.mkdir() _init_repo(repo) (repo / "plugin_config.py").write_text('plugin_names = ("local_plugin",)\n', encoding="utf-8") (repo / "local_plugin.py").write_text("VALUE = 1\n", encoding="utf-8") - (repo / "tests" / "conftest.py").write_text( - "from plugin_config import plugin_names\n\npytest_plugins = plugin_names\n", - encoding="utf-8", - ) + (repo / "tests" / "conftest.py").write_text(declaration, encoding="utf-8") (repo / "tests" / "test_body.py").write_text("def test_body():\n assert True\n", encoding="utf-8") _commit_all(repo, "fixture") origin = tmp_path / "origin.git" @@ -1223,7 +1227,42 @@ def test_production_verify_fails_closed_on_dynamic_pytest_plugins(tmp_path: Path assert completed.returncode == 125 assert payload["diagnosis"] == "native_testmon_preparation_failed" assert payload["release_baseline_allowed"] is False - assert "pytest_plugins declaration must be a literal" in completed.stderr + assert "pytest_plugins declaration must" in completed.stderr + + +def test_managed_native_launch_keeps_state_inode_bound_during_parent_replacement( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _init_repo(repo) + (repo / "tests" / "test_body.py").write_text("def test_body():\n assert True\n", encoding="utf-8") + _commit_all(repo, "fixture") + prepare_native_testmon_environment(repo) + external_cache = tmp_path / "external-cache" + external_cache.mkdir() + monkeypatch.setattr(verify, "ROOT", repo) + monkeypatch.setattr(verify, "TESTMON_DATA", repo / TESTMON_DATA_RELPATH) + + def replace_parent( + _label: str, + _cmd: list[str], + **kwargs: object, + ) -> tuple[int, float, dict[str, object]]: + bound_data = kwargs["native_testmon_data"] + assert isinstance(bound_data, Path) + (repo / ".cache").rename(repo / ".cache-owned") + (repo / ".cache").symlink_to(external_cache, target_is_directory=True) + bound_data.write_text("bound database", encoding="utf-8") + return 0, 0.01, {} + + with pytest.MonkeyPatch.context() as patcher: + patcher.setattr(verify, "_run_step", replace_parent) + assert verify._run("pytest native parallel (affected)", ["pytest"])[0] == 0 + + assert (repo / ".cache-owned" / "testmon" / "testmondata").read_text(encoding="utf-8") == "bound database" + assert list(external_cache.iterdir()) == [] def test_managed_native_routes_reject_replaced_cache_parent( diff --git a/tests/unit/devtools/test_testmon_bootstrap.py b/tests/unit/devtools/test_testmon_bootstrap.py index 6d37878e49..f18f182224 100644 --- a/tests/unit/devtools/test_testmon_bootstrap.py +++ b/tests/unit/devtools/test_testmon_bootstrap.py @@ -246,6 +246,13 @@ def test_dynamic_pytest_plugin_declaration_fails_closed(tmp_path: Path) -> None: with pytest.raises(NativeTestmonRepairError, match="must be one literal assignment"): _testmon_environment_digest(tmp_path) + conftest.write_text( + 'globals()["pytest_plugins"] = ("tests.infra.fixture_plugin",)\n', + encoding="utf-8", + ) + with pytest.raises(NativeTestmonRepairError, match="must be one literal assignment"): + _testmon_environment_digest(tmp_path) + def test_environment_digest_ignores_neutralized_pytest_plugins( tmp_path: Path, From 715408f4d3bdb9b9415ab80435ce42fdaa23f51d Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 14 Aug 2026 06:11:02 +0200 Subject: [PATCH 24/31] chore: drop stale Beads delta after rebase --- .beads/issues.jsonl | 1 - 1 file changed, 1 deletion(-) diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index e645d631c8..55bf665e37 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -1,4 +1,3 @@ -{"_type":"issue","id":"polylogue-2tk5h","title":"Nix: suppress stale free-threaded bootstrap checks","description":"On consumer nixpkgs where python314FreeThreading is uncached, the Polylogue flake closure fails in the pyproject-version-patch-hook helper fixpoint on stale python-discovery, virtualenv, and poetry-core checks. The fix must scope check suppression to the free-threaded interpreter package set and preserve standard interpreter checks.","acceptance_criteria":"1. A consumer nixpkgs build using an uncached python314FreeThreading closure reaches activation without the three stale bootstrap-check failures. 2. Suppression is scoped only to the free-threaded interpreter package set and does not disable checks for standard Python package sets. 3. The implementation records a recheck condition tied to upstream package versions or equivalent evidence. 4. A focused reproduction or build receipt names the failing leaves and the successful end-to-end route. 5. No unrelated Polylogue runtime or test behavior changes.","status":"closed","priority":1,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-08-11T02:56:39Z","created_by":"Sinity","updated_at":"2026-08-11T04:16:00Z","closed_at":"2026-08-11T04:16:00Z","close_reason":"Ref #3945 and commit 5d0e98c927f4feb77ed68fe807fbee4a4f038f33. PR #3945 merged at ce4dd629a5d53d312751ff3a484c86edc5665a56 from actual head fd999deebb48a4d6fdefd8b23cba651b2bc4c331; its embedded carrier named stale head 2aa9710d and is not used as provenance. The merged fix scopes suppressions to the free-threaded package set and records upstream-version rechecks; the installed Sinnix package is 0.3.0+ce4dd629. The follow-up commit binds both venv creation paths to the active devshell interpreter and proves a fresh detached worktree on Python 3.14.4 free-threaded, 2 focused tests, and a 25/25 quick gate. All acceptance criteria are satisfied; no Polylogue runtime behavior changed.","labels":["area:build","area:verification"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-2yivh","title":"test harness: recover partial seeded cache automatically","description":"Complete seeded-cache recovery for polylogue-9pf58. A SIGKILL or crash after a seeded database/build marker is written must be detected as incomplete, quarantined or rebuilt under ownership/lock, and never reused as a valid seed. Preserve active builders and foreign paths.","acceptance_criteria":"1. Partial seeded directories with a completion marker but missing required schema/data are detected. 2. Active or locked builders are preserved. 3. Dead partial builders are quarantined or atomically rebuilt. 4. A no-such-table failure cannot recur from a promoted partial seed. 5. Focused mutation tests cover marker-before-data and crash-before-marker cases.","status":"closed","priority":0,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-08-11T00:21:08Z","created_by":"Sinity","updated_at":"2026-08-11T00:31:06Z","closed_at":"2026-08-11T00:31:06Z","close_reason":"Satisfied by the seeded-artifact recovery path added in this branch: per-key flock excludes active builders, crash-left staging trees are swept before rebuild, final publication remains atomic, and the 9-test workload-artifact suite plus focused crash-recovery regression pass. Partial final artifacts continue to be rejected and rebuilt by the existing manifest/integrity validation.","dependencies":[{"issue_id":"polylogue-2yivh","depends_on_id":"polylogue-9pf58","type":"discovered-from","created_at":"2026-08-11T00:21:08Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-7wytz","title":"test harness: discover xdist workers from in-process receipts","description":"Complete the xdist stall AC for polylogue-9pf58. The supervisor must count real xdist workers from harness-owned in-process identity receipts or another process-visible authority, not only /proc exec-time environment. The six-worker D-state reproduction must classify all workers and terminate only the owned process group after the typed interval.","acceptance_criteria":"1. Every real worker emits an identity receipt before tests run. 2. The sampler resolves worker pid to worker id without relying on post-exec environment visibility. 3. A controlled six-worker D-state fixture produces the typed stall diagnosis. 4. Partial worker observation does not trigger all-workers termination.","status":"closed","priority":0,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-08-11T00:21:08Z","created_by":"Sinity","updated_at":"2026-08-11T00:43:33Z","closed_at":"2026-08-11T00:43:33Z","close_reason":"Satisfied by the controlled six-worker sampler fixture: six in-process session_started receipts resolve worker IDs without /proc environment reliance, all six workers in D state produce the typed stall diagnosis only after the full interval, and the existing partial-observation regression remains green. This validates the owned-process-group decision without manufacturing a live kernel D-state in the test suite.","dependencies":[{"issue_id":"polylogue-7wytz","depends_on_id":"polylogue-9pf58","type":"discovered-from","created_at":"2026-08-11T00:21:08Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-enl3l","title":"test harness: automatic reroute before resource termination","description":"Complete the remaining admission AC for polylogue-9pf58. A real verify/test run must estimate declared demand against every candidate root and reroute before the run can exceed the selected storage budget; a typed refusal is acceptable only when no supported candidate can satisfy the demand. Add a real-route regression proving the selected path never reaches supervisor termination for a known-demand run.","acceptance_criteria":"1. A supported alternate root is selected before execution when tmpfs demand cannot fit. 2. No run is allowed to start on a root that cannot satisfy declared demand plus reserve. 3. Focused mutation tests make the admission fail when reroute is removed. 4. Receipt records candidates, demand, reserve, selected root, and outcome.","status":"closed","priority":0,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-08-11T00:21:08Z","created_by":"Sinity","updated_at":"2026-08-11T00:34:55Z","closed_at":"2026-08-11T00:34:55Z","close_reason":"Satisfied by the merged basetemp admission resolver: declared demand is compared against each supported root before pytest starts, and the new real resolver regression proves a 2 GiB demand reroutes from insufficient tmpfs to NVMe scratch. Existing focused admission tests and the 25-step quick gate pass.","dependencies":[{"issue_id":"polylogue-enl3l","depends_on_id":"polylogue-9pf58","type":"discovered-from","created_at":"2026-08-11T00:21:08Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} From c7556c982384dac9948f142301769088d0eee2da Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 14 Aug 2026 06:15:46 +0200 Subject: [PATCH 25/31] style: format rebased authority monitor --- devtools/verify_runs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index f386b97bae..3ec33a1616 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -725,7 +725,8 @@ def _record_change(self, candidate: Path) -> None: nested_authorities = [ (authority_path, label) for authority_path, label in self._git_authority_paths.items() - if candidate != authority_path and authority_path.is_relative_to(candidate) + if candidate != authority_path + and authority_path.is_relative_to(candidate) and not (label == ".git/packed-refs" and self._git_current_ref_was_loose is True) ] if nested_authorities: From 25de372c70514990a72b65f767d1aa5f2855c2de Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 14 Aug 2026 11:18:30 +0200 Subject: [PATCH 26/31] fix: isolate managed verification Python startup --- devtools/verify.py | 53 +++++++- .../devtools/test_native_testmon_lifecycle.py | 125 +++++++++++++++++- tests/unit/devtools/test_verify.py | 30 ++++- 3 files changed, 199 insertions(+), 9 deletions(-) diff --git a/devtools/verify.py b/devtools/verify.py index d94cc34320..2cc2cb34bb 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -147,11 +147,23 @@ ) -def _normalize_managed_pytest_environment(env: dict[str, str]) -> None: +def _normalize_managed_pytest_environment( + env: dict[str, str], + *, + disable_plugin_autoload: bool = True, +) -> None: """Remove ambient pytest options and extensions from a managed child.""" env.pop("PYTEST_ADDOPTS", None) env.pop("PYTEST_PLUGINS", None) - env["PYTEST_DISABLE_PLUGIN_AUTOLOAD"] = "1" + if disable_plugin_autoload: + env["PYTEST_DISABLE_PLUGIN_AUTOLOAD"] = "1" + else: + env.pop("PYTEST_DISABLE_PLUGIN_AUTOLOAD", None) + + +def _python_optimization_level() -> int: + """Return the active interpreter optimization level.""" + return int(sys.flags.optimize) @dataclass(slots=True) @@ -244,6 +256,8 @@ def _mypy_cmd() -> list[str]: capture_output=True, text=True, timeout=5, + cwd=ROOT, + env=_subprocess_env(), ) if result.returncode == 0: return ["dmypy", "run", "--", "--no-error-summary"] @@ -1761,6 +1775,7 @@ def _run_step( # ``bench slo`` starts pytest-benchmark itself, so it needs the same # bounded temp policy and run marker as a direct pytest step. has_managed_pytest_child = label == "bench slo" + owns_pytest_environment = managed_native_lane or has_managed_pytest_child if is_pytest and run is not None: isolated_report = run.run_dir / f"pytest-report-{uuid.uuid4().hex}.json" cmd = [f"--json-report-file={isolated_report}" if arg.startswith("--json-report-file=") else arg for arg in cmd] @@ -1770,8 +1785,8 @@ def _run_step( env = _subprocess_env(native_testmon_data=native_testmon_data) external_addopts_neutralized = False external_plugins_neutralized = False - if managed_native_lane: - _normalize_managed_pytest_environment(env) + if owns_pytest_environment: + _normalize_managed_pytest_environment(env, disable_plugin_autoload=managed_native_lane) explicit_basetemp = _pytest_command_basetemp(cmd, cwd=cwd, env=env) if explicit_basetemp is not None: env[PYTEST_EXPLICIT_BASETEMP_ENV] = str(explicit_basetemp) @@ -1825,8 +1840,9 @@ def _run_step( env["POLYLOGUE_PYTEST_SELECTION_NODEID_LIMIT"] = "50000" if run is not None and artifacts is not None: env = env_for_pytest_step(env, run=run, artifacts=artifacts) + if owns_pytest_environment: + _normalize_managed_pytest_environment(env, disable_plugin_autoload=managed_native_lane) if managed_native_lane: - _normalize_managed_pytest_environment(env) external_addopts_neutralized = _PYTEST_CLEAR_CONFIGURED_ADDOPTS in cmd external_plugins_neutralized = ( "PYTEST_PLUGINS" not in env and env.get("PYTEST_DISABLE_PLUGIN_AUTOLOAD") == "1" @@ -2211,8 +2227,13 @@ def _subprocess_env(*, native_testmon_data: Path | None = None) -> dict[str, str env["GIT_OPTIONAL_LOCKS"] = "0" env["POLYLOGUE_ROOT"] = str(ROOT) env["POLYLOGUE_REPO_ROOT"] = str(ROOT) - inherited_pythonpath = env.get("PYTHONPATH", "") - env["PYTHONPATH"] = str(ROOT) if not inherited_pythonpath else f"{ROOT}{os.pathsep}{inherited_pythonpath}" + # Managed verification owns Python startup. Inherited paths can load a + # sitecustomize module before pytest gets a chance to neutralize addopts. + env["PYTHONPATH"] = str(ROOT) + env.pop("PYTHONOPTIMIZE", None) + env.pop("PYTHONHOME", None) + env.pop("PYTHONUSERBASE", None) + env["PYTHONNOUSERSITE"] = "1" env["PYTHONPYCACHEPREFIX"] = str(ROOT / ".cache" / "pycache") env["TESTMON_DATAFILE"] = str(native_testmon_data or TESTMON_DATA) env["POLYLOGUE_PYTEST_EVENTS_DIR"] = str(ROOT / PYTEST_EVENTS_DIR) @@ -2798,6 +2819,7 @@ def _main(argv: list[str] | None = None) -> int: ) head = _git_head() pytest_enabled = not (args.quick or args.commit) + managed_pytest_enabled = pytest_enabled or args.lab planned_scope = _planned_verification_scope( args, testmon_mode="full" if full_requested else None, @@ -2814,6 +2836,23 @@ def _main(argv: list[str] | None = None) -> int: head=head, ) + optimization_level = _python_optimization_level() + if managed_pytest_enabled and optimization_level > 0: + return _finalize_preflight_failure( + verify_run, + started_at=started_at, + tier=tier, + head=head, + verification_scope=planned_scope, + diagnosis="optimized_python_interpreter", + exit_code=125, + message=( + "Python optimization disables verification assertions; " + f"refusing managed pytest at optimization level {optimization_level}" + ), + use_json=bool(use_json), + ) + try: fingerprint = assert_polylogue_matches_checkout(ROOT, context="devtools verify") except CheckoutImportMismatchError as exc: diff --git a/tests/integration/devtools/test_native_testmon_lifecycle.py b/tests/integration/devtools/test_native_testmon_lifecycle.py index 23d40c618b..847bca4402 100644 --- a/tests/integration/devtools/test_native_testmon_lifecycle.py +++ b/tests/integration/devtools/test_native_testmon_lifecycle.py @@ -172,6 +172,8 @@ def _run_production_verify( repo: Path, *args: str, allow_rejection: bool = False, + environment_overrides: dict[str, str] | None = None, + interpreter_args: tuple[str, ...] = (), ) -> tuple[subprocess.CompletedProcess[str], dict[str, Any]]: """Run the production verifier orchestration against a tiny fixture corpus. @@ -227,9 +229,19 @@ def fixture_env_for_pytest_step(env, **kwargs): "GIT_OPTIONAL_LOCKS": "0", } ) + if environment_overrides is not None: + env.update(environment_overrides) try: completed = subprocess.run( - [sys.executable, "-c", driver, str(repo), *args, "--json"], + [ + sys.executable, + *interpreter_args, + "-c", + driver, + str(repo), + *args, + "--json", + ], cwd=PROJECT_ROOT, env=env, capture_output=True, @@ -266,6 +278,8 @@ def fixture_env_for_pytest_step(env, **kwargs): persisted = json.loads(receipt.read_text(encoding="utf-8")) assert persisted["invocation_id"] == invocation_id assert persisted["pytest_aggregate"] == payload["pytest_aggregate"] + for authority_field in ("diagnosis", "exit_code", "release_baseline_allowed"): + assert persisted.get(authority_field) == payload.get(authority_field) return completed, payload @@ -540,6 +554,115 @@ def test_production_verify_all_neutralizes_external_pytest_addopts( assert "serial body executed" in completed.stderr +def test_production_verify_all_drops_pythonpath_startup_injection(tmp_path: Path) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _init_repo(repo) + package = repo / "polylogue" + package.mkdir() + (package / "__init__.py").write_text("", encoding="utf-8") + release_source = ( + "import pytest\n\n" + "def test_parallel_passes():\n" + " assert True\n\n" + "@pytest.mark.load_sensitive\n" + "def test_serial_passes():\n" + " assert True\n\n" + "def test_omitted_failure():\n" + " assert False, 'PYTHONPATH startup injection did not narrow execution'\n" + ) + (repo / "tests" / "test_release.py").write_text(release_source, encoding="utf-8") + _commit_all(repo, "fixture") + origin = tmp_path / "origin.git" + _git(origin.parent, "init", "--bare", "-q", str(origin)) + _git(repo, "remote", "add", "origin", str(origin)) + _git(repo, "branch", "-M", "master") + _git(repo, "push", "-qu", "origin", "master") + startup_path = tmp_path / "ambient-pythonpath" + startup_path.mkdir() + (startup_path / "sitecustomize.py").write_text( + 'import os\nos.environ["PYTEST_ADDOPTS"] = "-k passes"\n', + encoding="utf-8", + ) + + completed, payload = _run_production_verify( + repo, + "--all", + environment_overrides={ + "PYTHONPATH": os.pathsep.join((str(PROJECT_ROOT), str(startup_path))), + }, + ) + + assert completed.returncode == 1 + assert payload["release_baseline_allowed"] is False + aggregate = payload["pytest_aggregate"] + assert aggregate["selected_union_count"] == 3 + assert aggregate["terminal_union_count"] == 3 + assert aggregate["outcomes"] == {"failed": 1, "passed": 2} + assert aggregate["terminal_green"] is False + assert "PYTHONPATH startup injection did not narrow execution" in completed.stderr + + +@pytest.mark.parametrize( + ("environment_overrides", "interpreter_args", "verify_args"), + [ + ({"PYTHONOPTIMIZE": "1"}, (), ("--all",)), + ({}, ("-O",), ("--all",)), + ({}, ("-OO",), ("--all",)), + ({"PYTHONOPTIMIZE": "1"}, (), ("--quick", "--lab")), + ({"PYTHONOPTIMIZE": "1"}, (), ("--commit", "--lab")), + ], + ids=("pythonoptimize", "dash-o", "dash-oo", "quick-lab", "commit-lab"), +) +def test_production_verify_rejects_optimized_managed_pytest_interpreter( + tmp_path: Path, + environment_overrides: dict[str, str], + interpreter_args: tuple[str, ...], + verify_args: tuple[str, ...], +) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _init_repo(repo) + package = repo / "polylogue" + package.mkdir() + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "invariant.py").write_text( + "def require_failure():\n assert False, 'product assertion executed'\n", + encoding="utf-8", + ) + (repo / "tests" / "test_release.py").write_text( + "import pytest\n" + "from polylogue.invariant import require_failure\n\n" + "def test_parallel_product_assertion():\n" + " require_failure()\n\n" + "@pytest.mark.load_sensitive\n" + "def test_serial_lane():\n" + " require_failure()\n", + encoding="utf-8", + ) + _commit_all(repo, "fixture") + origin = tmp_path / "origin.git" + _git(origin.parent, "init", "--bare", "-q", str(origin)) + _git(repo, "remote", "add", "origin", str(origin)) + _git(repo, "branch", "-M", "master") + _git(repo, "push", "-qu", "origin", "master") + + completed, payload = _run_production_verify( + repo, + *verify_args, + allow_rejection=True, + environment_overrides=environment_overrides, + interpreter_args=interpreter_args, + ) + + assert completed.returncode == 125 + assert payload["diagnosis"] == "optimized_python_interpreter" + assert payload["exit_code"] == 125 + assert payload["release_baseline_allowed"] is False + assert payload["pytest_aggregate"]["selection_mode"] == "none" + assert "Python optimization disables verification assertions" in completed.stderr + + @pytest.mark.parametrize("ambient_addopts", ["--collect-only", "--setup-only"]) def test_production_affected_verify_neutralizes_execution_suppressing_addopts( tmp_path: Path, diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 3ba1547735..0a4681467a 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -3288,6 +3288,9 @@ def test_bench_slo_forces_nested_pytest_to_managed_scratch( scratch.mkdir() run = VerifyRun(tier="lab", argv=[], git_head=None, root=tmp_path) monkeypatch.setenv("POLYLOGUE_PYTEST_BASETEMP_ROOT", str(inherited_tmpfs_root)) + monkeypatch.setenv("PYTEST_ADDOPTS", "--collect-only") + monkeypatch.setenv("PYTEST_PLUGINS", "ambient_plugin") + monkeypatch.setenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", "1") managed_env = { "POLYLOGUE_PYTEST_TMPFS": "0", "POLYLOGUE_PYTEST_BASETEMP_ROOT": str(scratch), @@ -3310,6 +3313,9 @@ def test_bench_slo_forces_nested_pytest_to_managed_scratch( assert env["POLYLOGUE_PYTEST_RUN_ID"] == run.run_id assert env["POLYLOGUE_PYTEST_TMPFS"] == "0" assert env["POLYLOGUE_PYTEST_BASETEMP_ROOT"] == str(scratch) + assert "PYTEST_ADDOPTS" not in env + assert "PYTEST_PLUGINS" not in env + assert "PYTEST_DISABLE_PLUGIN_AUTOLOAD" not in env def test_run_forces_subprocesses_to_current_checkout(monkeypatch: pytest.MonkeyPatch) -> None: @@ -3317,6 +3323,10 @@ def test_run_forces_subprocesses_to_current_checkout(monkeypatch: pytest.MonkeyP monkeypatch.setenv("POLYLOGUE_REPO_ROOT", "/stale/main") monkeypatch.setenv("PYTHONPYCACHEPREFIX", "/stale/main/.cache/pycache") monkeypatch.setenv("PYTHONPATH", "/stale/main") + monkeypatch.setenv("PYTHONOPTIMIZE", "1") + monkeypatch.setenv("PYTHONHOME", "/stale/python") + monkeypatch.setenv("PYTHONUSERBASE", "/stale/user-site") + monkeypatch.setenv("PYTHONNOUSERSITE", "0") completed = subprocess.CompletedProcess(args=["devtools"], returncode=0, stdout="", stderr="") with patch("devtools.verify.subprocess.run", return_value=completed) as run: @@ -3327,10 +3337,28 @@ def test_run_forces_subprocesses_to_current_checkout(monkeypatch: pytest.MonkeyP assert env["POLYLOGUE_ROOT"] == str(ROOT) assert env["POLYLOGUE_REPO_ROOT"] == str(ROOT) assert env["PYTHONPYCACHEPREFIX"] == str(ROOT / ".cache" / "pycache") - assert env["PYTHONPATH"].split(os.pathsep)[0] == str(ROOT) + assert env["PYTHONPATH"] == str(ROOT) + assert "PYTHONOPTIMIZE" not in env + assert "PYTHONHOME" not in env + assert "PYTHONUSERBASE" not in env + assert env["PYTHONNOUSERSITE"] == "1" assert env["POLYLOGUE_PYTEST_EVENTS_PATH"] == str(ROOT / PYTEST_EVENTS_PATH) +def test_mypy_probe_uses_managed_python_startup_environment(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("PYTHONPATH", "/caller/sitecustomize") + monkeypatch.setenv("PYTHONOPTIMIZE", "2") + completed = subprocess.CompletedProcess(args=["dmypy", "status"], returncode=1, stdout="", stderr="") + + with patch("devtools.verify.subprocess.run", return_value=completed) as run: + assert verify._mypy_cmd() == ["mypy"] + + assert run.call_args.kwargs["cwd"] == ROOT + env = run.call_args.kwargs["env"] + assert env["PYTHONPATH"] == str(ROOT) + assert "PYTHONOPTIMIZE" not in env + + def test_verify_subprocess_env_removes_cloud_basetemp_in_local_worktree(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("POLYLOGUE_PYTEST_BASETEMP_ROOT", "/tmp/polylogue-pytest") completed = subprocess.CompletedProcess(args=["devtools"], returncode=0, stdout="", stderr="") From 43fee85a2a47512e5da9efaebde7f38bc5aaed47 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 14 Aug 2026 13:49:12 +0200 Subject: [PATCH 27/31] fix(verify): harden native testmon release authority Force complete Hypothesis settings for release-baseline native lanes, bind lifecycle locks through checkout-owned no-follow descriptors, and retain preparation-window checkout mutations in final authority. Align merge-train guidance with release-baseline-only terminal receipts. --- devtools/merge_boundary.py | 7 +- devtools/testmon_bootstrap.py | 10 +- devtools/verify.py | 135 ++++++++++++++---- .../devtools/test_native_testmon_lifecycle.py | 49 +++++++ tests/unit/devtools/test_merge_boundary.py | 14 ++ tests/unit/devtools/test_verify.py | 92 +++++++++++- 6 files changed, 277 insertions(+), 30 deletions(-) diff --git a/devtools/merge_boundary.py b/devtools/merge_boundary.py index b6c10385dd..7bf07f30f1 100644 --- a/devtools/merge_boundary.py +++ b/devtools/merge_boundary.py @@ -37,8 +37,8 @@ 6. Runs the actual ``gh pr merge --squash``. 7. Appends a merge-train ledger entry (``.cache/verify/merge-gate/merge-train-ledger.json``) and, unless ``--with-verify`` was given, prints a reminder that the - ledger's terminal step -- one full-suite ``devtools verify --all`` (or - narrower agreed selection) since the last one -- has not yet been + ledger's terminal step -- one release-baseline ``devtools verify --all`` + since the last one -- has not yet been recorded for this train. ``devtools workspace merge train-status`` inspects the ledger and reports @@ -775,7 +775,8 @@ def cmd_train_status(as_json: bool) -> int: print(f" PR #{entry['pr']} @ {entry['head_sha'][:8]}: {entry['title']}") print( 'Run `devtools workspace merge record-full-verify --command "devtools verify --all"` ' - "(or the narrower agreed selection) before declaring this merge-train session done -- " + "before declaring this merge-train session done. A narrower successful selection does not " + "grant the release-baseline authority this ledger requires. " "per-PR CI skips the heavy suite, so nothing else will catch a master-red class only " "visible on the merged whole." ) diff --git a/devtools/testmon_bootstrap.py b/devtools/testmon_bootstrap.py index 27b1173e21..6fabef8efc 100644 --- a/devtools/testmon_bootstrap.py +++ b/devtools/testmon_bootstrap.py @@ -32,7 +32,7 @@ import sys import time import uuid -from collections.abc import Iterable, Sequence +from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass from pathlib import Path, PurePosixPath from typing import Literal @@ -286,6 +286,7 @@ def testmon_environment_digest( repo_root: Path, *, pytest_profile: str = "default", + pytest_environment: Mapping[str, str | None] | None = None, deadline_monotonic: float | None = None, ) -> str: """Return the native testmon environment name for collection semantics.""" @@ -306,7 +307,10 @@ def testmon_environment_digest( _environment_input_paths(root), deadline_monotonic=deadline_monotonic, ), - "pytest_environment": {key: os.environ.get(key) for key in _PYTEST_ENVIRONMENT_KEYS}, + "pytest_environment": { + key: (os.environ.get(key) if pytest_environment is None else pytest_environment.get(key)) + for key in _PYTEST_ENVIRONMENT_KEYS + }, "pytest_profile": pytest_profile, } encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() @@ -749,6 +753,7 @@ def prepare_native_testmon_environment( *, required_executable_paths: Sequence[str] = (), pytest_profile: str = "default", + pytest_environment: Mapping[str, str | None] | None = None, deadline_monotonic: float | None = None, ) -> NativeTestmonPreparation: """Repair derived local state and optionally reuse a matching main graph.""" @@ -757,6 +762,7 @@ def prepare_native_testmon_environment( environment_name = testmon_environment_digest( root, pytest_profile=pytest_profile, + pytest_environment=pytest_environment, deadline_monotonic=deadline_monotonic, ) local_data = root / TESTMON_DATA_RELPATH diff --git a/devtools/verify.py b/devtools/verify.py index 2cc2cb34bb..6eb0538d4a 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -215,24 +215,72 @@ def _native_testmon_lifecycle_lock(repo_root: Path) -> Iterator[None]: if not stat.S_ISDIR(mode): raise NativeTestmonRepairError(f"native testmon lock parent is not an owned directory: {cache}") lock_path = cache / "native-testmon-lifecycle.lock" - with lock_path.open("a+", encoding="utf-8") as handle: - try: - fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) - except OSError: - handle.seek(0) - holder = handle.read().strip() or "another verify invocation" - sys.stderr.write(f"verify: waiting for native testmon lifecycle lock ({holder})\n") - sys.stderr.flush() - fcntl.flock(handle.fileno(), fcntl.LOCK_EX) - handle.seek(0) - handle.truncate() - handle.write(f"pid={os.getpid()}") - handle.flush() - try: - yield - finally: + directory_descriptor: int | None = None + lock_descriptor: int | None = None + try: + directory_descriptor = os.open( + cache, + os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0), + ) + opened_directory = os.fstat(directory_descriptor) + current_directory = cache.lstat() + if not stat.S_ISDIR(opened_directory.st_mode) or (opened_directory.st_dev, opened_directory.st_ino) != ( + current_directory.st_dev, + current_directory.st_ino, + ): + raise NativeTestmonRepairError(f"native testmon lock parent changed while binding: {cache}") + lock_descriptor = os.open( + lock_path.name, + os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0), + 0o600, + dir_fd=directory_descriptor, + ) + opened_lock = os.fstat(lock_descriptor) + current_lock = os.stat(lock_path.name, dir_fd=directory_descriptor, follow_symlinks=False) + if ( + not stat.S_ISREG(opened_lock.st_mode) + or not stat.S_ISREG(current_lock.st_mode) + or (opened_lock.st_dev, opened_lock.st_ino) != (current_lock.st_dev, current_lock.st_ino) + ): + raise NativeTestmonRepairError(f"native testmon lifecycle lock is not an owned regular file: {lock_path}") + except OSError as exc: + if lock_descriptor is not None: + with contextlib.suppress(OSError): + os.close(lock_descriptor) + lock_descriptor = None + if directory_descriptor is not None: + with contextlib.suppress(OSError): + os.close(directory_descriptor) + directory_descriptor = None + raise NativeTestmonRepairError(f"cannot bind native testmon lifecycle lock {lock_path}: {exc}") from exc + try: + assert lock_descriptor is not None + with os.fdopen(lock_descriptor, "r+", encoding="utf-8") as handle: + lock_descriptor = None + try: + fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + handle.seek(0) + holder = handle.read().strip() or "another verify invocation" + sys.stderr.write(f"verify: waiting for native testmon lifecycle lock ({holder})\n") + sys.stderr.flush() + fcntl.flock(handle.fileno(), fcntl.LOCK_EX) handle.seek(0) handle.truncate() + handle.write(f"pid={os.getpid()}") + handle.flush() + try: + yield + finally: + handle.seek(0) + handle.truncate() + finally: + if lock_descriptor is not None: + with contextlib.suppress(OSError): + os.close(lock_descriptor) + if directory_descriptor is not None: + with contextlib.suppress(OSError): + os.close(directory_descriptor) def _anchor_verification_paths() -> None: @@ -1787,6 +1835,8 @@ def _run_step( external_plugins_neutralized = False if owns_pytest_environment: _normalize_managed_pytest_environment(env, disable_plugin_autoload=managed_native_lane) + if managed_native_lane and _pytest_uses_full_suite_basetemp(label): + env["HYPOTHESIS_PROFILE"] = "default" explicit_basetemp = _pytest_command_basetemp(cmd, cwd=cwd, env=env) if explicit_basetemp is not None: env[PYTEST_EXPLICIT_BASETEMP_ENV] = str(explicit_basetemp) @@ -2624,6 +2674,16 @@ def _pytest_profile() -> str: return "correctness=complete" +def _native_pytest_environment(*, force_release_profile: bool) -> dict[str, str | None]: + environment = { + "HYPOTHESIS_PROFILE": os.environ.get("HYPOTHESIS_PROFILE"), + "POLYLOGUE_CI": os.environ.get("POLYLOGUE_CI"), + } + if force_release_profile: + environment["HYPOTHESIS_PROFILE"] = "default" + return environment + + def _remaining_invocation_budget(started_at: float) -> float: return max(0.0, VERIFY_INVOCATION_BUDGET_S - (time.monotonic() - started_at)) @@ -2907,6 +2967,8 @@ def _main(argv: list[str] | None = None) -> int: runtime_data_paths: tuple[str, ...] = () preparation: NativeTestmonPreparation | None = None testmon_mode: str | None = None + native_pytest_environment = _native_pytest_environment(force_release_profile=full_requested) + preparation_mutation_observation: CheckoutMutationObservation | None = None if pytest_enabled: assert base_commit is not None assert head is not None @@ -2922,8 +2984,21 @@ def _main(argv: list[str] | None = None) -> int: ROOT, required_executable_paths=preparation_required_executable_paths, pytest_profile=_pytest_profile(), + pytest_environment=native_pytest_environment, deadline_monotonic=started_at + VERIFY_INVOCATION_BUDGET_S, ) + if ( + preparation.selection_mode == "bootstrap" + and native_pytest_environment["HYPOTHESIS_PROFILE"] != "default" + ): + native_pytest_environment = _native_pytest_environment(force_release_profile=True) + preparation = prepare_native_testmon_environment( + ROOT, + required_executable_paths=preparation_required_executable_paths, + pytest_profile=_pytest_profile(), + pytest_environment=native_pytest_environment, + deadline_monotonic=started_at + VERIFY_INVOCATION_BUDGET_S, + ) assert _ACTIVE_VERIFY_RUN is not None _ACTIVE_VERIFY_RUN.owned_native_testmon_state = _open_owned_native_testmon_state(ROOT) except NativeTestmonDeadlineError as exc: @@ -3005,9 +3080,10 @@ def _main(argv: list[str] | None = None) -> int: ) # Git probes and native testmon preparation can refresh the index as part - # of their own read path. Discard that preflight interval and begin the - # authority interval immediately before the verification steps. - _finish_active_checkout_mutation_monitor(mutation_monitor) + # of their own read path. Retain that interval's observation: a tracked + # file can be edited and restored while the graph is prepared, which makes + # any resulting selection unsuitable as exact-head authority. + preparation_mutation_observation = _finish_active_checkout_mutation_monitor(mutation_monitor) mutation_monitor = CheckoutMutationMonitor(ROOT) _start_active_checkout_mutation_monitor(mutation_monitor) step_results: list[dict[str, Any]] = [] @@ -3061,14 +3137,23 @@ def _main(argv: list[str] | None = None) -> int: final_checkout_fingerprint, } ) - if checkout_fingerprint_unavailable or mutation_observation.unavailable: + if ( + checkout_fingerprint_unavailable + or mutation_observation.unavailable + or preparation_mutation_observation.unavailable + ): checkout_stable = False diagnosis = ( "checkout_fingerprint_unavailable" if checkout_fingerprint_unavailable else "checkout_mutation_monitor_unavailable" ) - elif final_head != head or mutation_observation.changed or final_checkout_fingerprint != checkout_fingerprint: + elif ( + final_head != head + or preparation_mutation_observation.changed + or mutation_observation.changed + or final_checkout_fingerprint != checkout_fingerprint + ): checkout_stable = False diagnosis = "checkout_changed_during_verification" else: @@ -3083,8 +3168,10 @@ def _main(argv: list[str] | None = None) -> int: "final_git_head": final_head, "initial_worktree_fingerprint": checkout_fingerprint, "final_worktree_fingerprint": final_checkout_fingerprint, - "transient_checkout_mutation": mutation_observation.changed, - "checkout_mutation_path": mutation_observation.observed_path, + "transient_checkout_mutation": (preparation_mutation_observation.changed or mutation_observation.changed), + "checkout_mutation_path": ( + preparation_mutation_observation.observed_path or mutation_observation.observed_path + ), } step_results.append(stability_step) exit_code = 125 @@ -3246,7 +3333,7 @@ def _main(argv: list[str] | None = None) -> int: release_baseline_allowed=release_baseline_allowed, final_git_head=final_head, final_worktree_fingerprint=final_checkout_fingerprint, - checkout_mutation_path=mutation_observation.observed_path, + checkout_mutation_path=(preparation_mutation_observation.observed_path or mutation_observation.observed_path), checkout_diagnosis=checkout_diagnosis, pytest_aggregate=pytest_aggregate, invocation_budget_s=VERIFY_INVOCATION_BUDGET_S, diff --git a/tests/integration/devtools/test_native_testmon_lifecycle.py b/tests/integration/devtools/test_native_testmon_lifecycle.py index 847bca4402..fdaf919afe 100644 --- a/tests/integration/devtools/test_native_testmon_lifecycle.py +++ b/tests/integration/devtools/test_native_testmon_lifecycle.py @@ -474,6 +474,55 @@ def test_production_verify_all_grants_release_authority_after_complete_two_lane_ assert aggregate["deadline"] == {"budget_s": 3600.0, "met": True} +@pytest.mark.parametrize(("verify_args", "selection_mode"), [((), "bootstrap"), (("--all",), "full")]) +def test_release_native_runs_override_a_reduced_hypothesis_profile( + tmp_path: Path, + verify_args: tuple[str, ...], + selection_mode: str, +) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _init_repo( + repo, + conftest=( + "import os\n" + "from hypothesis import settings\n\n" + "settings.register_profile('default', max_examples=100)\n" + "settings.register_profile('verify', max_examples=10)\n" + "settings.load_profile(os.environ.get('HYPOTHESIS_PROFILE', 'default'))\n" + ), + ) + package = repo / "polylogue" + package.mkdir() + (package / "__init__.py").write_text("", encoding="utf-8") + (repo / "tests" / "test_hypothesis_profile.py").write_text( + "import pytest\n" + "from hypothesis import settings\n\n" + "def test_parallel_release_profile_is_complete():\n" + " assert settings().max_examples == 100\n\n" + "@pytest.mark.load_sensitive\n" + "def test_serial_release_profile_is_complete():\n" + " assert settings().max_examples == 100\n", + encoding="utf-8", + ) + _commit_all(repo, "fixture") + origin = tmp_path / "origin.git" + _git(origin.parent, "init", "--bare", "-q", str(origin)) + _git(repo, "remote", "add", "origin", str(origin)) + _git(repo, "branch", "-M", "master") + _git(repo, "push", "-qu", "origin", "master") + + completed, payload = _run_production_verify( + repo, + *verify_args, + environment_overrides={"HYPOTHESIS_PROFILE": "verify"}, + ) + + assert completed.returncode == 0, completed.stderr + assert payload["testmon_environment"]["selection_mode"] == selection_mode + assert payload["release_baseline_allowed"] is True + + @pytest.mark.parametrize( ("environment_addopts", "configured_addopts"), [ diff --git a/tests/unit/devtools/test_merge_boundary.py b/tests/unit/devtools/test_merge_boundary.py index f85af6f427..fff50d0a78 100644 --- a/tests/unit/devtools/test_merge_boundary.py +++ b/tests/unit/devtools/test_merge_boundary.py @@ -944,6 +944,20 @@ def test_train_status_blocks_when_pr_merged_after_last_full_verify( assert merge_boundary.cmd_train_status(as_json=False) == 1 +def test_train_status_requires_release_baseline_guidance( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + monkeypatch.chdir(tmp_path) + merge_boundary._append_merge_entry(1, "sha1", "some title") + + assert merge_boundary.cmd_train_status(as_json=False) == 1 + + output = capsys.readouterr().out + assert "devtools verify --all" in output + assert "narrower agreed selection" not in output + assert "does not grant the release-baseline authority" in output + + def test_train_status_reads_historical_scope_without_granting_release_authority( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 0a4681467a..d3976b5d90 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -20,7 +20,7 @@ from devtools import run_tests, verify, verify_runs from devtools.checkout_guard import CheckoutImportMismatchError -from devtools.testmon_bootstrap import NativeTestmonDeadlineError, executable_python_paths +from devtools.testmon_bootstrap import NativeTestmonDeadlineError, NativeTestmonRepairError, executable_python_paths from devtools.verification_contracts import VerificationScope from devtools.verify import ( PYTEST_CONTAINMENT_PATH, @@ -3124,6 +3124,20 @@ def contend_for_lock() -> None: assert contender_entered.is_set() +def test_native_testmon_lifecycle_lock_refuses_symlink_without_touching_target(tmp_path: Path) -> None: + cache = tmp_path / ".cache" + cache.mkdir() + target = tmp_path / "outside-checkout-target" + target.write_text("preserve this file\n", encoding="utf-8") + (cache / "native-testmon-lifecycle.lock").symlink_to(target) + + with pytest.raises(NativeTestmonRepairError, match="native testmon lifecycle lock"): + with verify._native_testmon_lifecycle_lock(tmp_path): + pytest.fail("symlinked lifecycle lock must not be acquired") + + assert target.read_text(encoding="utf-8") == "preserve this file\n" + + def test_run_clears_stale_current_statistics_before_an_interrupted_pytest_step(tmp_path: Path) -> None: stale_statistics = tmp_path / verify_runs.CURRENT_STATISTICS_PATH stale_statistics.parent.mkdir(parents=True) @@ -4355,6 +4369,82 @@ def test_release_authority_requires_current_complete_green_invocation() -> None: ) +def test_preparation_mutation_withholds_release_authority_after_restoration( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + observations = iter( + ( + CheckoutMutationObservation(changed=False, unavailable=False), + CheckoutMutationObservation(changed=True, unavailable=False, observed_path="polylogue/module.py"), + CheckoutMutationObservation(changed=False, unavailable=False), + ) + ) + + class _SequencedMonitor: + def __init__(self, _root: Path) -> None: + pass + + def start(self) -> None: + pass + + def finish(self) -> CheckoutMutationObservation: + return next(observations) + + preparation = SimpleNamespace( + environment_name="environment", + selection_mode="affected", + removed_paths=(), + copied_from=None, + ) + native_state = SimpleNamespace( + valid=True, + status="valid", + reason="current", + environment=SimpleNamespace(nodeids=("tests/test_owner.py::test_owner",)), + missing_executable_paths=(), + ) + release_aggregate = { + "complete_corpus_covered": True, + "terminal_green": True, + "external_addopts_neutralized": True, + "external_plugins_neutralized": True, + "closed_world_collection": True, + "cleanup": {"complete": True}, + "containment": {"complete": True}, + "deadline": {"met": True}, + } + monkeypatch.setattr(verify, "ROOT", tmp_path) + monkeypatch.setattr( + verify, + "assert_polylogue_matches_checkout", + lambda *_args, **_kwargs: SimpleNamespace(polylogue_import_path=tmp_path / "polylogue", as_dict=lambda: {}), + ) + + with ( + patch("devtools.verify._git_head", return_value="head"), + patch("devtools.verify._git_commit", return_value="base"), + patch("devtools.verify._changed_test_relevant_paths", return_value=()), + patch("devtools.verify.prepare_native_testmon_environment", return_value=preparation), + patch("devtools.verify._open_owned_native_testmon_state", return_value=SimpleNamespace(close=lambda: None)), + patch("devtools.verify._native_environment_after_run", return_value=native_state), + patch("devtools.verify.build_verify_steps", return_value=[]), + patch("devtools.verify.aggregate_native_testmon_run", return_value=release_aggregate), + patch("devtools.verify.CheckoutMutationMonitor", _SequencedMonitor), + patch("devtools.verify.worktree_fingerprint", return_value="stable"), + patch("devtools.verify._save_history"), + patch("devtools.verify._notify"), + ): + assert main(["--all", "--json"]) == 125 + + payload = json.loads(capsys.readouterr().out) + assert payload["release_baseline_allowed"] is False + stability = next(step for step in payload["steps"] if step["name"] == "checkout stability") + assert stability["diagnosis"] == "checkout_changed_during_verification" + assert stability["checkout_mutation_path"] == "polylogue/module.py" + + def test_collection_failure_still_persists_native_run_aggregate( capsys: pytest.CaptureFixture[str], ) -> None: From 567dc17986792b70b790393b362a2edacfc3626c Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 14 Aug 2026 14:03:30 +0200 Subject: [PATCH 28/31] test: isolate merge flow from Bead history Problem: merge-boundary unit tests used placeholder PR revisions. Strict carrier validation now resolves canonical Bead records from Git, so those tests stopped before exercising their intended merge behavior. The native testmon environment also treated an absent Hypothesis profile differently from the effective default profile, causing a redundant preparation pass.\n\nWhat changed: inject canonical synthetic Bead snapshots at the test storage boundary while retaining the real carrier validator and merge-gate path. Normalize the implicit Hypothesis profile to default in the native environment identity.\n\nVerification: direnv exec . devtools test exact 14 failing nodes (14 passed); direnv exec . devtools test tests/unit/devtools/test_merge_boundary.py tests/unit/devtools/test_verify.py (254 passed). --- devtools/verify.py | 5 ++++- tests/unit/devtools/test_merge_boundary.py | 17 +++++++++++++---- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/devtools/verify.py b/devtools/verify.py index 6eb0538d4a..0d72e5b70c 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -2676,7 +2676,10 @@ def _pytest_profile() -> str: def _native_pytest_environment(*, force_release_profile: bool) -> dict[str, str | None]: environment = { - "HYPOTHESIS_PROFILE": os.environ.get("HYPOTHESIS_PROFILE"), + # Hypothesis uses its default profile when the variable is absent. + # Record that effective value in the testmon environment identity so a + # bootstrap graph is reusable by the following affected invocation. + "HYPOTHESIS_PROFILE": os.environ.get("HYPOTHESIS_PROFILE") or "default", "POLYLOGUE_CI": os.environ.get("POLYLOGUE_CI"), } if force_release_profile: diff --git a/tests/unit/devtools/test_merge_boundary.py b/tests/unit/devtools/test_merge_boundary.py index fff50d0a78..a5cc84bf0a 100644 --- a/tests/unit/devtools/test_merge_boundary.py +++ b/tests/unit/devtools/test_merge_boundary.py @@ -26,10 +26,19 @@ @pytest.fixture(autouse=True) -def _scope_bead_record(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - beads_dir = tmp_path / ".beads" - beads_dir.mkdir() - (beads_dir / "issues.jsonl").write_text(json.dumps(_SCOPE_BEAD) + "\n") +def _scope_bead_record(monkeypatch: pytest.MonkeyPatch) -> None: + """Give merge-flow tests a canonical Bead snapshot without faking validation. + + Production scope validation reads the candidate and prospective records from + committed Git revisions. These unit tests deliberately use placeholder PR + SHAs, so inject that storage boundary and retain the real carrier, + disposition, digest, and merge-gate validation paths. + """ + + def canonical_records(_revision: str) -> dict[str, dict[str, object]]: + return {_SCOPE_BEAD["id"]: dict(_SCOPE_BEAD)} + + monkeypatch.setattr(pr_scope, "_bead_records_at", canonical_records) monkeypatch.setattr(pr_scope, "changed_bead_ids", lambda **_kwargs: []) From 9d3120c94d59a0c30d7e3ea58e22aaea2bbe8fc1 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 14 Aug 2026 14:26:44 +0200 Subject: [PATCH 29/31] fix(verify): harden native testmon review paths Problem: plugin discovery could outlive the verification deadline, topology rechecks could adopt changed Git authority, and integration helpers could inherit pytest controls or hide invalid JSON diagnostics. What changed: bound plugin discovery to the invocation deadline, retain first authority signatures, and align plain integration lanes with the managed pytest environment. Focused regressions cover every corrected path. Co-Authored-By: Codex --- devtools/testmon_bootstrap.py | 44 ++++++++++---- devtools/verify_runs.py | 6 +- .../devtools/test_native_testmon_lifecycle.py | 59 ++++++++++++++++++- tests/unit/devtools/test_testmon_bootstrap.py | 27 +++++++++ tests/unit/devtools/test_verify.py | 16 +++++ 5 files changed, 140 insertions(+), 12 deletions(-) diff --git a/devtools/testmon_bootstrap.py b/devtools/testmon_bootstrap.py index 6fabef8efc..90537dc626 100644 --- a/devtools/testmon_bootstrap.py +++ b/devtools/testmon_bootstrap.py @@ -181,12 +181,17 @@ def _indirect_pytest_plugins_declaration(node: ast.stmt) -> bool: return False -def _declared_pytest_plugin_names(root: Path) -> set[str]: +def _declared_pytest_plugin_names( + root: Path, + *, + deadline_monotonic: float | None = None, +) -> set[str]: """Read static local plugin declarations that pytest loads at collection.""" names: set[str] = set() - candidates = set(root.glob("tests/**/conftest.py")) + candidates: set[Path] = set() candidates.add(root / "conftest.py") for path in root.glob("tests/**/*.py"): + _ensure_deadline(deadline_monotonic) try: source = path.read_text(encoding="utf-8") except OSError: @@ -194,7 +199,11 @@ def _declared_pytest_plugin_names(root: Path) -> set[str]: if "pytest_plugins" not in source: continue candidates.add(path) - for path in candidates: + for path in root.glob("tests/**/conftest.py"): + _ensure_deadline(deadline_monotonic) + candidates.add(path) + for path in sorted(candidates): + _ensure_deadline(deadline_monotonic) try: tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) except (OSError, SyntaxError, UnicodeDecodeError): @@ -240,11 +249,16 @@ def _declared_pytest_plugin_names(root: Path) -> set[str]: return names -def _active_local_pytest_plugin_paths(root: Path) -> set[str]: +def _active_local_pytest_plugin_paths( + root: Path, + *, + deadline_monotonic: float | None = None, +) -> set[str]: """Resolve collection-active local pytest plugins regardless of filename.""" paths: set[str] = set() - plugin_names = _declared_pytest_plugin_names(root) + plugin_names = _declared_pytest_plugin_names(root, deadline_monotonic=deadline_monotonic) for raw_name in plugin_names: + _ensure_deadline(deadline_monotonic) module_name = raw_name.strip() if not module_name or any(part in {"", ".", ".."} for part in module_name.split(".")): continue @@ -254,11 +268,18 @@ def _active_local_pytest_plugin_paths(root: Path) -> set[str]: paths.add(module_file.relative_to(root).as_posix()) package = root / module_path if (package / "__init__.py").is_file(): - paths.update(path.relative_to(root).as_posix() for path in package.rglob("*.py") if path.is_file()) + for path in package.rglob("*.py"): + _ensure_deadline(deadline_monotonic) + if path.is_file(): + paths.add(path.relative_to(root).as_posix()) return paths -def _environment_input_paths(root: Path) -> tuple[str, ...]: +def _environment_input_paths( + root: Path, + *, + deadline_monotonic: float | None = None, +) -> tuple[str, ...]: """Discover collection and managed-pytest harness inputs.""" paths = set(_ENVIRONMENT_INPUTS) patterns = ( @@ -266,8 +287,11 @@ def _environment_input_paths(root: Path) -> tuple[str, ...]: "tests/**/conftest.py", ) for pattern in patterns: - paths.update(path.relative_to(root).as_posix() for path in root.glob(pattern) if path.is_file()) - paths.update(_active_local_pytest_plugin_paths(root)) + for path in root.glob(pattern): + _ensure_deadline(deadline_monotonic) + if path.is_file(): + paths.add(path.relative_to(root).as_posix()) + paths.update(_active_local_pytest_plugin_paths(root, deadline_monotonic=deadline_monotonic)) return tuple(sorted(paths)) @@ -304,7 +328,7 @@ def testmon_environment_digest( "distributions": _installed_distributions(), "inputs": _fingerprint_inputs( root, - _environment_input_paths(root), + _environment_input_paths(root, deadline_monotonic=deadline_monotonic), deadline_monotonic=deadline_monotonic, ), "pytest_environment": { diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index 3ec33a1616..adf32bb4b8 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -577,7 +577,11 @@ def walk_error(_error: OSError) -> None: if self._git_index_path is not None: self._git_authority_paths[self._git_index_path] = ".git/index" self._git_authority_paths.update(self._resolve_git_head_paths()) - self._git_authority_signatures = {path: self._authority_signature(path) for path in self._git_authority_paths} + for authority_path in self._git_authority_paths: + # The protected startup topology recheck discovers this set again. + # Retain each first baseline so it cannot adopt a write that the + # active watcher observed before processing its coalesced event. + self._git_authority_signatures.setdefault(authority_path, self._authority_signature(authority_path)) for authority_path in self._git_authority_paths: watched_parent = authority_path.parent while not watched_parent.exists() and watched_parent != watched_parent.parent: diff --git a/tests/integration/devtools/test_native_testmon_lifecycle.py b/tests/integration/devtools/test_native_testmon_lifecycle.py index fdaf919afe..4626e36b55 100644 --- a/tests/integration/devtools/test_native_testmon_lifecycle.py +++ b/tests/integration/devtools/test_native_testmon_lifecycle.py @@ -84,6 +84,9 @@ def _commit_all(root: Path, message: str) -> str: def _pytest_environment(repo: Path) -> dict[str, str]: (repo / TESTMON_DATA_RELPATH).parent.mkdir(parents=True, exist_ok=True) env = os.environ.copy() + env.pop("PYTEST_ADDOPTS", None) + env.pop("PYTEST_PLUGINS", None) + env["PYTEST_DISABLE_PLUGIN_AUTOLOAD"] = "1" env["TESTMON_DATAFILE"] = str(repo / TESTMON_DATA_RELPATH) env["PYTHONPATH"] = os.pathsep.join((str(repo), str(PROJECT_ROOT), env.get("PYTHONPATH", ""))) env["POLYLOGUE_PYTEST_SELECTION_NODEID_LIMIT"] = "50000" @@ -119,6 +122,7 @@ def _run_lane( "pytest", "-q", "--tb=short", + "--override-ini=addopts=", "--testmon", f"--testmon-env={environment_name}", selection, @@ -126,6 +130,12 @@ def _run_lane( marker, "-p", "devtools.pytest_progress_plugin", + "-p", + "pytest-testmon", + "-p", + "pytest_jsonreport", + "-p", + "xdist", "--json-report", "--json-report-omit=collectors,log,streams,warnings", f"--json-report-file={artifact_dir / PYTEST_CANONICAL_REPORT_NAME}", @@ -270,7 +280,13 @@ def fixture_env_for_pytest_step(env, **kwargs): pytest.fail( f"production verify wrote no invocation receipt\nstdout:\n{completed.stdout}\nstderr:\n{completed.stderr}" ) - payload = json.loads(completed.stdout) + try: + payload = json.loads(completed.stdout) + except json.JSONDecodeError as exc: + pytest.fail( + f"production verify emitted no JSON payload ({exc})\n" + f"returncode={completed.returncode}\nstdout:\n{completed.stdout}\nstderr:\n{completed.stderr}" + ) if completed.returncode == 125 and not allow_rejection: pytest.fail( f"production verify rejected the fixture checkout\npayload:\n{completed.stdout}\nstderr:\n{completed.stderr}" @@ -1338,6 +1354,25 @@ def test_neutralized_environment_and_declared_plugin_identity_are_owned( environment_unchanged = prepare_native_testmon_environment(repo) assert environment_unchanged.selection_mode == "affected" assert environment_unchanged.environment_name == initial.environment_name + test_file.write_text( + "import pytest\n\n" + "def test_parallel_body_must_run():\n" + " assert False, 'ambient pytest controls suppressed the parallel body'\n\n" + "@pytest.mark.load_sensitive\n" + "def test_serial_body_must_run():\n" + " assert False, 'ambient pytest controls suppressed the serial body'\n", + encoding="utf-8", + ) + ambient_results = _run_plain_verify_corpus( + repo, + mode="bootstrap", + environment_name=environment_unchanged.environment_name, + ) + assert [result.completed.returncode for result in ambient_results] == [1, 1] + assert _selected(*ambient_results) == { + "tests/test_identity.py::test_parallel_body_must_run", + "tests/test_identity.py::test_serial_body_must_run", + } monkeypatch.delenv("PYTEST_ADDOPTS") monkeypatch.delenv("PYTEST_PLUGINS") @@ -1372,6 +1407,28 @@ def test_neutralized_environment_and_declared_plugin_identity_are_owned( ] == [0, 0] +def test_production_verify_reports_stdout_when_json_payload_is_invalid( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + def invalid_json_result(command: list[str], **kwargs: Any) -> subprocess.CompletedProcess[str]: + environment = kwargs["env"] + assert isinstance(environment, dict) + receipt = Path(environment["POLYLOGUE_VERIFICATION_RECEIPT_PATH"]) + receipt.parent.mkdir(parents=True, exist_ok=True) + receipt.write_text("{}\n", encoding="utf-8") + return subprocess.CompletedProcess(command, 1, "not JSON", "verifier diagnostics") + + monkeypatch.setattr(subprocess, "run", invalid_json_result) + + with pytest.raises(pytest.fail.Exception) as failure: + _run_production_verify(tmp_path) + + assert "production verify emitted no JSON payload" in str(failure.value) + assert "stdout:\nnot JSON" in str(failure.value) + assert "stderr:\nverifier diagnostics" in str(failure.value) + + @pytest.mark.parametrize( "declaration", [ diff --git a/tests/unit/devtools/test_testmon_bootstrap.py b/tests/unit/devtools/test_testmon_bootstrap.py index f18f182224..35e6aa130b 100644 --- a/tests/unit/devtools/test_testmon_bootstrap.py +++ b/tests/unit/devtools/test_testmon_bootstrap.py @@ -300,6 +300,33 @@ def test_environment_digest_stops_at_invocation_deadline(tmp_path: Path) -> None _testmon_environment_digest(tmp_path, deadline_monotonic=0.0) +def test_plugin_declaration_discovery_stops_at_invocation_deadline( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + plugin_declaration = tmp_path / "tests" / "test_plugins.py" + plugin_declaration.parent.mkdir() + plugin_declaration.write_text('pytest_plugins = ("fixture_plugin",)\n', encoding="utf-8") + clock = {"value": 0.0} + original_read_text = Path.read_text + + def expire_after_plugin_discovery( + path: Path, + encoding: str | None = None, + errors: str | None = None, + ) -> str: + source = original_read_text(path, encoding=encoding, errors=errors) + if path == plugin_declaration: + clock["value"] = 1.0 + return source + + monkeypatch.setattr("devtools.testmon_bootstrap.time.monotonic", lambda: clock["value"]) + monkeypatch.setattr(Path, "read_text", expire_after_plugin_discovery) + + with pytest.raises(NativeTestmonDeadlineError, match="invocation deadline"): + _testmon_environment_digest(tmp_path, deadline_monotonic=0.5) + + def test_invalid_cleanup_removes_only_owned_sqlite_and_sidecars(tmp_path: Path) -> None: state_dir = tmp_path / ".cache" / "testmon" state_dir.mkdir(parents=True) diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index d3976b5d90..f5eb50c379 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -1489,6 +1489,22 @@ def delayed_watch(*_paths: Path, **kwargs: object) -> object: assert observation == CheckoutMutationObservation(changed=True, unavailable=False, observed_path=".git/index") +def test_checkout_mutation_monitor_keeps_initial_authority_signature_across_topology_recheck(tmp_path: Path) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + tracked = tmp_path / "tracked.py" + tracked.write_text("value = 1\n", encoding="utf-8") + subprocess.run(["git", "add", "tracked.py"], cwd=tmp_path, check=True) + + monitor = CheckoutMutationMonitor(tmp_path) + monitor._watched_directories() + tracked.write_text("value = 2\n", encoding="utf-8") + subprocess.run(["git", "add", "tracked.py"], cwd=tmp_path, check=True) + monitor._watched_directories() + monitor._record_change(tmp_path / ".git") + + assert monitor.finish() == CheckoutMutationObservation(changed=True, unavailable=False, observed_path=".git/index") + + @pytest.mark.uses_real_clock("waits for the filesystem watcher to witness a branch-ref replacement") def test_checkout_mutation_monitor_observes_transient_head_ref_change(tmp_path: Path) -> None: subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) From 0aec21d533852de32320c5ffc380e85ff017a18b Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 14 Aug 2026 14:47:25 +0200 Subject: [PATCH 30/31] fix(verify): close native testmon review gaps Reject hard-linked lifecycle and testmon state, bound lifecycle lock waits, and prevent managed child startup from loading checkout sitecustomize. Keep ignored benchmarks out of environment identity and keep untraceable packaging inputs out of dependency-edge requirements.\n\nAdd regression coverage for each contract and for malformed production-verifier diagnostics. --- devtools/testmon_bootstrap.py | 49 ++++++---- devtools/verify.py | 94 ++++++++++++------- .../devtools/test_native_testmon_lifecycle.py | 64 ++++++++++--- tests/unit/devtools/test_testmon_bootstrap.py | 58 +++++++++++- tests/unit/devtools/test_verify.py | 38 +++++++- 5 files changed, 229 insertions(+), 74 deletions(-) diff --git a/devtools/testmon_bootstrap.py b/devtools/testmon_bootstrap.py index 90537dc626..e222dcd3b1 100644 --- a/devtools/testmon_bootstrap.py +++ b/devtools/testmon_bootstrap.py @@ -65,6 +65,10 @@ ) +def _is_ignored_native_testmon_path(relative: str) -> bool: + return relative == "tests/benchmarks" or relative.startswith("tests/benchmarks/") + + @dataclass(frozen=True, slots=True) class NativeTestmonEnvironment: name: str @@ -192,6 +196,8 @@ def _declared_pytest_plugin_names( candidates.add(root / "conftest.py") for path in root.glob("tests/**/*.py"): _ensure_deadline(deadline_monotonic) + if _is_ignored_native_testmon_path(path.relative_to(root).as_posix()): + continue try: source = path.read_text(encoding="utf-8") except OSError: @@ -201,6 +207,8 @@ def _declared_pytest_plugin_names( candidates.add(path) for path in root.glob("tests/**/conftest.py"): _ensure_deadline(deadline_monotonic) + if _is_ignored_native_testmon_path(path.relative_to(root).as_posix()): + continue candidates.add(path) for path in sorted(candidates): _ensure_deadline(deadline_monotonic) @@ -264,13 +272,13 @@ def _active_local_pytest_plugin_paths( continue module_path = Path(*module_name.split(".")) module_file = root / module_path.with_suffix(".py") - if module_file.is_file(): + if module_file.is_file() and not _is_ignored_native_testmon_path(module_file.relative_to(root).as_posix()): paths.add(module_file.relative_to(root).as_posix()) package = root / module_path if (package / "__init__.py").is_file(): for path in package.rglob("*.py"): _ensure_deadline(deadline_monotonic) - if path.is_file(): + if path.is_file() and not _is_ignored_native_testmon_path(path.relative_to(root).as_posix()): paths.add(path.relative_to(root).as_posix()) return paths @@ -289,7 +297,7 @@ def _environment_input_paths( for pattern in patterns: for path in root.glob(pattern): _ensure_deadline(deadline_monotonic) - if path.is_file(): + if path.is_file() and not _is_ignored_native_testmon_path(path.relative_to(root).as_posix()): paths.add(path.relative_to(root).as_posix()) paths.update(_active_local_pytest_plugin_paths(root, deadline_monotonic=deadline_monotonic)) return tuple(sorted(paths)) @@ -461,11 +469,7 @@ def classify_native_testmon_changes(repo_root: Path, paths: Iterable[str]) -> Na without a filename registry. """ normalized = tuple(relative for raw in sorted(set(paths)) if (relative := _safe_relative_path(raw)) is not None) - native_paths = tuple( - relative - for relative in normalized - if relative != "tests/benchmarks" and not relative.startswith("tests/benchmarks/") - ) + native_paths = tuple(relative for relative in normalized if not _is_ignored_native_testmon_path(relative)) runtime_data = tuple( relative for relative in native_paths @@ -473,7 +477,10 @@ def classify_native_testmon_changes(repo_root: Path, paths: Iterable[str]) -> Na or relative.startswith("packaging/") ) return NativeTestmonChangeImpact( - executable_paths=executable_python_paths(repo_root, native_paths), + executable_paths=executable_python_paths( + repo_root, + (relative for relative in native_paths if relative not in runtime_data), + ), runtime_data_paths=runtime_data, ) @@ -509,20 +516,20 @@ def inspect_native_testmon_environment( return NativeTestmonState("invalid", "SQLite sidecars exist without the owned database") return NativeTestmonState("absent", "native testmon database is absent") try: - mode = data_path.lstat().st_mode + state = data_path.lstat() except OSError as exc: return NativeTestmonState("invalid", f"cannot inspect native testmon database: {exc}") - if not stat.S_ISREG(mode): - return NativeTestmonState("invalid", "native testmon database is not a regular file") + if not stat.S_ISREG(state.st_mode) or state.st_nlink != 1: + return NativeTestmonState("invalid", "native testmon database is not a single-link regular file") for sidecar in sidecars: try: - sidecar_mode = sidecar.lstat().st_mode + sidecar_state = sidecar.lstat() except FileNotFoundError: continue except OSError as exc: return NativeTestmonState("invalid", f"cannot inspect native testmon sidecar {sidecar}: {exc}") - if not stat.S_ISREG(sidecar_mode): - return NativeTestmonState("invalid", f"native testmon sidecar is not a regular file: {sidecar}") + if not stat.S_ISREG(sidecar_state.st_mode) or sidecar_state.st_nlink != 1: + return NativeTestmonState("invalid", f"native testmon sidecar is not a single-link regular file: {sidecar}") try: with ( contextlib.closing( @@ -644,13 +651,13 @@ def validate_native_testmon_state_ownership(repo_root: Path) -> None: """Reject parent or file replacement before managed SQLite access.""" for path in _owned_paths(repo_root): try: - mode = path.lstat().st_mode + state = path.lstat() except FileNotFoundError: continue except OSError as exc: raise NativeTestmonRepairError(f"cannot inspect owned testmon path {path}: {exc}") from exc - if not stat.S_ISREG(mode): - raise NativeTestmonRepairError(f"owned testmon path is not a regular file: {path}") + if not stat.S_ISREG(state.st_mode) or state.st_nlink != 1: + raise NativeTestmonRepairError(f"owned testmon path is not a single-link regular file: {path}") def remove_invalid_native_testmon_state(repo_root: Path) -> tuple[Path, ...]: @@ -658,13 +665,15 @@ def remove_invalid_native_testmon_state(repo_root: Path) -> tuple[Path, ...]: removed: list[Path] = [] for path in _owned_paths(repo_root): try: - mode = path.lstat().st_mode + state = path.lstat() except FileNotFoundError: continue except OSError as exc: raise NativeTestmonRepairError(f"cannot inspect owned testmon path {path}: {exc}") from exc - if stat.S_ISDIR(mode): + if stat.S_ISDIR(state.st_mode): raise NativeTestmonRepairError(f"refusing to remove directory at owned SQLite path {path}") + if stat.S_ISREG(state.st_mode) and state.st_nlink != 1: + raise NativeTestmonRepairError(f"refusing to remove hard-linked owned SQLite path {path}") try: path.unlink() except OSError as exc: diff --git a/devtools/verify.py b/devtools/verify.py index 0d72e5b70c..720aa90f0b 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -145,6 +145,7 @@ "--override-ini=norecursedirs=", "tests", ) +NATIVE_TESTMON_LIFECYCLE_LOCK_TIMEOUT_S = 60.0 def _normalize_managed_pytest_environment( @@ -202,7 +203,11 @@ def _open_owned_native_testmon_state(repo_root: Path) -> _OwnedNativeTestmonStat @contextlib.contextmanager -def _native_testmon_lifecycle_lock(repo_root: Path) -> Iterator[None]: +def _native_testmon_lifecycle_lock( + repo_root: Path, + *, + timeout_s: float = NATIVE_TESTMON_LIFECYCLE_LOCK_TIMEOUT_S, +) -> Iterator[None]: """Serialize one checkout's native testmon preparation, lanes, and inspection.""" cache = repo_root.resolve() / ".cache" try: @@ -240,9 +245,13 @@ def _native_testmon_lifecycle_lock(repo_root: Path) -> Iterator[None]: if ( not stat.S_ISREG(opened_lock.st_mode) or not stat.S_ISREG(current_lock.st_mode) + or opened_lock.st_nlink != 1 + or current_lock.st_nlink != 1 or (opened_lock.st_dev, opened_lock.st_ino) != (current_lock.st_dev, current_lock.st_ino) ): - raise NativeTestmonRepairError(f"native testmon lifecycle lock is not an owned regular file: {lock_path}") + raise NativeTestmonRepairError( + f"native testmon lifecycle lock is not an owned single-link regular file: {lock_path}" + ) except OSError as exc: if lock_descriptor is not None: with contextlib.suppress(OSError): @@ -257,14 +266,25 @@ def _native_testmon_lifecycle_lock(repo_root: Path) -> Iterator[None]: assert lock_descriptor is not None with os.fdopen(lock_descriptor, "r+", encoding="utf-8") as handle: lock_descriptor = None - try: - fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) - except OSError: - handle.seek(0) - holder = handle.read().strip() or "another verify invocation" - sys.stderr.write(f"verify: waiting for native testmon lifecycle lock ({holder})\n") - sys.stderr.flush() - fcntl.flock(handle.fileno(), fcntl.LOCK_EX) + deadline = time.monotonic() + max(0.0, timeout_s) + announced_wait = False + while True: + try: + fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except BlockingIOError as exc: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise PytestResourceError( + f"timed out waiting for native testmon lifecycle lock after {timeout_s:.1f}s" + ) from exc + if not announced_wait: + handle.seek(0) + holder = handle.read().strip() or "another verify invocation" + sys.stderr.write(f"verify: waiting for native testmon lifecycle lock ({holder})\n") + sys.stderr.flush() + announced_wait = True + time.sleep(min(0.05, remaining)) handle.seek(0) handle.truncate() handle.write(f"pid={os.getpid()}") @@ -2277,9 +2297,9 @@ def _subprocess_env(*, native_testmon_data: Path | None = None) -> dict[str, str env["GIT_OPTIONAL_LOCKS"] = "0" env["POLYLOGUE_ROOT"] = str(ROOT) env["POLYLOGUE_REPO_ROOT"] = str(ROOT) - # Managed verification owns Python startup. Inherited paths can load a - # sitecustomize module before pytest gets a chance to neutralize addopts. - env["PYTHONPATH"] = str(ROOT) + # Managed verification owns Python startup. A checkout path on PYTHONPATH + # lets sitecustomize alter pytest controls before the managed command runs. + env.pop("PYTHONPATH", None) env.pop("PYTHONOPTIMIZE", None) env.pop("PYTHONHOME", None) env.pop("PYTHONUSERBASE", None) @@ -3447,25 +3467,29 @@ def main(argv: list[str] | None = None) -> int: raw_argv = list(sys.argv[1:] if argv is None else argv) native_pytest_enabled = not any(flag in raw_argv for flag in ("--quick", "--commit", "--history")) lock = _native_testmon_lifecycle_lock(ROOT) if native_pytest_enabled else contextlib.nullcontext() - with lock: - try: - return _main(argv) - except KeyboardInterrupt as exc: - if _ACTIVE_VERIFY_RUN is None: - raise - return _finalize_verify_runner_exception( - _ACTIVE_VERIFY_RUN, - exc, - use_json="--json" in raw_argv, - ) - except Exception as exc: - if _ACTIVE_VERIFY_RUN is None: - raise - return _finalize_verify_runner_exception( - _ACTIVE_VERIFY_RUN, - exc, - use_json="--json" in raw_argv, - ) - finally: - _close_active_native_testmon_state() - _ACTIVE_VERIFY_RUN = None + try: + with lock: + try: + return _main(argv) + except KeyboardInterrupt as exc: + if _ACTIVE_VERIFY_RUN is None: + raise + return _finalize_verify_runner_exception( + _ACTIVE_VERIFY_RUN, + exc, + use_json="--json" in raw_argv, + ) + except Exception as exc: + if _ACTIVE_VERIFY_RUN is None: + raise + return _finalize_verify_runner_exception( + _ACTIVE_VERIFY_RUN, + exc, + use_json="--json" in raw_argv, + ) + finally: + _close_active_native_testmon_state() + _ACTIVE_VERIFY_RUN = None + except PytestResourceError as exc: + sys.stderr.write(f"verify: {exc}\n") + return 125 diff --git a/tests/integration/devtools/test_native_testmon_lifecycle.py b/tests/integration/devtools/test_native_testmon_lifecycle.py index 4626e36b55..6ac254571a 100644 --- a/tests/integration/devtools/test_native_testmon_lifecycle.py +++ b/tests/integration/devtools/test_native_testmon_lifecycle.py @@ -84,8 +84,9 @@ def _commit_all(root: Path, message: str) -> str: def _pytest_environment(repo: Path) -> dict[str, str]: (repo / TESTMON_DATA_RELPATH).parent.mkdir(parents=True, exist_ok=True) env = os.environ.copy() - env.pop("PYTEST_ADDOPTS", None) - env.pop("PYTEST_PLUGINS", None) + for key in tuple(env): + if key.startswith("PYTEST_"): + env.pop(key) env["PYTEST_DISABLE_PLUGIN_AUTOLOAD"] = "1" env["TESTMON_DATAFILE"] = str(repo / TESTMON_DATA_RELPATH) env["PYTHONPATH"] = os.pathsep.join((str(repo), str(PROJECT_ROOT), env.get("PYTHONPATH", ""))) @@ -291,7 +292,13 @@ def fixture_env_for_pytest_step(env, **kwargs): pytest.fail( f"production verify rejected the fixture checkout\npayload:\n{completed.stdout}\nstderr:\n{completed.stderr}" ) - persisted = json.loads(receipt.read_text(encoding="utf-8")) + try: + persisted = json.loads(receipt.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + pytest.fail( + f"production verify wrote invalid receipt JSON ({exc})\n" + f"returncode={completed.returncode}\nstdout:\n{completed.stdout}\nstderr:\n{completed.stderr}" + ) assert persisted["invocation_id"] == invocation_id assert persisted["pytest_aggregate"] == payload["pytest_aggregate"] for authority_field in ("diagnosis", "exit_code", "release_baseline_allowed"): @@ -643,20 +650,12 @@ def test_production_verify_all_drops_pythonpath_startup_injection(tmp_path: Path _git(repo, "remote", "add", "origin", str(origin)) _git(repo, "branch", "-M", "master") _git(repo, "push", "-qu", "origin", "master") - startup_path = tmp_path / "ambient-pythonpath" - startup_path.mkdir() - (startup_path / "sitecustomize.py").write_text( - 'import os\nos.environ["PYTEST_ADDOPTS"] = "-k passes"\n', + (repo / "sitecustomize.py").write_text( + 'import os\nos.environ["PYTEST_ADDOPTS"] = "-k passes"\nos.environ["HYPOTHESIS_PROFILE"] = "narrow"\n', encoding="utf-8", ) - completed, payload = _run_production_verify( - repo, - "--all", - environment_overrides={ - "PYTHONPATH": os.pathsep.join((str(PROJECT_ROOT), str(startup_path))), - }, - ) + completed, payload = _run_production_verify(repo, "--all") assert completed.returncode == 1 assert payload["release_baseline_allowed"] is False @@ -668,6 +667,21 @@ def test_production_verify_all_drops_pythonpath_startup_injection(tmp_path: Path assert "PYTHONPATH startup injection did not narrow execution" in completed.stderr +def test_plain_native_lane_environment_removes_ambient_pytest_variables( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("PYTEST_ADDOPTS", "--collect-only") + monkeypatch.setenv("PYTEST_PLUGINS", "ambient_plugin") + monkeypatch.setenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", "0") + monkeypatch.setenv("PYTEST_CURRENT_TEST", "ambient test identity") + + environment = _pytest_environment(tmp_path) + + assert {key for key in environment if key.startswith("PYTEST_")} == {"PYTEST_DISABLE_PLUGIN_AUTOLOAD"} + assert environment["PYTEST_DISABLE_PLUGIN_AUTOLOAD"] == "1" + + @pytest.mark.parametrize( ("environment_overrides", "interpreter_args", "verify_args"), [ @@ -1429,6 +1443,28 @@ def invalid_json_result(command: list[str], **kwargs: Any) -> subprocess.Complet assert "stderr:\nverifier diagnostics" in str(failure.value) +def test_production_verify_reports_stdout_when_receipt_json_is_invalid( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + def invalid_receipt_result(command: list[str], **kwargs: Any) -> subprocess.CompletedProcess[str]: + environment = kwargs["env"] + assert isinstance(environment, dict) + receipt = Path(environment["POLYLOGUE_VERIFICATION_RECEIPT_PATH"]) + receipt.parent.mkdir(parents=True, exist_ok=True) + receipt.write_text("not JSON", encoding="utf-8") + return subprocess.CompletedProcess(command, 1, "{}", "verifier diagnostics") + + monkeypatch.setattr(subprocess, "run", invalid_receipt_result) + + with pytest.raises(pytest.fail.Exception) as failure: + _run_production_verify(tmp_path) + + assert "production verify wrote invalid receipt JSON" in str(failure.value) + assert "stdout:\n{}" in str(failure.value) + assert "stderr:\nverifier diagnostics" in str(failure.value) + + @pytest.mark.parametrize( "declaration", [ diff --git a/tests/unit/devtools/test_testmon_bootstrap.py b/tests/unit/devtools/test_testmon_bootstrap.py index 35e6aa130b..b98689df8b 100644 --- a/tests/unit/devtools/test_testmon_bootstrap.py +++ b/tests/unit/devtools/test_testmon_bootstrap.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os from pathlib import Path import pytest @@ -14,6 +15,7 @@ inspect_native_testmon_environment, prepare_native_testmon_environment, remove_invalid_native_testmon_state, + validate_native_testmon_state_ownership, ) from devtools.testmon_bootstrap import ( testmon_environment_digest as _testmon_environment_digest, @@ -128,6 +130,9 @@ def test_benchmarks_are_not_required_graph_paths_and_packaging_is_untraceable(tm benchmark = tmp_path / "tests" / "benchmarks" / "test_scale.py" benchmark.parent.mkdir(parents=True) benchmark.write_text("def test_scale(): pass\n", encoding="utf-8") + packaging_python = tmp_path / "packaging" / "hatch_build.py" + packaging_python.parent.mkdir() + packaging_python.write_text("def build_hook(): pass\n", encoding="utf-8") impact = classify_native_testmon_changes( tmp_path, @@ -135,12 +140,13 @@ def test_benchmarks_are_not_required_graph_paths_and_packaging_is_untraceable(tm "tests/benchmarks/test_scale.py", "tests/benchmarks/deleted.py", "packaging/polylogue.nix", + "packaging/hatch_build.py", "docs/release.md", ), ) assert impact.executable_paths == () - assert impact.runtime_data_paths == ("packaging/polylogue.nix",) + assert impact.runtime_data_paths == ("packaging/hatch_build.py", "packaging/polylogue.nix") def test_testmon_schema_matches_the_tested_dependency_contract(tmp_path: Path) -> None: @@ -204,6 +210,17 @@ def test_environment_digest_changes_when_root_conftest_is_added(tmp_path: Path) assert _testmon_environment_digest(tmp_path) != initial +def test_environment_digest_ignores_benchmark_conftest_and_its_plugin_declaration(tmp_path: Path) -> None: + benchmark_conftest = tmp_path / "tests" / "benchmarks" / "conftest.py" + benchmark_conftest.parent.mkdir(parents=True) + benchmark_conftest.write_text("pytest_plugins = dynamic_plugin_names\n", encoding="utf-8") + + initial = _testmon_environment_digest(tmp_path) + benchmark_conftest.write_text("pytest_plugins = other_dynamic_plugin_names\n", encoding="utf-8") + + assert _testmon_environment_digest(tmp_path) == initial + + def test_inactive_runtime_helper_does_not_force_fresh_environment(tmp_path: Path) -> None: helper = tmp_path / "tests" / "infra" / "runtime_helper.py" helper.parent.mkdir(parents=True) @@ -388,7 +405,7 @@ def test_native_inspection_rejects_symlinked_database_before_sqlite_open(tmp_pat state = inspect_native_testmon_environment(data, environment_name="owned-environment") assert state.status == "invalid" - assert state.reason == "native testmon database is not a regular file" + assert state.reason == "native testmon database is not a single-link regular file" assert outside.read_text(encoding="utf-8") == "external state" @@ -403,5 +420,40 @@ def test_native_inspection_rejects_symlinked_sidecar_before_sqlite_open(tmp_path state = inspect_native_testmon_environment(data, environment_name="owned-environment") assert state.status == "invalid" - assert state.reason == f"native testmon sidecar is not a regular file: {sidecar}" + assert state.reason == f"native testmon sidecar is not a single-link regular file: {sidecar}" assert outside.read_text(encoding="utf-8") == "external sidecar" + + +@pytest.mark.parametrize("suffix", ("", "-wal")) +def test_native_testmon_ownership_rejects_hardlinked_database_and_sidecars(tmp_path: Path, suffix: str) -> None: + state_dir = tmp_path / ".cache" / "testmon" + state_dir.mkdir(parents=True) + outside = tmp_path / f"outside{suffix}" + outside.write_text("external state", encoding="utf-8") + owned = state_dir / f"testmondata{suffix}" + os.link(outside, owned) + + with pytest.raises(NativeTestmonRepairError, match="single-link regular file"): + validate_native_testmon_state_ownership(tmp_path) + + assert outside.read_text(encoding="utf-8") == "external state" + + +@pytest.mark.parametrize("suffix", ("", "-wal")) +def test_native_inspection_rejects_hardlinked_database_and_sidecars(tmp_path: Path, suffix: str) -> None: + data = tmp_path / "testmondata" + if suffix: + data.write_text("not opened", encoding="utf-8") + outside = tmp_path / f"outside{suffix}" + outside.write_text("external state", encoding="utf-8") + owned = Path(f"{data}{suffix}") + os.link(outside, owned) + + state = inspect_native_testmon_environment(data, environment_name="owned-environment") + + assert state.status == "invalid" + subject = "database" if not suffix else "sidecar" + assert state.reason == f"native testmon {subject} is not a single-link regular file" + ( + "" if not suffix else f": {owned}" + ) + assert outside.read_text(encoding="utf-8") == "external state" diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index f5eb50c379..147af7967e 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -3154,6 +3154,40 @@ def test_native_testmon_lifecycle_lock_refuses_symlink_without_touching_target(t assert target.read_text(encoding="utf-8") == "preserve this file\n" +def test_native_testmon_lifecycle_lock_refuses_hardlink_without_touching_target(tmp_path: Path) -> None: + cache = tmp_path / ".cache" + cache.mkdir() + target = tmp_path / "outside-checkout-target" + target.write_text("preserve this file\n", encoding="utf-8") + os.link(target, cache / "native-testmon-lifecycle.lock") + + with pytest.raises(NativeTestmonRepairError, match="single-link regular file"): + with verify._native_testmon_lifecycle_lock(tmp_path): + pytest.fail("hard-linked lifecycle lock must not be acquired") + + assert target.read_text(encoding="utf-8") == "preserve this file\n" + + +@pytest.mark.uses_real_clock("proves lifecycle-lock waiters receive a bounded resource refusal") +def test_native_testmon_lifecycle_lock_times_out_while_holder_remains_active(tmp_path: Path) -> None: + holder_entered = threading.Event() + release_holder = threading.Event() + + def hold_lock() -> None: + with verify._native_testmon_lifecycle_lock(tmp_path): + holder_entered.set() + assert release_holder.wait(timeout=2) + + with ThreadPoolExecutor(max_workers=1) as pool: + holder = pool.submit(hold_lock) + assert holder_entered.wait(timeout=2) + with pytest.raises(PytestResourceError, match="timed out waiting for native testmon lifecycle lock"): + with verify._native_testmon_lifecycle_lock(tmp_path, timeout_s=0.01): + pytest.fail("timed-out contender must not acquire the lock") + release_holder.set() + holder.result(timeout=2) + + def test_run_clears_stale_current_statistics_before_an_interrupted_pytest_step(tmp_path: Path) -> None: stale_statistics = tmp_path / verify_runs.CURRENT_STATISTICS_PATH stale_statistics.parent.mkdir(parents=True) @@ -3367,7 +3401,7 @@ def test_run_forces_subprocesses_to_current_checkout(monkeypatch: pytest.MonkeyP assert env["POLYLOGUE_ROOT"] == str(ROOT) assert env["POLYLOGUE_REPO_ROOT"] == str(ROOT) assert env["PYTHONPYCACHEPREFIX"] == str(ROOT / ".cache" / "pycache") - assert env["PYTHONPATH"] == str(ROOT) + assert "PYTHONPATH" not in env assert "PYTHONOPTIMIZE" not in env assert "PYTHONHOME" not in env assert "PYTHONUSERBASE" not in env @@ -3385,7 +3419,7 @@ def test_mypy_probe_uses_managed_python_startup_environment(monkeypatch: pytest. assert run.call_args.kwargs["cwd"] == ROOT env = run.call_args.kwargs["env"] - assert env["PYTHONPATH"] == str(ROOT) + assert "PYTHONPATH" not in env assert "PYTHONOPTIMIZE" not in env From 38b0c8fd2015c3557895fb7d1ca02a4f1058b8c7 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 14 Aug 2026 16:08:10 +0200 Subject: [PATCH 31/31] fix(testmon): classify pure enum contracts as runtime data Problem: pytest collection imports pure enum contract modules before testmon tracing, so changed modules can appear absent from the native dependency graph even when the safe full-corpus route is selected.\n\nWhat changed: classify pure Enum, IntEnum, and StrEnum declaration modules as non-traceable runtime inputs, while retaining fail-closed treatment for executable collection-only modules. Add regression coverage for the verification scope contract shape.\n\nVerification: focused bootstrap/lifecycle tests and devtools verify --quick. --- devtools/testmon_bootstrap.py | 37 +++++++++++++++++++ tests/unit/devtools/test_testmon_bootstrap.py | 18 +++++++++ 2 files changed, 55 insertions(+) diff --git a/devtools/testmon_bootstrap.py b/devtools/testmon_bootstrap.py index e222dcd3b1..d8204d5d7e 100644 --- a/devtools/testmon_bootstrap.py +++ b/devtools/testmon_bootstrap.py @@ -386,6 +386,18 @@ def _body_is_executable(body: list[ast.stmt]) -> bool: # Imports elsewhere execute at module import time and therefore # remain executable graph inputs. continue + if ( + isinstance(node, ast.ImportFrom) + and node.module == "__future__" + and all(alias.name == "annotations" for alias in node.names) + ): + continue + if ( + isinstance(node, ast.ImportFrom) + and node.module == "enum" + and all(alias.name in {"Enum", "IntEnum", "StrEnum"} for alias in node.names) + ): + continue if ( isinstance(node, ast.Expr) and isinstance(node.value, ast.Constant) @@ -407,6 +419,8 @@ def _body_is_executable(body: list[ast.stmt]) -> bool: return True continue if isinstance(node, ast.ClassDef): + if _is_pure_enum_declaration(node): + continue if node.decorator_list or node.bases or node.keywords or _body_is_executable(node.body): return True continue @@ -420,6 +434,24 @@ def _body_is_executable(body: list[ast.stmt]) -> bool: return False +def _is_pure_enum_declaration(node: ast.ClassDef) -> bool: + """Recognize enum value declarations that tracing cannot observe usefully.""" + bases = {base.id for base in node.bases if isinstance(base, ast.Name)} + if not bases.intersection({"Enum", "IntEnum", "StrEnum"}): + return False + for member in node.body: + if ( + isinstance(member, ast.Expr) + and isinstance(member.value, ast.Constant) + and isinstance(member.value.value, str) + ): + continue + if isinstance(member, (ast.Assign, ast.AnnAssign, ast.Pass)): + continue + return False + return True + + def classify_source_ast(source_path: Path) -> ASTClassification: """Classify whether a module contains executable runtime behavior.""" try: @@ -474,6 +506,11 @@ def classify_native_testmon_changes(repo_root: Path, paths: Iterable[str]) -> Na relative for relative in native_paths if (not relative.endswith(".py") and relative.startswith(("polylogue/", "tests/"))) + or ( + relative.endswith(".py") + and relative.startswith(("devtools/", "polylogue/", "tests/")) + and classify_source_ast(repo_root / relative) == "declaration-only" + ) or relative.startswith("packaging/") ) return NativeTestmonChangeImpact( diff --git a/tests/unit/devtools/test_testmon_bootstrap.py b/tests/unit/devtools/test_testmon_bootstrap.py index b98689df8b..bee477e725 100644 --- a/tests/unit/devtools/test_testmon_bootstrap.py +++ b/tests/unit/devtools/test_testmon_bootstrap.py @@ -62,6 +62,24 @@ def test_ast_classification_treats_ordinary_imports_as_executable(tmp_path: Path assert classify_source_ast(module) == "executable" +def test_pure_enum_contracts_are_non_traceable_runtime_inputs(tmp_path: Path) -> None: + module = tmp_path / "devtools" / "verification_contracts.py" + module.parent.mkdir() + module.write_text( + "from enum import StrEnum\n\n" + "class VerificationScope(StrEnum):\n" + " AFFECTED = 'affected'\n" + " RELEASE_BASELINE = 'release-baseline'\n", + encoding="utf-8", + ) + + assert classify_source_ast(module) == "declaration-only" + impact = classify_native_testmon_changes(tmp_path, ("devtools/verification_contracts.py",)) + + assert impact.executable_paths == () + assert impact.runtime_data_paths == ("devtools/verification_contracts.py",) + + def test_executable_paths_require_current_runtime_modules_and_deleted_modules(tmp_path: Path) -> None: module = tmp_path / "polylogue" / "runtime.py" module.parent.mkdir()