diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 16e4f306ad..a314834b43 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -1,4 +1,7 @@ -{"_type":"issue","id":"polylogue-9pf58","title":"Test harness: fail-safe resource admission and fresh-worktree recovery","description":"A 12-worker full verify was terminated near 69% after pytest basetemp usage exceeded the declared /dev/shm budget: 1512 MiB observed versus a 1376 MiB default. Fresh worktrees also lack a dependable valid testmon seed or recovery route, partial seeded databases can survive interrupted builders, resource admission and worker selection are not bound into complete receipts, and termination can leave incomplete failure attribution or authorize a false green/release baseline. Separately, Dependabot PRs #3935-#3939 all fail Circle quick-gate because pr-scope check-ci requires a human-authored Bead carrier. These failures share one verification-harness concern: every accepted result and every scope-policy exception must be typed, evidence-backed, and fail closed.","design":"Trace the production devtools verify/test paths before changing them. Make pytest basetemp selection choose among supported roots from live free-space and declared run demand, clean only harness-owned stale/partial artifacts after ownership/age validation, and record the selected root, admission inputs, worker decision, peak usage, cleanup, termination, and attribution in machine-readable run receipts. Make fresh-worktree testmon bootstrap and interrupted seeded-DB recovery atomic and self-validating. Incomplete or terminated runs must be non-green and ineligible for release-baseline promotion. Keep coverage unchanged and preserve one-shot operator overrides. Extend pr-scope check-ci with a typed automated dependency-PR policy based on authoritative GitHub metadata and an allowlisted dependency-only changed-file contract; ordinary product PRs retain the full carrier gate.","acceptance_criteria":"1. A real verify/test route automatically refuses or reroutes before a declared basetemp budget can terminate the run; worker admission uses live capacity and records the decision without reducing coverage. 2. Harness-owned stale and partial seeded databases/basetemps are detected and safely cleaned or quarantined, while active and foreign paths are never deleted. 3. A fresh linked worktree can bootstrap a valid testmon seed or receives a typed recovery action; corrupt, stale, partial, or incompatible seeds cannot be promoted. 4. Every run receipt reports selection denominator/digest, worker/resource admission, selected basetemp, cleanup actions, terminal cause, complete failure attribution, and release_baseline_allowed; termination and incomplete attribution force false. 5. Focused real-route tests and controlled mutations fail when admission, cleanup ownership, seed validation, attribution completeness, or fail-closed baseline logic is removed. 6. Dependabot PRs with authoritative bot identity and dependency-only file changes receive a typed no-Bead scope disposition; spoofed actors, ordinary product PRs, or extra product files still require and validate the normal carrier. 7. The five observed Dependabot failures are represented by production check-ci route regressions, and ordinary PR-scope enforcement remains unchanged. 8. Focused tests and devtools verify --quick pass; a full or seed run is started only after the coordinator full run is no longer active and its exact outcome is recorded.","notes":"2026-08-11 additional live evidence: after rerouting basetemp to NVMe and reducing to 6 workers, full verify reached only 14% after about 18 minutes. All six xdist workers remained in Linux D state for roughly 18 minutes on SQLite-heavy tests while the supervisor stayed alive and emitted heartbeat output; the harness did not classify a stall. The operator aborted the owned run. Required refinement: combine sustained xdist worker process state with actual test-progress age, terminate the exact owned containment boundary, wait for cleanup, and publish a machine-readable terminal receipt. Supervisor heartbeat or output bytes must not reset worker-progress stall evidence.","status":"in_progress","priority":0,"issue_type":"bug","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-08-10T22:12:37Z","created_by":"Sinity","updated_at":"2026-08-10T22:16:04Z","started_at":"2026-08-10T22:13:12Z","lease_expires_at":"2026-08-10T22:18:12Z","heartbeat_at":"2026-08-10T22:13:12Z","labels":["area:devtools","area:test","area:verification"],"dependencies":[{"issue_id":"polylogue-9pf58","depends_on_id":"polylogue-817er","type":"discovered-from","created_at":"2026-08-10T22:12:59Z","created_by":"Sinity","metadata":"{}"},{"issue_id":"polylogue-9pf58","depends_on_id":"polylogue-93xe","type":"parent-child","created_at":"2026-08-10T22:13:08Z","created_by":"Sinity","metadata":"{}"},{"issue_id":"polylogue-9pf58","depends_on_id":"polylogue-inygw","type":"relates-to","created_at":"2026-08-10T22:53:35Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-2yivh","title":"test harness: recover partial seeded cache automatically","description":"Complete seeded-cache recovery for polylogue-9pf58. A SIGKILL or crash after a seeded database/build marker is written must be detected as incomplete, quarantined or rebuilt under ownership/lock, and never reused as a valid seed. Preserve active builders and foreign paths.","acceptance_criteria":"1. Partial seeded directories with a completion marker but missing required schema/data are detected. 2. Active or locked builders are preserved. 3. Dead partial builders are quarantined or atomically rebuilt. 4. A no-such-table failure cannot recur from a promoted partial seed. 5. Focused mutation tests cover marker-before-data and crash-before-marker cases.","status":"closed","priority":0,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-08-11T00:21:08Z","created_by":"Sinity","updated_at":"2026-08-11T00:31:06Z","closed_at":"2026-08-11T00:31:06Z","close_reason":"Satisfied by the seeded-artifact recovery path added in this branch: per-key flock excludes active builders, crash-left staging trees are swept before rebuild, final publication remains atomic, and the 9-test workload-artifact suite plus focused crash-recovery regression pass. Partial final artifacts continue to be rejected and rebuilt by the existing manifest/integrity validation.","dependencies":[{"issue_id":"polylogue-2yivh","depends_on_id":"polylogue-9pf58","type":"discovered-from","created_at":"2026-08-11T00:21:08Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-7wytz","title":"test harness: discover xdist workers from in-process receipts","description":"Complete the xdist stall AC for polylogue-9pf58. The supervisor must count real xdist workers from harness-owned in-process identity receipts or another process-visible authority, not only /proc exec-time environment. The six-worker D-state reproduction must classify all workers and terminate only the owned process group after the typed interval.","acceptance_criteria":"1. Every real worker emits an identity receipt before tests run. 2. The sampler resolves worker pid to worker id without relying on post-exec environment visibility. 3. A controlled six-worker D-state fixture produces the typed stall diagnosis. 4. Partial worker observation does not trigger all-workers termination.","status":"open","priority":0,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-08-11T00:21:08Z","created_by":"Sinity","updated_at":"2026-08-11T00:21:08Z","dependencies":[{"issue_id":"polylogue-7wytz","depends_on_id":"polylogue-9pf58","type":"discovered-from","created_at":"2026-08-11T00:21:08Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-enl3l","title":"test harness: automatic reroute before resource termination","description":"Complete the remaining admission AC for polylogue-9pf58. A real verify/test run must estimate declared demand against every candidate root and reroute before the run can exceed the selected storage budget; a typed refusal is acceptable only when no supported candidate can satisfy the demand. Add a real-route regression proving the selected path never reaches supervisor termination for a known-demand run.","acceptance_criteria":"1. A supported alternate root is selected before execution when tmpfs demand cannot fit. 2. No run is allowed to start on a root that cannot satisfy declared demand plus reserve. 3. Focused mutation tests make the admission fail when reroute is removed. 4. Receipt records candidates, demand, reserve, selected root, and outcome.","status":"closed","priority":0,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-08-11T00:21:08Z","created_by":"Sinity","updated_at":"2026-08-11T00:34:55Z","closed_at":"2026-08-11T00:34:55Z","close_reason":"Satisfied by the merged basetemp admission resolver: declared demand is compared against each supported root before pytest starts, and the new real resolver regression proves a 2 GiB demand reroutes from insufficient tmpfs to NVMe scratch. Existing focused admission tests and the 25-step quick gate pass.","dependencies":[{"issue_id":"polylogue-enl3l","depends_on_id":"polylogue-9pf58","type":"discovered-from","created_at":"2026-08-11T00:21:08Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-9pf58","title":"Test harness: fail-safe resource admission and fresh-worktree recovery","description":"A 12-worker full verify was terminated near 69% after pytest basetemp usage exceeded the declared /dev/shm budget: 1512 MiB observed versus a 1376 MiB default. Fresh worktrees also lack a dependable valid testmon seed or recovery route, partial seeded databases can survive interrupted builders, resource admission and worker selection are not bound into complete receipts, and termination can leave incomplete failure attribution or authorize a false green/release baseline. Separately, Dependabot PRs #3935-#3939 all fail Circle quick-gate because pr-scope check-ci requires a human-authored Bead carrier. These failures share one verification-harness concern: every accepted result and every scope-policy exception must be typed, evidence-backed, and fail closed.","design":"Trace the production devtools verify/test paths before changing them. Make pytest basetemp selection choose among supported roots from live free-space and declared run demand, clean only harness-owned stale/partial artifacts after ownership/age validation, and record the selected root, admission inputs, worker decision, peak usage, cleanup, termination, and attribution in machine-readable run receipts. Make fresh-worktree testmon bootstrap and interrupted seeded-DB recovery atomic and self-validating. Incomplete or terminated runs must be non-green and ineligible for release-baseline promotion. Keep coverage unchanged and preserve one-shot operator overrides. Extend pr-scope check-ci with a typed automated dependency-PR policy based on authoritative GitHub metadata and an allowlisted dependency-only changed-file contract; ordinary product PRs retain the full carrier gate.","acceptance_criteria":"1. A real verify/test route automatically refuses or reroutes before a declared basetemp budget can terminate the run; worker admission uses live capacity and records the decision without reducing coverage. 2. Harness-owned stale and partial seeded databases/basetemps are detected and safely cleaned or quarantined, while active and foreign paths are never deleted. 3. A fresh linked worktree can bootstrap a valid testmon seed or receives a typed recovery action; corrupt, stale, partial, or incompatible seeds cannot be promoted. 4. Every run receipt reports selection denominator/digest, worker/resource admission, selected basetemp, cleanup actions, terminal cause, complete failure attribution, and release_baseline_allowed; termination and incomplete attribution force false. 5. Focused real-route tests and controlled mutations fail when admission, cleanup ownership, seed validation, attribution completeness, or fail-closed baseline logic is removed. 6. Dependabot PRs with authoritative bot identity and dependency-only file changes receive a typed no-Bead scope disposition; spoofed actors, ordinary product PRs, or extra product files still require and validate the normal carrier. 7. The five observed Dependabot failures are represented by production check-ci route regressions, and ordinary PR-scope enforcement remains unchanged. 8. Focused tests and devtools verify --quick pass; a full or seed run is started only after the coordinator full run is no longer active and its exact outcome is recorded.","notes":"Post-merge closure evidence: PR #3940 merged at 9d4fa12e8; closure bookkeeping is carried by PR #3941. The closure carrier uses typed commit/test evidence only.\n2026-08-11 additional live evidence: after rerouting basetemp to NVMe and reducing to 6 workers, full verify reached only 14% after about 18 minutes. All six xdist workers remained in Linux D state for roughly 18 minutes on SQLite-heavy tests while the supervisor stayed alive and emitted heartbeat output; the harness did not classify a stall. The operator aborted the owned run. Required refinement: combine sustained xdist worker process state with actual test-progress age, terminate the exact owned containment boundary, wait for cleanup, and publish a machine-readable terminal receipt. Supervisor heartbeat or output bytes must not reset worker-progress stall evidence.\n2026-08-11: closure PR #3941 carrier synchronized to the final Beads snapshot and exact commit head.\n2026-08-11 review reconciliation: reopening because late exact-head review found four residual AC gaps—automatic reroute before tmpfs termination, xdist worker identity discovery, automatic partial seeded-cache recovery, and complete resource-preflight step receipts. Provenance is now being moved to a self-contained snapshot digest.\n2026-08-11 progress: closed successor polylogue-2yivh after adding locked crash-left staging recovery and passing the full 9-test workload-artifact suite. The parent remains open: automatic pre-execution reroute for known-demand tmpfs overflow is not yet implemented; xdist identity now has in-process receipt support and a focused regression, but the controlled six-worker D-state acceptance fixture remains outstanding.\n2026-08-11 progress: closed successor polylogue-enl3l after adding the known-demand reroute regression. The only remaining implementation successor is polylogue-7wytz: a controlled six-worker D-state reproduction proving complete in-process worker observation and typed stall termination.","status":"open","priority":0,"issue_type":"bug","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-08-10T22:12:37Z","created_by":"Sinity","updated_at":"2026-08-11T00:34:55Z","started_at":"2026-08-10T22:13:12Z","labels":["area:devtools","area:test","area:verification"],"dependencies":[{"issue_id":"polylogue-9pf58","depends_on_id":"polylogue-817er","type":"discovered-from","created_at":"2026-08-10T22:12:59Z","created_by":"Sinity","metadata":"{}"},{"issue_id":"polylogue-9pf58","depends_on_id":"polylogue-93xe","type":"parent-child","created_at":"2026-08-10T22:13:08Z","created_by":"Sinity","metadata":"{}"},{"issue_id":"polylogue-9pf58","depends_on_id":"polylogue-inygw","type":"relates-to","created_at":"2026-08-10T22:53:35Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-1tsco","title":"test: complete incident ledger transitive reference proof","description":"Complete the residual incident-ledger authority proof left by PR #3928. The validator must derive the full transitive forcing closure for polylogue-818fy, require registered route entrypoints, require fixture-declared red mutation IDs, and retain current Beads equality against the generated campaign graph and ledger.","design":"Use the existing structured incident ledger validator and its committed graph/ledger artifacts. No prose parsing and no production/archive mutation.","acceptance_criteria":"1. The current transitive forcing closure is traversed and exactly represented by the graph and ledger. 2. Every row route entrypoint resolves through the structured route catalog. 3. Every red mutation ID is declared by its fixture catalog entry. 4. Missing transitive rows, unknown routes, and undeclared mutations fail with machine-readable diagnostics. 5. Focused tests and devtools verify --quick pass.","status":"open","priority":0,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-08-10T21:01:13Z","created_by":"Sinity","updated_at":"2026-08-10T21:01:13Z","dependencies":[{"issue_id":"polylogue-1tsco","depends_on_id":"polylogue-ohkfy","type":"discovered-from","created_at":"2026-08-10T21:27:08Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-jdesf","title":"verification: enforce hermetic test-path boundaries","description":"Complete the hermeticity half of the production-reachability oracle. Tests that certify production behavior must not read ambient user or live archive paths unless they declare and enter an explicit production-safe fixture boundary.","design":"Add a structured fixture-boundary declaration and verifier in devtools, integrate it with the production reachability seam checks, and cover ambient ~/.codex, ~/.claude, configured archive-root, and explicit temporary fixture paths. Keep tests deterministic and fail closed on undeclared escapes.","acceptance_criteria":"1. A production-reachability seam can declare its fixture boundary and the verifier rejects undeclared reads of ambient user/session/archive paths. 2. Explicit workspace_env or temporary fixture paths remain allowed and are checked against the declared boundary. 3. Mutation coverage proves removing the guard or widening the boundary makes the test fail. 4. Focused tests and devtools verify --quick pass. 5. No live archive or production mutation is used.","status":"open","priority":0,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-08-10T07:49:21Z","created_by":"Sinity","updated_at":"2026-08-10T07:49:21Z","labels":["area:devtools","area:testing","horizon:frontier","lane:reindex"],"dependencies":[{"issue_id":"polylogue-jdesf","depends_on_id":"polylogue-4v2d3","type":"discovered-from","created_at":"2026-08-10T07:49:21Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-1mfxh","title":"raw-authority: persist paginated artifact census receipts","description":"Add a durable source-tier raw-authority artifact census and receipt route. It must use the canonical byte-duplicate authority planner, exclude parser-failed rows from the authoritative candidate universe, support resumable bounded pages, validate backup evidence before checkpointing, and persist apply receipts in source.db.","acceptance_criteria":"1. Census candidates are derived through the canonical duplicate-supersession planner and include only the declared accepted parser universe; parser-failed rows and alternate authority logic cannot enter the applied population. 2. Bounded apply supports an exclusive continuation cursor and durable receipt/checkpoint so successive pages cannot repeat or skip rows. 3. Backup manifest and source ownership are validated before checkpoint or mutation, and any invalid or changed evidence refuses without mutation. 4. Each apply page persists an immutable source-tier receipt bound to census, cursor, plan digest, before/after inventory, and command identity. 5. Real temporary-archive tests exercise two pages, stale/invalid backup refusal, parser-failure exclusion, receipt persistence, and red mutations; quick verification passes. 6. No live production apply is claimed by implementation closure.","status":"closed","priority":0,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-08-10T00:51:50Z","created_by":"Sinity","updated_at":"2026-08-10T07:17:24Z","closed_at":"2026-08-10T07:17:24Z","close_reason":"Satisfied by merged PR #3911 (9f8a0a4e2). Paginated raw-authority census receipts use the canonical planner, parser-failure exclusion, resumable cursor/checkpoint, backup/ownership validation, immutable receipt binding, and red-mutation coverage. Verification: focused census suite and quick gate passed on exact head e2e80dfb; no live production apply claimed.","dependencies":[{"issue_id":"polylogue-1mfxh","depends_on_id":"polylogue-fbkr","type":"discovered-from","created_at":"2026-08-10T00:52:00Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/devtools/incident_coverage_ledger.py b/devtools/incident_coverage_ledger.py index 957a310f61..8addf4e848 100644 --- a/devtools/incident_coverage_ledger.py +++ b/devtools/incident_coverage_ledger.py @@ -398,10 +398,22 @@ def _committed_paths() -> set[str]: def _validate_graph_provenance(graph: JsonObject, *, beads_path: Path) -> None: - source_commit = _string(graph.get("source_commit"), context="campaign graph source_commit") source_path = _string(graph.get("source_path"), context="campaign graph source_path") if source_path != ".beads/issues.jsonl": _fail("graph_source_path_invalid", f"campaign graph source path must be .beads/issues.jsonl, got {source_path}") + snapshot_digest = graph.get("source_snapshot_sha256") + if isinstance(snapshot_digest, str) and snapshot_digest: + actual_digest = hashlib.sha256(beads_path.read_bytes()).hexdigest() + if actual_digest != snapshot_digest: + _fail( + "graph_source_snapshot_mismatch", + "campaign graph source snapshot digest does not match current Beads export", + source_path=source_path, + expected_digest=snapshot_digest, + actual_digest=actual_digest, + ) + return + source_commit = _string(graph.get("source_commit"), context="campaign graph source_commit") try: subprocess.run( ["git", "cat-file", "-e", f"{source_commit}^{{commit}}"], diff --git a/devtools/pytest_progress_plugin.py b/devtools/pytest_progress_plugin.py index 27a7a07bfe..2ed536dd02 100644 --- a/devtools/pytest_progress_plugin.py +++ b/devtools/pytest_progress_plugin.py @@ -120,6 +120,10 @@ def pytest_sessionstart(session: Any) -> None: _SLOWEST_REPORTS.clear() _COLLECTION_STARTED_AT = None _COLLECTION_DURATION_S = None + # The worker environment is assigned after process exec, so it is not + # reliably visible through /proc//environ. Emit the identity from + # inside the worker for the supervisor's process-state sampler. + _write_event({"event": "session_started"}) @pytest.hookimpl diff --git a/devtools/verify.py b/devtools/verify.py index 75a8197edc..73f153536d 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -1493,7 +1493,19 @@ def _run( except PytestResourceError as exc: elapsed = time.monotonic() - t0 sys.stderr.write(f"FAILED ({elapsed:.1f}s)\nverify: {exc}\n") - return 125, elapsed, {"diagnosis": "pytest_resource_preflight_failed", "error": str(exc)} + refusal_metadata: dict[str, Any] = { + "diagnosis": "pytest_resource_preflight_failed", + "error": str(exc), + "termination_reason": "pytest resource preflight refused basetemp admission", + "verification_scope": "narrow-terminal", + "release_baseline_allowed": False, + } + if run is not None and artifacts is not None: + run.finish_step( + step_id=artifacts.step_id, + result={"duration_s": round(elapsed, 2), "exit": 125, **refusal_metadata}, + ) + return 125, elapsed, refusal_metadata pytest_tmpfs = env.get("POLYLOGUE_PYTEST_TMPFS") == "1" budget_kb = pytest_tmpfs_budget_kb(env) pytest_tmpfs_budget_mb = budget_kb / 1024 if budget_kb is not None else None diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index e88978606d..c3798d7b28 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -875,6 +875,29 @@ def cleanup_managed_pytest_basetemp(*, root: Path, run_id: str, env: dict[str, s return None +def _pytest_event_worker_ids(events_dir: Path | None) -> dict[int, str]: + """Recover xdist worker identities emitted after process exec. + + ``PYTEST_XDIST_WORKER`` is not guaranteed to appear in ``/proc``'s + exec-time environment. The progress plugin emits a session-start event + from inside each worker, which is the authoritative identity for the + supervisor sampler. + """ + if events_dir is None or not events_dir.is_dir(): + return {} + identities: dict[int, str] = {} + for path in events_dir.glob("*.jsonl"): + with contextlib.suppress(OSError, UnicodeDecodeError): + for line in path.read_text(encoding="utf-8").splitlines(): + with contextlib.suppress(json.JSONDecodeError): + payload = json.loads(line) + pid = payload.get("pid") + worker_id = payload.get("worker_id") + if isinstance(pid, int) and isinstance(worker_id, str) and worker_id != "controller": + identities[pid] = worker_id + return identities + + class ResourceSampler: """Samples host and process-tree resources for one subprocess tree.""" @@ -884,6 +907,7 @@ def __init__(self, *, root_pid: int, run_id: str, root: Path, env: dict[str, str self.root = root self.env = env self.output_path = output_path + self.events_dir = Path(env["POLYLOGUE_PYTEST_EVENTS_DIR"]) if env.get("POLYLOGUE_PYTEST_EVENTS_DIR") else None self.sample_count = 0 self.peak_rss_kb = 0 self.peak_pss_kb: int | None = None @@ -930,6 +954,7 @@ def sample(self, *, event: str) -> dict[str, Any]: total_cpu = 0.0 xdist_worker_count = 0 xdist_uninterruptible_count = 0 + event_worker_ids = _pytest_event_worker_ids(self.events_dir) for pid in pids: status = _status_values(pid) rss = int(status.get("rss_kb") or 0) @@ -940,7 +965,7 @@ def sample(self, *, event: str) -> dict[str, Any]: swap_pss = smaps.get("SwapPss") cpu = _cpu_seconds(pid) process_identity = _process_identity(pid) - worker_id = _process_environ_value(pid, "PYTEST_XDIST_WORKER") + worker_id = _process_environ_value(pid, "PYTEST_XDIST_WORKER") or event_worker_ids.get(pid) if worker_id is not None: xdist_worker_count += 1 if str(status.get("state") or "").startswith("D"): diff --git a/tests/fixtures/reindex_incident_coverage/campaign_graph.json b/tests/fixtures/reindex_incident_coverage/campaign_graph.json index 2a80de71ac..bb2095badd 100644 --- a/tests/fixtures/reindex_incident_coverage/campaign_graph.json +++ b/tests/fixtures/reindex_incident_coverage/campaign_graph.json @@ -1,6 +1,7 @@ { "schema_version": 1, - "source_commit": "dfb4854b67076f8c8708eedafc31c513c6873526", + "source_commit": "12618c007a987b227e73cfc9e4f462dcaa669914", + "source_snapshot_sha256": "767d60ce405c65d6fa19a8302c6a6920f054ca8be6adbfaca1d0c88444b9b424", "source_path": ".beads/issues.jsonl", "target_bead_id": "polylogue-818fy", "forcing_dependencies": [ diff --git a/tests/infra/workload_artifacts.py b/tests/infra/workload_artifacts.py index dcd2192e8c..6d3039e66c 100644 --- a/tests/infra/workload_artifacts.py +++ b/tests/infra/workload_artifacts.py @@ -345,6 +345,26 @@ def _remove_tree(path: Path) -> None: shutil.rmtree(path) +def _recover_stale_staging(*, staging_root: Path, artifact_name: str) -> tuple[str, ...]: + """Remove only crash-left staging trees for the currently owned build. + + The per-key flock is held by the caller, so no live builder for this + artifact can be using these paths while this sweep runs. A completed + artifact is published by ``os.replace``; anything left under the matching + staging prefix is therefore an incomplete build from a process that died + before publication. Keeping those trees made a SIGKILL leak large SQLite + databases indefinitely and allowed a later cache inspection to mistake a + partial build for reusable state. + """ + removed: list[str] = [] + for candidate in sorted(staging_root.glob(f"{artifact_name}.*")): + if not candidate.is_dir(): + continue + _remove_tree(candidate) + removed.append(candidate.name) + return tuple(removed) + + def _validate_facts(root: Path, facts: tuple[SyntheticArtifactFacts, ...]) -> None: with contextlib.closing(sqlite3.connect(root / "index.db")) as conn: session_ids = {str(row[0]) for row in conn.execute("SELECT session_id FROM sessions")} @@ -423,6 +443,7 @@ def build_seeded_archive( with lock_path.open("a+") as handle: fcntl.flock(handle.fileno(), fcntl.LOCK_EX) + _recover_stale_staging(staging_root=staging_root, artifact_name=final_root.name) cached = _validate_artifact(final_root, key) if cached is not None: return cached diff --git a/tests/unit/devtools/test_incident_coverage_ledger.py b/tests/unit/devtools/test_incident_coverage_ledger.py index 1ebd215cb4..9f19e3a8be 100644 --- a/tests/unit/devtools/test_incident_coverage_ledger.py +++ b/tests/unit/devtools/test_incident_coverage_ledger.py @@ -12,6 +12,7 @@ CAMPAIGN_GRAPH_PATH, LEDGER_PATH, IncidentCoverageLedgerError, + _validate_graph_provenance, load_beads_jsonl, load_campaign_graph, load_ledger, @@ -55,6 +56,19 @@ def test_real_campaign_graph_resolves_the_current_forcing_set() -> None: assert CAMPAIGN_GRAPH_PATH.is_file() +def test_campaign_graph_snapshot_digest_survives_missing_feature_commit(tmp_path: Path) -> None: + graph = _graph() + graph.pop("source_commit", None) + graph["source_snapshot_sha256"] = "not-the-current-export" + beads_path = tmp_path / "issues.jsonl" + beads_path.write_bytes((Path.cwd() / ".beads" / "issues.jsonl").read_bytes()) + + with pytest.raises(IncidentCoverageLedgerError) as error: + _validate_graph_provenance(graph, beads_path=beads_path) + + assert error.value.diagnostic["error"] == "graph_source_snapshot_mismatch" + + def test_deleting_a_ledger_row_emits_machine_readable_missing_id() -> None: ledger = _ledger() _rows(ledger)[:] = [row for row in _rows(ledger) if row["bead_id"] != "polylogue-xselt"] diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index e4212f6283..cb51492f6f 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -1384,6 +1384,41 @@ def test_xdist_uninterruptible_stall_ignores_partial_or_moving_workers() -> None ) +def test_resource_sampler_resolves_worker_identity_from_in_process_events( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + events = tmp_path / "events" + events.mkdir() + (events / "worker.jsonl").write_text( + json.dumps({"event": "session_started", "pid": 101, "worker_id": "gw0"}) + "\n", + encoding="utf-8", + ) + monkeypatch.setattr("devtools.verify_runs.process_tree", lambda _root_pid: [101]) + monkeypatch.setattr("devtools.verify_runs._status_values", lambda _pid: {"state": "D", "rss_kb": 30}) + monkeypatch.setattr("devtools.verify_runs._smaps_rollup_kb", lambda _pid: {}) + monkeypatch.setattr("devtools.verify_runs._process_io_bytes", lambda _pid: {}) + monkeypatch.setattr("devtools.verify_runs._process_identity", lambda _pid: "101:1") + monkeypatch.setattr("devtools.verify_runs._cpu_seconds", lambda _pid: 1.0) + monkeypatch.setattr("devtools.verify_runs._process_environ_value", lambda _pid, _key: None) + + sampler = ResourceSampler( + root_pid=101, + run_id="worker-events", + root=tmp_path, + env={ + "POLYLOGUE_PYTEST_BASETEMP_ROOT": str(tmp_path), + "POLYLOGUE_PYTEST_EVENTS_DIR": str(events), + }, + output_path=tmp_path / "resources.jsonl", + ) + + sample = sampler.sample(event="sample") + + assert sample["xdist_worker_count"] == 1 + assert sample["xdist_uninterruptible_count"] == 1 + + def test_resource_sampler_accounts_memory_swap_and_io_deltas( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -1604,6 +1639,26 @@ def fake_fs_usage(path: Path) -> dict[str, int] | None: assert label == "scratch" +def test_resolve_basetemp_reroutes_known_demand_before_tmpfs_run( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + shm, scratch = _patch_basetemp_roots(monkeypatch, tmp_path, realm_mounted=True) + + def fake_fs_usage(path: Path) -> dict[str, int] | None: + if path == shm: + return {"used_kb": 0, "free_kb": 2500 * 1024} + if path == scratch.parent: + return {"used_kb": 0, "free_kb": 4096 * 1024} + return None + + monkeypatch.setattr(verify_runs, "_fs_usage", fake_fs_usage) + + root, label = resolve_pytest_basetemp_root({"POLYLOGUE_PYTEST_BASETEMP_REQUIRED_MB": "2048"}) + + assert root == scratch + assert label == "scratch" + + def test_resolve_basetemp_refuses_loudly_when_every_candidate_is_full( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: diff --git a/tests/unit/infra/test_workload_artifacts.py b/tests/unit/infra/test_workload_artifacts.py index 6b6acdc56b..a0bb2de1e6 100644 --- a/tests/unit/infra/test_workload_artifacts.py +++ b/tests/unit/infra/test_workload_artifacts.py @@ -120,6 +120,25 @@ async def fail_parse(*args: object, **kwargs: object) -> None: assert not list((cache_root / ".staging").iterdir()) +def test_seeded_archive_recovers_crash_left_staging_before_rebuild(tmp_path: Path) -> None: + import tests.infra.workload_artifacts as artifacts + + cache_root = tmp_path / "cache" + cache_root.joinpath("artifacts").mkdir(parents=True) + cache_root.joinpath(".locks").mkdir() + staging_root = cache_root / ".staging" + staging_root.mkdir() + stale = staging_root / "dead-build.123" + stale.mkdir() + stale.joinpath("index.db").write_bytes(b"partial sqlite") + stale.joinpath(".build.done").write_text("written before the crash", encoding="utf-8") + + removed = artifacts._recover_stale_staging(staging_root=staging_root, artifact_name="dead-build") + + assert removed == ("dead-build.123",) + assert not stale.exists() + + class _FlakyLockConnection: """Fakes ``PRAGMA journal_mode=DELETE`` raising a transient same-process lock.