diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 15731976e6..c34b92d542 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -61,7 +61,7 @@ {"_type":"issue","id":"polylogue-rrxe4","title":"test harness: hermetic convergence-property loop - generate corpus, ingest, converge, assert registry green; metamorphic properties on top","description":"The master loop unifying zoo (yazae), composer (amrpx), and registry (t0m73) into ONE elegant mechanism: property test = for corpus-program P and ingestion order sigma: build hermetic archive from (P, sigma) via PRODUCTION ingest+convergence (tests/infra/convergence_harness.py already adapts production writers/daemon stages and 'deliberately owns no alternate convergence state machine' - the exact seam), then assert the t0m73 registry green on the result. Every registry predicate thereby gets exercised against synthetic pathological corpora in CI, and every composer pathology is validated end-to-end through real code. METAMORPHIC properties on top (each kills a whole bug class): (1) INGESTION-ORDER INVARIANCE - final archive state equivalent for all sigma (catches the deferred-tail/parent-arrival order-dependence class); (2) INCREMENTAL==BULK - trickle-converged archive equivalent to bulk-rebuilt archive on the same corpus (this is hjwr done RIGHT: as a cheap synthetic property, not a live-archive lane - note on hjwr); (3) IDEMPOTENCE - re-ingest is a no-op; (4) APPEND-PREFIX consistency - ingest(full) == ingest(full_prefix)+ingest(append_delta). Use hypothesis.stateful RuleBasedStateMachine for order/interruption exploration (precedent: test_write_path_state_machine.py, test_fts_identity_state_machine.py); crash-injection at stage boundaries as a later extension. Depends on amrpx (corpus source) + t0m73 (predicates).","design":"DESIGN (2026-08-03; the description carries the mechanism — this fixes the file/seam map): SEAM: tests/infra/convergence_harness.py (already adapts production writers/daemon stages, owns no alternate state machine). LOOP: for corpus-program P (from the zoo/composer, yazae; amrpx closed = corpus source available) and ingestion order sigma: build a hermetic archive via PRODUCTION ingest+convergence, then assert the t0m73 registry green (binding (a) of its four). METAMORPHIC PROPERTIES, each its own test module, priority order: (1) ingestion-order invariance (kills the deferred-tail/parent-arrival class); (2) incremental==bulk (hjwr done right, synthetic); (3) idempotence (re-ingest no-op — hash-skip already exists, this proves it end-to-end); (4) append-prefix consistency (full == prefix+delta). MECHANISM: hypothesis.stateful RuleBasedStateMachine for order/interruption exploration (precedents: test_write_path_state_machine.py, test_fts_identity_state_machine.py); crash-injection at stage boundaries is a later extension, not v1. PITFALLS: archive equivalence needs a canonical comparison (session/message/block rows modulo generation-scoped ids and timestamps — define one comparator, reuse across all four properties); keep corpora small (seconds per example) and rely on Hypothesis shrinking; register slow variants behind --lab, not the default testmon loop. Depends on t0m73 (predicates) — start with the registry subset that exists today rather than waiting for full migration.\n","acceptance_criteria":"1. The property loop exists: (corpus-program P, order sigma) -> hermetic archive via production ingest+convergence (tests/infra/convergence_harness.py seam) -> t0m73 registry asserted green; runs in CI on small corpora.\n2. All four metamorphic properties implemented as separate modules sharing ONE canonical archive-equivalence comparator (also used by 0x7nh): order-invariance, incremental==bulk, idempotence, append-prefix consistency.\n3. Hypothesis stateful machine explores orders/interruptions (precedent files cited in design); failures shrink to minimal corpora.\n4. At least one historical bug class is demonstrably caught: re-introducing a known fixed order-dependence bug makes property (1) fail (anti-vacuity).\n5. Slow variants gated behind --lab; default testmon loop stays fast. Verify: devtools test -k convergence_property.","notes":"Promoted P0 2026-08-03, scope extended per operator direction: this is the closest existing bead to 'the test suite generated from schemas IS the integrity checker, run against the real archive when wanted.' Extend explicitly: corpus size can exceed the real archive, generated corpus is never checked into the repo (only the seeding schemas are, as the corpus's deterministic seeds), test invocations can specify which slice/intersection of the corpus to run. Depends on already-closed polylogue-amrpx (generator) and polylogue-t0m73 (registry) -- both landed, this is unblocked.\nSCOPE CORRECTION (operator-prompted verification, 2026-08-04): this bead's premise partially does not hold yet. Read tests/infra/pathology_composer.py (amrpx's actual output): it ships 6 fixed, non-composable pathology functions and ZERO ingestion-order control. rrxe4's own first metamorphic property (ingestion-order invariance over sigma) has no input to vary sigma OVER -- nothing generates multiple orderings of a corpus today. Filed as its own gap: [new bead, see notes]. rrxe4 cannot fully execute as designed until that lands (or rrxe4 absorbs building minimal order-control itself as part of its own scope, which may be the cheaper path -- rrxe4 is the one consumer that needs it, so building it inline here rather than reopening amrpx is worth considering).","status":"open","priority":0,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-08-03T07:09:11Z","created_by":"Sinity","updated_at":"2026-08-03T22:31:04Z","dependencies":[{"issue_id":"polylogue-rrxe4","depends_on_id":"polylogue-4v2d3","type":"blocks","created_at":"2026-08-06T07:02:25Z","created_by":"Sinity","metadata":"{}"},{"issue_id":"polylogue-rrxe4","depends_on_id":"polylogue-amrpx","type":"blocks","created_at":"2026-08-03T09:09:12Z","created_by":"Sinity","metadata":"{}"},{"issue_id":"polylogue-rrxe4","depends_on_id":"polylogue-canonical-snapshot","type":"blocks","created_at":"2026-08-06T07:02:24Z","created_by":"Sinity","metadata":"{}"},{"issue_id":"polylogue-rrxe4","depends_on_id":"polylogue-ehzfn","type":"blocks","created_at":"2026-08-06T17:13:59Z","created_by":"Sinity","metadata":"{}"},{"issue_id":"polylogue-rrxe4","depends_on_id":"polylogue-t0m73","type":"blocks","created_at":"2026-08-03T09:09:12Z","created_by":"Sinity","metadata":"{}"},{"issue_id":"polylogue-rrxe4","depends_on_id":"polylogue-un60n","type":"blocks","created_at":"2026-08-06T07:02:24Z","created_by":"Sinity","metadata":"{}"},{"issue_id":"polylogue-rrxe4","depends_on_id":"polylogue-yazae","type":"blocks","created_at":"2026-08-06T07:02:24Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":7,"dependent_count":2,"comment_count":0} {"_type":"issue","id":"polylogue-xselt","title":"reindex: stamp fingerprints at bootstrap - 818fy rebuild must write parser+lowering semantics fingerprints per session","description":"Split from polylogue-kea7p (its tiny prerequisite, decoupled from the big differential planner). Add two additive index-tier columns (sessions.parser_fingerprint, sessions.lowering_fingerprint) + per-origin fingerprint derivation (origin_specs home), and have the 818fy blue-green rebuild WRITE them for every session. Why P0 and why it gates 818fy: without stamps, the next semantic fix after the reindex forces another FULL rebuild (kea7p soundness analysis: hash-skip alone is unsound; the skip oracle needs fingerprint conjunction). With stamps, every future parser/lowering fix becomes an origin-scoped reparse and the whole 'we must root out ALL bugs before reindexing' pressure collapses to 'fix stamp-poisoners + durable-corrupters before; everything else cheaply after'. Blocked by the stamp-poisoner bugs (7zp4 NFC gaps, gysk3 position-derived identity) - fingerprints computed over buggy identity/hash semantics would bootstrap poisoned stamps. Scope: columns + derivation + rebuild wiring + registry check (every session row carries current-format stamps). NOT in scope: the differential planner/skip logic (stays kea7p).","design":"DESIGN (2026-08-03, grounded against current master: SOURCE=24, INDEX=57):\n\nPREMISE UPDATE: both declared blockers are now closed (7zp4 closed 2026-08-03, was already fixed at index-v46 commit 5e23e6abf; gysk3 closed via PR #3604). The stamp-poisoner gate on this bead is satisfied EXCEPT that fsgdd's K-class list also names 0qfy/uqwd (vintage-volatile comparison axes) as stamp-poisoners; coordinator should confirm whether those two must also gate this bead before stamps bootstrap. Otherwise this bead is ready to implement.\n\nWHAT GETS STAMPED (two additive nullable TEXT columns on index-tier `sessions`, archive_tiers/index.py):\n- sessions.parser_fingerprint: SHA-256 hex over the ORIGIN-SCOPED parser semantics — the normalized source of the parser module set that produced this session's parse. Derivation home is polylogue/sources/origin_specs.py (OriginSpec already carries the prose `semantic_reparse` field per origin; add a `parser_fingerprint()` that hashes the origin's declared parser surface). Mechanism precedent: docs/plans/classifier-fingerprints.json (polylogue-gucv gate) already computes per-function source fingerprints — reuse that function-source-hash machinery, do not invent a second normalizer.\n- sessions.lowering_fingerprint: SHA-256 over the SHARED lowering path semantics (sources/dispatch.py detect_provider/_lower_payload_specs/_parse_lowered_spec + pipeline/ids.py identity/hash functions). One global value per code vintage, same for all origins; a lowering change invalidates everything, a parser change invalidates one origin. This split is exactly what makes kea7p's T0 skip origin-scoped.\n\nNAMING HAZARD: source-tier census tables already have a `parser_fingerprint` column (revision_backfill.py:392 `_resource_blocked_parser_fingerprint`) — that fingerprints the RESOURCE ENVELOPE (max_payload_bytes), not parser semantics. Different concept, different tier. Document the distinction at both sites; do not unify them.\n\nWRITE POINT: write_parsed_session_to_archive (storage/sqlite/archive_tiers/write.py:307) — the single choke point both live incremental ingest and full raw replay/reindex go through, so ordinary ingest and the 818fy rebuild stamp identically for free. Stamps MUST be written in the same transaction as the session row, never backfilled asynchronously — an unstamped-but-indexed session makes kea7p's skip oracle unsound.\n\nSCHEMA MECHANICS: INDEX_SCHEMA_VERSION bump + IndexDeltaDeclaration in storage/sqlite/lifecycle.py. Correct class: SEMANTIC_REPARSE (the column's value is parser-semantics-dependent by definition; only reparse populates it). That routes existing archives to the full rebuild — which is precisely 818fy; land this bump so the one planned rebuild bootstraps the stamps, rather than paying a second rebuild later. `devtools lab policy schema-versioning` enforces the declaration.\n\nSKIP-PREDICATE CONTRACT (kea7p consumes, xselt guarantees): a session is reparse-candidate iff parser_fingerprint != current(origin) OR lowering_fingerprint != current() OR its raw head is unindexed; skippable otherwise. xselt's deliverable is that the conjunction inputs exist and are trustworthy on every row; the planner itself stays in kea7p.\n\nREGISTRY CHECK: add an ArchiveVerificationCheckSpec to ARCHIVE_VERIFICATION_CHECKS (polylogue/maintenance/archive_verification.py) asserting every sessions row carries non-NULL, hex-shaped fingerprints, with per-origin distinct-value counts as evidence (a healthy post-rebuild archive has exactly one parser_fingerprint per origin and one lowering_fingerprint globally). This is the \"registry check\" AC item and lives in both verification planes per 60gzo.\n","acceptance_criteria":"1. Additive nullable columns sessions.parser_fingerprint + sessions.lowering_fingerprint exist in the index-tier DDL (archive_tiers/index.py), with an INDEX_SCHEMA_VERSION bump carrying a declared IndexDeltaDeclaration (SEMANTIC_REPARSE) in storage/sqlite/lifecycle.py; `devtools lab policy schema-versioning` passes.\n2. write_parsed_session_to_archive stamps both columns in the same transaction as the session row, on both ordinary ingest and raw replay — proven by a unit test through the real write path (not a mock), verify: `devtools test -k fingerprint`.\n3. Per-origin derivation lives in origin_specs.py and reuses the classifier-fingerprints function-source-hash mechanism; a unit test proves the derived fingerprint CHANGES when a parser-semantics input changes and is STABLE across process restarts.\n4. After rebuilding a fixture archive, 100% of sessions rows carry both stamps, one distinct parser_fingerprint per origin, one lowering_fingerprint globally — enforced by a new ArchiveVerificationCheckSpec in ARCHIVE_VERIFICATION_CHECKS with a red-twin fixture test.\n5. Explicitly NOT in scope: the differential planner/skip logic (stays polylogue-kea7p); this bead only guarantees the skip oracle's inputs exist and are trustworthy.","notes":"2026-08-03 (reindex-gate-hunt, team-lead): PREMISE CORRECTION. The premise-update claiming \"both declared blockers are now closed (7zp4, gysk3) ... ready to implement\" is FALSE as stated: gysk3 closure fixed only the identity-hash call-site symptom and explicitly deferred the root cause (18 parser call sites baking positional strings into provider_message_id; the \"acks\" registry docs/plans/position-derived-identity-acks.json is a lint-suppression list whose every entry references the closed gysk3, not a tracking bead). New K-class blocker polylogue-slshy (blocks edge added) carries that root cause and MUST land before this bead writes bootstrap stamps — stamps computed over position-derived identity would poison every future differential reparse. Adjudicated K by the 2026-08-03 gate-hunt; ruling and evidence on that bead.\n2026-08-04 prerequisite update: polylogue-slshy landed as PR #3730. Current-master audit finds no position-derived provider_message_id assignments in parser routes, and docs/plans/position-derived-identity-acks.json is empty. The parser identity precondition for bootstrap stamps is satisfied; retain the remaining explicit xselt dependencies.\n2026-08-04 correction: slshy was reopened after an adversarial review found three identity defects in merged PR #3730. Do not implement or merge stamps until the reopened slshy repair is merged and re-verified.","status":"open","priority":0,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-08-03T06:24:45Z","created_by":"Sinity","updated_at":"2026-08-04T06:30:07Z","dependencies":[{"issue_id":"polylogue-xselt","depends_on_id":"polylogue-0qfy","type":"blocks","created_at":"2026-08-03T13:19:29Z","created_by":"Sinity","metadata":"{}"},{"issue_id":"polylogue-xselt","depends_on_id":"polylogue-7zp4","type":"blocks","created_at":"2026-08-03T08:24:57Z","created_by":"Sinity","metadata":"{}"},{"issue_id":"polylogue-xselt","depends_on_id":"polylogue-gysk3","type":"blocks","created_at":"2026-08-03T08:24:57Z","created_by":"Sinity","metadata":"{}"},{"issue_id":"polylogue-xselt","depends_on_id":"polylogue-slshy","type":"blocks","created_at":"2026-08-03T14:17:57Z","created_by":"Sinity","metadata":"{}"},{"issue_id":"polylogue-xselt","depends_on_id":"polylogue-uqwd","type":"blocks","created_at":"2026-08-03T13:19:29Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":5,"dependent_count":2,"comment_count":0} {"_type":"issue","id":"polylogue-4v2d3","title":"verification: oracle-integrity lint - every tested symbol must be production-reachable (dead-engine test suites certify nothing)","description":"From test-class taxonomy 2026-08-03. Two instances found in one day: test_topology_cycle_rejection.py + parts of test_delegations_view.py pin queries/session_links.py (zero production imports - H4/4ts.10); test_source_laws.py pins parse_drive_payload (zero production callers - M5/194qk). Both suites stay green forever regardless of live behavior. Class design: an import-graph reachability lint - for each test module, the primary tested symbols must be reachable from a production entry point (cli/mcp/daemon/api roots); a test whose entire target set is production-unreachable fails the lint with 'certifies dead code'. Needs a small allowlist for deliberate infra (tests/infra, fixtures). Cheap to approximate: intersect grep-derived test imports with the layering/topology import graph devtools already builds. This is the mechanical form of the anti-vacuity doctrine and directly attacks the 1:1-LoC-coverage paradox.","acceptance_criteria":"1. Every test module whose primary target is production behavior is reachable from a production entrypoint, or is explicitly classified as test infrastructure.\n2. Hermetic tests cannot read real user/archive paths unless an explicit production-safe fixture boundary is declared.\n3. A controlled dead-symbol mutation makes the reachability check fail, and a controlled path escape makes the hermeticity check fail.\n4. Focused devtools tests and devtools verify --quick pass.","notes":"2026-08-03: scope explicitly includes the HERMETICITY guard (kmqwm class - tests must not read real user paths like ~/.codex/sessions; enforce via fixture-env guard or lint), same family as reachability: both are 'the test exercises something other than what it claims'.\n2026-08-06 campaign graph promotion: this bead is a direct prerequisite or process guard for proof-carrying reindex acceptance. Its implementation cannot substitute for the terminal receipt, but its output is consumed by the campaign ledger and final gate.\nCodex closed-PR audit 2026-08-06: merged PR #3836 review found dead-engine and hermeticity gaps remain. This is a campaign prerequisite and cannot close from green tests that target production-unreachable symbols.","status":"open","priority":0,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-08-03T05:47:34Z","created_by":"Sinity","updated_at":"2026-08-06T19:24:17Z","dependency_count":0,"dependent_count":1,"comment_count":0} -{"_type":"issue","id":"polylogue-csx21","title":"verification: complexity/cost assertions - assert O-shape of work per operation on scaled synthetic corpora (nothing today can catch an O(archive)-per-item regression)","description":"From test-class taxonomy 2026-08-03. H7 (qsagp: archive-wide derived rebuild per component) was invisible to every existing test class: fixture-scale tests cannot see complexity bugs, benchmarks measure wall-clock on fixed inputs (noise-bound), and the 5000x commit-latency finding (7mtf) + 188s holds (de2a) were both discovered live instead. Class design: instrument work units (rows scanned/written via sqlite3 stmt counters or trace hooks, bytes, passes) and assert SHAPE across 2-3 corpus scales (e.g. materializing one new session must touch O(session) derived rows, not O(archive); a bounded pass's work must not scale with backlog size beyond its batch). Runs in CI at small scales (seconds); the assertion is the exponent, not the wall-clock. Candidates: per-component materialization cost (qsagp regression net), census cost per pass, FTS repair cost per drifted session, write-path cost per message. Home: the class-tagged check registry (t0m73) or tests/benchmarks reworked to counter-based assertions.","acceptance_criteria":"1. Scaled production-route workloads measure archive-wide refreshes from observed counters or production call spies, not fixture constants.\n2. The measured work bound rejects O(archive) derived rebuilds per component and reports component versus terminal refresh work separately.\n3. Reinstating the deleted archive-wide refresh produces a red result without changing the oracle.\n4. Focused complexity tests and devtools verify --quick pass.","notes":"2026-08-06 campaign graph promotion: this bead is a direct prerequisite or process guard for proof-carrying reindex acceptance. Its implementation cannot substitute for the terminal receipt, but its output is consumed by the campaign ledger and final gate.\nCodex audit of merged PR #3842 found the complexity assertion remains vacuous. Comment 3726263334 shows mutate_archive_wide_rebuild is hard-coded false and the growth budget permits linear archive work. Completion must derive archive-wide rebuild evidence from observed counters or production call spies, and compare scaled workloads with a sublinear bound that rejects O(archive) work per component. Add an anti-vacuity mutation that reinstates the archive-wide refresh and produces a red receipt. This remains a direct candidate-acceptance prerequisite.\nCodex closed-PR audit 2026-08-06, PR #3842 comments 3726263334 and related findings: the merged assertion still derives archive-wide work from fixture-controlled mutation state and permits linear growth. Keep this Bead open until observed counters, scaled sublinear bounds, and a red mutation are present.","status":"open","priority":0,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-08-03T05:47:33Z","created_by":"Sinity","updated_at":"2026-08-06T19:24:17Z","dependency_count":0,"dependent_count":1,"comment_count":0} +{"_type":"issue","id":"polylogue-csx21","title":"verification: complexity/cost assertions - assert O-shape of work per operation on scaled synthetic corpora (nothing today can catch an O(archive)-per-item regression)","description":"From test-class taxonomy 2026-08-03. H7 (qsagp: archive-wide derived rebuild per component) was invisible to every existing test class: fixture-scale tests cannot see complexity bugs, benchmarks measure wall-clock on fixed inputs (noise-bound), and the 5000x commit-latency finding (7mtf) + 188s holds (de2a) were both discovered live instead. Class design: instrument work units (rows scanned/written via sqlite3 stmt counters or trace hooks, bytes, passes) and assert SHAPE across 2-3 corpus scales (e.g. materializing one new session must touch O(session) derived rows, not O(archive); a bounded pass's work must not scale with backlog size beyond its batch). Runs in CI at small scales (seconds); the assertion is the exponent, not the wall-clock. Candidates: per-component materialization cost (qsagp regression net), census cost per pass, FTS repair cost per drifted session, write-path cost per message. Home: the class-tagged check registry (t0m73) or tests/benchmarks reworked to counter-based assertions.","acceptance_criteria":"1. Scaled production-route workloads measure archive-wide refreshes from observed counters or production call spies, not fixture constants.\n2. The measured work bound rejects O(archive) derived rebuilds per component and reports component versus terminal refresh work separately.\n3. Reinstating the deleted archive-wide refresh produces a red result without changing the oracle.\n4. Focused complexity tests and devtools verify --quick pass.","notes":"2026-08-06 campaign graph promotion: this bead is a direct prerequisite or process guard for proof-carrying reindex acceptance. Its implementation cannot substitute for the terminal receipt, but its output is consumed by the campaign ledger and final gate.\nCodex audit of merged PR #3842 found the complexity assertion remains vacuous. Comment 3726263334 shows mutate_archive_wide_rebuild is hard-coded false and the growth budget permits linear archive work. Completion must derive archive-wide rebuild evidence from observed counters or production call spies, and compare scaled workloads with a sublinear bound that rejects O(archive) work per component. Add an anti-vacuity mutation that reinstates the archive-wide refresh and produces a red receipt. This remains a direct candidate-acceptance prerequisite.\nCodex closed-PR audit 2026-08-06, PR #3842 comments 3726263334 and related findings: the merged assertion still derives archive-wide work from fixture-controlled mutation state and permits linear growth. Keep this Bead open until observed counters, scaled sublinear bounds, and a red mutation are present.","status":"closed","priority":0,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-08-03T05:47:33Z","created_by":"Sinity","updated_at":"2026-08-08T14:50:00Z","closed_at":"2026-08-08T14:50:00Z","close_reason":"Satisfied on final review-repair commit 3410cd0bf37f241accd7136fee615ef664cdcc44. AC1: scaled production repair workloads measure SQLite VM steps and archive-wide derived statements from real sqlite3 connections, plus repair-result row, byte, pass, and selected-component counters. AC2: the unchanged oracle reports selected-component derived work separately from the bounded terminal-refresh statement envelope; full-table delegation_refresh_scope deletion is counted and scoped deletion is not. AC3: the red twin reinstates the deleted FTS, command-trigram, action-pair, and delegation full-refresh quartet through the real revision-backfill seam, proves observed archive-wide statements exceed the terminal-refresh budget, then proves the same oracle fails. AC4: devtools test tests/unit/storage/test_rebuild_complexity.py passed 5 tests in 21.21s; devtools verify --quick passed all 24 steps in run 20260808T144757Z-quick-2979100-a34c9198. The default affected gate refused before collection because the fresh lane has no testmon seed; repository-wide seed and full-suite repair remains tracked by polylogue-93xe and is not an acceptance criterion for this proof bead.","dependency_count":0,"dependent_count":1,"comment_count":0} {"_type":"issue","id":"polylogue-in24n","title":"verification: source-index-coverage check uses the census's own ledger as its universe - the 7,200-source gap is invisible to it by construction","description":"Found 2026-08-03 while generalizing the invariant suite. maintenance/archive_verification.py:247-312: _check_source_index_coverage computes missing_work = censused_complete - indexed, where censused_complete = raw_membership_census WHERE status='complete' AND member_count>0. Raws the census never blessed (quarantined: 7,191 today; untyped: 9) never enter the universe, so the check reports OK while 25% of logical sources are unindexed. This is the wrong-oracle pattern inside the verification layer itself: the check audits the mechanism against the mechanism's own bookkeeping. Fix: universe = raw_sessions logical heads (ground truth); every unindexed head must be typed (parse_error / open blocker / declared non-session artifact) - i.e. adopt invariant I1 from .agent/scratch/archive-invariants-2026-08-03.py. Registry-wide rule to adopt in the same change (and lint if cheap): a verification check's universe must be a ground-truth table, never a derived ledger of the machinery under audit. Related: t0m73.","acceptance_criteria":"1. The source-index coverage universe is derived from raw logical heads in source.db, not from the census ledger under audit.\n2. Every raw logical head absent from the index is either indexed or has an explicit typed parse failure, unsupported/non-session disposition, quarantine blocker, or other accepted terminal state.\n3. A red mutation that removes a raw head from the derived census while leaving source.db unchanged makes the check fail.\n4. Focused registry tests and devtools verify --quick pass.","notes":"2026-08-06 campaign graph promotion: this bead is a direct prerequisite or process guard for proof-carrying reindex acceptance. Its implementation cannot substitute for the terminal receipt, but its output is consumed by the campaign ledger and final gate.\nCodex closed-PR audit 2026-08-06: review of PR #3836 confirmed the current check can bless its own incomplete census and miss the quarantined/untyped source population. Keep the ground-truth universe requirement explicit.","status":"open","priority":0,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-08-03T05:47:32Z","created_by":"Sinity","updated_at":"2026-08-06T19:24:18Z","dependency_count":0,"dependent_count":1,"comment_count":0} {"_type":"issue","id":"polylogue-t0m73","title":"verification: productize the whole-archive invariant suite (10 checks, 7 failing live) as a lab probe + reindex acceptance gate","description":"From inline audit continuation 2026-08-03. Prototype at .agent/scratch/archive-invariants-2026-08-03.py (repo checkout, gitignored) - 10 read-only ground-truth invariants runnable against any archive root; live run: 7 FAIL / 3 PASS in ~1.5s. Failing today: I1 coverage (7,200 unindexed logical sources: 7,191 quarantined + 9 UNTYPED), I2 enum-superset-CHECK (source tier 5 tables + live index generation sessions table all missing claude-design-session), I3 blob_refs join-liveness (73,427 raw_payload + 1,336 attachment orphans), I4 embeddings refs (4,186 orphaned = feu0's exact number, undrained), I5 session_links lifecycle (status NULL on all 9,497 rows = 4ts.10), I7 FTS drift (messages_fts missing 35,331; threads_fts 10), I8 message_count projection drift (1 session). Passing: I9 revision-head pointers, I10 user-tier refs, I6 (criterion too weak - detail shows 0 daemon stage events in 24h alongside a 7,200-source gap; productized version must fail on gap>0 AND no recent convergence activity, and re-check the 3 known convergence_debt rows my predicate missed). Productize as: devtools lab probe archive-invariants (against live/demo root) + pytest wrappers against corpus_seeded_db; each check documents the bug class and the incident that motivated it. These are checks against ground truth (cross-tier joins, enum-vs-DDL, liveness), NOT mechanism-vs-itself - the class the 1:1-LoC unit suite structurally lacks, which is why the suite is green while the archive is 25% unconverged. This suite green on the post-reindex archive should be part of 818fy's acceptance.","design":"DESIGN (2026-08-03 structured distillation of the rescope + classification notes — read them for evidence):\nTARGET SHAPE (per the operator rescope): do NOT build a bespoke lab probe. ONE class-tagged check registry, seeded from the two existing substrates: ARCHIVE_VERIFICATION_CHECKS (maintenance/archive_verification.py — read-only, any-root, per-check error isolation; I2/I3/I4/I5/I8 already migrated) and the daemon health tiers (daemon/health.py, ~20 checks). Class tags: state-invariant | liveness | freshness | complexity | fidelity | conservation | config.\nFOUR BINDINGS consuming the one registry: (a) pytest parametrized over registry × corpus_seeded_db/zoo fixtures (CI, Plane 1 per 60gzo); (b) promotion/readiness gate subset (818fy's rebuild-index promote step); (c) daemon health-tier scheduling for liveness/freshness classes (Plane 2); (d) operator CLI against any root incl. live.\nTWO CONTRACT RULES baked into the registry: GROUND-TRUTH-UNIVERSE (a check's universe is a ground-truth table, never the audited mechanism's own ledger — the wrong-universe coverage-check bug is filed separately) and RED-TWIN anti-vacuity (every check ships a fixture mutation that must make it fail; red twins run in binding (a)).\nWAIVERS: known-red-on-live carries a bead id and expires when that bead closes; red-without-waiver is the alarm.\nMIGRATION WORK REMAINING: lift the 5 not-yet-migrated prototype invariants (I1 coverage with the corrected third bucket byte_dup_of_indexed so the report can't overstate; I6 with the fixed criterion gap>0 AND no recent convergence activity => fail, re-checking convergence_debt rows; I7 FTS drift; I9/I10 as cheap passes) from .agent/scratch/archive-invariants-2026-08-03.py into the registry, each documenting its bug class + motivating incident; graduate the second-wave detectors (V2 capability-parity, V4 active-leaf, V1b vocabulary-honesty — all red live) per the wwph1 graduation rule.\nDEV LOOP: the canary reindex loop (partial --no-promote rebuild + registry against the canary, minutes) is the iteration mechanism; the full-registry green on the post-reindex archive is 818fy acceptance. Red-check-first workflow rule is ey4ro's contract — this bead provides the instruments.\n","acceptance_criteria":"1. One class-tagged registry exists (seeded from ARCHIVE_VERIFICATION_CHECKS + health-tier checks) with the GROUND-TRUTH-UNIVERSE and RED-TWIN contract rules enforced structurally (a check without a red twin fails a meta-test).\n2. All 10 prototype invariants migrated (I1 with byte_dup_of_indexed third bucket; I6 with gap>0 AND no-recent-convergence criterion), each documenting bug class + motivating incident.\n3. Four bindings live: pytest (registry x fixtures + red twins), promotion-gate subset wired into rebuild-index promote, daemon health scheduling for liveness/freshness classes, operator CLI against any root.\n4. Waiver mechanism: known-red-on-live rows carry a bead id and expire on close; red-without-waiver alarms.\n5. Registry green on the post-reindex archive is wired into 818fy acceptance (runbook step 5/6). Verify: devtools test -k archive_verification; devtools test -k registry.","notes":"2026-08-03 RESCOPE after substrate investigation (operator: 'not a bespoke probe - a probe against the archive where invariants are one kind of many'): do NOT build a new lab probe. The substrate exists twice already: (1) ARCHIVE_VERIFICATION_CHECKS (maintenance/archive_verification.py:612) - read-only, any-root, per-check error isolation, 7 checks, but run routinely by NOTHING (CLI manual, promotion gate runs only fts-parity subset, backup verify) and its coverage check has a wrong-universe bug (separate bead filed); (2) daemon health tiers (daemon/health.py, ~20 checks incl. convergence-debt/cursor-lag/insight-freshness) - the scheduled liveness/freshness home, currently dark past FAST (y0ven). Target shape: ONE class-tagged check registry (state-invariant | liveness | freshness | complexity | fidelity | conservation | config), four bindings consuming it: (a) pytest parametrized over registry x corpus_seeded_db fixtures (regular tests, CI); (b) promotion/readiness gate subset; (c) daemon health-tier scheduling (liveness classes); (d) operator CLI against any root incl. live. Two contract rules baked into the registry: GROUND-TRUTH-UNIVERSE (a check's universe is a ground-truth table, never the audited mechanism's ledger) and RED-TWIN anti-vacuity (every check ships a fixture mutation that must make it fail; the red twin runs in the pytest binding). Plus a waiver mechanism for known-red-on-live: waiver carries a bead id, expires when the bead closes; red-without-waiver is the alarm. Migrate the 10 prototype invariants into this registry; I6's criterion must become gap>0 AND no recent convergence activity => fail.\n2026-08-03 BACKLOG CLASSIFICATION (operator: 'are all bugs instances of a detectable class? classify the ~50, build the tests, get them red'). All 62 open 818fy-gating beads classified by detector family: ~48 map to one of EIGHT families - D1 state-invariants (14: 052vs, 4ts.10, 2tfug, i3zo, es7b, omsw, gxig, hjpx-symptoms, lkrc-symptoms, 5tkbt...), D2 liveness/freshness (6: 2qrx, ix5r, 5xxmc, tu1f, 5iz4-aging, hjpx-debt), D3 complexity-shape (3: qsagp, 5q2u, lyv4), D4 test-integrity/hermeticity (2: kmqwm, h7y0j-adjacent), D5 fidelity-differentials (8: c831, 6lyh1, 7zp4, gysk3, hjwr, uqwd, 0qfy, b5l.1), D6 capability-parity (6: ksgg, xofj, mvcbi, tu1f, 0qfy, 8ac0-coverage), D7 vocabulary-honesty (6: 6krh, cc4k, z22ml, h57ic, iuyr, vp2ky), D8 runtime/config-coherence (5: 9kc0, e98k, 9qnzy, swqu, f47j). ~14 are NOT detector-shaped: operational tasks (a7gmk, tnqqt, k8wv, lb39z, f1vg) and design decisions (cijx.2, ds4b4, w6hql, tw4ar, aex0, sp72, foee, ih67, 2qx.3, 6bebe) - honest limit of the approach. SECOND-WAVE RESULTS (built + ran today, .agent/scratch script extended inline): V2 capability-parity RED (codex 0% parent links of 2.47M msgs; hermes, aistudio 0%), V4 active-leaf RED (103 multi-leaf sessions), V1b vocab RED (deferrals as 'failed'), V5/V1a/V6 PASS and thereby flag gxig/cc4k/9kc0 as possibly-stale beads (detectors audit the backlog itself, both directions), V7 probe: 6lyh1 latent (0 of 4,344 APPEND raws divergent - un-gated from 818fy), V3: xofj needs parse-boundary conservation (silent drops invisible index-side). Cumulative scorecard: 18 detectors built today, 10 red, 3 bead-refuting passes, 2 scope-refining probes. WORKFLOW ADOPTION: red-check-first for bug-class gating beads - a fix PR must flip a named registry check red->green, check predates fix. ITERATION SPEED (operator concern - 'verification relies on actually reindexing'): add a CANARY REINDEX loop - partial selection rebuild into an inactive --no-promote generation (machinery exists) of a few hundred representative sessions per origin, run the registry against the canary in minutes, iterate red->green, full reindex once at the end. The gate is the backstop; the canary is the dev loop.\n2026-08-03 ~10:15 CORRECTION to invariant I1's framing: the prototype's 7,200/quarantined/9-untyped split is missing a THIRD bucket the operator's challenge surfaced - byte-identical-duplicate-of-already-indexed (measured: 4,305 of 7,200 heads, 77% of bytes). I1 as designed would still correctly flag the true ~2,895-head novel gap as ERROR, but its evidence/summary text should report the duplicate-vs-novel split, not present the raw unindexed count as if it were all 'missing'. When productizing into the registry: add a byte_dup_of_indexed classification (same blob_hash exists on an indexed raw) alongside untyped/quarantined so the check's own report can't repeat this overstatement.\n2026-08-06 audit reopens the acceptance claim. The implementation is useful, but the registry is not yet the sole self-describing source for red twins, waivers, acceptance selection, daemon scheduling, incident provenance, candidate applicability, or live receipts. Close only after registry-v2 and the campaign ledger consume one structured contract.","status":"open","priority":0,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-08-03T05:35:27Z","created_by":"Sinity","updated_at":"2026-08-06T04:58:17Z","dependencies":[{"issue_id":"polylogue-t0m73","depends_on_id":"polylogue-in24n","type":"blocks","created_at":"2026-08-06T07:02:25Z","created_by":"Sinity","metadata":"{}"},{"issue_id":"polylogue-t0m73","depends_on_id":"polylogue-reindex-registry-two-plane-subset","type":"blocks","created_at":"2026-08-06T07:02:25Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":2,"dependent_count":3,"comment_count":0} {"_type":"issue","id":"polylogue-fyyro","title":"storage: embeddings tier has no retired-generation GC - 555MB+ .retired files unowned; plus dead archive_tiers/self_verify.py","description":"Structural audit M8 (/realm/data/derived/reports/polylogue-structural-audit-2026-08-03.html). /realm/db/polylogue/embeddings.db.retired-20260627 (555MB) + embeddings.db.v2-retired-20260718-{shm,wal} referenced by zero code (repo-wide grep). Index tier has generation GC; embeddings has none. Also archive_tiers/self_verify.py:11-105 (build_archive_session_self_verify_envelope) has one caller: its own test; the real self-verify lives in devtools/self_verify.py. Reclaim files, add embeddings-generation retirement ownership, delete or fold the dead module.","notes":"2026-08-06 audit reopens the claim because the close reason says only the code half landed. Retired embeddings-generation retention and garbage collection must be automatic and receipt-backed before blue-green promotion. The embeddings-retention bead carries the residual implementation and proof scope.","status":"open","priority":0,"issue_type":"chore","owner":"ezo.dev@gmail.com","created_at":"2026-08-03T05:08:07Z","created_by":"Sinity","updated_at":"2026-08-06T05:00:40Z","dependency_count":0,"dependent_count":0,"comment_count":0,"metadata":{"acceptance_contract_v1":{"anti_vacuity":["A controlled mutation that removes the central guard or restores the pre-fix behavior makes the focused regression fail.","The test asserts durable/public behavior, not merely that a helper was called or returned a mocked value."],"bead_id":"polylogue-fyyro","closure":{"disposition":"whole-or-explicit-partial","rule":"Close `polylogue-fyyro` only when the criteria above are evidenced on the final head. Any residual operation, provider/origin, live population, or generalized bug class is transferred to a named successor with a dependency edge before closure.","successor_required_for_partial":true},"confidence":"medium","contract_type":"implementation","evidence":["The regression or audit preserves the motivating observation: Structural audit M8 (/realm/data/derived/reports/polylogue-structural-audit-2026-08-03.html). /realm/db/polylogue/embeddings.db.retired-20260627 (555MB) + embeddings.db.v2-retired-20260718-{shm,wal} referenced by zero code (repo-wide grep). Index tier has generation GC; embeddings has none.","Measured evidence remains reconciled with this recorded population: beddings tier has no retired-generation GC - 555MB+ .retired files unowned; plus dead archive_tiers/self_verify.py Stru","Measured evidence remains reconciled with this recorded population: a/derived/reports/polylogue-structural-audit-2026-08-03.html). /realm/db/polylogue/embeddings.db.retired-20260627 (555M"],"generated_at":"2026-08-07T00:00:00Z","outcome":"The production path no longer exhibits the defect or missing capability named “storage: embeddings tier has no retired-generation GC - 555MB+ .retired files unowned; plus dead archive_tiers/self_verify.py”; the result is observable through the public or operator-facing route.","retained_scope":[],"risk":"durable-mutation","routes":["Exercise the implementation through these named production surfaces: `archive_tiers/self_verify.py`, `devtools/self_verify.py`."],"safety":["No production mutation is performed by the implementation lane.","Any later apply is dry-run-first, backup-gated, exact-plan-bound, idempotent or resumable, and emits an immutable receipt."],"schema_version":1,"source_digest":"597108b615c52ffc98593f7f95042945dfaffb77e7bf70f01002f3b673f96922","verification":["Add a focused red-before/green-after regression carrying `polylogue-fyyro` or the incident name and executing the owning production route.","Run `devtools verify --quick` on the final head and record the exact head SHA in the closure evidence.","Run `devtools verify` on the final head so the testmon-affected regression set executes; `devtools verify --quick` alone is insufficient.","Run `devtools verify` on the final head so the testmon-affected regression set executes; `devtools verify --quick` alone is insufficient."]}},"acceptance_criteria":"1. Outcome: The production path no longer exhibits the defect or missing capability named “storage: embeddings tier has no retired-generation GC - 555MB+ .retired files unowned; plus dead archive_tiers/self_verify.py”; the result is observable through the public or operator-facing route.\n2. Production route: Exercise the implementation through these named production surfaces: `archive_tiers/self_verify.py`, `devtools/self_verify.py`.\n3. Evidence: The regression or audit preserves the motivating observation: Structural audit M8 (/realm/data/derived/reports/polylogue-structural-audit-2026-08-03.html). /realm/db/polylogue/embeddings.db.retired-20260627 (555MB) + embeddings.db.v2-retired-20260718-{shm,wal} referenced by zero code (repo-wide grep). Index tier has generation GC; embeddings has none.\n4. Evidence: Measured evidence remains reconciled with this recorded population: beddings tier has no retired-generation GC - 555MB+ .retired files unowned; plus dead archive_tiers/self_verify.py Stru\n5. Evidence: Measured evidence remains reconciled with this recorded population: a/derived/reports/polylogue-structural-audit-2026-08-03.html). /realm/db/polylogue/embeddings.db.retired-20260627 (555M\n6. Verification: Add a focused red-before/green-after regression carrying `polylogue-fyyro` or the incident name and executing the owning production route.\n7. Verification: Run `devtools verify --quick` on the final head and record the exact head SHA in the closure evidence.\n8. Verification: Run `devtools verify` on the final head so the testmon-affected regression set executes; `devtools verify --quick` alone is insufficient.\n9. Verification: Run `devtools verify` on the final head so the testmon-affected regression set executes; `devtools verify --quick` alone is insufficient.\n10. Anti-vacuity: A controlled mutation that removes the central guard or restores the pre-fix behavior makes the focused regression fail.\n11. Anti-vacuity: The test asserts durable/public behavior, not merely that a helper was called or returned a mocked value.\n12. Safety: No production mutation is performed by the implementation lane.\n13. Safety: Any later apply is dry-run-first, backup-gated, exact-plan-bound, idempotent or resumable, and emits an immutable receipt.\n14. Closure disposition: whole-or-explicit-partial\n15. Closure: Close `polylogue-fyyro` only when the criteria above are evidenced on the final head. Any residual operation, provider/origin, live population, or generalized bug class is transferred to a named successor with a dependency edge before closure."} diff --git a/tests/infra/sqlite_work_counter.py b/tests/infra/sqlite_work_counter.py index c9b70d64c1..60b61d55d9 100644 --- a/tests/infra/sqlite_work_counter.py +++ b/tests/infra/sqlite_work_counter.py @@ -13,9 +13,12 @@ _DERIVED_SURFACES = ( "messages_fts", + "messages_fts_identity", + "session_work_events_fts", "blocks_command_trigram", "action_pairs", "delegation_facts", + "delegation_refresh_scope", ) _SQL_SPACE = re.compile(r"\s+") @@ -48,16 +51,18 @@ def _is_archive_wide_derived_statement(sql: str) -> bool: """Recognize the deleted qsagp shape, without naming a private call site.""" if not _mentions_derived_surface(sql): return False + if " values " in sql: + return False if sql.startswith("delete from "): return " where " not in sql if sql.startswith("insert into action_pairs"): return "where u.session_id =" not in sql - if sql.startswith("insert into messages_fts") or sql.startswith("insert into blocks_command_trigram"): - return "where session_id =" not in sql + if sql.startswith("insert into messages_fts"): + return "target.session_id = b.session_id" not in sql + if sql.startswith("insert into blocks_command_trigram"): + return "session_id" not in sql if sql.startswith("insert or replace into delegation_refresh_scope"): return "select session_id from sessions" in sql - if sql.startswith("insert into delegation_facts"): - return "where parent_session_id =" not in sql return False diff --git a/tests/unit/storage/test_rebuild_complexity.py b/tests/unit/storage/test_rebuild_complexity.py index e583b8b860..29416e23a6 100644 --- a/tests/unit/storage/test_rebuild_complexity.py +++ b/tests/unit/storage/test_rebuild_complexity.py @@ -5,6 +5,7 @@ import json import sqlite3 from collections import Counter +from collections.abc import Callable from pathlib import Path from typing import Any @@ -16,9 +17,15 @@ from polylogue.storage import repair as repair_mod from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root -from tests.infra.growth_budgets import GrowthBudget, GrowthObservation, assert_growth_budgets +from tests.infra.growth_budgets import GrowthBudget, GrowthObservation, evaluate_growth_budgets from tests.infra.sqlite_work_counter import sqlite_work_counter +_COMPONENT_DERIVED_WORK_BUDGET = GrowthBudget(metric="component_derived_vm_steps", max_step_multiplier=4.0) +# A terminal refresh is allowed to emit the bounded quartet's observed +# archive-wide statement envelope once per production-route pass. The law +# rejects paying that envelope once per selected component. +_COMPONENT_TERMINAL_REFRESH_STATEMENT_BUDGET = 9 + def _config(root: Path) -> Config: return Config(archive_root=root, render_root=root, sources=[], db_path=root / "archive.db") @@ -96,76 +103,96 @@ def _run_component_measurement( archive_size: int, monkeypatch: pytest.MonkeyPatch, *, - mutate_archive_wide_rebuild: bool, + component_count: int, + install_mutation: Callable[[pytest.MonkeyPatch], None] | None = None, ) -> GrowthObservation: - root = tmp_path / (f"mutant-{archive_size}" if mutate_archive_wide_rebuild else f"healthy-{archive_size}") + if component_count < 1 or component_count > archive_size: + raise ValueError("component_count must be between one and archive_size") + root = tmp_path / f"component-{archive_size}" _seed_raw_archive(root, archive_size, prefix="existing") _materialize_all(root, archive_size) with ArchiveStore.open_existing(root, read_only=False) as archive: - archive.write_raw_payload( - provider=Provider.CODEX, - payload=_tool_call_payload("target"), - source_path="target.jsonl", - acquired_at_ms=archive_size + 1, - ) - - mutation_context = monkeypatch.context() if mutate_archive_wide_rebuild else None - if mutation_context is not None: - from polylogue.storage.fts import fts_lifecycle - from polylogue.storage.sqlite import action_pairs, delegation_facts - - original_backfill = revision_backfill.backfill_historical_revision_evidence - - def replay_with_deleted_regression(*args: Any, **kwargs: Any) -> Any: - result = original_backfill(*args, **kwargs) - archive_root = Path(args[0]) - with sqlite3.connect(archive_root / "index.db") as conn: - conn.execute("PRAGMA busy_timeout = 600000") - fts_lifecycle.rebuild_fts_index_sync(conn) - fts_lifecycle.rebuild_command_trigram_index_sync(conn) - action_pairs.rebuild_all_action_pairs_sync(conn) - delegation_facts.rebuild_all_delegation_facts_sync(conn) - conn.commit() - return result + for index in range(component_count): + native_id = f"target-{index}" + archive.write_raw_payload( + provider=Provider.CODEX, + payload=_tool_call_payload(native_id), + source_path=f"{native_id}.jsonl", + acquired_at_ms=archive_size + index + 1, + ) - with mutation_context as mutation: - mutation.setattr(revision_backfill, "backfill_historical_revision_evidence", replay_with_deleted_regression) - with sqlite_work_counter(step_interval=1) as counter: - result = repair_mod.repair_raw_materialization(_config(root), raw_artifact_limit=1) - else: + with monkeypatch.context() as mutation: + if install_mutation is not None: + install_mutation(mutation) with sqlite_work_counter(step_interval=1) as counter: - result = repair_mod.repair_raw_materialization(_config(root), raw_artifact_limit=1) + result = repair_mod.repair_raw_materialization(_config(root), raw_artifact_limit=component_count) - assert result.repaired_count == 1 + assert result.repaired_count == component_count return GrowthObservation( tier=str(archive_size), size=archive_size, metrics={ - "derived_vm_steps": float(counter.metric("derived_vm_steps")), - "archive_wide_rebuild_calls": float(4 if mutate_archive_wide_rebuild else 0), + "component_derived_vm_steps": float(counter.metric("derived_vm_steps")), + "archive_wide_derived_statements": float(counter.metric("archive_wide_derived_statements")), + "component_rows_scanned": result.metrics["raw_materialization_scanned_raw_count"], + "component_rows_written": result.metrics["raw_materialization_replayed_logical_source_count"], + "component_bytes": result.metrics["raw_materialization_selected_total_blob_bytes"], + "component_passes": result.metrics["raw_materialization_executed_count"], + "selected_component_count": float(component_count), }, ) def _assert_component_shape(observations: list[GrowthObservation]) -> None: - assert all(observation.metric("archive_wide_rebuild_calls") == 0 for observation in observations) - assert any(observation.metric("derived_vm_steps") > 0 for observation in observations) - assert_growth_budgets( - observations, - [GrowthBudget(metric="derived_vm_steps", max_step_multiplier=4.0)], + report = evaluate_growth_budgets(observations, [_COMPONENT_DERIVED_WORK_BUDGET]) + measured = "\n".join(f" {observation.tier}: {dict(observation.metrics)}" for observation in observations) + assert report.ok, ( + "component derived work exceeded the scale bound " + f"{_COMPONENT_DERIVED_WORK_BUDGET.max_step_multiplier}x; " + f"violations={report.violations}; measured counters:\n{measured}" + ) + archive_wide = [observation.metric("archive_wide_derived_statements") for observation in observations] + assert all(value <= _COMPONENT_TERMINAL_REFRESH_STATEMENT_BUDGET for value in archive_wide), ( + "incremental component route exceeded the one-terminal-refresh envelope; " + f"declared statement budget={_COMPONENT_TERMINAL_REFRESH_STATEMENT_BUDGET}; measured counters:\n{measured}" + ) + assert any(observation.metric("component_derived_vm_steps") > 0 for observation in observations), ( + f"production route reported no derived work; measured counters:\n{measured}" ) +def _restore_deleted_archive_wide_refresh(mutation: pytest.MonkeyPatch) -> None: + """Restore the deleted qsagp quartet through the real backfill seam.""" + from polylogue.storage.fts import fts_lifecycle + from polylogue.storage.sqlite import action_pairs, delegation_facts + + original_backfill = revision_backfill.backfill_historical_revision_evidence + + def replay_with_deleted_regression(*args: Any, **kwargs: Any) -> Any: + result = original_backfill(*args, **kwargs) + archive_root = Path(args[0]) + with sqlite3.connect(archive_root / "index.db") as conn: + conn.execute("PRAGMA busy_timeout = 600000") + fts_lifecycle.rebuild_fts_index_sync(conn) + fts_lifecycle.rebuild_command_trigram_index_sync(conn) + action_pairs.rebuild_all_action_pairs_sync(conn) + delegation_facts.rebuild_all_delegation_facts_sync(conn) + conn.commit() + return result + + mutation.setattr(revision_backfill, "backfill_historical_revision_evidence", replay_with_deleted_regression) + + def test_one_component_derived_work_is_archive_scale_stable(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: observations = [ _run_component_measurement( tmp_path, archive_size, monkeypatch, - mutate_archive_wide_rebuild=False, + component_count=component_count, ) - for archive_size in (2, 8, 32) + for archive_size, component_count in ((2, 1), (8, 2), (32, 4)) ] _assert_component_shape(observations) @@ -180,21 +207,30 @@ def test_component_law_rejects_qsagp_archive_wide_per_item_mutation( tmp_path, archive_size, monkeypatch, - mutate_archive_wide_rebuild=True, + component_count=component_count, + install_mutation=_restore_deleted_archive_wide_refresh, ) - for archive_size in (2, 8, 32) + for archive_size, component_count in ((2, 1), (8, 2), (32, 4)) ] + archive_wide = [observation.metric("archive_wide_derived_statements") for observation in observations] + assert any(value > _COMPONENT_TERMINAL_REFRESH_STATEMENT_BUDGET for value in archive_wide), ( + "red mutation did not exceed the terminal-refresh statement envelope; " + f"measured counters={[dict(observation.metrics) for observation in observations]}" + ) with pytest.raises(AssertionError): _assert_component_shape(observations) - assert all(observation.metric("archive_wide_rebuild_calls") == 4 for observation in observations) + assert all(value > 0 for value in archive_wide), ( + "red mutation did not reach observed archive-wide SQL; " + f"measured counters={[dict(observation.metrics) for observation in observations]}" + ) def test_bounded_replay_work_is_batch_bounded_independent_of_backlog( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: batch_size = 2 - observed: list[tuple[int, int, int]] = [] + observed: list[tuple[int, int, int, int]] = [] for archive_size in (4, 16, 64): root = tmp_path / f"batch-{archive_size}" _seed_raw_archive(root, archive_size, prefix="batch") @@ -214,9 +250,18 @@ def counted_backfill(*args: Any, _original: Any = original_backfill, **kwargs: A assert result.metrics["raw_materialization_executed_count"] == float(batch_size) assert result.metrics["raw_materialization_scanned_raw_count"] <= float(batch_size) - observed.append((archive_size, selected_work, result.repaired_count)) + observed.append( + ( + archive_size, + selected_work, + result.repaired_count, + int(result.metrics["raw_materialization_executed_count"]), + ) + ) - assert observed == [(4, 2, 2), (16, 2, 2), (64, 2, 2)] + assert observed == [(4, 2, 2, 2), (16, 2, 2, 2), (64, 2, 2, 2)], ( + f"bounded replay exceeded batch bound={batch_size}; observed={observed}" + ) def test_mixed_hot_cold_large_small_components_all_receive_a_turn(