From f7b4f1772d3335b0f22f137ccaaa9c111ad215d5 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sat, 8 Aug 2026 10:42:50 +0200 Subject: [PATCH 01/19] fix(reindex): isolate inactive candidates from durable writes Owned inactive generations previously reopened source and user tiers as writable through candidate read-through symlinks. The production bootstrap correctly rejected the layout, and bypassing that check would have let candidate replay mutate the frozen phase-2 source. Add an explicit owned-candidate mode that writes only index.db, validates every read-through target, opens durable tiers read-only, refuses blob publication, and requires current parser census plus persisted authority before allocating or accepting a candidate. Ref polylogue-inactive-candidate-durable-barrier. --- .beads/issues.jsonl | 1 + polylogue/maintenance/rebuild_index.py | 7 +- polylogue/sources/revision_backfill.py | 154 ++++++++++++-- .../storage/sqlite/archive_tiers/archive.py | 108 +++++++++- .../archive_tiers/revision_governance.py | 196 ++++++++++++++---- ...test_inactive_candidate_durable_barrier.py | 188 +++++++++++++++++ .../test_rebuild_index_provenance_gate.py | 4 +- tests/unit/maintenance/test_reindex_canary.py | 66 +++--- 8 files changed, 624 insertions(+), 100 deletions(-) create mode 100644 tests/unit/maintenance/test_inactive_candidate_durable_barrier.py diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 9067a3f07a..94bbb64a34 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -1749,3 +1749,4 @@ {"_type":"issue","id":"polylogue-fs1.2","title":"Importer: NeMo Relay ATOF/ATIF runtime spans","description":"Import Hermes observer-layer trace exports as runtime span evidence: pre/post_api_request -> LLM request spans; pre/post_tool_call -> tool execution spans with duration/status; approval hooks -> high-risk decision points; subagent hooks -> delegation graph; error hooks -> retry/fallback taxonomy. ATIF import + enrichment beats inventing another trajectory format — respect Hermes's actual extension seams and make Polylogue the normalizer.","design":"VERIFY first: current NeMo Relay plugin output shape in the Hermes repo (ATOF JSONL / ATIF JSON exported from observer hooks). Ingest route: new artifact kinds in the taxonomy (archive/artifact_taxonomy/) + a spans parser under sources/parsers/, landing as ObservedEvents/actions attached to the session (join key: Hermes session id from the trace envelope -> sessions.native_id). Map: pre/post_api_request pair -> LLM request span (duration, model, provider, token fields if present); pre/post_tool_call -> tool execution span with duration/status (structural outcome — feeds is_error/exit_code lanes where present); approval hooks -> decision-point events; subagent lifecycle -> topology_edges (subagent type); error hooks -> retry/fallback taxonomy events. Spans without a matching archived session become explicit acquisition debt rows, not silent drops.","acceptance_criteria":"`polylogue-fs1.2` adds or updates an origin contract with detector, parser, raw fixture, normalized fixture, parser fingerprint, and fidelity/completeness notes. Ambiguous inputs are handled deterministically. The regression suite proves idempotent replay and visible degraded/missing-field behavior. Verification artifact: OriginSpec detector/parser/fixture/fidelity suite and content-hash export/import roundtrip.","notes":"[Delivery upgrade 2026-07-07T00:05:00Z] Release=K-interop-origin-export; lane=origin-interop-export; readiness=D-horizon-ready; proof=OriginSpec detector/parser/fixture/fidelity suite and content-hash export/import roundtrip. Original readiness=E-spec-needed.\n2026-07-10 Hermes contract refinement: ingest context_injected with profile/session/turn/snapshot-revision correlation; unpaired spans remain explicit acquisition debt. fs1.7 owns atomic spool/export production; this bead owns normalization and reconciliation.\nImplemented and PR opened (not merged): #2876 (feature/hermes/lifecycle-spool-and-bridge).\n\nScope understood: import Hermes observer-layer (NeMo Relay) trace exports as runtime span evidence, normalized and reconciled per the design's mapping (pre/post_api_request -> LLM request spans, pre/post_tool_call -> tool spans, approvals -> decision points, subagent hooks -> delegation evidence, error hooks -> retry/fallback taxonomy).\n\nHonesty constraint documented explicitly in code + PR: the real ATOF/ATIF wire shape was not independently verifiable from this workspace -- no local checkout of the Hermes observer-plugin source was available. sources/parsers/hermes_spans.py implements a documented, testable, best-effort marker-based schema derived from this bead's own design notes and the shared lifecycle taxonomy (hermes_lifecycle.py, fs1.7). Every fidelity capability the parser declares tops out at \"inferred\", never \"exact\", for this reason -- filed as a concrete follow-up (fs1.2.1, not yet created as a bead by me -- flagging here so the orchestrator can file it) to re-verify against real Hermes source and tighten fidelity if it matches without changing the public contract.\n\nWhat changed: sources/parsers/hermes_spans.py (detector/parser/fidelity), wired into the real dispatch pipeline (sources/dispatch.py: detect_provider, lowering, parse_payload -- same path every other origin uses, not a bespoke test-only entrypoint); new artifact-taxonomy classification (archive/artifact_taxonomy/runtime.py).\n\nAC checklist: origin contract with detector/parser/raw fixture/normalized fixture/parser fingerprint/fidelity notes -- satisfied (marker_payload() is the raw-fixture generator used by every test; normalized output is the ParsedSession/session_events produced; fidelity via import_fidelity_declaration()). Ambiguous inputs handled deterministically -- satisfied: unrecognized hook_type -> generic hermes_observer_span event (never dropped, never misclassified as a known kind); malformed span entries (missing hook_type/span_id, non-dict entries) are skipped and counted, not crashing. Idempotent replay -- satisfied and tested (test_atif_parse_is_idempotent_and_deterministic: same document parsed twice -> byte-identical structural output). Visible degraded/missing-field behavior -- satisfied: unpaired spans (start without finish) are counted and surfaced as an explicit degraded fidelity capability with a caveat, never silently dropped.\n\nDesign gap explicitly NOT closed, documented not silently assumed: physical merge of observer spans into the state-db-ingested conversational session's message tree (the design's \"landing as ObservedEvents/actions attached to the session\"). This parser instead produces its own observer-evidence session (observer:) with a read-side correlation helper (hermes_observer_session_id_for) joining by the shared raw Hermes session id -- a physical content-tree merge across two independently-acquired artifacts is a session-identity/lineage design decision (topology_edges/session_links) I judged out of scope for this pass rather than improvising a schema-adjacent change.\n\nVerification: devtools test tests/unit/sources/parsers/test_hermes_spans.py -- 9/9 passed (subset of PR's 43-test combined run). devtools verify --quick exit 0.\n[gpt-5.6-terra integration refinement, 2026-07-14]\n\nReal producer evidence now exists: the bundled NousResearch Hermes observability/nemo_relay plugin emits ATIF v1.7 session documents and append-only ATOF JSONL through actual session, LLM, tool, approval, and subagent callbacks. ATIF import is live. The remaining producer-to-archive gap is ATOF materialization, not schema speculation.\n\nRefine this bead implementation order: retain byte-identified ATOF raw evidence first; incremental reader checkpoints file identity plus byte offset; tolerate partial final lines and rotation/truncation; validate/order/deduplicate events; materialize normalized lifecycle/action evidence idempotently; retain parent/child subagent links; surface unpaired/unmatched records as debt. Never synthesize ATIF from ATOF or duplicate transcript bodies into events. Update OriginSpec fidelity only where real exported fixtures prove a field mapping.","status":"closed","priority":4,"issue_type":"feature","owner":"ezo.dev@gmail.com","created_at":"2026-07-03T04:31:39Z","created_by":"Sinity","updated_at":"2026-07-31T22:35:43Z","closed_at":"2026-07-20T21:34:47Z","close_reason":"Complete in substance across the merged chain — every item of the 2026-07-14 refined implementation order shipped: byte-identified ATOF raw retention + incremental byte-offset reader with partial-line/rotation tolerance (pre-existing append-plan mechanism, verified fs1.2.1 notes); validate/order/dedup + idempotent lifecycle/action materialization (#3103); shared-file multi-session correctness (#3113/flxh); parent/child subagent links from producer-positive marks, fail-closed (#3231); unpaired/unmatched as explicit debt (#3103). Identity composed with profile+artifact-family qualification (#3224/#3225). OriginSpec detector/parser/fixture/fidelity satisfied against REAL producer fixtures with marker-only payloads as negative tests (#3231, fs1.2.1 closed). Force rationale: remaining blocker edge 2qx.1.1 (shared OriginSpec admission kernel/conformance law) is a lane-gate shared suite per the delivery-ac-template-interpretation adjudication (2026-07-07) — not a per-bead requirement; the Hermes origin will conform when that kernel lands, tracked there.","metadata":{"authored_by":"gpt-5.6-terra","authored_on":"2026-07-14"},"labels":["area:ingest","area:substrate","delivery:K-interop-origin-export","delivery:ac-patched","lane:origin-interop-export"],"dependencies":[{"issue_id":"polylogue-fs1.2","depends_on_id":"polylogue-2qx.1.1","type":"blocks","created_at":"2026-07-15T20:55:32Z","created_by":"Sinity","metadata":"{}"},{"issue_id":"polylogue-fs1.2","depends_on_id":"polylogue-fs1","type":"parent-child","created_at":"2026-07-03T06:31:39Z","created_by":"Sinity","metadata":"{}"},{"issue_id":"polylogue-fs1.2","depends_on_id":"polylogue-fs1.2.1","type":"blocks","created_at":"2026-07-14T11:39:19Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":2,"dependent_count":2,"comment_count":0} {"_type":"issue","id":"polylogue-harness-clone-contracts","title":"test(infra): restore archive-clone and embedding-stat contracts","description":"A current-master comparison for polylogue-mn0si exposed two independent shared-harness regressions. Cloned empty archives retained a fresh-bootstrap receipt bound to the template root, and the embedding-stat contract mock ignored bound sqlite_master parameters. Repair both without weakening durable identity checks or changing production embedding semantics.","acceptance_criteria":"1. Cloning the shared empty archive rebinds the fresh durable-bootstrap receipt to the destination identity, opens through the real ArchiveStore route, and leaves the source receipt unchanged. 2. The embedding-stat contract fixture resolves sqlite_master probes from bound parameters and preserves the intended legacy fallback count behavior. 3. The seven focused regressions pass, formatting and lint pass, and a complete current-master full-suite ledger records unrelated residual failures without claiming them fixed. 4. No production durable-train safety check or embedding provider behavior is weakened.","status":"closed","priority":1,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-08-08T06:54:56Z","created_by":"Sinity","updated_at":"2026-08-08T07:16:43Z","closed_at":"2026-08-08T07:16:43Z","close_reason":"Implemented archive-clone destination rebinding and embedding-stat bound-parameter fixtures in PR #3883. The exact seven-test focused batch and quick gate passed; the complete current-master full-suite ledger recorded unrelated residual failures without claiming them fixed.","dependencies":[{"issue_id":"polylogue-harness-clone-contracts","depends_on_id":"polylogue-mn0si","type":"discovered-from","created_at":"2026-08-08T08:54:56Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-seeded-clone-bootstrap-rebind","title":"test(infra): rebind durable bootstrap on seeded archive clones","description":"The immutable seeded-workload archive cache is copied by tests/infra/workload_artifacts.py::clone_seeded_archive. Unlike the empty-archive fixture path repaired in PR #3883, this clone path preserves the source artifact's fresh durable-bootstrap marker byte-for-byte. Its source/user durable identities no longer match the destination files, so the first real ArchiveStore open fails closed with fresh durable bootstrap marker durable identity mismatch. Candidate/canary tests that begin from seeded workload clones therefore cannot exercise the production route.","design":"Give all private seeded-archive clones destination-bound fresh-bootstrap evidence immediately after the copy or reflink completes, using the production receipt writer already used by the empty-archive clone path. Keep the immutable source artifact and its marker unchanged. Add a focused regression that records source marker bytes, clones, verifies marker divergence and destination identity, then opens the clone through the real ArchiveStore route.","acceptance_criteria":"1. clone_seeded_archive replaces only the copied fresh durable-bootstrap marker with a destination-bound production marker after making the clone writable. 2. The immutable cached artifact marker remains byte-identical. 3. Reflink and copy fallback clones both use the same rebinding boundary. 4. A real ArchiveStore writer open succeeds on the clone and durable continuity checks remain fail-closed for an actually foreign marker. 5. Focused workload-artifact, temp-policy, and durable-train regressions plus devtools verify --quick pass.","notes":"Implemented destination-bound bootstrap rebinding in tests/infra/workload_artifacts.py for both reflink and copy fallback clones. Added real ArchiveStore open, immutable-source-marker preservation, and foreign-marker fail-closed regressions. Verification: 26 focused tests passed; mypy strict passed on both touched files; devtools verify --quick run 20260808T083029Z-quick-3360251-3cef0cf0 passed all 24 steps.","status":"closed","priority":1,"issue_type":"bug","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-08-08T08:20:27Z","created_by":"Sinity","updated_at":"2026-08-08T08:33:10Z","closed_at":"2026-08-08T08:33:10Z","close_reason":"Seeded archive clones now replace copied bootstrap evidence with a destination-bound production marker on both clone paths. Real opens pass, the immutable artifact is unchanged, a restored foreign marker still fails closed, 26 focused tests pass, and quick verification is green.","labels":["area:test-infra","lane:reindex"],"dependencies":[{"issue_id":"polylogue-seeded-clone-bootstrap-rebind","depends_on_id":"polylogue-harness-clone-contracts","type":"discovered-from","created_at":"2026-08-08T10:20:27Z","created_by":"Sinity","metadata":"{}"},{"issue_id":"polylogue-seeded-clone-bootstrap-rebind","depends_on_id":"polylogue-inactive-candidate-durable-barrier","type":"discovered-from","created_at":"2026-08-08T10:20:27Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-inactive-candidate-durable-barrier","title":"fix: isolate inactive candidates from durable-tier writes","description":"The exact current-master rebuild route cannot open an owned inactive generation. IndexGenerationStore intentionally gives each candidate read-through symlinks to source.db, user.db, embeddings.db, ops.db, and blob, but ArchiveStore initializes the generation as an active writable archive. Durable change-train admission correctly rejects source.db at the symlink boundary. If that check were bypassed, the candidate replay path would open source.db and user.db writable through the links, publish blobs, update authority census and parse-state rows, and therefore mutate the phase-2 frozen source during phase 3. This defect currently accounts for 69 current-master full-suite failures and blocks polylogue-818fy.","design":"Introduce an explicit owned-inactive-candidate access mode. Validate generation ownership and exact read-through targets, initialize and write only the candidate index.db, open source.db and user.db read-only, and make every durable or blob mutation attempt fail before side effects. Split replay source remediation from candidate application: the candidate route must consume a complete, fingerprint-current frozen source census and persisted authority decisions, derive and write only index-side rows and receipts, and fail with a typed source-remediation-required outcome when the frozen source is incomplete or would need a changed authority decision. Do not relax active-archive durable change-train identity checks. Bind the candidate receipt to unchanged source/user identities and byte digests.","acceptance_criteria":"1. The real daemon/offline owned-inactive generation route opens an intentionally symlinked candidate without running active-root durable bootstrap against the generation root. 2. Candidate index writes succeed while source.db and user.db are opened read-only; blob publication, raw census, authority, parse-state, assertion, and other durable writes fail before side effects. 3. Candidate replay consumes only a complete current-parser census and persisted source authority. Missing/stale census or any replay decision that would change source state returns a typed source-remediation-required failure before candidate readiness. 4. A real-route fixture hashes source.db, user.db, and the blob namespace before and after a successful inactive build and proves byte and identity preservation, no active-pointer mutation, and a populated candidate index. 5. Adversarial tests restore the prior symlink/bootstrap path, make source/user writable, or permit one source mutation and fail. 6. Active archive startup still rejects unsafe durable symlinks or replaced files. 7. Focused rebuild, revision-governance, generation, and durable-train suites plus devtools verify --quick pass. 8. Production build, acceptance, promotion, restart, and postflight remain under their existing phase Beads.","status":"in_progress","priority":0,"issue_type":"bug","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-08-08T07:44:07Z","created_by":"Sinity","updated_at":"2026-08-08T07:44:47Z","started_at":"2026-08-08T07:44:47Z","lease_expires_at":"2026-08-08T07:49:47Z","heartbeat_at":"2026-08-08T07:44:47Z","labels":["area:maintenance","lane:reindex"],"dependency_count":0,"dependent_count":1,"comment_count":0} diff --git a/polylogue/maintenance/rebuild_index.py b/polylogue/maintenance/rebuild_index.py index 1fbb89beec..ece2c75839 100644 --- a/polylogue/maintenance/rebuild_index.py +++ b/polylogue/maintenance/rebuild_index.py @@ -1413,7 +1413,10 @@ async def _rebuild_index_from_source_owned( verify_archive, ) from polylogue.maintenance.replay import rebuild_index_from_source as replay_source - from polylogue.sources.revision_backfill import RebuildDeadlineExceededError + from polylogue.sources.revision_backfill import ( + RebuildDeadlineExceededError, + require_current_parser_source_census, + ) from polylogue.storage.archive_readiness import archive_readiness_status from polylogue.storage.index_generation import ( IndexGenerationStore, @@ -1458,6 +1461,7 @@ async def _rebuild_index_from_source_owned( page = None pass_started_at_ms = int(time.time() * 1000) if resumable_full_source: + require_current_parser_source_census(root) if request.operation_id is not None: transaction = generation_store.load_transaction(request.operation_id) transaction = _reconcile_active_generation_transaction(generation_store, transaction) @@ -1549,6 +1553,7 @@ async def _rebuild_index_from_source_owned( raw_count, selected_raw_ids, skipped_by_blob_limit_count = select_rebuild_raw_ids(request) selection_elapsed_s = time.perf_counter() - selection_started_at selected_raw_count = len(selected_raw_ids) + require_current_parser_source_census(root, selected_raw_ids=selected_raw_ids) provenance.validate() generation = generation_store.create(source_snapshot=rebuild_source_evidence_snapshot(root)) try: diff --git a/polylogue/sources/revision_backfill.py b/polylogue/sources/revision_backfill.py index 5c6f856500..9bd1b56fcc 100644 --- a/polylogue/sources/revision_backfill.py +++ b/polylogue/sources/revision_backfill.py @@ -60,6 +60,7 @@ SUPERSEDED_MEMBERSHIP_FINGERPRINTS, ) from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore +from polylogue.storage.sqlite.archive_tiers.revision_governance import FrozenSourceRemediationRequiredError from polylogue.storage.sqlite.archive_tiers.write import PreparedSessionRows, prepare_session_rows _LOGGER = _polylogue_logging.get_logger(__name__) @@ -806,6 +807,101 @@ def bind_byte_proven_older_member(raw_id: str, logical_key: str) -> None: return state +def _load_frozen_revision_evidence( + archive: ArchiveStore, + spill: _ParsedSessionSpill, + *, + selected_raw_ids: list[str] | None, + max_payload_bytes: int | None, + ingest_workers: int, + prefetch_cache: RawParsePrefetchCache | None, +) -> _RevisionCensusState: + """Parse a phase-2 source snapshot without changing its durable ledger.""" + expanded_raw_ids, _logical_keys = archive.expand_raw_membership_selection(selected_raw_ids) + require_current_parser_source_census( + archive.archive_root, + selected_raw_ids=expanded_raw_ids if selected_raw_ids is not None else None, + ) + rows = archive.raw_membership_census_rows(expanded_raw_ids if selected_raw_ids is not None else None) + if max_payload_bytes is not None: + payload_sizes = archive.raw_payload_sizes([raw_id for raw_id, _source_index in rows]) + total_payload_bytes = sum(payload_sizes.values()) + oversized = [raw_id for raw_id, size in payload_sizes.items() if size > max_payload_bytes] + if oversized or total_payload_bytes > max_payload_bytes: + raise RawRevisionReplayResourceBlockedError( + sorted(oversized or payload_sizes), max_payload_bytes, total_payload_bytes + ) + parseable_raw_ids = [raw_id for raw_id, source_index in rows if source_index >= 0] + parsed_outcomes = _parse_retained_raws( + archive, + parseable_raw_ids, + ingest_workers=ingest_workers, + prefetch_cache=prefetch_cache, + ) + state = _RevisionCensusState(0, 0, 0, set(), {}, {}) + for raw_id, source_index in rows: + state.scanned += 1 + state.censused.add(raw_id) + if source_index < 0: + state.quarantined += 1 + continue + outcome = parsed_outcomes[raw_id] + if isinstance(outcome, Exception): + raise FrozenSourceRemediationRequiredError( + f"inactive candidate could not parse frozen raw {raw_id}: {type(outcome).__name__}: {outcome}" + ) from outcome + sessions, payload_bytes, revision_kind = outcome + spill.add(raw_id, sessions, payload_bytes=payload_bytes) + state.classified += int(len(sessions) == 1) + if revision_kind is RawRevisionKind.UNKNOWN: + for session in sessions: + logical_key = f"{session.source_name.value}:{session.provider_session_id}" + state.membership_candidates.setdefault(logical_key, set()).add(raw_id) + return state + + +def require_current_parser_source_census( + archive_root: Path, + *, + selected_raw_ids: Sequence[str] | None = None, +) -> None: + """Require phase-2 parser receipts before allocating an index candidate.""" + stale_raw_ids: list[str] = [] + selections: tuple[tuple[str, ...] | None, ...] + if selected_raw_ids is None: + selections = (None,) + else: + selections = tuple( + tuple(selected_raw_ids[offset : offset + 500]) for offset in range(0, len(selected_raw_ids), 500) + ) + with sqlite3.connect(f"file:{archive_root / 'source.db'}?mode=ro", uri=True) as source_conn: + for selection in selections: + where = "" if selection is None else f"WHERE r.raw_id IN ({','.join('?' for _ in selection)})" + params: tuple[object, ...] = () if selection is None else selection + stale_raw_ids.extend( + str(row[0]) + for row in source_conn.execute( + f""" + SELECT r.raw_id + FROM raw_sessions AS r + LEFT JOIN raw_authority_parser_census AS c ON c.raw_id = r.raw_id + {where} + {"WHERE" if selection is None else "AND"} NOT COALESCE( + c.parser_fingerprint = ? AND c.status = 'complete', 0 + ) + ORDER BY r.raw_id + """, + (*params, RAW_AUTHORITY_PARSER_FINGERPRINT), + ) + ) + if stale_raw_ids: + sample = ", ".join(stale_raw_ids[:5]) + raise FrozenSourceRemediationRequiredError( + "inactive candidate requires a complete current-parser source census; " + f"{len(stale_raw_ids)} raw(s) are stale or incomplete (sample: {sample})" + ) + + def census_historical_revision_evidence( archive_root: Path, *, @@ -1089,15 +1185,25 @@ def backfill_historical_revision_evidence( prepare_pool if prepare_pool is not None else nullcontext(), ): census_started = time.perf_counter() - census = _census_historical_revision_evidence( - archive, - spill, - selected_raw_ids=selected_raw_ids, - max_payload_bytes=max_payload_bytes, - ingest_workers=ingest_workers, - commit_batch_size=commit_batch_size, - prefetch_cache=prefetch_cache, - ) + if owned_inactive_generation is not None: + census = _load_frozen_revision_evidence( + archive, + spill, + selected_raw_ids=selected_raw_ids, + max_payload_bytes=max_payload_bytes, + ingest_workers=ingest_workers, + prefetch_cache=prefetch_cache, + ) + else: + census = _census_historical_revision_evidence( + archive, + spill, + selected_raw_ids=selected_raw_ids, + max_payload_bytes=max_payload_bytes, + ingest_workers=ingest_workers, + commit_batch_size=commit_batch_size, + prefetch_cache=prefetch_cache, + ) stage_timings["census"] = time.perf_counter() - census_started receipt_started = time.perf_counter() censused_raw_ids, _censused_keys = archive.expand_raw_membership_selection(selected_raw_ids) @@ -1106,7 +1212,8 @@ def backfill_historical_revision_evidence( # any index plan. Commit the source census first so the separate # durable receipt writer observes one complete source snapshot. archive.commit() - _record_raw_authority_parser_census(archive_root, tuple(censused_raw_ids)) + if owned_inactive_generation is None: + _record_raw_authority_parser_census(archive_root, tuple(censused_raw_ids)) stage_timings["census_receipt"] = time.perf_counter() - receipt_started membership_candidates = census.membership_candidates provisional_full_raw_ids = census.provisional_full_raw_ids @@ -1181,13 +1288,17 @@ def commit_replay_unit() -> None: # different session's content, which must not be quarantined # as "divergent evidence". classify_started = time.perf_counter() - plan = archive.classify_raw_revision_cohort_for_rebuild_repair( - logical_key, - # Batched replay defers the classification's source.db - # authority updates into the same batch window as the replay - # writes (idempotent, re-derived on resume -- see - # classify_raw_revision_cohort_for_rebuild_repair's docstring). - manage_transaction=not replay_batched, + plan = ( + archive.classify_raw_revision_cohort_for_frozen_candidate(logical_key) + if owned_inactive_generation is not None + else archive.classify_raw_revision_cohort_for_rebuild_repair( + logical_key, + # Batched replay defers the classification's source.db + # authority updates into the same batch window as the replay + # writes (idempotent, re-derived on resume -- see + # classify_raw_revision_cohort_for_rebuild_repair's docstring). + manage_transaction=not replay_batched, + ) ) stage_timings["replay.classify_cohort"] = stage_timings.get("replay.classify_cohort", 0.0) + ( time.perf_counter() - classify_started @@ -1197,7 +1308,13 @@ def commit_replay_unit() -> None: # still carry semantic evidence. Move only that full-only # cohort to membership governance and let parsed-content # prefix rules decide it; append chains remain byte-governed. - for raw_id in archive.convertible_full_revision_raw_ids(logical_key): + convertible = archive.convertible_full_revision_raw_ids(logical_key) + if owned_inactive_generation is not None and convertible: + raise FrozenSourceRemediationRequiredError( + "inactive candidate found a full-revision cohort that still requires membership " + f"remediation in frozen source: {logical_key}" + ) + for raw_id in convertible: spill_started = time.perf_counter() sessions, _payload_bytes = spill.for_raw(archive, raw_id) stage_timings["spill_load"] = stage_timings.get("spill_load", 0.0) + ( @@ -2710,6 +2827,7 @@ def _parse_stream_raw( "census_historical_revision_evidence", "census_parse_worker", "record_resource_blocked_revision_census", + "require_current_parser_source_census", "uncensused_historical_revision_raw_ids", "parse_retained_raw_sessions", ] diff --git a/polylogue/storage/sqlite/archive_tiers/archive.py b/polylogue/storage/sqlite/archive_tiers/archive.py index 5baff3ee9f..915a475c5f 100644 --- a/polylogue/storage/sqlite/archive_tiers/archive.py +++ b/polylogue/storage/sqlite/archive_tiers/archive.py @@ -11,6 +11,7 @@ from __future__ import annotations +import hashlib import json import math import sqlite3 @@ -22,7 +23,7 @@ from datetime import UTC, datetime from pathlib import Path from types import TracebackType -from typing import Any, BinaryIO, Literal, TypedDict, cast +from typing import IO, Any, BinaryIO, Literal, NoReturn, TypedDict, cast from polylogue.annotations.batch import AnnotationBatch from polylogue.annotations.schema import AnnotationSchema @@ -134,6 +135,8 @@ from polylogue.insights.tool_usage import ToolUsageInsight, ToolUsageInsightQuery, build_tool_usage_insight from polylogue.pipeline.ids import SessionRevisionProjection from polylogue.sources.parsers.base import ParsedSession +from polylogue.storage.blob_publication import ArchiveBlobPublisher +from polylogue.storage.blob_store import Heartbeat from polylogue.storage.fts.sql import ( FTS_BULK_SESSION_WRITE_GUARD, delete_session_identity_rows_sql, @@ -183,6 +186,7 @@ apply_raw_revision_replay, bind_raw_revision, blob_path_for_hash, + classify_raw_revision_cohort_for_frozen_candidate, classify_raw_revision_cohort_for_live_watch, classify_raw_revision_cohort_for_rebuild_repair, classify_untyped_full_revision_groups, @@ -1637,6 +1641,32 @@ def __getattr__(self, name: str) -> Any: ) +class InactiveCandidateDurableWriteError(RuntimeError): + """An inactive generation attempted to mutate read-through durable state.""" + + +class _InactiveCandidateBlobPublisher(ArchiveBlobPublisher): + """Read frozen blob bytes while refusing candidate publication attempts.""" + + @staticmethod + def _refuse() -> NoReturn: + raise InactiveCandidateDurableWriteError( + "inactive candidate generations may read frozen blobs but may not publish or replace blob bytes" + ) + + def write_from_path(self, source: Path, *, heartbeat: Heartbeat | None = None) -> tuple[str, int]: + del source, heartbeat + return self._refuse() + + def write_from_fileobj(self, source: IO[bytes], *, heartbeat: Heartbeat | None = None) -> tuple[str, int]: + del source, heartbeat + return self._refuse() + + def write_from_bytes(self, data: bytes) -> tuple[str, int]: + del data + return self._refuse() + + class ArchiveStore: """Minimal archive-root façade for archive source/index/user tiers.""" @@ -1653,6 +1683,8 @@ def __init__( if source_tier_acquisition and read_only: raise ValueError("source_tier_acquisition mode is a writer mode; read_only must be False") self._source_tier_acquisition = source_tier_acquisition + self._owned_inactive_generation = owned_inactive_generation + self._inactive_candidate_durable_read_only = owned_inactive_generation is not None self._active_writer_lease = None if not read_only: from polylogue.paths import archive_root as configured_archive_root @@ -1689,10 +1721,29 @@ def __init__( or Path(generation.index_path).parent.resolve(strict=True) != archive_root.resolve(strict=True) ): raise RuntimeError("inactive index generation ownership validation failed") + declared_archive_root = Path(generation.archive_root).resolve(strict=True) + if declared_archive_root != generation_archive_root.resolve(strict=True): + raise RuntimeError("inactive index generation archive-root binding is stale") + for filename in ("source.db", "user.db", "embeddings.db", "ops.db", "blob"): + expected = declared_archive_root / filename + candidate = archive_root / filename + if expected.exists() or expected.is_symlink(): + if not candidate.is_symlink() or candidate.resolve(strict=True) != expected.resolve( + strict=True + ): + raise RuntimeError( + f"inactive index generation has an invalid read-through target: {filename}" + ) + elif candidate.exists() or candidate.is_symlink(): + raise RuntimeError(f"inactive index generation invented a read-through target: {filename}") try: self._initialize_store( archive_root, - initialize=initialize and not source_tier_acquisition, + # The generation store already initialized the candidate's + # sole owned tier, index.db. Active-root initialization would + # reinterpret deliberate source/user read-through symlinks as + # writable durable tiers. + initialize=initialize and not source_tier_acquisition and owned_inactive_generation is None, read_only=read_only, read_timeout=read_timeout, # polylogue-623q: only ever True for a write connection against @@ -1753,8 +1804,6 @@ def _initialize_store( f"source-tier acquisition refused: durable tier {spec.filename} " f"user_version {current} != expected {spec.version}" ) - from polylogue.storage.blob_publication import ArchiveBlobPublisher - self._conn = cast(sqlite3.Connection, _SourceTierOnlyIndexConnection()) self._user_tier_attached = False self._tags_relation = "session_tags" @@ -1767,7 +1816,11 @@ def _initialize_store( self._conn = sqlite3.connect(f"file:{self.index_db_path}?mode=ro", uri=True, timeout=read_timeout) pragma_statements = READ_CONNECTION_PRAGMA_STATEMENTS else: - self._conn = sqlite3.connect(self.index_db_path) + self._conn = ( + sqlite3.connect(f"file:{self.index_db_path}?mode=rw", uri=True) + if self._inactive_candidate_durable_read_only + else sqlite3.connect(self.index_db_path) + ) pragma_statements = ( BULK_BUILD_WRITE_CONNECTION_PRAGMA_STATEMENTS if bulk_build_profile @@ -1793,9 +1846,10 @@ def _initialize_store( self._user_tier_attached = False self._tags_relation = "session_tags" if not read_only: - from polylogue.storage.blob_publication import ArchiveBlobPublisher - - self._blob_publisher = ArchiveBlobPublisher(self.source_db_path, self.archive_root / "blob") + publisher_type = ( + _InactiveCandidateBlobPublisher if self._inactive_candidate_durable_read_only else ArchiveBlobPublisher + ) + self._blob_publisher = publisher_type(self.source_db_path, self.archive_root / "blob") self._attach_user_tier_if_present() @classmethod @@ -1893,7 +1947,11 @@ def interrupt_reads(self) -> None: def _ensure_source_conn(self) -> sqlite3.Connection: """Return the persistent source.db write connection, opening it lazily.""" if self._source_conn is None: - conn = sqlite3.connect(self.source_db_path) + if self._inactive_candidate_durable_read_only: + conn = sqlite3.connect(f"file:{self.source_db_path}?mode=ro", uri=True) + conn.execute("PRAGMA query_only = ON") + else: + conn = sqlite3.connect(self.source_db_path) conn.execute("PRAGMA foreign_keys = ON") self._source_conn = conn return self._source_conn @@ -2019,6 +2077,29 @@ def _preacquire_attachment_blobs( acquired: dict[int, tuple[bytes | None, int, str]] = {} refs: list[ArchiveSourceBlobRef] = [] for attachment in session.attachments: + if self._inactive_candidate_durable_read_only: + if attachment.inline_bytes is not None: + hash_hex = hashlib.sha256(attachment.inline_bytes).hexdigest() + size = len(attachment.inline_bytes) + elif attachment.precomputed_blob is not None: + hash_hex, size = attachment.precomputed_blob + else: + continue + blob_path = self._blob_publisher.blob_path(hash_hex) + if not blob_path.is_file() or blob_path.stat().st_size != size: + raise InactiveCandidateDurableWriteError( + "inactive candidate requires attachment bytes to be present in the frozen blob namespace: " + f"{hash_hex}" + ) + with blob_path.open("rb") as handle: + stored_hash = hashlib.file_digest(handle, "sha256").hexdigest() + if stored_hash != hash_hex: + raise InactiveCandidateDurableWriteError( + "inactive candidate found attachment bytes that do not match the frozen blob identity: " + f"{hash_hex}" + ) + acquired[id(attachment)] = (bytes.fromhex(hash_hex), size, "acquired") + continue if attachment.inline_bytes is None: continue hash_hex, size = self._blob_publisher.write_from_bytes(attachment.inline_bytes) @@ -2351,6 +2432,9 @@ def classify_raw_revision_cohort_for_rebuild_repair( manage_transaction=manage_transaction, ) + def classify_raw_revision_cohort_for_frozen_candidate(self, logical_source_key: str) -> RevisionReplayPlan: + return classify_raw_revision_cohort_for_frozen_candidate(self, logical_source_key) + def classify_raw_revision_cohort_for_live_watch( self, logical_source_key: str, @@ -5951,7 +6035,11 @@ def delete_sessions(self, session_ids: tuple[str, ...]) -> int: def _attach_user_tier_if_present(self) -> None: if self._user_tier_attached or not self.user_db_path.exists(): return - user_db_uri = f"file:{self.user_db_path}?mode=ro" if self._read_only else str(self.user_db_path) + user_db_uri = ( + f"file:{self.user_db_path}?mode=ro" + if self._read_only or self._inactive_candidate_durable_read_only + else str(self.user_db_path) + ) self._conn.execute("ATTACH DATABASE ? AS user_tier", (user_db_uri,)) self._user_tier_attached = True self._tags_relation = _all_session_tags_sql() diff --git a/polylogue/storage/sqlite/archive_tiers/revision_governance.py b/polylogue/storage/sqlite/archive_tiers/revision_governance.py index 91bb3be3c7..dc80ee1792 100644 --- a/polylogue/storage/sqlite/archive_tiers/revision_governance.py +++ b/polylogue/storage/sqlite/archive_tiers/revision_governance.py @@ -206,6 +206,15 @@ class MembershipReplayConflictError(RuntimeError): """ +class FrozenSourceRemediationRequiredError(RuntimeError): + """Candidate replay found source authority that phase 2 must update.""" + + +def _is_frozen_candidate(store: RawRevisionGovernanceHost) -> bool: + """Return whether this host is the owned inactive-generation adapter.""" + return bool(getattr(store, "_inactive_candidate_durable_read_only", False)) + + class RawRevisionGovernanceHost(Protocol): """The narrow slice of ``ArchiveStore`` this module is allowed to touch. @@ -982,6 +991,21 @@ def classify_raw_revision_cohort_for_rebuild_repair( logical_source_key, check_source_path_identity_split=True, manage_transaction=manage_transaction, + source_effects=True, + ) + + +def classify_raw_revision_cohort_for_frozen_candidate( + store: RawRevisionGovernanceHost, + logical_source_key: str, +) -> RevisionReplayPlan: + """Re-derive byte authority and require the frozen source to match it.""" + return _classify_raw_revision_cohort( + store, + logical_source_key, + check_source_path_identity_split=True, + manage_transaction=False, + source_effects=False, ) @@ -1011,6 +1035,7 @@ def classify_raw_revision_cohort_for_live_watch( logical_source_key, check_source_path_identity_split=False, manage_transaction=manage_transaction, + source_effects=True, ) @@ -1020,6 +1045,7 @@ def _classify_raw_revision_cohort( *, check_source_path_identity_split: bool, manage_transaction: bool = True, + source_effects: bool, ) -> RevisionReplayPlan: """Promote only a unique byte-prefix full chain and contiguous appends. @@ -1139,28 +1165,59 @@ def open_payload(blob_hash: str = str(row[1])) -> BinaryIO: for dup_decision in decisions: if dup_decision.relation == "duplicate" and dup_decision.duplicate_of_raw_id is not None: generation_by_raw_id[dup_decision.raw_id] = generation_by_raw_id.get(dup_decision.duplicate_of_raw_id, 0) - with source_conn if manage_transaction else nullcontext(): + if source_effects: + with source_conn if manage_transaction else nullcontext(): + for row in full_rows: + raw_id = str(row[0]) + decision = by_raw_id.get(raw_id) + authority = decision.authority if decision is not None else RawRevisionAuthority.QUARANTINED + predecessor_raw_id = decision.predecessor_raw_id if decision is not None else None + source_conn.execute( + """ + UPDATE raw_sessions + SET revision_authority = ?, predecessor_raw_id = ?, baseline_raw_id = ?, + acquisition_generation = ? + WHERE raw_id = ? + """, + ( + authority.value, + predecessor_raw_id, + baseline_raw_id if authority is RawRevisionAuthority.BYTE_PROVEN else None, + generation_by_raw_id.get(raw_id, 0), + raw_id, + ), + ) + _promote_contiguous_append_evidence(source_conn, logical_source_key) + else: for row in full_rows: raw_id = str(row[0]) decision = by_raw_id.get(raw_id) authority = decision.authority if decision is not None else RawRevisionAuthority.QUARANTINED predecessor_raw_id = decision.predecessor_raw_id if decision is not None else None - source_conn.execute( + expected = ( + authority.value, + predecessor_raw_id, + baseline_raw_id if authority is RawRevisionAuthority.BYTE_PROVEN else None, + generation_by_raw_id.get(raw_id, 0), + ) + persisted = source_conn.execute( """ - UPDATE raw_sessions - SET revision_authority = ?, predecessor_raw_id = ?, baseline_raw_id = ?, - acquisition_generation = ? - WHERE raw_id = ? + SELECT revision_authority, predecessor_raw_id, baseline_raw_id, + acquisition_generation + FROM raw_sessions WHERE raw_id = ? """, - ( - authority.value, - predecessor_raw_id, - baseline_raw_id if authority is RawRevisionAuthority.BYTE_PROVEN else None, - generation_by_raw_id.get(raw_id, 0), - raw_id, - ), + (raw_id,), + ).fetchone() + if persisted is None or tuple(persisted) != expected: + raise FrozenSourceRemediationRequiredError( + "inactive candidate re-derived different byte authority for frozen raw " + f"{raw_id}; complete source remediation before candidate construction" + ) + if _contiguous_append_authority_drift_exists(source_conn, logical_source_key): + raise FrozenSourceRemediationRequiredError( + "inactive candidate found promotable append authority in the frozen source; " + "complete source remediation before candidate construction" ) - _promote_contiguous_append_evidence(source_conn, logical_source_key) return raw_revision_replay_plan(store, logical_source_key) @@ -1237,10 +1294,11 @@ def open_payload(blob_hash: str = blob_hash) -> BinaryIO: return groups -def _promote_contiguous_append_evidence(conn: sqlite3.Connection, logical_source_key: str) -> None: - while True: - candidates = conn.execute( - """ +def _contiguous_append_authority_candidates( + conn: sqlite3.Connection, logical_source_key: str +) -> tuple[sqlite3.Row | tuple[object, ...], ...]: + candidates = conn.execute( + """ SELECT child.raw_id, parent.raw_id, COALESCE(parent.baseline_raw_id, parent.raw_id), parent.acquisition_generation + 1 FROM raw_sessions AS child @@ -1262,12 +1320,21 @@ def _promote_contiguous_append_evidence(conn: sqlite3.Connection, logical_source OR child.acquisition_generation != parent.acquisition_generation + 1 ) """, - (logical_source_key,), - ).fetchall() - by_child: dict[str, list[sqlite3.Row | tuple[object, ...]]] = {} - for row in candidates: - by_child.setdefault(str(row[0]), []).append(row) - promotable = [rows[0] for rows in by_child.values() if len(rows) == 1] + (logical_source_key,), + ).fetchall() + by_child: dict[str, list[sqlite3.Row | tuple[object, ...]]] = {} + for row in candidates: + by_child.setdefault(str(row[0]), []).append(row) + return tuple(rows[0] for rows in by_child.values() if len(rows) == 1) + + +def _contiguous_append_authority_drift_exists(conn: sqlite3.Connection, logical_source_key: str) -> bool: + return bool(_contiguous_append_authority_candidates(conn, logical_source_key)) + + +def _promote_contiguous_append_evidence(conn: sqlite3.Connection, logical_source_key: str) -> None: + while True: + promotable = _contiguous_append_authority_candidates(conn, logical_source_key) if not promotable: return changed = 0 @@ -2272,8 +2339,9 @@ def apply_raw_revision_replay( # semantics are identical to a smaller batch. store.commit() store._blob_publisher.flush() - for raw_id, refs in attachment_refs_by_raw_id.items(): - write_source_blob_refs(store._ensure_source_conn(), raw_id, refs) + if not _is_frozen_candidate(store): + for raw_id, refs in attachment_refs_by_raw_id.items(): + write_source_blob_refs(store._ensure_source_conn(), raw_id, refs) session_ids: set[str] = set() with store._conn if manage_transaction else nullcontext(): existing_head = store._conn.execute( @@ -2456,6 +2524,8 @@ def apply_raw_revision_replay( } for raw_id in terminal_raw_ids: provider, _blob_hash, _source_path, _kind, _blob_size = raw_revision_descriptor(store, raw_id) + if _is_frozen_candidate(store): + continue if manage_transaction: mark_raw_parse_succeeded(store, raw_id, provider=provider) else: @@ -2553,7 +2623,8 @@ def apply_raw_membership_classification( # a non-empty flush takes its separate source.db write lock. store.commit() store._blob_publisher.flush() - write_source_blob_refs(conn, accepted_raw_id, refs) + if not _is_frozen_candidate(store): + write_source_blob_refs(conn, accepted_raw_id, refs) with store._conn if manage_transaction else nullcontext(): existing_head = store._conn.execute( """ @@ -2836,27 +2907,52 @@ def apply_raw_membership_classification( if yield_to_head_raw_id is None: decisions[accepted_raw_id] = MembershipDecision.APPLIED - with conn if manage_transaction else nullcontext(): + if _is_frozen_candidate(store): for raw_id, decision in decisions.items(): - conn.execute( + expected = ( + decision.value, + "quarantined" + if decision in {MembershipDecision.AMBIGUOUS, MembershipDecision.DEFERRED} + else "byte_proven", + classification.accepted_raw_ids.index(raw_id) if raw_id in classification.accepted_raw_ids else 0, + ) + persisted = conn.execute( """ - UPDATE raw_session_memberships - SET decision = ?, decided_at_ms = ?, - revision_authority = ?, - acquisition_generation = ? + SELECT decision, revision_authority, acquisition_generation + FROM raw_session_memberships WHERE raw_id = ? AND logical_source_key = ? """, - ( - decision, - decided_at_ms, - "quarantined" - if decision in {MembershipDecision.AMBIGUOUS, MembershipDecision.DEFERRED} - else "byte_proven", - classification.accepted_raw_ids.index(raw_id) if raw_id in classification.accepted_raw_ids else 0, - raw_id, - logical_source_key, - ), - ) + (raw_id, logical_source_key), + ).fetchone() + if persisted is None or tuple(persisted) != expected: + raise FrozenSourceRemediationRequiredError( + "inactive candidate re-derived different membership authority for frozen raw " + f"{raw_id}; complete source remediation before candidate construction" + ) + else: + with conn if manage_transaction else nullcontext(): + for raw_id, decision in decisions.items(): + conn.execute( + """ + UPDATE raw_session_memberships + SET decision = ?, decided_at_ms = ?, + revision_authority = ?, + acquisition_generation = ? + WHERE raw_id = ? AND logical_source_key = ? + """, + ( + decision, + decided_at_ms, + "quarantined" + if decision in {MembershipDecision.AMBIGUOUS, MembershipDecision.DEFERRED} + else "byte_proven", + classification.accepted_raw_ids.index(raw_id) + if raw_id in classification.accepted_raw_ids + else 0, + raw_id, + logical_source_key, + ), + ) for raw_id in decisions: complete = conn.execute( """ @@ -2870,6 +2966,13 @@ def apply_raw_membership_classification( """, (raw_id,), ).fetchone() + if _is_frozen_candidate(store): + if complete is None or not bool(complete[0]): + raise FrozenSourceRemediationRequiredError( + "inactive candidate found incomplete frozen membership authority for raw " + f"{raw_id}; complete source remediation before candidate construction" + ) + continue if complete is not None and bool(complete[0]): provider, _blob_hash, _source_path, _kind, _blob_size = raw_revision_descriptor(store, raw_id) if manage_transaction: @@ -2997,9 +3100,10 @@ def _index_parsed_for_retained_raw( prepared=prepared, ) except Exception as exc: - finalize_raw_parse_state(store, raw_id, state=_raw_parse_failure_state(provider, exc)) + if not _is_frozen_candidate(store): + finalize_raw_parse_state(store, raw_id, state=_raw_parse_failure_state(provider, exc)) raise - if finalize_raw_parse: + if finalize_raw_parse and not _is_frozen_candidate(store): success_state = _raw_parse_success_state(provider) if manage_transaction: finalize_raw_parse_state(store, raw_id, state=success_state) diff --git a/tests/unit/maintenance/test_inactive_candidate_durable_barrier.py b/tests/unit/maintenance/test_inactive_candidate_durable_barrier.py new file mode 100644 index 0000000000..37e7032db7 --- /dev/null +++ b/tests/unit/maintenance/test_inactive_candidate_durable_barrier.py @@ -0,0 +1,188 @@ +"""Owned inactive generations may write only their derived index tier.""" + +from __future__ import annotations + +import hashlib +import sqlite3 +from pathlib import Path + +import pytest + +from polylogue.core.enums import Provider +from polylogue.maintenance.rebuild_index import RebuildIndexRequest, rebuild_index_from_source_sync +from polylogue.sources.revision_backfill import census_historical_revision_evidence +from polylogue.storage.index_generation import IndexGenerationStore, source_revision_snapshot +from polylogue.storage.sqlite.archive_tiers.archive import ( + ArchiveStore, + InactiveCandidateDurableWriteError, +) +from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root +from polylogue.storage.sqlite.archive_tiers.revision_governance import FrozenSourceRemediationRequiredError +from polylogue.storage.sqlite.durable_change_train import DurableChangeTrainError +from tests.infra.rebuild_receipt import write_valid_rebuild_receipt +from tests.infra.revision_backfill_benchmark import build_independent_raw_corpus + + +def _file_evidence(path: Path) -> tuple[int, int, str]: + stat = path.stat() + return stat.st_dev, stat.st_ino, hashlib.sha256(path.read_bytes()).hexdigest() + + +def _blob_evidence(root: Path) -> tuple[tuple[str, int, str], ...]: + return tuple( + (str(path.relative_to(root)), path.stat().st_size, hashlib.sha256(path.read_bytes()).hexdigest()) + for path in sorted(root.rglob("*")) + if path.is_file() + ) + + +def _prepare_frozen_source(root: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + build_independent_raw_corpus(root, raw_count=1, avg_payload_bytes=1_000) + monkeypatch.setenv("POLYLOGUE_ARCHIVE_ROOT", str(root)) + census = census_historical_revision_evidence(root) + assert census.scanned == 1 + assert census.classified_full == 1 + with sqlite3.connect(root / "source.db") as source: + source.execute( + """ + UPDATE raw_sessions + SET revision_authority = 'byte_proven', baseline_raw_id = raw_id, + predecessor_raw_id = NULL, acquisition_generation = 0 + """ + ) + source.commit() + return write_valid_rebuild_receipt(root, root.parent / "schema-inference-receipt.json") + + +def test_real_no_promote_candidate_preserves_frozen_durable_tiers( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = tmp_path / "archive" + receipt_path = _prepare_frozen_source(root, monkeypatch) + generation_store = IndexGenerationStore.for_archive_root(root) + active_target_before = generation_store.active_pointer.resolve(strict=True) + active_index_before = _file_evidence(active_target_before) + source_before = _file_evidence(root / "source.db") + user_before = _file_evidence(root / "user.db") + blobs_before = _blob_evidence(root / "blob") + + result = rebuild_index_from_source_sync( + RebuildIndexRequest( + archive_root=root, + schema_inference_receipt_path=receipt_path, + promote=False, + ) + ) + + assert result.status == "replayed" + assert result.transaction is not None + assert result.transaction["status"] == "ready" + generation = generation_store.load(str(result.transaction["generation_id"])) + assert generation.state == "inactive" + assert generation_store.active_pointer.resolve(strict=True) == active_target_before + assert _file_evidence(active_target_before) == active_index_before + assert _file_evidence(root / "source.db") == source_before + assert _file_evidence(root / "user.db") == user_before + assert _blob_evidence(root / "blob") == blobs_before + with sqlite3.connect(f"file:{generation.index_path}?mode=ro", uri=True) as candidate: + assert candidate.execute("SELECT COUNT(*) FROM sessions").fetchone()[0] == 1 + + +def test_owned_candidate_refuses_source_user_and_blob_writes( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = tmp_path / "archive" + _prepare_frozen_source(root, monkeypatch) + generation_store = IndexGenerationStore.for_archive_root(root) + generation = generation_store.create(source_snapshot=source_revision_snapshot(root)) + generation_root = Path(generation.index_path).parent + source_before = _file_evidence(root / "source.db") + user_before = _file_evidence(root / "user.db") + blobs_before = _blob_evidence(root / "blob") + + with ArchiveStore.open_owned_inactive_generation( + generation_root, + generation_id=generation.generation_id, + owner_id=generation.owner_id, + ) as candidate: + candidate._conn.execute("CREATE TABLE candidate_index_probe (value INTEGER) STRICT") + candidate.commit() + with pytest.raises(sqlite3.OperationalError, match="readonly"): + candidate._ensure_source_conn().execute("UPDATE raw_sessions SET parse_error = 'candidate-write'") + with pytest.raises(sqlite3.OperationalError, match="readonly"): + candidate._conn.execute("CREATE TABLE user_tier.candidate_user_probe (value INTEGER) STRICT") + assert candidate._blob_publisher is not None + with pytest.raises(InactiveCandidateDurableWriteError, match="may not publish"): + candidate._blob_publisher.write_from_bytes(b"candidate-write") + with pytest.raises(InactiveCandidateDurableWriteError, match="may not publish"): + candidate.write_raw_payload( + provider=Provider.CODEX, + payload=b"candidate-write", + source_path="candidate-write.jsonl", + acquired_at_ms=1, + ) + + assert _file_evidence(root / "source.db") == source_before + assert _file_evidence(root / "user.db") == user_before + assert _blob_evidence(root / "blob") == blobs_before + with sqlite3.connect(generation.index_path) as candidate_index: + assert candidate_index.execute( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'candidate_index_probe'" + ).fetchone() == (1,) + + +def test_candidate_requires_current_parser_census_before_generation_readiness( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = tmp_path / "archive" + build_independent_raw_corpus(root, raw_count=1, avg_payload_bytes=1_000) + monkeypatch.setenv("POLYLOGUE_ARCHIVE_ROOT", str(root)) + receipt_path = write_valid_rebuild_receipt(root, root.parent / "schema-inference-receipt.json") + + with pytest.raises(FrozenSourceRemediationRequiredError, match="complete current-parser source census"): + rebuild_index_from_source_sync( + RebuildIndexRequest( + archive_root=root, + schema_inference_receipt_path=receipt_path, + promote=False, + ) + ) + + assert not list((root / ".index-generations").glob("gen-*")) + + +def test_candidate_rejects_authority_drift_in_frozen_source( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = tmp_path / "archive" + _prepare_frozen_source(root, monkeypatch) + with sqlite3.connect(root / "source.db") as source: + source.execute("UPDATE raw_sessions SET revision_authority = 'asserted', baseline_raw_id = NULL") + source.commit() + receipt_path = write_valid_rebuild_receipt(root, root.parent / "post-drift-receipt.json") + + with pytest.raises(FrozenSourceRemediationRequiredError, match="re-derived different byte authority"): + rebuild_index_from_source_sync( + RebuildIndexRequest( + archive_root=root, + schema_inference_receipt_path=receipt_path, + promote=False, + ) + ) + + +def test_active_bootstrap_still_rejects_candidate_durable_symlinks( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = tmp_path / "archive" + _prepare_frozen_source(root, monkeypatch) + generation_store = IndexGenerationStore.for_archive_root(root) + generation = generation_store.create(source_snapshot=source_revision_snapshot(root)) + + with pytest.raises(DurableChangeTrainError, match="unsafe file"): + initialize_active_archive_root(Path(generation.index_path).parent) diff --git a/tests/unit/maintenance/test_rebuild_index_provenance_gate.py b/tests/unit/maintenance/test_rebuild_index_provenance_gate.py index 4f624e2011..564b8bd941 100644 --- a/tests/unit/maintenance/test_rebuild_index_provenance_gate.py +++ b/tests/unit/maintenance/test_rebuild_index_provenance_gate.py @@ -24,7 +24,7 @@ from polylogue.daemon import bulk_rebuild as bulk_rebuild_module from polylogue.maintenance.rebuild_index import RebuildIndexRequest, rebuild_index_from_source_sync from polylogue.maintenance.sharded_rebuild import shard_raw_ids -from polylogue.sources.revision_backfill import RebuildDeadlineExceededError +from polylogue.sources.revision_backfill import RebuildDeadlineExceededError, census_historical_revision_evidence from polylogue.storage.archive_identity import ArchiveLocation, ArchiveOwnershipError, OwnedArchiveLocation from polylogue.storage.blob_store import BlobStore from polylogue.storage.index_generation import ( @@ -78,6 +78,8 @@ def _seed(root: Path, count: int = 2) -> None: with sqlite3.connect(root / "source.db") as source: source.execute("UPDATE raw_sessions SET baseline_raw_id = raw_id, revision_authority = 'byte_proven'") source.commit() + census = census_historical_revision_evidence(root) + assert census.scanned == count def _active_bytes(root: Path) -> bytes: diff --git a/tests/unit/maintenance/test_reindex_canary.py b/tests/unit/maintenance/test_reindex_canary.py index 724679db8f..3f02e568aa 100644 --- a/tests/unit/maintenance/test_reindex_canary.py +++ b/tests/unit/maintenance/test_reindex_canary.py @@ -41,7 +41,11 @@ select_canary_sessions, write_canary_report, ) -from polylogue.sources.revision_backfill import RebuildDeadlineExceededError +from polylogue.sources.revision_backfill import ( + RebuildDeadlineExceededError, + backfill_historical_revision_evidence, + census_historical_revision_evidence, +) from polylogue.storage.archive_identity import ArchiveLocation from polylogue.storage.index_generation import rebuild_source_evidence_snapshot from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore @@ -84,6 +88,12 @@ def _receipt_path(tmp_path: Path) -> Path: return path +def _write_candidate_receipt(archive_root: Path, receipt_path: Path) -> Path: + """Bind a real receipt only after the fixture models phase-2 census.""" + census_historical_revision_evidence(archive_root) + return write_valid_rebuild_receipt(archive_root, receipt_path) + + def _seed_index( path: Path, *, @@ -567,7 +577,7 @@ def test_run_reindex_canary_rejects_missing_receipt_even_with_ambient_valid_rece ) -> None: artifact = build_seeded_archive(cache_root=tmp_path / "seeded-cache") root = clone_seeded_archive(artifact, tmp_path / "archive").root - ambient_receipt = write_valid_rebuild_receipt(root, tmp_path / "ambient-schema-inference-gate-receipt.json") + ambient_receipt = _write_candidate_receipt(root, tmp_path / "ambient-schema-inference-gate-receipt.json") monkeypatch.setenv("POLYLOGUE_SCHEMA_INFERENCE_RECEIPT", str(ambient_receipt)) with pytest.raises(CanarySelectionError, match="requires an explicit schema-inference receipt path"): @@ -580,7 +590,7 @@ def test_run_reindex_canary_cleans_candidate_after_comparison_failure( """A post-rebuild canary failure cannot strand its inactive candidate.""" artifact = build_seeded_archive(cache_root=tmp_path / "seeded-cache") root = clone_seeded_archive(artifact, tmp_path / "archive").root - receipt_path = write_valid_rebuild_receipt(root, tmp_path / "receipt.json") + receipt_path = _write_candidate_receipt(root, tmp_path / "receipt.json") def fail_compare(*args: object, **kwargs: object) -> CanaryDiffReport: raise RuntimeError("synthetic canary comparison failure") @@ -602,7 +612,7 @@ def test_run_reindex_canary_clean_success_retains_a_valid_inactive_candidate(tmp """A successful canary returns the candidate for its comparison evidence.""" artifact = build_seeded_archive(cache_root=tmp_path / "seeded-cache") root = clone_seeded_archive(artifact, tmp_path / "archive").root - receipt_path = write_valid_rebuild_receipt(root, tmp_path / "receipt.json") + receipt_path = _write_candidate_receipt(root, tmp_path / "receipt.json") result = run_reindex_canary( root, @@ -656,7 +666,7 @@ def test_run_reindex_canary_accepts_split_root_active_pointer_through_real_valid monkeypatch.setenv("POLYLOGUE_ARCHIVE_ROOT", str(tmp_path / "configured-live")) active_digest = hashlib.sha256(external_index.read_bytes()).hexdigest() evidence_before = rebuild_source_evidence_snapshot(root) - receipt_path = write_valid_rebuild_receipt(root, tmp_path / "schema-inference-gate-receipt.json") + receipt_path = _write_candidate_receipt(root, tmp_path / "schema-inference-gate-receipt.json") result = run_reindex_canary( root, @@ -705,8 +715,8 @@ def test_run_reindex_canary_accepts_split_root_active_pointer_through_real_valid assert operation_delta["source_snapshot_matches"] is True -def test_real_no_promote_rebuild_changes_parse_state_without_evidence_drift(tmp_path: Path) -> None: - """Rebuild output state may change while durable replay evidence remains fixed.""" +def test_real_no_promote_rebuild_preserves_remediated_source_state(tmp_path: Path) -> None: + """Candidate replay consumes phase-2 source state without changing it.""" root = tmp_path / "archive" initialize_active_archive_root(root) @@ -739,11 +749,20 @@ def test_real_no_promote_rebuild_changes_parse_state_without_evidence_drift(tmp_ native_id="fresh", acquired_at_ms=1, ) + backfill_historical_revision_evidence(root) with sqlite3.connect(root / "source.db") as connection: - assert ( - connection.execute("SELECT parsed_at_ms FROM raw_sessions WHERE raw_id = ?", (raw_id,)).fetchone()[0] - is None - ) + source_state_before = connection.execute( + """ + SELECT parsed_at_ms, parse_error, + (SELECT COUNT(*) FROM blob_refs WHERE ref_id = ? AND ref_type = 'attachment') + FROM raw_sessions WHERE raw_id = ? + """, + (raw_id, raw_id), + ).fetchone() + assert source_state_before is not None + assert source_state_before[0] is not None + assert source_state_before[1] is None + assert source_state_before[2] == 1 active_digest = hashlib.sha256((root / "index.db").read_bytes()).hexdigest() evidence_before = rebuild_source_evidence_snapshot(root) @@ -753,16 +772,15 @@ def test_real_no_promote_rebuild_changes_parse_state_without_evidence_drift(tmp_ ) with sqlite3.connect(root / "source.db") as connection: - assert ( - connection.execute("SELECT parsed_at_ms FROM raw_sessions WHERE raw_id = ?", (raw_id,)).fetchone()[0] - is not None - ) - assert ( - connection.execute( - "SELECT COUNT(*) FROM blob_refs WHERE ref_id = ? AND ref_type = 'attachment'", (raw_id,) - ).fetchone()[0] - == 1 - ) + source_state_after = connection.execute( + """ + SELECT parsed_at_ms, parse_error, + (SELECT COUNT(*) FROM blob_refs WHERE ref_id = ? AND ref_type = 'attachment') + FROM raw_sessions WHERE raw_id = ? + """, + (raw_id, raw_id), + ).fetchone() + assert source_state_after == source_state_before assert receipt.generation["state"] == "inactive" assert receipt.generation["source_snapshot"] == evidence_before == rebuild_source_evidence_snapshot(root) assert receipt.source_evidence_after == rebuild_source_evidence_snapshot(root) @@ -778,7 +796,7 @@ def test_run_reindex_canary_rejects_external_evidence_mutation_after_replay( root = clone_seeded_archive(artifact, tmp_path / "archive").root active_index = root / "index.db" active_digest = hashlib.sha256(active_index.read_bytes()).hexdigest() - receipt_path = write_valid_rebuild_receipt(root, tmp_path / "schema-inference-gate-receipt.json") + receipt_path = _write_candidate_receipt(root, tmp_path / "schema-inference-gate-receipt.json") monkeypatch.setenv("POLYLOGUE_ARCHIVE_ROOT", str(tmp_path / "configured-live")) from polylogue.maintenance import replay as rebuild_replay @@ -811,7 +829,7 @@ def test_run_reindex_canary_rejects_active_index_rotation_after_replay( rotated_index = tmp_path / "rotated" / "index.db" rotated_index.parent.mkdir(parents=True) shutil.copy2(current_index, rotated_index) - receipt_path = write_valid_rebuild_receipt(root, tmp_path / "schema-inference-gate-receipt.json") + receipt_path = _write_candidate_receipt(root, tmp_path / "schema-inference-gate-receipt.json") rebuild = rebuild_index_from_source_sync def rebuild_then_rotate(request: RebuildIndexRequest) -> object: @@ -841,7 +859,7 @@ async def mutate_source_then_interrupt(*args: Any, **kwargs: Any) -> dict[str, o raise RebuildDeadlineExceededError("synthetic deadline") monkeypatch.setattr(rebuild_replay, "rebuild_index_from_source", mutate_source_then_interrupt) - receipt_path = write_valid_rebuild_receipt(root, tmp_path / "schema-inference-gate-receipt.json") + receipt_path = _write_candidate_receipt(root, tmp_path / "schema-inference-gate-receipt.json") with pytest.raises(RuntimeError, match="schema-inference preflight gate failed"): rebuild_index_from_source_sync( From b095c1a6e7a8775f947aa468eaeda1699af240f4 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sat, 8 Aug 2026 11:22:07 +0200 Subject: [PATCH 02/19] fix(reindex): seal candidate durable capability leaks Problem Inactive candidates could reach live ops telemetry, convergence debt, and several inherited blob and user-tier mutation APIs through read-through links. Candidate allocation also preceded complete frozen-source authority. What changed Require current parser and settled authority evidence before allocating a generation. Validate split-root ownership against generation metadata, block every blob and user mutation surface, and suppress inactive-candidate writes to the live ops ledger while retaining active-generation telemetry. Verification Focused candidate barrier tests pass 6/6. The wider affected batch passed 162 tests; its 15 failures exactly match the frozen current-master full-suite ledger. Ruff and strict mypy pass for all touched source and test files. Ref polylogue-inactive-candidate-durable-barrier. Co-Authored-By: Codex --- polylogue/maintenance/rebuild_index.py | 1 + polylogue/sources/revision_backfill.py | 32 ++++ polylogue/storage/fts/drift_sampling.py | 16 ++ polylogue/storage/repair.py | 8 +- .../storage/sqlite/archive_tiers/archive.py | 142 ++++++++++++------ ...test_inactive_candidate_durable_barrier.py | 92 +++++++++++- tests/unit/maintenance/test_reindex_canary.py | 101 +++++++------ 7 files changed, 295 insertions(+), 97 deletions(-) diff --git a/polylogue/maintenance/rebuild_index.py b/polylogue/maintenance/rebuild_index.py index ece2c75839..3b16a0893d 100644 --- a/polylogue/maintenance/rebuild_index.py +++ b/polylogue/maintenance/rebuild_index.py @@ -2022,6 +2022,7 @@ def _check_pass_deadline() -> None: dry_run=False, archive_root_override=generation_root, owned_inactive_generation=(generation.generation_id, generation.owner_id), + resolve_convergence_debt=False, ) terminal_timings_s["terminal.session_insights"] = time.perf_counter() - terminal_started_at logger.info( diff --git a/polylogue/sources/revision_backfill.py b/polylogue/sources/revision_backfill.py index 9bd1b56fcc..63fcfbd977 100644 --- a/polylogue/sources/revision_backfill.py +++ b/polylogue/sources/revision_backfill.py @@ -901,6 +901,38 @@ def require_current_parser_source_census( f"{len(stale_raw_ids)} raw(s) are stale or incomplete (sample: {sample})" ) + unresolved_raw_ids: list[str] = [] + with sqlite3.connect(f"file:{archive_root / 'source.db'}?mode=ro", uri=True) as source_conn: + for selection in selections: + authority_where = "" if selection is None else f"AND r.raw_id IN ({','.join('?' for _ in selection)})" + authority_params: tuple[object, ...] = () if selection is None else selection + unresolved_raw_ids.extend( + str(row[0]) + for row in source_conn.execute( + f""" + SELECT DISTINCT r.raw_id + FROM raw_sessions AS r + LEFT JOIN raw_membership_census AS c ON c.raw_id = r.raw_id + LEFT JOIN raw_session_memberships AS m ON m.raw_id = r.raw_id + WHERE r.revision_authority = 'quarantined' + {authority_where} + AND ( + c.raw_id IS NULL OR c.status != 'complete' + OR m.raw_id IS NULL OR m.decision IS NULL + OR m.decision IN ('ambiguous', 'deferred') + ) + ORDER BY r.raw_id + """, + authority_params, + ) + ) + if unresolved_raw_ids: + sample = ", ".join(unresolved_raw_ids[:5]) + raise FrozenSourceRemediationRequiredError( + "inactive candidate requires complete frozen source authority; " + f"{len(unresolved_raw_ids)} raw(s) remain quarantined or undecided (sample: {sample})" + ) + def census_historical_revision_evidence( archive_root: Path, diff --git a/polylogue/storage/fts/drift_sampling.py b/polylogue/storage/fts/drift_sampling.py index a3a4306dd5..c320aebe5d 100644 --- a/polylogue/storage/fts/drift_sampling.py +++ b/polylogue/storage/fts/drift_sampling.py @@ -18,6 +18,7 @@ from __future__ import annotations import contextlib +import json import sqlite3 import time from pathlib import Path @@ -57,6 +58,19 @@ def _index_db_path_sync(conn: sqlite3.Connection) -> Path | None: return None +def _is_owned_inactive_generation_index(index_db_path: Path) -> bool: + """Return whether an index belongs to a validated inactive generation.""" + metadata_path = index_db_path.parent / "generation.json" + if not metadata_path.is_file(): + return False + try: + payload = json.loads(metadata_path.read_text(encoding="utf-8")) + declared_index = Path(str(payload["index_path"])).resolve(strict=True) + except (KeyError, OSError, TypeError, ValueError): + return False + return payload.get("state") == "inactive" and declared_index == index_db_path.resolve(strict=True) + + def sample_fts_drift_to_ops_sync(conn: sqlite3.Connection, *, archive_root: Path | None = None) -> int: """Append one ops.db drift sample per recorded FTS surface. @@ -89,6 +103,8 @@ def sample_fts_drift_to_ops_sync(conn: sqlite3.Connection, *, archive_root: Path index_db_path = _index_db_path_sync(conn) if index_db_path is None: return 0 + if _is_owned_inactive_generation_index(index_db_path): + return 0 ops_db_path = index_db_path.with_name("ops.db") if not ops_db_path.exists(): return 0 diff --git a/polylogue/storage/repair.py b/polylogue/storage/repair.py index 2d98244584..5f82b8171a 100644 --- a/polylogue/storage/repair.py +++ b/polylogue/storage/repair.py @@ -5817,6 +5817,7 @@ def repair_session_insights( session_ids: tuple[str, ...] | None = None, archive_root_override: Path | None = None, owned_inactive_generation: tuple[str, str] | None = None, + resolve_convergence_debt: bool = True, ) -> RepairResult: """Repair / rebuild session insights. @@ -5852,6 +5853,11 @@ def repair_session_insights( generation) cannot reach at all. Do not remove the mutate path without first giving thread/tag-rollup/day-summary aggregate staleness its own automatic convergence mechanism. + + ``resolve_convergence_debt=False`` is reserved for an owned inactive + generation. Its ``ops.db`` is a read-through link to live disposable + state, so candidate materialization may prove derived readiness without + clearing the active daemon's debt ledger. """ from polylogue.paths import archive_root as _resolve_archive_root from polylogue.storage.archive_identity import resolve_active_index_path @@ -5944,7 +5950,7 @@ def repair_session_insights( # A narrowed rebuild only attests its own slice; do not # demand global readiness for a scope-filtered call. success = True if session_ids is not None else assess_session_insight_repairs(refreshed).row_debt == 0 - if success: + if success and resolve_convergence_debt: _resolve_session_insight_convergence_debt( ops_db=config.archive_root / "ops.db", session_ids=session_ids, diff --git a/polylogue/storage/sqlite/archive_tiers/archive.py b/polylogue/storage/sqlite/archive_tiers/archive.py index 915a475c5f..55623a94d8 100644 --- a/polylogue/storage/sqlite/archive_tiers/archive.py +++ b/polylogue/storage/sqlite/archive_tiers/archive.py @@ -16,7 +16,7 @@ import math import sqlite3 import time -from collections.abc import Callable, Iterator, Mapping, Sequence +from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence from concurrent.futures import Future from contextlib import closing, contextmanager from dataclasses import dataclass, field, replace @@ -136,7 +136,7 @@ from polylogue.pipeline.ids import SessionRevisionProjection from polylogue.sources.parsers.base import ParsedSession from polylogue.storage.blob_publication import ArchiveBlobPublisher -from polylogue.storage.blob_store import Heartbeat +from polylogue.storage.blob_store import CleanupOrphansResult, Heartbeat, PreparedBlob from polylogue.storage.fts.sql import ( FTS_BULK_SESSION_WRITE_GUARD, delete_session_identity_rows_sql, @@ -1654,6 +1654,35 @@ def _refuse() -> NoReturn: "inactive candidate generations may read frozen blobs but may not publish or replace blob bytes" ) + def _queue(self, prepared: PreparedBlob) -> NoReturn: + del prepared + self._refuse() + + def prepare_from_path(self, source: Path, *, heartbeat: Heartbeat | None = None) -> NoReturn: + del source, heartbeat + self._refuse() + + def prepare_from_fileobj(self, source: IO[bytes], *, heartbeat: Heartbeat | None = None) -> NoReturn: + del source, heartbeat + self._refuse() + + def prepare_from_bytes(self, data: bytes) -> NoReturn: + del data + self._refuse() + + def publish_prepared(self, prepared: PreparedBlob) -> NoReturn: + del prepared + self._refuse() + + def publish_many(self, prepared: Iterable[PreparedBlob]) -> NoReturn: + del prepared + self._refuse() + + @staticmethod + def discard_prepared(prepared: PreparedBlob) -> NoReturn: + del prepared + _InactiveCandidateBlobPublisher._refuse() + def write_from_path(self, source: Path, *, heartbeat: Heartbeat | None = None) -> tuple[str, int]: del source, heartbeat return self._refuse() @@ -1666,6 +1695,26 @@ def write_from_bytes(self, data: bytes) -> tuple[str, int]: del data return self._refuse() + def remove(self, hash_hex: str) -> NoReturn: + del hash_hex + self._refuse() + + def cleanup_orphans( + self, + orphan_hashes: set[str], + *, + dry_run: bool = True, + ) -> CleanupOrphansResult: + if not dry_run: + return self._refuse() + return super().cleanup_orphans(orphan_hashes, dry_run=True) + + def flush(self) -> tuple[()]: + return () + + def discard_pending(self) -> None: + return None + class ArchiveStore: """Minimal archive-root façade for archive source/index/user tiers.""" @@ -1705,25 +1754,31 @@ def __init__( self._active_writer_lease = None raise else: - from polylogue.storage.index_generation import IndexGenerationStore + from polylogue.storage.index_generation import IndexGeneration, IndexGenerationStore generation_id, owner_id = owned_inactive_generation # An inactive generation is opened from its generation root, # while the configured root may intentionally point at a - # different live archive. Resolve the owning archive from the - # candidate path instead of routing this safety check through - # global configuration. - generation_archive_root = archive_root.parent.parent - generation = IndexGenerationStore.for_archive_root(generation_archive_root).load(generation_id) + # different live archive. Read the candidate's declared root, + # then require the store anchored at that root to return the + # exact same metadata. Deriving the root from ``../..`` fails + # for supported split-index layouts, where generations live + # beside the external active index rather than below the + # durable archive root. + generation = IndexGeneration( + **json.loads((archive_root / "generation.json").read_text(encoding="utf-8")) + ) + declared_archive_root = Path(generation.archive_root).resolve(strict=True) + authoritative_generation = IndexGenerationStore.for_archive_root(declared_archive_root).load( + generation_id + ) if ( - generation.owner_id != owner_id + generation != authoritative_generation + or generation.owner_id != owner_id or generation.state != "inactive" or Path(generation.index_path).parent.resolve(strict=True) != archive_root.resolve(strict=True) ): raise RuntimeError("inactive index generation ownership validation failed") - declared_archive_root = Path(generation.archive_root).resolve(strict=True) - if declared_archive_root != generation_archive_root.resolve(strict=True): - raise RuntimeError("inactive index generation archive-root binding is stale") for filename in ("source.db", "user.db", "embeddings.db", "ops.db", "blob"): expected = declared_archive_root / filename candidate = archive_root / filename @@ -1956,6 +2011,16 @@ def _ensure_source_conn(self) -> sqlite3.Connection: self._source_conn = conn return self._source_conn + def _open_user_write_connection(self, *, initialize: bool = False) -> sqlite3.Connection: + """Open user.db for mutation unless this store is an inactive candidate.""" + if self._inactive_candidate_durable_read_only: + raise InactiveCandidateDurableWriteError( + "inactive candidate generations may read frozen user assertions but may not mutate user.db" + ) + if initialize: + initialize_archive_database(self.user_db_path, ArchiveTier.USER) + return open_connection(self.user_db_path) + def commit(self) -> None: """Commit index.db and any source transaction left by other callers. @@ -4385,10 +4450,8 @@ def add_user_tags( author_kind: str | None = None, ) -> int: """Add user tag assertions to archive user.db and return changed count.""" - user_db_path = self.user_db_path - initialize_archive_database(user_db_path, ArchiveTier.USER) changed = 0 - user_conn = open_connection(user_db_path) + user_conn = self._open_user_write_connection(initialize=True) user_conn.row_factory = sqlite3.Row try: with user_conn: @@ -4427,7 +4490,7 @@ def remove_user_tags(self, session_ids: tuple[str, ...], tags: tuple[str, ...]) if not resolved_session_ids or not self.user_db_path.exists(): return 0 removed = 0 - user_conn = open_connection(self.user_db_path) + user_conn = self._open_user_write_connection() try: with user_conn: for session_id in resolved_session_ids: @@ -5224,9 +5287,7 @@ def _time_bucket_coverage_insights( def set_user_metadata(self, session_ids: tuple[str, ...], pairs: tuple[tuple[str, object], ...]) -> int: """Set human-owned metadata as archive user.db assertions.""" - user_db_path = self.user_db_path - initialize_archive_database(user_db_path, ArchiveTier.USER) - user_conn = open_connection(user_db_path) + user_conn = self._open_user_write_connection(initialize=True) user_conn.row_factory = sqlite3.Row try: changed = 0 @@ -5285,7 +5346,7 @@ def delete_user_metadata(self, session_id: str, key: str) -> int: raise ValueError("metadata key cannot be empty") if not self.user_db_path.exists(): return 0 - user_conn = open_connection(self.user_db_path) + user_conn = self._open_user_write_connection() try: with user_conn: assertion_id = assertion_id_for_session_metadata(resolved_session_id, normalized_key) @@ -5298,8 +5359,7 @@ def delete_user_metadata(self, session_id: str, key: str) -> int: def add_mark(self, target_type: str, target_id: str, mark_type: str) -> bool: """Add one user mark to archive user.db.""" - initialize_archive_database(self.user_db_path, ArchiveTier.USER) - user_conn = open_connection(self.user_db_path) + user_conn = self._open_user_write_connection(initialize=True) try: assertion = read_assertion_envelope(user_conn, assertion_id_for_mark(target_type, target_id, mark_type)) exists = assertion is not None and assertion.status != "deleted" @@ -5313,7 +5373,7 @@ def remove_mark(self, target_type: str, target_id: str, mark_type: str) -> bool: """Remove one user mark from archive user.db.""" if not self.user_db_path.exists(): return False - user_conn = open_connection(self.user_db_path) + user_conn = self._open_user_write_connection() try: with user_conn: return mark_assertion_status( @@ -5362,8 +5422,7 @@ def list_marks( def save_annotation(self, annotation_id: str, target_type: str, target_id: str, note_text: str) -> bool: """Create or update one annotation in archive user.db.""" - initialize_archive_database(self.user_db_path, ArchiveTier.USER) - user_conn = open_connection(self.user_db_path) + user_conn = self._open_user_write_connection(initialize=True) try: assertion = read_assertion_envelope(user_conn, assertion_id_for_annotation(annotation_id)) exists = assertion is not None and assertion.status != "deleted" @@ -5387,8 +5446,7 @@ def save_annotation_schema( ) -> DurableAnnotationSchema: """Persist an immutable annotation schema definition in ``user.db``.""" - initialize_archive_database(self.user_db_path, ArchiveTier.USER) - user_conn = open_connection(self.user_db_path) + user_conn = self._open_user_write_connection(initialize=True) user_conn.row_factory = sqlite3.Row try: with user_conn: @@ -5431,8 +5489,7 @@ def list_annotation_schemas(self) -> tuple[DurableAnnotationSchema, ...]: def save_annotation_batch(self, batch: AnnotationBatch) -> AnnotationBatch: """Persist one immutable annotation-batch provenance container.""" - initialize_archive_database(self.user_db_path, ArchiveTier.USER) - user_conn = open_connection(self.user_db_path) + user_conn = self._open_user_write_connection(initialize=True) user_conn.row_factory = sqlite3.Row try: with user_conn: @@ -5541,7 +5598,7 @@ def delete_annotation(self, annotation_id: str) -> bool: """Delete one annotation from archive user.db.""" if not self.user_db_path.exists(): return False - user_conn = open_connection(self.user_db_path) + user_conn = self._open_user_write_connection() try: with user_conn: return mark_assertion_status(user_conn, assertion_id_for_annotation(annotation_id), "deleted") @@ -5556,8 +5613,7 @@ def save_view(self, view_id: str, name: str, query_json: str) -> bool: query = json.loads(query_json) if not isinstance(query, dict): raise ValueError("query_json must encode an object") - initialize_archive_database(self.user_db_path, ArchiveTier.USER) - user_conn = open_connection(self.user_db_path) + user_conn = self._open_user_write_connection(initialize=True) try: assertion_id = assertion_id_for_saved_view(view_id) assertion = read_assertion_envelope(user_conn, assertion_id) @@ -5610,7 +5666,7 @@ def delete_view(self, view_id: str) -> bool: """Delete one saved view from archive user.db.""" if not self.user_db_path.exists(): return False - user_conn = open_connection(self.user_db_path) + user_conn = self._open_user_write_connection() try: with user_conn: return mark_assertion_status(user_conn, assertion_id_for_saved_view(view_id), "deleted") @@ -5630,8 +5686,7 @@ def save_recall_pack( raise ValueError("payload_json must encode an object") payload = dict(payload) payload["session_ids_json"] = session_ids_json - initialize_archive_database(self.user_db_path, ArchiveTier.USER) - user_conn = open_connection(self.user_db_path) + user_conn = self._open_user_write_connection(initialize=True) try: assertion = read_assertion_envelope(user_conn, assertion_id_for_recall_pack(pack_id)) exists = assertion is not None and assertion.status != "deleted" @@ -5679,7 +5734,7 @@ def delete_recall_pack(self, pack_id: str) -> bool: """Delete one recall pack from archive user.db.""" if not self.user_db_path.exists(): return False - user_conn = open_connection(self.user_db_path) + user_conn = self._open_user_write_connection() try: with user_conn: return mark_assertion_status(user_conn, assertion_id_for_recall_pack(pack_id), "deleted") @@ -5703,8 +5758,7 @@ def save_workspace( "layout_json": layout_json, "active_target_json": active_target_json, } - initialize_archive_database(self.user_db_path, ArchiveTier.USER) - user_conn = open_connection(self.user_db_path) + user_conn = self._open_user_write_connection(initialize=True) try: assertion_id = assertion_id_for_workspace(workspace_id) assertion = read_assertion_envelope(user_conn, assertion_id) @@ -5762,7 +5816,7 @@ def delete_workspace(self, workspace_id: str) -> bool: """Delete one workspace from archive user.db.""" if not self.user_db_path.exists(): return False - user_conn = open_connection(self.user_db_path) + user_conn = self._open_user_write_connection() try: with user_conn: return mark_assertion_status(user_conn, assertion_id_for_workspace(workspace_id), "deleted") @@ -5782,9 +5836,8 @@ def record_correction( """Record one learning correction in archive user.db.""" resolved_session_id = self.resolve_session_id(session_id) correction_kind = parse_correction_kind(kind) - initialize_archive_database(self.user_db_path, ArchiveTier.USER) stored_payload: dict[str, object] = {"payload": dict(payload), "note": note} - user_conn = open_connection(self.user_db_path) + user_conn = self._open_user_write_connection(initialize=True) try: with user_conn: upsert_correction( @@ -5837,7 +5890,7 @@ def delete_correction(self, session_id: str, kind: str) -> bool: correction_kind = parse_correction_kind(kind) if not self.user_db_path.exists(): return False - user_conn = open_connection(self.user_db_path) + user_conn = self._open_user_write_connection() try: with user_conn: correction_id = correction_id_for("insight", resolved_session_id, correction_kind.value) @@ -5850,7 +5903,7 @@ def clear_corrections(self, session_id: str) -> int: resolved_session_id = self.resolve_session_id(session_id) if not self.user_db_path.exists(): return 0 - user_conn = open_connection(self.user_db_path) + user_conn = self._open_user_write_connection() try: with user_conn: deleted_count = 0 @@ -5880,8 +5933,7 @@ def post_blackboard_note( context_policy: dict[str, object] | None = None, ) -> ArchiveBlackboardNoteEnvelope: """Insert-or-update one blackboard note in archive user.db.""" - initialize_archive_database(self.user_db_path, ArchiveTier.USER) - user_conn = open_connection(self.user_db_path) + user_conn = self._open_user_write_connection(initialize=True) try: envelope = upsert_blackboard_note( user_conn, diff --git a/tests/unit/maintenance/test_inactive_candidate_durable_barrier.py b/tests/unit/maintenance/test_inactive_candidate_durable_barrier.py index 37e7032db7..0f0111a7c8 100644 --- a/tests/unit/maintenance/test_inactive_candidate_durable_barrier.py +++ b/tests/unit/maintenance/test_inactive_candidate_durable_barrier.py @@ -4,6 +4,7 @@ import hashlib import sqlite3 +from io import BytesIO from pathlib import Path import pytest @@ -11,6 +12,7 @@ from polylogue.core.enums import Provider from polylogue.maintenance.rebuild_index import RebuildIndexRequest, rebuild_index_from_source_sync from polylogue.sources.revision_backfill import census_historical_revision_evidence +from polylogue.storage.blob_store import PreparedBlob from polylogue.storage.index_generation import IndexGenerationStore, source_revision_snapshot from polylogue.storage.sqlite.archive_tiers.archive import ( ArchiveStore, @@ -36,6 +38,14 @@ def _blob_evidence(root: Path) -> tuple[tuple[str, int, str], ...]: ) +def _symlink_evidence(path: Path) -> tuple[int, int, str]: + stat = path.lstat() + target = ( + f"symlink:{path.readlink()}" if path.is_symlink() else f"file:{hashlib.sha256(path.read_bytes()).hexdigest()}" + ) + return stat.st_dev, stat.st_ino, target + + def _prepare_frozen_source(root: Path, monkeypatch: pytest.MonkeyPatch) -> Path: build_independent_raw_corpus(root, raw_count=1, avg_payload_bytes=1_000) monkeypatch.setenv("POLYLOGUE_ARCHIVE_ROOT", str(root)) @@ -62,9 +72,23 @@ def test_real_no_promote_candidate_preserves_frozen_durable_tiers( receipt_path = _prepare_frozen_source(root, monkeypatch) generation_store = IndexGenerationStore.for_archive_root(root) active_target_before = generation_store.active_pointer.resolve(strict=True) + active_pointer_before = _symlink_evidence(generation_store.active_pointer) active_index_before = _file_evidence(active_target_before) source_before = _file_evidence(root / "source.db") user_before = _file_evidence(root / "user.db") + with sqlite3.connect(root / "ops.db") as ops: + ops.execute( + """ + INSERT INTO convergence_debt ( + debt_id, stage, target_type, target_id, status, priority, + attempts, last_error, created_at_ms, updated_at_ms + ) VALUES ('candidate-guard-debt', 'insights', 'session_id', + 'claude-ai-export:frozen-source', 'deferred', 0, 1, + 'candidate must not resolve live debt', 1, 1) + """ + ) + ops.commit() + ops_before = _file_evidence(root / "ops.db") blobs_before = _blob_evidence(root / "blob") result = rebuild_index_from_source_sync( @@ -81,10 +105,16 @@ def test_real_no_promote_candidate_preserves_frozen_durable_tiers( generation = generation_store.load(str(result.transaction["generation_id"])) assert generation.state == "inactive" assert generation_store.active_pointer.resolve(strict=True) == active_target_before + assert _symlink_evidence(generation_store.active_pointer) == active_pointer_before assert _file_evidence(active_target_before) == active_index_before assert _file_evidence(root / "source.db") == source_before assert _file_evidence(root / "user.db") == user_before + assert _file_evidence(root / "ops.db") == ops_before assert _blob_evidence(root / "blob") == blobs_before + with sqlite3.connect(root / "ops.db") as ops: + assert ops.execute( + "SELECT stage, target_id FROM convergence_debt WHERE debt_id = 'candidate-guard-debt'" + ).fetchone() == ("insights", "claude-ai-export:frozen-source") with sqlite3.connect(f"file:{generation.index_path}?mode=ro", uri=True) as candidate: assert candidate.execute("SELECT COUNT(*) FROM sessions").fetchone()[0] == 1 @@ -114,8 +144,37 @@ def test_owned_candidate_refuses_source_user_and_blob_writes( with pytest.raises(sqlite3.OperationalError, match="readonly"): candidate._conn.execute("CREATE TABLE user_tier.candidate_user_probe (value INTEGER) STRICT") assert candidate._blob_publisher is not None - with pytest.raises(InactiveCandidateDurableWriteError, match="may not publish"): - candidate._blob_publisher.write_from_bytes(b"candidate-write") + blob_publisher = candidate._blob_publisher + existing_blob_hash = next(blob_publisher.iter_all()) + staged_path = tmp_path / "prepared-candidate-blob" + staged_path.write_bytes(b"candidate-write") + prepared = PreparedBlob( + hash_hex=hashlib.sha256(b"candidate-write").hexdigest(), + size_bytes=len(b"candidate-write"), + temporary_path=staged_path, + ) + source_path = tmp_path / "candidate-source" + source_path.write_bytes(b"candidate-write") + refusing_blob_calls = ( + lambda: blob_publisher.prepare_from_path(source_path), + lambda: blob_publisher.prepare_from_fileobj(BytesIO(b"candidate-write")), + lambda: blob_publisher.prepare_from_bytes(b"candidate-write"), + lambda: blob_publisher.publish_prepared(prepared), + lambda: blob_publisher.publish_many((prepared,)), + lambda: blob_publisher.discard_prepared(prepared), + lambda: blob_publisher.write_from_path(source_path), + lambda: blob_publisher.write_from_fileobj(BytesIO(b"candidate-write")), + lambda: blob_publisher.write_from_bytes(b"candidate-write"), + lambda: blob_publisher.remove(existing_blob_hash), + lambda: blob_publisher.cleanup_orphans({existing_blob_hash}, dry_run=False), + ) + for refusing_call in refusing_blob_calls: + with pytest.raises(InactiveCandidateDurableWriteError, match="may not publish"): + refusing_call() + assert blob_publisher.flush() == () + blob_publisher.discard_pending() + assert staged_path.read_bytes() == b"candidate-write" + assert not tuple((root / "blob").glob(".blob.*")) with pytest.raises(InactiveCandidateDurableWriteError, match="may not publish"): candidate.write_raw_payload( provider=Provider.CODEX, @@ -123,6 +182,12 @@ def test_owned_candidate_refuses_source_user_and_blob_writes( source_path="candidate-write.jsonl", acquired_at_ms=1, ) + with pytest.raises(InactiveCandidateDurableWriteError, match="may not mutate user.db"): + candidate.add_user_tags(("candidate:session",), ("candidate",)) + with pytest.raises(InactiveCandidateDurableWriteError, match="may not mutate user.db"): + candidate.set_user_metadata(("candidate:session",), (("candidate", True),)) + with pytest.raises(InactiveCandidateDurableWriteError, match="may not mutate user.db"): + candidate.post_blackboard_note("candidate write") assert _file_evidence(root / "source.db") == source_before assert _file_evidence(root / "user.db") == user_before @@ -154,6 +219,29 @@ def test_candidate_requires_current_parser_census_before_generation_readiness( assert not list((root / ".index-generations").glob("gen-*")) +def test_candidate_requires_complete_source_authority_before_generation_readiness( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = tmp_path / "archive" + build_independent_raw_corpus(root, raw_count=1, avg_payload_bytes=1_000) + monkeypatch.setenv("POLYLOGUE_ARCHIVE_ROOT", str(root)) + census = census_historical_revision_evidence(root) + assert census.scanned == 1 + receipt_path = write_valid_rebuild_receipt(root, root.parent / "schema-inference-receipt.json") + + with pytest.raises(FrozenSourceRemediationRequiredError, match="complete frozen source authority"): + rebuild_index_from_source_sync( + RebuildIndexRequest( + archive_root=root, + schema_inference_receipt_path=receipt_path, + promote=False, + ) + ) + + assert not list((root / ".index-generations").glob("gen-*")) + + def test_candidate_rejects_authority_drift_in_frozen_source( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/maintenance/test_reindex_canary.py b/tests/unit/maintenance/test_reindex_canary.py index 3f02e568aa..7c255e9ca3 100644 --- a/tests/unit/maintenance/test_reindex_canary.py +++ b/tests/unit/maintenance/test_reindex_canary.py @@ -44,7 +44,6 @@ from polylogue.sources.revision_backfill import ( RebuildDeadlineExceededError, backfill_historical_revision_evidence, - census_historical_revision_evidence, ) from polylogue.storage.archive_identity import ArchiveLocation from polylogue.storage.index_generation import rebuild_source_evidence_snapshot @@ -52,7 +51,6 @@ from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root, initialize_archive_database from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier from tests.infra.rebuild_receipt import write_valid_rebuild_receipt -from tests.infra.workload_artifacts import build_seeded_archive, clone_seeded_archive @pytest.fixture(autouse=True) @@ -89,11 +87,46 @@ def _receipt_path(tmp_path: Path) -> Path: def _write_candidate_receipt(archive_root: Path, receipt_path: Path) -> Path: - """Bind a real receipt only after the fixture models phase-2 census.""" - census_historical_revision_evidence(archive_root) + """Bind a receipt to a fixture that already completed phase 2.""" return write_valid_rebuild_receipt(archive_root, receipt_path) +def _prepare_candidate_ready_archive(root: Path) -> str: + """Build one real-route archive whose source authority is fully settled.""" + initialize_active_archive_root(root) + payload = json.dumps( + { + "chat_messages": [ + {"uuid": "fresh-user", "sender": "human", "text": "hello"}, + { + "uuid": "fresh-assistant", + "sender": "assistant", + "text": "world", + "attachments": [ + { + "id": "fresh-attachment", + "name": "fresh.txt", + "mimeType": "text/plain", + "size": 16, + "extracted_content": "attachment bytes", + } + ], + }, + ] + } + ).encode() + with ArchiveStore.open_existing(root, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.CLAUDE_AI, + payload=payload, + source_path="fresh.json", + native_id="fresh", + acquired_at_ms=1, + ) + backfill_historical_revision_evidence(root) + return raw_id + + def _seed_index( path: Path, *, @@ -575,8 +608,8 @@ def fake_compare(current_index: Path, candidate_index: Path, *, session_ids: tup def test_run_reindex_canary_rejects_missing_receipt_even_with_ambient_valid_receipt( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - artifact = build_seeded_archive(cache_root=tmp_path / "seeded-cache") - root = clone_seeded_archive(artifact, tmp_path / "archive").root + root = tmp_path / "archive" + _prepare_candidate_ready_archive(root) ambient_receipt = _write_candidate_receipt(root, tmp_path / "ambient-schema-inference-gate-receipt.json") monkeypatch.setenv("POLYLOGUE_SCHEMA_INFERENCE_RECEIPT", str(ambient_receipt)) @@ -588,8 +621,8 @@ def test_run_reindex_canary_cleans_candidate_after_comparison_failure( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """A post-rebuild canary failure cannot strand its inactive candidate.""" - artifact = build_seeded_archive(cache_root=tmp_path / "seeded-cache") - root = clone_seeded_archive(artifact, tmp_path / "archive").root + root = tmp_path / "archive" + _prepare_candidate_ready_archive(root) receipt_path = _write_candidate_receipt(root, tmp_path / "receipt.json") def fail_compare(*args: object, **kwargs: object) -> CanaryDiffReport: @@ -610,8 +643,8 @@ def fail_compare(*args: object, **kwargs: object) -> CanaryDiffReport: def test_run_reindex_canary_clean_success_retains_a_valid_inactive_candidate(tmp_path: Path) -> None: """A successful canary returns the candidate for its comparison evidence.""" - artifact = build_seeded_archive(cache_root=tmp_path / "seeded-cache") - root = clone_seeded_archive(artifact, tmp_path / "archive").root + root = tmp_path / "archive" + _prepare_candidate_ready_archive(root) receipt_path = _write_candidate_receipt(root, tmp_path / "receipt.json") result = run_reindex_canary( @@ -656,8 +689,8 @@ def unexpected_selector(*args: object, **kwargs: object) -> None: def test_run_reindex_canary_accepts_split_root_active_pointer_through_real_validator( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - artifact = build_seeded_archive(cache_root=tmp_path / "seeded-cache") - root = clone_seeded_archive(artifact, tmp_path / "archive").root + root = tmp_path / "archive" + _prepare_candidate_ready_archive(root) external_index_root = tmp_path / "external-index-root" external_index_root.mkdir() external_index = external_index_root / "index.db" @@ -719,37 +752,7 @@ def test_real_no_promote_rebuild_preserves_remediated_source_state(tmp_path: Pat """Candidate replay consumes phase-2 source state without changing it.""" root = tmp_path / "archive" - initialize_active_archive_root(root) - payload = json.dumps( - { - "chat_messages": [ - {"uuid": "fresh-user", "sender": "human", "text": "hello"}, - { - "uuid": "fresh-assistant", - "sender": "assistant", - "text": "world", - "attachments": [ - { - "id": "fresh-attachment", - "name": "fresh.txt", - "mimeType": "text/plain", - "size": 16, - "extracted_content": "attachment bytes", - } - ], - }, - ] - } - ).encode() - with ArchiveStore.open_existing(root, read_only=False) as archive: - raw_id = archive.write_raw_payload( - provider=Provider.CLAUDE_AI, - payload=payload, - source_path="fresh.json", - native_id="fresh", - acquired_at_ms=1, - ) - backfill_historical_revision_evidence(root) + raw_id = _prepare_candidate_ready_archive(root) with sqlite3.connect(root / "source.db") as connection: source_state_before = connection.execute( """ @@ -792,8 +795,8 @@ def test_run_reindex_canary_rejects_external_evidence_mutation_after_replay( ) -> None: """A source identity mutation after replay fails before inactive readiness.""" - artifact = build_seeded_archive(cache_root=tmp_path / "seeded-cache") - root = clone_seeded_archive(artifact, tmp_path / "archive").root + root = tmp_path / "archive" + _prepare_candidate_ready_archive(root) active_index = root / "index.db" active_digest = hashlib.sha256(active_index.read_bytes()).hexdigest() receipt_path = _write_candidate_receipt(root, tmp_path / "schema-inference-gate-receipt.json") @@ -822,8 +825,8 @@ def test_run_reindex_canary_rejects_active_index_rotation_after_replay( ) -> None: """A canary cannot compare against an index that stopped being active.""" - artifact = build_seeded_archive(cache_root=tmp_path / "seeded-cache") - root = clone_seeded_archive(artifact, tmp_path / "archive").root + root = tmp_path / "archive" + _prepare_candidate_ready_archive(root) location = ArchiveLocation.resolve(root) current_index = location.active_index_path rotated_index = tmp_path / "rotated" / "index.db" @@ -848,8 +851,8 @@ def test_rebuild_rejects_evidence_mutation_in_deadline_interrupted_pass( ) -> None: """A deferred resumable pass cannot preserve a mutated source proof.""" - artifact = build_seeded_archive(cache_root=tmp_path / "seeded-cache") - root = clone_seeded_archive(artifact, tmp_path / "archive").root + root = tmp_path / "archive" + _prepare_candidate_ready_archive(root) from polylogue.maintenance import replay as rebuild_replay From 66b45e646c6bf994d74d53346f7500d181fbd668 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sat, 8 Aug 2026 11:26:14 +0200 Subject: [PATCH 03/19] chore(architecture): inventory frozen-candidate writer entrypoint Problem The writer-module policy discovered the new frozen-candidate classifier as a source/index mutation entrypoint but its declared inventory did not name it. What changed Add the classifier to the revision-governance writer inventory so the policy continues to audit its twin-tier reach. Verification `devtools verify layering --json` reports zero violations. Ref polylogue-inactive-candidate-durable-barrier. Co-Authored-By: Codex --- docs/plans/layering.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/plans/layering.yaml b/docs/plans/layering.yaml index 9f1093796f..91aa3c33e8 100644 --- a/docs/plans/layering.yaml +++ b/docs/plans/layering.yaml @@ -37,6 +37,7 @@ writer_modules: interruption: atomic entrypoints: [admit_raw_and_parsed_result, apply_raw_membership_classification, apply_raw_revision_replay, + classify_raw_revision_cohort_for_frozen_candidate, classify_raw_revision_cohort_for_live_watch, classify_raw_revision_cohort_for_rebuild_repair, release_provisional_full_revisions, replace_raw_membership_census, write_parsed_for_retained_raw, write_parsed_for_retained_raw_result, write_raw_and_parsed, From 3d9cfe1363b46e0a3a04b7a3ac499c07f993e9dc Mon Sep 17 00:00:00 2001 From: Sinity Date: Sat, 8 Aug 2026 11:33:59 +0200 Subject: [PATCH 04/19] fix(reindex): block explicit candidate ops telemetry routes Problem The inactive-generation telemetry guard applied only when the sampler inferred ops.db from the candidate index path. A caller supplying an explicit archive root could bypass the same durable boundary. What changed Resolve and classify the connected index before either ops path is selected. The candidate canary now exercises both inferred and explicit telemetry routes and proves live ops.db bytes remain unchanged. Verification Strict mypy and Ruff pass. The candidate barrier and active FTS convergence suites pass 10/10. Ref polylogue-inactive-candidate-durable-barrier. Co-Authored-By: Codex --- polylogue/storage/fts/drift_sampling.py | 6 +++--- .../maintenance/test_inactive_candidate_durable_barrier.py | 6 ++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/polylogue/storage/fts/drift_sampling.py b/polylogue/storage/fts/drift_sampling.py index c320aebe5d..663312dde5 100644 --- a/polylogue/storage/fts/drift_sampling.py +++ b/polylogue/storage/fts/drift_sampling.py @@ -97,14 +97,14 @@ def sample_fts_drift_to_ops_sync(conn: sqlite3.Connection, *, archive_root: Path if not rows: return 0 + index_db_path = _index_db_path_sync(conn) + if index_db_path is not None and _is_owned_inactive_generation_index(index_db_path): + return 0 if archive_root is not None: ops_db_path = archive_root / "ops.db" else: - index_db_path = _index_db_path_sync(conn) if index_db_path is None: return 0 - if _is_owned_inactive_generation_index(index_db_path): - return 0 ops_db_path = index_db_path.with_name("ops.db") if not ops_db_path.exists(): return 0 diff --git a/tests/unit/maintenance/test_inactive_candidate_durable_barrier.py b/tests/unit/maintenance/test_inactive_candidate_durable_barrier.py index 0f0111a7c8..ce4a3f8a2a 100644 --- a/tests/unit/maintenance/test_inactive_candidate_durable_barrier.py +++ b/tests/unit/maintenance/test_inactive_candidate_durable_barrier.py @@ -13,6 +13,8 @@ from polylogue.maintenance.rebuild_index import RebuildIndexRequest, rebuild_index_from_source_sync from polylogue.sources.revision_backfill import census_historical_revision_evidence from polylogue.storage.blob_store import PreparedBlob +from polylogue.storage.fts.drift_sampling import sample_fts_drift_to_ops_sync +from polylogue.storage.fts.fts_lifecycle import rebuild_fts_index_sync from polylogue.storage.index_generation import IndexGenerationStore, source_revision_snapshot from polylogue.storage.sqlite.archive_tiers.archive import ( ArchiveStore, @@ -130,6 +132,7 @@ def test_owned_candidate_refuses_source_user_and_blob_writes( generation_root = Path(generation.index_path).parent source_before = _file_evidence(root / "source.db") user_before = _file_evidence(root / "user.db") + ops_before = _file_evidence(root / "ops.db") blobs_before = _blob_evidence(root / "blob") with ArchiveStore.open_owned_inactive_generation( @@ -138,6 +141,8 @@ def test_owned_candidate_refuses_source_user_and_blob_writes( owner_id=generation.owner_id, ) as candidate: candidate._conn.execute("CREATE TABLE candidate_index_probe (value INTEGER) STRICT") + rebuild_fts_index_sync(candidate._conn) + assert sample_fts_drift_to_ops_sync(candidate._conn, archive_root=root) == 0 candidate.commit() with pytest.raises(sqlite3.OperationalError, match="readonly"): candidate._ensure_source_conn().execute("UPDATE raw_sessions SET parse_error = 'candidate-write'") @@ -191,6 +196,7 @@ def test_owned_candidate_refuses_source_user_and_blob_writes( assert _file_evidence(root / "source.db") == source_before assert _file_evidence(root / "user.db") == user_before + assert _file_evidence(root / "ops.db") == ops_before assert _blob_evidence(root / "blob") == blobs_before with sqlite3.connect(generation.index_path) as candidate_index: assert candidate_index.execute( From eaab4b266b7e007234d736d9ce85e242717002be Mon Sep 17 00:00:00 2001 From: Sinity Date: Sat, 8 Aug 2026 11:53:17 +0200 Subject: [PATCH 05/19] fix(reindex): validate frozen authority before candidate allocation Re-derive current byte and membership authority before creating an inactive generation or rebuild transaction. Add a non-repairing generation-store mode so candidate validation cannot rewrite a missing or poisoned active-pointer anchor. Prove refusal leaves pointer, generation, transaction, and durable-tier evidence unchanged. Ref polylogue-inactive-candidate-durable-barrier Co-Authored-By: Codex --- polylogue/maintenance/rebuild_index.py | 9 +- polylogue/sources/revision_backfill.py | 65 +++++++++ polylogue/storage/index_generation.py | 32 +++-- .../storage/sqlite/archive_tiers/archive.py | 45 +++++- .../archive_tiers/revision_governance.py | 111 ++++++++++----- ...test_inactive_candidate_durable_barrier.py | 133 +++++++++++++++++- 6 files changed, 330 insertions(+), 65 deletions(-) diff --git a/polylogue/maintenance/rebuild_index.py b/polylogue/maintenance/rebuild_index.py index 3b16a0893d..58111918b4 100644 --- a/polylogue/maintenance/rebuild_index.py +++ b/polylogue/maintenance/rebuild_index.py @@ -1415,7 +1415,7 @@ async def _rebuild_index_from_source_owned( from polylogue.maintenance.replay import rebuild_index_from_source as replay_source from polylogue.sources.revision_backfill import ( RebuildDeadlineExceededError, - require_current_parser_source_census, + validate_frozen_source_authority, ) from polylogue.storage.archive_readiness import archive_readiness_status from polylogue.storage.index_generation import ( @@ -1424,7 +1424,6 @@ async def _rebuild_index_from_source_owned( ) from polylogue.storage.repair import repair_session_insights - generation_store = IndexGenerationStore(owned.location) provenance = RebuildProvenanceContext( root=root, receipt_path=request.schema_inference_receipt_path, @@ -1461,7 +1460,8 @@ async def _rebuild_index_from_source_owned( page = None pass_started_at_ms = int(time.time() * 1000) if resumable_full_source: - require_current_parser_source_census(root) + validate_frozen_source_authority(root) + generation_store = IndexGenerationStore(owned.location) if request.operation_id is not None: transaction = generation_store.load_transaction(request.operation_id) transaction = _reconcile_active_generation_transaction(generation_store, transaction) @@ -1553,7 +1553,8 @@ async def _rebuild_index_from_source_owned( raw_count, selected_raw_ids, skipped_by_blob_limit_count = select_rebuild_raw_ids(request) selection_elapsed_s = time.perf_counter() - selection_started_at selected_raw_count = len(selected_raw_ids) - require_current_parser_source_census(root, selected_raw_ids=selected_raw_ids) + validate_frozen_source_authority(root, selected_raw_ids=selected_raw_ids) + generation_store = IndexGenerationStore(owned.location) provenance.validate() generation = generation_store.create(source_snapshot=rebuild_source_evidence_snapshot(root)) try: diff --git a/polylogue/sources/revision_backfill.py b/polylogue/sources/revision_backfill.py index 63fcfbd977..9b5302c971 100644 --- a/polylogue/sources/revision_backfill.py +++ b/polylogue/sources/revision_backfill.py @@ -934,6 +934,71 @@ def require_current_parser_source_census( ) +def validate_frozen_source_authority( + archive_root: Path, + *, + selected_raw_ids: list[str] | None = None, + max_payload_bytes: int | None = None, + ingest_workers: int = 1, + prefetch_cache: RawParsePrefetchCache | None = None, +) -> None: + """Re-derive every selected source decision before allocating a candidate.""" + with ( + ArchiveStore.open_frozen_source_validation(archive_root) as archive, + _ParsedSessionSpill(archive_root, max_cached_payload_bytes=max_payload_bytes) as spill, + ): + census = _load_frozen_revision_evidence( + archive, + spill, + selected_raw_ids=selected_raw_ids, + max_payload_bytes=max_payload_bytes, + ingest_workers=ingest_workers, + prefetch_cache=prefetch_cache, + ) + _unclassified, logical_keys = archive.raw_revision_rebuild_selection(selected_raw_ids) + _membership_raw_ids, persisted_membership_keys = archive.expand_raw_membership_selection(selected_raw_ids) + membership_keys = {*persisted_membership_keys, *census.membership_candidates} + byte_replayed_keys: set[str] = set() + + for logical_key in sorted(logical_keys): + plan = archive.classify_raw_revision_cohort_for_frozen_candidate(logical_key) + if not plan.accepted_raw_ids: + convertible = archive.convertible_full_revision_raw_ids(logical_key) + if convertible: + raise FrozenSourceRemediationRequiredError( + "inactive candidate found a full-revision cohort that still requires membership " + f"remediation in frozen source: {logical_key}" + ) + continue + byte_replayed_keys.add(logical_key) + + for logical_key in sorted(membership_keys - byte_replayed_keys): + candidate_raw_ids = set(archive.raw_membership_rebuild_raw_ids(logical_key)) + candidate_raw_ids.update(census.membership_candidates.get(logical_key, ())) + revisions: list[MembershipRevision] = [] + for raw_id in sorted(candidate_raw_ids): + sessions, _payload_bytes = spill.for_raw(archive, raw_id) + for session in sessions: + session_logical_key = f"{session.source_name.value}:{session.provider_session_id}" + if session_logical_key != logical_key: + continue + projection = session_revision_projection(session) + revisions.append( + MembershipRevision( + raw_id, + projection, + session.updated_at, + browser_snapshot_fidelity=_browser_snapshot_fidelity(session.ingest_flags), + provider_message_ids=frozenset(message.provider_message_id for message in session.messages), + provider_attachment_ids=frozenset( + attachment.provider_attachment_id for attachment in session.attachments + ), + ) + ) + classification = classify_membership_revisions(revisions, existing_accepted_raw_id=None) + archive.require_frozen_membership_authority(logical_key, classification) + + def census_historical_revision_evidence( archive_root: Path, *, diff --git a/polylogue/storage/index_generation.py b/polylogue/storage/index_generation.py index d7e27c4692..ef0738f51f 100644 --- a/polylogue/storage/index_generation.py +++ b/polylogue/storage/index_generation.py @@ -452,7 +452,7 @@ class IndexGenerationStore: authority belongs. """ - def __init__(self, location: ArchiveLocation) -> None: + def __init__(self, location: ArchiveLocation, *, repair_anchor: bool = True) -> None: self.archive_root = location.configured_root self.location = location anchor = location.configured_root / ".index-active-pointer" @@ -486,23 +486,29 @@ def __init__(self, location: ArchiveLocation) -> None: self.active_pointer = configured_index if _is_generation_member(resolved) else resolved else: self.active_pointer = configured_index - temporary = anchor.with_suffix(".tmp") - # Constructing the store must not require the archive root to have - # been materialized first. Daemon bulk-rebuild routing is now - # unconditional, so this runs on every convergence tick -- including - # against a configured-but-not-yet-created root, where the eager - # pointer write previously raised FileNotFoundError. - anchor.parent.mkdir(parents=True, exist_ok=True) - temporary.write_text(str(self.active_pointer.absolute()), encoding="utf-8") - os.replace(temporary, anchor) - _fsync_directory(anchor.parent) + if repair_anchor: + temporary = anchor.with_suffix(".tmp") + # Constructing the store must not require the archive root to have + # been materialized first. Daemon bulk-rebuild routing is now + # unconditional, so this runs on every convergence tick -- including + # against a configured-but-not-yet-created root, where the eager + # pointer write previously raised FileNotFoundError. + anchor.parent.mkdir(parents=True, exist_ok=True) + temporary.write_text(str(self.active_pointer.absolute()), encoding="utf-8") + os.replace(temporary, anchor) + _fsync_directory(anchor.parent) self.generations_root = self.active_pointer.parent / ".index-generations" self.transactions_root = self.active_pointer.parent / ".index-rebuild-transactions" @classmethod - def for_archive_root(cls, archive_root: Path) -> IndexGenerationStore: + def for_archive_root( + cls, + archive_root: Path, + *, + repair_anchor: bool = True, + ) -> IndexGenerationStore: """Convenience constructor resolving ``archive_root`` into an :class:`ArchiveLocation` first.""" - return cls(ArchiveLocation.resolve(archive_root)) + return cls(ArchiveLocation.resolve(archive_root), repair_anchor=repair_anchor) def create_transaction( self, diff --git a/polylogue/storage/sqlite/archive_tiers/archive.py b/polylogue/storage/sqlite/archive_tiers/archive.py index 55623a94d8..0c85267686 100644 --- a/polylogue/storage/sqlite/archive_tiers/archive.py +++ b/polylogue/storage/sqlite/archive_tiers/archive.py @@ -197,6 +197,7 @@ finalize_raw_parse_state, mark_raw_parse_failed, mark_raw_parse_succeeded, + membership_decisions_for_classification, open_raw_revision_material, pending_raw_revision_logical_keys, raw_append_revision_parent, @@ -221,6 +222,7 @@ record_raw_failure_evidence, release_provisional_full_revisions, replace_raw_membership_census, + require_frozen_membership_authority, unclassified_raw_revision_rows, write_parsed_for_retained_raw, write_parsed_for_retained_raw_result, @@ -1728,12 +1730,16 @@ def __init__( read_timeout: float = 5.0, owned_inactive_generation: tuple[str, str] | None = None, source_tier_acquisition: bool = False, + frozen_source_validation: bool = False, ) -> None: if source_tier_acquisition and read_only: raise ValueError("source_tier_acquisition mode is a writer mode; read_only must be False") + if frozen_source_validation and (not read_only or owned_inactive_generation is not None): + raise ValueError("frozen source validation requires a read-only active archive") self._source_tier_acquisition = source_tier_acquisition self._owned_inactive_generation = owned_inactive_generation - self._inactive_candidate_durable_read_only = owned_inactive_generation is not None + self._frozen_source_validation = frozen_source_validation + self._inactive_candidate_durable_read_only = owned_inactive_generation is not None or frozen_source_validation self._active_writer_lease = None if not read_only: from polylogue.paths import archive_root as configured_archive_root @@ -1769,9 +1775,10 @@ def __init__( **json.loads((archive_root / "generation.json").read_text(encoding="utf-8")) ) declared_archive_root = Path(generation.archive_root).resolve(strict=True) - authoritative_generation = IndexGenerationStore.for_archive_root(declared_archive_root).load( - generation_id - ) + authoritative_generation = IndexGenerationStore.for_archive_root( + declared_archive_root, + repair_anchor=False, + ).load(generation_id) if ( generation != authoritative_generation or generation.owner_id != owner_id @@ -1866,8 +1873,9 @@ def _initialize_store( return if initialize: initialize_active_archive_root(archive_root) - if read_only: + if read_only and not self._frozen_source_validation: self._ensure_read_runtime_indexes() + if read_only: self._conn = sqlite3.connect(f"file:{self.index_db_path}?mode=ro", uri=True, timeout=read_timeout) pragma_statements = READ_CONNECTION_PRAGMA_STATEMENTS else: @@ -1905,6 +1913,11 @@ def _initialize_store( _InactiveCandidateBlobPublisher if self._inactive_candidate_durable_read_only else ArchiveBlobPublisher ) self._blob_publisher = publisher_type(self.source_db_path, self.archive_root / "blob") + elif self._frozen_source_validation: + self._blob_publisher = _InactiveCandidateBlobPublisher( + self.source_db_path, + self.archive_root / "blob", + ) self._attach_user_tier_if_present() @classmethod @@ -1931,6 +1944,16 @@ def open_source_tier_acquisition(cls, archive_root: Path) -> ArchiveStore: """ return cls(archive_root, initialize=False, read_only=False, source_tier_acquisition=True) + @classmethod + def open_frozen_source_validation(cls, archive_root: Path) -> ArchiveStore: + """Open the live tiers without repairing or mutating any durable or pointer state.""" + return cls( + archive_root, + initialize=False, + read_only=True, + frozen_source_validation=True, + ) + @classmethod def open_owned_inactive_generation(cls, archive_root: Path, *, generation_id: str, owner_id: str) -> ArchiveStore: """Open a typed inactive generation without weakening normal identity checks.""" @@ -2500,6 +2523,18 @@ def classify_raw_revision_cohort_for_rebuild_repair( def classify_raw_revision_cohort_for_frozen_candidate(self, logical_source_key: str) -> RevisionReplayPlan: return classify_raw_revision_cohort_for_frozen_candidate(self, logical_source_key) + def require_frozen_membership_authority( + self, + logical_source_key: str, + classification: MembershipClassification, + ) -> None: + require_frozen_membership_authority( + self, + logical_source_key, + classification, + membership_decisions_for_classification(classification), + ) + def classify_raw_revision_cohort_for_live_watch( self, logical_source_key: str, diff --git a/polylogue/storage/sqlite/archive_tiers/revision_governance.py b/polylogue/storage/sqlite/archive_tiers/revision_governance.py index dc80ee1792..6b4d4c532d 100644 --- a/polylogue/storage/sqlite/archive_tiers/revision_governance.py +++ b/polylogue/storage/sqlite/archive_tiers/revision_governance.py @@ -2553,6 +2553,76 @@ def _application_decision_for(decision: MembershipDecision) -> ApplicationDecisi return ApplicationDecision.SELECTED_BASELINE +def membership_decisions_for_classification( + classification: MembershipClassification, +) -> dict[str, MembershipDecision]: + """Return the source-tier decisions a fresh replay must persist.""" + decisions: dict[str, MembershipDecision] = dict.fromkeys( + classification.ambiguous_raw_ids, + MembershipDecision.AMBIGUOUS, + ) + decisions.update( + dict.fromkeys( + classification.equivalent_raw_ids, + MembershipDecision.SUPERSEDED_EQUIVALENT + if classification.accepted_raw_ids + else MembershipDecision.AMBIGUOUS, + ) + ) + for raw_id in classification.accepted_raw_ids[:-1]: + decisions[raw_id] = MembershipDecision.SUPERSEDED_PREFIX + if classification.accepted_raw_ids: + decisions[classification.accepted_raw_ids[-1]] = MembershipDecision.APPLIED + return decisions + + +def require_frozen_membership_authority( + store: RawRevisionGovernanceHost, + logical_source_key: str, + classification: MembershipClassification, + decisions: dict[str, MembershipDecision] | None = None, +) -> None: + """Require persisted membership authority to equal a current re-derivation.""" + conn = store._ensure_source_conn() + expected_decisions = decisions or membership_decisions_for_classification(classification) + for raw_id, decision in expected_decisions.items(): + expected = ( + decision.value, + "quarantined" if decision in {MembershipDecision.AMBIGUOUS, MembershipDecision.DEFERRED} else "byte_proven", + classification.accepted_raw_ids.index(raw_id) if raw_id in classification.accepted_raw_ids else 0, + ) + persisted = conn.execute( + """ + SELECT decision, revision_authority, acquisition_generation + FROM raw_session_memberships + WHERE raw_id = ? AND logical_source_key = ? + """, + (raw_id, logical_source_key), + ).fetchone() + if persisted is None or tuple(persisted) != expected: + raise FrozenSourceRemediationRequiredError( + "inactive candidate re-derived different membership authority for frozen raw " + f"{raw_id}; complete source remediation before candidate construction" + ) + complete = conn.execute( + """ + SELECT c.status = 'complete' + AND NOT EXISTS ( + SELECT 1 FROM raw_session_memberships AS m + WHERE m.raw_id = c.raw_id + AND (m.decision IS NULL OR m.decision IN ('ambiguous', 'deferred')) + ) + FROM raw_membership_census AS c WHERE c.raw_id = ? + """, + (raw_id,), + ).fetchone() + if complete is None or not bool(complete[0]): + raise FrozenSourceRemediationRequiredError( + "inactive candidate found incomplete frozen membership authority for raw " + f"{raw_id}; complete source remediation before candidate construction" + ) + + def apply_raw_membership_classification( store: RawRevisionGovernanceHost, logical_source_key: str, @@ -2584,25 +2654,13 @@ def apply_raw_membership_classification( """ conn = store._ensure_source_conn() decided_at_ms = int(datetime.now(UTC).timestamp() * 1000) - decisions: dict[str, MembershipDecision] = dict.fromkeys( - classification.ambiguous_raw_ids, MembershipDecision.AMBIGUOUS - ) + decisions = membership_decisions_for_classification(classification) # "superseded_equivalent" asserts an accepted chain superseded the # member. With no accepted head, equivalence collapses back into the # unresolved cohort: labeling it superseded (and, downstream, # byte_proven) fabricates authority for a head that was never written # -- 914 headless-but-"byte_proven" logical sources on the 2026-07-20 # rebuild walk came from exactly this mislabel. - decisions.update( - dict.fromkeys( - classification.equivalent_raw_ids, - MembershipDecision.SUPERSEDED_EQUIVALENT - if classification.accepted_raw_ids - else MembershipDecision.AMBIGUOUS, - ) - ) - for raw_id in classification.accepted_raw_ids[:-1]: - decisions[raw_id] = MembershipDecision.SUPERSEDED_PREFIX session_id: str | None = None # Ambiguous evidence is debt, not deletion authority. A later branch # must not erase the last accepted session/head; a cold rebuild simply @@ -2908,27 +2966,7 @@ def apply_raw_membership_classification( decisions[accepted_raw_id] = MembershipDecision.APPLIED if _is_frozen_candidate(store): - for raw_id, decision in decisions.items(): - expected = ( - decision.value, - "quarantined" - if decision in {MembershipDecision.AMBIGUOUS, MembershipDecision.DEFERRED} - else "byte_proven", - classification.accepted_raw_ids.index(raw_id) if raw_id in classification.accepted_raw_ids else 0, - ) - persisted = conn.execute( - """ - SELECT decision, revision_authority, acquisition_generation - FROM raw_session_memberships - WHERE raw_id = ? AND logical_source_key = ? - """, - (raw_id, logical_source_key), - ).fetchone() - if persisted is None or tuple(persisted) != expected: - raise FrozenSourceRemediationRequiredError( - "inactive candidate re-derived different membership authority for frozen raw " - f"{raw_id}; complete source remediation before candidate construction" - ) + require_frozen_membership_authority(store, logical_source_key, classification, decisions) else: with conn if manage_transaction else nullcontext(): for raw_id, decision in decisions.items(): @@ -2967,11 +3005,6 @@ def apply_raw_membership_classification( (raw_id,), ).fetchone() if _is_frozen_candidate(store): - if complete is None or not bool(complete[0]): - raise FrozenSourceRemediationRequiredError( - "inactive candidate found incomplete frozen membership authority for raw " - f"{raw_id}; complete source remediation before candidate construction" - ) continue if complete is not None and bool(complete[0]): provider, _blob_hash, _source_path, _kind, _blob_size = raw_revision_descriptor(store, raw_id) diff --git a/tests/unit/maintenance/test_inactive_candidate_durable_barrier.py b/tests/unit/maintenance/test_inactive_candidate_durable_barrier.py index ce4a3f8a2a..57da403458 100644 --- a/tests/unit/maintenance/test_inactive_candidate_durable_barrier.py +++ b/tests/unit/maintenance/test_inactive_candidate_durable_barrier.py @@ -3,6 +3,7 @@ from __future__ import annotations import hashlib +import json import sqlite3 from io import BytesIO from pathlib import Path @@ -11,7 +12,10 @@ from polylogue.core.enums import Provider from polylogue.maintenance.rebuild_index import RebuildIndexRequest, rebuild_index_from_source_sync -from polylogue.sources.revision_backfill import census_historical_revision_evidence +from polylogue.sources.revision_backfill import ( + backfill_historical_revision_evidence, + census_historical_revision_evidence, +) from polylogue.storage.blob_store import PreparedBlob from polylogue.storage.fts.drift_sampling import sample_fts_drift_to_ops_sync from polylogue.storage.fts.fts_lifecycle import rebuild_fts_index_sync @@ -48,6 +52,47 @@ def _symlink_evidence(path: Path) -> tuple[int, int, str]: return stat.st_dev, stat.st_ino, target +def _optional_path_evidence(path: Path) -> tuple[int, int, str] | None: + if not path.exists() and not path.is_symlink(): + return None + return _symlink_evidence(path) + + +def _assert_no_candidate_bookkeeping(root: Path) -> None: + assert not (root / ".index-generations").exists() + assert not (root / ".index-rebuild-transactions").exists() + + +def _chatgpt_bundle(*native_ids: str) -> bytes: + sessions = [] + for native_id in native_ids: + node_id = f"{native_id}-node" + sessions.append( + { + "id": native_id, + "conversation_id": native_id, + "title": native_id, + "create_time": 1_700_000_000, + "update_time": 1_700_000_001, + "current_node": node_id, + "mapping": { + node_id: { + "id": node_id, + "parent": None, + "children": [], + "message": { + "id": f"{native_id}-message", + "author": {"role": "user"}, + "content": {"content_type": "text", "parts": [native_id]}, + "create_time": 1_700_000_000, + }, + } + }, + } + ) + return json.dumps(sessions, sort_keys=True).encode() + + def _prepare_frozen_source(root: Path, monkeypatch: pytest.MonkeyPatch) -> Path: build_independent_raw_corpus(root, raw_count=1, avg_payload_bytes=1_000) monkeypatch.setenv("POLYLOGUE_ARCHIVE_ROOT", str(root)) @@ -73,6 +118,8 @@ def test_real_no_promote_candidate_preserves_frozen_durable_tiers( root = tmp_path / "archive" receipt_path = _prepare_frozen_source(root, monkeypatch) generation_store = IndexGenerationStore.for_archive_root(root) + anchor = root / ".index-active-pointer" + anchor_before = _optional_path_evidence(anchor) active_target_before = generation_store.active_pointer.resolve(strict=True) active_pointer_before = _symlink_evidence(generation_store.active_pointer) active_index_before = _file_evidence(active_target_before) @@ -107,6 +154,7 @@ def test_real_no_promote_candidate_preserves_frozen_durable_tiers( generation = generation_store.load(str(result.transaction["generation_id"])) assert generation.state == "inactive" assert generation_store.active_pointer.resolve(strict=True) == active_target_before + assert _optional_path_evidence(anchor) == anchor_before assert _symlink_evidence(generation_store.active_pointer) == active_pointer_before assert _file_evidence(active_target_before) == active_index_before assert _file_evidence(root / "source.db") == source_before @@ -130,6 +178,9 @@ def test_owned_candidate_refuses_source_user_and_blob_writes( generation_store = IndexGenerationStore.for_archive_root(root) generation = generation_store.create(source_snapshot=source_revision_snapshot(root)) generation_root = Path(generation.index_path).parent + anchor = root / ".index-active-pointer" + anchor.write_text(str(generation.index_path), encoding="utf-8") + poisoned_anchor_before = _optional_path_evidence(anchor) source_before = _file_evidence(root / "source.db") user_before = _file_evidence(root / "user.db") ops_before = _file_evidence(root / "ops.db") @@ -198,20 +249,30 @@ def test_owned_candidate_refuses_source_user_and_blob_writes( assert _file_evidence(root / "user.db") == user_before assert _file_evidence(root / "ops.db") == ops_before assert _blob_evidence(root / "blob") == blobs_before + assert _optional_path_evidence(anchor) == poisoned_anchor_before with sqlite3.connect(generation.index_path) as candidate_index: assert candidate_index.execute( "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'candidate_index_probe'" ).fetchone() == (1,) +@pytest.mark.parametrize("anchor_state", ["missing", "poisoned"]) def test_candidate_requires_current_parser_census_before_generation_readiness( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, + anchor_state: str, ) -> None: root = tmp_path / "archive" build_independent_raw_corpus(root, raw_count=1, avg_payload_bytes=1_000) monkeypatch.setenv("POLYLOGUE_ARCHIVE_ROOT", str(root)) receipt_path = write_valid_rebuild_receipt(root, root.parent / "schema-inference-receipt.json") + anchor = root / ".index-active-pointer" + if anchor_state == "poisoned": + anchor.write_text( + str(root / ".index-generations" / "gen-poisoned" / "index.db"), + encoding="utf-8", + ) + anchor_before = _optional_path_evidence(anchor) with pytest.raises(FrozenSourceRemediationRequiredError, match="complete current-parser source census"): rebuild_index_from_source_sync( @@ -222,7 +283,8 @@ def test_candidate_requires_current_parser_census_before_generation_readiness( ) ) - assert not list((root / ".index-generations").glob("gen-*")) + assert _optional_path_evidence(anchor) == anchor_before + _assert_no_candidate_bookkeeping(root) def test_candidate_requires_complete_source_authority_before_generation_readiness( @@ -245,19 +307,26 @@ def test_candidate_requires_complete_source_authority_before_generation_readines ) ) - assert not list((root / ".index-generations").glob("gen-*")) + _assert_no_candidate_bookkeeping(root) +@pytest.mark.parametrize("drift", ["asserted", "stale-byte-proven"]) def test_candidate_rejects_authority_drift_in_frozen_source( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, + drift: str, ) -> None: root = tmp_path / "archive" _prepare_frozen_source(root, monkeypatch) with sqlite3.connect(root / "source.db") as source: - source.execute("UPDATE raw_sessions SET revision_authority = 'asserted', baseline_raw_id = NULL") + if drift == "asserted": + source.execute("UPDATE raw_sessions SET revision_authority = 'asserted', baseline_raw_id = NULL") + else: + source.execute("UPDATE raw_sessions SET acquisition_generation = 7") source.commit() receipt_path = write_valid_rebuild_receipt(root, root.parent / "post-drift-receipt.json") + anchor = root / ".index-active-pointer" + anchor_before = _optional_path_evidence(anchor) with pytest.raises(FrozenSourceRemediationRequiredError, match="re-derived different byte authority"): rebuild_index_from_source_sync( @@ -268,6 +337,62 @@ def test_candidate_rejects_authority_drift_in_frozen_source( ) ) + assert _optional_path_evidence(anchor) == anchor_before + _assert_no_candidate_bookkeeping(root) + + +def test_candidate_rejects_membership_authority_drift_before_allocation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = tmp_path / "archive" + monkeypatch.setenv("POLYLOGUE_ARCHIVE_ROOT", str(root)) + with ArchiveStore.open_existing(root, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.CHATGPT, + payload=_chatgpt_bundle("membership-a", "membership-b"), + source_path="conversations.json", + acquired_at_ms=1, + ) + result = backfill_historical_revision_evidence(root) + assert result.replayed_logical_sources == 2 + with sqlite3.connect(root / "source.db") as source: + decisions = source.execute( + """ + SELECT logical_source_key, decision + FROM raw_session_memberships WHERE raw_id = ? + ORDER BY logical_source_key + """, + (raw_id,), + ).fetchall() + assert decisions == [ + ("chatgpt:membership-a", "applied"), + ("chatgpt:membership-b", "applied"), + ] + source.execute( + """ + UPDATE raw_session_memberships SET decision = 'superseded_prefix' + WHERE raw_id = ? AND logical_source_key = ? + """, + (raw_id, "chatgpt:membership-a"), + ) + source.commit() + receipt_path = write_valid_rebuild_receipt(root, root.parent / "membership-drift-receipt.json") + anchor = root / ".index-active-pointer" + anchor_before = _optional_path_evidence(anchor) + + with pytest.raises(FrozenSourceRemediationRequiredError, match="different membership authority"): + rebuild_index_from_source_sync( + RebuildIndexRequest( + archive_root=root, + schema_inference_receipt_path=receipt_path, + promote=False, + ) + ) + + assert _optional_path_evidence(anchor) == anchor_before + _assert_no_candidate_bookkeeping(root) + def test_active_bootstrap_still_rejects_candidate_durable_symlinks( tmp_path: Path, From 78650b69b46a4c960c8cc62eec6d190212528cbf Mon Sep 17 00:00:00 2001 From: Sinity Date: Sat, 8 Aug 2026 12:01:59 +0200 Subject: [PATCH 06/19] fix(reindex): validate split-root candidate authority Bind frozen source validation to the canonical active index selected by the owned archive identity. Keep the durable archive root for source, user, and blob reads while ignoring missing or poisoned generation-member anchors without repairing them. Ref polylogue-inactive-candidate-durable-barrier Co-Authored-By: Codex --- polylogue/maintenance/rebuild_index.py | 12 ++++- polylogue/sources/revision_backfill.py | 6 ++- polylogue/storage/index_generation.py | 46 ++++++++++--------- .../storage/sqlite/archive_tiers/archive.py | 14 +++++- 4 files changed, 52 insertions(+), 26 deletions(-) diff --git a/polylogue/maintenance/rebuild_index.py b/polylogue/maintenance/rebuild_index.py index 58111918b4..02de7463df 100644 --- a/polylogue/maintenance/rebuild_index.py +++ b/polylogue/maintenance/rebuild_index.py @@ -1420,6 +1420,7 @@ async def _rebuild_index_from_source_owned( from polylogue.storage.archive_readiness import archive_readiness_status from polylogue.storage.index_generation import ( IndexGenerationStore, + canonical_active_index_path, rebuild_source_evidence_snapshot, ) from polylogue.storage.repair import repair_session_insights @@ -1460,7 +1461,10 @@ async def _rebuild_index_from_source_owned( page = None pass_started_at_ms = int(time.time() * 1000) if resumable_full_source: - validate_frozen_source_authority(root) + validate_frozen_source_authority( + root, + active_index_path=canonical_active_index_path(owned.location), + ) generation_store = IndexGenerationStore(owned.location) if request.operation_id is not None: transaction = generation_store.load_transaction(request.operation_id) @@ -1553,7 +1557,11 @@ async def _rebuild_index_from_source_owned( raw_count, selected_raw_ids, skipped_by_blob_limit_count = select_rebuild_raw_ids(request) selection_elapsed_s = time.perf_counter() - selection_started_at selected_raw_count = len(selected_raw_ids) - validate_frozen_source_authority(root, selected_raw_ids=selected_raw_ids) + validate_frozen_source_authority( + root, + active_index_path=canonical_active_index_path(owned.location), + selected_raw_ids=selected_raw_ids, + ) generation_store = IndexGenerationStore(owned.location) provenance.validate() generation = generation_store.create(source_snapshot=rebuild_source_evidence_snapshot(root)) diff --git a/polylogue/sources/revision_backfill.py b/polylogue/sources/revision_backfill.py index 9b5302c971..52666fa217 100644 --- a/polylogue/sources/revision_backfill.py +++ b/polylogue/sources/revision_backfill.py @@ -937,6 +937,7 @@ def require_current_parser_source_census( def validate_frozen_source_authority( archive_root: Path, *, + active_index_path: Path | None = None, selected_raw_ids: list[str] | None = None, max_payload_bytes: int | None = None, ingest_workers: int = 1, @@ -944,7 +945,10 @@ def validate_frozen_source_authority( ) -> None: """Re-derive every selected source decision before allocating a candidate.""" with ( - ArchiveStore.open_frozen_source_validation(archive_root) as archive, + ArchiveStore.open_frozen_source_validation( + archive_root, + active_index_path=active_index_path, + ) as archive, _ParsedSessionSpill(archive_root, max_cached_payload_bytes=max_payload_bytes) as spill, ): census = _load_frozen_revision_evidence( diff --git a/polylogue/storage/index_generation.py b/polylogue/storage/index_generation.py index ef0738f51f..cc18a47e92 100644 --- a/polylogue/storage/index_generation.py +++ b/polylogue/storage/index_generation.py @@ -126,6 +126,19 @@ def _is_generation_member(path: Path) -> bool: return len(parts) - depth > 2 +def canonical_active_index_path(location: ArchiveLocation) -> Path: + """Resolve the active index path without repairing a missing or poisoned anchor.""" + anchored = location.active_pointer + if anchored is not None and not _is_generation_member(anchored): + return anchored + configured_index = location.configured_tier("index").configured_path + if configured_index.is_symlink(): + target = Path(os.readlink(configured_index)) + resolved = target if target.is_absolute() else configured_index.parent / target + return configured_index if _is_generation_member(resolved) else resolved + return configured_index + + @dataclass(frozen=True, slots=True) class IndexGeneration: generation_id: str @@ -456,11 +469,9 @@ def __init__(self, location: ArchiveLocation, *, repair_anchor: bool = True) -> self.archive_root = location.configured_root self.location = location anchor = location.configured_root / ".index-active-pointer" - configured_index = location.configured_tier("index").configured_path anchored = location.active_pointer - if anchored is not None and not _is_generation_member(anchored): - self.active_pointer = anchored - else: + self.active_pointer = canonical_active_index_path(location) + if (anchored is None or _is_generation_member(anchored)) and repair_anchor: # Recompute, and rewrite the anchor, when it is absent OR poisoned. # # The canonical pointer is the path ``promote()`` replaces with a @@ -480,23 +491,16 @@ def __init__(self, location: ArchiveLocation, *, repair_anchor: bool = True) -> # anchor as recoverable rather than fatal lets an archive already # carrying one heal on next open, instead of needing the file # repaired by hand. - if configured_index.is_symlink(): - target = Path(os.readlink(configured_index)) - resolved = target if target.is_absolute() else configured_index.parent / target - self.active_pointer = configured_index if _is_generation_member(resolved) else resolved - else: - self.active_pointer = configured_index - if repair_anchor: - temporary = anchor.with_suffix(".tmp") - # Constructing the store must not require the archive root to have - # been materialized first. Daemon bulk-rebuild routing is now - # unconditional, so this runs on every convergence tick -- including - # against a configured-but-not-yet-created root, where the eager - # pointer write previously raised FileNotFoundError. - anchor.parent.mkdir(parents=True, exist_ok=True) - temporary.write_text(str(self.active_pointer.absolute()), encoding="utf-8") - os.replace(temporary, anchor) - _fsync_directory(anchor.parent) + temporary = anchor.with_suffix(".tmp") + # Constructing the store must not require the archive root to have + # been materialized first. Daemon bulk-rebuild routing is now + # unconditional, so this runs on every convergence tick -- including + # against a configured-but-not-yet-created root, where the eager + # pointer write previously raised FileNotFoundError. + anchor.parent.mkdir(parents=True, exist_ok=True) + temporary.write_text(str(self.active_pointer.absolute()), encoding="utf-8") + os.replace(temporary, anchor) + _fsync_directory(anchor.parent) self.generations_root = self.active_pointer.parent / ".index-generations" self.transactions_root = self.active_pointer.parent / ".index-rebuild-transactions" diff --git a/polylogue/storage/sqlite/archive_tiers/archive.py b/polylogue/storage/sqlite/archive_tiers/archive.py index 0c85267686..23be4513ea 100644 --- a/polylogue/storage/sqlite/archive_tiers/archive.py +++ b/polylogue/storage/sqlite/archive_tiers/archive.py @@ -1731,14 +1731,18 @@ def __init__( owned_inactive_generation: tuple[str, str] | None = None, source_tier_acquisition: bool = False, frozen_source_validation: bool = False, + frozen_index_path: Path | None = None, ) -> None: if source_tier_acquisition and read_only: raise ValueError("source_tier_acquisition mode is a writer mode; read_only must be False") if frozen_source_validation and (not read_only or owned_inactive_generation is not None): raise ValueError("frozen source validation requires a read-only active archive") + if frozen_index_path is not None and not frozen_source_validation: + raise ValueError("a frozen index path is valid only for frozen source validation") self._source_tier_acquisition = source_tier_acquisition self._owned_inactive_generation = owned_inactive_generation self._frozen_source_validation = frozen_source_validation + self._frozen_index_path = frozen_index_path self._inactive_candidate_durable_read_only = owned_inactive_generation is not None or frozen_source_validation self._active_writer_lease = None if not read_only: @@ -1836,7 +1840,7 @@ def _initialize_store( ) -> None: self.archive_root = archive_root self.source_db_path = archive_root / "source.db" - self.index_db_path = archive_root / "index.db" + self.index_db_path = self._frozen_index_path or archive_root / "index.db" self.embeddings_db_path = archive_root / "embeddings.db" self.user_db_path = archive_root / "user.db" self.ops_db_path = archive_root / "ops.db" @@ -1945,13 +1949,19 @@ def open_source_tier_acquisition(cls, archive_root: Path) -> ArchiveStore: return cls(archive_root, initialize=False, read_only=False, source_tier_acquisition=True) @classmethod - def open_frozen_source_validation(cls, archive_root: Path) -> ArchiveStore: + def open_frozen_source_validation( + cls, + archive_root: Path, + *, + active_index_path: Path | None = None, + ) -> ArchiveStore: """Open the live tiers without repairing or mutating any durable or pointer state.""" return cls( archive_root, initialize=False, read_only=True, frozen_source_validation=True, + frozen_index_path=active_index_path, ) @classmethod From f23d620d45a4440971e1eddf015748b7f74be62f Mon Sep 17 00:00:00 2001 From: Sinity Date: Sat, 8 Aug 2026 12:08:43 +0200 Subject: [PATCH 07/19] fix(reindex): gate daemon candidate allocation on frozen source Run the same current byte and membership authority validation inside the daemon bulk resolver after ownership and provenance checks, before constructing the generation store or creating a transaction. Update daemon fixtures to represent a phase-2-complete source and prove missing or poisoned anchors remain untouched on refusal. Ref polylogue-inactive-candidate-durable-barrier Co-Authored-By: Codex --- polylogue/daemon/bulk_rebuild.py | 6 ++++ tests/unit/daemon/test_bulk_rebuild.py | 2 ++ ...test_inactive_candidate_durable_barrier.py | 29 +++++++++++++++++++ 3 files changed, 37 insertions(+) diff --git a/polylogue/daemon/bulk_rebuild.py b/polylogue/daemon/bulk_rebuild.py index 276b51f900..288ef96821 100644 --- a/polylogue/daemon/bulk_rebuild.py +++ b/polylogue/daemon/bulk_rebuild.py @@ -54,6 +54,7 @@ from polylogue.storage.index_generation import ( IndexGenerationStore, IndexRebuildTransaction, + canonical_active_index_path, rebuild_source_evidence_snapshot, ) @@ -189,6 +190,7 @@ def resolve_or_start_daemon_bulk_rebuild_transaction( archive location before touching disk, not just the eventual write pass. """ from polylogue.maintenance.rebuild_index import require_rebuild_schema_currency + from polylogue.sources.revision_backfill import validate_frozen_source_authority require_rebuild_schema_currency(root) _validate_rebuild_provenance_receipt(root, schema_inference_receipt_path) @@ -210,6 +212,10 @@ def resolve_or_start_daemon_bulk_rebuild_transaction( # after ownership acquisition so receipt expiry, source revision, or # external-corpus drift cannot reach generation bookkeeping. _validate_rebuild_provenance_receipt(root, schema_inference_receipt_path) + validate_frozen_source_authority( + root, + active_index_path=canonical_active_index_path(location), + ) store = IndexGenerationStore(location) transaction: IndexRebuildTransaction | None try: diff --git a/tests/unit/daemon/test_bulk_rebuild.py b/tests/unit/daemon/test_bulk_rebuild.py index 342a87c2e7..413edfe958 100644 --- a/tests/unit/daemon/test_bulk_rebuild.py +++ b/tests/unit/daemon/test_bulk_rebuild.py @@ -53,6 +53,7 @@ RebuildSchemaCurrencyError, rebuild_index_from_source_sync, ) +from polylogue.sources.revision_backfill import backfill_historical_revision_evidence from polylogue.storage.archive_identity import ArchiveLocation, OwnedArchiveLocation, assert_owns_archive_location from polylogue.storage.archive_readiness import probe_archive_tier from polylogue.storage.index_generation import ( @@ -111,6 +112,7 @@ def _seed_corpus(root: Path, *, count: int = _RAW_COUNT) -> None: source_path=f"gd6v-corpus-{index}.jsonl", acquired_at_ms=index, ) + backfill_historical_revision_evidence(root) def _connect(path: Path) -> sqlite3.Connection: diff --git a/tests/unit/maintenance/test_inactive_candidate_durable_barrier.py b/tests/unit/maintenance/test_inactive_candidate_durable_barrier.py index 57da403458..a59b2727c6 100644 --- a/tests/unit/maintenance/test_inactive_candidate_durable_barrier.py +++ b/tests/unit/maintenance/test_inactive_candidate_durable_barrier.py @@ -11,6 +11,7 @@ import pytest from polylogue.core.enums import Provider +from polylogue.daemon.bulk_rebuild import resolve_or_start_daemon_bulk_rebuild_transaction from polylogue.maintenance.rebuild_index import RebuildIndexRequest, rebuild_index_from_source_sync from polylogue.sources.revision_backfill import ( backfill_historical_revision_evidence, @@ -287,6 +288,34 @@ def test_candidate_requires_current_parser_census_before_generation_readiness( _assert_no_candidate_bookkeeping(root) +@pytest.mark.parametrize("anchor_state", ["missing", "poisoned"]) +def test_daemon_candidate_requires_source_admission_before_transaction_allocation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + anchor_state: str, +) -> None: + root = tmp_path / "archive" + build_independent_raw_corpus(root, raw_count=1, avg_payload_bytes=1_000) + monkeypatch.setenv("POLYLOGUE_ARCHIVE_ROOT", str(root)) + receipt_path = write_valid_rebuild_receipt(root, root.parent / "daemon-schema-inference-receipt.json") + anchor = root / ".index-active-pointer" + if anchor_state == "poisoned": + anchor.write_text( + str(root / ".index-generations" / "gen-poisoned" / "index.db"), + encoding="utf-8", + ) + anchor_before = _optional_path_evidence(anchor) + + with pytest.raises(FrozenSourceRemediationRequiredError, match="complete current-parser source census"): + resolve_or_start_daemon_bulk_rebuild_transaction( + root, + schema_inference_receipt_path=receipt_path, + ) + + assert _optional_path_evidence(anchor) == anchor_before + _assert_no_candidate_bookkeeping(root) + + def test_candidate_requires_complete_source_authority_before_generation_readiness( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, From f6b5d9db4198a97d602365909bad88354a527937 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sat, 8 Aug 2026 12:11:52 +0200 Subject: [PATCH 08/19] refactor(reindex): route daemon admission through maintenance Keep daemon adapters out of source-layer internals by exposing the frozen-source admission boundary from the maintenance operation layer. The wrapper retains the exact owned active-index identity and leaves generation allocation ordering unchanged. Ref polylogue-inactive-candidate-durable-barrier Co-Authored-By: Codex --- polylogue/daemon/bulk_rebuild.py | 8 ++------ polylogue/maintenance/rebuild_index.py | 11 +++++++++++ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/polylogue/daemon/bulk_rebuild.py b/polylogue/daemon/bulk_rebuild.py index 288ef96821..0b37d8baec 100644 --- a/polylogue/daemon/bulk_rebuild.py +++ b/polylogue/daemon/bulk_rebuild.py @@ -49,12 +49,12 @@ from polylogue.maintenance.rebuild_index import ( _REBUILD_TERMINAL_NOT_RESUMABLE, _reconcile_active_generation_transaction, + validate_rebuild_source_admission, ) from polylogue.storage.archive_identity import ArchiveLocation, OwnedArchiveLocation, assert_owns_archive_location from polylogue.storage.index_generation import ( IndexGenerationStore, IndexRebuildTransaction, - canonical_active_index_path, rebuild_source_evidence_snapshot, ) @@ -190,7 +190,6 @@ def resolve_or_start_daemon_bulk_rebuild_transaction( archive location before touching disk, not just the eventual write pass. """ from polylogue.maintenance.rebuild_index import require_rebuild_schema_currency - from polylogue.sources.revision_backfill import validate_frozen_source_authority require_rebuild_schema_currency(root) _validate_rebuild_provenance_receipt(root, schema_inference_receipt_path) @@ -212,10 +211,7 @@ def resolve_or_start_daemon_bulk_rebuild_transaction( # after ownership acquisition so receipt expiry, source revision, or # external-corpus drift cannot reach generation bookkeeping. _validate_rebuild_provenance_receipt(root, schema_inference_receipt_path) - validate_frozen_source_authority( - root, - active_index_path=canonical_active_index_path(location), - ) + validate_rebuild_source_admission(root, location) store = IndexGenerationStore(location) transaction: IndexRebuildTransaction | None try: diff --git a/polylogue/maintenance/rebuild_index.py b/polylogue/maintenance/rebuild_index.py index 02de7463df..1ac2e630c4 100644 --- a/polylogue/maintenance/rebuild_index.py +++ b/polylogue/maintenance/rebuild_index.py @@ -131,6 +131,17 @@ def require_rebuild_schema_currency(root: Path) -> dict[str, object]: return diagnostic +def validate_rebuild_source_admission(root: Path, location: ArchiveLocation) -> None: + """Validate frozen source authority through the owned active-index identity.""" + from polylogue.sources.revision_backfill import validate_frozen_source_authority + from polylogue.storage.index_generation import canonical_active_index_path + + validate_frozen_source_authority( + root, + active_index_path=canonical_active_index_path(location), + ) + + @dataclass(slots=True) class RebuildProvenanceContext: """Validated evidence shared by every mutation in one rebuild pass. From 7f137af54cde11587674940ca5c902a4699bde80 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sat, 8 Aug 2026 12:19:41 +0200 Subject: [PATCH 09/19] fix(reindex): preserve anchor during stale retirement A resumable rebuild that failed provenance admission marked its transaction stale through an IndexGenerationStore constructor that also repaired missing or poisoned active-pointer anchors. That lifecycle-only failure path must not alter active archive authority. Construct the store with anchor repair disabled and cover both missing and poisoned anchors on the real operation-id route while preserving stale transaction evidence. Ref polylogue-inactive-candidate-durable-barrier. --- polylogue/maintenance/rebuild_index.py | 2 +- .../test_rebuild_index_provenance_gate.py | 42 +++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/polylogue/maintenance/rebuild_index.py b/polylogue/maintenance/rebuild_index.py index 1ac2e630c4..c52ba86a42 100644 --- a/polylogue/maintenance/rebuild_index.py +++ b/polylogue/maintenance/rebuild_index.py @@ -285,7 +285,7 @@ def _mark_rebuild_transaction_stale_after_provenance_failure( from polylogue.storage.index_generation import IndexGenerationStore try: - store = IndexGenerationStore.for_archive_root(root) + store = IndexGenerationStore.for_archive_root(root, repair_anchor=False) transaction = store.load_transaction(operation_id) transaction = _reconcile_active_generation_transaction(store, transaction) except Exception as load_error: diff --git a/tests/unit/maintenance/test_rebuild_index_provenance_gate.py b/tests/unit/maintenance/test_rebuild_index_provenance_gate.py index 564b8bd941..b1515128fd 100644 --- a/tests/unit/maintenance/test_rebuild_index_provenance_gate.py +++ b/tests/unit/maintenance/test_rebuild_index_provenance_gate.py @@ -406,6 +406,48 @@ def test_invalid_receipt_preserves_provenance_error_when_recovery_metadata_is_re ) +@pytest.mark.parametrize("anchor_state", ["missing", "poisoned"]) +def test_invalid_resume_marks_transaction_stale_without_repairing_active_anchor( + tmp_path: Path, anchor_state: str +) -> None: + """Stale-retirement bookkeeping cannot mutate active-pointer authority.""" + root = tmp_path / "archive" + _seed(root, count=1) + receipt_path = write_valid_rebuild_receipt(root, tmp_path / "receipt.json") + store = IndexGenerationStore.for_archive_root(root) + transaction = store.create_transaction( + source_snapshot=rebuild_source_evidence_snapshot(root), + operation_id=f"stale-with-{anchor_state}-anchor", + ) + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + receipt["generated_at"] = "2000-01-01T00:00:00Z" + receipt_path.write_text(json.dumps(receipt), encoding="utf-8") + + anchor = root / ".index-active-pointer" + if anchor_state == "missing": + anchor.unlink() + expected_anchor: bytes | None = None + else: + poisoned_path = Path(store.load(transaction.generation_id).index_path) + anchor.write_text(str(poisoned_path), encoding="utf-8") + expected_anchor = anchor.read_bytes() + + with pytest.raises(rebuild_index_module.RebuildProvenanceError, match="schema-inference preflight gate failed"): + rebuild_index_from_source_sync( + RebuildIndexRequest( + archive_root=root, + schema_inference_receipt_path=receipt_path, + operation_id=transaction.operation_id, + ) + ) + + assert (anchor.read_bytes() if anchor.exists() else None) == expected_anchor + checkpoint = IndexGenerationStore.for_archive_root(root, repair_anchor=False).load_transaction( + transaction.operation_id + ) + assert checkpoint.status == "stale" + + def test_resume_revalidates_external_mapping_before_more_replay(tmp_path: Path) -> None: root = tmp_path / "archive" _seed(root, count=2) From 3a2116d0e7bcd98c0c611805ef06c4df8d1848c6 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sat, 8 Aug 2026 12:41:33 +0200 Subject: [PATCH 10/19] fix(reindex): bind candidate census identity Inactive admission trusted a parser-census status without comparing its persisted logical-key identity to a fresh current-parser derivation. Candidate transaction setup also repaired missing or poisoned active-pointer anchors before any promotion boundary. Decode and normalize every current parser census key set, compare parseable raws against fresh parser identity, and construct candidate admission stores without pointer repair. Promotion-authorized offline requests retain their explicit repair authority. Ref polylogue-inactive-candidate-durable-barrier. --- polylogue/daemon/bulk_rebuild.py | 4 +- polylogue/maintenance/rebuild_index.py | 4 +- polylogue/sources/revision_backfill.py | 56 ++++++++++----- ...test_inactive_candidate_durable_barrier.py | 69 +++++++++++++++++++ 4 files changed, 112 insertions(+), 21 deletions(-) diff --git a/polylogue/daemon/bulk_rebuild.py b/polylogue/daemon/bulk_rebuild.py index 0b37d8baec..a573e8aa61 100644 --- a/polylogue/daemon/bulk_rebuild.py +++ b/polylogue/daemon/bulk_rebuild.py @@ -212,7 +212,7 @@ def resolve_or_start_daemon_bulk_rebuild_transaction( # external-corpus drift cannot reach generation bookkeeping. _validate_rebuild_provenance_receipt(root, schema_inference_receipt_path) validate_rebuild_source_admission(root, location) - store = IndexGenerationStore(location) + store = IndexGenerationStore(location, repair_anchor=False) transaction: IndexRebuildTransaction | None try: transaction = store.load_transaction(DAEMON_BULK_REBUILD_OPERATION_ID) @@ -381,7 +381,7 @@ async def run_daemon_bulk_rebuild_pass( # Recheck before consuming the receipt or selecting source material. await asyncio.to_thread(require_rebuild_schema_currency, root) await asyncio.to_thread(_validate_rebuild_provenance_receipt, root, receipt_path) - store = IndexGenerationStore(location) + store = IndexGenerationStore(location, repair_anchor=False) await asyncio.to_thread(_validate_rebuild_provenance_receipt, root, receipt_path) page = await asyncio.to_thread(store.next_raw_page, transaction, limit=batch_size) finally: diff --git a/polylogue/maintenance/rebuild_index.py b/polylogue/maintenance/rebuild_index.py index c52ba86a42..b00be63464 100644 --- a/polylogue/maintenance/rebuild_index.py +++ b/polylogue/maintenance/rebuild_index.py @@ -1476,7 +1476,7 @@ async def _rebuild_index_from_source_owned( root, active_index_path=canonical_active_index_path(owned.location), ) - generation_store = IndexGenerationStore(owned.location) + generation_store = IndexGenerationStore(owned.location, repair_anchor=request.promote) if request.operation_id is not None: transaction = generation_store.load_transaction(request.operation_id) transaction = _reconcile_active_generation_transaction(generation_store, transaction) @@ -1573,7 +1573,7 @@ async def _rebuild_index_from_source_owned( active_index_path=canonical_active_index_path(owned.location), selected_raw_ids=selected_raw_ids, ) - generation_store = IndexGenerationStore(owned.location) + generation_store = IndexGenerationStore(owned.location, repair_anchor=request.promote) provenance.validate() generation = generation_store.create(source_snapshot=rebuild_source_evidence_snapshot(root)) try: diff --git a/polylogue/sources/revision_backfill.py b/polylogue/sources/revision_backfill.py index 52666fa217..f74d2b2321 100644 --- a/polylogue/sources/revision_backfill.py +++ b/polylogue/sources/revision_backfill.py @@ -818,7 +818,7 @@ def _load_frozen_revision_evidence( ) -> _RevisionCensusState: """Parse a phase-2 source snapshot without changing its durable ledger.""" expanded_raw_ids, _logical_keys = archive.expand_raw_membership_selection(selected_raw_ids) - require_current_parser_source_census( + recorded_logical_keys = require_current_parser_source_census( archive.archive_root, selected_raw_ids=expanded_raw_ids if selected_raw_ids is not None else None, ) @@ -851,6 +851,14 @@ def _load_frozen_revision_evidence( f"inactive candidate could not parse frozen raw {raw_id}: {type(outcome).__name__}: {outcome}" ) from outcome sessions, payload_bytes, revision_kind = outcome + parsed_logical_keys = tuple( + sorted({f"{session.source_name.value}:{session.provider_session_id}" for session in sessions}) + ) + if recorded_logical_keys[raw_id] != parsed_logical_keys: + raise FrozenSourceRemediationRequiredError( + "inactive candidate re-derived different current-parser logical keys for frozen raw " + f"{raw_id}: recorded={recorded_logical_keys[raw_id]!r}, parsed={parsed_logical_keys!r}" + ) spill.add(raw_id, sessions, payload_bytes=payload_bytes) state.classified += int(len(sessions) == 1) if revision_kind is RawRevisionKind.UNKNOWN: @@ -864,9 +872,10 @@ def require_current_parser_source_census( archive_root: Path, *, selected_raw_ids: Sequence[str] | None = None, -) -> None: +) -> dict[str, tuple[str, ...]]: """Require phase-2 parser receipts before allocating an index candidate.""" stale_raw_ids: list[str] = [] + recorded_logical_keys: dict[str, tuple[str, ...]] = {} selections: tuple[tuple[str, ...] | None, ...] if selected_raw_ids is None: selections = (None,) @@ -878,22 +887,34 @@ def require_current_parser_source_census( for selection in selections: where = "" if selection is None else f"WHERE r.raw_id IN ({','.join('?' for _ in selection)})" params: tuple[object, ...] = () if selection is None else selection - stale_raw_ids.extend( - str(row[0]) - for row in source_conn.execute( - f""" - SELECT r.raw_id - FROM raw_sessions AS r - LEFT JOIN raw_authority_parser_census AS c ON c.raw_id = r.raw_id - {where} - {"WHERE" if selection is None else "AND"} NOT COALESCE( - c.parser_fingerprint = ? AND c.status = 'complete', 0 - ) - ORDER BY r.raw_id - """, - (*params, RAW_AUTHORITY_PARSER_FINGERPRINT), - ) + rows = source_conn.execute( + f""" + SELECT r.raw_id, c.parser_fingerprint, c.status, c.logical_keys_json + FROM raw_sessions AS r + LEFT JOIN raw_authority_parser_census AS c ON c.raw_id = r.raw_id + {where} + ORDER BY r.raw_id + """, + params, ) + for raw_id_value, fingerprint, status, logical_keys_json in rows: + raw_id = str(raw_id_value) + if fingerprint != RAW_AUTHORITY_PARSER_FINGERPRINT or status != "complete": + stale_raw_ids.append(raw_id) + continue + try: + decoded_keys = json.loads(str(logical_keys_json)) + except (TypeError, ValueError, json.JSONDecodeError): + stale_raw_ids.append(raw_id) + continue + if not isinstance(decoded_keys, list) or not all(isinstance(value, str) for value in decoded_keys): + stale_raw_ids.append(raw_id) + continue + normalized_keys = tuple(sorted(set(decoded_keys))) + if tuple(decoded_keys) != normalized_keys: + stale_raw_ids.append(raw_id) + continue + recorded_logical_keys[raw_id] = normalized_keys if stale_raw_ids: sample = ", ".join(stale_raw_ids[:5]) raise FrozenSourceRemediationRequiredError( @@ -932,6 +953,7 @@ def require_current_parser_source_census( "inactive candidate requires complete frozen source authority; " f"{len(unresolved_raw_ids)} raw(s) remain quarantined or undecided (sample: {sample})" ) + return recorded_logical_keys def validate_frozen_source_authority( diff --git a/tests/unit/maintenance/test_inactive_candidate_durable_barrier.py b/tests/unit/maintenance/test_inactive_candidate_durable_barrier.py index a59b2727c6..1c383783cd 100644 --- a/tests/unit/maintenance/test_inactive_candidate_durable_barrier.py +++ b/tests/unit/maintenance/test_inactive_candidate_durable_barrier.py @@ -316,6 +316,75 @@ def test_daemon_candidate_requires_source_admission_before_transaction_allocatio _assert_no_candidate_bookkeeping(root) +@pytest.mark.parametrize("route", ["offline", "daemon"]) +@pytest.mark.parametrize("anchor_state", ["missing", "poisoned"]) +def test_valid_candidate_admission_does_not_repair_active_pointer_anchor( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + route: str, + anchor_state: str, +) -> None: + root = tmp_path / "archive" + receipt_path = _prepare_frozen_source(root, monkeypatch) + anchor = root / ".index-active-pointer" + if anchor.exists() or anchor.is_symlink(): + anchor.unlink() + if anchor_state == "poisoned": + anchor.write_text( + str(root / ".index-generations" / "gen-poisoned" / "index.db"), + encoding="utf-8", + ) + anchor_before = _optional_path_evidence(anchor) + + if route == "offline": + result = rebuild_index_from_source_sync( + RebuildIndexRequest( + archive_root=root, + schema_inference_receipt_path=receipt_path, + promote=False, + ) + ) + assert result.transaction is not None + assert result.transaction["status"] == "ready" + else: + transaction = resolve_or_start_daemon_bulk_rebuild_transaction( + root, + schema_inference_receipt_path=receipt_path, + ) + assert transaction.status == "running" + + assert _optional_path_evidence(anchor) == anchor_before + + +def test_candidate_rejects_poisoned_current_parser_logical_keys_before_allocation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = tmp_path / "archive" + _prepare_frozen_source(root, monkeypatch) + with sqlite3.connect(root / "source.db") as source: + source.execute( + "UPDATE raw_authority_parser_census SET logical_keys_json = ?", + (json.dumps(["codex-session:poisoned-census-key"]),), + ) + source.commit() + receipt_path = write_valid_rebuild_receipt(root, root.parent / "poisoned-census-receipt.json") + anchor = root / ".index-active-pointer" + anchor_before = _optional_path_evidence(anchor) + + with pytest.raises(FrozenSourceRemediationRequiredError, match="re-derived different current-parser logical keys"): + rebuild_index_from_source_sync( + RebuildIndexRequest( + archive_root=root, + schema_inference_receipt_path=receipt_path, + promote=False, + ) + ) + + assert _optional_path_evidence(anchor) == anchor_before + _assert_no_candidate_bookkeeping(root) + + def test_candidate_requires_complete_source_authority_before_generation_readiness( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, From 86b802bc289446445fdd5997cdee537c918e87d7 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sat, 8 Aug 2026 12:46:29 +0200 Subject: [PATCH 11/19] fix(reindex): normalize census identity vocabulary Frozen authority receipts can contain either transitional Provider prefixes or public Origin prefixes depending on the acquisition route. Comparing either raw spelling to freshly parsed public identity rejected valid Codex and ChatGPT archives. Normalize both persisted aliases to Origin while preserving the native id, reject unknown or collapsing key sets, and compare that canonical identity to the current parser result. Ref polylogue-inactive-candidate-durable-barrier. --- polylogue/sources/revision_backfill.py | 35 +++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/polylogue/sources/revision_backfill.py b/polylogue/sources/revision_backfill.py index f74d2b2321..04c02e1e24 100644 --- a/polylogue/sources/revision_backfill.py +++ b/polylogue/sources/revision_backfill.py @@ -36,7 +36,7 @@ ) from polylogue.archive.session_revision_membership import MembershipRevision, classify_membership_revisions from polylogue.core.enums import Origin, Provider -from polylogue.core.sources import provider_from_origin +from polylogue.core.sources import origin_from_provider, provider_from_origin from polylogue.pipeline.ids import session_revision_projection from polylogue.pipeline.parsed_tree_size import effective_physical_memory_bytes, estimate_parsed_tree_bytes from polylogue.pipeline.services.process_pool import ( @@ -66,6 +66,21 @@ _LOGGER = _polylogue_logging.get_logger(__name__) +def _canonical_authority_logical_key(logical_key: str) -> str: + """Normalize transitional provider and public-origin authority prefixes.""" + prefix, separator, native_id = logical_key.partition(":") + if not separator or not native_id: + raise ValueError(f"invalid logical source key: {logical_key!r}") + try: + origin = Origin(prefix) + except ValueError: + try: + origin = origin_from_provider(Provider(prefix)) + except ValueError as exc: + raise ValueError(f"unknown logical source key prefix: {prefix!r}") from exc + return f"{origin.value}:{native_id}" + + def _browser_snapshot_fidelity(ingest_flags: Sequence[str]) -> Literal["dom", "native"] | None: """Derive membership-classification browser fidelity from parser ingest flags. @@ -852,7 +867,12 @@ def _load_frozen_revision_evidence( ) from outcome sessions, payload_bytes, revision_kind = outcome parsed_logical_keys = tuple( - sorted({f"{session.source_name.value}:{session.provider_session_id}" for session in sessions}) + sorted( + { + f"{origin_from_provider(session.source_name).value}:{session.provider_session_id}" + for session in sessions + } + ) ) if recorded_logical_keys[raw_id] != parsed_logical_keys: raise FrozenSourceRemediationRequiredError( @@ -910,8 +930,15 @@ def require_current_parser_source_census( if not isinstance(decoded_keys, list) or not all(isinstance(value, str) for value in decoded_keys): stale_raw_ids.append(raw_id) continue - normalized_keys = tuple(sorted(set(decoded_keys))) - if tuple(decoded_keys) != normalized_keys: + if tuple(decoded_keys) != tuple(sorted(set(decoded_keys))): + stale_raw_ids.append(raw_id) + continue + try: + normalized_keys = tuple(sorted({_canonical_authority_logical_key(value) for value in decoded_keys})) + except ValueError: + stale_raw_ids.append(raw_id) + continue + if len(normalized_keys) != len(decoded_keys): stale_raw_ids.append(raw_id) continue recorded_logical_keys[raw_id] = normalized_keys From 3124cb6b269ee7855edb991d8ab82f8e45aa6640 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sat, 8 Aug 2026 12:53:29 +0200 Subject: [PATCH 12/19] fix(reindex): require exact durable authority keys Parser receipts were compared to fresh parses only for standalone payloads. Extra durable membership bindings and typed continuation rows could therefore carry identity outside that comparison and survive frozen-source admission. Require each canonical census key set to equal the durable typed and membership authority bindings before replay. Add real-route regressions for an extra membership, a poisoned typed append receipt, and jointly poisoned durable plus parser identity. Ref polylogue-inactive-candidate-durable-barrier. --- polylogue/sources/revision_backfill.py | 47 ++++++++++ ...test_inactive_candidate_durable_barrier.py | 94 +++++++++++++++++++ 2 files changed, 141 insertions(+) diff --git a/polylogue/sources/revision_backfill.py b/polylogue/sources/revision_backfill.py index 04c02e1e24..1fed739baf 100644 --- a/polylogue/sources/revision_backfill.py +++ b/polylogue/sources/revision_backfill.py @@ -949,6 +949,53 @@ def require_current_parser_source_census( f"{len(stale_raw_ids)} raw(s) are stale or incomplete (sample: {sample})" ) + durable_logical_keys: dict[str, set[str]] = {raw_id: set() for raw_id in recorded_logical_keys} + invalid_durable_bindings: set[str] = set() + with sqlite3.connect(f"file:{archive_root / 'source.db'}?mode=ro", uri=True) as source_conn: + for selection in selections: + where = "" if selection is None else f"WHERE r.raw_id IN ({','.join('?' for _ in selection)})" + params = () if selection is None else selection + rows = source_conn.execute( + f""" + SELECT r.raw_id, r.logical_source_key, r.revision_kind, m.logical_source_key + FROM raw_sessions AS r + LEFT JOIN raw_session_memberships AS m ON m.raw_id = r.raw_id + {where} + ORDER BY r.raw_id, m.logical_source_key + """, + params, + ) + for raw_id_value, typed_key, revision_kind, membership_key in rows: + raw_id = str(raw_id_value) + persisted_keys = durable_logical_keys.setdefault(raw_id, set()) + raw_keys = [ + value + for value in ( + typed_key if typed_key is not None and revision_kind != RawRevisionKind.UNKNOWN.value else None, + membership_key, + ) + if value is not None + ] + try: + persisted_keys.update(_canonical_authority_logical_key(str(value)) for value in raw_keys) + except ValueError: + invalid_durable_bindings.add(raw_id) + + authority_binding_drift = sorted( + invalid_durable_bindings + | { + raw_id + for raw_id, census_keys in recorded_logical_keys.items() + if tuple(sorted(durable_logical_keys.get(raw_id, ()))) != census_keys + } + ) + if authority_binding_drift: + sample = ", ".join(authority_binding_drift[:5]) + raise FrozenSourceRemediationRequiredError( + "inactive candidate current-parser census differs from frozen durable authority bindings; " + f"{len(authority_binding_drift)} raw(s) require source remediation (sample: {sample})" + ) + unresolved_raw_ids: list[str] = [] with sqlite3.connect(f"file:{archive_root / 'source.db'}?mode=ro", uri=True) as source_conn: for selection in selections: diff --git a/tests/unit/maintenance/test_inactive_candidate_durable_barrier.py b/tests/unit/maintenance/test_inactive_candidate_durable_barrier.py index 1c383783cd..59a0f6807e 100644 --- a/tests/unit/maintenance/test_inactive_candidate_durable_barrier.py +++ b/tests/unit/maintenance/test_inactive_candidate_durable_barrier.py @@ -10,6 +10,7 @@ import pytest +from polylogue.archive.revision_authority import RawRevisionAuthority, RawRevisionEnvelope, RawRevisionKind from polylogue.core.enums import Provider from polylogue.daemon.bulk_rebuild import resolve_or_start_daemon_bulk_rebuild_transaction from polylogue.maintenance.rebuild_index import RebuildIndexRequest, rebuild_index_from_source_sync @@ -363,6 +364,10 @@ def test_candidate_rejects_poisoned_current_parser_logical_keys_before_allocatio root = tmp_path / "archive" _prepare_frozen_source(root, monkeypatch) with sqlite3.connect(root / "source.db") as source: + source.execute( + "UPDATE raw_sessions SET logical_source_key = ?", + ("codex-session:poisoned-census-key",), + ) source.execute( "UPDATE raw_authority_parser_census SET logical_keys_json = ?", (json.dumps(["codex-session:poisoned-census-key"]),), @@ -385,6 +390,95 @@ def test_candidate_rejects_poisoned_current_parser_logical_keys_before_allocatio _assert_no_candidate_bookkeeping(root) +def test_candidate_rejects_extra_membership_binding_before_allocation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = tmp_path / "archive" + _prepare_frozen_source(root, monkeypatch) + with sqlite3.connect(root / "source.db") as source: + raw_id = str(source.execute("SELECT raw_id FROM raw_sessions").fetchone()[0]) + source.execute( + """ + INSERT INTO raw_session_memberships ( + raw_id, logical_source_key, provider_session_id, source_revision, + normalized_content_hash, message_count, acquisition_generation, + revision_authority, decision, decided_at_ms + ) VALUES (?, 'codex-session:stale-extra', 'stale-extra', ?, ?, 0, 0, + 'byte_proven', 'applied', 0) + """, + (raw_id, raw_id, b"\x00" * 32), + ) + source.commit() + receipt_path = write_valid_rebuild_receipt(root, root.parent / "extra-membership-receipt.json") + + with pytest.raises(FrozenSourceRemediationRequiredError, match="frozen durable authority bindings"): + rebuild_index_from_source_sync( + RebuildIndexRequest( + archive_root=root, + schema_inference_receipt_path=receipt_path, + promote=False, + ) + ) + + _assert_no_candidate_bookkeeping(root) + + +def test_candidate_rejects_poisoned_typed_append_census_before_allocation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = tmp_path / "archive" + _prepare_frozen_source(root, monkeypatch) + with sqlite3.connect(root / "source.db") as source: + baseline_raw_id, logical_key = source.execute("SELECT raw_id, logical_source_key FROM raw_sessions").fetchone() + append_payload = b'{"type":"response_item","payload":{"type":"message","id":"append"}}\n' + with ArchiveStore.open_existing(root, read_only=False) as archive: + append_raw_id = archive.write_raw_payload( + provider=Provider.CODEX, + payload=append_payload, + source_path="current/append.jsonl", + source_index=-1, + acquired_at_ms=2, + revision=RawRevisionEnvelope( + logical_source_key=str(logical_key), + kind=RawRevisionKind.APPEND, + source_revision="append-revision", + predecessor_source_revision=str(baseline_raw_id), + predecessor_raw_id=str(baseline_raw_id), + baseline_raw_id=str(baseline_raw_id), + acquisition_generation=1, + append_start_offset=0, + append_end_offset=len(append_payload), + authority=RawRevisionAuthority.BYTE_PROVEN, + ), + ) + with sqlite3.connect(root / "source.db") as source: + source.execute( + """ + INSERT INTO raw_authority_parser_census ( + raw_id, parser_fingerprint, status, logical_keys_json, detail, censused_at_ms + ) + SELECT ?, parser_fingerprint, 'complete', ?, 'poisoned typed append census', 0 + FROM raw_authority_parser_census LIMIT 1 + """, + (append_raw_id, json.dumps(["codex-session:poisoned-append-key"])), + ) + source.commit() + receipt_path = write_valid_rebuild_receipt(root, root.parent / "poisoned-append-receipt.json") + + with pytest.raises(FrozenSourceRemediationRequiredError, match="frozen durable authority bindings"): + rebuild_index_from_source_sync( + RebuildIndexRequest( + archive_root=root, + schema_inference_receipt_path=receipt_path, + promote=False, + ) + ) + + _assert_no_candidate_bookkeeping(root) + + def test_candidate_requires_complete_source_authority_before_generation_readiness( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, From 8622b2f0ebdd64a2f66c0cf72a9f5652e7433f23 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sat, 8 Aug 2026 13:04:48 +0200 Subject: [PATCH 13/19] fix(reindex): bind append and canary cleanup authority Typed append rows cannot be parsed independently, so jointly poisoned durable and census keys could evade fresh full-row identity checks. Canary failure cleanup also constructed a generation store with pointer repair enabled. Require each append key to match its linked predecessor and baseline authority, whose full rows remain fresh-parser validated. Keep canary discard bookkeeping from repairing missing or poisoned active anchors. Ref polylogue-inactive-candidate-durable-barrier. --- polylogue/maintenance/reindex_canary.py | 2 +- polylogue/sources/revision_backfill.py | 36 +++++++++++++++++++ ...test_inactive_candidate_durable_barrier.py | 6 +++- tests/unit/maintenance/test_reindex_canary.py | 14 +++++++- 4 files changed, 55 insertions(+), 3 deletions(-) diff --git a/polylogue/maintenance/reindex_canary.py b/polylogue/maintenance/reindex_canary.py index 8526f07be9..0db2b885df 100644 --- a/polylogue/maintenance/reindex_canary.py +++ b/polylogue/maintenance/reindex_canary.py @@ -388,7 +388,7 @@ def _discard_canary_candidate(archive_root: Path, receipt: object) -> list[BaseE generation_id = generation.get("generation_id") if not isinstance(generation_id, str) or not generation_id: return [RuntimeError("canary receipt candidate generation id is missing")] - store = IndexGenerationStore(ArchiveLocation.resolve(archive_root)) + store = IndexGenerationStore(ArchiveLocation.resolve(archive_root), repair_anchor=False) try: candidate = store.load(generation_id) except BaseException as exc: diff --git a/polylogue/sources/revision_backfill.py b/polylogue/sources/revision_backfill.py index 1fed739baf..9cb1c4d8cd 100644 --- a/polylogue/sources/revision_backfill.py +++ b/polylogue/sources/revision_backfill.py @@ -996,6 +996,42 @@ def require_current_parser_source_census( f"{len(authority_binding_drift)} raw(s) require source remediation (sample: {sample})" ) + append_identity_drift: set[str] = set() + with sqlite3.connect(f"file:{archive_root / 'source.db'}?mode=ro", uri=True) as source_conn: + for selection in selections: + where = "" if selection is None else f"AND a.raw_id IN ({','.join('?' for _ in selection)})" + params = () if selection is None else selection + rows = source_conn.execute( + f""" + SELECT a.raw_id, a.logical_source_key, + predecessor.logical_source_key, baseline.logical_source_key + FROM raw_sessions AS a + LEFT JOIN raw_sessions AS predecessor ON predecessor.raw_id = a.predecessor_raw_id + LEFT JOIN raw_sessions AS baseline ON baseline.raw_id = a.baseline_raw_id + WHERE a.revision_kind = 'append' {where} + ORDER BY a.raw_id + """, + params, + ) + for raw_id_value, append_key, predecessor_key, baseline_key in rows: + linked_keys = [value for value in (predecessor_key, baseline_key) if value is not None] + if append_key is None or not linked_keys: + continue + try: + canonical_append_key = _canonical_authority_logical_key(str(append_key)) + canonical_linked_keys = {_canonical_authority_logical_key(str(value)) for value in linked_keys} + except ValueError: + append_identity_drift.add(str(raw_id_value)) + continue + if canonical_linked_keys != {canonical_append_key}: + append_identity_drift.add(str(raw_id_value)) + if append_identity_drift: + sample = ", ".join(sorted(append_identity_drift)[:5]) + raise FrozenSourceRemediationRequiredError( + "inactive candidate typed continuation identity differs from linked byte authority; " + f"{len(append_identity_drift)} raw(s) require source remediation (sample: {sample})" + ) + unresolved_raw_ids: list[str] = [] with sqlite3.connect(f"file:{archive_root / 'source.db'}?mode=ro", uri=True) as source_conn: for selection in selections: diff --git a/tests/unit/maintenance/test_inactive_candidate_durable_barrier.py b/tests/unit/maintenance/test_inactive_candidate_durable_barrier.py index 59a0f6807e..b4e34fc24c 100644 --- a/tests/unit/maintenance/test_inactive_candidate_durable_barrier.py +++ b/tests/unit/maintenance/test_inactive_candidate_durable_barrier.py @@ -454,6 +454,10 @@ def test_candidate_rejects_poisoned_typed_append_census_before_allocation( ), ) with sqlite3.connect(root / "source.db") as source: + source.execute( + "UPDATE raw_sessions SET logical_source_key = ? WHERE raw_id = ?", + ("codex-session:poisoned-append-key", append_raw_id), + ) source.execute( """ INSERT INTO raw_authority_parser_census ( @@ -467,7 +471,7 @@ def test_candidate_rejects_poisoned_typed_append_census_before_allocation( source.commit() receipt_path = write_valid_rebuild_receipt(root, root.parent / "poisoned-append-receipt.json") - with pytest.raises(FrozenSourceRemediationRequiredError, match="frozen durable authority bindings"): + with pytest.raises(FrozenSourceRemediationRequiredError, match="typed continuation identity"): rebuild_index_from_source_sync( RebuildIndexRequest( archive_root=root, diff --git a/tests/unit/maintenance/test_reindex_canary.py b/tests/unit/maintenance/test_reindex_canary.py index 7c255e9ca3..f7bebf583c 100644 --- a/tests/unit/maintenance/test_reindex_canary.py +++ b/tests/unit/maintenance/test_reindex_canary.py @@ -617,15 +617,26 @@ def test_run_reindex_canary_rejects_missing_receipt_even_with_ambient_valid_rece run_reindex_canary(root, schema_inference_receipt_path=None, sessions_per_origin=1, no_promote=True) +@pytest.mark.parametrize("anchor_state", ["missing", "poisoned"]) def test_run_reindex_canary_cleans_candidate_after_comparison_failure( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, anchor_state: str ) -> None: """A post-rebuild canary failure cannot strand its inactive candidate.""" root = tmp_path / "archive" _prepare_candidate_ready_archive(root) receipt_path = _write_candidate_receipt(root, tmp_path / "receipt.json") + anchor = root / ".index-active-pointer" + expected_anchor: bytes | None = None def fail_compare(*args: object, **kwargs: object) -> CanaryDiffReport: + nonlocal expected_anchor + del kwargs + if anchor.exists() or anchor.is_symlink(): + anchor.unlink() + if anchor_state == "poisoned": + candidate_path = Path(str(args[1])) + anchor.write_text(str(candidate_path), encoding="utf-8") + expected_anchor = anchor.read_bytes() raise RuntimeError("synthetic canary comparison failure") monkeypatch.setattr(reindex_canary_module, "compare_reindex_generations", fail_compare) @@ -639,6 +650,7 @@ def fail_compare(*args: object, **kwargs: object) -> CanaryDiffReport: ) assert not list((root / ".index-generations").glob("gen-*")) + assert (anchor.read_bytes() if anchor.exists() else None) == expected_anchor def test_run_reindex_canary_clean_success_retains_a_valid_inactive_candidate(tmp_path: Path) -> None: From 92a9d4ac3a62e81a01d2ad832acd1060d5657c02 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sat, 8 Aug 2026 13:16:15 +0200 Subject: [PATCH 14/19] fix(reindex): anchor append identity to full authority A self-linked or incomplete APPEND row could make its poisoned durable and census identity agree without reaching any independently parseable authority. String equality alone could not establish that the continuation belonged to a real full source. Expand selected validation through revision links, require byte-proven acyclic predecessor chains that terminate at a distinct parseable FULL baseline, and enforce one canonical identity across the chain. Ref polylogue-inactive-candidate-durable-barrier. --- polylogue/sources/revision_backfill.py | 131 +++++++++++++++--- ...test_inactive_candidate_durable_barrier.py | 7 + 2 files changed, 117 insertions(+), 21 deletions(-) diff --git a/polylogue/sources/revision_backfill.py b/polylogue/sources/revision_backfill.py index 9cb1c4d8cd..83041cc6cf 100644 --- a/polylogue/sources/revision_backfill.py +++ b/polylogue/sources/revision_backfill.py @@ -81,6 +81,32 @@ def _canonical_authority_logical_key(logical_key: str) -> str: return f"{origin.value}:{native_id}" +def _expand_frozen_revision_link_selection(archive_root: Path, raw_ids: Sequence[str]) -> tuple[str, ...]: + """Include every predecessor and baseline needed to validate selected APPEND authority.""" + expanded = set(raw_ids) + pending = set(raw_ids) + with sqlite3.connect(f"file:{archive_root / 'source.db'}?mode=ro", uri=True) as source_conn: + while pending: + current = tuple(sorted(pending)) + pending.clear() + for offset in range(0, len(current), 500): + chunk = current[offset : offset + 500] + placeholders = ",".join("?" for _ in chunk) + rows = source_conn.execute( + f""" + SELECT predecessor_raw_id, baseline_raw_id + FROM raw_sessions WHERE raw_id IN ({placeholders}) + """, + chunk, + ) + for predecessor_raw_id, baseline_raw_id in rows: + for linked_raw_id in (predecessor_raw_id, baseline_raw_id): + if linked_raw_id is not None and str(linked_raw_id) not in expanded: + expanded.add(str(linked_raw_id)) + pending.add(str(linked_raw_id)) + return tuple(sorted(expanded)) + + def _browser_snapshot_fidelity(ingest_flags: Sequence[str]) -> Literal["dom", "native"] | None: """Derive membership-classification browser fidelity from parser ingest flags. @@ -833,6 +859,8 @@ def _load_frozen_revision_evidence( ) -> _RevisionCensusState: """Parse a phase-2 source snapshot without changing its durable ledger.""" expanded_raw_ids, _logical_keys = archive.expand_raw_membership_selection(selected_raw_ids) + if selected_raw_ids is not None: + expanded_raw_ids = _expand_frozen_revision_link_selection(archive.archive_root, expanded_raw_ids) recorded_logical_keys = require_current_parser_source_census( archive.archive_root, selected_raw_ids=expanded_raw_ids if selected_raw_ids is not None else None, @@ -996,35 +1024,96 @@ def require_current_parser_source_census( f"{len(authority_binding_drift)} raw(s) require source remediation (sample: {sample})" ) - append_identity_drift: set[str] = set() + authority_rows: dict[str, tuple[str | None, str, str, int, str | None, str | None]] = {} with sqlite3.connect(f"file:{archive_root / 'source.db'}?mode=ro", uri=True) as source_conn: for selection in selections: - where = "" if selection is None else f"AND a.raw_id IN ({','.join('?' for _ in selection)})" + where = "" if selection is None else f"WHERE raw_id IN ({','.join('?' for _ in selection)})" params = () if selection is None else selection rows = source_conn.execute( f""" - SELECT a.raw_id, a.logical_source_key, - predecessor.logical_source_key, baseline.logical_source_key - FROM raw_sessions AS a - LEFT JOIN raw_sessions AS predecessor ON predecessor.raw_id = a.predecessor_raw_id - LEFT JOIN raw_sessions AS baseline ON baseline.raw_id = a.baseline_raw_id - WHERE a.revision_kind = 'append' {where} - ORDER BY a.raw_id + SELECT raw_id, logical_source_key, revision_kind, revision_authority, + source_index, predecessor_raw_id, baseline_raw_id + FROM raw_sessions {where} + ORDER BY raw_id """, params, ) - for raw_id_value, append_key, predecessor_key, baseline_key in rows: - linked_keys = [value for value in (predecessor_key, baseline_key) if value is not None] - if append_key is None or not linked_keys: - continue - try: - canonical_append_key = _canonical_authority_logical_key(str(append_key)) - canonical_linked_keys = {_canonical_authority_logical_key(str(value)) for value in linked_keys} - except ValueError: - append_identity_drift.add(str(raw_id_value)) - continue - if canonical_linked_keys != {canonical_append_key}: - append_identity_drift.add(str(raw_id_value)) + for raw_id_value, logical_key, revision_kind, authority, source_index, predecessor_id, baseline_id in rows: + authority_rows[str(raw_id_value)] = ( + str(logical_key) if logical_key is not None else None, + str(revision_kind), + str(authority), + int(source_index), + str(predecessor_id) if predecessor_id is not None else None, + str(baseline_id) if baseline_id is not None else None, + ) + + append_identity_drift: set[str] = set() + for raw_id, ( + append_key, + revision_kind, + authority, + _source_index, + predecessor_id, + baseline_id, + ) in authority_rows.items(): + if revision_kind != RawRevisionKind.APPEND.value: + continue + predecessor = authority_rows.get(predecessor_id or "") + baseline = authority_rows.get(baseline_id or "") + if ( + append_key is None + or authority != RawRevisionAuthority.BYTE_PROVEN.value + or predecessor_id is None + or baseline_id is None + or predecessor_id == raw_id + or baseline_id == raw_id + or predecessor is None + or baseline is None + or predecessor[2] != RawRevisionAuthority.BYTE_PROVEN.value + or baseline[1] != RawRevisionKind.FULL.value + or baseline[2] != RawRevisionAuthority.BYTE_PROVEN.value + or baseline[3] < 0 + ): + append_identity_drift.add(raw_id) + continue + try: + canonical_append_key = _canonical_authority_logical_key(append_key) + canonical_predecessor_key = _canonical_authority_logical_key(predecessor[0] or "") + canonical_baseline_key = _canonical_authority_logical_key(baseline[0] or "") + except ValueError: + append_identity_drift.add(raw_id) + continue + if {canonical_predecessor_key, canonical_baseline_key} != {canonical_append_key}: + append_identity_drift.add(raw_id) + continue + + seen = {raw_id} + cursor_id = predecessor_id + while True: + if cursor_id in seen: + append_identity_drift.add(raw_id) + break + seen.add(cursor_id) + cursor = authority_rows.get(cursor_id) + if cursor is None: + append_identity_drift.add(raw_id) + break + if cursor[1] == RawRevisionKind.FULL.value: + if cursor_id != baseline_id: + append_identity_drift.add(raw_id) + break + if cursor[1] != RawRevisionKind.APPEND.value or cursor[4] is None: + append_identity_drift.add(raw_id) + break + try: + if _canonical_authority_logical_key(cursor[0] or "") != canonical_append_key: + append_identity_drift.add(raw_id) + break + except ValueError: + append_identity_drift.add(raw_id) + break + cursor_id = cursor[4] if append_identity_drift: sample = ", ".join(sorted(append_identity_drift)[:5]) raise FrozenSourceRemediationRequiredError( diff --git a/tests/unit/maintenance/test_inactive_candidate_durable_barrier.py b/tests/unit/maintenance/test_inactive_candidate_durable_barrier.py index b4e34fc24c..2e5b29ebe3 100644 --- a/tests/unit/maintenance/test_inactive_candidate_durable_barrier.py +++ b/tests/unit/maintenance/test_inactive_candidate_durable_barrier.py @@ -424,9 +424,11 @@ def test_candidate_rejects_extra_membership_binding_before_allocation( _assert_no_candidate_bookkeeping(root) +@pytest.mark.parametrize("link_shape", ["linked", "self-linked"]) def test_candidate_rejects_poisoned_typed_append_census_before_allocation( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, + link_shape: str, ) -> None: root = tmp_path / "archive" _prepare_frozen_source(root, monkeypatch) @@ -458,6 +460,11 @@ def test_candidate_rejects_poisoned_typed_append_census_before_allocation( "UPDATE raw_sessions SET logical_source_key = ? WHERE raw_id = ?", ("codex-session:poisoned-append-key", append_raw_id), ) + if link_shape == "self-linked": + source.execute( + "UPDATE raw_sessions SET predecessor_raw_id = raw_id, baseline_raw_id = raw_id WHERE raw_id = ?", + (append_raw_id,), + ) source.execute( """ INSERT INTO raw_authority_parser_census ( From 6bf48c08eb9c2dd5052bae4a066222ee58db7612 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sat, 8 Aug 2026 13:23:36 +0200 Subject: [PATCH 15/19] chore(beads): close inactive candidate durable barrier Record the exact implementation head, focused verification, quick-gate receipt, and independent no-blockers verdict. Production candidate execution and downstream phases remain open under their existing Beads. --- .beads/issues.jsonl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 94bbb64a34..709dfc3386 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -1749,4 +1749,4 @@ {"_type":"issue","id":"polylogue-fs1.2","title":"Importer: NeMo Relay ATOF/ATIF runtime spans","description":"Import Hermes observer-layer trace exports as runtime span evidence: pre/post_api_request -> LLM request spans; pre/post_tool_call -> tool execution spans with duration/status; approval hooks -> high-risk decision points; subagent hooks -> delegation graph; error hooks -> retry/fallback taxonomy. ATIF import + enrichment beats inventing another trajectory format — respect Hermes's actual extension seams and make Polylogue the normalizer.","design":"VERIFY first: current NeMo Relay plugin output shape in the Hermes repo (ATOF JSONL / ATIF JSON exported from observer hooks). Ingest route: new artifact kinds in the taxonomy (archive/artifact_taxonomy/) + a spans parser under sources/parsers/, landing as ObservedEvents/actions attached to the session (join key: Hermes session id from the trace envelope -> sessions.native_id). Map: pre/post_api_request pair -> LLM request span (duration, model, provider, token fields if present); pre/post_tool_call -> tool execution span with duration/status (structural outcome — feeds is_error/exit_code lanes where present); approval hooks -> decision-point events; subagent lifecycle -> topology_edges (subagent type); error hooks -> retry/fallback taxonomy events. Spans without a matching archived session become explicit acquisition debt rows, not silent drops.","acceptance_criteria":"`polylogue-fs1.2` adds or updates an origin contract with detector, parser, raw fixture, normalized fixture, parser fingerprint, and fidelity/completeness notes. Ambiguous inputs are handled deterministically. The regression suite proves idempotent replay and visible degraded/missing-field behavior. Verification artifact: OriginSpec detector/parser/fixture/fidelity suite and content-hash export/import roundtrip.","notes":"[Delivery upgrade 2026-07-07T00:05:00Z] Release=K-interop-origin-export; lane=origin-interop-export; readiness=D-horizon-ready; proof=OriginSpec detector/parser/fixture/fidelity suite and content-hash export/import roundtrip. Original readiness=E-spec-needed.\n2026-07-10 Hermes contract refinement: ingest context_injected with profile/session/turn/snapshot-revision correlation; unpaired spans remain explicit acquisition debt. fs1.7 owns atomic spool/export production; this bead owns normalization and reconciliation.\nImplemented and PR opened (not merged): #2876 (feature/hermes/lifecycle-spool-and-bridge).\n\nScope understood: import Hermes observer-layer (NeMo Relay) trace exports as runtime span evidence, normalized and reconciled per the design's mapping (pre/post_api_request -> LLM request spans, pre/post_tool_call -> tool spans, approvals -> decision points, subagent hooks -> delegation evidence, error hooks -> retry/fallback taxonomy).\n\nHonesty constraint documented explicitly in code + PR: the real ATOF/ATIF wire shape was not independently verifiable from this workspace -- no local checkout of the Hermes observer-plugin source was available. sources/parsers/hermes_spans.py implements a documented, testable, best-effort marker-based schema derived from this bead's own design notes and the shared lifecycle taxonomy (hermes_lifecycle.py, fs1.7). Every fidelity capability the parser declares tops out at \"inferred\", never \"exact\", for this reason -- filed as a concrete follow-up (fs1.2.1, not yet created as a bead by me -- flagging here so the orchestrator can file it) to re-verify against real Hermes source and tighten fidelity if it matches without changing the public contract.\n\nWhat changed: sources/parsers/hermes_spans.py (detector/parser/fidelity), wired into the real dispatch pipeline (sources/dispatch.py: detect_provider, lowering, parse_payload -- same path every other origin uses, not a bespoke test-only entrypoint); new artifact-taxonomy classification (archive/artifact_taxonomy/runtime.py).\n\nAC checklist: origin contract with detector/parser/raw fixture/normalized fixture/parser fingerprint/fidelity notes -- satisfied (marker_payload() is the raw-fixture generator used by every test; normalized output is the ParsedSession/session_events produced; fidelity via import_fidelity_declaration()). Ambiguous inputs handled deterministically -- satisfied: unrecognized hook_type -> generic hermes_observer_span event (never dropped, never misclassified as a known kind); malformed span entries (missing hook_type/span_id, non-dict entries) are skipped and counted, not crashing. Idempotent replay -- satisfied and tested (test_atif_parse_is_idempotent_and_deterministic: same document parsed twice -> byte-identical structural output). Visible degraded/missing-field behavior -- satisfied: unpaired spans (start without finish) are counted and surfaced as an explicit degraded fidelity capability with a caveat, never silently dropped.\n\nDesign gap explicitly NOT closed, documented not silently assumed: physical merge of observer spans into the state-db-ingested conversational session's message tree (the design's \"landing as ObservedEvents/actions attached to the session\"). This parser instead produces its own observer-evidence session (observer:) with a read-side correlation helper (hermes_observer_session_id_for) joining by the shared raw Hermes session id -- a physical content-tree merge across two independently-acquired artifacts is a session-identity/lineage design decision (topology_edges/session_links) I judged out of scope for this pass rather than improvising a schema-adjacent change.\n\nVerification: devtools test tests/unit/sources/parsers/test_hermes_spans.py -- 9/9 passed (subset of PR's 43-test combined run). devtools verify --quick exit 0.\n[gpt-5.6-terra integration refinement, 2026-07-14]\n\nReal producer evidence now exists: the bundled NousResearch Hermes observability/nemo_relay plugin emits ATIF v1.7 session documents and append-only ATOF JSONL through actual session, LLM, tool, approval, and subagent callbacks. ATIF import is live. The remaining producer-to-archive gap is ATOF materialization, not schema speculation.\n\nRefine this bead implementation order: retain byte-identified ATOF raw evidence first; incremental reader checkpoints file identity plus byte offset; tolerate partial final lines and rotation/truncation; validate/order/deduplicate events; materialize normalized lifecycle/action evidence idempotently; retain parent/child subagent links; surface unpaired/unmatched records as debt. Never synthesize ATIF from ATOF or duplicate transcript bodies into events. Update OriginSpec fidelity only where real exported fixtures prove a field mapping.","status":"closed","priority":4,"issue_type":"feature","owner":"ezo.dev@gmail.com","created_at":"2026-07-03T04:31:39Z","created_by":"Sinity","updated_at":"2026-07-31T22:35:43Z","closed_at":"2026-07-20T21:34:47Z","close_reason":"Complete in substance across the merged chain — every item of the 2026-07-14 refined implementation order shipped: byte-identified ATOF raw retention + incremental byte-offset reader with partial-line/rotation tolerance (pre-existing append-plan mechanism, verified fs1.2.1 notes); validate/order/dedup + idempotent lifecycle/action materialization (#3103); shared-file multi-session correctness (#3113/flxh); parent/child subagent links from producer-positive marks, fail-closed (#3231); unpaired/unmatched as explicit debt (#3103). Identity composed with profile+artifact-family qualification (#3224/#3225). OriginSpec detector/parser/fixture/fidelity satisfied against REAL producer fixtures with marker-only payloads as negative tests (#3231, fs1.2.1 closed). Force rationale: remaining blocker edge 2qx.1.1 (shared OriginSpec admission kernel/conformance law) is a lane-gate shared suite per the delivery-ac-template-interpretation adjudication (2026-07-07) — not a per-bead requirement; the Hermes origin will conform when that kernel lands, tracked there.","metadata":{"authored_by":"gpt-5.6-terra","authored_on":"2026-07-14"},"labels":["area:ingest","area:substrate","delivery:K-interop-origin-export","delivery:ac-patched","lane:origin-interop-export"],"dependencies":[{"issue_id":"polylogue-fs1.2","depends_on_id":"polylogue-2qx.1.1","type":"blocks","created_at":"2026-07-15T20:55:32Z","created_by":"Sinity","metadata":"{}"},{"issue_id":"polylogue-fs1.2","depends_on_id":"polylogue-fs1","type":"parent-child","created_at":"2026-07-03T06:31:39Z","created_by":"Sinity","metadata":"{}"},{"issue_id":"polylogue-fs1.2","depends_on_id":"polylogue-fs1.2.1","type":"blocks","created_at":"2026-07-14T11:39:19Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":2,"dependent_count":2,"comment_count":0} {"_type":"issue","id":"polylogue-harness-clone-contracts","title":"test(infra): restore archive-clone and embedding-stat contracts","description":"A current-master comparison for polylogue-mn0si exposed two independent shared-harness regressions. Cloned empty archives retained a fresh-bootstrap receipt bound to the template root, and the embedding-stat contract mock ignored bound sqlite_master parameters. Repair both without weakening durable identity checks or changing production embedding semantics.","acceptance_criteria":"1. Cloning the shared empty archive rebinds the fresh durable-bootstrap receipt to the destination identity, opens through the real ArchiveStore route, and leaves the source receipt unchanged. 2. The embedding-stat contract fixture resolves sqlite_master probes from bound parameters and preserves the intended legacy fallback count behavior. 3. The seven focused regressions pass, formatting and lint pass, and a complete current-master full-suite ledger records unrelated residual failures without claiming them fixed. 4. No production durable-train safety check or embedding provider behavior is weakened.","status":"closed","priority":1,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-08-08T06:54:56Z","created_by":"Sinity","updated_at":"2026-08-08T07:16:43Z","closed_at":"2026-08-08T07:16:43Z","close_reason":"Implemented archive-clone destination rebinding and embedding-stat bound-parameter fixtures in PR #3883. The exact seven-test focused batch and quick gate passed; the complete current-master full-suite ledger recorded unrelated residual failures without claiming them fixed.","dependencies":[{"issue_id":"polylogue-harness-clone-contracts","depends_on_id":"polylogue-mn0si","type":"discovered-from","created_at":"2026-08-08T08:54:56Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-seeded-clone-bootstrap-rebind","title":"test(infra): rebind durable bootstrap on seeded archive clones","description":"The immutable seeded-workload archive cache is copied by tests/infra/workload_artifacts.py::clone_seeded_archive. Unlike the empty-archive fixture path repaired in PR #3883, this clone path preserves the source artifact's fresh durable-bootstrap marker byte-for-byte. Its source/user durable identities no longer match the destination files, so the first real ArchiveStore open fails closed with fresh durable bootstrap marker durable identity mismatch. Candidate/canary tests that begin from seeded workload clones therefore cannot exercise the production route.","design":"Give all private seeded-archive clones destination-bound fresh-bootstrap evidence immediately after the copy or reflink completes, using the production receipt writer already used by the empty-archive clone path. Keep the immutable source artifact and its marker unchanged. Add a focused regression that records source marker bytes, clones, verifies marker divergence and destination identity, then opens the clone through the real ArchiveStore route.","acceptance_criteria":"1. clone_seeded_archive replaces only the copied fresh durable-bootstrap marker with a destination-bound production marker after making the clone writable. 2. The immutable cached artifact marker remains byte-identical. 3. Reflink and copy fallback clones both use the same rebinding boundary. 4. A real ArchiveStore writer open succeeds on the clone and durable continuity checks remain fail-closed for an actually foreign marker. 5. Focused workload-artifact, temp-policy, and durable-train regressions plus devtools verify --quick pass.","notes":"Implemented destination-bound bootstrap rebinding in tests/infra/workload_artifacts.py for both reflink and copy fallback clones. Added real ArchiveStore open, immutable-source-marker preservation, and foreign-marker fail-closed regressions. Verification: 26 focused tests passed; mypy strict passed on both touched files; devtools verify --quick run 20260808T083029Z-quick-3360251-3cef0cf0 passed all 24 steps.","status":"closed","priority":1,"issue_type":"bug","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-08-08T08:20:27Z","created_by":"Sinity","updated_at":"2026-08-08T08:33:10Z","closed_at":"2026-08-08T08:33:10Z","close_reason":"Seeded archive clones now replace copied bootstrap evidence with a destination-bound production marker on both clone paths. Real opens pass, the immutable artifact is unchanged, a restored foreign marker still fails closed, 26 focused tests pass, and quick verification is green.","labels":["area:test-infra","lane:reindex"],"dependencies":[{"issue_id":"polylogue-seeded-clone-bootstrap-rebind","depends_on_id":"polylogue-harness-clone-contracts","type":"discovered-from","created_at":"2026-08-08T10:20:27Z","created_by":"Sinity","metadata":"{}"},{"issue_id":"polylogue-seeded-clone-bootstrap-rebind","depends_on_id":"polylogue-inactive-candidate-durable-barrier","type":"discovered-from","created_at":"2026-08-08T10:20:27Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"polylogue-inactive-candidate-durable-barrier","title":"fix: isolate inactive candidates from durable-tier writes","description":"The exact current-master rebuild route cannot open an owned inactive generation. IndexGenerationStore intentionally gives each candidate read-through symlinks to source.db, user.db, embeddings.db, ops.db, and blob, but ArchiveStore initializes the generation as an active writable archive. Durable change-train admission correctly rejects source.db at the symlink boundary. If that check were bypassed, the candidate replay path would open source.db and user.db writable through the links, publish blobs, update authority census and parse-state rows, and therefore mutate the phase-2 frozen source during phase 3. This defect currently accounts for 69 current-master full-suite failures and blocks polylogue-818fy.","design":"Introduce an explicit owned-inactive-candidate access mode. Validate generation ownership and exact read-through targets, initialize and write only the candidate index.db, open source.db and user.db read-only, and make every durable or blob mutation attempt fail before side effects. Split replay source remediation from candidate application: the candidate route must consume a complete, fingerprint-current frozen source census and persisted authority decisions, derive and write only index-side rows and receipts, and fail with a typed source-remediation-required outcome when the frozen source is incomplete or would need a changed authority decision. Do not relax active-archive durable change-train identity checks. Bind the candidate receipt to unchanged source/user identities and byte digests.","acceptance_criteria":"1. The real daemon/offline owned-inactive generation route opens an intentionally symlinked candidate without running active-root durable bootstrap against the generation root. 2. Candidate index writes succeed while source.db and user.db are opened read-only; blob publication, raw census, authority, parse-state, assertion, and other durable writes fail before side effects. 3. Candidate replay consumes only a complete current-parser census and persisted source authority. Missing/stale census or any replay decision that would change source state returns a typed source-remediation-required failure before candidate readiness. 4. A real-route fixture hashes source.db, user.db, and the blob namespace before and after a successful inactive build and proves byte and identity preservation, no active-pointer mutation, and a populated candidate index. 5. Adversarial tests restore the prior symlink/bootstrap path, make source/user writable, or permit one source mutation and fail. 6. Active archive startup still rejects unsafe durable symlinks or replaced files. 7. Focused rebuild, revision-governance, generation, and durable-train suites plus devtools verify --quick pass. 8. Production build, acceptance, promotion, restart, and postflight remain under their existing phase Beads.","status":"in_progress","priority":0,"issue_type":"bug","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-08-08T07:44:07Z","created_by":"Sinity","updated_at":"2026-08-08T07:44:47Z","started_at":"2026-08-08T07:44:47Z","lease_expires_at":"2026-08-08T07:49:47Z","heartbeat_at":"2026-08-08T07:44:47Z","labels":["area:maintenance","lane:reindex"],"dependency_count":0,"dependent_count":1,"comment_count":0} +{"_type":"issue","id":"polylogue-inactive-candidate-durable-barrier","title":"fix: isolate inactive candidates from durable-tier writes","description":"The exact current-master rebuild route cannot open an owned inactive generation. IndexGenerationStore intentionally gives each candidate read-through symlinks to source.db, user.db, embeddings.db, ops.db, and blob, but ArchiveStore initializes the generation as an active writable archive. Durable change-train admission correctly rejects source.db at the symlink boundary. If that check were bypassed, the candidate replay path would open source.db and user.db writable through the links, publish blobs, update authority census and parse-state rows, and therefore mutate the phase-2 frozen source during phase 3. This defect currently accounts for 69 current-master full-suite failures and blocks polylogue-818fy.","design":"Introduce an explicit owned-inactive-candidate access mode. Validate generation ownership and exact read-through targets, initialize and write only the candidate index.db, open source.db and user.db read-only, and make every durable or blob mutation attempt fail before side effects. Split replay source remediation from candidate application: the candidate route must consume a complete, fingerprint-current frozen source census and persisted authority decisions, derive and write only index-side rows and receipts, and fail with a typed source-remediation-required outcome when the frozen source is incomplete or would need a changed authority decision. Do not relax active-archive durable change-train identity checks. Bind the candidate receipt to unchanged source/user identities and byte digests.","acceptance_criteria":"1. The real daemon/offline owned-inactive generation route opens an intentionally symlinked candidate without running active-root durable bootstrap against the generation root. 2. Candidate index writes succeed while source.db and user.db are opened read-only; blob publication, raw census, authority, parse-state, assertion, and other durable writes fail before side effects. 3. Candidate replay consumes only a complete current-parser census and persisted source authority. Missing/stale census or any replay decision that would change source state returns a typed source-remediation-required failure before candidate readiness. 4. A real-route fixture hashes source.db, user.db, and the blob namespace before and after a successful inactive build and proves byte and identity preservation, no active-pointer mutation, and a populated candidate index. 5. Adversarial tests restore the prior symlink/bootstrap path, make source/user writable, or permit one source mutation and fail. 6. Active archive startup still rejects unsafe durable symlinks or replaced files. 7. Focused rebuild, revision-governance, generation, and durable-train suites plus devtools verify --quick pass. 8. Production build, acceptance, promotion, restart, and postflight remain under their existing phase Beads.","notes":"Satisfied at code head 92a9d4ac3a62e81a01d2ad832acd1060d5657c02. The real daemon and offline owned-inactive routes validate generation ownership plus exact read-through targets, open source/user read-only, write only candidate index state, refuse blob/user/source/ops mutations, validate complete frozen parser census and persisted source authority before generation allocation, preserve active pointer and durable identities, and reject poisoned append authority. Verification: 130 focused rebuild/revision tests passed; canary cleanup variants passed; devtools verify --quick run 20260808T111649Z-quick-3111380-e69d77f2 passed all 24 steps. Independent adversarial reviewer Volta returned: No legitimate blockers found at exact HEAD 92a9d4ac3a62e81a01d2ad832acd1060d5657c02. Production candidate construction, acceptance, promotion, restart, and postflight remain open under the existing phase Beads.","status":"closed","priority":0,"issue_type":"bug","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-08-08T07:44:07Z","created_by":"Sinity","updated_at":"2026-08-08T11:21:48Z","started_at":"2026-08-08T07:44:47Z","closed_at":"2026-08-08T11:21:48Z","close_reason":"Implemented and verified the owned inactive-candidate durable barrier, frozen-source admission, exact authority binding, and adversarial coverage. Production phase execution remains downstream.","labels":["area:maintenance","lane:reindex"],"dependency_count":0,"dependent_count":1,"comment_count":0} From 9fe7cc0bb87bf91fcfd2657b0b44d79ab176b9c8 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sat, 8 Aug 2026 13:41:55 +0200 Subject: [PATCH 16/19] fix(reindex): close frozen candidate review gaps Close read-only source census connections explicitly and document the intentional Provider-to-Origin normalization boundary. Expose frozen candidate mode through the governance protocol and stop source-tier finalization work once frozen authority has already been proven. Focused candidate, provenance, and revision-backfill verification passes with 130 tests. --- polylogue/sources/revision_backfill.py | 16 +++-- .../archive_tiers/revision_governance.py | 64 +++++++++---------- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/polylogue/sources/revision_backfill.py b/polylogue/sources/revision_backfill.py index 83041cc6cf..6d6eab68cd 100644 --- a/polylogue/sources/revision_backfill.py +++ b/polylogue/sources/revision_backfill.py @@ -12,7 +12,7 @@ from collections import OrderedDict from collections.abc import Callable, Iterator, Sequence from concurrent.futures import Future, ThreadPoolExecutor -from contextlib import contextmanager, nullcontext +from contextlib import closing, contextmanager, nullcontext from dataclasses import dataclass, field from io import BytesIO from itertools import chain, islice @@ -85,7 +85,7 @@ def _expand_frozen_revision_link_selection(archive_root: Path, raw_ids: Sequence """Include every predecessor and baseline needed to validate selected APPEND authority.""" expanded = set(raw_ids) pending = set(raw_ids) - with sqlite3.connect(f"file:{archive_root / 'source.db'}?mode=ro", uri=True) as source_conn: + with closing(sqlite3.connect(f"file:{archive_root / 'source.db'}?mode=ro", uri=True)) as source_conn: while pending: current = tuple(sorted(pending)) pending.clear() @@ -897,6 +897,8 @@ def _load_frozen_revision_evidence( parsed_logical_keys = tuple( sorted( { + # Parser output uses Provider internally, while the + # persisted census is normalized to public Origin keys. f"{origin_from_provider(session.source_name).value}:{session.provider_session_id}" for session in sessions } @@ -911,6 +913,8 @@ def _load_frozen_revision_evidence( state.classified += int(len(sessions) == 1) if revision_kind is RawRevisionKind.UNKNOWN: for session in sessions: + # Membership rows intentionally retain their provider-wire + # identity until durable authority comparison normalizes it. logical_key = f"{session.source_name.value}:{session.provider_session_id}" state.membership_candidates.setdefault(logical_key, set()).add(raw_id) return state @@ -931,7 +935,7 @@ def require_current_parser_source_census( selections = tuple( tuple(selected_raw_ids[offset : offset + 500]) for offset in range(0, len(selected_raw_ids), 500) ) - with sqlite3.connect(f"file:{archive_root / 'source.db'}?mode=ro", uri=True) as source_conn: + with closing(sqlite3.connect(f"file:{archive_root / 'source.db'}?mode=ro", uri=True)) as source_conn: for selection in selections: where = "" if selection is None else f"WHERE r.raw_id IN ({','.join('?' for _ in selection)})" params: tuple[object, ...] = () if selection is None else selection @@ -979,7 +983,7 @@ def require_current_parser_source_census( durable_logical_keys: dict[str, set[str]] = {raw_id: set() for raw_id in recorded_logical_keys} invalid_durable_bindings: set[str] = set() - with sqlite3.connect(f"file:{archive_root / 'source.db'}?mode=ro", uri=True) as source_conn: + with closing(sqlite3.connect(f"file:{archive_root / 'source.db'}?mode=ro", uri=True)) as source_conn: for selection in selections: where = "" if selection is None else f"WHERE r.raw_id IN ({','.join('?' for _ in selection)})" params = () if selection is None else selection @@ -1025,7 +1029,7 @@ def require_current_parser_source_census( ) authority_rows: dict[str, tuple[str | None, str, str, int, str | None, str | None]] = {} - with sqlite3.connect(f"file:{archive_root / 'source.db'}?mode=ro", uri=True) as source_conn: + with closing(sqlite3.connect(f"file:{archive_root / 'source.db'}?mode=ro", uri=True)) as source_conn: for selection in selections: where = "" if selection is None else f"WHERE raw_id IN ({','.join('?' for _ in selection)})" params = () if selection is None else selection @@ -1122,7 +1126,7 @@ def require_current_parser_source_census( ) unresolved_raw_ids: list[str] = [] - with sqlite3.connect(f"file:{archive_root / 'source.db'}?mode=ro", uri=True) as source_conn: + with closing(sqlite3.connect(f"file:{archive_root / 'source.db'}?mode=ro", uri=True)) as source_conn: for selection in selections: authority_where = "" if selection is None else f"AND r.raw_id IN ({','.join('?' for _ in selection)})" authority_params: tuple[object, ...] = () if selection is None else selection diff --git a/polylogue/storage/sqlite/archive_tiers/revision_governance.py b/polylogue/storage/sqlite/archive_tiers/revision_governance.py index 6b4d4c532d..886c3a0a38 100644 --- a/polylogue/storage/sqlite/archive_tiers/revision_governance.py +++ b/polylogue/storage/sqlite/archive_tiers/revision_governance.py @@ -225,6 +225,7 @@ class RawRevisionGovernanceHost(Protocol): _conn: sqlite3.Connection _blob_publisher: ArchiveBlobPublisher | None + _inactive_candidate_durable_read_only: bool _pending_raw_parse_states: list[tuple[str, RawSessionStateUpdate]] def _ensure_source_conn(self) -> sqlite3.Connection: ... @@ -2522,14 +2523,13 @@ def apply_raw_revision_replay( ApplicationDecision.SUPERSEDED, } } - for raw_id in terminal_raw_ids: - provider, _blob_hash, _source_path, _kind, _blob_size = raw_revision_descriptor(store, raw_id) - if _is_frozen_candidate(store): - continue - if manage_transaction: - mark_raw_parse_succeeded(store, raw_id, provider=provider) - else: - store._pending_raw_parse_states.append((raw_id, _raw_parse_success_state(provider))) + if not _is_frozen_candidate(store): + for raw_id in terminal_raw_ids: + provider, _blob_hash, _source_path, _kind, _blob_size = raw_revision_descriptor(store, raw_id) + if manage_transaction: + mark_raw_parse_succeeded(store, raw_id, provider=provider) + else: + store._pending_raw_parse_states.append((raw_id, _raw_parse_success_state(provider))) return session_id, plan.accepted_raw_ids @@ -2967,30 +2967,28 @@ def apply_raw_membership_classification( if _is_frozen_candidate(store): require_frozen_membership_authority(store, logical_source_key, classification, decisions) - else: - with conn if manage_transaction else nullcontext(): - for raw_id, decision in decisions.items(): - conn.execute( - """ - UPDATE raw_session_memberships - SET decision = ?, decided_at_ms = ?, - revision_authority = ?, - acquisition_generation = ? - WHERE raw_id = ? AND logical_source_key = ? - """, - ( - decision, - decided_at_ms, - "quarantined" - if decision in {MembershipDecision.AMBIGUOUS, MembershipDecision.DEFERRED} - else "byte_proven", - classification.accepted_raw_ids.index(raw_id) - if raw_id in classification.accepted_raw_ids - else 0, - raw_id, - logical_source_key, - ), - ) + return session_id + with conn if manage_transaction else nullcontext(): + for raw_id, decision in decisions.items(): + conn.execute( + """ + UPDATE raw_session_memberships + SET decision = ?, decided_at_ms = ?, + revision_authority = ?, + acquisition_generation = ? + WHERE raw_id = ? AND logical_source_key = ? + """, + ( + decision, + decided_at_ms, + "quarantined" + if decision in {MembershipDecision.AMBIGUOUS, MembershipDecision.DEFERRED} + else "byte_proven", + classification.accepted_raw_ids.index(raw_id) if raw_id in classification.accepted_raw_ids else 0, + raw_id, + logical_source_key, + ), + ) for raw_id in decisions: complete = conn.execute( """ @@ -3004,8 +3002,6 @@ def apply_raw_membership_classification( """, (raw_id,), ).fetchone() - if _is_frozen_candidate(store): - continue if complete is not None and bool(complete[0]): provider, _blob_hash, _source_path, _kind, _blob_size = raw_revision_descriptor(store, raw_id) if manage_transaction: From 8e4d6f4a706bd99adf610added66b9b8ab6e1fc8 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sat, 8 Aug 2026 13:43:59 +0200 Subject: [PATCH 17/19] fix(ingest): declare mutable Drive governance mode Declare the Drive revision adapter as a normal mutable governance host so the frozen-candidate protocol remains explicit and structurally type-safe. --- polylogue/pipeline/services/ingest_batch/_core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/polylogue/pipeline/services/ingest_batch/_core.py b/polylogue/pipeline/services/ingest_batch/_core.py index 73002dba49..6b317da74b 100644 --- a/polylogue/pipeline/services/ingest_batch/_core.py +++ b/polylogue/pipeline/services/ingest_batch/_core.py @@ -629,6 +629,7 @@ class _DriveRevisionGovernanceAdapter: def __init__(self, source_conn: sqlite3.Connection, blob_publisher: ArchiveBlobPublisher | None) -> None: self._source_conn = source_conn self._blob_publisher = blob_publisher + self._inactive_candidate_durable_read_only = False # Never read by bind_raw_revision/classify_raw_revision_cohort; see # class docstring for why this is a harmless placeholder value. self._conn = source_conn From 329cdf5767bed0619f7edea6ec9adffae1ef81fa Mon Sep 17 00:00:00 2001 From: Sinity Date: Sat, 8 Aug 2026 13:59:52 +0200 Subject: [PATCH 18/19] fix(reindex): make source admission rebuild-safe Validate the full frozen source once when a resumable transaction is created, then revalidate only each selected authority component on resumed pages. Treat terminal non-session census rows as resolved and let source-only admission operate when the derived index is missing or unreadable. Regression coverage proves bounded resume, non-session authority, and missing-index recovery. The 133-test candidate/provenance/revision suite passes. The daemon suite has seven passes plus two failures already present in the exact current-master baseline. --- polylogue/daemon/bulk_rebuild.py | 6 +- polylogue/maintenance/rebuild_index.py | 41 ++++--- polylogue/sources/revision_backfill.py | 11 +- .../storage/sqlite/archive_tiers/archive.py | 29 +++-- ...test_inactive_candidate_durable_barrier.py | 108 +++++++++++++++++- 5 files changed, 160 insertions(+), 35 deletions(-) diff --git a/polylogue/daemon/bulk_rebuild.py b/polylogue/daemon/bulk_rebuild.py index a573e8aa61..09a7423cfa 100644 --- a/polylogue/daemon/bulk_rebuild.py +++ b/polylogue/daemon/bulk_rebuild.py @@ -211,7 +211,6 @@ def resolve_or_start_daemon_bulk_rebuild_transaction( # after ownership acquisition so receipt expiry, source revision, or # external-corpus drift cannot reach generation bookkeeping. _validate_rebuild_provenance_receipt(root, schema_inference_receipt_path) - validate_rebuild_source_admission(root, location) store = IndexGenerationStore(location, repair_anchor=False) transaction: IndexRebuildTransaction | None try: @@ -232,6 +231,11 @@ def resolve_or_start_daemon_bulk_rebuild_transaction( _validate_rebuild_provenance_receipt(root, schema_inference_receipt_path) return transaction + # A fresh transaction receives one archive-wide source admission. + # Resumed passes validate only their selected authority component in + # rebuild_index_from_source_sync, avoiding a full reparse per page. + validate_rebuild_source_admission(root, location) + if transaction is not None: if transaction.status == "promoted-attestation-failed": # The generation is already active. Preserve the terminal diff --git a/polylogue/maintenance/rebuild_index.py b/polylogue/maintenance/rebuild_index.py index b00be63464..0317ffe239 100644 --- a/polylogue/maintenance/rebuild_index.py +++ b/polylogue/maintenance/rebuild_index.py @@ -132,14 +132,12 @@ def require_rebuild_schema_currency(root: Path) -> dict[str, object]: def validate_rebuild_source_admission(root: Path, location: ArchiveLocation) -> None: - """Validate frozen source authority through the owned active-index identity.""" + """Validate frozen source authority under the owned archive identity.""" from polylogue.sources.revision_backfill import validate_frozen_source_authority - from polylogue.storage.index_generation import canonical_active_index_path - validate_frozen_source_authority( - root, - active_index_path=canonical_active_index_path(location), - ) + if location.configured_root != root.absolute(): + raise RuntimeError("rebuild source admission received a foreign archive location") + validate_frozen_source_authority(root) @dataclass(slots=True) @@ -1343,8 +1341,16 @@ async def rebuild_index_from_source(request: RebuildIndexRequest) -> RebuildInde # route below, so a missing root/index.db must not masquerade as raw debt. if count_source_raw_sessions(root) and location.active_index_path.parent == root: from polylogue.readiness.capability import raw_frontier_source_selection_block_reason - - if reason := raw_frontier_source_selection_block_reason(root): + from polylogue.storage.archive_readiness import raw_materialization_readiness_snapshot + + materialization = raw_materialization_readiness_snapshot(root) + # An available active index must agree with source and cursor + # authority before it can seed a rebuild. A missing or unreadable + # derived tier is the recovery case this operation exists to handle; + # source-only candidate admission below remains mandatory. + if materialization.get("available") is True and ( + reason := raw_frontier_source_selection_block_reason(root, materialization) + ): raise RuntimeError(f"reindex source preflight gate failed: raw frontier integrity: {reason}") active_config = Config( archive_root=root, @@ -1429,11 +1435,7 @@ async def _rebuild_index_from_source_owned( validate_frozen_source_authority, ) from polylogue.storage.archive_readiness import archive_readiness_status - from polylogue.storage.index_generation import ( - IndexGenerationStore, - canonical_active_index_path, - rebuild_source_evidence_snapshot, - ) + from polylogue.storage.index_generation import IndexGenerationStore, rebuild_source_evidence_snapshot from polylogue.storage.repair import repair_session_insights provenance = RebuildProvenanceContext( @@ -1472,15 +1474,12 @@ async def _rebuild_index_from_source_owned( page = None pass_started_at_ms = int(time.time() * 1000) if resumable_full_source: - validate_frozen_source_authority( - root, - active_index_path=canonical_active_index_path(owned.location), - ) - generation_store = IndexGenerationStore(owned.location, repair_anchor=request.promote) + generation_store = IndexGenerationStore(owned.location, repair_anchor=False) if request.operation_id is not None: transaction = generation_store.load_transaction(request.operation_id) transaction = _reconcile_active_generation_transaction(generation_store, transaction) else: + validate_frozen_source_authority(root) transaction = _create_rebuild_transaction_after_receipt_validation( generation_store, request, provenance ) @@ -1563,6 +1562,11 @@ async def _rebuild_index_from_source_owned( selected_raw_ids = [raw_id for raw_id, _blob_hash_hex, _blob_size in page.rows] selected_raw_count = len(selected_raw_ids) skipped_by_blob_limit_count = 0 + if not transaction_created_here and selected_raw_ids: + validate_frozen_source_authority( + root, + selected_raw_ids=selected_raw_ids, + ) else: selection_started_at = time.perf_counter() raw_count, selected_raw_ids, skipped_by_blob_limit_count = select_rebuild_raw_ids(request) @@ -1570,7 +1574,6 @@ async def _rebuild_index_from_source_owned( selected_raw_count = len(selected_raw_ids) validate_frozen_source_authority( root, - active_index_path=canonical_active_index_path(owned.location), selected_raw_ids=selected_raw_ids, ) generation_store = IndexGenerationStore(owned.location, repair_anchor=request.promote) diff --git a/polylogue/sources/revision_backfill.py b/polylogue/sources/revision_backfill.py index 6d6eab68cd..a1e8f8e5dd 100644 --- a/polylogue/sources/revision_backfill.py +++ b/polylogue/sources/revision_backfill.py @@ -1141,9 +1141,14 @@ def require_current_parser_source_census( WHERE r.revision_authority = 'quarantined' {authority_where} AND ( - c.raw_id IS NULL OR c.status != 'complete' - OR m.raw_id IS NULL OR m.decision IS NULL - OR m.decision IN ('ambiguous', 'deferred') + c.raw_id IS NULL OR c.status NOT IN ('complete', 'non_session') + OR ( + c.status = 'complete' + AND ( + m.raw_id IS NULL OR m.decision IS NULL + OR m.decision IN ('ambiguous', 'deferred') + ) + ) ) ORDER BY r.raw_id """, diff --git a/polylogue/storage/sqlite/archive_tiers/archive.py b/polylogue/storage/sqlite/archive_tiers/archive.py index 23be4513ea..3b0e9825c7 100644 --- a/polylogue/storage/sqlite/archive_tiers/archive.py +++ b/polylogue/storage/sqlite/archive_tiers/archive.py @@ -1623,7 +1623,7 @@ def _session_filter_is_active(session_filters: Mapping[str, object] | None) -> b class _SourceTierOnlyIndexConnection: - """Loud placeholder for the index connection in source-tier acquisition mode. + """Loud placeholder for an archive mode that must not open ``index.db``. Acquire-only ingestion (polylogue-gbs02) deliberately never opens ``index.db`` — a derived tier awaiting rebuild may be at an older schema @@ -1633,13 +1633,16 @@ class _SourceTierOnlyIndexConnection: instead of writing through a stale-schema handle. """ + def __init__(self, mode: str = "source-tier acquisition") -> None: + self._mode = mode + def __getattr__(self, name: str) -> Any: if name == "close": return lambda: None raise RuntimeError( - "index tier is unavailable in source-tier acquisition mode " + f"index tier is unavailable in {self._mode} mode " f"(attempted connection attribute {name!r}); only raw source-tier " - "admission is permitted while derived tiers await rebuild" + "access is permitted while the derived tier is unavailable" ) @@ -1875,6 +1878,21 @@ def _initialize_store( self._tags_relation = "session_tags" self._blob_publisher = ArchiveBlobPublisher(self.source_db_path, self.archive_root / "blob") return + if self._frozen_source_validation: + # Candidate admission derives every decision from source.db and + # frozen blob bytes. Requiring an index handle here would make the + # derived tier being rebuilt a prerequisite for its own rebuild. + self._conn = cast( + sqlite3.Connection, + _SourceTierOnlyIndexConnection("frozen source validation"), + ) + self._user_tier_attached = False + self._tags_relation = "session_tags" + self._blob_publisher = _InactiveCandidateBlobPublisher( + self.source_db_path, + self.archive_root / "blob", + ) + return if initialize: initialize_active_archive_root(archive_root) if read_only and not self._frozen_source_validation: @@ -1917,11 +1935,6 @@ def _initialize_store( _InactiveCandidateBlobPublisher if self._inactive_candidate_durable_read_only else ArchiveBlobPublisher ) self._blob_publisher = publisher_type(self.source_db_path, self.archive_root / "blob") - elif self._frozen_source_validation: - self._blob_publisher = _InactiveCandidateBlobPublisher( - self.source_db_path, - self.archive_root / "blob", - ) self._attach_user_tier_if_present() @classmethod diff --git a/tests/unit/maintenance/test_inactive_candidate_durable_barrier.py b/tests/unit/maintenance/test_inactive_candidate_durable_barrier.py index 2e5b29ebe3..0c3342c28d 100644 --- a/tests/unit/maintenance/test_inactive_candidate_durable_barrier.py +++ b/tests/unit/maintenance/test_inactive_candidate_durable_barrier.py @@ -10,6 +10,7 @@ import pytest +import polylogue.sources.revision_backfill as revision_backfill_module from polylogue.archive.revision_authority import RawRevisionAuthority, RawRevisionEnvelope, RawRevisionKind from polylogue.core.enums import Provider from polylogue.daemon.bulk_rebuild import resolve_or_start_daemon_bulk_rebuild_transaction @@ -17,6 +18,7 @@ from polylogue.sources.revision_backfill import ( backfill_historical_revision_evidence, census_historical_revision_evidence, + validate_frozen_source_authority, ) from polylogue.storage.blob_store import PreparedBlob from polylogue.storage.fts.drift_sampling import sample_fts_drift_to_ops_sync @@ -95,12 +97,12 @@ def _chatgpt_bundle(*native_ids: str) -> bytes: return json.dumps(sessions, sort_keys=True).encode() -def _prepare_frozen_source(root: Path, monkeypatch: pytest.MonkeyPatch) -> Path: - build_independent_raw_corpus(root, raw_count=1, avg_payload_bytes=1_000) +def _prepare_frozen_source(root: Path, monkeypatch: pytest.MonkeyPatch, *, raw_count: int = 1) -> Path: + build_independent_raw_corpus(root, raw_count=raw_count, avg_payload_bytes=1_000) monkeypatch.setenv("POLYLOGUE_ARCHIVE_ROOT", str(root)) census = census_historical_revision_evidence(root) - assert census.scanned == 1 - assert census.classified_full == 1 + assert census.scanned == raw_count + assert census.classified_full == raw_count with sqlite3.connect(root / "source.db") as source: source.execute( """ @@ -113,6 +115,104 @@ def _prepare_frozen_source(root: Path, monkeypatch: pytest.MonkeyPatch) -> Path: return write_valid_rebuild_receipt(root, root.parent / "schema-inference-receipt.json") +def test_resumed_candidate_validates_only_selected_raw_page( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = tmp_path / "archive" + receipt_path = _prepare_frozen_source(root, monkeypatch, raw_count=2) + original_validate = revision_backfill_module.validate_frozen_source_authority + selections: list[tuple[str, ...] | None] = [] + + def record_validation( + archive_root: Path, + *, + selected_raw_ids: list[str] | None = None, + **kwargs: object, + ) -> None: + selections.append(None if selected_raw_ids is None else tuple(selected_raw_ids)) + original_validate(archive_root, selected_raw_ids=selected_raw_ids, **kwargs) + + monkeypatch.setattr(revision_backfill_module, "validate_frozen_source_authority", record_validation) + first = rebuild_index_from_source_sync( + RebuildIndexRequest( + archive_root=root, + schema_inference_receipt_path=receipt_path, + raw_batch_size=1, + promote=False, + ) + ) + + assert first.status == "paused" + assert first.transaction is not None + assert selections == [None] + + selections.clear() + second = rebuild_index_from_source_sync( + RebuildIndexRequest( + archive_root=root, + operation_id=str(first.transaction["operation_id"]), + schema_inference_receipt_path=receipt_path, + raw_batch_size=1, + promote=False, + ) + ) + + assert second.status == "replayed" + assert len(selections) == 1 + assert selections[0] is not None + assert len(selections[0]) == 1 + + +def test_frozen_source_admission_treats_non_session_census_as_terminal( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = tmp_path / "archive" + initialize_active_archive_root(root) + monkeypatch.setenv("POLYLOGUE_ARCHIVE_ROOT", str(root)) + with ArchiveStore.open_existing(root, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.CHATGPT, + payload=b"[]", + source_path="empty-conversations.json", + acquired_at_ms=1, + ) + census_historical_revision_evidence(root) + + with sqlite3.connect(root / "source.db") as source: + assert source.execute( + "SELECT status, member_count FROM raw_membership_census WHERE raw_id = ?", + (raw_id,), + ).fetchone() == ("non_session", 0) + assert source.execute( + "SELECT revision_authority FROM raw_sessions WHERE raw_id = ?", + (raw_id,), + ).fetchone() == ("quarantined",) + + validate_frozen_source_authority(root) + + +def test_candidate_rebuild_does_not_require_the_missing_active_index( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = tmp_path / "archive" + receipt_path = _prepare_frozen_source(root, monkeypatch) + (root / "index.db").unlink() + + result = rebuild_index_from_source_sync( + RebuildIndexRequest( + archive_root=root, + schema_inference_receipt_path=receipt_path, + promote=False, + ) + ) + + assert result.status == "replayed" + assert result.transaction is not None + + def test_real_no_promote_candidate_preserves_frozen_durable_tiers( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, From 4d02ba7e83b75049e60c65f049aa78dfc73add53 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sat, 8 Aug 2026 14:02:05 +0200 Subject: [PATCH 19/19] test(reindex): type bounded admission probe Keep the resumed-page validation spy faithful to the production validator signature so strict type checking covers the regression harness. --- .../maintenance/test_inactive_candidate_durable_barrier.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/unit/maintenance/test_inactive_candidate_durable_barrier.py b/tests/unit/maintenance/test_inactive_candidate_durable_barrier.py index 0c3342c28d..7d60c28e92 100644 --- a/tests/unit/maintenance/test_inactive_candidate_durable_barrier.py +++ b/tests/unit/maintenance/test_inactive_candidate_durable_barrier.py @@ -128,10 +128,9 @@ def record_validation( archive_root: Path, *, selected_raw_ids: list[str] | None = None, - **kwargs: object, ) -> None: selections.append(None if selected_raw_ids is None else tuple(selected_raw_ids)) - original_validate(archive_root, selected_raw_ids=selected_raw_ids, **kwargs) + original_validate(archive_root, selected_raw_ids=selected_raw_ids) monkeypatch.setattr(revision_backfill_module, "validate_frozen_source_authority", record_validation) first = rebuild_index_from_source_sync(