From 94b7b1e7be2a28aea9615807c4f7e6d16626bb90 Mon Sep 17 00:00:00 2001 From: Sinity Date: Tue, 11 Aug 2026 15:44:20 +0200 Subject: [PATCH 01/65] fix(live): repair cursor authority across active archives Problem: live ingestion could advance cursors over typed terminal artifacts without retention authority, write a stale conventional index after generation promotion, and misclassify sidecar inputs. A hidden-directory watch race also could miss an immediately created source file. What changed: align retention, cursor integrity, bootstrap, and writable archive opens with active source authority. Preserve semantic heads without byte-chain reinterpretation, follow the active index pointer, exclude only non-session sidecars with no decoded session evidence, and scan newly added watched directories. Tests now assert the real authority and shutdown routes. --- polylogue/sources/live/batch.py | 17 +++- polylogue/sources/live/watcher.py | 41 +++++++- polylogue/storage/raw_retention.py | 94 ++++++++++++++++++- .../storage/sqlite/archive_tiers/archive.py | 11 ++- .../storage/sqlite/archive_tiers/bootstrap.py | 21 ++++- tests/unit/sources/test_live_watcher.py | 5 + .../unit/sources/test_live_watcher_locking.py | 21 ++++- tests/unit/storage/test_raw_retention.py | 90 ++++++++++++++++++ 8 files changed, 289 insertions(+), 11 deletions(-) diff --git a/polylogue/sources/live/batch.py b/polylogue/sources/live/batch.py index b7ed29f790..d15bb2d870 100644 --- a/polylogue/sources/live/batch.py +++ b/polylogue/sources/live/batch.py @@ -20,6 +20,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, Literal, ParamSpec, TypeVar, cast +from polylogue.archive.artifact_taxonomy import classify_artifact_path from polylogue.archive.ingest_flags import ( COMPACT_BROWSER_CAPTURE_INGEST_FLAG, DOM_FALLBACK_INGEST_FLAG, @@ -144,6 +145,7 @@ ZipEntryReadContext, iter_zip_entry_raw_data, ) +from polylogue.sources.source_parsing import has_decoded_session_evidence from polylogue.sources.sqlite_snapshot import ( codex_state_raw_id, hermes_profile_raw_id, @@ -1877,6 +1879,7 @@ def _ingest_full_paths_sync( continue captured_file_observations[path] = _file_observation(stat) origin_artifact_rule = artifact_rule_for_path(fallback_provider, str(path)) + path_artifact = classify_artifact_path(path, provider=fallback_provider) if heartbeat is not None: heartbeat( "full_file_scan", @@ -1994,6 +1997,18 @@ def _ingest_full_paths_sync( # the bytes as a generic session artifact. self._mark_excluded_cursor(path, stat, source_name=fallback_provider.value) continue + elif ( + origin_artifact_rule is None + and path.suffix.lower() != ".jsonl" + and path_artifact is not None + and not path_artifact.parse_as_session + and not has_decoded_session_evidence(path, provider=fallback_provider) + ): + # Keep path-only metadata out of the generic JSON fallback, + # but let real decoded session evidence outrank a stale or + # overbroad filename rule just as offline source parsing does. + self._mark_excluded_cursor(path, stat, source_name=fallback_provider.value) + continue elif origin_artifact_rule is not None and origin_artifact_rule.parse_policy != "session": provider = fallback_provider source_name = provider.value @@ -2037,7 +2052,7 @@ def _ingest_full_paths_sync( elif path.suffix.lower() == ".jsonl": provider, parse_as_session = _jsonl_provider_and_session_artifact(path, fallback_provider) source_name = provider.value - if not parse_as_session and provider is not Provider.UNKNOWN: + if not parse_as_session: self._mark_excluded_cursor(path, stat, source_name=source_name) continue if stat.st_size >= _STREAMING_FULL_INGEST_BYTES: diff --git a/polylogue/sources/live/watcher.py b/polylogue/sources/live/watcher.py index a57ac556ef..708485444d 100644 --- a/polylogue/sources/live/watcher.py +++ b/polylogue/sources/live/watcher.py @@ -354,7 +354,11 @@ async def _watch_changes(self, roots: list[Path]) -> None: for change, raw_path in changes: if change is Change.deleted: continue - path = self._canonical_watch_path(Path(raw_path)) + observed_path = Path(raw_path) + if change is Change.added and observed_path.is_dir(): + self._enqueue_added_directory(observed_path) + continue + path = self._canonical_watch_path(observed_path) if path is None: continue if not self._source_accepts(path): @@ -1522,7 +1526,7 @@ def _source_name_for(self, path: Path) -> str: try: if resolved.is_relative_to(source.root.resolve()): return source.name - except OSError: + except (OSError, ValueError): continue return path.parent.name @@ -1574,6 +1578,33 @@ def _canonical_watch_path(self, path: Path) -> Path | None: return database return None + def _source_for_directory(self, path: Path) -> WatchSource | None: + """Return the watched source owning a non-ignored directory.""" + + resolved = path.resolve() + for source in self._sources: + try: + relative = resolved.relative_to(source.root.resolve()) + except (OSError, ValueError): + continue + if any(source.ignores_directory(Path(part)) for part in relative.parts): + return None + return source + return None + + def _enqueue_added_directory(self, directory: Path) -> None: + """Cover files created before a recursive watcher installs its new sub-watch.""" + + source = self._source_for_directory(directory) + if source is None: + return + for parent, dir_names, file_names in os.walk(directory): + dir_names[:] = [name for name in dir_names if not source.ignores_directory(Path(name))] + for name in file_names: + candidate = Path(parent) / name + if source.accepts(candidate): + self._enqueue(candidate) + def _watch_filter(self, _change: object, path: str) -> bool: """Accept configured source files under hidden canonical roots. @@ -1583,7 +1614,11 @@ def _watch_filter(self, _change: object, path: str) -> bool: writes. This filter keeps the project's own source/suffix predicate as the gate instead. """ - return self._canonical_watch_path(Path(path)) is not None + observed_path = Path(path) + return ( + self._canonical_watch_path(observed_path) is not None + or self._source_for_directory(observed_path) is not None + ) def _interleave_by_source(candidates: list[CandidateSourceFile]) -> list[CandidateSourceFile]: diff --git a/polylogue/storage/raw_retention.py b/polylogue/storage/raw_retention.py index 830c5afaa1..5ca094d900 100644 --- a/polylogue/storage/raw_retention.py +++ b/polylogue/storage/raw_retention.py @@ -390,16 +390,33 @@ def active_raw_retention_authority( session_raw_ids, heads, eligible_receipts = _active_index_raw_authority(index_db_path) seeds = set(session_raw_ids) seeds.update(head.accepted_raw_id for head in heads) + all_raw_ids = frozenset(str(row[0]) for row in conn.execute("SELECT raw_id FROM raw_sessions").fetchall()) + terminal_artifact_raw_ids = _terminal_artifact_raw_ids(conn) if not seeds: - if conn.execute("SELECT 1 FROM raw_sessions LIMIT 1").fetchone() is not None: + if all_raw_ids and all_raw_ids.issubset(terminal_artifact_raw_ids): + return RawRetentionAuthority(protected_raw_ids=all_raw_ids, eligible_raw_ids=frozenset()) + if all_raw_ids: raise RawRetentionSafetyError("source tier contains raw evidence but index has no raw authority") return RawRetentionAuthority(protected_raw_ids=frozenset(), eligible_raw_ids=frozenset()) authority_raw_ids = seeds.union(receipt.raw_id for receipt in eligible_receipts) rows_by_id = _raw_revision_rows(conn, authority_raw_ids) protected: set[str] = set() + byte_head_raw_ids = {head.accepted_raw_id for head in heads if head.accepted_frontier_kind == "byte"} + semantic_only_raw_ids = { + head.accepted_raw_id for head in heads if head.accepted_frontier_kind != "byte" + }.difference(byte_head_raw_ids) + # A semantic membership head is accepted authority for retention, but + # it is deliberately not a byte-predecessor proof. Keep it protected + # without reinterpreting it as one. + protected.update(semantic_only_raw_ids) + protected.update(terminal_artifact_raw_ids) for seed_raw_id in sorted(session_raw_ids): + if seed_raw_id in semantic_only_raw_ids: + continue protected.update(_validate_active_revision_chain(rows_by_id, seed_raw_id)) for head in heads: + if head.accepted_raw_id in semantic_only_raw_ids: + continue row = rows_by_id[head.accepted_raw_id] if head.accepted_frontier_kind == "byte": _validate_byte_head(row, head) @@ -1369,6 +1386,15 @@ def _check_broken_active_chains( for head in heads: heads_by_raw_id.setdefault(head.accepted_raw_id, []).append(head) seed_raw_ids = set(session_raw_ids).union(heads_by_raw_id) + # Membership-governed snapshots carry a semantic head, not a byte + # predecessor chain. They remain active source authority and must be + # retained, but applying byte-chain validation to them turns a normal + # membership snapshot into a false broken-head violation. A raw selected + # by both regimes remains byte-validated. + byte_head_raw_ids = {head.accepted_raw_id for head in heads if head.accepted_frontier_kind == "byte"} + semantic_only_raw_ids = { + head.accepted_raw_id for head in heads if head.accepted_frontier_kind != "byte" + }.difference(byte_head_raw_ids) try: rows_by_id = _raw_revision_rows(conn, seed_raw_ids, allow_missing=True) except _RawRevisionAuthorityUnavailableError as exc: @@ -1379,6 +1405,8 @@ def _check_broken_active_chains( for seed_raw_id in sorted(seed_raw_ids): seed_heads = heads_by_raw_id.get(seed_raw_id, []) row = rows_by_id.get(seed_raw_id) + if seed_raw_id in semantic_only_raw_ids: + continue if row is None and not seed_heads: # Directly missing sessions.raw_id rows are counted once by the # canonical lost-source-evidence projection. There is no chain to @@ -1479,6 +1507,11 @@ def _check_cursor_ahead_of_accepted( except sqlite3.Error as exc: logger.warning("raw frontier integrity: cursor source path lookup failed: %s", exc) return "unknown", 0, 0, 0, 0, (), 0, (), f"cursor source path lookup failed: {exc}" + try: + terminal_artifact_paths = _terminal_artifact_paths(conn, set(cursor_map)) + except sqlite3.Error as exc: + logger.warning("raw frontier integrity: terminal artifact authority lookup failed: %s", exc) + return "unknown", 0, 0, 0, 0, (), 0, (), f"terminal artifact authority is unreadable: {exc}" for path, cursor in cursor_map.items(): cursor_offset = cursor.byte_offset if cursor.is_deferred: @@ -1501,7 +1534,7 @@ def _check_cursor_ahead_of_accepted( if not comparable_heads: # A path governed exclusively by membership authority has no # comparable byte frontier and is intentionally out of scope. - if path in all_head_paths: + if path in all_head_paths or path in terminal_artifact_paths: continue gap_count += 1 if len(gaps) < sample_limit: @@ -1593,6 +1626,63 @@ def _source_paths_for_paths(conn: sqlite3.Connection, source_paths: set[str]) -> return result +def _terminal_artifact_paths(conn: sqlite3.Connection, source_paths: set[str]) -> set[str]: + """Return paths whose current raw observation is typed non-session evidence. + + A full-route cursor can legitimately advance over a workflow/fact artifact + that has no session head. ``raw_artifacts.parse_as_session = 0`` is the + source-tier terminal authority for that case. Ordinary artifact upserts + retain the source coordinate's latest receipt while ``raw_sessions`` + retains its historical acquisition evidence, so authority attaches to the + newest raw observation rather than requiring a duplicate receipt on every + historical raw. A later conversational raw cannot inherit the exemption: + it becomes the newest observation and leaves the path without terminal + authority until it gains a comparable accepted head. + """ + + result: set[str] = set() + pending = set(source_paths) + while pending: + batch = tuple(sorted(pending)[:500]) + pending.difference_update(batch) + placeholders = ", ".join("?" for _ in batch) + rows = conn.execute( + f""" + SELECT DISTINCT artifact.source_path + FROM raw_artifacts AS artifact + JOIN raw_sessions AS terminal_raw ON terminal_raw.raw_id = artifact.raw_id + WHERE artifact.parse_as_session = 0 + AND artifact.source_path IN ({placeholders}) + AND terminal_raw.raw_id = ( + SELECT newest.raw_id + FROM raw_sessions AS newest + WHERE newest.source_path = artifact.source_path + ORDER BY newest.acquired_at_ms DESC, newest.rowid DESC + LIMIT 1 + ) + """, + batch, + ).fetchall() + result.update(str(row[0]) for row in rows) + return result + + +def _terminal_artifact_raw_ids(conn: sqlite3.Connection) -> frozenset[str]: + """Return all retained raw evidence for paths with terminal current observations.""" + + source_paths = {str(row[0]) for row in conn.execute("SELECT DISTINCT source_path FROM raw_sessions").fetchall()} + terminal_paths = _terminal_artifact_paths(conn, source_paths) + if not terminal_paths: + return frozenset() + ordered_paths = tuple(sorted(terminal_paths)) + placeholders = ", ".join("?" for _ in ordered_paths) + rows = conn.execute( + f"SELECT raw_id FROM raw_sessions WHERE source_path IN ({placeholders})", + ordered_paths, + ).fetchall() + return frozenset(str(row[0]) for row in rows) + + def _ops_cursor_byte_offsets(ops_db_path: Path) -> dict[str, _OpsCursorAuthority]: if not ops_db_path.is_file(): raise RawRetentionSafetyError(f"ops tier is unavailable: {ops_db_path}") diff --git a/polylogue/storage/sqlite/archive_tiers/archive.py b/polylogue/storage/sqlite/archive_tiers/archive.py index 3841333e58..9875935900 100644 --- a/polylogue/storage/sqlite/archive_tiers/archive.py +++ b/polylogue/storage/sqlite/archive_tiers/archive.py @@ -1867,7 +1867,16 @@ def _initialize_store( ) -> None: self.archive_root = archive_root self.source_db_path = archive_root / "source.db" - self.index_db_path = self._frozen_index_path or archive_root / "index.db" + if self._frozen_index_path is not None: + self.index_db_path = self._frozen_index_path + else: + # The configured root owns the durable tiers, while an active + # generation can keep index.db elsewhere. A writable open must + # follow the same pointer as readiness and live ingest instead of + # silently mutating a stale conventional root/index.db shadow. + from polylogue.storage.archive_identity import resolve_active_index_path + + self.index_db_path = resolve_active_index_path(archive_root) self.embeddings_db_path = archive_root / "embeddings.db" self.user_db_path = archive_root / "user.db" self.ops_db_path = archive_root / "ops.db" diff --git a/polylogue/storage/sqlite/archive_tiers/bootstrap.py b/polylogue/storage/sqlite/archive_tiers/bootstrap.py index e124c0ff10..d736cdb9d7 100644 --- a/polylogue/storage/sqlite/archive_tiers/bootstrap.py +++ b/polylogue/storage/sqlite/archive_tiers/bootstrap.py @@ -4,6 +4,7 @@ import os import sqlite3 +import threading from dataclasses import dataclass from pathlib import Path from typing import Literal @@ -76,6 +77,13 @@ def ddl(self) -> str: } +# ``OwnedArchiveLocation`` protects against other processes. Its deliberate +# reentrancy permits nested production opens in one process, so bootstrap also +# needs this process-local serialization around the fresh durable receipt +# protocol. +_ACTIVE_ARCHIVE_BOOTSTRAP_LOCK = threading.RLock() + + def archive_tier_spec(tier: ArchiveTier) -> ArchiveTierSpec: """Return the database-file spec for one durability tier.""" return ARCHIVE_TIER_SPECS[tier] @@ -310,7 +318,7 @@ def initialize_archive_database( conn.close() -def initialize_active_archive_root(root: Path) -> None: +def _initialize_active_archive_root(root: Path) -> None: """Create or initialize every tier database in an archive root.""" from polylogue.operations.durable_change_train import audit_adoption_receipt_path, recover_pending_audit_adoption from polylogue.storage.archive_identity import ( @@ -429,9 +437,11 @@ def classify_paths() -> tuple[bool, bool]: if not recovering_fresh_durable_bootstrap and not pre_marker_adoption: assert_owned_root() reconcile_durable_change_trains_on_startup(root) + location = ArchiveLocation.resolve(root) for spec in ARCHIVE_TIER_SPECS.values(): assert_owned_root() - initialize_archive_database(root / spec.filename, spec.tier) + tier_path = location.active_index_path if spec.tier is ArchiveTier.INDEX else root / spec.filename + initialize_archive_database(tier_path, spec.tier) # Mutation composition performs source/audit reconciliation immediately # before it consumes authority. Ordinary archive opens stay read-only # with respect to continuity, including their steady-state path. @@ -453,6 +463,13 @@ def classify_paths() -> tuple[bool, bool]: pending_bootstrap_path.unlink(missing_ok=True) +def initialize_active_archive_root(root: Path) -> None: + """Create or initialize every active archive tier under one local bootstrap owner.""" + + with _ACTIVE_ARCHIVE_BOOTSTRAP_LOCK: + _initialize_active_archive_root(root) + + def reconcile_durable_change_trains_on_startup(root: Path) -> tuple[Path, ...]: """Reconcile persisted durable trains without executing migration SQL.""" from polylogue.storage.sqlite.durable_change_train import reconcile_durable_change_train_startup diff --git a/tests/unit/sources/test_live_watcher.py b/tests/unit/sources/test_live_watcher.py index 7f508c3175..779b405f71 100644 --- a/tests/unit/sources/test_live_watcher.py +++ b/tests/unit/sources/test_live_watcher.py @@ -357,6 +357,7 @@ async def test_active_index_pointer_keeps_shadow_index_unmodified(tmp_path: Path projection = reconcile._projection_for(tmp_path) sample = projection.cursor_ahead_samples[0] shadow_before = shadow_index.read_bytes() + active_before = active_index.read_bytes() with scoped_cursor_authority_authorization( source_path_digest=cursor_authority_path_digest(source_path), cursor_byte_offset=sample.cursor_byte_offset, @@ -368,6 +369,7 @@ async def test_active_index_pointer_keeps_shadow_index_unmodified(tmp_path: Path assert metrics.full_file_count == 1 assert shadow_index.read_bytes() == shadow_before + assert active_index.read_bytes() != active_before watcher.stop() @@ -2176,6 +2178,7 @@ async def test_live_full_ingest_preserves_complete_workflow_journal_revisions( } assert summary.call_count == 1 assert summary.journal_result_count == 1 + assert processor.require_cursor_authority() is None finally: await archive.close() @@ -2294,6 +2297,7 @@ async def test_live_append_atof_shared_file_multi_session_boundary_retains_all_e await processor.ingest_files([source_path], emit_event=False) replayed = _atof_event_uuids_by_session(workspace_env["archive_root"]) assert replayed == event_uuids_by_session + assert processor.require_cursor_authority() is None finally: await archive.close() @@ -2433,6 +2437,7 @@ async def test_live_full_ingest_over_ambiguous_membership_preserves_durable_debt second = await processor.ingest_files([source_path], emit_event=False) assert second.succeeded_file_count == 1, "ambiguous membership debt is not retried as a file failure (#3282)" assert second.failed_file_count == 0 + assert processor.require_cursor_authority() is None record = cursor.get_record(source_path) assert record is not None diff --git a/tests/unit/sources/test_live_watcher_locking.py b/tests/unit/sources/test_live_watcher_locking.py index edc3b09e99..a6ba716848 100644 --- a/tests/unit/sources/test_live_watcher_locking.py +++ b/tests/unit/sources/test_live_watcher_locking.py @@ -61,11 +61,18 @@ async def main() -> None: cursor=cursor, write_coordinator=coordinator, ) + # This proof targets the writer bridge's process-exit semantics. + # Disable the independent prefetch lane so an executor worker + # cannot determine the subprocess lifetime instead. + watcher._parse_stage.shutdown() + watcher._parse_stage = None + watcher._batch_processor._parse_stage = None started = threading.Event() + release = threading.Event() def stuck(*args, **kwargs): started.set() - threading.Event().wait() + release.wait() if {route!r} == "append": stat = path.stat() @@ -94,7 +101,17 @@ def stuck(*args, **kwargs): caller.cancel() with contextlib.suppress(asyncio.CancelledError): await caller - assert await coordinator.shutdown(timeout=0.01) is False + try: + assert await coordinator.shutdown(timeout=0.01) is False + finally: + # The injected thread is intentionally unlike production + # parsing: it has no natural completion condition. Releasing + # it after the coordinator's fail-safe result proves the + # process-exit assertion without making asyncio's executor + # shutdown permanently unreleasable. + release.set() + assert await coordinator.shutdown(timeout=1.0) is True + watcher.stop() asyncio.run(main()) """ diff --git a/tests/unit/storage/test_raw_retention.py b/tests/unit/storage/test_raw_retention.py index 60011c82ca..d3adc1ea4e 100644 --- a/tests/unit/storage/test_raw_retention.py +++ b/tests/unit/storage/test_raw_retention.py @@ -556,6 +556,96 @@ def test_active_raw_protection_rejects_empty_index_over_retained_source(tmp_path assert conn.execute("SELECT COUNT(*) FROM raw_sessions").fetchone() == (2,) +def test_current_terminal_artifact_authorizes_historical_raws_but_not_later_session_raw(tmp_path: Path) -> None: + """Terminal artifact authority follows the current coordinate receipt, not every old raw. + + ``raw_artifacts`` intentionally keeps one current carrier per ordinary + source coordinate while ``raw_sessions`` keeps every acquisition. A + current workflow/fact artifact may therefore retain historical raw rows + without a duplicate artifact receipt. Conversely, a later unclassified + raw must remove that exemption rather than allowing the old terminal + receipt to mask a cursor-authority gap. + """ + source_db = tmp_path / "source.db" + index_db = tmp_path / "index.db" + ops_db = tmp_path / "ops.db" + source_path = tmp_path / "journal.jsonl" + initialize_archive_database(source_db, ArchiveTier.SOURCE) + initialize_archive_database(index_db, ArchiveTier.INDEX) + with sqlite3.connect(source_db) as conn: + _insert_revision_raw( + conn, + raw_id="raw-journal-old", + source_path=source_path, + acquired_at_ms=1, + kind="unknown", + source_revision="old", + generation=0, + blob_size=10, + authority="quarantined", + ) + _insert_revision_raw( + conn, + raw_id="raw-journal-current", + source_path=source_path, + acquired_at_ms=2, + kind="unknown", + source_revision="current", + generation=0, + blob_size=20, + authority="quarantined", + ) + conn.execute( + """ + INSERT INTO raw_artifacts ( + artifact_id, raw_id, origin, source_path, source_index, + artifact_kind, support_status, classification_reason, + parse_as_session, schema_eligible, malformed_jsonl_lines, + first_observed_at_ms, last_observed_at_ms + ) VALUES (?, ?, 'claude-code-session', ?, 0, 'workflow_journal', + 'unknown', 'typed terminal artifact', 0, 0, 0, 1, 2) + """, + ("artifact-journal", "raw-journal-current", str(source_path)), + ) + conn.commit() + _seed_ops_cursor(ops_db, source_path=source_path, byte_offset=20) + + with sqlite3.connect(source_db) as conn: + authority = active_raw_retention_authority(conn, index_db_path=index_db) + snapshot = raw_frontier_integrity_snapshot(conn, index_db_path=index_db, ops_db_path=ops_db) + + assert authority == RawRetentionAuthority( + protected_raw_ids=frozenset({"raw-journal-old", "raw-journal-current"}), + eligible_raw_ids=frozenset(), + ) + assert snapshot.cursor_ahead_status == "healthy" + assert snapshot.cursor_authority_gap_count == 0 + + with sqlite3.connect(source_db) as conn: + _insert_revision_raw( + conn, + raw_id="raw-conversational-later", + source_path=source_path, + acquired_at_ms=3, + kind="full", + source_revision="later", + generation=0, + blob_size=30, + authority="asserted", + ) + conn.commit() + _seed_ops_cursor(ops_db, source_path=source_path, byte_offset=30) + + with sqlite3.connect(source_db) as conn: + with pytest.raises(RawRetentionSafetyError, match="index has no raw authority"): + active_raw_retention_authority(conn, index_db_path=index_db) + snapshot = raw_frontier_integrity_snapshot(conn, index_db_path=index_db, ops_db_path=ops_db) + + assert snapshot.cursor_ahead_status == "unknown" + assert snapshot.cursor_authority_gap_count == 1 + assert snapshot.cursor_authority_gap_samples[0].state == "source_raws_without_accepted_head" + + def test_active_raw_protection_rejects_incomplete_predecessor_chain(tmp_path: Path) -> None: source_db = tmp_path / "source.db" index_db = tmp_path / "index.db" From 995eb8663827d3b0a1e6a1ea4f38f2916c12da5c Mon Sep 17 00:00:00 2001 From: Sinity Date: Tue, 11 Aug 2026 19:22:59 +0200 Subject: [PATCH 02/65] fix(live): follow active index authority in every route Problem: cursor reconciliation, append bootstrap, raw-frontier reporting, and maintenance raw retention still opened the conventional index path after a generation promotion. They could therefore read stale authority even while live writes followed the promoted index.\n\nWhat changed: resolve the active index pointer in each remaining authority reader. Pointer-backed tests cover watcher cursor recovery, frontier health, and superseded-raw cleanup against an intentionally stale shadow index. --- polylogue/sources/live/append_ingest.py | 3 +- polylogue/sources/live/watcher.py | 7 +-- polylogue/storage/raw_retention.py | 3 +- polylogue/storage/repair.py | 4 +- tests/unit/sources/test_live_watcher.py | 4 ++ tests/unit/storage/test_raw_retention.py | 48 +++++++++++++++++++ tests/unit/storage/test_repair.py | 59 ++++++++++++++++++++++++ 7 files changed, 121 insertions(+), 7 deletions(-) diff --git a/polylogue/sources/live/append_ingest.py b/polylogue/sources/live/append_ingest.py index 91fcb4e231..54b52f9ec5 100644 --- a/polylogue/sources/live/append_ingest.py +++ b/polylogue/sources/live/append_ingest.py @@ -24,6 +24,7 @@ from polylogue.sources.live.batch_support import _AppendPlan, _AppendResult from polylogue.sources.live.cursor import CursorStore from polylogue.sources.live.sqlite_locking import is_transient_sqlite_lock +from polylogue.storage.archive_identity import resolve_active_index_path from polylogue.storage.raw.models import RawSessionStateUpdate from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root from polylogue.storage.sqlite.archive_tiers.raw_admission import RawAdmissionArm @@ -72,7 +73,7 @@ def _ingest_append_plans_archive( archive_root: Path, ) -> _AppendResult: timings: dict[str, float] = {} - index_db = archive_root / "index.db" + index_db = resolve_active_index_path(archive_root) source_db = archive_root / "source.db" if not index_db.exists() or not source_db.exists(): t0 = time.perf_counter() diff --git a/polylogue/sources/live/watcher.py b/polylogue/sources/live/watcher.py index 708485444d..1705f48839 100644 --- a/polylogue/sources/live/watcher.py +++ b/polylogue/sources/live/watcher.py @@ -52,6 +52,7 @@ from polylogue.sources.live.metrics import LiveBatchMetrics from polylogue.sources.live.parse_prefetch import LiveParseStage from polylogue.sources.sqlite_snapshot import is_sqlite_path, sqlite_database_for_sidecar, sqlite_source_revision +from polylogue.storage.archive_identity import resolve_active_index_path if TYPE_CHECKING: from polylogue.api import Polylogue @@ -1207,7 +1208,7 @@ def _archived_cursor_reconciliation_scope(self) -> Iterator[None]: """ archive_root = Path(getattr(self._polylogue, "archive_root", self._cursor._db_path.parent)) source_db = archive_root / "source.db" - index_db = archive_root / "index.db" + index_db = resolve_active_index_path(archive_root) conns: tuple[sqlite3.Connection, sqlite3.Connection] | None = None if source_db.exists() and index_db.exists(): try: @@ -1334,7 +1335,7 @@ def _cursor_skip_corroborated_by_index(self, path: Path) -> bool: return self._path_corroborated_by_index(path, source_conn=shared[0], index_conn=shared[1]) archive_root = Path(getattr(self._polylogue, "archive_root", self._cursor._db_path.parent)) source_db = archive_root / "source.db" - index_db = archive_root / "index.db" + index_db = resolve_active_index_path(archive_root) if not source_db.exists() or not index_db.exists(): return True with ( @@ -1370,7 +1371,7 @@ def _reconcile_archived_cursor_outcome( row = self._archived_cursor_row(path, source_conn=shared[0], index_conn=shared[1]) else: source_db = archive_root / "source.db" - index_db = archive_root / "index.db" + index_db = resolve_active_index_path(archive_root) if not source_db.exists() or not index_db.exists(): return _ArchivedCursorReconciliation.UNAVAILABLE with ( diff --git a/polylogue/storage/raw_retention.py b/polylogue/storage/raw_retention.py index 5ca094d900..38ba716df5 100644 --- a/polylogue/storage/raw_retention.py +++ b/polylogue/storage/raw_retention.py @@ -11,6 +11,7 @@ from typing import Literal from polylogue.logging import get_logger +from polylogue.storage.archive_identity import resolve_active_index_path from polylogue.storage.blob_store import BlobStore, get_blob_store from polylogue.storage.introspection import column_exists as _column_exists from polylogue.storage.introspection import table_exists as _table_exists @@ -1173,7 +1174,7 @@ def raw_frontier_integrity_projection( raw_materialization_readiness, sample_limit=sample_limit, ) - index_db_path = archive_root / "index.db" + index_db_path = resolve_active_index_path(archive_root) source_db_path = archive_root / "source.db" ops_db_path = archive_root / "ops.db" snapshot = _unavailable_frontier_integrity_snapshot(f"source tier is unavailable: {source_db_path}") diff --git a/polylogue/storage/repair.py b/polylogue/storage/repair.py index ff9102247e..6429abffa5 100644 --- a/polylogue/storage/repair.py +++ b/polylogue/storage/repair.py @@ -45,7 +45,7 @@ ) from polylogue.pipeline.ids import session_content_hash, session_revision_projection from polylogue.pipeline.ids import session_id as make_session_id -from polylogue.storage.archive_identity import archive_file_set_root +from polylogue.storage.archive_identity import archive_file_set_root, resolve_active_index_path from polylogue.storage.blob_repair import count_orphaned_blobs_sync, repair_orphaned_blobs_data from polylogue.storage.blob_store import BlobStore from polylogue.storage.insights.session.repair_assessment import ( @@ -5890,7 +5890,7 @@ def repair_superseded_raw_snapshots(config: Config, dry_run: bool = False) -> Re archive_root = _raw_materialization_archive_root(config) repair_db_path = archive_root / "source.db" if repair_db_path.exists(): - index_db_path = archive_root / "index.db" + index_db_path = resolve_active_index_path(archive_root) with closing(open_connection(repair_db_path)) as conn, conn: conn.row_factory = sqlite3.Row try: diff --git a/tests/unit/sources/test_live_watcher.py b/tests/unit/sources/test_live_watcher.py index 779b405f71..8a59950683 100644 --- a/tests/unit/sources/test_live_watcher.py +++ b/tests/unit/sources/test_live_watcher.py @@ -370,6 +370,10 @@ async def test_active_index_pointer_keeps_shadow_index_unmodified(tmp_path: Path assert metrics.full_file_count == 1 assert shadow_index.read_bytes() == shadow_before assert active_index.read_bytes() != active_before + with sqlite3.connect(shadow_index) as conn: + conn.execute("DELETE FROM sessions") + conn.commit() + assert watcher._reconcile_archived_cursor(source_path, stat=source_path.stat()) is True watcher.stop() diff --git a/tests/unit/storage/test_raw_retention.py b/tests/unit/storage/test_raw_retention.py index d3adc1ea4e..bfaf5861e5 100644 --- a/tests/unit/storage/test_raw_retention.py +++ b/tests/unit/storage/test_raw_retention.py @@ -2042,6 +2042,54 @@ def test_raw_frontier_integrity_projection_preserves_violation_when_sibling_is_u assert projection.available is False +def test_raw_frontier_integrity_projection_follows_active_index_pointer(tmp_path: Path) -> None: + """A promoted index, rather than a stale conventional shadow, governs frontier health.""" + + source_db = tmp_path / "source.db" + shadow_index = tmp_path / "index.db" + active_index = tmp_path / "generations" / "active" / "index.db" + ops_db = tmp_path / "ops.db" + source_path = tmp_path / "session.jsonl" + source_path.write_text("{}\n", encoding="utf-8") + initialize_archive_database(source_db, ArchiveTier.SOURCE) + initialize_archive_database(shadow_index, ArchiveTier.INDEX) + initialize_archive_database(active_index, ArchiveTier.INDEX) + initialize_archive_database(ops_db, ArchiveTier.OPS) + with sqlite3.connect(source_db) as conn: + _insert_revision_raw( + conn, + raw_id="raw-active", + source_path=source_path, + acquired_at_ms=1, + kind="full", + source_revision="revision-1", + generation=1, + blob_size=10, + ) + conn.commit() + _seed_index_authority( + active_index, + session_raw_id="raw-active", + accepted_raw_id="raw-active", + accepted_revision="revision-1", + generation=1, + frontier=10, + append_end_offset=None, + ) + _seed_ops_cursor(ops_db, source_path=source_path, byte_offset=10) + (tmp_path / ".index-active-pointer").write_text(f"{active_index}\n", encoding="utf-8") + + projection = raw_frontier_integrity_projection( + tmp_path, + {"available": True, "lost_source_evidence_count": 0}, + ) + + assert projection.broken_head_status == "healthy" + assert projection.cursor_ahead_status == "healthy" + assert projection.overall_status == "healthy" + assert projection.available is True + + @pytest.mark.parametrize("index_kind", ["missing", "malformed"]) def test_raw_frontier_integrity_snapshot_unavailable_index_tier_is_unknown_never_healthy( tmp_path: Path, diff --git a/tests/unit/storage/test_repair.py b/tests/unit/storage/test_repair.py index 7cb78bcbe7..9469920808 100644 --- a/tests/unit/storage/test_repair.py +++ b/tests/unit/storage/test_repair.py @@ -1453,6 +1453,65 @@ def test_superseded_raw_cleanup_protects_split_index_referenced_raw_ids(tmp_path assert "skipped 1 active revision raw rows" in result.detail +def test_superseded_raw_cleanup_follows_active_index_pointer(tmp_path: Path) -> None: + config = _config(tmp_path) + initialize_archive_database(tmp_path / "source.db", ArchiveTier.SOURCE) + shadow_index = tmp_path / "index.db" + active_index = tmp_path / "generations" / "active" / "index.db" + initialize_archive_database(shadow_index, ArchiveTier.INDEX) + initialize_archive_database(active_index, ArchiveTier.INDEX) + source_file = tmp_path / "source.jsonl" + source_file.write_text("{}", encoding="utf-8") + + with sqlite3.connect(tmp_path / "source.db") as source_conn: + source_conn.executemany( + """ + INSERT INTO raw_sessions ( + raw_id, origin, native_id, source_path, source_index, blob_hash, blob_size, acquired_at_ms + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + ( + "raw-referenced-old", + "chatgpt-export", + "native-old", + str(source_file), + 0, + bytes.fromhex("11" * 32), + 10, + 1, + ), + ( + "raw-newer", + "chatgpt-export", + "native-newer", + str(source_file), + 0, + bytes.fromhex("22" * 32), + 11, + 2, + ), + ), + ) + source_conn.commit() + with sqlite3.connect(active_index) as index_conn: + index_conn.execute( + """ + INSERT INTO sessions (native_id, origin, raw_id, title, content_hash) + VALUES (?, ?, ?, ?, ?) + """, + ("native-old", "chatgpt-export", "raw-referenced-old", "old", bytes(32)), + ) + index_conn.commit() + (tmp_path / ".index-active-pointer").write_text(f"{active_index}\n", encoding="utf-8") + + result = repair_mod.repair_superseded_raw_snapshots(config, dry_run=True) + + assert result.success is True + assert result.repaired_count == 0 + assert "skipped 1 active revision raw rows" in result.detail + + def test_superseded_raw_cleanup_allows_history_before_active_full(tmp_path: Path) -> None: config = _config(tmp_path) initialize_archive_database(tmp_path / "source.db", ArchiveTier.SOURCE) From 5795e3caf3a778658d5b6e66fbcf67811162108b Mon Sep 17 00:00:00 2001 From: Sinity Date: Tue, 11 Aug 2026 19:48:28 +0200 Subject: [PATCH 03/65] fix(live): preserve terminal evidence for new source paths Keep unknown JSONL on the full acquisition route so strict classification records terminal source evidence. Drain hook spool shards immediately when their directory is added to the watcher. --- polylogue/sources/live/batch.py | 7 ++++- polylogue/sources/live/watcher.py | 3 ++ tests/unit/sources/test_hook_spool.py | 39 +++++++++++++++++++++++++ tests/unit/sources/test_live_watcher.py | 4 +-- 4 files changed, 50 insertions(+), 3 deletions(-) diff --git a/polylogue/sources/live/batch.py b/polylogue/sources/live/batch.py index d15bb2d870..659413a62e 100644 --- a/polylogue/sources/live/batch.py +++ b/polylogue/sources/live/batch.py @@ -2052,7 +2052,12 @@ def _ingest_full_paths_sync( elif path.suffix.lower() == ".jsonl": provider, parse_as_session = _jsonl_provider_and_session_artifact(path, fallback_provider) source_name = provider.value - if not parse_as_session: + # An unknown JSONL cannot be safely excluded from acquire: the + # strict parse route persists typed terminal evidence for empty + # and malformed exports. Known-provider sidecars remain + # cursor-excluded here because their classification is already + # authoritative. + if not parse_as_session and provider is not Provider.UNKNOWN: self._mark_excluded_cursor(path, stat, source_name=source_name) continue if stat.st_size >= _STREAMING_FULL_INGEST_BYTES: diff --git a/polylogue/sources/live/watcher.py b/polylogue/sources/live/watcher.py index 1705f48839..597e5ed645 100644 --- a/polylogue/sources/live/watcher.py +++ b/polylogue/sources/live/watcher.py @@ -357,6 +357,9 @@ async def _watch_changes(self, roots: list[Path]) -> None: continue observed_path = Path(raw_path) if change is Change.added and observed_path.is_dir(): + if self._is_hook_spool_path(observed_path): + await self._drain_hook_spool() + continue self._enqueue_added_directory(observed_path) continue path = self._canonical_watch_path(observed_path) diff --git a/tests/unit/sources/test_hook_spool.py b/tests/unit/sources/test_hook_spool.py index a93e2ad1f8..117b986463 100644 --- a/tests/unit/sources/test_hook_spool.py +++ b/tests/unit/sources/test_hook_spool.py @@ -267,6 +267,45 @@ async def emit_first_hook(*roots: Path, **_kwargs: object) -> AsyncIterator[set[ assert conn.execute("SELECT session_native_id FROM raw_hook_events").fetchone() == ("session-1",) +@pytest.mark.asyncio +async def test_live_watcher_drains_hook_spool_from_added_directory_notification( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """An added hook shard drains immediately without waiting for catch-up.""" + + spool_root = tmp_path / "hooks" + pending = pending_hook_spool_dir(spool_root) + archive_root = tmp_path / "archive" + archive_root.mkdir() + watcher = LiveWatcher( + cast(Any, SimpleNamespace(archive_root=archive_root, backend=None)), + (WatchSource(name="hooks", root=pending, suffixes=(".json",)),), + cursor=CursorStore(archive_root / "ops.db"), + ) + + async def emit_added_shard(*roots: Path, **_kwargs: object) -> AsyncIterator[set[tuple[Change, str]]]: + assert roots == (pending,) + event_path = enqueue_hook_event( + event_id="directory-notification", + provider="codex", + event_type="SessionStart", + session_id="session-1", + timestamp="2026-07-12T10:00:00Z", + payload={"cwd": "/workspace"}, + root=spool_root, + ) + yield {(Change.added, str(event_path.parent))} + + monkeypatch.setattr(watchfiles, "awatch", emit_added_shard) + + await watcher._watch_changes([pending]) + + assert list(acknowledged_hook_spool_dir(spool_root).rglob("directory-notification.json")) != [] + with sqlite3.connect(archive_root / "source.db") as conn: + assert conn.execute("SELECT session_native_id FROM raw_hook_events").fetchone() == ("session-1",) + + def test_hook_spool_retains_sqlite_failures_for_retry( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/sources/test_live_watcher.py b/tests/unit/sources/test_live_watcher.py index 8a59950683..371bf2459e 100644 --- a/tests/unit/sources/test_live_watcher.py +++ b/tests/unit/sources/test_live_watcher.py @@ -2871,7 +2871,7 @@ async def test_live_full_ingest_excludes_non_session_sidecars_before_raw_storage @pytest.mark.asyncio -async def test_live_full_ingest_excludes_invalid_jsonl_sidecars_before_raw_storage( +async def test_live_full_ingest_excludes_known_provider_invalid_jsonl_sidecars_before_raw_storage( workspace_env: dict[str, Path], ) -> None: root = workspace_env["data_root"] / "projects" @@ -2884,7 +2884,7 @@ async def test_live_full_ingest_excludes_invalid_jsonl_sidecars_before_raw_stora cursor = CursorStore(db_path) processor = LiveBatchProcessor( archive, - (WatchSource(name="projects", root=root),), + (WatchSource(name="claude-code", root=root),), cursor=cursor, parser_fingerprint=live_watcher._PARSER_FINGERPRINT, ) From 5743f4cbe5810c3ef1a453d1155a31be5b8fd3cd Mon Sep 17 00:00:00 2001 From: Sinity Date: Tue, 11 Aug 2026 20:05:06 +0200 Subject: [PATCH 04/65] fix(live): redrain newly added hook spool shards Retry once after the first drain so a directory event that precedes atomic envelope publication does not defer hook evidence to periodic catch-up. --- polylogue/sources/live/watcher.py | 8 +++++ tests/unit/sources/test_hook_spool.py | 51 +++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/polylogue/sources/live/watcher.py b/polylogue/sources/live/watcher.py index 597e5ed645..aa3d8b70f0 100644 --- a/polylogue/sources/live/watcher.py +++ b/polylogue/sources/live/watcher.py @@ -62,6 +62,12 @@ # One bounded writer hold per hook-spool drain batch; the drain loops until # the backlog is gone, releasing the writer between batches. _HOOK_SPOOL_DRAIN_BATCH_LIMIT = 250 +# A hook creates a day-shard directory before atomically publishing its first +# envelope. An added-directory event can therefore precede the child-file +# event that a recursive watcher is about to install. Re-drain once after this +# short publication grace period rather than leaving the envelope to periodic +# catch-up. +_HOOK_SPOOL_DIRECTORY_RETRY_DELAY_S = 0.05 # A catch-up writer owns the only archive writer for the whole chunk. The # former 50-file/64-MiB envelope held it for 14+ minutes on the real archive, # starving fresh watcher events. Keep historical convergence fair by @@ -358,6 +364,8 @@ async def _watch_changes(self, roots: list[Path]) -> None: observed_path = Path(raw_path) if change is Change.added and observed_path.is_dir(): if self._is_hook_spool_path(observed_path): + await self._drain_hook_spool() + await asyncio.sleep(_HOOK_SPOOL_DIRECTORY_RETRY_DELAY_S) await self._drain_hook_spool() continue self._enqueue_added_directory(observed_path) diff --git a/tests/unit/sources/test_hook_spool.py b/tests/unit/sources/test_hook_spool.py index 117b986463..4406ebff7b 100644 --- a/tests/unit/sources/test_hook_spool.py +++ b/tests/unit/sources/test_hook_spool.py @@ -306,6 +306,57 @@ async def emit_added_shard(*roots: Path, **_kwargs: object) -> AsyncIterator[set assert conn.execute("SELECT session_native_id FROM raw_hook_events").fetchone() == ("session-1",) +@pytest.mark.asyncio +async def test_live_watcher_redrains_added_hook_shard_after_atomic_publish( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The directory event can precede its first atomically published envelope.""" + + spool_root = tmp_path / "hooks" + pending = pending_hook_spool_dir(spool_root) + archive_root = tmp_path / "archive" + archive_root.mkdir() + watcher = LiveWatcher( + cast(Any, SimpleNamespace(archive_root=archive_root, backend=None)), + (WatchSource(name="hooks", root=pending, suffixes=(".json",)),), + cursor=CursorStore(archive_root / "ops.db"), + ) + original_drain = watcher._drain_hook_spool + drain_calls = 0 + + async def drain_then_publish_first_envelope() -> None: + nonlocal drain_calls + await original_drain() + drain_calls += 1 + if drain_calls == 1: + enqueue_hook_event( + event_id="published-after-directory-event", + provider="codex", + event_type="SessionStart", + session_id="session-2", + timestamp="2026-07-12T10:00:00Z", + payload={"cwd": "/workspace"}, + root=spool_root, + ) + + async def emit_empty_shard(*roots: Path, **_kwargs: object) -> AsyncIterator[set[tuple[Change, str]]]: + assert roots == (pending,) + shard = pending / "2026-08-11" + shard.mkdir(parents=True) + yield {(Change.added, str(shard))} + + monkeypatch.setattr(watcher, "_drain_hook_spool", drain_then_publish_first_envelope) + monkeypatch.setattr(watchfiles, "awatch", emit_empty_shard) + + await watcher._watch_changes([pending]) + + assert drain_calls == 2 + assert list(acknowledged_hook_spool_dir(spool_root).rglob("published-after-directory-event.json")) != [] + with sqlite3.connect(archive_root / "source.db") as conn: + assert conn.execute("SELECT session_native_id FROM raw_hook_events").fetchone() == ("session-2",) + + def test_hook_spool_retains_sqlite_failures_for_retry( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, From ec1756befdbb63fee66eb51528538d431fea0b07 Mon Sep 17 00:00:00 2001 From: Sinity Date: Tue, 11 Aug 2026 20:24:06 +0200 Subject: [PATCH 05/65] fix(live): retry hook shard publication races Drain an added hook shard immediately, then retry only an empty day shard until its first atomically published envelope is visible. Cancel those retry tasks with the watcher lifecycle.\n\nCover a producer delayed beyond the former fixed grace period without a child-file notification. --- polylogue/sources/live/watcher.py | 72 ++++++++++++++++++++++++--- tests/unit/sources/test_hook_spool.py | 50 +++++++++++-------- 2 files changed, 95 insertions(+), 27 deletions(-) diff --git a/polylogue/sources/live/watcher.py b/polylogue/sources/live/watcher.py index aa3d8b70f0..1b67653871 100644 --- a/polylogue/sources/live/watcher.py +++ b/polylogue/sources/live/watcher.py @@ -64,10 +64,10 @@ _HOOK_SPOOL_DRAIN_BATCH_LIMIT = 250 # A hook creates a day-shard directory before atomically publishing its first # envelope. An added-directory event can therefore precede the child-file -# event that a recursive watcher is about to install. Re-drain once after this -# short publication grace period rather than leaving the envelope to periodic -# catch-up. -_HOOK_SPOOL_DIRECTORY_RETRY_DELAY_S = 0.05 +# event that a recursive watcher is about to install. Poll only that new shard +# until its first envelope is visible, rather than leaving it to periodic +# catch-up or relying on a scheduler-dependent fixed grace period. +_HOOK_SPOOL_DIRECTORY_RETRY_POLL_S = 0.05 # A catch-up writer owns the only archive writer for the whole chunk. The # former 50-file/64-MiB envelope held it for 14+ minutes on the real archive, # starving fresh watcher events. Keep historical convergence fair by @@ -269,6 +269,7 @@ def __init__( self._drain_task: asyncio.Task[None] | None = None self._failed_retry_task: asyncio.Task[None] | None = None self._periodic_catch_up_task: asyncio.Task[None] | None = None + self._hook_spool_directory_retry_tasks: set[asyncio.Task[None]] = set() self._failed_retry_deadline: float | None = None self._last_enqueue_at = 0.0 self._last_batch_at: float = 0.0 @@ -348,6 +349,7 @@ async def run(self) -> None: with suppress(asyncio.CancelledError): await watch_task self._cancel_periodic_catch_up() + self._cancel_hook_spool_directory_retries() async def _watch_changes(self, roots: list[Path]) -> None: from watchfiles import Change, awatch @@ -364,9 +366,12 @@ async def _watch_changes(self, roots: list[Path]) -> None: observed_path = Path(raw_path) if change is Change.added and observed_path.is_dir(): if self._is_hook_spool_path(observed_path): + needs_first_envelope_retry = self._is_hook_spool_shard_directory( + observed_path + ) and not self._hook_spool_directory_has_envelope(observed_path) await self._drain_hook_spool() - await asyncio.sleep(_HOOK_SPOOL_DIRECTORY_RETRY_DELAY_S) - await self._drain_hook_spool() + if needs_first_envelope_retry: + self._schedule_hook_spool_directory_retry(observed_path) continue self._enqueue_added_directory(observed_path) continue @@ -384,6 +389,7 @@ def stop(self) -> None: self._stop.set() self._cancel_failed_retry_task() self._cancel_periodic_catch_up() + self._cancel_hook_spool_directory_retries() if self._parse_stage is not None and self._owns_parse_stage: self._parse_stage.shutdown() @@ -395,6 +401,7 @@ def cancel_pending(self) -> None: self._pending_scheduled = False self._cancel_failed_retry_task() self._cancel_periodic_catch_up() + self._cancel_hook_spool_directory_retries() async def _periodic_catch_up(self, roots: list[Path]) -> None: delay_s = _PERIODIC_CATCH_UP_INTERVAL_S @@ -420,6 +427,40 @@ def _cancel_periodic_catch_up(self) -> None: task.cancel() self._periodic_catch_up_task = None + def _schedule_hook_spool_directory_retry(self, directory: Path) -> None: + """Drain a newly added hook shard when its first envelope appears. + + The initial drain above handles an already-published envelope. This + task covers the narrow event-ordering race where the directory arrives + first and the recursive watcher misses the first child notification. + It does not block subsequent watcher events or start a source-tree + catch-up scan. + """ + + task = asyncio.create_task(self._retry_hook_spool_directory_until_populated(directory)) + self._hook_spool_directory_retry_tasks.add(task) + task.add_done_callback(self._hook_spool_directory_retry_tasks.discard) + + async def _retry_hook_spool_directory_until_populated(self, directory: Path) -> None: + """Wait for an added shard's first JSON envelope, then drain once.""" + + while not self._stop.is_set(): + try: + if not directory.exists(): + return + if any(directory.glob("*.json")): + await self._drain_hook_spool() + return + except OSError: + return + await asyncio.sleep(_HOOK_SPOOL_DIRECTORY_RETRY_POLL_S) + + def _cancel_hook_spool_directory_retries(self) -> None: + for task in tuple(self._hook_spool_directory_retry_tasks): + if not task.done(): + task.cancel() + self._hook_spool_directory_retry_tasks.clear() + # ------------------------------------------------------------------ # Catch-up: batch all changed files # ------------------------------------------------------------------ @@ -1562,6 +1603,25 @@ def _is_hook_spool_path(self, path: Path) -> bool: return False return False + def _is_hook_spool_shard_directory(self, path: Path) -> bool: + """Return whether ``path`` is a direct day shard beneath ``pending``.""" + + for source in self._sources: + if source.name != "hooks": + continue + try: + return path.resolve().parent == source.root.resolve() + except OSError: + return False + return False + + @staticmethod + def _hook_spool_directory_has_envelope(directory: Path) -> bool: + try: + return next(directory.glob("*.json"), None) is not None + except OSError: + return False + def _hook_spool_root(self) -> Path: """Return the root paired with this watcher's hook source.""" diff --git a/tests/unit/sources/test_hook_spool.py b/tests/unit/sources/test_hook_spool.py index 4406ebff7b..60613665c8 100644 --- a/tests/unit/sources/test_hook_spool.py +++ b/tests/unit/sources/test_hook_spool.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio import json import os import re @@ -300,6 +301,7 @@ async def emit_added_shard(*roots: Path, **_kwargs: object) -> AsyncIterator[set monkeypatch.setattr(watchfiles, "awatch", emit_added_shard) await watcher._watch_changes([pending]) + watcher.stop() assert list(acknowledged_hook_spool_dir(spool_root).rglob("directory-notification.json")) != [] with sqlite3.connect(archive_root / "source.db") as conn: @@ -307,11 +309,11 @@ async def emit_added_shard(*roots: Path, **_kwargs: object) -> AsyncIterator[set @pytest.mark.asyncio -async def test_live_watcher_redrains_added_hook_shard_after_atomic_publish( +async def test_live_watcher_retries_added_hook_shard_until_atomic_publish( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - """The directory event can precede its first atomically published envelope.""" + """A late first envelope drains without its own child-file notification.""" spool_root = tmp_path / "hooks" pending = pending_hook_spool_dir(spool_root) @@ -322,36 +324,42 @@ async def test_live_watcher_redrains_added_hook_shard_after_atomic_publish( (WatchSource(name="hooks", root=pending, suffixes=(".json",)),), cursor=CursorStore(archive_root / "ops.db"), ) - original_drain = watcher._drain_hook_spool - drain_calls = 0 - - async def drain_then_publish_first_envelope() -> None: - nonlocal drain_calls - await original_drain() - drain_calls += 1 - if drain_calls == 1: - enqueue_hook_event( - event_id="published-after-directory-event", - provider="codex", - event_type="SessionStart", - session_id="session-2", - timestamp="2026-07-12T10:00:00Z", - payload={"cwd": "/workspace"}, - root=spool_root, - ) + publish_task: asyncio.Task[None] | None = None + + async def publish_after_fixed_grace() -> None: + # The old one-shot 50 ms re-drain has already completed by the time + # this producer publishes. The retry must keep watching this shard. + await asyncio.sleep(0.10) + enqueue_hook_event( + event_id="published-after-directory-event", + provider="codex", + event_type="SessionStart", + session_id="session-2", + timestamp="2026-07-12T10:00:00Z", + payload={"cwd": "/workspace"}, + root=spool_root, + ) async def emit_empty_shard(*roots: Path, **_kwargs: object) -> AsyncIterator[set[tuple[Change, str]]]: + nonlocal publish_task assert roots == (pending,) shard = pending / "2026-08-11" shard.mkdir(parents=True) + publish_task = asyncio.create_task(publish_after_fixed_grace()) yield {(Change.added, str(shard))} - monkeypatch.setattr(watcher, "_drain_hook_spool", drain_then_publish_first_envelope) monkeypatch.setattr(watchfiles, "awatch", emit_empty_shard) await watcher._watch_changes([pending]) + assert publish_task is not None + await publish_task + + for _ in range(30): + if list(acknowledged_hook_spool_dir(spool_root).rglob("published-after-directory-event.json")): + break + await asyncio.sleep(0.01) + watcher.stop() - assert drain_calls == 2 assert list(acknowledged_hook_spool_dir(spool_root).rglob("published-after-directory-event.json")) != [] with sqlite3.connect(archive_root / "source.db") as conn: assert conn.execute("SELECT session_native_id FROM raw_hook_events").fetchone() == ("session-2",) From 843ca58e33e77a3622f496a23ad7492c415fc155 Mon Sep 17 00:00:00 2001 From: Sinity Date: Tue, 11 Aug 2026 20:35:23 +0200 Subject: [PATCH 06/65] fix(live): bound hook shard retries Deduplicate per-shard publication retries and stop polling after a bounded backoff window when an interrupted hook writer leaves an empty directory. --- polylogue/sources/live/watcher.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/polylogue/sources/live/watcher.py b/polylogue/sources/live/watcher.py index 1b67653871..68f2ec0074 100644 --- a/polylogue/sources/live/watcher.py +++ b/polylogue/sources/live/watcher.py @@ -68,6 +68,7 @@ # until its first envelope is visible, rather than leaving it to periodic # catch-up or relying on a scheduler-dependent fixed grace period. _HOOK_SPOOL_DIRECTORY_RETRY_POLL_S = 0.05 +_HOOK_SPOOL_DIRECTORY_RETRY_MAX_SECONDS = 5.0 # A catch-up writer owns the only archive writer for the whole chunk. The # former 50-file/64-MiB envelope held it for 14+ minutes on the real archive, # starving fresh watcher events. Keep historical convergence fair by @@ -269,7 +270,7 @@ def __init__( self._drain_task: asyncio.Task[None] | None = None self._failed_retry_task: asyncio.Task[None] | None = None self._periodic_catch_up_task: asyncio.Task[None] | None = None - self._hook_spool_directory_retry_tasks: set[asyncio.Task[None]] = set() + self._hook_spool_directory_retry_tasks: dict[Path, asyncio.Task[None]] = {} self._failed_retry_deadline: float | None = None self._last_enqueue_at = 0.0 self._last_batch_at: float = 0.0 @@ -437,14 +438,20 @@ def _schedule_hook_spool_directory_retry(self, directory: Path) -> None: catch-up scan. """ + directory = directory.resolve() + existing = self._hook_spool_directory_retry_tasks.get(directory) + if existing is not None and not existing.done(): + return task = asyncio.create_task(self._retry_hook_spool_directory_until_populated(directory)) - self._hook_spool_directory_retry_tasks.add(task) - task.add_done_callback(self._hook_spool_directory_retry_tasks.discard) + self._hook_spool_directory_retry_tasks[directory] = task + task.add_done_callback(lambda _task: self._hook_spool_directory_retry_tasks.pop(directory, None)) async def _retry_hook_spool_directory_until_populated(self, directory: Path) -> None: """Wait for an added shard's first JSON envelope, then drain once.""" - while not self._stop.is_set(): + deadline = asyncio.get_running_loop().time() + _HOOK_SPOOL_DIRECTORY_RETRY_MAX_SECONDS + delay_s = _HOOK_SPOOL_DIRECTORY_RETRY_POLL_S + while not self._stop.is_set() and asyncio.get_running_loop().time() < deadline: try: if not directory.exists(): return @@ -453,10 +460,11 @@ async def _retry_hook_spool_directory_until_populated(self, directory: Path) -> return except OSError: return - await asyncio.sleep(_HOOK_SPOOL_DIRECTORY_RETRY_POLL_S) + await asyncio.sleep(delay_s) + delay_s = min(delay_s * 2, 0.5) def _cancel_hook_spool_directory_retries(self) -> None: - for task in tuple(self._hook_spool_directory_retry_tasks): + for task in tuple(self._hook_spool_directory_retry_tasks.values()): if not task.done(): task.cancel() self._hook_spool_directory_retry_tasks.clear() From fae5087f8cea531efde63524fc0c8d7873db8d71 Mon Sep 17 00:00:00 2001 From: Sinity Date: Wed, 12 Aug 2026 22:54:39 +0200 Subject: [PATCH 07/65] fix: harden live ingest authority repairs --- polylogue/sources/live/batch.py | 12 +- polylogue/sources/live/batch_observability.py | 3 +- polylogue/sources/live/batch_support.py | 14 +-- .../sources/live/convergence_debt_retry.py | 3 +- polylogue/sources/live/watcher.py | 10 +- polylogue/sources/source_parsing.py | 3 +- polylogue/storage/raw_retention.py | 62 +++++++--- polylogue/storage/repair.py | 6 +- ...st_convergence_debt_deferred_vocabulary.py | 55 +++++++++ tests/unit/sources/test_hook_spool.py | 45 +++++-- tests/unit/sources/test_live_batch_support.py | 95 ++++++++++++++- tests/unit/sources/test_live_watcher.py | 19 +++ .../unit/sources/test_live_watcher_locking.py | 33 +++--- tests/unit/storage/test_raw_retention.py | 112 ++++++++++++++++++ tests/unit/storage/test_repair.py | 42 +++++++ 15 files changed, 446 insertions(+), 68 deletions(-) diff --git a/polylogue/sources/live/batch.py b/polylogue/sources/live/batch.py index 659413a62e..aab85bb632 100644 --- a/polylogue/sources/live/batch.py +++ b/polylogue/sources/live/batch.py @@ -71,6 +71,7 @@ from polylogue.sources.decoders import JsonlDecodeError, _iter_json_stream, _ZipEntryValidator from polylogue.sources.dispatch import ( _detect_provider_from_raw_bytes, + is_jsonl_source_path, is_stream_record_provider, parse_payload, parse_stream_payload, @@ -453,7 +454,7 @@ def _blob_jsonl_has_session_evidence( provider: Provider, source_path: str, ) -> bool: - if Path(source_path).suffix.lower() != ".jsonl": + if not is_jsonl_source_path(source_path): return False try: return jsonl_session_artifact(blob_store.blob_path(blob_hash), provider=provider) is not None @@ -464,7 +465,7 @@ def _blob_jsonl_has_session_evidence( def _live_parse_stage_candidates(paths: list[Path], *, fallback_provider: Provider) -> list[LiveParseCandidate]: """Select and read eligible files for off-writer-hold pre-parse (polylogue-wf8a). - Deliberately narrow scope: only plain ``.jsonl`` provider-session files + Deliberately narrow scope: only JSONL/NDJSON provider-session files below ``_STREAMING_FULL_INGEST_BYTES`` are eligible -- exactly the branch at lines ~1377-1425 of ``_ingest_full_paths_sync`` that reads the whole payload into memory and later parses it via ``parse_payload``/ @@ -478,7 +479,7 @@ def _live_parse_stage_candidates(paths: list[Path], *, fallback_provider: Provid """ candidates: list[LiveParseCandidate] = [] for path in paths: - if path.suffix.lower() != ".jsonl": + if not is_jsonl_source_path(str(path)): continue try: stat = path.stat() @@ -1999,9 +2000,10 @@ def _ingest_full_paths_sync( continue elif ( origin_artifact_rule is None - and path.suffix.lower() != ".jsonl" + and not is_jsonl_source_path(str(path)) and path_artifact is not None and not path_artifact.parse_as_session + and stat.st_size < _STREAMING_FULL_INGEST_BYTES and not has_decoded_session_evidence(path, provider=fallback_provider) ): # Keep path-only metadata out of the generic JSON fallback, @@ -2049,7 +2051,7 @@ def _ingest_full_paths_sync( current_path=path, source_payload_read_bytes=source_payload_read_bytes, ) - elif path.suffix.lower() == ".jsonl": + elif is_jsonl_source_path(str(path)): provider, parse_as_session = _jsonl_provider_and_session_artifact(path, fallback_provider) source_name = provider.value # An unknown JSONL cannot be safely excluded from acquire: the diff --git a/polylogue/sources/live/batch_observability.py b/polylogue/sources/live/batch_observability.py index 6f907e6d35..8f90699fe8 100644 --- a/polylogue/sources/live/batch_observability.py +++ b/polylogue/sources/live/batch_observability.py @@ -17,6 +17,7 @@ read_peak_rss_children_mb, read_peak_rss_self_mb, ) +from polylogue.storage.archive_identity import resolve_active_index_path def record_attempt_progress( @@ -126,7 +127,7 @@ def session_ids_for_source_path(path: Path, *, archive_root: Path | None = None) def _schema_archive_session_ids_for_source_path(archive_root: Path, path: Path) -> tuple[str, ...]: - index_db = archive_root / "index.db" + index_db = resolve_active_index_path(archive_root) source_db = archive_root / "source.db" if not index_db.exists() or not source_db.exists(): return () diff --git a/polylogue/sources/live/batch_support.py b/polylogue/sources/live/batch_support.py index cd4ff8efe7..13ee51035f 100644 --- a/polylogue/sources/live/batch_support.py +++ b/polylogue/sources/live/batch_support.py @@ -18,7 +18,7 @@ from polylogue.core.json import JSONDecodeError, JSONValue from polylogue.core.json import loads as json_loads from polylogue.pipeline.services.ingest_batch._core import _select_ingest_worker_count -from polylogue.sources.dispatch import _detect_provider_from_raw_bytes, detect_provider +from polylogue.sources.dispatch import _detect_provider_from_raw_bytes, detect_provider, is_jsonl_source_path from polylogue.sources.parsers import hermes_state, hermes_verification from polylogue.storage.runtime import RawSessionRecord @@ -556,7 +556,7 @@ def _detect_provider_from_path_sample(path: Path, fallback_provider: Provider) - path ): return Provider.HERMES - if path.suffix.lower() == ".jsonl": + if is_jsonl_source_path(str(path)): records = _jsonl_sample_from_path(path) if records: return detect_provider(records) or fallback_provider @@ -591,19 +591,19 @@ def _parse_path_as_session_artifact(path: Path, *, provider: Provider) -> bool: or hermes_verification.looks_like_verification_evidence_db_path(path) ): return True - if path.suffix.lower() == ".jsonl": + if is_jsonl_source_path(str(path)): if jsonl_session_artifact(path, provider=provider) is not None: return True path_classification = classify_artifact_path(path, provider=provider) return path_classification.parse_as_session if path_classification is not None else False - path_classification = classify_artifact_path(path, provider=provider) - if path_classification is not None: - return path_classification.parse_as_session if _path_size(path) > _STREAMING_FULL_INGEST_BYTES: browser_capture, _browser_provider = _browser_capture_prefix_probe(path) if browser_capture: return True return _large_non_jsonl_path_can_stream(path, provider=provider) + path_classification = classify_artifact_path(path, provider=provider) + if path_classification is not None: + return path_classification.parse_as_session try: document = json_loads(path.read_bytes()) except JSONDecodeError: @@ -638,7 +638,7 @@ def _parse_payload_as_session_artifact(path: Path, *, provider: Provider, payloa return hermes_state.looks_like_state_db_path( path ) or hermes_verification.looks_like_verification_evidence_db_path(path) - if path.suffix.lower() == ".jsonl": + if is_jsonl_source_path(str(path)): if jsonl_session_artifact(payload, provider=provider) is not None: return True path_classification = classify_artifact_path(path, provider=provider) diff --git a/polylogue/sources/live/convergence_debt_retry.py b/polylogue/sources/live/convergence_debt_retry.py index cab92122e2..22f4730d2c 100644 --- a/polylogue/sources/live/convergence_debt_retry.py +++ b/polylogue/sources/live/convergence_debt_retry.py @@ -6,6 +6,7 @@ from datetime import UTC, datetime from pathlib import Path +from polylogue.storage.archive_identity import resolve_active_index_path from polylogue.storage.introspection import table_exists as _table_exists _HOT_INSIGHT_DEFERRED = "insights deferred until source quiet" @@ -69,7 +70,7 @@ def convergence_debt_source_path( def _archive_convergence_debt_source_path_from_root(archive_root: Path, session_id: str) -> Path | None: - index_db = archive_root / "index.db" + index_db = resolve_active_index_path(archive_root) source_db = archive_root / "source.db" if not index_db.exists() or not source_db.exists(): return None diff --git a/polylogue/sources/live/watcher.py b/polylogue/sources/live/watcher.py index 68f2ec0074..7496ead9a8 100644 --- a/polylogue/sources/live/watcher.py +++ b/polylogue/sources/live/watcher.py @@ -447,7 +447,7 @@ def _schedule_hook_spool_directory_retry(self, directory: Path) -> None: task.add_done_callback(lambda _task: self._hook_spool_directory_retry_tasks.pop(directory, None)) async def _retry_hook_spool_directory_until_populated(self, directory: Path) -> None: - """Wait for an added shard's first JSON envelope, then drain once.""" + """Wait for an added shard's first envelope until it is acknowledged.""" deadline = asyncio.get_running_loop().time() + _HOOK_SPOOL_DIRECTORY_RETRY_MAX_SECONDS delay_s = _HOOK_SPOOL_DIRECTORY_RETRY_POLL_S @@ -457,7 +457,8 @@ async def _retry_hook_spool_directory_until_populated(self, directory: Path) -> return if any(directory.glob("*.json")): await self._drain_hook_spool() - return + if not any(directory.glob("*.json")): + return except OSError: return await asyncio.sleep(delay_s) @@ -1695,9 +1696,8 @@ def _watch_filter(self, _change: object, path: str) -> bool: the gate instead. """ observed_path = Path(path) - return ( - self._canonical_watch_path(observed_path) is not None - or self._source_for_directory(observed_path) is not None + return self._canonical_watch_path(observed_path) is not None or ( + observed_path.is_dir() and self._source_for_directory(observed_path) is not None ) diff --git a/polylogue/sources/source_parsing.py b/polylogue/sources/source_parsing.py index 3214b4e589..93d580c78f 100644 --- a/polylogue/sources/source_parsing.py +++ b/polylogue/sources/source_parsing.py @@ -22,6 +22,7 @@ from .cursor import _log_source_iteration_summary, _ParseContext, _record_cursor_failure from .decoders import _process_zip from .dispatch import GROUP_PROVIDERS as _GROUP_PROVIDERS +from .dispatch import is_jsonl_source_path from .emitter import _SessionEmitter from .parsers import antigravity, hermes_state, hermes_verification from .parsers.base import ParsedSession, RawSessionData @@ -35,7 +36,7 @@ def has_decoded_session_evidence(path: Path, *, provider: Provider) -> bool: """Return whether decoded JSON content outranks a non-session path rule.""" - if path.suffix.lower() == ".jsonl": + if is_jsonl_source_path(str(path)): return jsonl_session_artifact(path, provider=provider) is not None if path.suffix.lower() != ".json": diff --git a/polylogue/storage/raw_retention.py b/polylogue/storage/raw_retention.py index 38ba716df5..5dcb9708f4 100644 --- a/polylogue/storage/raw_retention.py +++ b/polylogue/storage/raw_retention.py @@ -1406,8 +1406,6 @@ def _check_broken_active_chains( for seed_raw_id in sorted(seed_raw_ids): seed_heads = heads_by_raw_id.get(seed_raw_id, []) row = rows_by_id.get(seed_raw_id) - if seed_raw_id in semantic_only_raw_ids: - continue if row is None and not seed_heads: # Directly missing sessions.raw_id rows are counted once by the # canonical lost-source-evidence projection. There is no chain to @@ -1417,6 +1415,8 @@ def _check_broken_active_chains( try: if row is None: raise RawRetentionSafetyError(f"active index raw is missing from source tier: {seed_raw_id}") + if seed_raw_id in semantic_only_raw_ids: + continue for head in seed_heads: if head.accepted_frontier_kind == "byte": _validate_byte_head(row, head) @@ -1628,17 +1628,16 @@ def _source_paths_for_paths(conn: sqlite3.Connection, source_paths: set[str]) -> def _terminal_artifact_paths(conn: sqlite3.Connection, source_paths: set[str]) -> set[str]: - """Return paths whose current raw observation is typed non-session evidence. + """Return paths whose every current source coordinate is terminal evidence. A full-route cursor can legitimately advance over a workflow/fact artifact that has no session head. ``raw_artifacts.parse_as_session = 0`` is the source-tier terminal authority for that case. Ordinary artifact upserts retain the source coordinate's latest receipt while ``raw_sessions`` - retains its historical acquisition evidence, so authority attaches to the - newest raw observation rather than requiring a duplicate receipt on every - historical raw. A later conversational raw cannot inherit the exemption: - it becomes the newest observation and leaves the path without terminal - authority until it gains a comparable accepted head. + retains its historical acquisition evidence, so authority attaches to each + coordinate's newest raw observation rather than requiring a duplicate + receipt on every historical raw. Every ``(origin, source_index)`` member + of a physical path must be terminal before the cursor path is exempt. """ result: set[str] = set() @@ -1649,18 +1648,40 @@ def _terminal_artifact_paths(conn: sqlite3.Connection, source_paths: set[str]) - placeholders = ", ".join("?" for _ in batch) rows = conn.execute( f""" - SELECT DISTINCT artifact.source_path + SELECT DISTINCT terminal_raw.source_path FROM raw_artifacts AS artifact JOIN raw_sessions AS terminal_raw ON terminal_raw.raw_id = artifact.raw_id WHERE artifact.parse_as_session = 0 - AND artifact.source_path IN ({placeholders}) + AND terminal_raw.source_path IN ({placeholders}) AND terminal_raw.raw_id = ( SELECT newest.raw_id FROM raw_sessions AS newest - WHERE newest.source_path = artifact.source_path + WHERE newest.source_path = terminal_raw.source_path + AND newest.origin = terminal_raw.origin + AND newest.source_index = terminal_raw.source_index ORDER BY newest.acquired_at_ms DESC, newest.rowid DESC LIMIT 1 ) + AND NOT EXISTS ( + SELECT 1 + FROM raw_sessions AS coordinate + WHERE coordinate.source_path = terminal_raw.source_path + AND coordinate.raw_id = ( + SELECT newest.raw_id + FROM raw_sessions AS newest + WHERE newest.source_path = coordinate.source_path + AND newest.origin = coordinate.origin + AND newest.source_index = coordinate.source_index + ORDER BY newest.acquired_at_ms DESC, newest.rowid DESC + LIMIT 1 + ) + AND NOT EXISTS ( + SELECT 1 + FROM raw_artifacts AS current_artifact + WHERE current_artifact.raw_id = coordinate.raw_id + AND current_artifact.parse_as_session = 0 + ) + ) """, batch, ).fetchall() @@ -1675,13 +1696,18 @@ def _terminal_artifact_raw_ids(conn: sqlite3.Connection) -> frozenset[str]: terminal_paths = _terminal_artifact_paths(conn, source_paths) if not terminal_paths: return frozenset() - ordered_paths = tuple(sorted(terminal_paths)) - placeholders = ", ".join("?" for _ in ordered_paths) - rows = conn.execute( - f"SELECT raw_id FROM raw_sessions WHERE source_path IN ({placeholders})", - ordered_paths, - ).fetchall() - return frozenset(str(row[0]) for row in rows) + raw_ids: set[str] = set() + pending = set(terminal_paths) + while pending: + batch = tuple(sorted(pending)[:500]) + pending.difference_update(batch) + placeholders = ", ".join("?" for _ in batch) + rows = conn.execute( + f"SELECT raw_id FROM raw_sessions WHERE source_path IN ({placeholders})", + batch, + ).fetchall() + raw_ids.update(str(row[0]) for row in rows) + return frozenset(raw_ids) def _ops_cursor_byte_offsets(ops_db_path: Path) -> dict[str, _OpsCursorAuthority]: diff --git a/polylogue/storage/repair.py b/polylogue/storage/repair.py index 6429abffa5..cc23d0a02c 100644 --- a/polylogue/storage/repair.py +++ b/polylogue/storage/repair.py @@ -3772,8 +3772,8 @@ def _raw_materialization_archive_root(config: Config) -> Path: def _raw_materialization_index_path(config: Config, archive_root: Path) -> Path: - """Return the active derived tier while keeping durable tiers at root.""" - return config.db_path if config.db_path.name == "index.db" else archive_root / "index.db" + """Return an explicit index override or the archive's active generation.""" + return config.db_path if config.db_path.name == "index.db" else resolve_active_index_path(archive_root) def _raw_artifact_coordinate_predicate(*, artifact_alias: str, raw_alias: str) -> str: @@ -5890,7 +5890,7 @@ def repair_superseded_raw_snapshots(config: Config, dry_run: bool = False) -> Re archive_root = _raw_materialization_archive_root(config) repair_db_path = archive_root / "source.db" if repair_db_path.exists(): - index_db_path = resolve_active_index_path(archive_root) + index_db_path = _raw_materialization_index_path(config, archive_root) with closing(open_connection(repair_db_path)) as conn, conn: conn.row_factory = sqlite3.Row try: diff --git a/tests/unit/sources/test_convergence_debt_deferred_vocabulary.py b/tests/unit/sources/test_convergence_debt_deferred_vocabulary.py index e62422944a..3694b1e480 100644 --- a/tests/unit/sources/test_convergence_debt_deferred_vocabulary.py +++ b/tests/unit/sources/test_convergence_debt_deferred_vocabulary.py @@ -32,8 +32,63 @@ convergence_debt_from_states, is_deferred_stage_state, ) +from polylogue.sources.live.convergence_debt_retry import convergence_debt_source_path from polylogue.sources.live.convergence_outcome import record_convergence_outcome from polylogue.sources.live.cursor import CursorStore +from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_archive_database +from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier + + +def test_convergence_debt_lookups_follow_the_active_index_generation(tmp_path: Path) -> None: + """Outcome and retry lookups ignore a stale conventional index database.""" + + source_db = tmp_path / "source.db" + shadow_index = tmp_path / "index.db" + active_index = tmp_path / "generations" / "active" / "index.db" + initialize_archive_database(source_db, ArchiveTier.SOURCE) + initialize_archive_database(shadow_index, ArchiveTier.INDEX) + initialize_archive_database(active_index, ArchiveTier.INDEX) + source_path = tmp_path / "active.jsonl" + source_path.write_text("{}", encoding="utf-8") + with sqlite3.connect(source_db) as conn: + conn.execute( + """ + INSERT INTO raw_sessions ( + raw_id, origin, native_id, source_path, source_index, blob_hash, blob_size, acquired_at_ms + ) VALUES ('raw-active', 'codex-session', 'active', ?, 0, ?, 1, 1) + """, + (str(source_path), bytes(32)), + ) + conn.commit() + with sqlite3.connect(active_index) as conn: + conn.execute( + """ + INSERT INTO sessions (native_id, origin, raw_id, title, content_hash) + VALUES ('active', 'codex-session', 'raw-active', 'active', ?) + """, + (bytes(32),), + ) + conn.commit() + (tmp_path / ".index-active-pointer").write_text(f"{active_index}\n", encoding="utf-8") + cursor = CursorStore(tmp_path / "ops.db") + debt = ConvergenceDebt(path=source_path, stage="fts", error="deferred", deferred=True) + + record_convergence_outcome(cursor, source_path, (debt,), archive_root=tmp_path) + with sqlite3.connect(tmp_path / "ops.db") as conn: + session_debts = conn.execute( + "SELECT target_id FROM convergence_debt WHERE target_type = 'session_id'" + ).fetchall() + assert ( + convergence_debt_source_path( + conn, + subject_type="session_id", + subject_id="codex-session:active", + archive_root=tmp_path, + ) + == source_path + ) + + assert session_debts == [("codex-session:active",)] def test_is_deferred_stage_state_true_only_for_pending() -> None: diff --git a/tests/unit/sources/test_hook_spool.py b/tests/unit/sources/test_hook_spool.py index 60613665c8..ab6a7c5e78 100644 --- a/tests/unit/sources/test_hook_spool.py +++ b/tests/unit/sources/test_hook_spool.py @@ -308,6 +308,7 @@ async def emit_added_shard(*roots: Path, **_kwargs: object) -> AsyncIterator[set assert conn.execute("SELECT session_native_id FROM raw_hook_events").fetchone() == ("session-1",) +@pytest.mark.uses_real_clock("exercises production retry polling against a delayed atomic hook publish") @pytest.mark.asyncio async def test_live_watcher_retries_added_hook_shard_until_atomic_publish( tmp_path: Path, @@ -324,6 +325,8 @@ async def test_live_watcher_retries_added_hook_shard_until_atomic_publish( (WatchSource(name="hooks", root=pending, suffixes=(".json",)),), cursor=CursorStore(archive_root / "ops.db"), ) + monkeypatch.setattr("polylogue.sources.hooks._day_shard", lambda: "2026-08-12") + shard = pending / "2026-08-12" publish_task: asyncio.Task[None] | None = None async def publish_after_fixed_grace() -> None: @@ -343,7 +346,6 @@ async def publish_after_fixed_grace() -> None: async def emit_empty_shard(*roots: Path, **_kwargs: object) -> AsyncIterator[set[tuple[Change, str]]]: nonlocal publish_task assert roots == (pending,) - shard = pending / "2026-08-11" shard.mkdir(parents=True) publish_task = asyncio.create_task(publish_after_fixed_grace()) yield {(Change.added, str(shard))} @@ -353,18 +355,47 @@ async def emit_empty_shard(*roots: Path, **_kwargs: object) -> AsyncIterator[set await watcher._watch_changes([pending]) assert publish_task is not None await publish_task - - for _ in range(30): - if list(acknowledged_hook_spool_dir(spool_root).rglob("published-after-directory-event.json")): - break - await asyncio.sleep(0.01) - watcher.stop() + retry_task = watcher._hook_spool_directory_retry_tasks[shard.resolve()] + await asyncio.wait_for(retry_task, timeout=1.0) assert list(acknowledged_hook_spool_dir(spool_root).rglob("published-after-directory-event.json")) != [] with sqlite3.connect(archive_root / "source.db") as conn: assert conn.execute("SELECT session_native_id FROM raw_hook_events").fetchone() == ("session-2",) +@pytest.mark.uses_real_clock("exercises retry polling while a pending hook envelope remains unacknowledged") +@pytest.mark.asyncio +async def test_hook_shard_retry_waits_for_durable_acknowledgement( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A failed drain keeps the directory retry alive until the envelope moves.""" + + directory = tmp_path / "hooks" / "pending" / "2026-08-12" + directory.mkdir(parents=True) + envelope = directory / "retry.json" + envelope.write_text("{}", encoding="utf-8") + watcher = LiveWatcher( + cast(Any, SimpleNamespace(archive_root=tmp_path / "archive", backend=None)), + (WatchSource(name="hooks", root=directory.parent, suffixes=(".json",)),), + cursor=CursorStore(tmp_path / "ops.db"), + ) + drains = 0 + + async def drain_until_acknowledged() -> None: + nonlocal drains + drains += 1 + if drains == 2: + envelope.unlink() + + monkeypatch.setattr(watcher, "_drain_hook_spool", drain_until_acknowledged) + + await watcher._retry_hook_spool_directory_until_populated(directory) + + assert drains == 2 + assert not envelope.exists() + + def test_hook_spool_retains_sqlite_failures_for_retry( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/sources/test_live_batch_support.py b/tests/unit/sources/test_live_batch_support.py index 513df715ce..575313d62f 100644 --- a/tests/unit/sources/test_live_batch_support.py +++ b/tests/unit/sources/test_live_batch_support.py @@ -457,7 +457,7 @@ def test_full_ingest_empty_jsonl_is_not_misclassified_as_truncated( def test_full_ingest_unknown_export_without_sessions_records_terminal_evidence(tmp_path: Path) -> None: - root = tmp_path / "unknown" + root = tmp_path / "chatgpt" root.mkdir() path = root / "export.jsonl" path.write_bytes(b"") @@ -477,6 +477,29 @@ def test_full_ingest_unknown_export_without_sessions_records_terminal_evidence(t assert artifact == ("terminal_unknown_export_no_session", "unsupported_parseable", 0) +def test_full_ingest_unknown_weak_path_ndjson_records_terminal_evidence(tmp_path: Path) -> None: + """NDJSON takes the same strict terminal classification route as JSONL.""" + + root = tmp_path / "chatgpt" + path = root / "analysis" / "export.ndjson" + path.parent.mkdir(parents=True) + path.write_bytes(b"") + db_path = tmp_path / "archive.sqlite" + processor = LiveBatchProcessor( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=db_path))), + (WatchSource(name="unknown", root=root, suffixes=(".jsonl", ".ndjson")),), + cursor=CursorStore(db_path), + parser_fingerprint="test-parser", + ) + + result = processor._ingest_full_paths_sync([path], source_name="unknown") + + assert result.succeeded == [path] + with sqlite3.connect(tmp_path / "source.db") as conn: + artifact = conn.execute("SELECT artifact_kind, parse_as_session FROM raw_artifacts").fetchone() + assert artifact == ("terminal_unknown_export_no_session", 0) + + def test_full_ingest_unknown_malformed_jsonl_records_terminal_decode_and_stops_retrying(tmp_path: Path) -> None: """Complete malformed JSONL lines are terminal decode evidence, not no-session evidence.""" root = tmp_path / "unknown" @@ -923,6 +946,76 @@ def test_streaming_sized_full_ingest_uses_archive( assert conn.execute("SELECT COUNT(*) FROM raw_sessions").fetchone()[0] == 1 +def test_large_weak_path_uses_streaming_route_before_decoded_evidence( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A weak path cannot force an eager whole-file evidence decode.""" + + root = tmp_path / "unknown" + path = root / "analysis" / "export.json" + path.parent.mkdir(parents=True) + path.write_bytes( + json.dumps( + { + "id": "weak-large", + "title": "weak large export", + "create_time": 1781442866.0, + "update_time": 1781442966.0, + "current_node": "assistant-node", + "mapping": { + "root": {"id": "root", "message": None, "parent": None, "children": ["user-node"]}, + "user-node": { + "id": "user-node", + "parent": "root", + "children": ["assistant-node"], + "message": { + "id": "weak-u1", + "author": {"role": "user"}, + "content": {"content_type": "text", "parts": ["question"]}, + "metadata": {}, + }, + }, + "assistant-node": { + "id": "assistant-node", + "parent": "user-node", + "children": [], + "message": { + "id": "weak-a1", + "author": {"role": "assistant"}, + "content": {"content_type": "text", "parts": ["answer"]}, + "metadata": {}, + }, + }, + }, + } + ).encode() + + (b" " * (9 * 1024 * 1024)) + ) + db_path = tmp_path / "archive.sqlite" + processor = LiveBatchProcessor( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=db_path))), + (WatchSource(name="chatgpt", root=root, suffixes=(".json",)),), + cursor=CursorStore(db_path), + parser_fingerprint="test-parser", + ) + monkeypatch.setattr("polylogue.sources.live.batch._STREAMING_FULL_INGEST_BYTES", 1) + monkeypatch.setattr("polylogue.sources.live.batch_support._STREAMING_FULL_INGEST_BYTES", 1) + monkeypatch.setattr( + "polylogue.sources.live.batch.has_decoded_session_evidence", + lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("large input decoded before streaming route")), + ) + phases: list[str] = [] + + def heartbeat(phase: str, **_kwargs: object) -> None: + phases.append(phase) + + result = processor._ingest_full_paths_sync([path], source_name="chatgpt", heartbeat=heartbeat) + + assert result.failed == [] + assert "full_blob_copy" in phases + + def test_full_ingest_writes_archive_with_route_observability( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/sources/test_live_watcher.py b/tests/unit/sources/test_live_watcher.py index 371bf2459e..8ccc2fe6a9 100644 --- a/tests/unit/sources/test_live_watcher.py +++ b/tests/unit/sources/test_live_watcher.py @@ -1574,6 +1574,25 @@ def test_hermes_wal_revision_triggers_resnapshot_and_maps_sidecar_event(tmp_path writer.close() +def test_watch_filter_accepts_directories_but_not_unmatched_files_under_broad_roots(tmp_path: Path) -> None: + """The watch backend wakes only for source suffixes or real directories.""" + + root = tmp_path / "codex-state" + root.mkdir() + unmatched = root / "history.log" + unmatched.write_text("noise", encoding="utf-8") + child_directory = root / "new-session" + child_directory.mkdir() + watcher, _full_ingest = _make_watcher( + tmp_path, + root, + sources=(WatchSource(name="codex-state", root=root, suffixes=(".jsonl",)),), + ) + + assert watcher._watch_filter(object(), str(unmatched)) is False + assert watcher._watch_filter(object(), str(child_directory)) is True + + def test_hermes_cursor_records_acquisition_revision_not_live_tail(tmp_path: Path) -> None: root = tmp_path / "hermes" root.mkdir() diff --git a/tests/unit/sources/test_live_watcher_locking.py b/tests/unit/sources/test_live_watcher_locking.py index a6ba716848..a38831a453 100644 --- a/tests/unit/sources/test_live_watcher_locking.py +++ b/tests/unit/sources/test_live_watcher_locking.py @@ -31,11 +31,11 @@ def _make_watcher(tmp_path: Path, root: Path, *, debounce_s: float = 0.01) -> Li @pytest.mark.parametrize("route", ["append", "full"]) +@pytest.mark.uses_real_clock("requires a bounded subprocess exit deadline while the injected writer remains blocked") def test_real_watcher_writer_routes_cannot_pin_process_exit(route: str) -> None: script = textwrap.dedent( f""" import asyncio - import contextlib import tempfile import threading from pathlib import Path @@ -68,11 +68,9 @@ async def main() -> None: watcher._parse_stage = None watcher._batch_processor._parse_stage = None started = threading.Event() - release = threading.Event() - def stuck(*args, **kwargs): started.set() - release.wait() + threading.Event().wait() if {route!r} == "append": stat = path.stat() @@ -99,21 +97,18 @@ def stuck(*args, **kwargs): while not started.is_set(): await asyncio.sleep(0.001) caller.cancel() - with contextlib.suppress(asyncio.CancelledError): - await caller - try: - assert await coordinator.shutdown(timeout=0.01) is False - finally: - # The injected thread is intentionally unlike production - # parsing: it has no natural completion condition. Releasing - # it after the coordinator's fail-safe result proves the - # process-exit assertion without making asyncio's executor - # shutdown permanently unreleasable. - release.set() - assert await coordinator.shutdown(timeout=1.0) is True - watcher.stop() - - asyncio.run(main()) + assert await coordinator.shutdown(timeout=0.01) is False + # The injected writer remains blocked through interpreter + # termination. A non-daemon bridge thread would pin this + # subprocess after the loop closes. + watcher.stop() + + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + try: + loop.run_until_complete(main()) + finally: + loop.close() """ ) diff --git a/tests/unit/storage/test_raw_retention.py b/tests/unit/storage/test_raw_retention.py index bfaf5861e5..3c10d035e9 100644 --- a/tests/unit/storage/test_raw_retention.py +++ b/tests/unit/storage/test_raw_retention.py @@ -646,6 +646,86 @@ def test_current_terminal_artifact_authorizes_historical_raws_but_not_later_sess assert snapshot.cursor_authority_gap_samples[0].state == "source_raws_without_accepted_head" +def test_terminal_cursor_exemption_requires_every_source_coordinate(tmp_path: Path) -> None: + """A terminal sibling cannot hide an unheaded conversational coordinate.""" + + source_db = tmp_path / "source.db" + index_db = tmp_path / "index.db" + ops_db = tmp_path / "ops.db" + source_path = tmp_path / "bundle.jsonl" + initialize_archive_database(source_db, ArchiveTier.SOURCE) + initialize_archive_database(index_db, ArchiveTier.INDEX) + with sqlite3.connect(source_db) as conn: + conn.executemany( + """ + INSERT INTO raw_sessions ( + raw_id, origin, native_id, source_path, source_index, blob_hash, blob_size, acquired_at_ms + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + ("raw-terminal", "claude-code-session", "terminal", str(source_path), 0, bytes(32), 1, 1), + ("raw-session", "claude-code-session", "session", str(source_path), 1, bytes(1) * 32, 1, 2), + ), + ) + conn.execute( + """ + INSERT INTO raw_artifacts ( + artifact_id, raw_id, origin, source_path, source_index, artifact_kind, + support_status, classification_reason, parse_as_session, schema_eligible, + malformed_jsonl_lines, first_observed_at_ms, last_observed_at_ms + ) VALUES (?, ?, ?, ?, ?, 'workflow_journal', 'unknown', 'terminal coordinate', 0, 0, 0, 1, 1) + """, + ("artifact-terminal", "raw-terminal", "claude-code-session", str(source_path), 0), + ) + conn.commit() + _seed_ops_cursor(ops_db, source_path=source_path, byte_offset=2) + + with sqlite3.connect(source_db) as conn: + snapshot = raw_frontier_integrity_snapshot(conn, index_db_path=index_db, ops_db_path=ops_db) + + assert snapshot.cursor_ahead_status == "unknown" + assert snapshot.cursor_authority_gap_count == 1 + assert snapshot.cursor_authority_gap_samples[0].state == "source_raws_without_accepted_head" + + +def test_terminal_artifact_retention_batches_source_paths_below_sqlite_limit(tmp_path: Path) -> None: + """Terminal evidence remains protectable when more than one SQL batch is needed.""" + + source_db = tmp_path / "source.db" + index_db = tmp_path / "index.db" + initialize_archive_database(source_db, ArchiveTier.SOURCE) + initialize_archive_database(index_db, ArchiveTier.INDEX) + with sqlite3.connect(source_db) as conn: + for number in range(501): + source_path = tmp_path / f"terminal-{number}.jsonl" + raw_id = f"raw-terminal-{number}" + conn.execute( + """ + INSERT INTO raw_sessions ( + raw_id, origin, native_id, source_path, source_index, blob_hash, blob_size, acquired_at_ms + ) VALUES (?, 'claude-code-session', ?, ?, 0, ?, 1, ?) + """, + (raw_id, raw_id, str(source_path), number.to_bytes(32, "big"), number), + ) + conn.execute( + """ + INSERT INTO raw_artifacts ( + artifact_id, raw_id, origin, source_path, source_index, artifact_kind, + support_status, classification_reason, parse_as_session, schema_eligible, + malformed_jsonl_lines, first_observed_at_ms, last_observed_at_ms + ) VALUES (?, ?, 'claude-code-session', ?, 0, 'workflow_journal', + 'unknown', 'terminal', 0, 0, 0, ?, ?) + """, + (f"artifact-{number}", raw_id, str(source_path), number, number), + ) + conn.commit() + conn.setlimit(sqlite3.SQLITE_LIMIT_VARIABLE_NUMBER, 500) + authority = active_raw_retention_authority(conn, index_db_path=index_db) + + assert authority.protected_raw_ids == frozenset(f"raw-terminal-{number}" for number in range(501)) + assert authority.eligible_raw_ids == frozenset() + + def test_active_raw_protection_rejects_incomplete_predecessor_chain(tmp_path: Path) -> None: source_db = tmp_path / "source.db" index_db = tmp_path / "index.db" @@ -1858,6 +1938,38 @@ def test_raw_frontier_integrity_semantic_membership_cursor_is_intentionally_not_ assert snapshot.overall_status == "healthy" +def test_raw_frontier_integrity_reports_missing_semantic_head_source_raw(tmp_path: Path) -> None: + """Semantic membership skips byte validation only after finding its source raw.""" + + source_db = tmp_path / "source.db" + index_db = tmp_path / "index.db" + ops_db = tmp_path / "ops.db" + initialize_archive_database(source_db, ArchiveTier.SOURCE) + initialize_archive_database(index_db, ArchiveTier.INDEX) + _seed_index_authority( + index_db, + session_raw_id="raw-missing-semantic", + accepted_raw_id="raw-missing-semantic", + accepted_revision="semantic-revision", + generation=0, + frontier=1, + append_end_offset=None, + ) + with sqlite3.connect(index_db) as conn: + conn.execute("UPDATE raw_revision_heads SET accepted_frontier_kind = 'semantic'") + conn.commit() + initialize_archive_database(ops_db, ArchiveTier.OPS) + + with sqlite3.connect(source_db) as conn: + snapshot = raw_frontier_integrity_snapshot(conn, index_db_path=index_db, ops_db_path=ops_db) + + assert snapshot.broken_head_status == "violated" + assert snapshot.broken_head_count == 1 + assert snapshot.broken_head_checked_count == 1 + assert snapshot.broken_head_samples[0].accepted_raw_id == "raw-missing-semantic" + assert "missing from source tier" in snapshot.broken_head_samples[0].reason + + def test_raw_frontier_integrity_snapshot_cursor_at_exact_accepted_frontier_is_healthy(tmp_path: Path) -> None: """A cursor sitting exactly at the accepted frontier (not past it) is healthy. diff --git a/tests/unit/storage/test_repair.py b/tests/unit/storage/test_repair.py index 9469920808..5ea5a0bd50 100644 --- a/tests/unit/storage/test_repair.py +++ b/tests/unit/storage/test_repair.py @@ -1512,6 +1512,48 @@ def test_superseded_raw_cleanup_follows_active_index_pointer(tmp_path: Path) -> assert "skipped 1 active revision raw rows" in result.detail +def test_superseded_raw_cleanup_preserves_explicit_index_override(tmp_path: Path) -> None: + """An explicit generation remains cleanup authority even when a pointer differs.""" + + initialize_archive_database(tmp_path / "source.db", ArchiveTier.SOURCE) + pointer_index = tmp_path / "generations" / "pointer" / "index.db" + explicit_index = tmp_path / "generations" / "explicit" / "index.db" + initialize_archive_database(pointer_index, ArchiveTier.INDEX) + initialize_archive_database(explicit_index, ArchiveTier.INDEX) + source_file = tmp_path / "source.jsonl" + source_file.write_text("{}", encoding="utf-8") + with sqlite3.connect(tmp_path / "source.db") as source_conn: + source_conn.executemany( + """ + INSERT INTO raw_sessions ( + raw_id, origin, native_id, source_path, source_index, blob_hash, blob_size, acquired_at_ms + ) VALUES (?, 'chatgpt-export', ?, ?, 0, ?, ?, ?) + """, + ( + ("raw-explicit", "native-explicit", str(source_file), bytes.fromhex("11" * 32), 10, 1), + ("raw-newer", "native-newer", str(source_file), bytes.fromhex("22" * 32), 11, 2), + ), + ) + source_conn.commit() + with sqlite3.connect(explicit_index) as index_conn: + index_conn.execute( + """ + INSERT INTO sessions (native_id, origin, raw_id, title, content_hash) + VALUES ('native-explicit', 'chatgpt-export', 'raw-explicit', 'explicit', ?) + """, + (bytes(32),), + ) + index_conn.commit() + (tmp_path / ".index-active-pointer").write_text(f"{pointer_index}\n", encoding="utf-8") + config = Config(archive_root=tmp_path, render_root=tmp_path, sources=[], db_path=explicit_index) + + result = repair_mod.repair_superseded_raw_snapshots(config, dry_run=True) + + assert result.success is True + assert result.repaired_count == 0 + assert "skipped 1 active revision raw rows" in result.detail + + def test_superseded_raw_cleanup_allows_history_before_active_full(tmp_path: Path) -> None: config = _config(tmp_path) initialize_archive_database(tmp_path / "source.db", ArchiveTier.SOURCE) From 22f4ab28d25d3846e91e311f6ab9e0779626aa55 Mon Sep 17 00:00:00 2001 From: Sinity Date: Wed, 12 Aug 2026 23:55:44 +0200 Subject: [PATCH 08/65] test(live): align synthetic archive authority fixtures Bootstrap full synthetic archives and assert the durable census and cursor-authority contracts used by production ingest. --- tests/unit/sources/test_live_batch_support.py | 115 +++++--------- tests/unit/storage/test_repair.py | 149 ++++++++---------- 2 files changed, 106 insertions(+), 158 deletions(-) diff --git a/tests/unit/sources/test_live_batch_support.py b/tests/unit/sources/test_live_batch_support.py index 575313d62f..fb4b211234 100644 --- a/tests/unit/sources/test_live_batch_support.py +++ b/tests/unit/sources/test_live_batch_support.py @@ -30,6 +30,7 @@ from polylogue.sources.live.append_ingest import ingest_append_plans from polylogue.sources.live.batch import ( _MAX_APPEND_PLAN_PAYLOAD_BYTES, + CursorAuthorityBlockedError, LiveBatchProcessor, _ArchiveFullWriteResult, append_capability_receipt, @@ -102,8 +103,6 @@ def test_append_capability_receipt_is_keyed_to_live_identity_contract( initialize_active_archive_root, initialize_archive_database, ) -from polylogue.storage.sqlite.archive_tiers.index import INDEX_SCHEMA_VERSION -from polylogue.storage.sqlite.archive_tiers.source import SOURCE_SCHEMA_VERSION from polylogue.storage.sqlite.archive_tiers.source_write import ( ArchiveSourceArtifact, read_archive_raw_session_envelope, @@ -114,6 +113,13 @@ def test_append_capability_receipt_is_keyed_to_live_identity_contract( _ARCHIVE_STORAGE_TIERS = ",".join(spec.tier.value for spec in ARCHIVE_TIER_SPECS.values()) +def _complete_archive_storage_probe_fields() -> dict[str, object]: + return _archive_storage_probe_fields( + present={spec.tier for spec in ARCHIVE_TIER_SPECS.values()}, + versions={spec.tier: spec.version for spec in ARCHIVE_TIER_SPECS.values()}, + ) + + def _archive_storage_probe_fields( *, present: set[ArchiveTier], @@ -714,7 +720,7 @@ def grow_source_after_capture(**kwargs: object) -> bool: assert artifact == ("deferred_claude_code_partial_jsonl", "partial_decode", 1) -def test_streamed_incomplete_jsonl_capture_defers_then_replays_completed_source( +def test_streamed_incomplete_jsonl_capture_defers_completed_source_until_authority_recovers( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -764,16 +770,16 @@ def complete_source_after_capture(**kwargs: object) -> bool: artifact = conn.execute("SELECT artifact_kind FROM raw_artifacts ORDER BY last_observed_at_ms DESC").fetchone() assert artifact == ("deferred_hot_jsonl_capture",) - replayed = asyncio.run(processor.ingest_files([path])) + with pytest.raises(CursorAuthorityBlockedError, match="source-selection gate blocked"): + asyncio.run(processor.ingest_files([path])) - assert replayed.full_file_count == 1 - assert replayed.append_file_count == 0 - assert replayed.succeeded_file_count == 1 final_cursor = cursor.get_record(path) assert final_cursor is not None - assert final_cursor.failure_count == 0 + assert final_cursor.byte_offset == 0 + assert final_cursor.byte_size == len(completed) + assert final_cursor.deferred_end_offset is None with sqlite3.connect(index_db) as conn: - assert conn.execute("SELECT native_id FROM messages").fetchall() == [("message-0",)] + assert conn.execute("SELECT native_id FROM messages").fetchall() == [] def test_full_ingest_rejects_incomplete_jsonl_without_hot_prefix_proof( @@ -1020,9 +1026,6 @@ def test_full_ingest_writes_archive_with_route_observability( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_archive_database - from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier - root = tmp_path / "sessions" root.mkdir() source = root / "full-v1.jsonl" @@ -1033,8 +1036,7 @@ def test_full_ingest_writes_archive_with_route_observability( source.write_bytes(payload) index_db = tmp_path / "index.db" source_db = tmp_path / "source.db" - initialize_archive_database(index_db, ArchiveTier.INDEX) - initialize_archive_database(source_db, ArchiveTier.SOURCE) + initialize_active_archive_root(tmp_path) cursor = CursorStore(index_db) processor = LiveBatchProcessor( cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=index_db))), @@ -1090,14 +1092,7 @@ def heartbeat( "storage_write_tiers": "source,index", "archive_active": True, "archive_bootstrapped": False, - **_archive_storage_probe_fields( - present={ArchiveTier.SOURCE, ArchiveTier.INDEX, ArchiveTier.OPS}, - versions={ - ArchiveTier.SOURCE: SOURCE_SCHEMA_VERSION, - ArchiveTier.INDEX: INDEX_SCHEMA_VERSION, - ArchiveTier.OPS: 1, - }, - ), + **_complete_archive_storage_probe_fields(), } write_event = next(payload for phase, payload in stage_events if phase == "full_archive_write") assert write_event == { @@ -1126,9 +1121,6 @@ def test_streaming_full_ingest_writes_archive_from_blob( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_archive_database - from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier - root = tmp_path / "sessions" root.mkdir() source = root / "stream-v1.jsonl" @@ -1139,8 +1131,7 @@ def test_streaming_full_ingest_writes_archive_from_blob( source.write_bytes(payload) index_db = tmp_path / "index.db" source_db = tmp_path / "source.db" - initialize_archive_database(index_db, ArchiveTier.INDEX) - initialize_archive_database(source_db, ArchiveTier.SOURCE) + initialize_active_archive_root(tmp_path) cursor = CursorStore(index_db) processor = LiveBatchProcessor( cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=index_db))), @@ -1183,14 +1174,7 @@ def heartbeat( "storage_write_tiers": "source,index", "archive_active": True, "archive_bootstrapped": False, - **_archive_storage_probe_fields( - present={ArchiveTier.SOURCE, ArchiveTier.INDEX, ArchiveTier.OPS}, - versions={ - ArchiveTier.SOURCE: SOURCE_SCHEMA_VERSION, - ArchiveTier.INDEX: INDEX_SCHEMA_VERSION, - ArchiveTier.OPS: 1, - }, - ), + **_complete_archive_storage_probe_fields(), } write_event = next(payload for phase, payload in stage_events if phase == "full_archive_write") assert write_event == { @@ -1220,9 +1204,6 @@ def test_streaming_sized_browser_capture_json_uses_native_payload_detection( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_archive_database - from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier - root = tmp_path / "browser-capture" / "chatgpt" root.mkdir(parents=True) source = root / "native-capture.json" @@ -1288,8 +1269,7 @@ def test_streaming_sized_browser_capture_json_uses_native_payload_detection( source.write_text(json.dumps(capture_payload), encoding="utf-8") index_db = tmp_path / "index.db" source_db = tmp_path / "source.db" - initialize_archive_database(index_db, ArchiveTier.INDEX) - initialize_archive_database(source_db, ArchiveTier.SOURCE) + initialize_active_archive_root(tmp_path) cursor = CursorStore(index_db) processor = LiveBatchProcessor( cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=index_db))), @@ -1342,9 +1322,6 @@ def test_generic_large_browser_capture_json_uses_prefix_detection_without_unknow tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_archive_database - from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier - root = tmp_path / "inbox" root.mkdir() source = root / "large-browser-capture.json" @@ -1374,8 +1351,7 @@ def test_generic_large_browser_capture_json_uses_prefix_detection_without_unknow source.write_text(json.dumps(capture_payload), encoding="utf-8") index_db = tmp_path / "index.db" source_db = tmp_path / "source.db" - initialize_archive_database(index_db, ArchiveTier.INDEX) - initialize_archive_database(source_db, ArchiveTier.SOURCE) + initialize_active_archive_root(tmp_path) cursor = CursorStore(index_db) processor = LiveBatchProcessor( cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=index_db))), @@ -2782,8 +2758,6 @@ def test_live_append_chain_survives_post_ingest_compaction( protect_chain: bool, ) -> None: from polylogue.storage.blob_publication import ArchiveBlobPublisher - from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_archive_database - from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier root = tmp_path / "sessions" root.mkdir() @@ -2796,8 +2770,7 @@ def test_live_append_chain_survives_post_ingest_compaction( path.write_bytes(payload) index_db = tmp_path / "index.db" source_db = tmp_path / "source.db" - initialize_archive_database(index_db, ArchiveTier.INDEX) - initialize_archive_database(source_db, ArchiveTier.SOURCE) + initialize_active_archive_root(tmp_path) cursor = CursorStore(index_db) processor = LiveBatchProcessor( cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=index_db))), @@ -2967,11 +2940,7 @@ def test_append_ingest_proves_byte_authority_at_capture_without_reconciler(tmp_p path.write_bytes(baseline) index_db = tmp_path / "index.db" source_db = tmp_path / "source.db" - from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_archive_database - from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier - - initialize_archive_database(index_db, ArchiveTier.INDEX) - initialize_archive_database(source_db, ArchiveTier.SOURCE) + initialize_active_archive_root(tmp_path) cursor = CursorStore(index_db) processor = LiveBatchProcessor( cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=index_db))), @@ -4681,7 +4650,7 @@ def fail_bind(self: ArchiveStore, raw_id: str, revision: RawRevisionEnvelope, ** assert conn.execute("SELECT COUNT(*) FROM raw_sessions WHERE source_index = -1").fetchone() == (1,) -def test_public_full_blob_batch_bind_failure_persists_bytes_and_retries( +def test_public_full_blob_batch_bind_failure_persists_bytes_and_blocks_unsafe_retry( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -4744,13 +4713,12 @@ def fail_bind(self: ArchiveStore, raw_id: str, revision: RawRevisionEnvelope, ** ) assert isinstance(row[13], str) and "injected blob bind failure" in row[13] - retry = asyncio.run(processor.ingest_files([source], emit_event=False)) + with pytest.raises(CursorAuthorityBlockedError, match="source-selection gate blocked"): + asyncio.run(processor.ingest_files([source], emit_event=False)) - assert retry.full_file_count == 1 - assert retry.failed_file_count == 0 with sqlite3.connect(tmp_path / "source.db") as conn: assert conn.execute("SELECT COUNT(*) FROM raw_sessions").fetchone() == (1,) - bound = conn.execute( + retained = conn.execute( """ SELECT logical_source_key, revision_kind, source_revision, predecessor_source_revision, predecessor_raw_id, baseline_raw_id, @@ -4759,18 +4727,18 @@ def fail_bind(self: ArchiveStore, raw_id: str, revision: RawRevisionEnvelope, ** FROM raw_sessions """ ).fetchone() - assert bound == ( - "codex:blob-retry", + assert retained == ( + f"pending-raw:codex-session:0:{source}:{raw_id}", "full", sha256(payload).hexdigest(), None, None, - raw_id, None, None, - 0, - "byte_proven", None, + 0, + "quarantined", + row[13], ) @@ -4883,14 +4851,7 @@ def test_append_multi_session_payload_is_rejected_before_index_write( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - path = tmp_path / "append-multi.jsonl" - # Real, parseable content -- not a bare `{}` -- because polylogue-xwkh's - # declared-non-session-artifact gate now refuses that shape before this - # test's mocked parse_payload (returning two sessions) is ever reached. - payload = b'{"type":"event_msg","payload":{"type":"user_message","message":"hello"}}\n' - path.write_bytes(payload) - plan = _append_plan(path, payload, payload_hash="multi") - owner = _append_owner(tmp_path) + path, plan, owner = _seed_live_append_plan(tmp_path, native_id="append-multi") # polylogue-9ykn: a message-less ParsedSession carries no positive # conversational evidence and is refused before this test's own # "more than one session" check ever runs -- give each session one real @@ -4907,15 +4868,15 @@ def test_append_multi_session_payload_is_rejected_before_index_write( messages=[ParsedMessage(provider_message_id="multi-2-0", role=Role.USER, text="hello")], ), ] - monkeypatch.setattr("polylogue.sources.dispatch.parse_payload", lambda *_args, **_kwargs: sessions) + monkeypatch.setattr("polylogue.sources.dispatch.parse_stream_payload", lambda *_args, **_kwargs: sessions) result = ingest_append_plans(cast(Any, owner), [plan]) assert result.failed == [plan] - parsed_at_ms, parse_error = _raw_parse_state(tmp_path) + parsed_at_ms, parse_error = _append_raw_parse_state(tmp_path) assert parsed_at_ms is None assert isinstance(parse_error, str) and "did not prove one session and cursor identity" in parse_error with sqlite3.connect(tmp_path / "index.db") as conn: - assert conn.execute("SELECT COUNT(*) FROM sessions").fetchone()[0] == 0 + assert conn.execute("SELECT native_id FROM sessions").fetchall() == [("append-multi",)] def test_full_multi_session_failure_retries_without_success_mapping( @@ -5028,12 +4989,10 @@ def test_full_ingest_skips_durably_excised_content_without_aborting_batch( ``write_raw_payload`` -> ``write_source_raw_session`` gate, which is a different call site (polylogue-re4a). """ - from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_archive_database from polylogue.storage.sqlite.archive_tiers.source_write import ( deterministic_blob_hash, record_excised_blob_hash, ) - from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier root = tmp_path / "sessions" root.mkdir() @@ -5051,7 +5010,7 @@ def test_full_ingest_skips_durably_excised_content_without_aborting_batch( # Pre-mark the excised file's exact content hash as durably excised, # mirroring a prior real `polylogue ops excise` apply. - initialize_archive_database(tmp_path / "source.db", ArchiveTier.SOURCE) + initialize_active_archive_root(tmp_path) source_conn = sqlite3.connect(tmp_path / "source.db") try: record_excised_blob_hash( diff --git a/tests/unit/storage/test_repair.py b/tests/unit/storage/test_repair.py index 5ea5a0bd50..4552ebb15e 100644 --- a/tests/unit/storage/test_repair.py +++ b/tests/unit/storage/test_repair.py @@ -24,7 +24,7 @@ from polylogue.storage.insights.session.repair_assessment import assess_session_insight_repairs from polylogue.storage.insights.session.runtime import SessionInsightCounts, SessionInsightStatusSnapshot from polylogue.storage.raw_authority import RawReplayPlan, RawReplayPlanOutcome -from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_archive_database +from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root, initialize_archive_database from polylogue.storage.sqlite.archive_tiers.source_write import ArchiveSourceArtifact, upsert_raw_artifact from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier @@ -140,6 +140,17 @@ def _complete_bounded_raw_census(config: Config, *, limit: int) -> tuple[repair_ raise AssertionError("bounded raw census did not quiesce") +def _repair_after_persisted_census( + config: Config, + *, + dry_run: bool = False, + raw_artifact_id: str | None = None, +) -> repair_mod.RepairResult: + """Exercise replay only after the durable parser census reaches quiescence.""" + _complete_bounded_raw_census(config, limit=1_000) + return repair_mod.repair_raw_materialization(config, dry_run=dry_run, raw_artifact_id=raw_artifact_id) + + def _status( *, source_documents: int = 0, @@ -376,8 +387,7 @@ def fail_unrelated(*_args: object, **_kwargs: object) -> int: def test_raw_materialization_preview_counts_replayable_rows_without_erasing_missing_blobs(tmp_path: Path) -> None: config = _config(tmp_path) - initialize_archive_database(tmp_path / "source.db", ArchiveTier.SOURCE) - initialize_archive_database(tmp_path / "index.db", ArchiveTier.INDEX) + initialize_active_archive_root(tmp_path) blob_store = BlobStore(tmp_path / "blob") replayable_raw_id, replayable_size = blob_store.write_from_bytes(b'{"mapping":{}}') materialized_raw_id, materialized_size = blob_store.write_from_bytes(b'{"mapping":{"done":{}}}') @@ -449,39 +459,17 @@ def test_raw_materialization_preview_counts_replayable_rows_without_erasing_miss result = repair_mod.repair_raw_materialization(config, dry_run=True) assert result.repaired_count == 0 - assert result.success is True - assert result.metrics == { - "raw_materialization_candidate_count": 1.0, - "raw_materialization_selected_count": 1.0, - "raw_materialization_missing_blob_count": 1.0, - "raw_materialization_missing_blob_source_available_count": 0.0, - "raw_materialization_missing_blob_source_missing_count": 1.0, - "raw_materialization_already_parsed_count": 0.0, - "raw_materialization_total_blob_bytes": float(replayable_size), - "raw_materialization_max_blob_bytes": float(replayable_size), - "raw_materialization_selected_total_blob_bytes": float(replayable_size), - "raw_materialization_selected_max_blob_bytes": float(replayable_size), - "raw_materialization_adoption_deferred_count": 0.0, - "raw_materialization_authority_quarantined_count": 0.0, - "raw_materialization_byte_authority_fragment_count": 0.0, - "raw_materialization_byte_authority_pending_count": 0.0, - "raw_materialization_byte_authority_quarantined_count": 0.0, - "raw_materialization_before_component_count": 1.0, - "raw_materialization_selected_executable_component_count": 1.0, - "raw_materialization_selected_blocked_component_count": 0.0, - "raw_materialization_census_sequence": 1.0, - "raw_materialization_census_fixed_point": 0.0, - } - assert "per-session revision authority" in result.detail - assert "selected raw payload bytes total=" in result.detail - assert "largest=" in result.detail - assert "1 raw rows remain blocked by missing blobs (1 with source paths missing)" in result.detail + assert result.success is False + assert result.census_receipt is not None + assert result.census_receipt.quiescent is False + assert result.metrics["raw_materialization_census_incomplete_raw_count"] == 1.0 + assert result.metrics["raw_materialization_missing_blob_count"] == 1.0 + assert "persisted parser census" in result.detail def test_raw_materialization_replays_same_native_when_index_raw_link_is_dangling(tmp_path: Path) -> None: config = _config(tmp_path) - initialize_archive_database(tmp_path / "source.db", ArchiveTier.SOURCE) - initialize_archive_database(tmp_path / "index.db", ArchiveTier.INDEX) + initialize_active_archive_root(tmp_path) blob_store = BlobStore(tmp_path / "blob") replacement_raw_id, replacement_size = blob_store.write_from_bytes(b'{"mapping":{"replacement":{}}}') @@ -515,7 +503,7 @@ def test_raw_materialization_replays_same_native_when_index_raw_link_is_dangling ) index_conn.commit() - result = repair_mod.repair_raw_materialization(config, dry_run=True) + result = _repair_after_persisted_census(config, dry_run=True) assert result.success is True assert result.repaired_count == 0 @@ -526,9 +514,7 @@ def test_raw_materialization_split_root_routes_authority_replay(tmp_path: Path) configured_root = tmp_path / "configured" routed_root = tmp_path / "routed" configured_root.mkdir() - routed_root.mkdir() - initialize_archive_database(routed_root / "source.db", ArchiveTier.SOURCE) - initialize_archive_database(routed_root / "index.db", ArchiveTier.INDEX) + initialize_active_archive_root(routed_root) raw_id, raw_size = BlobStore(routed_root / "blob").write_from_bytes( b'{"mapping":{"routed":{"id":"routed","message":{"id":"m1","author":{"role":"user"},' b'"content":{"content_type":"text","parts":["hi"]}},"parent":null,"children":[]}},' @@ -561,7 +547,7 @@ def test_raw_materialization_split_root_routes_authority_replay(tmp_path: Path) ) backlog = repair_mod.raw_materialization_replay_backlog(config) - result = repair_mod.repair_raw_materialization(config) + result = _repair_after_persisted_census(config) assert backlog["execution_blocked"] is False assert backlog["execution_block_reason"] is None @@ -1359,9 +1345,7 @@ def test_raw_materialization_split_root_classifies_parsed_sidecar_from_routed_bl configured_root = tmp_path / "configured" routed_root = tmp_path / "routed" configured_root.mkdir() - routed_root.mkdir() - initialize_archive_database(routed_root / "source.db", ArchiveTier.SOURCE) - initialize_archive_database(routed_root / "index.db", ArchiveTier.INDEX) + initialize_active_archive_root(routed_root) raw_id, raw_size = BlobStore(routed_root / "blob").write_from_bytes(b'{"type":"session_meta"}\n') with sqlite3.connect(routed_root / "source.db") as source_conn: source_conn.execute( @@ -1391,7 +1375,7 @@ def test_raw_materialization_split_root_classifies_parsed_sidecar_from_routed_bl db_path=routed_root / "index.db", ) - result = repair_mod.repair_raw_materialization(config, dry_run=True) + result = _repair_after_persisted_census(config, dry_run=True) assert result.success is True assert result.repaired_count == 0 @@ -1701,8 +1685,7 @@ def test_raw_materialization_retries_restored_missing_blob_parse_errors(tmp_path def test_raw_materialization_replays_parsed_rows_when_index_is_empty(tmp_path: Path) -> None: config = _config(tmp_path) - initialize_archive_database(tmp_path / "source.db", ArchiveTier.SOURCE) - initialize_archive_database(tmp_path / "index.db", ArchiveTier.INDEX) + initialize_active_archive_root(tmp_path) blob_store = BlobStore(tmp_path / "blob") raw_id, blob_size = blob_store.write_from_bytes(b'{"mapping":{"already-parsed":{}}}') @@ -1728,7 +1711,7 @@ def test_raw_materialization_replays_parsed_rows_when_index_is_empty(tmp_path: P ) source_conn.commit() - result = repair_mod.repair_raw_materialization(config, dry_run=True) + result = _repair_after_persisted_census(config, dry_run=True) assert result.repaired_count == 0 assert result.success is True @@ -1739,8 +1722,7 @@ def test_raw_materialization_replays_parsed_rows_when_index_is_empty(tmp_path: P def test_raw_materialization_replays_parsed_rows_after_interrupted_index_rebuild(tmp_path: Path) -> None: config = _config(tmp_path) - initialize_archive_database(tmp_path / "source.db", ArchiveTier.SOURCE) - initialize_archive_database(tmp_path / "index.db", ArchiveTier.INDEX) + initialize_active_archive_root(tmp_path) blob_store = BlobStore(tmp_path / "blob") remaining_raw_id, remaining_size = blob_store.write_from_bytes(b'{"mapping":{"remaining":{}}}') done_raw_id, done_size = blob_store.write_from_bytes(b'{"mapping":{"done":{}}}') @@ -1790,7 +1772,7 @@ def test_raw_materialization_replays_parsed_rows_after_interrupted_index_rebuild ) index_conn.commit() - result = repair_mod.repair_raw_materialization(config, dry_run=True) + result = _repair_after_persisted_census(config, dry_run=True) assert result.repaired_count == 0 assert result.success is True @@ -2004,8 +1986,7 @@ def conversation(session_id: str) -> dict[str, object]: def test_raw_materialization_reports_uncensused_append_fragments_as_pending_debt(tmp_path: Path) -> None: config = _config(tmp_path) - initialize_archive_database(tmp_path / "source.db", ArchiveTier.SOURCE) - initialize_archive_database(tmp_path / "index.db", ArchiveTier.INDEX) + initialize_active_archive_root(tmp_path) blob_store = BlobStore(tmp_path / "blob") raw_id, blob_size = blob_store.write_from_bytes(b'{"fragment":true}') with sqlite3.connect(tmp_path / "source.db") as source_conn: @@ -2030,18 +2011,21 @@ def test_raw_materialization_reports_uncensused_append_fragments_as_pending_debt assert backlog["durable_authority_debt_count"] == 1 assert backlog["byte_authority_pending_count"] == 1 assert targeted.success is False - assert "pending byte-authority adjudication" in targeted.detail + assert targeted.census_receipt is not None + assert targeted.census_receipt.quiescent is False + assert "persisted parser census" in targeted.detail with sqlite3.connect(tmp_path / "source.db") as source_conn: source_conn.execute( """ - INSERT INTO raw_membership_census ( - raw_id, parser_fingerprint, status, member_count, censused_at_ms, detail - ) VALUES (?, 'test', 'failed', 0, 2, - 'append fragments are governed by byte revision authority') + UPDATE raw_membership_census + SET parser_fingerprint = 'test', status = 'failed', member_count = 0, + censused_at_ms = 2, detail = 'append fragments are governed by byte revision authority' + WHERE raw_id = ? """, (raw_id,), ) + assert source_conn.total_changes == 1 source_conn.commit() governed = repair_mod._raw_materialization_candidate_ids(config) @@ -2065,8 +2049,7 @@ def test_raw_materialization_reports_uncensused_append_fragments_as_pending_debt def test_raw_materialization_ordinary_replay_reaches_two_call_fixed_point(tmp_path: Path) -> None: config = _config(tmp_path) - initialize_archive_database(tmp_path / "source.db", ArchiveTier.SOURCE) - initialize_archive_database(tmp_path / "index.db", ArchiveTier.INDEX) + initialize_active_archive_root(tmp_path) payload = b"""{ "id": "fixed-point", "title": "fixed point", @@ -2109,7 +2092,7 @@ def test_raw_materialization_ordinary_replay_reaches_two_call_fixed_point(tmp_pa ) source_conn.commit() - first = repair_mod.repair_raw_materialization(config) + first = _repair_after_persisted_census(config) with sqlite3.connect(tmp_path / "index.db") as index_conn: receipts_after_first = index_conn.execute( "SELECT decision_id, raw_id, decision FROM raw_revision_applications ORDER BY decision_id" @@ -2149,8 +2132,7 @@ def test_raw_materialization_no_progress_component_terminalizes_instead_of_loopi automatically reselected on the next pass. """ config = _config(tmp_path) - initialize_archive_database(tmp_path / "source.db", ArchiveTier.SOURCE) - initialize_archive_database(tmp_path / "index.db", ArchiveTier.INDEX) + initialize_active_archive_root(tmp_path) payload = b"""{ "id": "orphan-append", "title": "orphan append", @@ -2208,7 +2190,7 @@ def test_raw_materialization_no_progress_component_terminalizes_instead_of_loopi ) source_conn.commit() - first = repair_mod.repair_raw_materialization(config) + first = _repair_after_persisted_census(config) assert first.success is False assert first.repaired_count == 0 assert first.metrics.get("raw_materialization_no_progress_count") == 1.0 @@ -2249,8 +2231,7 @@ def test_raw_materialization_uses_authority_replay_not_legacy_batch_parser( monkeypatch: pytest.MonkeyPatch, ) -> None: config = _config(tmp_path) - initialize_archive_database(tmp_path / "source.db", ArchiveTier.SOURCE) - initialize_archive_database(tmp_path / "index.db", ArchiveTier.INDEX) + initialize_active_archive_root(tmp_path) blob_store = BlobStore(tmp_path / "blob") first_raw_id, first_size = blob_store.write_from_bytes( b'{"mapping":{"first":{"id":"first","message":{"id":"m1","author":{"role":"user"},' @@ -2309,7 +2290,7 @@ async def parse_from_raw(self, *, raw_ids: list[str], **kwargs: object) -> objec monkeypatch.setattr(parsing_module, "ParsingService", FakeParsingService) - result = repair_mod.repair_raw_materialization(config) + result = _repair_after_persisted_census(config) assert result.success is True assert result.repaired_count == 2 @@ -2322,8 +2303,7 @@ def test_raw_materialization_ordinary_repair_preserves_newer_index_state( monkeypatch: pytest.MonkeyPatch, ) -> None: config = _config(tmp_path) - initialize_archive_database(tmp_path / "source.db", ArchiveTier.SOURCE) - initialize_archive_database(tmp_path / "index.db", ArchiveTier.INDEX) + initialize_active_archive_root(tmp_path) older_payload = b"""{ "id": "logical-session", "title": "older raw snapshot", @@ -2392,6 +2372,13 @@ def test_raw_materialization_ordinary_repair_preserves_newer_index_state( fts_hits_before = index_conn.execute( "SELECT rowid FROM messages_fts WHERE messages_fts MATCH 'newer' ORDER BY rowid" ).fetchall() + message_ids_before = [ + str(message_id) + for (message_id,) in index_conn.execute( + "SELECT message_id FROM messages WHERE session_id = ? ORDER BY position", + (session_id,), + ).fetchall() + ] assert len(fts_hits_before) == 1 class UnexpectedParsingService: @@ -2400,7 +2387,7 @@ def __init__(self, *_args: object, **_kwargs: object) -> None: monkeypatch.setattr("polylogue.pipeline.services.parsing.ParsingService", UnexpectedParsingService) - result = repair_mod.repair_raw_materialization(config, dry_run=False) + result = _repair_after_persisted_census(config) assert result.success is False assert result.repaired_count == 0 @@ -2422,7 +2409,7 @@ def __init__(self, *_args: object, **_kwargs: object) -> None: "SELECT rowid FROM messages_fts WHERE messages_fts MATCH 'newer' ORDER BY rowid" ).fetchall() assert row == ("newer-index-raw", "newer indexed state", newer_hash, 1) - assert message_ids == ["chatgpt-export:logical-session:newer-message"] + assert message_ids == message_ids_before assert fts_hits_after == fts_hits_before with sqlite3.connect(tmp_path / "source.db") as source_conn: raw_state = source_conn.execute( @@ -2586,8 +2573,7 @@ def test_raw_materialization_raw_artifact_filter_counts_only_target(tmp_path: Pa def test_raw_materialization_excludes_already_parsed_non_materialized_rows(tmp_path: Path) -> None: config = _config(tmp_path) - initialize_archive_database(tmp_path / "source.db", ArchiveTier.SOURCE) - initialize_archive_database(tmp_path / "index.db", ArchiveTier.INDEX) + initialize_active_archive_root(tmp_path) blob_store = BlobStore(tmp_path / "blob") replayable_raw_id, replayable_size = blob_store.write_from_bytes(b'{"mapping":{"pending":{}}}') parsed_raw_id, parsed_size = blob_store.write_from_bytes(b'{"mapping":{"parsed":{}}}') @@ -2626,13 +2612,13 @@ def test_raw_materialization_excludes_already_parsed_non_materialized_rows(tmp_p ) source_conn.commit() - result = repair_mod.repair_raw_materialization(config, dry_run=True) + result = _repair_after_persisted_census(config, dry_run=True) assert result.repaired_count == 0 assert result.metrics["raw_materialization_candidate_count"] == 2.0 assert "1 already parsed but not materialized" in result.detail - scoped = repair_mod.repair_raw_materialization(config, dry_run=True, raw_artifact_id=parsed_raw_id) + scoped = _repair_after_persisted_census(config, dry_run=True, raw_artifact_id=parsed_raw_id) assert scoped.repaired_count == 0 assert scoped.metrics["raw_materialization_candidate_count"] == 1.0 @@ -2682,8 +2668,7 @@ def test_raw_materialization_excludes_parsed_non_session_artifacts(tmp_path: Pat def test_raw_materialization_explicit_scope_includes_already_parsed_rows(tmp_path: Path) -> None: config = _config(tmp_path) - initialize_archive_database(tmp_path / "source.db", ArchiveTier.SOURCE) - initialize_archive_database(tmp_path / "index.db", ArchiveTier.INDEX) + initialize_active_archive_root(tmp_path) blob_store = BlobStore(tmp_path / "blob") parsed_raw_id, parsed_size = blob_store.write_from_bytes(b'{"items":[]}') @@ -2709,6 +2694,7 @@ def test_raw_materialization_explicit_scope_includes_already_parsed_rows(tmp_pat ) source_conn.commit() + _complete_bounded_raw_census(config, limit=1_000) broad = repair_mod.repair_raw_materialization(config, dry_run=True) by_family = repair_mod.repair_raw_materialization(config, dry_run=True, source_family="gemini-cli-session") by_root = repair_mod.repair_raw_materialization(config, dry_run=True, source_root=Path("/captures/gemini")) @@ -2907,8 +2893,10 @@ def __init__(self, **_kwargs: object) -> None: assert result.metrics["raw_materialization_executed_count"] == 0.0 assert result.metrics["raw_materialization_execute_blob_limit_bytes"] == float(1024 * 1024 * 1024) assert parser_fingerprint.endswith(":resource-blocked:1073741824") - assert repeated.success is True - assert repeated_census_count == first_census_count + assert repeated.success is False + assert len(repeated.plan_outcomes) == 1 + assert repeated.plan_outcomes[0].status.value == "terminal" + assert repeated_census_count == first_census_count + 1 def test_raw_materialization_classifies_oversized_stream_record_replay( @@ -3085,10 +3073,11 @@ def test_raw_materialization_blocks_aggregate_sub_limit_cohort_before_blob_open( assert result.success is False assert result.metrics["raw_materialization_resource_blocked_count"] == 2.0 assert len(result.plan_outcomes) == 1 - assert result.plan_outcomes[0].status.value == "deferred" - assert repeated.success is True - assert repeated.plan_outcomes == () - assert "unchanged plan(s) remain deferred" in repeated.detail + assert result.plan_outcomes[0].status.value == "terminal" + assert repeated.success is False + assert len(repeated.plan_outcomes) == 1 + assert repeated.plan_outcomes[0].status.value == "terminal" + assert "aggregate payload exceeds 1.0 GiB" in repeated.detail assert "aggregate payload exceeds 1.0 GiB" in result.detail @@ -3319,7 +3308,7 @@ def test_raw_materialization_durable_ledger_survives_ops_reset_for_fairness( ) first = repair_mod.repair_raw_materialization(_config(tmp_path), raw_artifact_limit=1) - assert first.plan_outcomes[0].status.value == "deferred" + assert first.plan_outcomes[0].status.value == "terminal" (tmp_path / "ops.db").unlink() second = repair_mod.repair_raw_materialization(_config(tmp_path), raw_artifact_limit=1) From 46891e7f0d6a99ed52f837d82cc762c446f9b5ee Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 00:25:51 +0200 Subject: [PATCH 09/65] fix(live): retain unknown weak-path JSON Problem: weak analysis-path classification excluded unknown ordinary JSON before the generic route could retain malformed or empty bytes and terminal evidence.\n\nWhat changed: exempt only unknown .json from that pre-decode exclusion. The generic route now persists the raw input and records the existing typed terminal decode outcome.\n\nThe regression exercises malformed and empty weak-path JSON, asserts the path-only exclusion preconditions, and verifies durable raw rows and terminal artifacts. --- polylogue/sources/live/batch.py | 5 ++ tests/unit/sources/test_live_batch_support.py | 56 +++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/polylogue/sources/live/batch.py b/polylogue/sources/live/batch.py index aab85bb632..5ca8a58599 100644 --- a/polylogue/sources/live/batch.py +++ b/polylogue/sources/live/batch.py @@ -2004,6 +2004,11 @@ def _ingest_full_paths_sync( and path_artifact is not None and not path_artifact.parse_as_session and stat.st_size < _STREAMING_FULL_INGEST_BYTES + # An unknown ordinary JSON payload must reach the generic + # JSON route. That route retains raw bytes and records a + # typed terminal outcome for malformed or empty input. + # Other weak-path artifacts remain excluded before acquisition. + and not (fallback_provider is Provider.UNKNOWN and path.suffix.lower() == ".json") and not has_decoded_session_evidence(path, provider=fallback_provider) ): # Keep path-only metadata out of the generic JSON fallback, diff --git a/tests/unit/sources/test_live_batch_support.py b/tests/unit/sources/test_live_batch_support.py index fb4b211234..e6bf5e183b 100644 --- a/tests/unit/sources/test_live_batch_support.py +++ b/tests/unit/sources/test_live_batch_support.py @@ -14,6 +14,7 @@ import pytest import polylogue.sources.live.watcher as live_watcher +from polylogue.archive.artifact_taxonomy import classify_artifact_path from polylogue.archive.message.roles import Role from polylogue.archive.revision_authority import ( HISTORICAL_NON_PREFIX_GOVERNANCE_DETAIL, @@ -51,6 +52,7 @@ ) from polylogue.sources.live.cursor import CursorStore from polylogue.sources.parsers.base import ParsedMessage, ParsedSession +from polylogue.sources.source_parsing import has_decoded_session_evidence from polylogue.storage.blob_store import BlobStore from polylogue.storage.raw_authority import RAW_AUTHORITY_PARSER_FINGERPRINT from polylogue.storage.raw_failure_lifecycle import read_raw_failure_lifecycle @@ -506,6 +508,60 @@ def test_full_ingest_unknown_weak_path_ndjson_records_terminal_evidence(tmp_path assert artifact == ("terminal_unknown_export_no_session", 0) +@pytest.mark.parametrize( + ("payload", "expected_artifact"), + [ + (b"{", ("terminal_unknown_json_decode", "decode_failed")), + (b"", ("terminal_unknown_json_decode", "decode_failed")), + ], +) +def test_full_ingest_unknown_weak_path_json_retains_terminal_evidence( + tmp_path: Path, + payload: bytes, + expected_artifact: tuple[str, str], +) -> None: + """Unknown weak-path JSON reaches durable generic terminal handling.""" + + root = tmp_path / "unknown" + path = root / "analysis" / "export.json" + path.parent.mkdir(parents=True) + path.write_bytes(payload) + path_artifact = classify_artifact_path(path, provider=Provider.UNKNOWN) + assert path_artifact is not None and not path_artifact.parse_as_session + assert not has_decoded_session_evidence(path, provider=Provider.UNKNOWN) + + db_path = tmp_path / "archive.sqlite" + processor = LiveBatchProcessor( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=db_path))), + (WatchSource(name="unknown", root=root),), + cursor=CursorStore(db_path), + parser_fingerprint="test-parser", + ) + + result = processor._ingest_full_paths_sync([path], source_name="unknown") + + assert result.succeeded == [path] + assert result.failed == [] + with sqlite3.connect(tmp_path / "source.db") as conn: + raw = conn.execute( + "SELECT raw_id, blob_size, parse_error FROM raw_sessions WHERE source_path = ?", (str(path),) + ).fetchone() + artifact = conn.execute( + """ + SELECT artifact_kind, support_status + FROM raw_artifacts + WHERE raw_id = ? + """, + (raw[0],) if raw is not None else (None,), + ).fetchone() + # The preconditions above would take the weak path-exclusion branch if + # the production unknown-JSON exemption were removed. + assert raw is not None + assert raw[1] == len(payload) + assert isinstance(raw[2], str) + assert artifact == expected_artifact + + def test_full_ingest_unknown_malformed_jsonl_records_terminal_decode_and_stops_retrying(tmp_path: Path) -> None: """Complete malformed JSONL lines are terminal decode evidence, not no-session evidence.""" root = tmp_path / "unknown" From 2c7ab729c9ee375f3e1b5cfc1bf55270e5dd1dbc Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 01:01:51 +0200 Subject: [PATCH 10/65] fix(live): repair ingest authority lifecycles Problem: resolution carriers could authorize a missing accepted head, retry cleanup could drop a replacement task, unknown broad-source JSON treated named sidecars as ordinary payloads, and the writer exit proof left a pending caller on a manually closed loop. What changed: terminal cursor authority now admits only genuine terminal raw-failure kinds, hook retry cleanup is task-identity conditional, named sidecars retain path authority while weak unknown metadata is retained for strict classification, and the subprocess proof closes its caller lifecycle without unblocking the daemon writer. Verification: focused watcher failures passed 3/3; the specified seven-file gate passed 446/446; devtools verify --quick passed. --- polylogue/sources/live/batch.py | 17 ++++-- polylogue/sources/live/watcher.py | 7 ++- polylogue/storage/raw_retention.py | 31 +++++++--- tests/unit/sources/test_hook_spool.py | 59 ++++++++++++++++++- .../unit/sources/test_live_watcher_locking.py | 10 ++-- tests/unit/storage/test_raw_retention.py | 46 +++++++++++++++ 6 files changed, 149 insertions(+), 21 deletions(-) diff --git a/polylogue/sources/live/batch.py b/polylogue/sources/live/batch.py index 5ca8a58599..b12594ca00 100644 --- a/polylogue/sources/live/batch.py +++ b/polylogue/sources/live/batch.py @@ -20,7 +20,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, Literal, ParamSpec, TypeVar, cast -from polylogue.archive.artifact_taxonomy import classify_artifact_path +from polylogue.archive.artifact_taxonomy import ArtifactKind, classify_artifact_path from polylogue.archive.ingest_flags import ( COMPACT_BROWSER_CAPTURE_INGEST_FLAG, DOM_FALLBACK_INGEST_FLAG, @@ -2004,11 +2004,16 @@ def _ingest_full_paths_sync( and path_artifact is not None and not path_artifact.parse_as_session and stat.st_size < _STREAMING_FULL_INGEST_BYTES - # An unknown ordinary JSON payload must reach the generic - # JSON route. That route retains raw bytes and records a - # typed terminal outcome for malformed or empty input. - # Other weak-path artifacts remain excluded before acquisition. - and not (fallback_provider is Provider.UNKNOWN and path.suffix.lower() == ".json") + # An unknown JSON payload under the weak ``analysis/`` path + # heuristic must reach the generic JSON route. That route + # retains raw bytes and records a typed terminal outcome for + # malformed or empty input. Strong named sidecars such as + # ``sessions-index.json`` remain excluded before acquisition. + and not ( + fallback_provider is Provider.UNKNOWN + and path.suffix.lower() == ".json" + and path_artifact.kind is ArtifactKind.METADATA_DOCUMENT + ) and not has_decoded_session_evidence(path, provider=fallback_provider) ): # Keep path-only metadata out of the generic JSON fallback, diff --git a/polylogue/sources/live/watcher.py b/polylogue/sources/live/watcher.py index 7496ead9a8..0ed99a9fa2 100644 --- a/polylogue/sources/live/watcher.py +++ b/polylogue/sources/live/watcher.py @@ -444,7 +444,12 @@ def _schedule_hook_spool_directory_retry(self, directory: Path) -> None: return task = asyncio.create_task(self._retry_hook_spool_directory_until_populated(directory)) self._hook_spool_directory_retry_tasks[directory] = task - task.add_done_callback(lambda _task: self._hook_spool_directory_retry_tasks.pop(directory, None)) + + def discard_completed_task(completed: asyncio.Task[None]) -> None: + if self._hook_spool_directory_retry_tasks.get(directory) is completed: + self._hook_spool_directory_retry_tasks.pop(directory, None) + + task.add_done_callback(discard_completed_task) async def _retry_hook_spool_directory_until_populated(self, directory: Path) -> None: """Wait for an added shard's first envelope until it is acknowledged.""" diff --git a/polylogue/storage/raw_retention.py b/polylogue/storage/raw_retention.py index 5dcb9708f4..228043adaa 100644 --- a/polylogue/storage/raw_retention.py +++ b/polylogue/storage/raw_retention.py @@ -10,6 +10,7 @@ from pathlib import Path from typing import Literal +from polylogue.core.raw_failure_evidence import RAW_FAILURE_EVIDENCE_KINDS, RawFailureEvidenceKind from polylogue.logging import get_logger from polylogue.storage.archive_identity import resolve_active_index_path from polylogue.storage.blob_store import BlobStore, get_blob_store @@ -18,6 +19,10 @@ logger = get_logger(__name__) +_TERMINAL_RAW_FAILURE_EVIDENCE_KINDS = frozenset( + kind.value for kind in RawFailureEvidenceKind if kind.lifecycle == "terminal" +) + _V1_RAW_CANDIDATE_SQL = """ WITH ranked AS ( SELECT @@ -1641,18 +1646,31 @@ def _terminal_artifact_paths(conn: sqlite3.Connection, source_paths: set[str]) - """ result: set[str] = set() + raw_failure_kinds = tuple(sorted(RAW_FAILURE_EVIDENCE_KINDS)) + terminal_raw_failure_kinds = tuple(sorted(_TERMINAL_RAW_FAILURE_EVIDENCE_KINDS)) + raw_failure_placeholders = ", ".join("?" for _ in raw_failure_kinds) + terminal_raw_failure_placeholders = ", ".join("?" for _ in terminal_raw_failure_kinds) + path_batch_size = 500 - len(raw_failure_kinds) - len(terminal_raw_failure_kinds) pending = set(source_paths) while pending: - batch = tuple(sorted(pending)[:500]) + batch = tuple(sorted(pending)[:path_batch_size]) pending.difference_update(batch) placeholders = ", ".join("?" for _ in batch) rows = conn.execute( f""" + WITH terminal_artifacts AS ( + SELECT raw_id + FROM raw_artifacts + WHERE parse_as_session = 0 + AND ( + artifact_kind NOT IN ({raw_failure_placeholders}) + OR artifact_kind IN ({terminal_raw_failure_placeholders}) + ) + ) SELECT DISTINCT terminal_raw.source_path - FROM raw_artifacts AS artifact + FROM terminal_artifacts AS artifact JOIN raw_sessions AS terminal_raw ON terminal_raw.raw_id = artifact.raw_id - WHERE artifact.parse_as_session = 0 - AND terminal_raw.source_path IN ({placeholders}) + WHERE terminal_raw.source_path IN ({placeholders}) AND terminal_raw.raw_id = ( SELECT newest.raw_id FROM raw_sessions AS newest @@ -1677,13 +1695,12 @@ def _terminal_artifact_paths(conn: sqlite3.Connection, source_paths: set[str]) - ) AND NOT EXISTS ( SELECT 1 - FROM raw_artifacts AS current_artifact + FROM terminal_artifacts AS current_artifact WHERE current_artifact.raw_id = coordinate.raw_id - AND current_artifact.parse_as_session = 0 ) ) """, - batch, + (*raw_failure_kinds, *terminal_raw_failure_kinds, *batch), ).fetchall() result.update(str(row[0]) for row in rows) return result diff --git a/tests/unit/sources/test_hook_spool.py b/tests/unit/sources/test_hook_spool.py index ab6a7c5e78..b6028e4411 100644 --- a/tests/unit/sources/test_hook_spool.py +++ b/tests/unit/sources/test_hook_spool.py @@ -9,7 +9,7 @@ import sqlite3 import subprocess import sys -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Callable, Coroutine from io import StringIO from pathlib import Path from types import SimpleNamespace @@ -363,6 +363,63 @@ async def emit_empty_shard(*roots: Path, **_kwargs: object) -> AsyncIterator[set assert conn.execute("SELECT session_native_id FROM raw_hook_events").fetchone() == ("session-2",) +@pytest.mark.asyncio +async def test_hook_shard_retry_replacement_remains_tracked_until_stop( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """An old done callback cannot untrack its replacement retry task.""" + + class ControlledTask: + def __init__(self) -> None: + self._done = False + self.cancelled = False + self.callbacks: list[Callable[[ControlledTask], None]] = [] + + def done(self) -> bool: + return self._done + + def cancel(self) -> None: + self.cancelled = True + + def add_done_callback(self, callback: Callable[[ControlledTask], None]) -> None: + self.callbacks.append(callback) + + def finish(self) -> None: + self._done = True + for callback in self.callbacks: + callback(self) + + created: list[ControlledTask] = [] + + def create_task(coro: Coroutine[Any, Any, None]) -> ControlledTask: + coro.close() + task = ControlledTask() + created.append(task) + return task + + spool_root = tmp_path / "hooks" + watcher = LiveWatcher( + cast(Any, SimpleNamespace(archive_root=tmp_path / "archive", backend=None)), + (WatchSource(name="hooks", root=pending_hook_spool_dir(spool_root), suffixes=(".json",)),), + cursor=CursorStore(tmp_path / "archive" / "ops.db"), + ) + monkeypatch.setattr(asyncio, "create_task", create_task) + directory = pending_hook_spool_dir(spool_root) / "2026-08-13" + + watcher._schedule_hook_spool_directory_retry(directory) + first = created[0] + first._done = True + watcher._schedule_hook_spool_directory_retry(directory) + replacement = created[1] + first.finish() + + tracked_replacement = cast(object, watcher._hook_spool_directory_retry_tasks[directory.resolve()]) + assert tracked_replacement is replacement + watcher.stop() + assert replacement.cancelled is True + + @pytest.mark.uses_real_clock("exercises retry polling while a pending hook envelope remains unacknowledged") @pytest.mark.asyncio async def test_hook_shard_retry_waits_for_durable_acknowledgement( diff --git a/tests/unit/sources/test_live_watcher_locking.py b/tests/unit/sources/test_live_watcher_locking.py index a38831a453..655b3303b2 100644 --- a/tests/unit/sources/test_live_watcher_locking.py +++ b/tests/unit/sources/test_live_watcher_locking.py @@ -36,6 +36,7 @@ def test_real_watcher_writer_routes_cannot_pin_process_exit(route: str) -> None: script = textwrap.dedent( f""" import asyncio + import contextlib import tempfile import threading from pathlib import Path @@ -97,18 +98,15 @@ def stuck(*args, **kwargs): while not started.is_set(): await asyncio.sleep(0.001) caller.cancel() + with contextlib.suppress(asyncio.CancelledError): + await caller assert await coordinator.shutdown(timeout=0.01) is False # The injected writer remains blocked through interpreter # termination. A non-daemon bridge thread would pin this # subprocess after the loop closes. watcher.stop() - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - try: - loop.run_until_complete(main()) - finally: - loop.close() + asyncio.run(main()) """ ) diff --git a/tests/unit/storage/test_raw_retention.py b/tests/unit/storage/test_raw_retention.py index 3c10d035e9..b943be1dfe 100644 --- a/tests/unit/storage/test_raw_retention.py +++ b/tests/unit/storage/test_raw_retention.py @@ -688,6 +688,52 @@ def test_terminal_cursor_exemption_requires_every_source_coordinate(tmp_path: Pa assert snapshot.cursor_authority_gap_samples[0].state == "source_raws_without_accepted_head" +def test_resolution_carrier_cannot_authorize_cursor_without_accepted_head(tmp_path: Path) -> None: + """A superseded deferred-CAS receipt is resolution evidence, not terminal authority.""" + + source_db = tmp_path / "source.db" + index_db = tmp_path / "index.db" + ops_db = tmp_path / "ops.db" + source_path = tmp_path / "replaced-attempt.jsonl" + initialize_archive_database(source_db, ArchiveTier.SOURCE) + initialize_archive_database(index_db, ArchiveTier.INDEX) + with sqlite3.connect(source_db) as conn: + conn.execute( + """ + INSERT INTO raw_sessions ( + raw_id, origin, native_id, source_path, source_index, blob_hash, blob_size, acquired_at_ms + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, + ("raw-resolution", "claude-code-session", "resolution", str(source_path), 0, bytes(32), 1, 1), + ) + conn.execute( + """ + INSERT INTO raw_artifacts ( + artifact_id, raw_id, origin, source_path, source_index, artifact_kind, + support_status, classification_reason, parse_as_session, schema_eligible, + malformed_jsonl_lines, first_observed_at_ms, last_observed_at_ms + ) VALUES (?, ?, ?, ?, ?, ?, 'unknown', 'deferred attempt replaced', 0, 0, 0, 1, 1) + """, + ( + "artifact-resolution", + "raw-resolution", + "claude-code-session", + str(source_path), + 0, + "terminal_superseded_deferred_cas_frontier", + ), + ) + conn.commit() + _seed_ops_cursor(ops_db, source_path=source_path, byte_offset=1) + + with sqlite3.connect(source_db) as conn: + snapshot = raw_frontier_integrity_snapshot(conn, index_db_path=index_db, ops_db_path=ops_db) + + assert snapshot.cursor_ahead_status == "unknown" + assert snapshot.cursor_authority_gap_count == 1 + assert snapshot.cursor_authority_gap_samples[0].state == "source_raws_without_accepted_head" + + def test_terminal_artifact_retention_batches_source_paths_below_sqlite_limit(tmp_path: Path) -> None: """Terminal evidence remains protectable when more than one SQL batch is needed.""" From bb041aa939e0c8f4466196fd4875194f657ccd50 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 01:37:24 +0200 Subject: [PATCH 11/65] fix(live): prioritize strong sidecars before streaming Problem: large non-JSONL sidecars could reach generic streaming admission before definitive path classification.\n\nWhat changed: expose the existing strong path rules for live admission, preserve weak analysis-path payload and streaming handling, and cover both the full-ingest and preloaded-payload routes.\n\nCompatibility: the public path classifier preserves its output; the fingerprint manifest records the safe internal delegation.\n\nRef #3952 --- docs/plans/classifier-fingerprints.json | 6 +-- .../archive/artifact_taxonomy/__init__.py | 7 +++- .../archive/artifact_taxonomy/runtime.py | 15 +++++++ polylogue/sources/live/batch_support.py | 14 ++++--- tests/unit/sources/test_live_batch_support.py | 41 +++++++++++++++++++ 5 files changed, 74 insertions(+), 9 deletions(-) diff --git a/docs/plans/classifier-fingerprints.json b/docs/plans/classifier-fingerprints.json index ddde05b6e5..3439772203 100644 --- a/docs/plans/classifier-fingerprints.json +++ b/docs/plans/classifier-fingerprints.json @@ -10,11 +10,11 @@ } }, "polylogue/archive/artifact_taxonomy/runtime.py:classify_artifact_path": { - "fingerprint": "10f54d1827fdca585985cf3180e54c27098de26121530076e51a26cef7b2de24", + "fingerprint": "32b6f26516b4cc9ed0342c262e492fca469aaca9fb47a66e4ca1c76b7f58987a", "covered_by": { "kind": "acknowledged_safe", - "reason": "Baseline snapshot at classifier-fingerprint gate introduction (polylogue-gucv).", - "ref": "polylogue-gucv" + "reason": "Strong-only admission helper preserves this classifier output; no archived payload classification changes.", + "ref": "#3952" } }, "polylogue/archive/artifact_taxonomy/support.py:looks_like_beads_interaction": { diff --git a/polylogue/archive/artifact_taxonomy/__init__.py b/polylogue/archive/artifact_taxonomy/__init__.py index ec44d19408..79396181e9 100644 --- a/polylogue/archive/artifact_taxonomy/__init__.py +++ b/polylogue/archive/artifact_taxonomy/__init__.py @@ -8,11 +8,16 @@ from __future__ import annotations from polylogue.archive.artifact_taxonomy.models import ArtifactClassification, ArtifactKind -from polylogue.archive.artifact_taxonomy.runtime import classify_artifact, classify_artifact_path +from polylogue.archive.artifact_taxonomy.runtime import ( + classify_artifact, + classify_artifact_path, + strong_path_classification, +) __all__ = [ "ArtifactClassification", "ArtifactKind", "classify_artifact", "classify_artifact_path", + "strong_path_classification", ] diff --git a/polylogue/archive/artifact_taxonomy/runtime.py b/polylogue/archive/artifact_taxonomy/runtime.py index 95499d9375..ddf2f38b47 100644 --- a/polylogue/archive/artifact_taxonomy/runtime.py +++ b/polylogue/archive/artifact_taxonomy/runtime.py @@ -102,6 +102,21 @@ def classify_artifact_path( """ if weak := _self_generated_artifact_dir_classification(source_path, provider=provider): return weak + return strong_path_classification(source_path, provider=provider) + + +def strong_path_classification( + source_path: str | Path | None, + *, + provider: str | Provider, +) -> ArtifactClassification | None: + """Classify only definitive path rules. + + Live admission uses this before deciding whether a payload may enter a + bounded streaming route. The weak ``analysis/`` location heuristic is + deliberately excluded there because it must yield to bounded payload + evidence or the streaming policy. + """ return _classify_artifact_path_strong(source_path, provider=provider) diff --git a/polylogue/sources/live/batch_support.py b/polylogue/sources/live/batch_support.py index 13ee51035f..8e3c7579ef 100644 --- a/polylogue/sources/live/batch_support.py +++ b/polylogue/sources/live/batch_support.py @@ -12,7 +12,11 @@ import ijson -from polylogue.archive.artifact_taxonomy import classify_artifact, classify_artifact_path +from polylogue.archive.artifact_taxonomy import ( + classify_artifact, + classify_artifact_path, + strong_path_classification, +) from polylogue.archive.raw_payload.decode import jsonl_session_artifact from polylogue.core.enums import Provider from polylogue.core.json import JSONDecodeError, JSONValue @@ -596,14 +600,14 @@ def _parse_path_as_session_artifact(path: Path, *, provider: Provider) -> bool: return True path_classification = classify_artifact_path(path, provider=provider) return path_classification.parse_as_session if path_classification is not None else False + path_classification = strong_path_classification(path, provider=provider) + if path_classification is not None: + return path_classification.parse_as_session if _path_size(path) > _STREAMING_FULL_INGEST_BYTES: browser_capture, _browser_provider = _browser_capture_prefix_probe(path) if browser_capture: return True return _large_non_jsonl_path_can_stream(path, provider=provider) - path_classification = classify_artifact_path(path, provider=provider) - if path_classification is not None: - return path_classification.parse_as_session try: document = json_loads(path.read_bytes()) except JSONDecodeError: @@ -643,7 +647,7 @@ def _parse_payload_as_session_artifact(path: Path, *, provider: Provider, payloa return True path_classification = classify_artifact_path(path, provider=provider) return path_classification.parse_as_session if path_classification is not None else False - path_classification = classify_artifact_path(path, provider=provider) + path_classification = strong_path_classification(path, provider=provider) if path_classification is not None: return path_classification.parse_as_session try: diff --git a/tests/unit/sources/test_live_batch_support.py b/tests/unit/sources/test_live_batch_support.py index e6bf5e183b..28ceb5c461 100644 --- a/tests/unit/sources/test_live_batch_support.py +++ b/tests/unit/sources/test_live_batch_support.py @@ -1076,6 +1076,47 @@ def heartbeat(phase: str, **_kwargs: object) -> None: assert result.failed == [] assert "full_blob_copy" in phases + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT COUNT(*) FROM raw_sessions").fetchone() == (1,) + + +def test_threshold_crossing_strong_sidecar_is_excluded_before_streaming( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A definitive sidecar path must never reach large-JSON admission.""" + + root = tmp_path / "chatgpt" + root.mkdir() + path = root / "sessions-index.json" + path.write_bytes(b"{}") + db_path = tmp_path / "archive.sqlite" + processor = LiveBatchProcessor( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=db_path))), + (WatchSource(name="chatgpt", root=root, suffixes=(".json",)),), + cursor=CursorStore(db_path), + parser_fingerprint="test-parser", + ) + monkeypatch.setattr("polylogue.sources.live.batch._STREAMING_FULL_INGEST_BYTES", 1) + monkeypatch.setattr("polylogue.sources.live.batch_support._STREAMING_FULL_INGEST_BYTES", 1) + monkeypatch.setattr( + "polylogue.sources.live.batch_support._large_non_jsonl_path_can_stream", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("strong sidecar reached large-file streaming admission") + ), + ) + + result = processor._ingest_full_paths_sync([path], source_name="chatgpt") + + assert result.succeeded == [] + assert result.failed == [] + assert not _parse_payload_as_session_artifact( + path, + provider=Provider.CHATGPT, + payload=b'{"mapping":{"session":"would otherwise look like an export"}}', + ) + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT COUNT(*) FROM raw_sessions").fetchone() == (0,) def test_full_ingest_writes_archive_with_route_observability( From e0cda0808bb88ba996095665ae58ef65aedfca6b Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 02:07:36 +0200 Subject: [PATCH 12/65] test(live): isolate writer exit deadline --- .../unit/sources/test_live_watcher_locking.py | 28 +++++++++++++++---- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/tests/unit/sources/test_live_watcher_locking.py b/tests/unit/sources/test_live_watcher_locking.py index 655b3303b2..15fc4b1c94 100644 --- a/tests/unit/sources/test_live_watcher_locking.py +++ b/tests/unit/sources/test_live_watcher_locking.py @@ -3,6 +3,8 @@ from __future__ import annotations import asyncio +import contextlib +import selectors import sqlite3 import subprocess import sys @@ -105,21 +107,35 @@ def stuck(*args, **kwargs): # termination. A non-daemon bridge thread would pin this # subprocess after the loop closes. watcher.stop() + print("ready-for-interpreter-exit", flush=True) asyncio.run(main()) """ ) - completed = subprocess.run( + process = subprocess.Popen( [sys.executable, "-c", script], cwd=Path(__file__).parents[3], - capture_output=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True, - timeout=6.0, - check=False, ) - - assert completed.returncode == 0, completed.stderr + assert process.stdout is not None + selector = selectors.DefaultSelector() + selector.register(process.stdout, selectors.EVENT_READ) + try: + assert selector.select(timeout=30.0), "writer subprocess did not reach its exit boundary" + assert process.stdout.readline().strip() == "ready-for-interpreter-exit" + stdout, stderr = process.communicate(timeout=2.0) + except BaseException: + process.kill() + with contextlib.suppress(subprocess.TimeoutExpired): + process.communicate(timeout=2.0) + raise + finally: + selector.close() + + assert process.returncode == 0, stdout + stderr @pytest.mark.asyncio From 354cc348f103a738878fd4c44b612daafe215814 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 02:18:09 +0200 Subject: [PATCH 13/65] fix(live): close ingest authority review gaps --- polylogue/sources/live/batch.py | 10 ++++- polylogue/sources/live/watcher.py | 7 +-- polylogue/storage/raw_retention.py | 20 ++++++--- tests/unit/sources/test_live_batch_support.py | 24 ++++++++++ tests/unit/sources/test_live_watcher.py | 29 ++++++++++++ tests/unit/storage/test_raw_retention.py | 45 +++++++++++++++++++ 6 files changed, 125 insertions(+), 10 deletions(-) diff --git a/polylogue/sources/live/batch.py b/polylogue/sources/live/batch.py index b12594ca00..0b89f366b4 100644 --- a/polylogue/sources/live/batch.py +++ b/polylogue/sources/live/batch.py @@ -20,7 +20,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, Literal, ParamSpec, TypeVar, cast -from polylogue.archive.artifact_taxonomy import ArtifactKind, classify_artifact_path +from polylogue.archive.artifact_taxonomy import ArtifactKind, classify_artifact_path, strong_path_classification from polylogue.archive.ingest_flags import ( COMPACT_BROWSER_CAPTURE_INGEST_FLAG, DOM_FALLBACK_INGEST_FLAG, @@ -1881,6 +1881,7 @@ def _ingest_full_paths_sync( captured_file_observations[path] = _file_observation(stat) origin_artifact_rule = artifact_rule_for_path(fallback_provider, str(path)) path_artifact = classify_artifact_path(path, provider=fallback_provider) + strong_path_artifact = strong_path_classification(path, provider=fallback_provider) if heartbeat is not None: heartbeat( "full_file_scan", @@ -2013,6 +2014,7 @@ def _ingest_full_paths_sync( fallback_provider is Provider.UNKNOWN and path.suffix.lower() == ".json" and path_artifact.kind is ArtifactKind.METADATA_DOCUMENT + and (strong_path_artifact is None or strong_path_artifact.parse_as_session) ) and not has_decoded_session_evidence(path, provider=fallback_provider) ): @@ -3881,7 +3883,11 @@ def _compact_superseded_raw_snapshots(self, paths: list[Path]) -> None: with closing(sqlite3.connect(source_db)) as conn, conn: conn.row_factory = sqlite3.Row try: - retention_authority = active_raw_retention_authority(conn, index_db_path=index_db) + retention_authority = active_raw_retention_authority( + conn, + index_db_path=index_db, + terminal_source_paths=paths, + ) except RawRetentionSafetyError as exc: logger.warning("live.watcher: skipped unsafe raw snapshot compaction: %s", exc) return diff --git a/polylogue/sources/live/watcher.py b/polylogue/sources/live/watcher.py index 0ed99a9fa2..d34cf4cfbc 100644 --- a/polylogue/sources/live/watcher.py +++ b/polylogue/sources/live/watcher.py @@ -1605,7 +1605,7 @@ def _source_accepts(self, path: Path) -> bool: return source.accepts(path) except OSError: continue - return path.suffix == ".jsonl" + return False def _is_hook_spool_path(self, path: Path) -> bool: for source in self._sources: @@ -1688,8 +1688,9 @@ def _enqueue_added_directory(self, directory: Path) -> None: dir_names[:] = [name for name in dir_names if not source.ignores_directory(Path(name))] for name in file_names: candidate = Path(parent) / name - if source.accepts(candidate): - self._enqueue(candidate) + canonical = self._canonical_watch_path(candidate) + if canonical is not None: + self._enqueue(canonical) def _watch_filter(self, _change: object, path: str) -> bool: """Accept configured source files under hidden canonical roots. diff --git a/polylogue/storage/raw_retention.py b/polylogue/storage/raw_retention.py index 228043adaa..29f88fa58f 100644 --- a/polylogue/storage/raw_retention.py +++ b/polylogue/storage/raw_retention.py @@ -381,6 +381,7 @@ def active_raw_retention_authority( conn: sqlite3.Connection, *, index_db_path: Path, + terminal_source_paths: Iterable[Path] | None = None, ) -> RawRetentionAuthority: """Return current protection plus explicitly authorized deletion rows. @@ -396,14 +397,15 @@ def active_raw_retention_authority( session_raw_ids, heads, eligible_receipts = _active_index_raw_authority(index_db_path) seeds = set(session_raw_ids) seeds.update(head.accepted_raw_id for head in heads) - all_raw_ids = frozenset(str(row[0]) for row in conn.execute("SELECT raw_id FROM raw_sessions").fetchall()) - terminal_artifact_raw_ids = _terminal_artifact_raw_ids(conn) if not seeds: + all_raw_ids = frozenset(str(row[0]) for row in conn.execute("SELECT raw_id FROM raw_sessions").fetchall()) + terminal_artifact_raw_ids = _terminal_artifact_raw_ids(conn) if all_raw_ids and all_raw_ids.issubset(terminal_artifact_raw_ids): return RawRetentionAuthority(protected_raw_ids=all_raw_ids, eligible_raw_ids=frozenset()) if all_raw_ids: raise RawRetentionSafetyError("source tier contains raw evidence but index has no raw authority") return RawRetentionAuthority(protected_raw_ids=frozenset(), eligible_raw_ids=frozenset()) + terminal_artifact_raw_ids = _terminal_artifact_raw_ids(conn, source_paths=terminal_source_paths) authority_raw_ids = seeds.union(receipt.raw_id for receipt in eligible_receipts) rows_by_id = _raw_revision_rows(conn, authority_raw_ids) protected: set[str] = set() @@ -1706,11 +1708,19 @@ def _terminal_artifact_paths(conn: sqlite3.Connection, source_paths: set[str]) - return result -def _terminal_artifact_raw_ids(conn: sqlite3.Connection) -> frozenset[str]: +def _terminal_artifact_raw_ids( + conn: sqlite3.Connection, + *, + source_paths: Iterable[Path] | None = None, +) -> frozenset[str]: """Return all retained raw evidence for paths with terminal current observations.""" - source_paths = {str(row[0]) for row in conn.execute("SELECT DISTINCT source_path FROM raw_sessions").fetchall()} - terminal_paths = _terminal_artifact_paths(conn, source_paths) + selected_paths = ( + {str(row[0]) for row in conn.execute("SELECT DISTINCT source_path FROM raw_sessions").fetchall()} + if source_paths is None + else {str(path) for path in source_paths} + ) + terminal_paths = _terminal_artifact_paths(conn, selected_paths) if not terminal_paths: return frozenset() raw_ids: set[str] = set() diff --git a/tests/unit/sources/test_live_batch_support.py b/tests/unit/sources/test_live_batch_support.py index 28ceb5c461..e5a43b3197 100644 --- a/tests/unit/sources/test_live_batch_support.py +++ b/tests/unit/sources/test_live_batch_support.py @@ -562,6 +562,30 @@ def test_full_ingest_unknown_weak_path_json_retains_terminal_evidence( assert artifact == expected_artifact +def test_full_ingest_unknown_weak_directory_still_excludes_strong_sidecar(tmp_path: Path) -> None: + """A weak directory cannot override a definitive non-session filename.""" + + root = tmp_path / "unknown" + path = root / "analysis" / "sessions-index.json" + path.parent.mkdir(parents=True) + path.write_text('{"mapping":{"looks":"conversational"}}', encoding="utf-8") + path_artifact = classify_artifact_path(path, provider=Provider.UNKNOWN) + assert path_artifact is not None and path_artifact.kind.value == "metadata_document" + processor = LiveBatchProcessor( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=tmp_path / "ops.db"))), + (WatchSource(name="unknown", root=root, suffixes=(".json",)),), + cursor=CursorStore(tmp_path / "ops.db"), + parser_fingerprint="test-parser", + ) + + result = processor._ingest_full_paths_sync([path], source_name="unknown") + + assert result.succeeded == [] + assert result.failed == [] + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT COUNT(*) FROM raw_sessions").fetchone() == (0,) + + def test_full_ingest_unknown_malformed_jsonl_records_terminal_decode_and_stops_retrying(tmp_path: Path) -> None: """Complete malformed JSONL lines are terminal decode evidence, not no-session evidence.""" root = tmp_path / "unknown" diff --git a/tests/unit/sources/test_live_watcher.py b/tests/unit/sources/test_live_watcher.py index 8ccc2fe6a9..ef42996a4d 100644 --- a/tests/unit/sources/test_live_watcher.py +++ b/tests/unit/sources/test_live_watcher.py @@ -1593,6 +1593,35 @@ def test_watch_filter_accepts_directories_but_not_unmatched_files_under_broad_ro assert watcher._watch_filter(object(), str(child_directory)) is True +def test_added_directory_scan_rejects_file_symlinks_escaping_source_root( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Recursive add recovery applies the same resolved containment as live events.""" + + root = tmp_path / "watched" + added = root / "new-directory" + added.mkdir(parents=True) + internal = added / "inside.jsonl" + internal.write_text("{}\n", encoding="utf-8") + external = tmp_path / "outside.jsonl" + external.write_text("secret\n", encoding="utf-8") + escaping = added / "escaping.jsonl" + escaping.symlink_to(external) + watcher, _full_ingest = _make_watcher( + tmp_path, + root, + sources=(WatchSource(name="codex", root=root, suffixes=(".jsonl",)),), + ) + enqueued: list[Path] = [] + monkeypatch.setattr(watcher, "_enqueue", enqueued.append) + + assert watcher._canonical_watch_path(escaping) is None + watcher._enqueue_added_directory(added) + + assert enqueued == [internal] + + def test_hermes_cursor_records_acquisition_revision_not_live_tail(tmp_path: Path) -> None: root = tmp_path / "hermes" root.mkdir() diff --git a/tests/unit/storage/test_raw_retention.py b/tests/unit/storage/test_raw_retention.py index b943be1dfe..a1ad439c46 100644 --- a/tests/unit/storage/test_raw_retention.py +++ b/tests/unit/storage/test_raw_retention.py @@ -417,6 +417,51 @@ def test_real_revision_receipt_authorizes_only_current_byte_head_supersession(tm ) +def test_scoped_terminal_retention_avoids_archive_wide_raw_inventory(tmp_path: Path) -> None: + """Healthy live compaction reads terminal authority only for its input paths.""" + + old_raw_id, new_raw_id = _seed_real_full_supersession(tmp_path) + source_db = tmp_path / "source.db" + source_path = tmp_path / "session.jsonl" + unrelated_path = tmp_path / "unrelated-terminal.json" + with sqlite3.connect(source_db) as conn: + conn.execute( + """ + INSERT INTO raw_sessions ( + raw_id, origin, native_id, source_path, source_index, blob_hash, blob_size, acquired_at_ms + ) VALUES ('raw-unrelated-terminal', 'unknown-export', 'terminal', ?, 0, ?, 1, 3) + """, + (str(unrelated_path), bytes.fromhex("03" * 32)), + ) + conn.execute( + """ + INSERT INTO raw_artifacts ( + artifact_id, raw_id, origin, source_path, source_index, artifact_kind, + support_status, classification_reason, parse_as_session, schema_eligible, + malformed_jsonl_lines, first_observed_at_ms, last_observed_at_ms + ) VALUES ('artifact-unrelated-terminal', 'raw-unrelated-terminal', 'unknown-export', ?, 0, + 'workflow_journal', 'unknown', 'terminal', 0, 0, 0, 3, 3) + """, + (str(unrelated_path),), + ) + conn.commit() + statements: list[str] = [] + conn.set_trace_callback(statements.append) + authority = active_raw_retention_authority( + conn, + index_db_path=tmp_path / "index.db", + terminal_source_paths=(source_path,), + ) + + normalized = {" ".join(statement.split()) for statement in statements} + assert "SELECT raw_id FROM raw_sessions" not in normalized + assert "SELECT DISTINCT source_path FROM raw_sessions" not in normalized + assert authority == RawRetentionAuthority( + protected_raw_ids=frozenset({new_raw_id}), + eligible_raw_ids=frozenset({old_raw_id}), + ) + + def test_semantic_head_receipt_authorizes_no_raw_deletion(tmp_path: Path) -> None: old_raw_id, new_raw_id = _seed_real_full_supersession(tmp_path) with sqlite3.connect(tmp_path / "index.db") as conn: From 584aed1fa3c743203a8aca552f6adf4cc345f663 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 02:18:49 +0200 Subject: [PATCH 14/65] docs(storage): define scoped retention authority --- polylogue/storage/raw_retention.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/polylogue/storage/raw_retention.py b/polylogue/storage/raw_retention.py index 29f88fa58f..c55dca24df 100644 --- a/polylogue/storage/raw_retention.py +++ b/polylogue/storage/raw_retention.py @@ -390,6 +390,9 @@ def active_raw_retention_authority( immutable ``superseded`` receipt tied to the current head authorizes raw deletion. Callers must serialize this read with source deletion under the daemon's single-writer contract, or stop the daemon for manual cleanup. + ``terminal_source_paths`` scopes terminal-artifact protection only for a + deletion operation constrained to those same physical paths; callers that + may delete archive-wide must leave it unset. """ original_row_factory = conn.row_factory conn.row_factory = sqlite3.Row From 041a9caa5f188d3ffa9a358f90a1c820015a862d Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 02:43:54 +0200 Subject: [PATCH 15/65] fix(retention): revoke stale terminal parse authority A successful reparse clears the raw failure state but retains its historical artifact carrier. Require current parse or validation failure before a failure-kind carrier can exempt a cursor path, while preserving ordinary non-session artifact authority. --- polylogue/storage/raw_retention.py | 24 +++++--- tests/unit/storage/test_raw_retention.py | 77 ++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 7 deletions(-) diff --git a/polylogue/storage/raw_retention.py b/polylogue/storage/raw_retention.py index c55dca24df..a2e9e1667c 100644 --- a/polylogue/storage/raw_retention.py +++ b/polylogue/storage/raw_retention.py @@ -1646,8 +1646,11 @@ def _terminal_artifact_paths(conn: sqlite3.Connection, source_paths: set[str]) - retain the source coordinate's latest receipt while ``raw_sessions`` retains its historical acquisition evidence, so authority attaches to each coordinate's newest raw observation rather than requiring a duplicate - receipt on every historical raw. Every ``(origin, source_index)`` member - of a physical path must be terminal before the cursor path is exempt. + receipt on every historical raw. A failure-kind carrier remains authority + only while that raw's current parse or validation state is failed; a later + successful reparse makes the retained carrier historical evidence. Every + ``(origin, source_index)`` member of a physical path must be terminal before + the cursor path is exempt. """ result: set[str] = set() @@ -1664,12 +1667,19 @@ def _terminal_artifact_paths(conn: sqlite3.Connection, source_paths: set[str]) - rows = conn.execute( f""" WITH terminal_artifacts AS ( - SELECT raw_id - FROM raw_artifacts - WHERE parse_as_session = 0 + SELECT artifact.raw_id + FROM raw_artifacts AS artifact + JOIN raw_sessions AS evidence_raw ON evidence_raw.raw_id = artifact.raw_id + WHERE artifact.parse_as_session = 0 AND ( - artifact_kind NOT IN ({raw_failure_placeholders}) - OR artifact_kind IN ({terminal_raw_failure_placeholders}) + artifact.artifact_kind NOT IN ({raw_failure_placeholders}) + OR ( + artifact.artifact_kind IN ({terminal_raw_failure_placeholders}) + AND ( + evidence_raw.parse_error IS NOT NULL + OR evidence_raw.validation_status = 'failed' + ) + ) ) ) SELECT DISTINCT terminal_raw.source_path diff --git a/tests/unit/storage/test_raw_retention.py b/tests/unit/storage/test_raw_retention.py index a1ad439c46..25d1538979 100644 --- a/tests/unit/storage/test_raw_retention.py +++ b/tests/unit/storage/test_raw_retention.py @@ -779,6 +779,83 @@ def test_resolution_carrier_cannot_authorize_cursor_without_accepted_head(tmp_pa assert snapshot.cursor_authority_gap_samples[0].state == "source_raws_without_accepted_head" +def test_successful_reparse_revokes_stale_terminal_failure_cursor_authority(tmp_path: Path) -> None: + """A successful production parse state makes an old terminal carrier historical.""" + + initialize_active_archive_root(tmp_path) + source_path = tmp_path / "reparsed-export.json" + with sqlite3.connect(tmp_path / "source.db") as conn: + conn.execute( + """ + INSERT INTO raw_sessions ( + raw_id, origin, native_id, source_path, source_index, + blob_hash, blob_size, acquired_at_ms, parse_error + ) VALUES (?, ?, ?, ?, 0, ?, 1, 1, ?) + """, + ( + "raw-terminal-reparsed", + "codex-session", + "reparsed", + str(source_path), + bytes(32), + "ValueError: unsupported export shape", + ), + ) + conn.execute( + """ + INSERT INTO raw_artifacts ( + artifact_id, raw_id, origin, source_path, source_index, + artifact_kind, support_status, classification_reason, + parse_as_session, schema_eligible, malformed_jsonl_lines, + first_observed_at_ms, last_observed_at_ms + ) VALUES (?, ?, ?, ?, 0, ?, ?, ?, 0, 0, 0, 1, 1) + """, + ( + "artifact-terminal-reparsed", + "raw-terminal-reparsed", + "codex-session", + str(source_path), + "terminal_unsupported_shape", + "unsupported_parseable", + "terminal parse failure", + ), + ) + conn.commit() + _seed_ops_cursor(tmp_path / "ops.db", source_path=source_path, byte_offset=1) + + with sqlite3.connect(tmp_path / "source.db") as conn: + before = raw_frontier_integrity_snapshot( + conn, + index_db_path=tmp_path / "index.db", + ops_db_path=tmp_path / "ops.db", + ) + assert before.cursor_ahead_status == "healthy" + + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + archive.mark_raw_parse_succeeded("raw-terminal-reparsed", provider=Provider.CODEX) + + with sqlite3.connect(tmp_path / "source.db") as conn: + state = conn.execute( + "SELECT parsed_at_ms, parse_error FROM raw_sessions WHERE raw_id = ?", + ("raw-terminal-reparsed",), + ).fetchone() + stale_artifact = conn.execute( + "SELECT artifact_kind FROM raw_artifacts WHERE raw_id = ?", + ("raw-terminal-reparsed",), + ).fetchone() + after = raw_frontier_integrity_snapshot( + conn, + index_db_path=tmp_path / "index.db", + ops_db_path=tmp_path / "ops.db", + ) + + assert state is not None and state[0] is not None and state[1] is None + assert stale_artifact == ("terminal_unsupported_shape",) + assert after.cursor_ahead_status == "unknown" + assert after.cursor_authority_gap_count == 1 + assert after.cursor_authority_gap_samples[0].state == "source_raws_without_accepted_head" + + def test_terminal_artifact_retention_batches_source_paths_below_sqlite_limit(tmp_path: Path) -> None: """Terminal evidence remains protectable when more than one SQL batch is needed.""" From f649f4c959553818efb8f085c87a9549c59892cd Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 03:03:46 +0200 Subject: [PATCH 16/65] fix(raw): clear stale validation after reparse --- .../archive_tiers/revision_governance.py | 7 +++++ polylogue/storage/sqlite/queries/raw_state.py | 4 +++ polylogue/storage/sqlite/raw_state_update.py | 5 +++- tests/unit/storage/test_parse_tracking.py | 14 +++++++++- tests/unit/storage/test_raw_retention.py | 26 ++++++++++++++++--- 5 files changed, 50 insertions(+), 6 deletions(-) diff --git a/polylogue/storage/sqlite/archive_tiers/revision_governance.py b/polylogue/storage/sqlite/archive_tiers/revision_governance.py index ba32d52a0d..20b9a73cfa 100644 --- a/polylogue/storage/sqlite/archive_tiers/revision_governance.py +++ b/polylogue/storage/sqlite/archive_tiers/revision_governance.py @@ -3490,6 +3490,13 @@ def _raw_parse_success_state(provider: Provider) -> RawSessionStateUpdate: parsed_at=datetime.now(UTC).isoformat(), parse_error=None, payload_provider=provider, + # Validation describes the previous parse result. Once a later parse + # succeeds, retaining either verdict would make stale validation + # evidence authoritative for different material. + validation_status=None, + validation_error=None, + validation_drift_count=0, + validation_mode=None, ) diff --git a/polylogue/storage/sqlite/queries/raw_state.py b/polylogue/storage/sqlite/queries/raw_state.py index 93f707ee69..3cfe7c8878 100644 --- a/polylogue/storage/sqlite/queries/raw_state.py +++ b/polylogue/storage/sqlite/queries/raw_state.py @@ -95,6 +95,10 @@ async def mark_raw_parsed( parsed_at=datetime.now(timezone.utc).isoformat(), parse_error=None, payload_provider=provider_token, + validation_status=None, + validation_error=None, + validation_drift_count=0, + validation_mode=None, ) else: state = RawSessionStateUpdate( diff --git a/polylogue/storage/sqlite/raw_state_update.py b/polylogue/storage/sqlite/raw_state_update.py index c145503b4f..d7ab38784f 100644 --- a/polylogue/storage/sqlite/raw_state_update.py +++ b/polylogue/storage/sqlite/raw_state_update.py @@ -53,7 +53,10 @@ def compile_raw_state_update( warnings = state.detection_warnings set_clauses.append("detection_warnings_json = ?") params.append(json.dumps([warnings[:2000]]) if isinstance(warnings, str) and warnings else "[]") - if state.validation_status is not UNSET or state.validation_error is not UNSET: + if state.validation_status is not UNSET: + set_clauses.append("validated_at_ms = ?") + params.append(now_ms if isinstance(state.validation_status, ValidationStatus) else None) + elif state.validation_error is not UNSET: set_clauses.append("validated_at_ms = ?") params.append(now_ms) return tuple(set_clauses), tuple(params) diff --git a/tests/unit/storage/test_parse_tracking.py b/tests/unit/storage/test_parse_tracking.py index 35a5251de4..87977eb732 100644 --- a/tests/unit/storage/test_parse_tracking.py +++ b/tests/unit/storage/test_parse_tracking.py @@ -70,15 +70,27 @@ async def test_mark_failure(self, backend: SQLiteBackend) -> None: assert rec.parse_error == "JSON decode error" async def test_mark_success_after_failure(self, backend: SQLiteBackend) -> None: - """Successful parse after failure clears the error.""" + """Successful parse clears stale parse and validation failures.""" await self._save_raw(backend) await backend.mark_raw_parsed("test-raw", error="first attempt failed") + await backend.mark_raw_validated( + "test-raw", + status="failed", + error="validation failed", + drift_count=2, + mode="strict", + ) await backend.mark_raw_parsed("test-raw") # Success rec = await backend.get_raw_session("test-raw") assert rec is not None assert rec.parsed_at is not None assert rec.parse_error is None + assert rec.validated_at is None + assert rec.validation_status is None + assert rec.validation_error is None + assert rec.validation_drift_count == 0 + assert rec.validation_mode is None async def test_error_truncation(self, backend: SQLiteBackend) -> None: """Long error messages are truncated to prevent DB bloat.""" diff --git a/tests/unit/storage/test_raw_retention.py b/tests/unit/storage/test_raw_retention.py index 25d1538979..1dba4e00ad 100644 --- a/tests/unit/storage/test_raw_retention.py +++ b/tests/unit/storage/test_raw_retention.py @@ -789,8 +789,10 @@ def test_successful_reparse_revokes_stale_terminal_failure_cursor_authority(tmp_ """ INSERT INTO raw_sessions ( raw_id, origin, native_id, source_path, source_index, - blob_hash, blob_size, acquired_at_ms, parse_error - ) VALUES (?, ?, ?, ?, 0, ?, 1, 1, ?) + blob_hash, blob_size, acquired_at_ms, parse_error, + validated_at_ms, validation_status, validation_error, + validation_drift_count, validation_mode + ) VALUES (?, ?, ?, ?, 0, ?, 1, 1, ?, 1, 'failed', ?, 3, 'strict') """, ( "raw-terminal-reparsed", @@ -799,6 +801,7 @@ def test_successful_reparse_revokes_stale_terminal_failure_cursor_authority(tmp_ str(source_path), bytes(32), "ValueError: unsupported export shape", + "schema validation failed", ), ) conn.execute( @@ -836,7 +839,12 @@ def test_successful_reparse_revokes_stale_terminal_failure_cursor_authority(tmp_ with sqlite3.connect(tmp_path / "source.db") as conn: state = conn.execute( - "SELECT parsed_at_ms, parse_error FROM raw_sessions WHERE raw_id = ?", + """ + SELECT parsed_at_ms, parse_error, validated_at_ms, validation_status, + validation_error, validation_drift_count, validation_mode + FROM raw_sessions + WHERE raw_id = ? + """, ("raw-terminal-reparsed",), ).fetchone() stale_artifact = conn.execute( @@ -849,7 +857,17 @@ def test_successful_reparse_revokes_stale_terminal_failure_cursor_authority(tmp_ ops_db_path=tmp_path / "ops.db", ) - assert state is not None and state[0] is not None and state[1] is None + assert state is not None + parsed_at_ms, parse_error, validated_at_ms, validation_status, validation_error, drift_count, mode = state + assert parsed_at_ms is not None + assert (parse_error, validated_at_ms, validation_status, validation_error, drift_count, mode) == ( + None, + None, + None, + None, + 0, + None, + ) assert stale_artifact == ("terminal_unsupported_shape",) assert after.cursor_ahead_status == "unknown" assert after.cursor_authority_gap_count == 1 From cd2bf4ab3dc6a84a417680f20d905cd2aff575e0 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 03:06:06 +0200 Subject: [PATCH 17/65] fix(raw): retain current validation after reparse --- polylogue/storage/raw_retention.py | 5 ++++- .../sqlite/archive_tiers/revision_governance.py | 7 ------- polylogue/storage/sqlite/queries/raw_state.py | 4 ---- polylogue/storage/sqlite/raw_state_update.py | 5 +---- tests/unit/storage/test_parse_tracking.py | 14 +------------- tests/unit/storage/test_raw_retention.py | 10 +++++----- 6 files changed, 11 insertions(+), 34 deletions(-) diff --git a/polylogue/storage/raw_retention.py b/polylogue/storage/raw_retention.py index a2e9e1667c..a1652b2b44 100644 --- a/polylogue/storage/raw_retention.py +++ b/polylogue/storage/raw_retention.py @@ -1677,7 +1677,10 @@ def _terminal_artifact_paths(conn: sqlite3.Connection, source_paths: set[str]) - artifact.artifact_kind IN ({terminal_raw_failure_placeholders}) AND ( evidence_raw.parse_error IS NOT NULL - OR evidence_raw.validation_status = 'failed' + OR ( + evidence_raw.validation_status = 'failed' + AND evidence_raw.parsed_at_ms IS NULL + ) ) ) ) diff --git a/polylogue/storage/sqlite/archive_tiers/revision_governance.py b/polylogue/storage/sqlite/archive_tiers/revision_governance.py index 20b9a73cfa..ba32d52a0d 100644 --- a/polylogue/storage/sqlite/archive_tiers/revision_governance.py +++ b/polylogue/storage/sqlite/archive_tiers/revision_governance.py @@ -3490,13 +3490,6 @@ def _raw_parse_success_state(provider: Provider) -> RawSessionStateUpdate: parsed_at=datetime.now(UTC).isoformat(), parse_error=None, payload_provider=provider, - # Validation describes the previous parse result. Once a later parse - # succeeds, retaining either verdict would make stale validation - # evidence authoritative for different material. - validation_status=None, - validation_error=None, - validation_drift_count=0, - validation_mode=None, ) diff --git a/polylogue/storage/sqlite/queries/raw_state.py b/polylogue/storage/sqlite/queries/raw_state.py index 3cfe7c8878..93f707ee69 100644 --- a/polylogue/storage/sqlite/queries/raw_state.py +++ b/polylogue/storage/sqlite/queries/raw_state.py @@ -95,10 +95,6 @@ async def mark_raw_parsed( parsed_at=datetime.now(timezone.utc).isoformat(), parse_error=None, payload_provider=provider_token, - validation_status=None, - validation_error=None, - validation_drift_count=0, - validation_mode=None, ) else: state = RawSessionStateUpdate( diff --git a/polylogue/storage/sqlite/raw_state_update.py b/polylogue/storage/sqlite/raw_state_update.py index d7ab38784f..c145503b4f 100644 --- a/polylogue/storage/sqlite/raw_state_update.py +++ b/polylogue/storage/sqlite/raw_state_update.py @@ -53,10 +53,7 @@ def compile_raw_state_update( warnings = state.detection_warnings set_clauses.append("detection_warnings_json = ?") params.append(json.dumps([warnings[:2000]]) if isinstance(warnings, str) and warnings else "[]") - if state.validation_status is not UNSET: - set_clauses.append("validated_at_ms = ?") - params.append(now_ms if isinstance(state.validation_status, ValidationStatus) else None) - elif state.validation_error is not UNSET: + if state.validation_status is not UNSET or state.validation_error is not UNSET: set_clauses.append("validated_at_ms = ?") params.append(now_ms) return tuple(set_clauses), tuple(params) diff --git a/tests/unit/storage/test_parse_tracking.py b/tests/unit/storage/test_parse_tracking.py index 87977eb732..35a5251de4 100644 --- a/tests/unit/storage/test_parse_tracking.py +++ b/tests/unit/storage/test_parse_tracking.py @@ -70,27 +70,15 @@ async def test_mark_failure(self, backend: SQLiteBackend) -> None: assert rec.parse_error == "JSON decode error" async def test_mark_success_after_failure(self, backend: SQLiteBackend) -> None: - """Successful parse clears stale parse and validation failures.""" + """Successful parse after failure clears the error.""" await self._save_raw(backend) await backend.mark_raw_parsed("test-raw", error="first attempt failed") - await backend.mark_raw_validated( - "test-raw", - status="failed", - error="validation failed", - drift_count=2, - mode="strict", - ) await backend.mark_raw_parsed("test-raw") # Success rec = await backend.get_raw_session("test-raw") assert rec is not None assert rec.parsed_at is not None assert rec.parse_error is None - assert rec.validated_at is None - assert rec.validation_status is None - assert rec.validation_error is None - assert rec.validation_drift_count == 0 - assert rec.validation_mode is None async def test_error_truncation(self, backend: SQLiteBackend) -> None: """Long error messages are truncated to prevent DB bloat.""" diff --git a/tests/unit/storage/test_raw_retention.py b/tests/unit/storage/test_raw_retention.py index 1dba4e00ad..342784477c 100644 --- a/tests/unit/storage/test_raw_retention.py +++ b/tests/unit/storage/test_raw_retention.py @@ -862,11 +862,11 @@ def test_successful_reparse_revokes_stale_terminal_failure_cursor_authority(tmp_ assert parsed_at_ms is not None assert (parse_error, validated_at_ms, validation_status, validation_error, drift_count, mode) == ( None, - None, - None, - None, - 0, - None, + 1, + "failed", + "schema validation failed", + 3, + "strict", ) assert stale_artifact == ("terminal_unsupported_shape",) assert after.cursor_ahead_status == "unknown" From 8eaa421eb778256b0f1f338dbb0e8873acf11be5 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 03:52:25 +0200 Subject: [PATCH 18/65] fix(raw): replay raws after successful reparse --- polylogue/storage/repair.py | 20 +++++++-- tests/unit/storage/test_repair.py | 68 +++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 4 deletions(-) diff --git a/polylogue/storage/repair.py b/polylogue/storage/repair.py index cc23d0a02c..9b08f23912 100644 --- a/polylogue/storage/repair.py +++ b/polylogue/storage/repair.py @@ -3949,10 +3949,14 @@ def _raw_materialization_candidate_ids( s_by_native.native_id IS NULL OR existing_native_raw.raw_id IS NULL ) - -- A failed worker validation is not replay authority. Keep - -- the raw bytes and their diagnostics, but require a fresh - -- validation outcome before materialization can select them. - AND COALESCE(r.validation_status, '') != 'failed' + -- A failed worker validation is replay authority only until a + -- successful parse records a durable parsed timestamp. Keep + -- that historical diagnostic, but do not let it block an + -- index reset from replaying successfully parsed raw bytes. + AND NOT ( + COALESCE(r.validation_status, '') = 'failed' + AND r.parsed_at_ms IS NULL + ) AND ( r.parse_error IS NULL OR r.parse_error = 'OperationalError: database is locked' @@ -3981,6 +3985,13 @@ def _raw_materialization_candidate_ids( AND (terminal_evidence.artifact_kind, terminal_evidence.support_status) IN ( {terminal_pair_placeholders} ) + AND ( + r.parse_error IS NOT NULL + OR ( + r.validation_status = 'failed' + AND r.parsed_at_ms IS NULL + ) + ) ) AND NOT ( COALESCE(r.validation_status, '') = 'skipped' @@ -4688,6 +4699,7 @@ def _raw_replay_plan_outcome( FROM raw_sessions WHERE raw_id IN ({placeholders}) AND validation_status = 'failed' + AND parsed_at_ms IS NULL UNION ALL SELECT 1 FROM raw_sessions diff --git a/tests/unit/storage/test_repair.py b/tests/unit/storage/test_repair.py index 4552ebb15e..3fa3c3e8f0 100644 --- a/tests/unit/storage/test_repair.py +++ b/tests/unit/storage/test_repair.py @@ -23,6 +23,7 @@ from polylogue.storage.blob_store import BlobStore from polylogue.storage.insights.session.repair_assessment import assess_session_insight_repairs from polylogue.storage.insights.session.runtime import SessionInsightCounts, SessionInsightStatusSnapshot +from polylogue.storage.raw.models import RawSessionStateUpdate from polylogue.storage.raw_authority import RawReplayPlan, RawReplayPlanOutcome from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root, initialize_archive_database from polylogue.storage.sqlite.archive_tiers.source_write import ArchiveSourceArtifact, upsert_raw_artifact @@ -828,6 +829,73 @@ def test_raw_materialization_validation_failure_cannot_reuse_deferred_authority( assert backlog["candidate_count"] == 0 +def test_raw_materialization_replays_successful_raw_with_historical_validation_failure(tmp_path: Path) -> None: + """Index reset replays a successful raw while retaining its failed-validation history.""" + from polylogue.core.enums import Provider + from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore + from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root + + initialize_active_archive_root(tmp_path) + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.CODEX, + payload=( + b'{"type":"session_meta","payload":{"id":"historical-validation"}}\n' + b'{"type":"response_item","payload":{"type":"message","id":"m1","role":"user",' + b'"content":[{"type":"input_text","text":"repair retained validation"}]}}\n' + ), + source_path="historical-validation.jsonl", + acquired_at_ms=1, + ) + + assert repair_mod.repair_raw_materialization(_config(tmp_path)).success is True + + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + archive.finalize_raw_parse_state( + raw_id, + state=RawSessionStateUpdate( + parsed_at=None, + parse_error=None, + validation_status="failed", + validation_error="validator rejected an earlier observation", + ), + ) + archive.record_raw_failure_evidence( + raw_id, + provider=Provider.CODEX, + source_path="historical-validation.jsonl", + source_index=0, + acquired_at_ms=1, + kind=RawFailureEvidenceKind.TERMINAL_CORRUPT_INPUT, + ) + archive.mark_raw_parse_succeeded(raw_id, provider=Provider.CODEX) + + with sqlite3.connect(tmp_path / "source.db") as conn: + raw_state = conn.execute( + "SELECT parsed_at_ms, parse_error, validation_status, validation_error FROM raw_sessions WHERE raw_id = ?", + (raw_id,), + ).fetchone() + assert raw_state is not None + assert raw_state[0] is not None + assert raw_state[1:] == (None, "failed", "validator rejected an earlier observation") + assert conn.execute( + "SELECT artifact_kind, support_status FROM raw_artifacts WHERE raw_id = ?", + (raw_id,), + ).fetchone() == ("terminal_corrupt_input", "decode_failed") + + # A reset removes only the derived projection; durable raw evidence and + # its historical validation diagnosis remain available to replay. + (tmp_path / "index.db").unlink() + initialize_archive_database(tmp_path / "index.db", ArchiveTier.INDEX) + + replay = repair_mod.repair_raw_materialization(_config(tmp_path)) + + assert replay.success is True + assert replay.repaired_count == 1 + with sqlite3.connect(tmp_path / "index.db") as conn: + assert conn.execute("SELECT COUNT(*) FROM sessions WHERE raw_id = ?", (raw_id,)).fetchone() == (1,) + + @pytest.mark.parametrize("artifact_kind", ["deferred_hot_jsonl_capture", "deferred_claude_code_partial_jsonl"]) def test_raw_materialization_does_not_replay_hot_partial_capture(tmp_path: Path, artifact_kind: str) -> None: """Hot partial evidence stays deferred until a complete source observation arrives.""" From 9e63e8f158d4c3705709e6e8a8da37f86afcd14c Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 04:47:20 +0200 Subject: [PATCH 19/65] fix(storage): close live archive authority gaps --- polylogue/browser_capture/receiver.py | 14 ++-- polylogue/daemon/convergence_stages.py | 5 +- polylogue/daemon/provenance.py | 2 +- polylogue/schemas/sampling_db.py | 11 +-- polylogue/sources/live/batch.py | 5 +- polylogue/sources/live/watcher.py | 23 ++++++- polylogue/storage/raw_retention.py | 67 ++++++++++--------- polylogue/storage/repair.py | 10 ++- .../storage/sqlite/archive_tiers/archive.py | 20 +++--- tests/unit/browser_capture/test_receiver.py | 49 ++++++++++++-- tests/unit/core/test_sampling.py | 29 ++++++++ tests/unit/daemon/test_provenance_endpoint.py | 19 ++++++ tests/unit/daemon/test_raw_parse_recovery.py | 3 + tests/unit/sources/test_live_batch_support.py | 37 ++++++++++ tests/unit/sources/test_live_watcher.py | 50 ++++++++++++++ .../storage/test_archive_tiers_archive.py | 15 +++++ tests/unit/storage/test_raw_retention.py | 37 +++++----- tests/unit/storage/test_repair.py | 11 ++- 18 files changed, 317 insertions(+), 90 deletions(-) diff --git a/polylogue/browser_capture/receiver.py b/polylogue/browser_capture/receiver.py index aa96933acc..4ad95ac4ce 100644 --- a/polylogue/browser_capture/receiver.py +++ b/polylogue/browser_capture/receiver.py @@ -39,6 +39,7 @@ browser_capture_receiver_token_path, browser_capture_spool_root, ) +from polylogue.storage.archive_identity import resolve_active_index_path from polylogue.storage.introspection import table_exists as _table_exists logger = get_logger(__name__) @@ -370,7 +371,7 @@ def _lookup_raw_archive_state( return _RawArchiveLookup() columns = _columns(conn, "raw_sessions") select = ["raw_id"] if "raw_id" in columns else [] - for optional in ("parse_error", "validation_error", "validation_status"): + for optional in ("parse_error", "validation_error", "validation_status", "parsed_at_ms"): if optional in columns: select.append(optional) if not select: @@ -404,13 +405,18 @@ def _lookup_raw_archive_state( validation_status = ( str(row["validation_status"]) if "validation_status" in row_keys and row["validation_status"] else None ) + validation_is_current = "parsed_at_ms" not in row_keys or row["parsed_at_ms"] is None if isinstance(parse_error, str) and parse_error: latest_failure = parse_error failure_source = "raw_parse" - elif isinstance(validation_error, str) and validation_error: + elif validation_is_current and isinstance(validation_error, str) and validation_error: latest_failure = validation_error failure_source = "raw_validation" - elif validation_status is not None and validation_status not in {"passed", "valid", "ok"}: + elif ( + validation_is_current + and validation_status is not None + and validation_status not in {"passed", "valid", "ok"} + ): latest_failure = validation_status failure_source = "raw_validation" return _RawArchiveLookup( @@ -432,7 +438,7 @@ def _lookup_index_archive_state( provider: str, provider_session_id: str, ) -> _IndexArchiveLookup: - conn = _open_readonly_sqlite(archive_root / "index.db") + conn = _open_readonly_sqlite(resolve_active_index_path(archive_root)) if conn is None: return _IndexArchiveLookup() try: diff --git a/polylogue/daemon/convergence_stages.py b/polylogue/daemon/convergence_stages.py index a511de244c..a2ce44d18e 100644 --- a/polylogue/daemon/convergence_stages.py +++ b/polylogue/daemon/convergence_stages.py @@ -926,7 +926,10 @@ def _raw_parse_recovery_pending_count(db_path: Path, path: Path, *, archive_root FROM raw_sessions AS r {materialized_join} WHERE (r.source_path = ? OR r.source_path LIKE ?) - AND COALESCE(r.validation_status, '') != 'failed' + AND NOT ( + COALESCE(r.validation_status, '') = 'failed' + AND r.parsed_at_ms IS NULL + ) AND ( ( r.parsed_at_ms IS NULL diff --git a/polylogue/daemon/provenance.py b/polylogue/daemon/provenance.py index 407a6dca9d..aea6e09a70 100644 --- a/polylogue/daemon/provenance.py +++ b/polylogue/daemon/provenance.py @@ -272,7 +272,7 @@ def _quarantine_state(row: ProvenanceRow) -> tuple[bool, str | None]: return True, "no_raw_artifact" if row.parse_error: return True, "parse_error" - if row.validation_status == "failed": + if row.validation_status == "failed" and row.parsed_at is None: return True, "validation_failed" return False, None diff --git a/polylogue/schemas/sampling_db.py b/polylogue/schemas/sampling_db.py index f072789dfb..5bf4c1f37d 100644 --- a/polylogue/schemas/sampling_db.py +++ b/polylogue/schemas/sampling_db.py @@ -74,6 +74,7 @@ class _RawSessionRow: blob_hash: bytes file_mtime_ms: int | None acquired_at_ms: int | None + parsed_at_ms: int | None validation_status: str | None @property @@ -117,6 +118,7 @@ def _coerce_schema_row(row: sqlite3.Row) -> _RawSessionRow: blob_hash=bytes(row["blob_hash"]) if row["blob_hash"] is not None else b"", file_mtime_ms=row["file_mtime_ms"], acquired_at_ms=row["acquired_at_ms"], + parsed_at_ms=row["parsed_at_ms"], validation_status=row["validation_status"], ) @@ -302,7 +304,7 @@ def _iter_schema_units_from_db( query = f""" WITH heads AS ( SELECT - source_path, origin, raw_id, blob_hash, file_mtime_ms, acquired_at_ms, + source_path, origin, raw_id, blob_hash, file_mtime_ms, acquired_at_ms, parsed_at_ms, validation_status, ROW_NUMBER() OVER ( PARTITION BY origin, {logical_cohort_expr} @@ -311,12 +313,13 @@ def _iter_schema_units_from_db( FROM raw_sessions WHERE origin IN ({placeholders}) ) - SELECT source_path, origin, raw_id, blob_hash, file_mtime_ms, acquired_at_ms, validation_status + SELECT source_path, origin, raw_id, blob_hash, file_mtime_ms, acquired_at_ms, parsed_at_ms, + validation_status FROM heads WHERE rn = 1 """ else: query = f""" - SELECT source_path, origin, raw_id, blob_hash, file_mtime_ms, acquired_at_ms, + SELECT source_path, origin, raw_id, blob_hash, file_mtime_ms, acquired_at_ms, parsed_at_ms, validation_status FROM raw_sessions WHERE origin IN ({placeholders}) @@ -367,7 +370,7 @@ def _iter_schema_units_from_db( ) continue - if row.validation_status == "failed": + if row.validation_status == "failed" and row.parsed_at_ms is None: _record_terminal( terminal_recorder, row, diff --git a/polylogue/sources/live/batch.py b/polylogue/sources/live/batch.py index 0b89f366b4..dec0645cc7 100644 --- a/polylogue/sources/live/batch.py +++ b/polylogue/sources/live/batch.py @@ -517,8 +517,7 @@ def _captured_jsonl_ends_at_record_boundary( blob_hash: str, blob_size: int, ) -> bool: - path = Path(source_path) - if not required or path.suffix.lower() not in {".jsonl", ".ndjson"}: + if not required or not is_jsonl_source_path(source_path): return True if blob_size <= 0: # A zero-byte capture has zero records -- none complete, none @@ -2241,7 +2240,7 @@ def _ingest_full_paths_sync( acquired_at=datetime.now(UTC).isoformat(), file_mtime=datetime.fromtimestamp(stat.st_mtime_ns / 1_000_000_000, UTC).isoformat(), captured_source_revision=raw_source_revisions.get(path, raw_id), - requires_complete_record_boundary=path.suffix.lower() in {".jsonl", ".ndjson"}, + requires_complete_record_boundary=is_jsonl_source_path(str(path)), ) ) raw_source_revisions.setdefault(path, raw_id) diff --git a/polylogue/sources/live/watcher.py b/polylogue/sources/live/watcher.py index d34cf4cfbc..e2b1434920 100644 --- a/polylogue/sources/live/watcher.py +++ b/polylogue/sources/live/watcher.py @@ -464,6 +464,12 @@ async def _retry_hook_spool_directory_until_populated(self, directory: Path) -> await self._drain_hook_spool() if not any(directory.glob("*.json")): return + except sqlite3.OperationalError: + # The normal periodic catch-up route retries transient source + # tier contention. A just-created shard must get the same + # treatment instead of letting this narrow event-ordering + # recovery task die before its envelope is acknowledged. + pass except OSError: return await asyncio.sleep(delay_s) @@ -1598,13 +1604,24 @@ def _source_name_for(self, path: Path) -> str: return path.parent.name def _source_accepts(self, path: Path) -> bool: - resolved = path.resolve() + try: + resolved = path.resolve() + except OSError: + return False + matches: list[tuple[int, WatchSource]] = [] for source in self._sources: try: - if resolved.is_relative_to(source.root.resolve()): - return source.accepts(path) + source_root = source.root.resolve() + if resolved.is_relative_to(source_root): + matches.append((len(source_root.parts), source)) except OSError: continue + if matches: + # Default roots deliberately overlap (~/.codex contains its + # sessions subroot). The deepest root owns a path, independent + # of declaration order or whether the roots were supplied by the + # defaults or explicit configuration. + return max(matches, key=lambda match: match[0])[1].accepts(path) return False def _is_hook_spool_path(self, path: Path) -> bool: diff --git a/polylogue/storage/raw_retention.py b/polylogue/storage/raw_retention.py index a1652b2b44..3217c72de5 100644 --- a/polylogue/storage/raw_retention.py +++ b/polylogue/storage/raw_retention.py @@ -1666,10 +1666,30 @@ def _terminal_artifact_paths(conn: sqlite3.Connection, source_paths: set[str]) - placeholders = ", ".join("?" for _ in batch) rows = conn.execute( f""" - WITH terminal_artifacts AS ( + WITH newest_per_coordinate AS ( + SELECT raw_id, source_path, origin, source_index, parse_error, validation_status, parsed_at_ms + FROM ( + SELECT + raw_id, + source_path, + origin, + source_index, + parse_error, + validation_status, + parsed_at_ms, + ROW_NUMBER() OVER ( + PARTITION BY source_path, origin, source_index + ORDER BY acquired_at_ms DESC, rowid DESC + ) AS coordinate_rank + FROM raw_sessions + WHERE source_path IN ({placeholders}) + ) + WHERE coordinate_rank = 1 + ), + terminal_artifacts AS ( SELECT artifact.raw_id FROM raw_artifacts AS artifact - JOIN raw_sessions AS evidence_raw ON evidence_raw.raw_id = artifact.raw_id + JOIN newest_per_coordinate AS evidence_raw ON evidence_raw.raw_id = artifact.raw_id WHERE artifact.parse_as_session = 0 AND ( artifact.artifact_kind NOT IN ({raw_failure_placeholders}) @@ -1687,38 +1707,19 @@ def _terminal_artifact_paths(conn: sqlite3.Connection, source_paths: set[str]) - ) SELECT DISTINCT terminal_raw.source_path FROM terminal_artifacts AS artifact - JOIN raw_sessions AS terminal_raw ON terminal_raw.raw_id = artifact.raw_id - WHERE terminal_raw.source_path IN ({placeholders}) - AND terminal_raw.raw_id = ( - SELECT newest.raw_id - FROM raw_sessions AS newest - WHERE newest.source_path = terminal_raw.source_path - AND newest.origin = terminal_raw.origin - AND newest.source_index = terminal_raw.source_index - ORDER BY newest.acquired_at_ms DESC, newest.rowid DESC - LIMIT 1 - ) - AND NOT EXISTS ( - SELECT 1 - FROM raw_sessions AS coordinate - WHERE coordinate.source_path = terminal_raw.source_path - AND coordinate.raw_id = ( - SELECT newest.raw_id - FROM raw_sessions AS newest - WHERE newest.source_path = coordinate.source_path - AND newest.origin = coordinate.origin - AND newest.source_index = coordinate.source_index - ORDER BY newest.acquired_at_ms DESC, newest.rowid DESC - LIMIT 1 - ) - AND NOT EXISTS ( - SELECT 1 - FROM terminal_artifacts AS current_artifact - WHERE current_artifact.raw_id = coordinate.raw_id - ) - ) + JOIN newest_per_coordinate AS terminal_raw ON terminal_raw.raw_id = artifact.raw_id + WHERE NOT EXISTS ( + SELECT 1 + FROM newest_per_coordinate AS coordinate + WHERE coordinate.source_path = terminal_raw.source_path + AND NOT EXISTS ( + SELECT 1 + FROM terminal_artifacts AS current_artifact + WHERE current_artifact.raw_id = coordinate.raw_id + ) + ) """, - (*raw_failure_kinds, *terminal_raw_failure_kinds, *batch), + (*batch, *raw_failure_kinds, *terminal_raw_failure_kinds), ).fetchall() result.update(str(row[0]) for row in rows) return result diff --git a/polylogue/storage/repair.py b/polylogue/storage/repair.py index 9b08f23912..99712976a5 100644 --- a/polylogue/storage/repair.py +++ b/polylogue/storage/repair.py @@ -4768,6 +4768,7 @@ def _raw_replay_plan_outcome( def _raw_replay_plan_outcomes( archive_root: Path, + index_db: Path, plans: Sequence[RawReplayPlan], *, remaining: RawMaterializationCandidates, @@ -4777,7 +4778,7 @@ def _raw_replay_plan_outcomes( return () with closing(sqlite3.connect(f"file:{archive_root / 'source.db'}?mode=ro", uri=True)) as conn: conn.row_factory = sqlite3.Row - conn.execute("ATTACH DATABASE ? AS index_tier", (str(archive_root / "index.db"),)) + conn.execute("ATTACH DATABASE ? AS index_tier", (str(index_db),)) return tuple( _raw_replay_plan_outcome(conn, plan, remaining=remaining, no_progress=no_progress) for plan in plans ) @@ -6840,7 +6841,8 @@ def _pass_deadline_exceeded() -> bool: # writer-hot table before this bounded live pass; this is the same # planner invariant seeded for a fresh index bootstrap, without turning # raw materialization into a full rebuild. - with closing(sqlite3.connect(archive_root / "index.db", timeout=60)) as planner_conn: + index_db = _raw_materialization_index_path(config, archive_root) + with closing(sqlite3.connect(index_db, timeout=60)) as planner_conn: planner_conn.execute("PRAGMA busy_timeout = 60000") # A freshly reset index uses representative bootstrap statistics. # ``ANALYZE blocks`` on an empty table deletes that seed and brings @@ -6996,7 +6998,9 @@ def _pass_deadline_exceeded() -> bool: # ``_raw_replay_plan_outcome`` types this TERMINAL (not RETRYABLE) so # it stops being silently reselected forever. no_progress = part.replayed_logical_sources == 0 and part.quarantined == 0 and part.adoption_deferred == 0 - component_outcomes = _raw_replay_plan_outcomes(archive_root, [plan], remaining=current, no_progress=no_progress) + component_outcomes = _raw_replay_plan_outcomes( + archive_root, index_db, [plan], remaining=current, no_progress=no_progress + ) for outcome in component_outcomes: application_receipt = raw_replay_application_receipt(archive_root, plan) receipted = dataclasses.replace(outcome, application_receipt=application_receipt) diff --git a/polylogue/storage/sqlite/archive_tiers/archive.py b/polylogue/storage/sqlite/archive_tiers/archive.py index 9875935900..65abd3adb6 100644 --- a/polylogue/storage/sqlite/archive_tiers/archive.py +++ b/polylogue/storage/sqlite/archive_tiers/archive.py @@ -1867,16 +1867,6 @@ def _initialize_store( ) -> None: self.archive_root = archive_root self.source_db_path = archive_root / "source.db" - if self._frozen_index_path is not None: - self.index_db_path = self._frozen_index_path - else: - # The configured root owns the durable tiers, while an active - # generation can keep index.db elsewhere. A writable open must - # follow the same pointer as readiness and live ingest instead of - # silently mutating a stale conventional root/index.db shadow. - from polylogue.storage.archive_identity import resolve_active_index_path - - self.index_db_path = resolve_active_index_path(archive_root) self.embeddings_db_path = archive_root / "embeddings.db" self.user_db_path = archive_root / "user.db" self.ops_db_path = archive_root / "ops.db" @@ -1913,6 +1903,16 @@ def _initialize_store( self._tags_relation = "session_tags" self._blob_publisher = ArchiveBlobPublisher(self.source_db_path, self.archive_root / "blob") return + if self._frozen_index_path is not None: + self.index_db_path = self._frozen_index_path + else: + # The configured root owns the durable tiers, while an active + # generation can keep index.db elsewhere. A writable open must + # follow the same pointer as readiness and live ingest instead of + # silently mutating a stale conventional root/index.db shadow. + from polylogue.storage.archive_identity import resolve_active_index_path + + self.index_db_path = resolve_active_index_path(archive_root) if self._frozen_source_validation: # Candidate admission derives every decision from source.db and # frozen blob bytes. Requiring an index handle here would make the diff --git a/tests/unit/browser_capture/test_receiver.py b/tests/unit/browser_capture/test_receiver.py index f2f6aaadf8..3c55193618 100644 --- a/tests/unit/browser_capture/test_receiver.py +++ b/tests/unit/browser_capture/test_receiver.py @@ -100,6 +100,8 @@ def _seed_browser_capture_archive( raw_id: str = "raw-capture", message_count: int = 1, parse_error: str | None = None, + validation_status: str | None = None, + parsed_at_ms: int | None = None, updated_at_ms: int | None = None, ) -> None: with sqlite3.connect(archive_root / "source.db") as conn: @@ -110,16 +112,27 @@ def _seed_browser_capture_archive( origin TEXT, native_id TEXT, source_path TEXT, - parse_error TEXT + parse_error TEXT, + validation_status TEXT, + parsed_at_ms INTEGER ) """ ) conn.execute( """ - INSERT INTO raw_sessions (raw_id, origin, native_id, source_path, parse_error) - VALUES (?, ?, ?, ?, ?) + INSERT INTO raw_sessions ( + raw_id, origin, native_id, source_path, parse_error, validation_status, parsed_at_ms + ) VALUES (?, ?, ?, ?, ?, ?, ?) """, - (raw_id, "chatgpt-export", native_id, f"browser-capture/chatgpt/{native_id}.json", parse_error), + ( + raw_id, + "chatgpt-export", + native_id, + f"browser-capture/chatgpt/{native_id}.json", + parse_error, + validation_status, + parsed_at_ms, + ), ) with sqlite3.connect(archive_root / "index.db") as conn: conn.execute( @@ -763,6 +776,34 @@ def test_receiver_archive_state_surfaces_raw_failure(tmp_path: Path) -> None: assert state.failure_source == "raw_parse" +def test_receiver_uses_active_index_and_ignores_historical_validation_failure(tmp_path: Path) -> None: + """The public state reads the promoted generation, not its stale shadow.""" + envelope = BrowserCaptureEnvelope.model_validate(_payload()) + write_capture_envelope(envelope, spool_path=tmp_path) + _seed_browser_capture_archive( + tmp_path, + validation_status="failed", + parsed_at_ms=1, + message_count=0, + ) + active_index = tmp_path / "generations" / "active" / "index.db" + active_index.parent.mkdir(parents=True) + with sqlite3.connect(active_index) as conn: + conn.execute( + "CREATE TABLE sessions (session_id TEXT, raw_id TEXT, native_id TEXT, message_count INTEGER, updated_at_ms INTEGER)" + ) + conn.execute("INSERT INTO sessions VALUES ('chatgpt-export:conv-123', 'raw-capture', 'conv-123', 1, NULL)") + (tmp_path / ".index-active-pointer").write_text(f"{active_index}\n", encoding="utf-8") + + state = BrowserCaptureArchiveStatePayload.model_validate( + existing_capture_state("chatgpt", "conv-123", spool_path=tmp_path, archive_root=tmp_path) + ) + + assert state.state == "archived" + assert state.latest_failure is None + assert state.indexed_message_count == 1 + + def test_receiver_echoes_safe_request_id_header(tmp_path: Path) -> None: with _running_receiver(tmp_path) as (host, port): conn = HTTPConnection(host, port) diff --git a/tests/unit/core/test_sampling.py b/tests/unit/core/test_sampling.py index bda1bb458e..3aa39d22a4 100644 --- a/tests/unit/core/test_sampling.py +++ b/tests/unit/core/test_sampling.py @@ -262,6 +262,35 @@ def test_claude_ai_reads_db_rows_stored_under_claude(self, tmp_path: Path) -> No assert len(result) == 1 assert result[0]["uuid"] == "conv-1" + def test_sampling_keeps_successfully_reparsed_historical_validation_failure(self, tmp_path: Path) -> None: + db = _archive_index_db(tmp_path) + _insert_raw_session( + db_path=db, + origin="claude-ai-export", + source_path="/tmp/sessions.json", + raw_content=json.dumps( + [ + { + "uuid": "reparsed", + "name": "Retained", + "summary": "successful reparse", + "created_at": "2026-01-01T00:00:00Z", + "updated_at": "2026-01-01T00:05:00Z", + "account": {"uuid": "acct-reparsed"}, + "chat_messages": [], + } + ] + ).encode(), + ) + with sqlite3.connect(db.with_name("source.db")) as conn: + conn.execute("UPDATE raw_sessions SET parsed_at_ms = 1, validation_status = 'failed'") + conn.commit() + + result = load_samples_from_db("claude-ai", db_path=db) + + assert len(result) == 1 + assert result[0]["uuid"] == "reparsed" + def test_record_provider_sampling_streams_without_full_envelope( self, tmp_path: Path, diff --git a/tests/unit/daemon/test_provenance_endpoint.py b/tests/unit/daemon/test_provenance_endpoint.py index 49f59431ef..06e7841d85 100644 --- a/tests/unit/daemon/test_provenance_endpoint.py +++ b/tests/unit/daemon/test_provenance_endpoint.py @@ -387,6 +387,7 @@ def test_quarantine_surfaces_when_validation_failed(self, workspace_env: dict[st raw_id=raw_id, source_path="/tmp/x.json", blob_size=len(payload_bytes), + parsed_at_ms=None, validation_status="failed", ) @@ -395,6 +396,24 @@ def test_quarantine_surfaces_when_validation_failed(self, workspace_env: dict[st assert result["quarantined"] is True assert result["quarantine_reason"] == "validation_failed" + def test_historical_validation_failure_is_not_current_quarantine(self, workspace_env: dict[str, Path]) -> None: + raw_id = _seed_raw_blob(b"{}") + session_id = _seed_archive_provenance( + session_id="c-historical-validation", + raw_id=raw_id, + source_path="/tmp/x.json", + blob_size=2, + validation_status="failed", + ) + + result = build_provenance_payload(session_id) + + assert result is not None + assert result["validation_status"] == "failed" + assert result["parsed_at"] is not None + assert result["quarantined"] is False + assert result["quarantine_reason"] is None + def test_quarantine_surfaces_when_no_raw_artifact(self, workspace_env: dict[str, Path]) -> None: session_id = _seed_archive_provenance( session_id="c-orphan", diff --git a/tests/unit/daemon/test_raw_parse_recovery.py b/tests/unit/daemon/test_raw_parse_recovery.py index 513c3b0ac6..d37bbba808 100644 --- a/tests/unit/daemon/test_raw_parse_recovery.py +++ b/tests/unit/daemon/test_raw_parse_recovery.py @@ -331,6 +331,9 @@ def test_raw_parse_recovery_drains_previously_parsed_cas_frontier_failure(tmp_pa provider=Provider.CHATGPT, error=RawCASFrontierError("frontier changed after parsing completed"), ) + with sqlite3.connect(tmp_path / "source.db") as conn: + conn.execute("UPDATE raw_sessions SET validation_status = 'failed' WHERE raw_id = ?", (raw_id,)) + conn.commit() stage = make_raw_parse_recovery_stage(tmp_path / "index.db") diff --git a/tests/unit/sources/test_live_batch_support.py b/tests/unit/sources/test_live_batch_support.py index e5a43b3197..15e05ea3f9 100644 --- a/tests/unit/sources/test_live_batch_support.py +++ b/tests/unit/sources/test_live_batch_support.py @@ -762,6 +762,43 @@ def grow_source_after_capture(**kwargs: object) -> bool: assert artifact == ("deferred_hot_jsonl_capture", "partial_decode", 1) +def test_full_ingest_applies_incomplete_record_guard_to_jsonl_txt( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The supported ``.jsonl.txt`` wire suffix has JSONL tail authority too.""" + from polylogue.sources.live import batch as live_batch + + root = tmp_path / "sessions" + root.mkdir() + path = root / "active.jsonl.txt" + captured = b'{"type":"session_meta"' + path.write_bytes(captured) + processor = LiveBatchProcessor( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=tmp_path / "archive.sqlite"))), + (WatchSource(name="codex", root=root),), + cursor=CursorStore(tmp_path / "archive.sqlite"), + parser_fingerprint="test-parser", + ) + monkeypatch.setattr( + "polylogue.sources.live.batch._jsonl_provider_and_session_artifact", + lambda _path, fallback_provider: (fallback_provider, True), + ) + boundary_check = live_batch._captured_jsonl_ends_at_record_boundary + + def grow_source_after_capture(**kwargs: object) -> bool: + path.write_bytes(captured + b"\n") + return boundary_check(**kwargs) # type: ignore[arg-type] + + monkeypatch.setattr(live_batch, "_captured_jsonl_ends_at_record_boundary", grow_source_after_capture) + + result = processor._ingest_full_paths_sync([path], source_name="codex") + + assert result.succeeded == [path] + _parsed_at_ms, parse_error = _raw_parse_state(tmp_path) + assert isinstance(parse_error, str) and parse_error.endswith("complete record boundary") + + def test_full_ingest_claude_partial_jsonl_has_provider_specific_evidence( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/sources/test_live_watcher.py b/tests/unit/sources/test_live_watcher.py index ef42996a4d..21ecae7be9 100644 --- a/tests/unit/sources/test_live_watcher.py +++ b/tests/unit/sources/test_live_watcher.py @@ -3535,6 +3535,56 @@ def test_watch_source_accepts_configured_suffixes(tmp_path: Path) -> None: assert src.accepts(tmp_path / "README.md") is False +def test_source_accepts_prefers_most_specific_nested_root(tmp_path: Path) -> None: + """A nested explicit root owns its files regardless of source order.""" + root = tmp_path / "codex" + sessions = root / "sessions" + sessions.mkdir(parents=True) + path = sessions / "session.jsonl" + watcher = LiveWatcher( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=tmp_path / "index.db"))), + ( + WatchSource(name="codex-state", root=root, suffixes=(".sqlite",)), + WatchSource(name="codex", root=sessions, suffixes=(".jsonl",)), + ), + cursor=CursorStore(tmp_path / "cursor.db"), + ) + + try: + assert watcher._source_accepts(path) is True + finally: + watcher._parse_stage.shutdown() + + +@pytest.mark.asyncio +async def test_hook_spool_directory_retry_retries_sqlite_operational_error(tmp_path: Path) -> None: + """A transient spool-drain lock follows the normal delayed retry path.""" + shard = tmp_path / "pending" / "2026-08-13" + shard.mkdir(parents=True) + (shard / "event.json").write_text("{}", encoding="utf-8") + watcher = LiveWatcher( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=tmp_path / "index.db"))), + (), + cursor=CursorStore(tmp_path / "cursor.db"), + ) + calls = 0 + + async def drain() -> None: + nonlocal calls + calls += 1 + if calls == 1: + raise sqlite3.OperationalError("database is locked") + (shard / "event.json").unlink() + + watcher._drain_hook_spool = drain # type: ignore[method-assign] + try: + await watcher._retry_hook_spool_directory_until_populated(shard) + finally: + watcher._parse_stage.shutdown() + + assert calls == 2 + + def test_inbox_source_accepts_zip_and_archive_formats() -> None: """#1683: inbox must accept .zip (GDPR exports), .json, .jsonl, .ndjson.""" from polylogue.sources.live.watcher import default_sources diff --git a/tests/unit/storage/test_archive_tiers_archive.py b/tests/unit/storage/test_archive_tiers_archive.py index 41fb39da51..7444950709 100644 --- a/tests/unit/storage/test_archive_tiers_archive.py +++ b/tests/unit/storage/test_archive_tiers_archive.py @@ -105,6 +105,21 @@ def acquire_then_replace( assert not (root / ".maintenance-state").exists() +def test_source_tier_acquisition_does_not_resolve_active_index(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Acquire-only writes remain available while the derived pointer is unreadable.""" + from polylogue.storage import archive_identity + + initialize_active_archive_root(tmp_path) + monkeypatch.setattr( + archive_identity, + "resolve_active_index_path", + lambda _root: (_ for _ in ()).throw(AssertionError("source acquisition must not resolve index")), + ) + + with ArchiveStore.open_source_tier_acquisition(tmp_path) as archive: + assert archive.source_db_path == tmp_path / "source.db" + + def test_active_archive_root_facade_writes_reads_and_searches_archive_db(tmp_path: Path) -> None: session = ParsedSession( source_name=Provider.CODEX, diff --git a/tests/unit/storage/test_raw_retention.py b/tests/unit/storage/test_raw_retention.py index 342784477c..909a9a6b62 100644 --- a/tests/unit/storage/test_raw_retention.py +++ b/tests/unit/storage/test_raw_retention.py @@ -12,6 +12,7 @@ from polylogue.archive.revision_authority import RawRevisionAuthority, RawRevisionEnvelope, RawRevisionKind from polylogue.core.enums import Provider from polylogue.sources.parsers.base import ParsedMessage, ParsedSession +from polylogue.storage import raw_retention as raw_retention_mod from polylogue.storage.archive_readiness import raw_materialization_readiness_snapshot from polylogue.storage.blob_store import BlobStore from polylogue.storage.raw_retention import ( @@ -418,20 +419,19 @@ def test_real_revision_receipt_authorizes_only_current_byte_head_supersession(tm def test_scoped_terminal_retention_avoids_archive_wide_raw_inventory(tmp_path: Path) -> None: - """Healthy live compaction reads terminal authority only for its input paths.""" + """Terminal authority scans only the caller's source-path scope.""" - old_raw_id, new_raw_id = _seed_real_full_supersession(tmp_path) source_db = tmp_path / "source.db" - source_path = tmp_path / "session.jsonl" - unrelated_path = tmp_path / "unrelated-terminal.json" + source_path = tmp_path / "terminal.json" + initialize_archive_database(source_db, ArchiveTier.SOURCE) with sqlite3.connect(source_db) as conn: conn.execute( """ INSERT INTO raw_sessions ( raw_id, origin, native_id, source_path, source_index, blob_hash, blob_size, acquired_at_ms - ) VALUES ('raw-unrelated-terminal', 'unknown-export', 'terminal', ?, 0, ?, 1, 3) + ) VALUES ('raw-terminal', 'unknown-export', 'terminal', ?, 0, ?, 1, 3) """, - (str(unrelated_path), bytes.fromhex("03" * 32)), + (str(source_path), bytes.fromhex("03" * 32)), ) conn.execute( """ @@ -439,27 +439,22 @@ def test_scoped_terminal_retention_avoids_archive_wide_raw_inventory(tmp_path: P artifact_id, raw_id, origin, source_path, source_index, artifact_kind, support_status, classification_reason, parse_as_session, schema_eligible, malformed_jsonl_lines, first_observed_at_ms, last_observed_at_ms - ) VALUES ('artifact-unrelated-terminal', 'raw-unrelated-terminal', 'unknown-export', ?, 0, + ) VALUES ('artifact-terminal', 'raw-terminal', 'unknown-export', ?, 0, 'workflow_journal', 'unknown', 'terminal', 0, 0, 0, 3, 3) """, - (str(unrelated_path),), + (str(source_path),), ) conn.commit() statements: list[str] = [] conn.set_trace_callback(statements.append) - authority = active_raw_retention_authority( - conn, - index_db_path=tmp_path / "index.db", - terminal_source_paths=(source_path,), - ) - - normalized = {" ".join(statement.split()) for statement in statements} - assert "SELECT raw_id FROM raw_sessions" not in normalized - assert "SELECT DISTINCT source_path FROM raw_sessions" not in normalized - assert authority == RawRetentionAuthority( - protected_raw_ids=frozenset({new_raw_id}), - eligible_raw_ids=frozenset({old_raw_id}), - ) + terminal_paths = raw_retention_mod._terminal_artifact_paths(conn, {str(source_path)}) + + raw_reads = [" ".join(statement.split()).upper() for statement in statements if "RAW_SESSIONS" in statement.upper()] + assert raw_reads + # Every raw_sessions scan in this cursor-scoped route must carry the + # source-path scope. Assert the SQL shape, not one historical rendering. + assert all("SOURCE_PATH IN (" in statement for statement in raw_reads) + assert terminal_paths == {str(source_path)} def test_semantic_head_receipt_authorizes_no_raw_deletion(tmp_path: Path) -> None: diff --git a/tests/unit/storage/test_repair.py b/tests/unit/storage/test_repair.py index 3fa3c3e8f0..4ade56413b 100644 --- a/tests/unit/storage/test_repair.py +++ b/tests/unit/storage/test_repair.py @@ -884,14 +884,19 @@ def test_raw_materialization_replays_successful_raw_with_historical_validation_f ).fetchone() == ("terminal_corrupt_input", "decode_failed") # A reset removes only the derived projection; durable raw evidence and - # its historical validation diagnosis remain available to replay. - (tmp_path / "index.db").unlink() - initialize_archive_database(tmp_path / "index.db", ArchiveTier.INDEX) + # its historical validation diagnosis remain available to replay. Leave + # the populated conventional index as a stale shadow: the production + # planner and replay postcondition must use this promoted empty generation. + active_index = tmp_path / "generations" / "active" / "index.db" + initialize_archive_database(active_index, ArchiveTier.INDEX) + (tmp_path / ".index-active-pointer").write_text(f"{active_index}\n", encoding="utf-8") replay = repair_mod.repair_raw_materialization(_config(tmp_path)) assert replay.success is True assert replay.repaired_count == 1 + with sqlite3.connect(active_index) as conn: + assert conn.execute("SELECT COUNT(*) FROM sessions WHERE raw_id = ?", (raw_id,)).fetchone() == (1,) with sqlite3.connect(tmp_path / "index.db") as conn: assert conn.execute("SELECT COUNT(*) FROM sessions WHERE raw_id = ?", (raw_id,)).fetchone() == (1,) From 8f57ade3fa4315bfd4f21bb6d9873f8333aa0f48 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 05:18:30 +0200 Subject: [PATCH 20/65] fix(watcher): surface non-transient spool failures Retry only SQLite busy or locked errors while draining a newly created hook shard. Retrieve detached task results and log unexpected failures so schema or corruption errors cannot disappear as unhandled task warnings. --- polylogue/sources/live/watcher.py | 21 +++++++-- tests/unit/sources/test_live_watcher.py | 62 +++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 3 deletions(-) diff --git a/polylogue/sources/live/watcher.py b/polylogue/sources/live/watcher.py index e2b1434920..71922afda7 100644 --- a/polylogue/sources/live/watcher.py +++ b/polylogue/sources/live/watcher.py @@ -448,6 +448,12 @@ def _schedule_hook_spool_directory_retry(self, directory: Path) -> None: def discard_completed_task(completed: asyncio.Task[None]) -> None: if self._hook_spool_directory_retry_tasks.get(directory) is completed: self._hook_spool_directory_retry_tasks.pop(directory, None) + if completed.cancelled(): + return + try: + completed.result() + except Exception: + logger.exception("live.watcher: hook spool directory retry failed for %s", directory) task.add_done_callback(discard_completed_task) @@ -464,12 +470,14 @@ async def _retry_hook_spool_directory_until_populated(self, directory: Path) -> await self._drain_hook_spool() if not any(directory.glob("*.json")): return - except sqlite3.OperationalError: + except sqlite3.OperationalError as exc: # The normal periodic catch-up route retries transient source # tier contention. A just-created shard must get the same # treatment instead of letting this narrow event-ordering # recovery task die before its envelope is acknowledged. - pass + if not _is_database_locked(exc): + raise + logger.warning("live.watcher: archive busy while draining new hook shard; will retry") except OSError: return await asyncio.sleep(delay_s) @@ -1867,7 +1875,14 @@ def _cursor_db_path(polylogue: Polylogue) -> Path: def _is_database_locked(exc: sqlite3.OperationalError) -> bool: - return "database is locked" in str(exc).lower() + error_code = getattr(exc, "sqlite_errorcode", None) + if error_code in {sqlite3.SQLITE_BUSY, sqlite3.SQLITE_LOCKED}: + return True + message = str(exc).lower() + return any( + locked_message in message + for locked_message in ("database is locked", "database table is locked", "database schema is locked") + ) def _cursor_age_exceeds(cursor: CursorRecord, min_age_s: float) -> bool: diff --git a/tests/unit/sources/test_live_watcher.py b/tests/unit/sources/test_live_watcher.py index 21ecae7be9..c11e71d8d1 100644 --- a/tests/unit/sources/test_live_watcher.py +++ b/tests/unit/sources/test_live_watcher.py @@ -3585,6 +3585,68 @@ async def drain() -> None: assert calls == 2 +@pytest.mark.asyncio +async def test_hook_spool_directory_retry_rejects_non_lock_sqlite_error(tmp_path: Path) -> None: + """A corrupt or incompatible spool database is not misclassified as contention.""" + + shard = tmp_path / "pending" / "2026-08-13" + shard.mkdir(parents=True) + (shard / "event.json").write_text("{}", encoding="utf-8") + watcher = LiveWatcher( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=tmp_path / "index.db"))), + (), + cursor=CursorStore(tmp_path / "cursor.db"), + ) + + async def drain() -> None: + raise sqlite3.OperationalError("no such table: hook_events") + + watcher._drain_hook_spool = drain # type: ignore[method-assign] + try: + with pytest.raises(sqlite3.OperationalError, match="no such table"): + await watcher._retry_hook_spool_directory_until_populated(shard) + finally: + watcher._parse_stage.shutdown() + + +@pytest.mark.asyncio +async def test_scheduled_hook_spool_retry_observes_and_logs_failure( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A failed detached retry is retrieved and reported instead of becoming an unhandled task.""" + + shard = tmp_path / "pending" / "2026-08-13" + shard.mkdir(parents=True) + (shard / "event.json").write_text("{}", encoding="utf-8") + watcher = LiveWatcher( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=tmp_path / "index.db"))), + (), + cursor=CursorStore(tmp_path / "cursor.db"), + ) + recorded_logger = MagicMock() + monkeypatch.setattr(live_watcher, "logger", recorded_logger) + + async def drain() -> None: + raise sqlite3.OperationalError("database disk image is malformed") + + watcher._drain_hook_spool = drain # type: ignore[method-assign] + try: + watcher._schedule_hook_spool_directory_retry(shard) + task = watcher._hook_spool_directory_retry_tasks[shard.resolve()] + while not task.done(): + await asyncio.sleep(0) + await asyncio.sleep(0) + finally: + watcher._parse_stage.shutdown() + + assert watcher._hook_spool_directory_retry_tasks == {} + recorded_logger.exception.assert_called_once() + assert recorded_logger.exception.call_args.args == ( + "live.watcher: hook spool directory retry failed for %s", + shard.resolve(), + ) + + def test_inbox_source_accepts_zip_and_archive_formats() -> None: """#1683: inbox must accept .zip (GDPR exports), .json, .jsonl, .ndjson.""" from polylogue.sources.live.watcher import default_sources From 7c59321063a05d74b7c6056f106914cc2b2f7567 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 05:58:53 +0200 Subject: [PATCH 21/65] fix(ingest): preserve live and generation authority --- polylogue/config.py | 8 +++ polylogue/daemon/cli.py | 4 +- polylogue/sources/live/append_ingest.py | 9 ++- polylogue/sources/live/archive_open.py | 10 +++- polylogue/sources/live/batch.py | 55 +++++++++++------- polylogue/sources/live/source_selection.py | 36 ++++++++++++ polylogue/sources/live/watcher.py | 32 ++-------- polylogue/storage/raw_authority.py | 14 ++++- polylogue/storage/repair.py | 58 +++++++++++++++++-- .../storage/sqlite/archive_tiers/archive.py | 19 +++--- tests/unit/daemon/test_daemon_cli.py | 7 ++- tests/unit/sources/test_live_batch_support.py | 22 +++++++ tests/unit/sources/test_live_watcher.py | 2 + .../storage/test_archive_tiers_archive.py | 10 +--- .../unit/storage/test_raw_authority_ledger.py | 35 +++++++++-- tests/unit/storage/test_repair.py | 30 ++++++++++ 16 files changed, 269 insertions(+), 82 deletions(-) create mode 100644 polylogue/sources/live/source_selection.py diff --git a/polylogue/config.py b/polylogue/config.py index cf215e96f1..ee0443f1bc 100644 --- a/polylogue/config.py +++ b/polylogue/config.py @@ -122,6 +122,7 @@ def __init__( self.archive_root = archive_root self.render_root = render_root self.sources = sources + self._db_path_explicit = db_path is not None self.db_path = db_path if db_path is not None else resolve_active_index_path(archive_root) self.drive_config = drive_config self.index_config = index_config @@ -137,6 +138,13 @@ def __init__( if isinstance(judgment_automation_interval_s, bool) or not isinstance(judgment_automation_interval_s, int): raise ConfigError("Config.judgment_automation_interval_s must be an integer") + def current_db_path(self) -> Path: + """Resolve the current generation unless the caller pinned an override.""" + + if self._db_path_explicit and self.db_path.name == "index.db": + return self.db_path + return resolve_active_index_path(self.archive_root) + def __eq__(self, other: object) -> bool: if not isinstance(other, Config): return NotImplemented diff --git a/polylogue/daemon/cli.py b/polylogue/daemon/cli.py index 753bab6703..7d227028ed 100644 --- a/polylogue/daemon/cli.py +++ b/polylogue/daemon/cli.py @@ -1319,7 +1319,7 @@ def _drain_raw_materialization_once( max_pass_seconds=_RAW_MATERIALIZATION_MAX_PASS_SECONDS, ) finally: - _close_raw_materialization_fts(config.archive_root / "index.db") + _close_raw_materialization_fts(config.current_db_path()) _emit_raw_materialization_pass(result) frontier_repaired = _converge_raw_authority_frontier(config, limit=min(limit, 8)) if not result.success: @@ -1387,7 +1387,7 @@ def _run_raw_materialization_whale_pass_once(*, raw_artifact_id: str, max_payloa raw_artifact_id=raw_artifact_id, ) finally: - _close_raw_materialization_fts(config.archive_root / "index.db") + _close_raw_materialization_fts(config.current_db_path()) _emit_raw_materialization_pass(result) if not result.success: logger.warning("raw materialization: whale pass for %s incomplete: %s", raw_artifact_id, result.detail) diff --git a/polylogue/sources/live/append_ingest.py b/polylogue/sources/live/append_ingest.py index 54b52f9ec5..1ef5720512 100644 --- a/polylogue/sources/live/append_ingest.py +++ b/polylogue/sources/live/append_ingest.py @@ -20,7 +20,7 @@ from polylogue.core.degraded import degraded_reason from polylogue.core.enums import Provider from polylogue.logging import get_logger -from polylogue.sources.live.archive_open import _open_archive_for_live_write +from polylogue.sources.live.archive_open import _open_archive_for_live_write, _source_tier_acquisition_required from polylogue.sources.live.batch_support import _AppendPlan, _AppendResult from polylogue.sources.live.cursor import CursorStore from polylogue.sources.live.sqlite_locking import is_transient_sqlite_lock @@ -73,9 +73,12 @@ def _ingest_append_plans_archive( archive_root: Path, ) -> _AppendResult: timings: dict[str, float] = {} - index_db = resolve_active_index_path(archive_root) source_db = archive_root / "source.db" - if not index_db.exists() or not source_db.exists(): + source_only = _source_tier_acquisition_required() + archive_missing = not source_db.exists() + if not source_only: + archive_missing = archive_missing or not resolve_active_index_path(archive_root).exists() + if archive_missing: t0 = time.perf_counter() initialize_active_archive_root(archive_root) _add_timing(timings, "append.archive_init", t0) diff --git a/polylogue/sources/live/archive_open.py b/polylogue/sources/live/archive_open.py index 379eb36738..1eedfefc02 100644 --- a/polylogue/sources/live/archive_open.py +++ b/polylogue/sources/live/archive_open.py @@ -19,6 +19,13 @@ from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore +def _source_tier_acquisition_required() -> bool: + """Return whether live ingest must avoid every derived-tier read.""" + + reason = degraded_reason() + return reason is not None and reason.derived_only + + def _open_archive_for_live_write(archive_root: Path) -> ArchiveStore: """Open the archive for a live ingest write pass. @@ -29,7 +36,6 @@ def _open_archive_for_live_write(archive_root: Path) -> ArchiveStore: nothing beyond raw admission is reached. Otherwise returns the ordinary full writer, preserving its all-tier validation exactly. """ - reason = degraded_reason() - if reason is not None and reason.derived_only: + if _source_tier_acquisition_required(): return ArchiveStore.open_source_tier_acquisition(archive_root) return ArchiveStore.open_existing(archive_root, read_only=False) diff --git a/polylogue/sources/live/batch.py b/polylogue/sources/live/batch.py index dec0645cc7..21860d9c73 100644 --- a/polylogue/sources/live/batch.py +++ b/polylogue/sources/live/batch.py @@ -78,7 +78,7 @@ require_positive_conversational_evidence, ) from polylogue.sources.live.append_ingest import ingest_append_plans, reset_transient_raw_parse_state -from polylogue.sources.live.archive_open import _open_archive_for_live_write +from polylogue.sources.live.archive_open import _open_archive_for_live_write, _source_tier_acquisition_required from polylogue.sources.live.batch_observability import ( record_attempt_progress, ) @@ -133,6 +133,7 @@ from polylogue.sources.live.deferred_cursor import record_deferred_append_cursor from polylogue.sources.live.metrics import LiveBatchMetrics, LiveFullIngestAggregate from polylogue.sources.live.parse_prefetch import LiveParseCandidate, LiveParseStage +from polylogue.sources.live.source_selection import deepest_source_for_path from polylogue.sources.live.sqlite_locking import is_transient_sqlite_lock from polylogue.sources.origin_specs import artifact_rule_for_path from polylogue.sources.parsers import codex_state, hermes_state, hermes_verification @@ -159,11 +160,13 @@ from polylogue.storage.sqlite.archive_tiers.archive import ActiveByteRevisionChainError from polylogue.storage.sqlite.archive_tiers.bootstrap import ( ARCHIVE_TIER_SPECS, + archive_tier_spec, ) from polylogue.storage.sqlite.archive_tiers.bootstrap import ( initialize_active_archive_root as initialize_archive_root, ) from polylogue.storage.sqlite.archive_tiers.source_write import ContentExcisedError +from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier if TYPE_CHECKING: from polylogue.api import Polylogue @@ -1744,13 +1747,9 @@ def _current_parser_fingerprint(self) -> str: return self._parser_fingerprint def _source_name_for(self, path: Path) -> str: - resolved = path.resolve() - for source in self._sources: - try: - if resolved.is_relative_to(source.root.resolve()): - return str(source.name) - except OSError: - continue + source = deepest_source_for_path(path, self._sources) + if source is not None: + return str(source.name) return path.parent.name def _can_ingest_appends_directly(self) -> bool: @@ -1852,7 +1851,9 @@ def _ingest_full_paths_sync( fallback_provider = Provider.from_string(canonical_acquisition_provider(source_name, source_name=source_name)) acquisition_capture_mode = fallback_provider - archive_bootstrapped = not self._archive_active(archive_root) + source_only = _source_tier_acquisition_required() + archive_active = self._archive_active(archive_root) + archive_bootstrapped = not archive_active and (not source_only or not (archive_root / "source.db").exists()) if archive_bootstrapped: initialize_archive_root(archive_root) archive_active = self._archive_active(archive_root) @@ -2350,6 +2351,8 @@ def _ingest_full_paths_sync( return result def _archive_active(self, archive_root: Path) -> bool: + if _source_tier_acquisition_required(): + return (archive_root / "source.db").exists() and (archive_root / "user.db").exists() return ( ArchiveLocation.resolve(archive_root).active_index_path.exists() and (archive_root / "source.db").exists() ) @@ -2361,14 +2364,26 @@ def _archive_storage_probe_payload( archive_active: bool, archive_bootstrapped: bool, ) -> dict[str, object]: - tier_paths = { - spec.tier.value: ( - ArchiveLocation.resolve(archive_root).active_index_path - if spec.tier.value == "index" - else archive_root / spec.filename - ) - for spec in ARCHIVE_TIER_SPECS.values() - } + if _source_tier_acquisition_required(): + tier_paths = { + tier.value: archive_root / archive_tier_spec(tier).filename + for tier in (ArchiveTier.SOURCE, ArchiveTier.USER) + } + storage_route = "archive_source_acquisition" + storage_tiers = ",".join(tier_paths) + storage_write_tiers = ArchiveTier.SOURCE.value + else: + tier_paths = { + spec.tier.value: ( + ArchiveLocation.resolve(archive_root).active_index_path + if spec.tier.value == "index" + else archive_root / spec.filename + ) + for spec in ARCHIVE_TIER_SPECS.values() + } + storage_route = "archive_full" + storage_tiers = _ARCHIVE_RUNTIME_TIERS + storage_write_tiers = _ARCHIVE_NATIVE_WRITE_TIERS present = [tier for tier, path in tier_paths.items() if path.exists()] missing = [tier for tier, path in tier_paths.items() if not path.exists()] user_versions: dict[str, int | None] = {} @@ -2385,9 +2400,9 @@ def _archive_storage_probe_payload( except sqlite3.Error: user_versions[tier] = -1 return { - "storage_route": "archive_full", - "storage_tiers": _ARCHIVE_RUNTIME_TIERS, - "storage_write_tiers": _ARCHIVE_NATIVE_WRITE_TIERS, + "storage_route": storage_route, + "storage_tiers": storage_tiers, + "storage_write_tiers": storage_write_tiers, "archive_active": archive_active, "archive_bootstrapped": archive_bootstrapped, "archive_present_tiers": ",".join(present), diff --git a/polylogue/sources/live/source_selection.py b/polylogue/sources/live/source_selection.py new file mode 100644 index 0000000000..b24ca87fc6 --- /dev/null +++ b/polylogue/sources/live/source_selection.py @@ -0,0 +1,36 @@ +"""Deterministic ownership for overlapping live-source roots.""" + +from __future__ import annotations + +from collections.abc import Iterable +from pathlib import Path +from typing import Protocol, TypeVar + + +class RootedSource(Protocol): + @property + def root(self) -> Path: ... + + +SourceT = TypeVar("SourceT", bound=RootedSource) + + +def deepest_source_for_path(path: Path, sources: Iterable[SourceT]) -> SourceT | None: + """Return the most-specific configured source owning ``path``.""" + + try: + resolved = path.resolve() + except OSError: + return None + matches: list[tuple[int, SourceT]] = [] + for source in sources: + try: + source_root = source.root.resolve() + if resolved.is_relative_to(source_root): + matches.append((len(source_root.parts), source)) + except (OSError, ValueError): + continue + return max(matches, key=lambda match: match[0])[1] if matches else None + + +__all__ = ["deepest_source_for_path"] diff --git a/polylogue/sources/live/watcher.py b/polylogue/sources/live/watcher.py index 71922afda7..7452716b51 100644 --- a/polylogue/sources/live/watcher.py +++ b/polylogue/sources/live/watcher.py @@ -51,6 +51,7 @@ from polylogue.sources.live.deferred_cursor import record_deferred_append_cursor from polylogue.sources.live.metrics import LiveBatchMetrics from polylogue.sources.live.parse_prefetch import LiveParseStage +from polylogue.sources.live.source_selection import deepest_source_for_path from polylogue.sources.sqlite_snapshot import is_sqlite_path, sqlite_database_for_sidecar, sqlite_source_revision from polylogue.storage.archive_identity import resolve_active_index_path @@ -1602,35 +1603,14 @@ async def _run_coordinated(self, actor: str, operation: Callable[[], Awaitable[N await operation() def _source_name_for(self, path: Path) -> str: - resolved = path.resolve() - for source in self._sources: - try: - if resolved.is_relative_to(source.root.resolve()): - return source.name - except (OSError, ValueError): - continue + source = deepest_source_for_path(path, self._sources) + if source is not None: + return source.name return path.parent.name def _source_accepts(self, path: Path) -> bool: - try: - resolved = path.resolve() - except OSError: - return False - matches: list[tuple[int, WatchSource]] = [] - for source in self._sources: - try: - source_root = source.root.resolve() - if resolved.is_relative_to(source_root): - matches.append((len(source_root.parts), source)) - except OSError: - continue - if matches: - # Default roots deliberately overlap (~/.codex contains its - # sessions subroot). The deepest root owns a path, independent - # of declaration order or whether the roots were supplied by the - # defaults or explicit configuration. - return max(matches, key=lambda match: match[0])[1].accepts(path) - return False + source = deepest_source_for_path(path, self._sources) + return source.accepts(path) if source is not None else False def _is_hook_spool_path(self, path: Path) -> bool: for source in self._sources: diff --git a/polylogue/storage/raw_authority.py b/polylogue/storage/raw_authority.py index 899f4535c0..8e9a2f08f1 100644 --- a/polylogue/storage/raw_authority.py +++ b/polylogue/storage/raw_authority.py @@ -1284,10 +1284,19 @@ def validate_raw_replay_plan(archive_root: Path, plan: RawReplayPlan) -> tuple[b return observed == plan, observed.to_dict() -def raw_replay_application_receipt(archive_root: Path, plan: RawReplayPlan) -> JSONDocument: +def raw_replay_application_receipt( + archive_root: Path, + plan: RawReplayPlan, + *, + index_db_path: Path | None = None, +) -> JSONDocument: + if index_db_path is None: + from polylogue.storage.archive_identity import resolve_active_index_path + + index_db_path = resolve_active_index_path(archive_root) marks = ",".join("?" for _ in plan.input_raw_ids) with closing(sqlite3.connect(f"file:{archive_root / 'source.db'}?mode=ro", uri=True)) as conn: - conn.execute("ATTACH DATABASE ? AS index_tier", (str(archive_root / "index.db"),)) + conn.execute("ATTACH DATABASE ? AS index_tier", (str(index_db_path),)) source = _rows( conn, f""" @@ -1349,6 +1358,7 @@ def raw_replay_application_receipt(archive_root: Path, plan: RawReplayPlan) -> J return json_document( { "schema": "polylogue.raw-replay-application-receipt.v2", + "index_db_path": str(index_db_path), "source_rows": source, "membership_rows": memberships, "application_rows": applications, diff --git a/polylogue/storage/repair.py b/polylogue/storage/repair.py index 99712976a5..78f73067ab 100644 --- a/polylogue/storage/repair.py +++ b/polylogue/storage/repair.py @@ -3773,7 +3773,8 @@ def _raw_materialization_archive_root(config: Config) -> Path: def _raw_materialization_index_path(config: Config, archive_root: Path) -> Path: """Return an explicit index override or the archive's active generation.""" - return config.db_path if config.db_path.name == "index.db" else resolve_active_index_path(archive_root) + del archive_root + return config.current_db_path() def _raw_artifact_coordinate_predicate(*, artifact_alias: str, raw_alias: str) -> str: @@ -5126,7 +5127,6 @@ def _source_path_native_id_candidates(source_path: str) -> tuple[str, ...]: def _open_archive_index_connection() -> sqlite3.Connection: from polylogue.paths import archive_root - from polylogue.storage.archive_identity import resolve_active_index_path conn = sqlite3.connect(resolve_active_index_path(archive_root())) conn.row_factory = sqlite3.Row @@ -6064,7 +6064,6 @@ def repair_session_insights( clearing the active daemon's debt ledger. """ from polylogue.paths import archive_root as _resolve_archive_root - from polylogue.storage.archive_identity import resolve_active_index_path from polylogue.storage.insights.session.rebuild import ( rebuild_archive_session_insights, refresh_session_insight_aggregates_sync, @@ -6222,6 +6221,55 @@ def repair_raw_materialization( progress_callback: ProgressCallback | None = None, prefetch_cache: RawParsePrefetchCache | None = None, max_pass_seconds: float | None = None, +) -> RepairResult: + """Converge one raw-materialization pass under active-generation ownership.""" + + def run() -> RepairResult: + return _repair_raw_materialization( + config, + dry_run=dry_run, + raw_artifact_id=raw_artifact_id, + provider=provider, + source_family=source_family, + source_root=source_root, + raw_artifact_limit=raw_artifact_limit, + max_payload_bytes=max_payload_bytes, + ingest_workers=ingest_workers, + commit_batch_size=commit_batch_size, + progress_callback=progress_callback, + prefetch_cache=prefetch_cache, + max_pass_seconds=max_pass_seconds, + ) + + if dry_run: + return run() + + from polylogue.storage.index_generation import ActiveWriterLease + + archive_root = _raw_materialization_archive_root(config) + lease = ActiveWriterLease(archive_root) + lease.acquire() + try: + return run() + finally: + lease.close() + + +def _repair_raw_materialization( + config: Config, + dry_run: bool = False, + *, + raw_artifact_id: str | None = None, + provider: str | None = None, + source_family: str | None = None, + source_root: Path | None = None, + raw_artifact_limit: int | None = None, + max_payload_bytes: int = RAW_MATERIALIZATION_EXECUTE_BLOB_LIMIT_BYTES, + ingest_workers: int | None = None, + commit_batch_size: int | None = None, + progress_callback: ProgressCallback | None = None, + prefetch_cache: RawParsePrefetchCache | None = None, + max_pass_seconds: float | None = None, ) -> RepairResult: """Converge retained raws through typed per-session revision authority. @@ -6941,7 +6989,7 @@ def _pass_deadline_exceeded() -> bool: continue except Exception as exc: logger.exception("raw replay plan %s failed", plan.plan_id) - application_receipt = raw_replay_application_receipt(archive_root, plan) + application_receipt = raw_replay_application_receipt(archive_root, plan, index_db_path=index_db) receipt_valid, receipt_problems = validate_raw_replay_application_receipt(plan, application_receipt) if receipt_valid: outcome = RawReplayPlanOutcome( @@ -7002,7 +7050,7 @@ def _pass_deadline_exceeded() -> bool: archive_root, index_db, [plan], remaining=current, no_progress=no_progress ) for outcome in component_outcomes: - application_receipt = raw_replay_application_receipt(archive_root, plan) + application_receipt = raw_replay_application_receipt(archive_root, plan, index_db_path=index_db) receipted = dataclasses.replace(outcome, application_receipt=application_receipt) if outcome.status is RawReplayPlanStatus.EXECUTED: receipt_valid, receipt_problems = validate_raw_replay_application_receipt(plan, application_receipt) diff --git a/polylogue/storage/sqlite/archive_tiers/archive.py b/polylogue/storage/sqlite/archive_tiers/archive.py index 65abd3adb6..445aec8590 100644 --- a/polylogue/storage/sqlite/archive_tiers/archive.py +++ b/polylogue/storage/sqlite/archive_tiers/archive.py @@ -1779,15 +1779,16 @@ def __init__( self._active_writer_lease = ActiveWriterLease(archive_root) self._active_writer_lease.acquire() - try: - assert_writable_archive_identity( - configured_root=configured_archive_root(), - active_root=archive_root, - ) - except Exception: - self._active_writer_lease.close() - self._active_writer_lease = None - raise + if not source_tier_acquisition: + try: + assert_writable_archive_identity( + configured_root=configured_archive_root(), + active_root=archive_root, + ) + except Exception: + self._active_writer_lease.close() + self._active_writer_lease = None + raise else: from polylogue.storage.index_generation import IndexGeneration, IndexGenerationStore diff --git a/tests/unit/daemon/test_daemon_cli.py b/tests/unit/daemon/test_daemon_cli.py index c13cf41a1a..28c4b59da0 100644 --- a/tests/unit/daemon/test_daemon_cli.py +++ b/tests/unit/daemon/test_daemon_cli.py @@ -1054,6 +1054,11 @@ def test_raw_materialization_closes_fts_on_cancellation( from polylogue.daemon import cli as daemon_cli archive = tmp_path / "archive" + active_index = tmp_path / "generations" / "active" / "index.db" + active_index.parent.mkdir(parents=True) + active_index.touch() + archive.mkdir() + (archive / ".index-active-pointer").write_text(str(active_index), encoding="utf-8") closed: list[Path] = [] class FakeRestoreResult: @@ -1075,7 +1080,7 @@ def cancel_repair(*_args: object, **_kwargs: object) -> object: with pytest.raises(asyncio.CancelledError): daemon_cli._drain_raw_materialization_once() - assert closed == [archive / "index.db"] + assert closed == [active_index] def test_raw_materialization_fts_failure_records_durable_debt( diff --git a/tests/unit/sources/test_live_batch_support.py b/tests/unit/sources/test_live_batch_support.py index 15e05ea3f9..b5b8d54e87 100644 --- a/tests/unit/sources/test_live_batch_support.py +++ b/tests/unit/sources/test_live_batch_support.py @@ -271,6 +271,27 @@ def test_live_append_replay_streams_retained_jsonl_raw( assert result.failed == [] +def test_live_append_acquires_with_unreadable_active_pointer(tmp_path: Path) -> None: + from polylogue.core.degraded import DegradedReason, clear_degraded, set_degraded + + _path, plan, owner = _seed_live_append_plan(tmp_path, native_id="degraded-append") + (tmp_path / ".index-active-pointer").write_bytes(b"\xff") + set_degraded( + DegradedReason( + code="schema_version_mismatch", + message="derived generation unavailable", + derived_only=True, + ) + ) + try: + result = ingest_append_plans(cast(Any, owner), [plan]) + finally: + clear_degraded() + + assert result.succeeded == [plan] + assert result.failed == [] + + def test_live_full_replay_streams_retained_jsonl_raw( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -402,6 +423,7 @@ def test_full_ingest_acquires_when_index_is_genuinely_semantic_distance_stale( finally: conn.close() index_digest_before = hashlib.sha256(index_db.read_bytes()).hexdigest() + (tmp_path / ".index-active-pointer").write_bytes(b"\xff") processor = LiveBatchProcessor( cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=index_db))), diff --git a/tests/unit/sources/test_live_watcher.py b/tests/unit/sources/test_live_watcher.py index c11e71d8d1..7fc0d6bbfa 100644 --- a/tests/unit/sources/test_live_watcher.py +++ b/tests/unit/sources/test_live_watcher.py @@ -3552,6 +3552,8 @@ def test_source_accepts_prefers_most_specific_nested_root(tmp_path: Path) -> Non try: assert watcher._source_accepts(path) is True + assert watcher._source_name_for(path) == "codex" + assert watcher._batch_processor._source_name_for(path) == "codex" finally: watcher._parse_stage.shutdown() diff --git a/tests/unit/storage/test_archive_tiers_archive.py b/tests/unit/storage/test_archive_tiers_archive.py index 7444950709..99c548d462 100644 --- a/tests/unit/storage/test_archive_tiers_archive.py +++ b/tests/unit/storage/test_archive_tiers_archive.py @@ -105,16 +105,10 @@ def acquire_then_replace( assert not (root / ".maintenance-state").exists() -def test_source_tier_acquisition_does_not_resolve_active_index(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: +def test_source_tier_acquisition_does_not_resolve_active_index(tmp_path: Path) -> None: """Acquire-only writes remain available while the derived pointer is unreadable.""" - from polylogue.storage import archive_identity - initialize_active_archive_root(tmp_path) - monkeypatch.setattr( - archive_identity, - "resolve_active_index_path", - lambda _root: (_ for _ in ()).throw(AssertionError("source acquisition must not resolve index")), - ) + (tmp_path / ".index-active-pointer").write_bytes(b"\xff") with ArchiveStore.open_source_tier_acquisition(tmp_path) as archive: assert archive.source_db_path == tmp_path / "source.db" diff --git a/tests/unit/storage/test_raw_authority_ledger.py b/tests/unit/storage/test_raw_authority_ledger.py index 945c57ab9d..873311280e 100644 --- a/tests/unit/storage/test_raw_authority_ledger.py +++ b/tests/unit/storage/test_raw_authority_ledger.py @@ -20,6 +20,7 @@ from polylogue.storage import raw_authority as raw_authority_mod from polylogue.storage import raw_reconciler as raw_reconciler_mod from polylogue.storage import repair as repair_mod +from polylogue.storage.archive_identity import resolve_active_index_path from polylogue.storage.archive_readiness import raw_materialization_readiness_snapshot, raw_materialization_ready from polylogue.storage.blob_store import BlobStore from polylogue.storage.raw_authority import ( @@ -42,7 +43,8 @@ from polylogue.storage.raw_reconciler import RawAuthorityFrontierState, inspect_raw_authority_frontier from polylogue.storage.repair import RepairResult, repair_raw_materialization from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore -from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root +from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root, initialize_archive_database +from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier def _config(root: Path) -> Config: @@ -905,8 +907,13 @@ def test_parsed_timestamp_without_exact_application_receipt_fails_closed(tmp_pat _write_codex_raw(tmp_path, native_id="receipt", source_path="receipt.jsonl", acquired_at_ms=1) real_receipt = raw_authority_mod.raw_replay_application_receipt - def incomplete_receipt(root: Path, plan: RawReplayPlan) -> JSONDocument: - payload = dict(real_receipt(root, plan)) + def incomplete_receipt( + root: Path, + plan: RawReplayPlan, + *, + index_db_path: Path | None = None, + ) -> JSONDocument: + payload = dict(real_receipt(root, plan, index_db_path=index_db_path)) payload["head_rows"] = [] return json_document(payload) @@ -920,6 +927,21 @@ def incomplete_receipt(root: Path, plan: RawReplayPlan) -> JSONDocument: ) +def test_application_receipt_reads_the_active_generation_not_shadow_index(tmp_path: Path) -> None: + initialize_active_archive_root(tmp_path) + raw_id = _write_codex_raw(tmp_path, native_id="active-receipt", source_path="active.jsonl", acquired_at_ms=1) + assert repair_raw_materialization(_config(tmp_path)).success is True + plan = build_raw_replay_plans(tmp_path, ((raw_id,),))[0] + active_index = tmp_path / "generations" / "active" / "index.db" + initialize_archive_database(active_index, ArchiveTier.INDEX) + (tmp_path / ".index-active-pointer").write_text(str(active_index), encoding="utf-8") + + receipt = raw_authority_mod.raw_replay_application_receipt(tmp_path, plan) + + assert receipt["index_db_path"] == str(active_index) + assert receipt["application_rows"] == [] + + @pytest.mark.parametrize("field", ["session_id", "accepted_raw_id", "accepted_content_hash"]) def test_application_receipt_requires_exact_application_authority(tmp_path: Path, field: str) -> None: initialize_active_archive_root(tmp_path) @@ -1305,7 +1327,12 @@ def _seed_ambiguous_membership_component( conn.commit() (plan,) = build_raw_replay_plans(tmp_path, [(raw_id,)]) empty_remaining = repair_mod.RawMaterializationCandidates([], 0, 0) - (outcome,) = repair_mod._raw_replay_plan_outcomes(tmp_path, [plan], remaining=empty_remaining) + (outcome,) = repair_mod._raw_replay_plan_outcomes( + tmp_path, + resolve_active_index_path(tmp_path), + [plan], + remaining=empty_remaining, + ) return raw_id, outcome diff --git a/tests/unit/storage/test_repair.py b/tests/unit/storage/test_repair.py index 4ade56413b..fe465b6392 100644 --- a/tests/unit/storage/test_repair.py +++ b/tests/unit/storage/test_repair.py @@ -34,6 +34,36 @@ def _config(tmp_path: Path) -> Config: return Config(archive_root=tmp_path, render_root=tmp_path, sources=[], db_path=tmp_path / "archive.db") +def test_raw_materialization_binds_current_generation_under_writer_lease( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Promotion cannot race generation resolution, replay, and postconditions.""" + from polylogue.storage.index_generation import RebuildLease, RebuildLeaseUnavailableError + + initialize_active_archive_root(tmp_path) + config = Config(archive_root=tmp_path, render_root=tmp_path, sources=[]) + active_index = tmp_path / "generations" / "active" / "index.db" + initialize_archive_database(active_index, ArchiveTier.INDEX) + (tmp_path / ".index-active-pointer").write_text(str(active_index), encoding="utf-8") + + class InnerReachedError(RuntimeError): + pass + + def inspect_inner(*_args: object, **_kwargs: object) -> Any: + assert config.current_db_path() == active_index + with pytest.raises(RebuildLeaseUnavailableError): + with RebuildLease(tmp_path): + pass + raise InnerReachedError + + monkeypatch.setattr(repair_mod, "_repair_raw_materialization", inspect_inner) + + with pytest.raises(InnerReachedError): + repair_mod.repair_raw_materialization(config) + with RebuildLease(tmp_path): + pass + + def test_raw_materialization_reparses_legacy_indexed_raw_before_receipting(tmp_path: Path) -> None: """The daemon reopens legacy bytes instead of certifying old durable bindings.""" from polylogue.archive.message.roles import Role From 1242cc639a12555e41cd2ea598652b4e2a972357 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 06:23:32 +0200 Subject: [PATCH 22/65] fix(ingest): bind source writes to live authority --- polylogue/sources/live/batch.py | 55 +++++++++++------- polylogue/sources/live/watcher.py | 20 +++---- polylogue/storage/raw_reconciler.py | 4 +- polylogue/storage/repair.py | 16 ++++++ tests/unit/sources/test_live_batch_support.py | 56 ++++++++++++++++++- tests/unit/sources/test_live_watcher.py | 4 ++ .../unit/storage/test_raw_authority_ledger.py | 13 +++++ tests/unit/storage/test_repair.py | 27 +++++++++ 8 files changed, 159 insertions(+), 36 deletions(-) diff --git a/polylogue/sources/live/batch.py b/polylogue/sources/live/batch.py index 21860d9c73..bf65abb189 100644 --- a/polylogue/sources/live/batch.py +++ b/polylogue/sources/live/batch.py @@ -644,6 +644,12 @@ def cursor_authority_block_reason(self) -> str | None: exist. Once they do, the readiness proof is fail-closed and shared with raw convergence, recovery, and reindex. """ + if _source_tier_acquisition_required(): + # The derived tier is explicitly unavailable in this mode. Raw + # admission establishes source authority without consulting or + # mutating it; trying to resolve the active index pointer here + # would defeat the acquire-only route before it can run. + return None archive_root = Path(getattr(self._polylogue, "archive_root", self._cursor._db_path.parent)) if ( not (archive_root / "source.db").is_file() @@ -1114,7 +1120,8 @@ async def flush_append_plans() -> None: ) if self._last_cursor_write_stale: stale_cursor_write_count += 1 - self._record_convergence_outcome(path, debt_by_source_path.get(path, ())) + if not _source_tier_acquisition_required(): + self._record_convergence_outcome(path, debt_by_source_path.get(path, ())) for path in full_result.failed: failed_paths.append(str(path)) cursor_fingerprint_read_bytes += self._record_failed_cursor(path) @@ -1143,7 +1150,7 @@ async def flush_append_plans() -> None: stage_payload=summary_stage_payload, ) - if succeeded_paths: + if succeeded_paths and not _source_tier_acquisition_required(): await self._run_sync( "watcher.live_ingest.raw_compaction", self._compact_superseded_raw_snapshots, @@ -3881,8 +3888,9 @@ def _ingest_append_plans(self, plans: list[_AppendPlan]) -> _AppendResult: return ingest_append_plans(self, plans) def _compact_superseded_raw_snapshots(self, paths: list[Path]) -> None: - if not paths: + if not paths or _source_tier_acquisition_required(): return + from polylogue.storage.index_generation import ActiveWriterLease from polylogue.storage.raw_retention import ( RawRetentionSafetyError, active_raw_retention_authority, @@ -3891,28 +3899,33 @@ def _compact_superseded_raw_snapshots(self, paths: list[Path]) -> None: archive_root = Path(getattr(self._polylogue, "archive_root", self._cursor._db_path.parent)) source_db = archive_root / "source.db" - index_db = ArchiveLocation.resolve(archive_root).active_index_path if not source_db.exists(): return - with closing(sqlite3.connect(source_db)) as conn, conn: - conn.row_factory = sqlite3.Row - try: - retention_authority = active_raw_retention_authority( + lease = ActiveWriterLease(archive_root) + lease.acquire() + try: + index_db = ArchiveLocation.resolve(archive_root).active_index_path + with closing(sqlite3.connect(source_db)) as conn, conn: + conn.row_factory = sqlite3.Row + try: + retention_authority = active_raw_retention_authority( + conn, + index_db_path=index_db, + terminal_source_paths=paths, + ) + except RawRetentionSafetyError as exc: + logger.warning("live.watcher: skipped unsafe raw snapshot compaction: %s", exc) + return + result = compact_paths_superseded_raw_snapshots( conn, - index_db_path=index_db, - terminal_source_paths=paths, + paths, + limit_per_path=25, + min_acquired_at=self._raw_compaction_min_acquired_at, + protected_raw_ids=retention_authority.protected_raw_ids, + eligible_raw_ids=retention_authority.eligible_raw_ids, ) - except RawRetentionSafetyError as exc: - logger.warning("live.watcher: skipped unsafe raw snapshot compaction: %s", exc) - return - result = compact_paths_superseded_raw_snapshots( - conn, - paths, - limit_per_path=25, - min_acquired_at=self._raw_compaction_min_acquired_at, - protected_raw_ids=retention_authority.protected_raw_ids, - eligible_raw_ids=retention_authority.eligible_raw_ids, - ) + finally: + lease.close() if result.errors: logger.warning("live.watcher: raw snapshot compaction errors: %s", "; ".join(result.errors[:3])) diff --git a/polylogue/sources/live/watcher.py b/polylogue/sources/live/watcher.py index 7452716b51..5496ebf500 100644 --- a/polylogue/sources/live/watcher.py +++ b/polylogue/sources/live/watcher.py @@ -706,6 +706,8 @@ def _scan_catch_up_candidates(self, roots: list[Path]) -> tuple[CandidateSourceF ] for filename in filenames: path = Path(directory) / filename + if deepest_source_for_path(path, self._sources) is not source: + continue if not source.accepts(path): # Unclaimed-file sweep (mission item 2): a file this # source's own root walk reached but whose suffix no @@ -1672,16 +1674,14 @@ def _canonical_watch_path(self, path: Path) -> Path | None: def _source_for_directory(self, path: Path) -> WatchSource | None: """Return the watched source owning a non-ignored directory.""" - resolved = path.resolve() - for source in self._sources: - try: - relative = resolved.relative_to(source.root.resolve()) - except (OSError, ValueError): - continue - if any(source.ignores_directory(Path(part)) for part in relative.parts): - return None - return source - return None + source = deepest_source_for_path(path, self._sources) + if source is None: + return None + try: + relative = path.resolve().relative_to(source.root.resolve()) + except (OSError, ValueError): + return None + return None if any(source.ignores_directory(Path(part)) for part in relative.parts) else source def _enqueue_added_directory(self, directory: Path) -> None: """Cover files created before a recursive watcher installs its new sub-watch.""" diff --git a/polylogue/storage/raw_reconciler.py b/polylogue/storage/raw_reconciler.py index 0fb12f9c08..d02d65c9e5 100644 --- a/polylogue/storage/raw_reconciler.py +++ b/polylogue/storage/raw_reconciler.py @@ -1194,7 +1194,7 @@ def _plan(item: RawAuthorityFrontierItem) -> RawReplayPlan: def _frontier_items(config: Config) -> tuple[tuple[RawAuthorityFrontierItem, ...], int, int]: root = _archive_root(config) source_db = root / "source.db" - index_db = root / "index.db" + index_db = config.current_db_path() if not source_db.is_file() or not index_db.is_file(): raise RuntimeError("raw authority frontier census requires initialized source and index tiers") with closing(sqlite3.connect(source_db)) as conn, conn: @@ -1352,7 +1352,7 @@ def _apply_strategy( root = _archive_root(config) source_db = root / "source.db" - index_db = root / "index.db" + index_db = config.current_db_path() if item.actuator is RawAuthorityActuator.RESOLVE_CONFLICT: conflict = item.strategy_witness.get("conflict") diff --git a/polylogue/storage/repair.py b/polylogue/storage/repair.py index 78f73067ab..75e97f182b 100644 --- a/polylogue/storage/repair.py +++ b/polylogue/storage/repair.py @@ -5893,6 +5893,22 @@ def count_superseded_raw_snapshots_sync(conn: sqlite3.Connection) -> int: def repair_superseded_raw_snapshots(config: Config, dry_run: bool = False) -> RepairResult: + """Delete redundant raw snapshots while promotion cannot change the protected set.""" + + if dry_run: + return _repair_superseded_raw_snapshots(config, dry_run=True) + + from polylogue.storage.index_generation import ActiveWriterLease + + lease = ActiveWriterLease(_raw_materialization_archive_root(config)) + lease.acquire() + try: + return _repair_superseded_raw_snapshots(config, dry_run=False) + finally: + lease.close() + + +def _repair_superseded_raw_snapshots(config: Config, dry_run: bool = False) -> RepairResult: from polylogue.storage.raw_retention import ( RawRetentionSafetyError, active_raw_retention_authority, diff --git a/tests/unit/sources/test_live_batch_support.py b/tests/unit/sources/test_live_batch_support.py index b5b8d54e87..0f05410254 100644 --- a/tests/unit/sources/test_live_batch_support.py +++ b/tests/unit/sources/test_live_batch_support.py @@ -423,7 +423,8 @@ def test_full_ingest_acquires_when_index_is_genuinely_semantic_distance_stale( finally: conn.close() index_digest_before = hashlib.sha256(index_db.read_bytes()).hexdigest() - (tmp_path / ".index-active-pointer").write_bytes(b"\xff") + pointer = tmp_path / ".index-active-pointer" + pointer.write_bytes(b"\xff") processor = LiveBatchProcessor( cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=index_db))), @@ -439,17 +440,66 @@ def test_full_ingest_acquires_when_index_is_genuinely_semantic_distance_stale( ) ) try: - result = processor._ingest_full_paths_sync([path], source_name="codex") + metrics = asyncio.run(processor.ingest_files([path], emit_event=False)) finally: clear_degraded() - assert result.succeeded == [path], f"failed={result.failed}" + assert metrics.succeeded_file_count == 1 + assert metrics.failed_file_count == 0 parsed_at_ms, parse_error = _raw_parse_state(tmp_path) assert parsed_at_ms is None assert parse_error is None assert hashlib.sha256(index_db.read_bytes()).hexdigest() == index_digest_before, ( "the stale index tier must never be opened for write during acquire-only ingest" ) + assert pointer.read_bytes() == b"\xff" + + +def test_live_raw_compaction_holds_generation_lease_through_delete( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The protected set and destructive cleanup observe one unpromotable generation.""" + + from polylogue.storage import raw_retention + from polylogue.storage.index_generation import RebuildLease, RebuildLeaseUnavailableError + from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root + + initialize_active_archive_root(tmp_path) + root = tmp_path / "sessions" + root.mkdir() + path = root / "session.jsonl" + path.write_text("{}\n", encoding="utf-8") + processor = LiveBatchProcessor( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=tmp_path / "index.db"))), + (WatchSource(name="codex", root=root),), + cursor=CursorStore(tmp_path / "ops.db"), + parser_fingerprint="test-parser", + ) + phases: list[str] = [] + + def assert_promotion_excluded(*_args: object, **_kwargs: object) -> SimpleNamespace: + with pytest.raises(RebuildLeaseUnavailableError): + with RebuildLease(tmp_path): + pass + phases.append("authority") + return SimpleNamespace(protected_raw_ids=frozenset(), eligible_raw_ids=frozenset()) + + def assert_delete_excluded(*_args: object, **_kwargs: object) -> SimpleNamespace: + with pytest.raises(RebuildLeaseUnavailableError): + with RebuildLease(tmp_path): + pass + phases.append("delete") + return SimpleNamespace(errors=()) + + monkeypatch.setattr(raw_retention, "active_raw_retention_authority", assert_promotion_excluded) + monkeypatch.setattr(raw_retention, "compact_paths_superseded_raw_snapshots", assert_delete_excluded) + + processor._compact_superseded_raw_snapshots([path]) + + assert phases == ["authority", "delete"] + with RebuildLease(tmp_path): + pass def test_full_ingest_empty_jsonl_is_not_misclassified_as_truncated( diff --git a/tests/unit/sources/test_live_watcher.py b/tests/unit/sources/test_live_watcher.py index 7fc0d6bbfa..bf0f5812a4 100644 --- a/tests/unit/sources/test_live_watcher.py +++ b/tests/unit/sources/test_live_watcher.py @@ -3541,6 +3541,7 @@ def test_source_accepts_prefers_most_specific_nested_root(tmp_path: Path) -> Non sessions = root / "sessions" sessions.mkdir(parents=True) path = sessions / "session.jsonl" + path.write_text("{}\n", encoding="utf-8") watcher = LiveWatcher( cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=tmp_path / "index.db"))), ( @@ -3554,6 +3555,9 @@ def test_source_accepts_prefers_most_specific_nested_root(tmp_path: Path) -> Non assert watcher._source_accepts(path) is True assert watcher._source_name_for(path) == "codex" assert watcher._batch_processor._source_name_for(path) == "codex" + assert watcher._source_for_directory(sessions).name == "codex" + candidates = watcher._scan_catch_up_candidates([root, sessions]) + assert [(candidate.path, candidate.source_name) for candidate in candidates] == [(path, "codex")] finally: watcher._parse_stage.shutdown() diff --git a/tests/unit/storage/test_raw_authority_ledger.py b/tests/unit/storage/test_raw_authority_ledger.py index 873311280e..438345e834 100644 --- a/tests/unit/storage/test_raw_authority_ledger.py +++ b/tests/unit/storage/test_raw_authority_ledger.py @@ -942,6 +942,19 @@ def test_application_receipt_reads_the_active_generation_not_shadow_index(tmp_pa assert receipt["application_rows"] == [] +def test_frontier_census_reads_the_active_generation_not_shadow_index(tmp_path: Path) -> None: + initialize_active_archive_root(tmp_path) + active_index = tmp_path / "generations" / "active" / "index.db" + initialize_archive_database(active_index, ArchiveTier.INDEX) + (tmp_path / ".index-active-pointer").write_text(str(active_index), encoding="utf-8") + (tmp_path / "index.db").write_bytes(b"not a sqlite database") + + census = inspect_raw_authority_frontier(_config(tmp_path)) + + assert census.accepted_head_count == 0 + assert census.plan_count == 0 + + @pytest.mark.parametrize("field", ["session_id", "accepted_raw_id", "accepted_content_hash"]) def test_application_receipt_requires_exact_application_authority(tmp_path: Path, field: str) -> None: initialize_active_archive_root(tmp_path) diff --git a/tests/unit/storage/test_repair.py b/tests/unit/storage/test_repair.py index fe465b6392..a97cd8037e 100644 --- a/tests/unit/storage/test_repair.py +++ b/tests/unit/storage/test_repair.py @@ -64,6 +64,33 @@ def inspect_inner(*_args: object, **_kwargs: object) -> Any: pass +def test_raw_snapshot_cleanup_binds_authority_and_delete_under_writer_lease( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Promotion cannot change the protected generation during destructive raw cleanup.""" + + from polylogue.storage.index_generation import RebuildLease, RebuildLeaseUnavailableError + + initialize_active_archive_root(tmp_path) + config = Config(archive_root=tmp_path, render_root=tmp_path, sources=[]) + + class InnerReachedError(RuntimeError): + pass + + def inspect_inner(*_args: object, **_kwargs: object) -> Any: + with pytest.raises(RebuildLeaseUnavailableError): + with RebuildLease(tmp_path): + pass + raise InnerReachedError + + monkeypatch.setattr(repair_mod, "_repair_superseded_raw_snapshots", inspect_inner) + + with pytest.raises(InnerReachedError): + repair_mod.repair_superseded_raw_snapshots(config) + with RebuildLease(tmp_path): + pass + + def test_raw_materialization_reparses_legacy_indexed_raw_before_receipting(tmp_path: Path) -> None: """The daemon reopens legacy bytes instead of certifying old durable bindings.""" from polylogue.archive.message.roles import Role From c6f3e692fd62a20120b04e91ceff1b3cc68cc0dd Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 06:45:04 +0200 Subject: [PATCH 23/65] fix(watcher): keep degraded acquisition source-only --- polylogue/sources/live/batch.py | 7 ++ polylogue/sources/live/watcher.py | 35 +++++- tests/unit/sources/test_live_batch_support.py | 59 ++++++++-- tests/unit/sources/test_live_watcher.py | 101 +++++++++++++++++- 4 files changed, 185 insertions(+), 17 deletions(-) diff --git a/polylogue/sources/live/batch.py b/polylogue/sources/live/batch.py index bf65abb189..4a6f09a7ef 100644 --- a/polylogue/sources/live/batch.py +++ b/polylogue/sources/live/batch.py @@ -902,6 +902,13 @@ async def flush_append_plans() -> None: if authorization is not None and authorization.force_full_ingest: full_paths.append(path) continue + if _source_tier_acquisition_required(): + # Append planning and replay both consult the active index to + # prove lineage. In acquire-only mode the derived tier is the + # unavailable component, so capture the complete source + # observation through the source-only full route instead. + full_paths.append(path) + continue if is_fully_degraded(): full_paths.append(path) continue diff --git a/polylogue/sources/live/watcher.py b/polylogue/sources/live/watcher.py index 5496ebf500..e0a27b06ad 100644 --- a/polylogue/sources/live/watcher.py +++ b/polylogue/sources/live/watcher.py @@ -31,6 +31,7 @@ from polylogue.logging import get_logger from polylogue.sources.hooks import drain_hook_event_spool, hook_spool_root, pending_hook_spool_dir from polylogue.sources.live.acquisition_log import log_unclaimed_file +from polylogue.sources.live.archive_open import _source_tier_acquisition_required from polylogue.sources.live.batch import ( CursorAuthorityBlockedError, LiveBatchEventEmitter, @@ -1289,6 +1290,11 @@ def _archived_cursor_reconciliation_scope(self) -> Iterator[None]: cached connection would keep reading a replaced index.db inode across a blue-green generation swap. """ + if _source_tier_acquisition_required(): + self._archived_cursor_conns = None + self._archived_cursor_index_untrusted = False + yield + return archive_root = Path(getattr(self._polylogue, "archive_root", self._cursor._db_path.parent)) source_db = archive_root / "source.db" index_db = resolve_active_index_path(archive_root) @@ -1447,6 +1453,11 @@ def _reconcile_archived_cursor_outcome( archived prefix so catch-up can take the append path instead of parsing the whole active JSONL again. """ + if _source_tier_acquisition_required(): + # Derived corroboration is inapplicable in acquire-only mode. + # Force a fresh source observation instead of deferring on an + # index that this mode is explicitly forbidden to read. + return _ArchivedCursorReconciliation.INCOMPATIBLE shared = self._archived_cursor_conns archive_root = Path(getattr(self._polylogue, "archive_root", self._cursor._db_path.parent)) try: @@ -1683,14 +1694,30 @@ def _source_for_directory(self, path: Path) -> WatchSource | None: return None return None if any(source.ignores_directory(Path(part)) for part in relative.parts) else source + def _directory_is_watch_relevant(self, path: Path) -> bool: + """Return whether a directory is owned or leads to a configured source root.""" + + if self._source_for_directory(path) is not None: + return True + try: + resolved = path.resolve() + except OSError: + return False + for source in self._sources: + try: + if source.root.resolve().is_relative_to(resolved): + return True + except (OSError, ValueError): + continue + return False + def _enqueue_added_directory(self, directory: Path) -> None: """Cover files created before a recursive watcher installs its new sub-watch.""" - source = self._source_for_directory(directory) - if source is None: + if not self._directory_is_watch_relevant(directory): return for parent, dir_names, file_names in os.walk(directory): - dir_names[:] = [name for name in dir_names if not source.ignores_directory(Path(name))] + dir_names[:] = [name for name in dir_names if self._directory_is_watch_relevant(Path(parent) / name)] for name in file_names: candidate = Path(parent) / name canonical = self._canonical_watch_path(candidate) @@ -1708,7 +1735,7 @@ def _watch_filter(self, _change: object, path: str) -> bool: """ observed_path = Path(path) return self._canonical_watch_path(observed_path) is not None or ( - observed_path.is_dir() and self._source_for_directory(observed_path) is not None + observed_path.is_dir() and self._directory_is_watch_relevant(observed_path) ) diff --git a/tests/unit/sources/test_live_batch_support.py b/tests/unit/sources/test_live_batch_support.py index 0f05410254..d6d258e2bb 100644 --- a/tests/unit/sources/test_live_batch_support.py +++ b/tests/unit/sources/test_live_batch_support.py @@ -220,7 +220,7 @@ def _seed_live_append_plan( archive_root: Path, *, native_id: str, -) -> tuple[Path, _AppendPlan, object]: +) -> tuple[Path, _AppendPlan, object, LiveBatchProcessor]: root = archive_root / "sessions" root.mkdir() path = root / f"{native_id}.jsonl" @@ -248,7 +248,7 @@ def _seed_live_append_plan( handle.write(append) plan = processor._append_plan(path) assert isinstance(plan, _AppendPlan) - return path, plan, _append_owner(archive_root) + return path, plan, _append_owner(archive_root), processor def test_live_append_replay_streams_retained_jsonl_raw( @@ -258,7 +258,7 @@ def test_live_append_replay_streams_retained_jsonl_raw( """Append replay must not resurrect eager blob materialization.""" from polylogue.storage.blob_publication import ArchiveBlobPublisher - _path, plan, owner = _seed_live_append_plan(tmp_path, native_id="streamed-append") + _path, plan, owner, _processor = _seed_live_append_plan(tmp_path, native_id="streamed-append") monkeypatch.setattr( ArchiveBlobPublisher, "read_all", @@ -274,7 +274,7 @@ def test_live_append_replay_streams_retained_jsonl_raw( def test_live_append_acquires_with_unreadable_active_pointer(tmp_path: Path) -> None: from polylogue.core.degraded import DegradedReason, clear_degraded, set_degraded - _path, plan, owner = _seed_live_append_plan(tmp_path, native_id="degraded-append") + _path, plan, owner, _processor = _seed_live_append_plan(tmp_path, native_id="degraded-append") (tmp_path / ".index-active-pointer").write_bytes(b"\xff") set_degraded( DegradedReason( @@ -292,6 +292,49 @@ def test_live_append_acquires_with_unreadable_active_pointer(tmp_path: Path) -> assert result.failed == [] +def test_derived_only_live_append_candidate_uses_source_acquisition(tmp_path: Path) -> None: + """The managed batch route must not plan an index-backed append while derived-only.""" + + import hashlib + + from polylogue.core.degraded import DegradedReason, clear_degraded, set_degraded + + path, _plan, _owner, processor = _seed_live_append_plan(tmp_path, native_id="degraded-managed-append") + index_db = tmp_path / "index.db" + index_digest_before = hashlib.sha256(index_db.read_bytes()).hexdigest() + with sqlite3.connect(tmp_path / "source.db") as conn: + raw_count_before = int( + conn.execute("SELECT COUNT(*) FROM raw_sessions WHERE source_path = ?", (str(path),)).fetchone()[0] + ) + pointer = tmp_path / ".index-active-pointer" + pointer.write_bytes(b"\xff") + set_degraded( + DegradedReason( + code="schema_version_mismatch", + message="derived generation unavailable", + derived_only=True, + ) + ) + try: + metrics = asyncio.run(processor.ingest_files([path], emit_event=False)) + finally: + clear_degraded() + + assert metrics.succeeded_file_count == 1 + assert metrics.append_file_count == 0 + assert metrics.full_file_count == 1 + assert pointer.read_bytes() == b"\xff" + assert hashlib.sha256(index_db.read_bytes()).hexdigest() == index_digest_before + with sqlite3.connect(tmp_path / "source.db") as conn: + rows = conn.execute( + """SELECT parsed_at_ms, parse_error FROM raw_sessions + WHERE source_path = ? ORDER BY acquired_at_ms DESC, raw_id DESC""", + (str(path),), + ).fetchall() + assert len(rows) == raw_count_before + 1 + assert rows[0] == (None, None) + + def test_live_full_replay_streams_retained_jsonl_raw( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -4816,7 +4859,7 @@ def test_append_admission_bind_failure_persists_exact_pending_envelope_and_retri tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - _path, plan, owner = _seed_live_append_plan(tmp_path, native_id="append-admission-retry") + _path, plan, owner, _processor = _seed_live_append_plan(tmp_path, native_id="append-admission-retry") original_bind = ArchiveStore.bind_raw_revision fail_once = True @@ -5060,7 +5103,7 @@ def test_append_index_failure_never_marks_raw_success( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - _path, plan, owner = _seed_live_append_plan(tmp_path, native_id="index-fail") + _path, plan, owner, _processor = _seed_live_append_plan(tmp_path, native_id="index-fail") def fail_index(*_args: object, **_kwargs: object) -> object: raise sqlite3.IntegrityError("injected index commit failure") @@ -5081,7 +5124,7 @@ def test_append_multi_session_payload_is_rejected_before_index_write( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - path, plan, owner = _seed_live_append_plan(tmp_path, native_id="append-multi") + path, plan, owner, _processor = _seed_live_append_plan(tmp_path, native_id="append-multi") # polylogue-9ykn: a message-less ParsedSession carries no positive # conversational evidence and is refused before this test's own # "more than one session" check ever runs -- give each session one real @@ -6762,7 +6805,7 @@ def test_append_crash_after_index_commit_repairs_idempotently( class SimulatedProcessCrash(BaseException): pass - _path, plan, owner = _seed_live_append_plan(tmp_path, native_id="crash-retry") + _path, plan, owner, _processor = _seed_live_append_plan(tmp_path, native_id="crash-retry") # polylogue-1r9c: mark_raw_parse_succeeded is called internally by # revision_governance.py (a direct module-internal function reference), # not through ArchiveStore's `self.` dispatch -- patch it there. diff --git a/tests/unit/sources/test_live_watcher.py b/tests/unit/sources/test_live_watcher.py index bf0f5812a4..46b1a62f52 100644 --- a/tests/unit/sources/test_live_watcher.py +++ b/tests/unit/sources/test_live_watcher.py @@ -1622,6 +1622,40 @@ def test_added_directory_scan_rejects_file_symlinks_escaping_source_root( assert enqueued == [internal] +def test_added_directory_scan_retains_a_deeper_root_under_outer_ignore( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """An outer ignore rule cannot hide a configured nested source root.""" + + outer = tmp_path / "codex-state" + ignored = outer / "runtime" + inner = ignored / "sessions" + inner.mkdir(parents=True) + session = inner / "session.jsonl" + session.write_text("{}\n", encoding="utf-8") + watcher, _full_ingest = _make_watcher( + tmp_path, + outer, + sources=( + WatchSource( + name="codex-state", + root=outer, + suffixes=(".sqlite",), + ignored_dir_names=frozenset({"runtime"}), + ), + WatchSource(name="codex", root=inner, suffixes=(".jsonl",)), + ), + ) + enqueued: list[Path] = [] + monkeypatch.setattr(watcher, "_enqueue", enqueued.append) + + assert watcher._watch_filter(object(), str(ignored)) is True + watcher._enqueue_added_directory(ignored) + + assert enqueued == [session] + + def test_hermes_cursor_records_acquisition_revision_not_live_tail(tmp_path: Path) -> None: root = tmp_path / "hermes" root.mkdir() @@ -3290,6 +3324,53 @@ def test_catch_up_processes_pre_existing_files(tmp_path: Path) -> None: assert parse_sources.await_count == 1 +def test_catch_up_acquires_source_without_reading_unavailable_index(tmp_path: Path) -> None: + """The real catch-up planner and batch route remain source-only while derived-only.""" + + from polylogue.core.degraded import DegradedReason, clear_degraded, set_degraded + + with ArchiveStore.open_existing(tmp_path, read_only=False): + pass + root = tmp_path / "sessions" + root.mkdir() + path = root / "degraded-catch-up.jsonl" + path.write_bytes( + b'{"type":"session_meta","payload":{"id":"degraded-catch-up"}}\n' + b'{"type":"response_item","payload":{"type":"message","id":"message-0","role":"user",' + b'"content":[{"type":"input_text","text":"zero"}]}}\n' + ) + pointer = tmp_path / ".index-active-pointer" + pointer.write_bytes(b"\xff") + watcher = LiveWatcher( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=tmp_path / "index.db"))), + (WatchSource(name="codex", root=root),), + cursor=CursorStore(tmp_path / "cursor.sqlite", ops_db_path=tmp_path / "ops.db"), + ) + parse_stage = watcher._parse_stage + assert parse_stage is not None + set_degraded( + DegradedReason( + code="schema_version_mismatch", + message="derived generation unavailable", + derived_only=True, + ) + ) + try: + asyncio.run(watcher._catch_up([root])) + finally: + clear_degraded() + parse_stage.shutdown() + + assert pointer.read_bytes() == b"\xff" + with sqlite3.connect(tmp_path / "source.db") as conn: + row = conn.execute( + """SELECT parsed_at_ms, parse_error FROM raw_sessions + WHERE source_path = ? ORDER BY acquired_at_ms DESC, raw_id DESC LIMIT 1""", + (str(path),), + ).fetchone() + assert row == (None, None) + + def test_catch_up_skips_already_processed(tmp_path: Path) -> None: root = tmp_path / "src" root.mkdir() @@ -3550,16 +3631,20 @@ def test_source_accepts_prefers_most_specific_nested_root(tmp_path: Path) -> Non ), cursor=CursorStore(tmp_path / "cursor.db"), ) + parse_stage = watcher._parse_stage + assert parse_stage is not None try: assert watcher._source_accepts(path) is True assert watcher._source_name_for(path) == "codex" assert watcher._batch_processor._source_name_for(path) == "codex" - assert watcher._source_for_directory(sessions).name == "codex" + directory_source = watcher._source_for_directory(sessions) + assert directory_source is not None + assert directory_source.name == "codex" candidates = watcher._scan_catch_up_candidates([root, sessions]) assert [(candidate.path, candidate.source_name) for candidate in candidates] == [(path, "codex")] finally: - watcher._parse_stage.shutdown() + parse_stage.shutdown() @pytest.mark.asyncio @@ -3573,6 +3658,8 @@ async def test_hook_spool_directory_retry_retries_sqlite_operational_error(tmp_p (), cursor=CursorStore(tmp_path / "cursor.db"), ) + parse_stage = watcher._parse_stage + assert parse_stage is not None calls = 0 async def drain() -> None: @@ -3586,7 +3673,7 @@ async def drain() -> None: try: await watcher._retry_hook_spool_directory_until_populated(shard) finally: - watcher._parse_stage.shutdown() + parse_stage.shutdown() assert calls == 2 @@ -3603,6 +3690,8 @@ async def test_hook_spool_directory_retry_rejects_non_lock_sqlite_error(tmp_path (), cursor=CursorStore(tmp_path / "cursor.db"), ) + parse_stage = watcher._parse_stage + assert parse_stage is not None async def drain() -> None: raise sqlite3.OperationalError("no such table: hook_events") @@ -3612,7 +3701,7 @@ async def drain() -> None: with pytest.raises(sqlite3.OperationalError, match="no such table"): await watcher._retry_hook_spool_directory_until_populated(shard) finally: - watcher._parse_stage.shutdown() + parse_stage.shutdown() @pytest.mark.asyncio @@ -3629,6 +3718,8 @@ async def test_scheduled_hook_spool_retry_observes_and_logs_failure( (), cursor=CursorStore(tmp_path / "cursor.db"), ) + parse_stage = watcher._parse_stage + assert parse_stage is not None recorded_logger = MagicMock() monkeypatch.setattr(live_watcher, "logger", recorded_logger) @@ -3643,7 +3734,7 @@ async def drain() -> None: await asyncio.sleep(0) await asyncio.sleep(0) finally: - watcher._parse_stage.shutdown() + parse_stage.shutdown() assert watcher._hook_spool_directory_retry_tasks == {} recorded_logger.exception.assert_called_once() From 8ef9e727ff0444be29f8312a09e693f62e9371b7 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 07:27:00 +0200 Subject: [PATCH 24/65] fix(storage): preserve live generation authority --- polylogue/sources/hooks.py | 12 +++- polylogue/storage/raw_reconciler.py | 7 +- .../storage/sqlite/archive_tiers/archive.py | 8 +++ tests/unit/sources/test_hook_spool.py | 66 +++++++++++++++---- .../test_duplicate_raw_identity_repair.py | 21 ++++++ 5 files changed, 98 insertions(+), 16 deletions(-) diff --git a/polylogue/sources/hooks.py b/polylogue/sources/hooks.py index 4cffd7182a..88b916b643 100644 --- a/polylogue/sources/hooks.py +++ b/polylogue/sources/hooks.py @@ -297,8 +297,16 @@ def drain_hook_event_spool( acknowledged = 0 failed = 0 try: - initialize_active_archive_root(archive_root) - store = ArchiveStore.open_existing(archive_root, read_only=False) + # Import after this module has initialized: ``sources.live.__init__`` + # exposes the watcher, and the watcher imports this spool module. + from polylogue.sources.live.archive_open import ( + _open_archive_for_live_write, + _source_tier_acquisition_required, + ) + + if not _source_tier_acquisition_required(): + initialize_active_archive_root(archive_root) + store = _open_archive_for_live_write(archive_root) except (OSError, sqlite3.Error, ValueError): logger.warning("hook spool drain could not open the archive; all events remain pending", exc_info=True) return HookSpoolDrainResult( diff --git a/polylogue/storage/raw_reconciler.py b/polylogue/storage/raw_reconciler.py index d02d65c9e5..6eb429696f 100644 --- a/polylogue/storage/raw_reconciler.py +++ b/polylogue/storage/raw_reconciler.py @@ -630,6 +630,7 @@ def _item( def _classify_frontier( conn: sqlite3.Connection, blob_store: BlobStore, + index_db: Path, row: dict[str, object], strategy_override: _StrategyOverride | None, ) -> RawAuthorityFrontierItem: @@ -671,7 +672,7 @@ def _classify_frontier( if len(duplicate_siblings) != 1: raise RuntimeError(f"duplicate alias classification is not injective for {raw_id}") - with closing(sqlite3.connect(f"file:{blob_store.root.parent / 'index.db'}?mode=ro", uri=True)) as proof_conn: + with closing(sqlite3.connect(f"file:{index_db}?mode=ro", uri=True)) as proof_conn: proof_conn.row_factory = sqlite3.Row proof_conn.execute( "ATTACH DATABASE ? AS source", @@ -1216,7 +1217,9 @@ def _override_for(row: dict[str, object]) -> _StrategyOverride | None: # through this same connection; the outer ``conn`` context manager commits # those writes on clean exit (or rolls back on exception), so a receipt is # never durably recorded for bytes this pass didn't finish inspecting. - head_items = [_classify_frontier(conn, BlobStore(root / "blob"), row, _override_for(row)) for row in head_rows] + head_items = [ + _classify_frontier(conn, BlobStore(root / "blob"), index_db, row, _override_for(row)) for row in head_rows + ] superseded_items = _terminal_superseded_items(conn) all_items = _apply_judgment_dispositions(config, (*head_items, *superseded_items)) return ( diff --git a/polylogue/storage/sqlite/archive_tiers/archive.py b/polylogue/storage/sqlite/archive_tiers/archive.py index 445aec8590..aacae2f72a 100644 --- a/polylogue/storage/sqlite/archive_tiers/archive.py +++ b/polylogue/storage/sqlite/archive_tiers/archive.py @@ -2121,6 +2121,10 @@ def commit(self) -> None: publication receipts; bulk cadence applies to the derived index. """ self._require_writable("commit archive writes") + if self._source_tier_acquisition: + if self._source_conn is not None: + self._source_conn.commit() + return self._conn.commit() self._consume_index_blob_receipts() self._flush_pending_raw_parse_states() @@ -2133,6 +2137,10 @@ def rollback(self) -> None: Used by a bulk caller to discard an uncommitted, half-applied batch when a write raises, before propagating the error. """ + if self._source_tier_acquisition: + if self._source_conn is not None: + self._source_conn.rollback() + return self._conn.rollback() self._pending_index_blob_receipts.clear() self._pending_raw_parse_states.clear() diff --git a/tests/unit/sources/test_hook_spool.py b/tests/unit/sources/test_hook_spool.py index b6028e4411..d44aa60301 100644 --- a/tests/unit/sources/test_hook_spool.py +++ b/tests/unit/sources/test_hook_spool.py @@ -33,6 +33,7 @@ from polylogue.sources.live import LiveWatcher, WatchSource from polylogue.sources.live.cursor import CursorStore from polylogue.sources.parsers.hermes_lifecycle import DURABLE_FINALIZE, PER_TURN_END +from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root @pytest.mark.parametrize( @@ -373,14 +374,20 @@ async def test_hook_shard_retry_replacement_remains_tracked_until_stop( class ControlledTask: def __init__(self) -> None: self._done = False - self.cancelled = False + self._cancelled = False self.callbacks: list[Callable[[ControlledTask], None]] = [] def done(self) -> bool: return self._done def cancel(self) -> None: - self.cancelled = True + self._cancelled = True + + def cancelled(self) -> bool: + return self._cancelled + + def result(self) -> None: + return None def add_done_callback(self, callback: Callable[[ControlledTask], None]) -> None: self.callbacks.append(callback) @@ -417,7 +424,7 @@ def create_task(coro: Coroutine[Any, Any, None]) -> ControlledTask: tracked_replacement = cast(object, watcher._hook_spool_directory_retry_tasks[directory.resolve()]) assert tracked_replacement is replacement watcher.stop() - assert replacement.cancelled is True + assert replacement.cancelled() is True @pytest.mark.uses_real_clock("exercises retry polling while a pending hook envelope remains unacknowledged") @@ -481,6 +488,44 @@ def fail_persistence(*_args: object, **_kwargs: object) -> None: assert event_path.exists() +def test_hook_spool_drain_remains_source_only_when_derived_generation_is_unavailable(tmp_path: Path) -> None: + from polylogue.core.degraded import DegradedReason, clear_degraded, set_degraded + + archive_root = tmp_path / "archive" + spool_root = tmp_path / "hooks" + initialize_active_archive_root(archive_root) + pointer = archive_root / ".index-active-pointer" + pointer.write_bytes(b"\xff") + enqueue_hook_event( + event_id="derived-only-hook", + provider="codex", + event_type="PostToolUse", + session_id="session-1", + timestamp="2026-08-13T05:00:00Z", + payload={"tool_name": "exec"}, + root=spool_root, + ) + set_degraded( + DegradedReason( + code="schema_version_mismatch", + message="derived generation unavailable", + derived_only=True, + ) + ) + try: + result = drain_hook_event_spool(archive_root, root=spool_root) + finally: + clear_degraded() + + assert result.acknowledged == 1 + assert pointer.read_bytes() == b"\xff" + with sqlite3.connect(archive_root / "source.db") as conn: + assert conn.execute( + "SELECT hook_event_id FROM raw_hook_events WHERE hook_event_id = ?", + ("hook:derived-only-hook",), + ).fetchone() == ("hook:derived-only-hook",) + + @pytest.mark.parametrize( ("provider", "session_id"), [("claude-code", "claude-session"), ("codex", "codex-session")], @@ -737,8 +782,6 @@ def test_drain_opens_archive_once_per_pass_and_honors_limit( ) -> None: """One archive open per drain pass (never per record), bounded by limit, with remaining telling the caller to drain again.""" - from types import SimpleNamespace - from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore spool_root = tmp_path / "hooks" @@ -754,17 +797,16 @@ def test_drain_opens_archive_once_per_pass_and_honors_limit( ) archive_root = tmp_path / "archive" open_calls = 0 - real_open = ArchiveStore.open_existing + from polylogue.sources.live import archive_open - def counting_open(root: Path, *, read_only: bool = True, read_timeout: float = 5.0) -> ArchiveStore: + real_open = archive_open._open_archive_for_live_write + + def counting_open(root: Path) -> ArchiveStore: nonlocal open_calls open_calls += 1 - return real_open(root, read_only=read_only, read_timeout=read_timeout) + return real_open(root) - monkeypatch.setattr( - "polylogue.sources.hooks.ArchiveStore", - SimpleNamespace(open_existing=counting_open), - ) + monkeypatch.setattr(archive_open, "_open_archive_for_live_write", counting_open) first = drain_hook_event_spool(archive_root, root=spool_root, limit=2) assert first.acknowledged == 2 diff --git a/tests/unit/storage/test_duplicate_raw_identity_repair.py b/tests/unit/storage/test_duplicate_raw_identity_repair.py index 4d8f6a6134..66764ff717 100644 --- a/tests/unit/storage/test_duplicate_raw_identity_repair.py +++ b/tests/unit/storage/test_duplicate_raw_identity_repair.py @@ -184,6 +184,27 @@ def test_unified_frontier_census_plans_duplicate_alias_with_stable_evidence(tmp_ assert json.loads(persisted[3])["accepted_content_hash"] +def test_duplicate_alias_census_uses_active_generation_not_shadow_index(tmp_path: Path) -> None: + stale_raw_id, _canonical_raw_id, _session_id, _logical_key = _seed_duplicate_raw_pair(tmp_path) + active_index = tmp_path / "generations" / "active" / "index.db" + active_index.parent.mkdir(parents=True) + (tmp_path / "index.db").replace(active_index) + (tmp_path / ".index-active-pointer").write_text(str(active_index), encoding="utf-8") + (tmp_path / "index.db").write_bytes(b"not a sqlite database") + + config = Config( + archive_root=tmp_path, + render_root=tmp_path / "render", + sources=[], + db_path=tmp_path / "archive.db", + ) + census = inspect_raw_authority_frontier(config) + + duplicate = next(item for item in census.items if item.raw_id == stale_raw_id) + assert duplicate.state is RawAuthorityFrontierState.DUPLICATE_ALIAS + assert duplicate.actuator is RawAuthorityActuator.FOLD_DUPLICATE_ALIAS + + def test_unified_frontier_census_prioritizes_missing_bytes_over_safe_actuation(tmp_path: Path) -> None: stale_raw_id, _canonical_raw_id, _session_id, _logical_key = _seed_duplicate_raw_pair(tmp_path) with sqlite3.connect(tmp_path / "source.db") as conn: From 39181b8901883b9662e8a3de11732a89035f7063 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 07:49:33 +0200 Subject: [PATCH 25/65] fix(storage): order retained evidence by observation --- polylogue/storage/raw_retention.py | 18 ++++++-- polylogue/storage/repair.py | 14 +++++-- tests/unit/storage/test_raw_retention.py | 53 ++++++++++++++++++++++++ tests/unit/storage/test_repair.py | 43 +++++++++++++++++++ 4 files changed, 122 insertions(+), 6 deletions(-) diff --git a/polylogue/storage/raw_retention.py b/polylogue/storage/raw_retention.py index 3217c72de5..23e25f8328 100644 --- a/polylogue/storage/raw_retention.py +++ b/polylogue/storage/raw_retention.py @@ -1667,7 +1667,8 @@ def _terminal_artifact_paths(conn: sqlite3.Connection, source_paths: set[str]) - rows = conn.execute( f""" WITH newest_per_coordinate AS ( - SELECT raw_id, source_path, origin, source_index, parse_error, validation_status, parsed_at_ms + SELECT raw_id, source_path, origin, source_index, parse_error, + validation_status, validated_at_ms, parsed_at_ms FROM ( SELECT raw_id, @@ -1676,6 +1677,7 @@ def _terminal_artifact_paths(conn: sqlite3.Connection, source_paths: set[str]) - source_index, parse_error, validation_status, + validated_at_ms, parsed_at_ms, ROW_NUMBER() OVER ( PARTITION BY source_path, origin, source_index @@ -1692,14 +1694,24 @@ def _terminal_artifact_paths(conn: sqlite3.Connection, source_paths: set[str]) - JOIN newest_per_coordinate AS evidence_raw ON evidence_raw.raw_id = artifact.raw_id WHERE artifact.parse_as_session = 0 AND ( - artifact.artifact_kind NOT IN ({raw_failure_placeholders}) + ( + artifact.artifact_kind NOT IN ({raw_failure_placeholders}) + AND ( + evidence_raw.parsed_at_ms IS NULL + OR artifact.last_observed_at_ms >= evidence_raw.parsed_at_ms + ) + ) OR ( artifact.artifact_kind IN ({terminal_raw_failure_placeholders}) AND ( evidence_raw.parse_error IS NOT NULL OR ( evidence_raw.validation_status = 'failed' - AND evidence_raw.parsed_at_ms IS NULL + AND ( + evidence_raw.parsed_at_ms IS NULL + OR evidence_raw.validated_at_ms IS NULL + OR evidence_raw.validated_at_ms >= evidence_raw.parsed_at_ms + ) ) ) ) diff --git a/polylogue/storage/repair.py b/polylogue/storage/repair.py index 75e97f182b..2f8e0af623 100644 --- a/polylogue/storage/repair.py +++ b/polylogue/storage/repair.py @@ -3868,7 +3868,7 @@ def _raw_materialization_candidate_ids( rows = conn.execute( f""" SELECT r.raw_id, r.origin, r.native_id, r.source_path, r.blob_hash, r.blob_size, - r.acquired_at_ms, r.parsed_at_ms, + r.acquired_at_ms, r.parsed_at_ms, r.validated_at_ms, r.parse_error, ( SELECT a.artifact_kind @@ -3956,7 +3956,11 @@ def _raw_materialization_candidate_ids( -- index reset from replaying successfully parsed raw bytes. AND NOT ( COALESCE(r.validation_status, '') = 'failed' - AND r.parsed_at_ms IS NULL + AND ( + r.parsed_at_ms IS NULL + OR r.validated_at_ms IS NULL + OR r.validated_at_ms >= r.parsed_at_ms + ) ) AND ( r.parse_error IS NULL @@ -3990,7 +3994,11 @@ def _raw_materialization_candidate_ids( r.parse_error IS NOT NULL OR ( r.validation_status = 'failed' - AND r.parsed_at_ms IS NULL + AND ( + r.parsed_at_ms IS NULL + OR r.validated_at_ms IS NULL + OR r.validated_at_ms >= r.parsed_at_ms + ) ) ) ) diff --git a/tests/unit/storage/test_raw_retention.py b/tests/unit/storage/test_raw_retention.py index 909a9a6b62..8a09643fb1 100644 --- a/tests/unit/storage/test_raw_retention.py +++ b/tests/unit/storage/test_raw_retention.py @@ -869,6 +869,59 @@ def test_successful_reparse_revokes_stale_terminal_failure_cursor_authority(tmp_ assert after.cursor_authority_gap_samples[0].state == "source_raws_without_accepted_head" +def test_successful_reparse_revokes_stale_ordinary_artifact_cursor_authority(tmp_path: Path) -> None: + """An ordinary sidecar classification cannot stay terminal after a later parse.""" + + initialize_active_archive_root(tmp_path) + source_path = tmp_path / "reparsed-sidecar.json" + with sqlite3.connect(tmp_path / "source.db") as conn: + conn.execute( + """ + INSERT INTO raw_sessions ( + raw_id, origin, native_id, source_path, source_index, + blob_hash, blob_size, acquired_at_ms, parsed_at_ms + ) VALUES (?, ?, ?, ?, 0, ?, 1, 1, 20) + """, + ( + "raw-ordinary-reparsed", + "codex-session", + "ordinary-reparsed", + str(source_path), + bytes(32), + ), + ) + conn.execute( + """ + INSERT INTO raw_artifacts ( + artifact_id, raw_id, origin, source_path, source_index, + artifact_kind, support_status, classification_reason, + parse_as_session, schema_eligible, malformed_jsonl_lines, + first_observed_at_ms, last_observed_at_ms + ) VALUES (?, ?, ?, ?, 0, 'session_metadata', 'supported_parseable', ?, 0, 0, 0, 10, 10) + """, + ( + "artifact-ordinary-reparsed", + "raw-ordinary-reparsed", + "codex-session", + str(source_path), + "ordinary sidecar classification before parser support", + ), + ) + conn.commit() + _seed_ops_cursor(tmp_path / "ops.db", source_path=source_path, byte_offset=1) + + with sqlite3.connect(tmp_path / "source.db") as conn: + snapshot = raw_frontier_integrity_snapshot( + conn, + index_db_path=tmp_path / "index.db", + ops_db_path=tmp_path / "ops.db", + ) + + assert snapshot.cursor_ahead_status == "unknown" + assert snapshot.cursor_authority_gap_count == 1 + assert snapshot.cursor_authority_gap_samples[0].state == "source_raws_without_accepted_head" + + def test_terminal_artifact_retention_batches_source_paths_below_sqlite_limit(tmp_path: Path) -> None: """Terminal evidence remains protectable when more than one SQL batch is needed.""" diff --git a/tests/unit/storage/test_repair.py b/tests/unit/storage/test_repair.py index a97cd8037e..05bb6a544a 100644 --- a/tests/unit/storage/test_repair.py +++ b/tests/unit/storage/test_repair.py @@ -958,6 +958,49 @@ def test_raw_materialization_replays_successful_raw_with_historical_validation_f assert conn.execute("SELECT COUNT(*) FROM sessions WHERE raw_id = ?", (raw_id,)).fetchone() == (1,) +def test_raw_materialization_refuses_validation_failure_newer_than_parse(tmp_path: Path) -> None: + """A later validation failure remains current authority after an earlier parse.""" + from polylogue.core.enums import Provider + from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore + from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root + + initialize_active_archive_root(tmp_path) + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.CODEX, + payload=( + b'{"type":"session_meta","payload":{"id":"later-validation-failure"}}\n' + b'{"type":"response_item","payload":{"type":"message","id":"m1","role":"user",' + b'"content":[{"type":"input_text","text":"current failure"}]}}\n' + ), + source_path="later-validation-failure.jsonl", + acquired_at_ms=1, + ) + + config = _config(tmp_path) + assert repair_mod.repair_raw_materialization(config).success is True + with sqlite3.connect(tmp_path / "source.db") as conn: + parsed_at_ms = int( + conn.execute("SELECT parsed_at_ms FROM raw_sessions WHERE raw_id = ?", (raw_id,)).fetchone()[0] + ) + conn.execute( + """ + UPDATE raw_sessions + SET validation_status = 'failed', validation_error = ?, validated_at_ms = ? + WHERE raw_id = ? + """, + ("strict validation rejected the later observation", parsed_at_ms + 1, raw_id), + ) + conn.commit() + + active_index = tmp_path / "generations" / "after-validation" / "index.db" + initialize_archive_database(active_index, ArchiveTier.INDEX) + (tmp_path / ".index-active-pointer").write_text(f"{active_index}\n", encoding="utf-8") + + assert raw_id not in repair_mod._raw_materialization_candidate_ids(config).raw_ids + assert repair_mod.raw_materialization_replay_backlog(config)["candidate_count"] == 0 + + @pytest.mark.parametrize("artifact_kind", ["deferred_hot_jsonl_capture", "deferred_claude_code_partial_jsonl"]) def test_raw_materialization_does_not_replay_hot_partial_capture(tmp_path: Path, artifact_kind: str) -> None: """Hot partial evidence stays deferred until a complete source observation arrives.""" From bedb18644341be4cf297581aeba121147066a663 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 08:06:15 +0200 Subject: [PATCH 26/65] fix(storage): bind raw authority to active generation --- polylogue/storage/raw_authority.py | 26 +++++++-- polylogue/storage/raw_reconciler.py | 22 +++++-- polylogue/storage/repair.py | 58 ++++++++++++++----- .../test_browser_capture_origin_repair.py | 22 +++++++ .../unit/storage/test_raw_authority_ledger.py | 19 ++++++ 5 files changed, 125 insertions(+), 22 deletions(-) diff --git a/polylogue/storage/raw_authority.py b/polylogue/storage/raw_authority.py index 8e9a2f08f1..f99a34e4cb 100644 --- a/polylogue/storage/raw_authority.py +++ b/polylogue/storage/raw_authority.py @@ -744,11 +744,20 @@ def build_raw_replay_plan(conn: sqlite3.Connection, input_raw_ids: Sequence[str] ) -def build_raw_replay_plans(archive_root: Path, components: Sequence[tuple[str, ...]]) -> tuple[RawReplayPlan, ...]: +def build_raw_replay_plans( + archive_root: Path, + components: Sequence[tuple[str, ...]], + *, + index_db_path: Path | None = None, +) -> tuple[RawReplayPlan, ...]: if not components: return () + if index_db_path is None: + from polylogue.storage.archive_identity import resolve_active_index_path + + index_db_path = resolve_active_index_path(archive_root) with closing(sqlite3.connect(f"file:{archive_root / 'source.db'}?mode=ro", uri=True)) as conn: - conn.execute("ATTACH DATABASE ? AS index_tier", (str(archive_root / "index.db"),)) + conn.execute("ATTACH DATABASE ? AS index_tier", (str(index_db_path),)) return tuple(build_raw_replay_plan(conn, component) for component in components) @@ -1275,9 +1284,18 @@ def record_raw_authority_census( ) -def validate_raw_replay_plan(archive_root: Path, plan: RawReplayPlan) -> tuple[bool, JSONDocument]: +def validate_raw_replay_plan( + archive_root: Path, + plan: RawReplayPlan, + *, + index_db_path: Path | None = None, +) -> tuple[bool, JSONDocument]: try: - observed = build_raw_replay_plans(archive_root, (plan.input_raw_ids,))[0] + observed = build_raw_replay_plans( + archive_root, + (plan.input_raw_ids,), + index_db_path=index_db_path, + )[0] except Exception as exc: logger.warning("raw replay plan validation could not rebuild %s", plan.plan_id, exc_info=True) return False, json_document({"error": f"{type(exc).__name__}: {exc}"}) diff --git a/polylogue/storage/raw_reconciler.py b/polylogue/storage/raw_reconciler.py index 6eb429696f..c17ee56818 100644 --- a/polylogue/storage/raw_reconciler.py +++ b/polylogue/storage/raw_reconciler.py @@ -783,6 +783,8 @@ def _classify_frontier( def _strategy_overrides( config: Config, rows: list[dict[str, object]], + *, + index_db_path: Path, ) -> dict[str, _StrategyOverride]: """Ask legacy incident inspectors for proofs, never for plan identity.""" from polylogue.storage.repair import ( @@ -800,7 +802,11 @@ def _strategy_overrides( } ) for browser_chunk in _chunks(browser_ids): - browser_items = inspect_browser_capture_origin_mismatches(config, browser_chunk) + browser_items = inspect_browser_capture_origin_mismatches( + config, + browser_chunk, + index_db_path=index_db_path, + ) for browser_item in browser_items: if browser_item.status in {"eligible", "already_repaired"}: overrides[browser_item.raw_id] = _StrategyOverride( @@ -810,7 +816,11 @@ def _strategy_overrides( witness=_browser_strategy_witness(browser_item), input_raw_ids=_browser_strategy_raw_ids(browser_item), ) - conflicts = inspect_browser_canonical_authority_conflicts(config, browser_chunk) + conflicts = inspect_browser_canonical_authority_conflicts( + config, + browser_chunk, + index_db_path=index_db_path, + ) for conflict_item in conflicts.items: if conflict_item.raw_id in overrides: continue @@ -861,7 +871,11 @@ def _strategy_overrides( } ) for quarantine_chunk in _chunks(quarantine_pairs, size=100): - quarantine_items = inspect_quarantined_accepted_raws(config, quarantine_chunk) + quarantine_items = inspect_quarantined_accepted_raws( + config, + quarantine_chunk, + index_db_path=index_db_path, + ) for (raw_id, logical_source_key), quarantine_item in zip(quarantine_chunk, quarantine_items, strict=True): if quarantine_item.status in {"eligible", "already_repaired"}: overrides[_quarantine_override_key(raw_id, logical_source_key)] = _StrategyOverride( @@ -1202,7 +1216,7 @@ def _frontier_items(config: Config) -> tuple[tuple[RawAuthorityFrontierItem, ... conn.row_factory = sqlite3.Row conn.execute("ATTACH DATABASE ? AS index_tier", (str(index_db),)) head_rows = _frontier_rows(conn) - overrides = _strategy_overrides(config, head_rows) + overrides = _strategy_overrides(config, head_rows, index_db_path=index_db) def _override_for(row: dict[str, object]) -> _StrategyOverride | None: raw_id = str(row["accepted_raw_id"]) diff --git a/polylogue/storage/repair.py b/polylogue/storage/repair.py index 2f8e0af623..6f2b5df8f4 100644 --- a/polylogue/storage/repair.py +++ b/polylogue/storage/repair.py @@ -1228,6 +1228,8 @@ def _cas_refine_quarantined_accepted_raw( def inspect_quarantined_accepted_raws( config: Config, raw_ids_with_keys: list[tuple[str, str]], + *, + index_db_path: Path | None = None, ) -> tuple[QuarantinedAcceptedRawRepairItem, ...]: """Return exact typed quarantine-refinement proofs without mutation. @@ -1246,7 +1248,7 @@ def inspect_quarantined_accepted_raws( raise ValueError("raw ids must be lowercase SHA-256 identifiers") archive_root = _raw_materialization_archive_root(config) source_db = archive_root / "source.db" - index_db = archive_root / "index.db" + index_db = index_db_path or config.current_db_path() if not source_db.exists() or not index_db.exists(): raise RuntimeError("source or index tier is missing") with closing(sqlite3.connect(f"file:{source_db}?mode=ro", uri=True)) as conn: @@ -2995,6 +2997,8 @@ def _inspect_browser_capture_origin_strategy( def inspect_browser_capture_origin_mismatches( config: Config, raw_ids: list[str], + *, + index_db_path: Path | None = None, ) -> tuple[BrowserCaptureOriginRepairItem, ...]: """Return the exact admitted browser-origin strategy for each raw. @@ -3010,7 +3014,7 @@ def inspect_browser_capture_origin_mismatches( raise ValueError("raw ids must be lowercase SHA-256 identifiers") archive_root = _raw_materialization_archive_root(config) source_db = archive_root / "source.db" - index_db = archive_root / "index.db" + index_db = index_db_path or config.current_db_path() if not source_db.exists() or not index_db.exists(): raise RuntimeError("source or index tier is missing") with closing(sqlite3.connect(f"file:{index_db}?mode=ro", uri=True)) as conn: @@ -3241,7 +3245,10 @@ def ineligible(reason: str) -> BrowserCanonicalAuthorityConflictWitness: def inspect_browser_canonical_authority_conflicts( - config: Config, raw_ids: list[str] + config: Config, + raw_ids: list[str], + *, + index_db_path: Path | None = None, ) -> BrowserCanonicalAuthorityConflictReport: """Build read-only evidence packets for browser-capture raws a safe rekey refuses. @@ -3268,7 +3275,7 @@ def inspect_browser_canonical_authority_conflicts( raise ValueError("raw ids must be lowercase SHA-256 identifiers") archive_root = _raw_materialization_archive_root(config) source_db = archive_root / "source.db" - index_db = archive_root / "index.db" + index_db = index_db_path or config.current_db_path() if not source_db.exists() or not index_db.exists(): raise RuntimeError("source or index tier is missing") @@ -4375,12 +4382,13 @@ def _raw_materialization_ordered_components( candidates: RawMaterializationCandidates, *, archive_root: Path, + index_db_path: Path | None = None, ) -> list[tuple[str, ...]]: """Order complete components fairly without splitting authority cohorts.""" candidate_ids = set(candidates.raw_ids) source_components = candidates.authority_components or tuple((raw_id,) for raw_id in candidates.raw_ids) components = [component for component in source_components if candidate_ids.intersection(component)] - plans = build_raw_replay_plans(archive_root, components) + plans = build_raw_replay_plans(archive_root, components, index_db_path=index_db_path) plan_ids = {plan.input_raw_ids: plan.plan_id for plan in plans} last_attempts = raw_replay_plan_last_attempts(archive_root) @@ -4547,10 +4555,15 @@ def _raw_authority_postflight_snapshot( candidates: RawMaterializationCandidates, *, max_payload_bytes: int, + index_db_path: Path | None = None, ) -> tuple[tuple[RawReplayPlan, ...], dict[str, object]]: """Build the complete post-pass plan inventory and typed residual debt.""" - components = _raw_materialization_ordered_components(candidates, archive_root=archive_root) - plans = build_raw_replay_plans(archive_root, components) + components = _raw_materialization_ordered_components( + candidates, + archive_root=archive_root, + index_db_path=index_db_path, + ) + plans = build_raw_replay_plans(archive_root, components, index_db_path=index_db_path) blocked_plan_ids = tuple( sorted( plan.plan_id @@ -6363,6 +6376,7 @@ def _pass_deadline_exceeded() -> bool: return max_pass_seconds is not None and (time.monotonic() - pass_started_monotonic) >= max_pass_seconds archive_root = _raw_materialization_archive_root(config) + index_db = _raw_materialization_index_path(config, archive_root) recovered_censuses = recover_interrupted_raw_authority_censuses(archive_root) for recovered_census_id, recovered_scope in recovered_censuses: recovered_envelope = recovered_scope.get("max_payload_bytes") @@ -6374,6 +6388,7 @@ def _pass_deadline_exceeded() -> bool: archive_root, recovered_candidates, max_payload_bytes=recovered_max_payload_bytes, + index_db_path=index_db, ) finalize_raw_authority_census( archive_root, @@ -6429,7 +6444,11 @@ def _pass_deadline_exceeded() -> bool: raise ValueError("raw_artifact_limit must be positive") census_components_attempted = 0 if uncensused_raw_ids: - preliminary_components = _raw_materialization_ordered_components(census_candidates, archive_root=archive_root) + preliminary_components = _raw_materialization_ordered_components( + census_candidates, + archive_root=archive_root, + index_db_path=index_db, + ) for component in preliminary_components: if not uncensused_raw_ids.intersection(component): continue @@ -6576,8 +6595,12 @@ def _pass_deadline_exceeded() -> bool: census_receipt=census_receipt, ) candidate_raw_ids = candidates.raw_ids - ordered_components = _raw_materialization_ordered_components(candidates, archive_root=archive_root) - plans = build_raw_replay_plans(archive_root, ordered_components) + ordered_components = _raw_materialization_ordered_components( + candidates, + archive_root=archive_root, + index_db_path=index_db, + ) + plans = build_raw_replay_plans(archive_root, ordered_components, index_db_path=index_db) plan_by_component = {plan.input_raw_ids: plan for plan in plans} all_blocked_components = [ component @@ -6844,7 +6867,7 @@ def _pass_deadline_exceeded() -> bool: stale_outcomes: list[RawReplayPlanOutcome] = [] validated_plans: list[RawReplayPlan] = [] for plan in executable_plans: - valid, observed = validate_raw_replay_plan(archive_root, plan) + valid, observed = validate_raw_replay_plan(archive_root, plan, index_db_path=index_db) if valid: validated_plans.append(plan) else: @@ -6873,6 +6896,7 @@ def _pass_deadline_exceeded() -> bool: archive_root, stale_candidates, max_payload_bytes=max_payload_bytes, + index_db_path=index_db, ) census_receipt = finalize_raw_authority_census( archive_root, @@ -6913,7 +6937,6 @@ def _pass_deadline_exceeded() -> bool: # writer-hot table before this bounded live pass; this is the same # planner invariant seeded for a fresh index bootstrap, without turning # raw materialization into a full rebuild. - index_db = _raw_materialization_index_path(config, archive_root) with closing(sqlite3.connect(index_db, timeout=60)) as planner_conn: planner_conn.execute("PRAGMA busy_timeout = 60000") # A freshly reset index uses representative bootstrap statistics. @@ -7026,7 +7049,11 @@ def _pass_deadline_exceeded() -> bool: ) record_raw_replay_outcome(archive_root, census_receipt.census_id, outcome) else: - plan_still_valid, _ = validate_raw_replay_plan(archive_root, plan) + plan_still_valid, _ = validate_raw_replay_plan( + archive_root, + plan, + index_db_path=index_db, + ) if plan_still_valid: outcome = RawReplayPlanOutcome( plan.plan_id, @@ -7135,7 +7162,10 @@ def _pass_deadline_exceeded() -> bool: ) plan_outcomes = tuple(execution_outcomes) + blocked_plan_outcomes post_plans, post_residual = _raw_authority_postflight_snapshot( - archive_root, remaining, max_payload_bytes=max_payload_bytes + archive_root, + remaining, + max_payload_bytes=max_payload_bytes, + index_db_path=index_db, ) census_receipt = finalize_raw_authority_census( archive_root, diff --git a/tests/unit/storage/test_browser_capture_origin_repair.py b/tests/unit/storage/test_browser_capture_origin_repair.py index b39a8ee732..cf2cfb0fc6 100644 --- a/tests/unit/storage/test_browser_capture_origin_repair.py +++ b/tests/unit/storage/test_browser_capture_origin_repair.py @@ -606,6 +606,28 @@ def test_unified_frontier_applies_browser_origin_without_incident_receipt(tmp_pa assert not (tmp_path / "recovery").exists() +def test_unified_frontier_strategy_uses_the_selected_active_generation(tmp_path: Path) -> None: + raw_id = _seed_mismatched_browser_head(tmp_path) + active_dir = tmp_path / "generations" / "active" + active_dir.mkdir(parents=True) + active_index = active_dir / "index.db" + (tmp_path / "index.db").rename(active_index) + (tmp_path / "index.db").write_bytes(b"shadow index is not authoritative") + (tmp_path / ".index-active-pointer").write_text(str(active_index), encoding="utf-8") + config = Config( + archive_root=tmp_path, + render_root=tmp_path / "render", + sources=[], + db_path=active_index, + ) + + census = inspect_raw_authority_frontier(config) + selected = next(item for item in census.items if item.raw_id == raw_id) + + assert selected.state is RawAuthorityFrontierState.SAFELY_REKEYABLE + assert selected.actuator is RawAuthorityActuator.COPY_FORWARD_ORIGIN + + def test_unified_frontier_restores_equivalent_canonical_browser_head(tmp_path: Path) -> None: mismatched_raw_id = _seed_mismatched_browser_head(tmp_path) canonical_raw_id = _seed_equivalent_canonical_head(tmp_path, mismatched_raw_id) diff --git a/tests/unit/storage/test_raw_authority_ledger.py b/tests/unit/storage/test_raw_authority_ledger.py index 438345e834..67ccee1711 100644 --- a/tests/unit/storage/test_raw_authority_ledger.py +++ b/tests/unit/storage/test_raw_authority_ledger.py @@ -942,6 +942,25 @@ def test_application_receipt_reads_the_active_generation_not_shadow_index(tmp_pa assert receipt["application_rows"] == [] +def test_replay_plan_build_and_validation_read_the_active_generation(tmp_path: Path) -> None: + initialize_active_archive_root(tmp_path) + raw_id = _write_codex_raw(tmp_path, native_id="active-plan", source_path="active-plan.jsonl", acquired_at_ms=1) + assert repair_raw_materialization(_config(tmp_path)).success is True + shadow_plan = build_raw_replay_plans(tmp_path, ((raw_id,),))[0] + assert shadow_plan.index_preconditions["sessions"] + + active_index = tmp_path / "generations" / "active" / "index.db" + initialize_archive_database(active_index, ArchiveTier.INDEX) + (tmp_path / ".index-active-pointer").write_text(str(active_index), encoding="utf-8") + + active_plan = build_raw_replay_plans(tmp_path, ((raw_id,),))[0] + valid, observed = validate_raw_replay_plan(tmp_path, shadow_plan) + + assert active_plan.index_preconditions["sessions"] == [] + assert valid is False + assert observed == active_plan.to_dict() + + def test_frontier_census_reads_the_active_generation_not_shadow_index(tmp_path: Path) -> None: initialize_active_archive_root(tmp_path) active_index = tmp_path / "generations" / "active" / "index.db" From 6dc78df34c0442097b8698f716ebf684af07b794 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 08:16:15 +0200 Subject: [PATCH 27/65] fix(storage): pin whale planning to active index --- polylogue/storage/repair.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/polylogue/storage/repair.py b/polylogue/storage/repair.py index 6f2b5df8f4..b9817a3a30 100644 --- a/polylogue/storage/repair.py +++ b/polylogue/storage/repair.py @@ -4484,7 +4484,11 @@ def raw_materialization_whale_pass_candidate( ): if not candidates.raw_ids: continue - ordered_components = _raw_materialization_ordered_components(candidates, archive_root=archive_root) + ordered_components = _raw_materialization_ordered_components( + candidates, + archive_root=archive_root, + index_db_path=_raw_materialization_index_path(config, archive_root), + ) for component in ordered_components: if census_only and not blocked_raw_ids.intersection(component): continue From 5f0acd4cdfa7e309f3310c90ac057b01e739543d Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 08:59:07 +0200 Subject: [PATCH 28/65] fix(live): honor source and generation authority Keep source-only acquisition out of the parse and evidence paths, make parse and validation ordering deterministic, and bind repair work to the selected generation under its lease.\n\nCover pointer resolution, interrupted recovery, watcher failure handling, retention progress, and non-vacuous production-route regressions. --- polylogue/browser_capture/receiver.py | 12 +++++- polylogue/config.py | 4 +- polylogue/daemon/provenance.py | 4 +- polylogue/schemas/sampling_db.py | 12 ++++-- polylogue/sources/live/batch.py | 31 +++++++------- polylogue/sources/live/watcher.py | 14 +++++-- polylogue/storage/raw_authority.py | 6 ++- polylogue/storage/raw_reconciler.py | 12 +++--- polylogue/storage/raw_retention.py | 4 +- polylogue/storage/repair.py | 6 +-- tests/unit/browser_capture/test_receiver.py | 33 +++++++++++++-- tests/unit/core/test_config.py | 32 ++++++++++++++ tests/unit/core/test_sampling.py | 39 ++++++++++++++++- tests/unit/daemon/test_provenance_endpoint.py | 1 + tests/unit/daemon/test_raw_parse_recovery.py | 10 ++++- tests/unit/sources/test_live_batch_support.py | 5 +++ tests/unit/sources/test_live_watcher.py | 5 ++- .../test_browser_capture_origin_repair.py | 1 - .../test_duplicate_raw_identity_repair.py | 1 - .../unit/storage/test_raw_authority_ledger.py | 25 ++++++++++- tests/unit/storage/test_raw_retention.py | 42 ++++++++++++++++++- tests/unit/storage/test_repair.py | 14 +++++-- 22 files changed, 258 insertions(+), 55 deletions(-) diff --git a/polylogue/browser_capture/receiver.py b/polylogue/browser_capture/receiver.py index 4ad95ac4ce..b1e912175c 100644 --- a/polylogue/browser_capture/receiver.py +++ b/polylogue/browser_capture/receiver.py @@ -371,7 +371,7 @@ def _lookup_raw_archive_state( return _RawArchiveLookup() columns = _columns(conn, "raw_sessions") select = ["raw_id"] if "raw_id" in columns else [] - for optional in ("parse_error", "validation_error", "validation_status", "parsed_at_ms"): + for optional in ("parse_error", "validation_error", "validation_status", "parsed_at_ms", "validated_at_ms"): if optional in columns: select.append(optional) if not select: @@ -405,7 +405,15 @@ def _lookup_raw_archive_state( validation_status = ( str(row["validation_status"]) if "validation_status" in row_keys and row["validation_status"] else None ) - validation_is_current = "parsed_at_ms" not in row_keys or row["parsed_at_ms"] is None + validation_is_current = ( + "parsed_at_ms" not in row_keys + or row["parsed_at_ms"] is None + or ( + "validated_at_ms" in row_keys + and row["validated_at_ms"] is not None + and row["validated_at_ms"] > row["parsed_at_ms"] + ) + ) if isinstance(parse_error, str) and parse_error: latest_failure = parse_error failure_source = "raw_parse" diff --git a/polylogue/config.py b/polylogue/config.py index ee0443f1bc..83c0253279 100644 --- a/polylogue/config.py +++ b/polylogue/config.py @@ -141,7 +141,7 @@ def __init__( def current_db_path(self) -> Path: """Resolve the current generation unless the caller pinned an override.""" - if self._db_path_explicit and self.db_path.name == "index.db": + if self._db_path_explicit: return self.db_path return resolve_active_index_path(self.archive_root) @@ -174,7 +174,7 @@ def with_sources(self, sources: list[Source]) -> Config: archive_root=self.archive_root, render_root=self.render_root, sources=sources, - db_path=self.db_path, + db_path=self.db_path if self._db_path_explicit else None, drive_config=self.drive_config, index_config=self.index_config, embedding_model=self.embedding_model, diff --git a/polylogue/daemon/provenance.py b/polylogue/daemon/provenance.py index aea6e09a70..4a350ae83f 100644 --- a/polylogue/daemon/provenance.py +++ b/polylogue/daemon/provenance.py @@ -272,7 +272,9 @@ def _quarantine_state(row: ProvenanceRow) -> tuple[bool, str | None]: return True, "no_raw_artifact" if row.parse_error: return True, "parse_error" - if row.validation_status == "failed" and row.parsed_at is None: + if row.validation_status == "failed" and ( + row.parsed_at is None or row.validated_at is None or row.validated_at > row.parsed_at + ): return True, "validation_failed" return False, None diff --git a/polylogue/schemas/sampling_db.py b/polylogue/schemas/sampling_db.py index 5bf4c1f37d..d8fac0cd7e 100644 --- a/polylogue/schemas/sampling_db.py +++ b/polylogue/schemas/sampling_db.py @@ -75,6 +75,7 @@ class _RawSessionRow: file_mtime_ms: int | None acquired_at_ms: int | None parsed_at_ms: int | None + validated_at_ms: int | None validation_status: str | None @property @@ -119,6 +120,7 @@ def _coerce_schema_row(row: sqlite3.Row) -> _RawSessionRow: file_mtime_ms=row["file_mtime_ms"], acquired_at_ms=row["acquired_at_ms"], parsed_at_ms=row["parsed_at_ms"], + validated_at_ms=row["validated_at_ms"], validation_status=row["validation_status"], ) @@ -305,7 +307,7 @@ def _iter_schema_units_from_db( WITH heads AS ( SELECT source_path, origin, raw_id, blob_hash, file_mtime_ms, acquired_at_ms, parsed_at_ms, - validation_status, + validated_at_ms, validation_status, ROW_NUMBER() OVER ( PARTITION BY origin, {logical_cohort_expr} ORDER BY acquired_at_ms DESC, raw_id DESC @@ -314,13 +316,13 @@ def _iter_schema_units_from_db( WHERE origin IN ({placeholders}) ) SELECT source_path, origin, raw_id, blob_hash, file_mtime_ms, acquired_at_ms, parsed_at_ms, - validation_status + validated_at_ms, validation_status FROM heads WHERE rn = 1 """ else: query = f""" SELECT source_path, origin, raw_id, blob_hash, file_mtime_ms, acquired_at_ms, parsed_at_ms, - validation_status + validated_at_ms, validation_status FROM raw_sessions WHERE origin IN ({placeholders}) """ @@ -370,7 +372,9 @@ def _iter_schema_units_from_db( ) continue - if row.validation_status == "failed" and row.parsed_at_ms is None: + if row.validation_status == "failed" and ( + row.parsed_at_ms is None or row.validated_at_ms is None or row.validated_at_ms > row.parsed_at_ms + ): _record_terminal( terminal_recorder, row, diff --git a/polylogue/sources/live/batch.py b/polylogue/sources/live/batch.py index 4a6f09a7ef..fc8ed56056 100644 --- a/polylogue/sources/live/batch.py +++ b/polylogue/sources/live/batch.py @@ -1780,7 +1780,7 @@ async def _ingest_full_paths( max_pass_seconds: float | None = None, pass_started: float | None = None, ) -> _FullIngestResult: - if self._parse_stage is not None: + if self._parse_stage is not None and not _source_tier_acquisition_required(): # polylogue-wf8a: pre-parse eligible candidates BEFORE ever # asking the write coordinator for the writer hold below -- # identical sequencing guarantee to ``DaemonParseStage.warm`` @@ -2439,6 +2439,7 @@ def _ingest_full_records_archive( result = _ArchiveFullWriteResult() pass_clock_started = pass_started if pass_started is not None else time.monotonic() with _open_archive_for_live_write(archive_root) as archive: + source_only = _source_tier_acquisition_required() for record_index, record in enumerate(records): # polylogue-11cg9: a single logical session write cannot be # split mid-transaction (it must remain atomic), so the @@ -2473,20 +2474,22 @@ def _ingest_full_records_archive( provider, record.source_path, ) - session_evidence = ( - _blob_jsonl_has_session_evidence( - blob_store, - blob_hash, - provider=provider, - source_path=record.source_path, - ) - if payload is None - else _parse_payload_as_session_artifact( - Path(record.source_path), - provider=provider, - payload=payload, + session_evidence = False + if artifact_classification is not None and not source_only: + session_evidence = ( + _blob_jsonl_has_session_evidence( + blob_store, + blob_hash, + provider=provider, + source_path=record.source_path, + ) + if payload is None + else _parse_payload_as_session_artifact( + Path(record.source_path), + provider=provider, + payload=payload, + ) ) - ) if artifact_classification is not None and not session_evidence: explicit_raw_id = record.raw_id if record.blob_hash is not None else None if payload is None: diff --git a/polylogue/sources/live/watcher.py b/polylogue/sources/live/watcher.py index e0a27b06ad..dee13fd265 100644 --- a/polylogue/sources/live/watcher.py +++ b/polylogue/sources/live/watcher.py @@ -54,7 +54,7 @@ from polylogue.sources.live.parse_prefetch import LiveParseStage from polylogue.sources.live.source_selection import deepest_source_for_path from polylogue.sources.sqlite_snapshot import is_sqlite_path, sqlite_database_for_sidecar, sqlite_source_revision -from polylogue.storage.archive_identity import resolve_active_index_path +from polylogue.storage.archive_identity import ArchiveLocationError, resolve_active_index_path if TYPE_CHECKING: from polylogue.api import Polylogue @@ -1297,7 +1297,13 @@ def _archived_cursor_reconciliation_scope(self) -> Iterator[None]: return archive_root = Path(getattr(self._polylogue, "archive_root", self._cursor._db_path.parent)) source_db = archive_root / "source.db" - index_db = resolve_active_index_path(archive_root) + try: + index_db = resolve_active_index_path(archive_root) + except (ArchiveLocationError, OSError, UnicodeError): + self._archived_cursor_conns = None + self._archived_cursor_index_untrusted = False + yield + return conns: tuple[sqlite3.Connection, sqlite3.Connection] | None = None if source_db.exists() and index_db.exists(): try: @@ -1432,7 +1438,7 @@ def _cursor_skip_corroborated_by_index(self, path: Path) -> bool: closing(sqlite3.connect(f"file:{index_db}?mode=ro", uri=True, timeout=1.0)) as index_conn, ): return self._path_corroborated_by_index(path, source_conn=source_conn, index_conn=index_conn) - except sqlite3.Error: + except (ArchiveLocationError, OSError, UnicodeError, sqlite3.Error): # Cannot prove absence on a transient DB error -- don't force a # spurious re-ingest of an otherwise-healthy cursor. return True @@ -1473,7 +1479,7 @@ def _reconcile_archived_cursor_outcome( closing(sqlite3.connect(f"file:{index_db}?mode=ro", uri=True, timeout=1.0)) as index_conn, ): row = self._archived_cursor_row(path, source_conn=source_conn, index_conn=index_conn) - except sqlite3.Error: + except (ArchiveLocationError, OSError, UnicodeError, sqlite3.Error): return _ArchivedCursorReconciliation.UNAVAILABLE if row is None: return _ArchivedCursorReconciliation.INCOMPATIBLE diff --git a/polylogue/storage/raw_authority.py b/polylogue/storage/raw_authority.py index f99a34e4cb..873838eee9 100644 --- a/polylogue/storage/raw_authority.py +++ b/polylogue/storage/raw_authority.py @@ -1763,6 +1763,8 @@ def finalize_raw_authority_census( def recover_interrupted_raw_authority_censuses( archive_root: Path, + *, + index_db_path: Path | None = None, ) -> tuple[tuple[str, JSONDocument], ...]: """Reconcile unfinished apply censuses from durable postconditions.""" source_db = archive_root / "source.db" @@ -1799,7 +1801,7 @@ def recover_interrupted_raw_authority_censuses( for row in rows: census_id = str(row["census_id"]) plan = _raw_replay_plan_from_row(row) - receipt = raw_replay_application_receipt(archive_root, plan) + receipt = raw_replay_application_receipt(archive_root, plan, index_db_path=index_db_path) valid_receipt, problems = validate_raw_replay_application_receipt(plan, receipt) if valid_receipt: outcome = RawReplayPlanOutcome( @@ -1812,7 +1814,7 @@ def recover_interrupted_raw_authority_censuses( ) record_raw_replay_outcome(archive_root, census_id, outcome) continue - valid_plan, observed = validate_raw_replay_plan(archive_root, plan) + valid_plan, observed = validate_raw_replay_plan(archive_root, plan, index_db_path=index_db_path) if not valid_plan: reject_stale_raw_replay_plan(archive_root, census_id, plan, observed) else: diff --git a/polylogue/storage/raw_reconciler.py b/polylogue/storage/raw_reconciler.py index c17ee56818..e4a7450533 100644 --- a/polylogue/storage/raw_reconciler.py +++ b/polylogue/storage/raw_reconciler.py @@ -1369,8 +1369,6 @@ def _apply_strategy( root = _archive_root(config) source_db = root / "source.db" - index_db = config.current_db_path() - if item.actuator is RawAuthorityActuator.RESOLVE_CONFLICT: conflict = item.strategy_witness.get("conflict") judgment = item.strategy_witness.get("judgment") @@ -1379,7 +1377,7 @@ def _apply_strategy( evidence = conflict.get("evidence") if not isinstance(evidence, dict) or judgment.get("disposition") != "retain_canonical_authority": raise RuntimeError("conflict-resolution strategy is not explicitly authorized") - with RebuildLease(root), closing(sqlite3.connect(f"file:{index_db}?mode=rw", uri=True)) as conn: + with RebuildLease(root), closing(sqlite3.connect(f"file:{config.current_db_path()}?mode=rw", uri=True)) as conn: conn.row_factory = sqlite3.Row conn.execute("PRAGMA foreign_keys = ON") conn.execute("ATTACH DATABASE ? AS source", (f"file:{source_db}?mode=ro",)) @@ -1404,7 +1402,7 @@ def _apply_strategy( if item.logical_source_key is None: raise RuntimeError("duplicate-alias plan is missing the logical source key it was proven against") logical_source_key = item.logical_source_key - with RebuildLease(root), closing(sqlite3.connect(f"file:{index_db}?mode=rw", uri=True)) as conn: + with RebuildLease(root), closing(sqlite3.connect(f"file:{config.current_db_path()}?mode=rw", uri=True)) as conn: conn.row_factory = sqlite3.Row conn.execute("PRAGMA foreign_keys = ON") conn.execute("ATTACH DATABASE ? AS source", (f"file:{source_db}?mode=ro",)) @@ -1471,7 +1469,7 @@ def _apply_strategy( from polylogue.storage.blob_publication import exclude_archive_blob_publishers with RebuildLease(root), exclude_archive_blob_publishers(source_db): - with closing(sqlite3.connect(f"file:{index_db}?mode=ro", uri=True)) as proof_conn: + with closing(sqlite3.connect(f"file:{config.current_db_path()}?mode=ro", uri=True)) as proof_conn: proof_conn.execute("ATTACH DATABASE ? AS source", (str(source_db),)) preview = _inspect_browser_capture_origin_strategy(root, item.raw_id, conn=proof_conn) if _browser_strategy_witness(preview) != item.strategy_witness: @@ -1492,7 +1490,7 @@ def _apply_strategy( pass elif preview.status != "already_repaired": raise RuntimeError(f"browser-origin strategy lost its exact proof: {preview.reason}") - with closing(sqlite3.connect(f"file:{index_db}?mode=rw", uri=True)) as conn: + with closing(sqlite3.connect(f"file:{config.current_db_path()}?mode=rw", uri=True)) as conn: conn.execute("PRAGMA foreign_keys = ON") conn.execute("ATTACH DATABASE ? AS source", (str(source_db),)) conn.execute("BEGIN IMMEDIATE") @@ -1523,7 +1521,7 @@ def _apply_strategy( logical_source_key = item.logical_source_key with RebuildLease(root), closing(sqlite3.connect(f"file:{source_db}?mode=rw", uri=True)) as source_conn: source_conn.execute("PRAGMA foreign_keys = ON") - _attach_repair_index(source_conn, index_db) + _attach_repair_index(source_conn, config.current_db_path()) source_conn.execute("BEGIN IMMEDIATE") try: # Apply-side stays fail-closed: an authorized plan whose single diff --git a/polylogue/storage/raw_retention.py b/polylogue/storage/raw_retention.py index 23e25f8328..8fc55218a0 100644 --- a/polylogue/storage/raw_retention.py +++ b/polylogue/storage/raw_retention.py @@ -1658,7 +1658,7 @@ def _terminal_artifact_paths(conn: sqlite3.Connection, source_paths: set[str]) - terminal_raw_failure_kinds = tuple(sorted(_TERMINAL_RAW_FAILURE_EVIDENCE_KINDS)) raw_failure_placeholders = ", ".join("?" for _ in raw_failure_kinds) terminal_raw_failure_placeholders = ", ".join("?" for _ in terminal_raw_failure_kinds) - path_batch_size = 500 - len(raw_failure_kinds) - len(terminal_raw_failure_kinds) + path_batch_size = max(1, 500 - len(raw_failure_kinds) - len(terminal_raw_failure_kinds)) pending = set(source_paths) while pending: batch = tuple(sorted(pending)[:path_batch_size]) @@ -1710,7 +1710,7 @@ def _terminal_artifact_paths(conn: sqlite3.Connection, source_paths: set[str]) - AND ( evidence_raw.parsed_at_ms IS NULL OR evidence_raw.validated_at_ms IS NULL - OR evidence_raw.validated_at_ms >= evidence_raw.parsed_at_ms + OR evidence_raw.validated_at_ms > evidence_raw.parsed_at_ms ) ) ) diff --git a/polylogue/storage/repair.py b/polylogue/storage/repair.py index b9817a3a30..0446393594 100644 --- a/polylogue/storage/repair.py +++ b/polylogue/storage/repair.py @@ -3966,7 +3966,7 @@ def _raw_materialization_candidate_ids( AND ( r.parsed_at_ms IS NULL OR r.validated_at_ms IS NULL - OR r.validated_at_ms >= r.parsed_at_ms + OR r.validated_at_ms > r.parsed_at_ms ) ) AND ( @@ -4004,7 +4004,7 @@ def _raw_materialization_candidate_ids( AND ( r.parsed_at_ms IS NULL OR r.validated_at_ms IS NULL - OR r.validated_at_ms >= r.parsed_at_ms + OR r.validated_at_ms > r.parsed_at_ms ) ) ) @@ -6381,7 +6381,7 @@ def _pass_deadline_exceeded() -> bool: archive_root = _raw_materialization_archive_root(config) index_db = _raw_materialization_index_path(config, archive_root) - recovered_censuses = recover_interrupted_raw_authority_censuses(archive_root) + recovered_censuses = recover_interrupted_raw_authority_censuses(archive_root, index_db_path=index_db) for recovered_census_id, recovered_scope in recovered_censuses: recovered_envelope = recovered_scope.get("max_payload_bytes") recovered_max_payload_bytes = ( diff --git a/tests/unit/browser_capture/test_receiver.py b/tests/unit/browser_capture/test_receiver.py index 3c55193618..cea57e9fbb 100644 --- a/tests/unit/browser_capture/test_receiver.py +++ b/tests/unit/browser_capture/test_receiver.py @@ -101,7 +101,9 @@ def _seed_browser_capture_archive( message_count: int = 1, parse_error: str | None = None, validation_status: str | None = None, + validation_error: str | None = None, parsed_at_ms: int | None = None, + validated_at_ms: int | None = None, updated_at_ms: int | None = None, ) -> None: with sqlite3.connect(archive_root / "source.db") as conn: @@ -114,15 +116,17 @@ def _seed_browser_capture_archive( source_path TEXT, parse_error TEXT, validation_status TEXT, - parsed_at_ms INTEGER + validation_error TEXT, + parsed_at_ms INTEGER, + validated_at_ms INTEGER ) """ ) conn.execute( """ INSERT INTO raw_sessions ( - raw_id, origin, native_id, source_path, parse_error, validation_status, parsed_at_ms - ) VALUES (?, ?, ?, ?, ?, ?, ?) + raw_id, origin, native_id, source_path, parse_error, validation_status, validation_error, parsed_at_ms, validated_at_ms + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( raw_id, @@ -131,7 +135,9 @@ def _seed_browser_capture_archive( f"browser-capture/chatgpt/{native_id}.json", parse_error, validation_status, + validation_error, parsed_at_ms, + validated_at_ms, ), ) with sqlite3.connect(archive_root / "index.db") as conn: @@ -784,6 +790,7 @@ def test_receiver_uses_active_index_and_ignores_historical_validation_failure(tm tmp_path, validation_status="failed", parsed_at_ms=1, + validated_at_ms=1, message_count=0, ) active_index = tmp_path / "generations" / "active" / "index.db" @@ -804,6 +811,26 @@ def test_receiver_uses_active_index_and_ignores_historical_validation_failure(tm assert state.indexed_message_count == 1 +def test_receiver_surfaces_validation_failure_newer_than_parse(tmp_path: Path) -> None: + envelope = BrowserCaptureEnvelope.model_validate(_payload()) + write_capture_envelope(envelope, spool_path=tmp_path) + _seed_browser_capture_archive( + tmp_path, + validation_status="failed", + parsed_at_ms=1, + validated_at_ms=2, + validation_error="strict validation rejected current bytes", + ) + + state = BrowserCaptureArchiveStatePayload.model_validate( + existing_capture_state("chatgpt", "conv-123", spool_path=tmp_path, archive_root=tmp_path) + ) + + assert state.state == "failed" + assert state.latest_failure == "strict validation rejected current bytes" + assert state.failure_source == "raw_validation" + + def test_receiver_echoes_safe_request_id_header(tmp_path: Path) -> None: with _running_receiver(tmp_path) as (host, port): conn = HTTPConnection(host, port) diff --git a/tests/unit/core/test_config.py b/tests/unit/core/test_config.py index 161ce26744..151e7613b9 100644 --- a/tests/unit/core/test_config.py +++ b/tests/unit/core/test_config.py @@ -99,6 +99,38 @@ def test_config_db_path_follows_active_generation_pointer(self, tmp_path: Path) assert config.db_path == active_index + def test_with_sources_keeps_implicit_active_generation_tracking(self, tmp_path: Path) -> None: + archive_root = tmp_path / "archive" + archive_root.mkdir() + first = tmp_path / "first" / "index.db" + second = tmp_path / "second" / "index.db" + first.parent.mkdir() + second.parent.mkdir() + first.touch() + second.touch() + pointer = archive_root / ".index-active-pointer" + pointer.write_text(str(first), encoding="utf-8") + + clone = Config(archive_root=archive_root, render_root=tmp_path / "render", sources=[]).with_sources([]) + pointer.write_text(str(second), encoding="utf-8") + + assert clone.current_db_path() == second + + def test_current_db_path_honors_explicit_nonstandard_filename(self, tmp_path: Path) -> None: + archive_root = tmp_path / "archive" + archive_root.mkdir() + active = tmp_path / "generation" / "index.db" + explicit = tmp_path / "selected" / "archive.db" + active.parent.mkdir() + explicit.parent.mkdir() + active.touch() + explicit.touch() + (archive_root / ".index-active-pointer").write_text(str(active), encoding="utf-8") + + config = Config(archive_root=archive_root, render_root=tmp_path / "render", sources=[], db_path=explicit) + + assert config.current_db_path() == explicit + def test_config_db_path_warns_on_stale_conventional_index_shadowing_pointer( self, tmp_path: Path, caplog: pytest.LogCaptureFixture ) -> None: diff --git a/tests/unit/core/test_sampling.py b/tests/unit/core/test_sampling.py index 3aa39d22a4..76ed5baf0b 100644 --- a/tests/unit/core/test_sampling.py +++ b/tests/unit/core/test_sampling.py @@ -283,7 +283,7 @@ def test_sampling_keeps_successfully_reparsed_historical_validation_failure(self ).encode(), ) with sqlite3.connect(db.with_name("source.db")) as conn: - conn.execute("UPDATE raw_sessions SET parsed_at_ms = 1, validation_status = 'failed'") + conn.execute("UPDATE raw_sessions SET parsed_at_ms = 1, validated_at_ms = 0, validation_status = 'failed'") conn.commit() result = load_samples_from_db("claude-ai", db_path=db) @@ -291,6 +291,43 @@ def test_sampling_keeps_successfully_reparsed_historical_validation_failure(self assert len(result) == 1 assert result[0]["uuid"] == "reparsed" + def test_sampling_quarantines_validation_failure_newer_than_parse(self, tmp_path: Path) -> None: + db = _archive_index_db(tmp_path) + raw_id = _insert_raw_session( + db_path=db, + origin="claude-ai-export", + source_path="/tmp/rejected.json", + raw_content=b'{"uuid":"rejected","chat_messages":[]}', + ) + with sqlite3.connect(db.with_name("source.db")) as conn: + cursor = conn.execute( + "UPDATE raw_sessions SET parsed_at_ms = 1, validated_at_ms = 2, validation_status = 'failed' WHERE raw_id = ?", + (raw_id,), + ) + assert cursor.rowcount == 1 + conn.commit() + outcomes: list[dict[str, object]] = [] + + result = list( + iter_schema_units( + "claude-ai", + db_path=db, + full_corpus=True, + terminal_recorder=lambda **outcome: outcomes.append(outcome), + ) + ) + + assert result == [] + assert outcomes == [ + { + "raw_id": raw_id, + "status": "quarantined", + "artifact_kind": None, + "source_path": "/tmp/rejected.json", + "reason": "source_validation_failed", + } + ] + def test_record_provider_sampling_streams_without_full_envelope( self, tmp_path: Path, diff --git a/tests/unit/daemon/test_provenance_endpoint.py b/tests/unit/daemon/test_provenance_endpoint.py index 06e7841d85..8ed269cd2d 100644 --- a/tests/unit/daemon/test_provenance_endpoint.py +++ b/tests/unit/daemon/test_provenance_endpoint.py @@ -403,6 +403,7 @@ def test_historical_validation_failure_is_not_current_quarantine(self, workspace raw_id=raw_id, source_path="/tmp/x.json", blob_size=2, + validated_at_ms=1_767_225_602_000, validation_status="failed", ) diff --git a/tests/unit/daemon/test_raw_parse_recovery.py b/tests/unit/daemon/test_raw_parse_recovery.py index d37bbba808..56087ce26b 100644 --- a/tests/unit/daemon/test_raw_parse_recovery.py +++ b/tests/unit/daemon/test_raw_parse_recovery.py @@ -313,6 +313,7 @@ def test_raw_parse_recovery_skips_validation_failed_cas_frontier_failure(tmp_pat ) with sqlite3.connect(tmp_path / "source.db") as conn: conn.execute("UPDATE raw_sessions SET validation_status = 'failed' WHERE raw_id = ?", (raw_id,)) + assert conn.total_changes == 1 conn.commit() assert make_raw_parse_recovery_stage(tmp_path / "index.db").check(path) is False @@ -332,7 +333,14 @@ def test_raw_parse_recovery_drains_previously_parsed_cas_frontier_failure(tmp_pa error=RawCASFrontierError("frontier changed after parsing completed"), ) with sqlite3.connect(tmp_path / "source.db") as conn: - conn.execute("UPDATE raw_sessions SET validation_status = 'failed' WHERE raw_id = ?", (raw_id,)) + parsed_at_ms = int( + conn.execute("SELECT parsed_at_ms FROM raw_sessions WHERE raw_id = ?", (raw_id,)).fetchone()[0] + ) + conn.execute( + "UPDATE raw_sessions SET validation_status = 'failed', validated_at_ms = ? WHERE raw_id = ?", + (parsed_at_ms, raw_id), + ) + assert conn.total_changes == 1 conn.commit() stage = make_raw_parse_recovery_stage(tmp_path / "index.db") diff --git a/tests/unit/sources/test_live_batch_support.py b/tests/unit/sources/test_live_batch_support.py index d6d258e2bb..906d1e09c1 100644 --- a/tests/unit/sources/test_live_batch_support.py +++ b/tests/unit/sources/test_live_batch_support.py @@ -377,6 +377,7 @@ def test_live_full_replay_streams_retained_jsonl_raw( def test_full_ingest_acquires_but_does_not_parse_when_derived_tier_degraded( tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, ) -> None: """polylogue-gbs02: a derived-only degraded reason must still acquire raw content. @@ -413,6 +414,10 @@ def test_full_ingest_acquires_but_does_not_parse_when_derived_tier_degraded( derived_only=True, ) ) + monkeypatch.setattr( + "polylogue.sources.live.batch._parse_payload_as_session_artifact", + lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("must not decode source-only evidence")), + ) try: result = processor._ingest_full_paths_sync([path], source_name="codex") finally: diff --git a/tests/unit/sources/test_live_watcher.py b/tests/unit/sources/test_live_watcher.py index 46b1a62f52..85e4757f3d 100644 --- a/tests/unit/sources/test_live_watcher.py +++ b/tests/unit/sources/test_live_watcher.py @@ -3324,7 +3324,9 @@ def test_catch_up_processes_pre_existing_files(tmp_path: Path) -> None: assert parse_sources.await_count == 1 -def test_catch_up_acquires_source_without_reading_unavailable_index(tmp_path: Path) -> None: +def test_catch_up_acquires_source_without_reading_unavailable_index( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: """The real catch-up planner and batch route remain source-only while derived-only.""" from polylogue.core.degraded import DegradedReason, clear_degraded, set_degraded @@ -3348,6 +3350,7 @@ def test_catch_up_acquires_source_without_reading_unavailable_index(tmp_path: Pa ) parse_stage = watcher._parse_stage assert parse_stage is not None + monkeypatch.setattr(parse_stage, "warm", lambda *_args: (_ for _ in ()).throw(AssertionError("must not prewarm"))) set_degraded( DegradedReason( code="schema_version_mismatch", diff --git a/tests/unit/storage/test_browser_capture_origin_repair.py b/tests/unit/storage/test_browser_capture_origin_repair.py index cf2cfb0fc6..591d45d781 100644 --- a/tests/unit/storage/test_browser_capture_origin_repair.py +++ b/tests/unit/storage/test_browser_capture_origin_repair.py @@ -618,7 +618,6 @@ def test_unified_frontier_strategy_uses_the_selected_active_generation(tmp_path: archive_root=tmp_path, render_root=tmp_path / "render", sources=[], - db_path=active_index, ) census = inspect_raw_authority_frontier(config) diff --git a/tests/unit/storage/test_duplicate_raw_identity_repair.py b/tests/unit/storage/test_duplicate_raw_identity_repair.py index 66764ff717..1f8105d9d3 100644 --- a/tests/unit/storage/test_duplicate_raw_identity_repair.py +++ b/tests/unit/storage/test_duplicate_raw_identity_repair.py @@ -196,7 +196,6 @@ def test_duplicate_alias_census_uses_active_generation_not_shadow_index(tmp_path archive_root=tmp_path, render_root=tmp_path / "render", sources=[], - db_path=tmp_path / "archive.db", ) census = inspect_raw_authority_frontier(config) diff --git a/tests/unit/storage/test_raw_authority_ledger.py b/tests/unit/storage/test_raw_authority_ledger.py index 67ccee1711..4c4064927b 100644 --- a/tests/unit/storage/test_raw_authority_ledger.py +++ b/tests/unit/storage/test_raw_authority_ledger.py @@ -48,7 +48,7 @@ def _config(root: Path) -> Config: - return Config(archive_root=root, render_root=root / "render", sources=[], db_path=root / "archive.db") + return Config(archive_root=root, render_root=root / "render", sources=[]) def _read_detail_document(root: Path, query_handle: str, *, chunk_chars: int = 256) -> dict[str, object]: @@ -902,6 +902,29 @@ def test_interrupted_apply_recovers_exact_durable_postconditions(tmp_path: Path) assert fts_hits_after_resume == fts_hits_before_resume +def test_interrupted_recovery_receives_repair_pinned_index_path(tmp_path: Path) -> None: + initialize_active_archive_root(tmp_path) + _write_codex_raw(tmp_path, native_id="pinned-recovery", source_path="pinned-recovery.jsonl", acquired_at_ms=1) + + with patch.object(repair_mod, "raw_replay_application_receipt", side_effect=RuntimeError("synthetic crash")): + with pytest.raises(RuntimeError, match="synthetic crash"): + repair_raw_materialization(_config(tmp_path)) + + expected_index = resolve_active_index_path(tmp_path) + recover = raw_authority_mod.recover_interrupted_raw_authority_censuses + received: list[Path | None] = [] + + def capture_pinned_index(root: Path, *, index_db_path: Path | None = None) -> tuple[tuple[str, JSONDocument], ...]: + received.append(index_db_path) + return recover(root, index_db_path=index_db_path) + + with patch.object(repair_mod, "recover_interrupted_raw_authority_censuses", side_effect=capture_pinned_index): + result = repair_raw_materialization(_config(tmp_path)) + + assert result.metrics["raw_materialization_recovered_census_count"] == 1.0 + assert received == [expected_index] + + def test_parsed_timestamp_without_exact_application_receipt_fails_closed(tmp_path: Path) -> None: initialize_active_archive_root(tmp_path) _write_codex_raw(tmp_path, native_id="receipt", source_path="receipt.jsonl", acquired_at_ms=1) diff --git a/tests/unit/storage/test_raw_retention.py b/tests/unit/storage/test_raw_retention.py index 8a09643fb1..3b6353c913 100644 --- a/tests/unit/storage/test_raw_retention.py +++ b/tests/unit/storage/test_raw_retention.py @@ -457,6 +457,46 @@ def test_scoped_terminal_retention_avoids_archive_wide_raw_inventory(tmp_path: P assert terminal_paths == {str(source_path)} +def test_terminal_retention_batches_make_progress_when_failure_kinds_fill_budget( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source_db = tmp_path / "source.db" + source_paths = {tmp_path / "first.json", tmp_path / "second.json"} + initialize_archive_database(source_db, ArchiveTier.SOURCE) + with sqlite3.connect(source_db) as conn: + for index, source_path in enumerate(sorted(source_paths)): + raw_id = f"raw-{index}" + conn.execute( + """ + INSERT INTO raw_sessions ( + raw_id, origin, native_id, source_path, source_index, blob_hash, blob_size, acquired_at_ms + ) VALUES (?, 'unknown-export', ?, ?, 0, ?, 1, ?) + """, + (raw_id, raw_id, str(source_path), bytes([index + 1]) * 32, index + 1), + ) + conn.execute( + """ + INSERT INTO raw_artifacts ( + artifact_id, raw_id, origin, source_path, source_index, artifact_kind, + support_status, classification_reason, parse_as_session, schema_eligible, + malformed_jsonl_lines, first_observed_at_ms, last_observed_at_ms + ) VALUES (?, ?, 'unknown-export', ?, 0, + 'workflow_journal', 'unknown', 'terminal', 0, 0, 0, ?, ?) + """, + (f"artifact-{index}", raw_id, str(source_path), index + 1, index + 1), + ) + monkeypatch.setattr(raw_retention_mod, "RAW_FAILURE_EVIDENCE_KINDS", frozenset(f"raw-{i}" for i in range(250))) + monkeypatch.setattr( + raw_retention_mod, + "_TERMINAL_RAW_FAILURE_EVIDENCE_KINDS", + frozenset(f"terminal-{i}" for i in range(250)), + ) + + assert raw_retention_mod._terminal_artifact_paths(conn, {str(path) for path in source_paths}) == { + str(path) for path in source_paths + } + + def test_semantic_head_receipt_authorizes_no_raw_deletion(tmp_path: Path) -> None: old_raw_id, new_raw_id = _seed_real_full_supersession(tmp_path) with sqlite3.connect(tmp_path / "index.db") as conn: @@ -704,7 +744,7 @@ def test_terminal_cursor_exemption_requires_every_source_coordinate(tmp_path: Pa """, ( ("raw-terminal", "claude-code-session", "terminal", str(source_path), 0, bytes(32), 1, 1), - ("raw-session", "claude-code-session", "session", str(source_path), 1, bytes(1) * 32, 1, 2), + ("raw-session", "claude-code-session", "session", str(source_path), 1, bytes([1]) * 32, 1, 2), ), ) conn.execute( diff --git a/tests/unit/storage/test_repair.py b/tests/unit/storage/test_repair.py index 05bb6a544a..a59b4bf66d 100644 --- a/tests/unit/storage/test_repair.py +++ b/tests/unit/storage/test_repair.py @@ -31,7 +31,7 @@ def _config(tmp_path: Path) -> Config: - return Config(archive_root=tmp_path, render_root=tmp_path, sources=[], db_path=tmp_path / "archive.db") + return Config(archive_root=tmp_path, render_root=tmp_path, sources=[]) def test_raw_materialization_binds_current_generation_under_writer_lease( @@ -1777,7 +1777,7 @@ def test_superseded_raw_cleanup_fails_closed_without_index(tmp_path: Path) -> No initialize_archive_database(tmp_path / "source.db", ArchiveTier.SOURCE) # This valid but unrelated legacy anchor must never authorize deletion # from the split archive_root/source.db file set. - initialize_archive_database(config.db_path, ArchiveTier.INDEX) + initialize_archive_database(tmp_path / "archive.db", ArchiveTier.INDEX) source_file = tmp_path / "source.jsonl" source_file.write_text("{}", encoding="utf-8") with sqlite3.connect(tmp_path / "source.db") as conn: @@ -3524,7 +3524,10 @@ def retry_oldest(*args: Any, selected_raw_ids: list[str] | None = None, **kwargs mutation.setattr(revision_backfill, "backfill_historical_revision_evidence", retry_oldest) if remove_fair_rotation: - def acquisition_only_order(candidates: Any, *, archive_root: Path) -> list[tuple[str, ...]]: + def acquisition_only_order( + candidates: Any, *, archive_root: Path, index_db_path: Path + ) -> list[tuple[str, ...]]: + del index_db_path return sorted( candidates.authority_components, key=lambda component: min(candidates.raw_acquired_at_ms[raw_id] for raw_id in component), @@ -3593,7 +3596,10 @@ def run(*, prefer_cheap: bool) -> tuple[tuple[str, ...], str]: with monkeypatch.context() as mutation: if prefer_cheap: - def cheap_first_order(candidates: Any, *, archive_root: Path) -> list[tuple[str, ...]]: + def cheap_first_order( + candidates: Any, *, archive_root: Path, index_db_path: Path + ) -> list[tuple[str, ...]]: + del index_db_path candidate_ids = set(candidates.raw_ids) source_components = candidates.authority_components or tuple( (raw_id,) for raw_id in candidates.raw_ids From f7cb9e329b8822e8a28a0ea421463e9adfa1b3ca Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 09:31:26 +0200 Subject: [PATCH 29/65] fix(ingest): avoid decode in source-only acquisition Problem: derived-only ingestion still ran provider and session-evidence decoders before durable raw admission.\n\nWhat changed: admit ordinary full-route files by configured source identity while the derived tier is unavailable. The production-route regression makes every skipped decoder raise and proves both JSONL and JSON bytes remain pending for replay. --- polylogue/sources/live/batch.py | 35 +++++++++++++++++++ tests/unit/sources/test_live_batch_support.py | 24 ++++++++++--- 2 files changed, 54 insertions(+), 5 deletions(-) diff --git a/polylogue/sources/live/batch.py b/polylogue/sources/live/batch.py index fc8ed56056..8b22bf8d4b 100644 --- a/polylogue/sources/live/batch.py +++ b/polylogue/sources/live/batch.py @@ -2013,6 +2013,41 @@ def _ingest_full_paths_sync( # the bytes as a generic session artifact. self._mark_excluded_cursor(path, stat, source_name=fallback_provider.value) continue + elif source_only: + # A derived-only outage must not turn durable acquisition into + # an ad hoc parse pass. Provider detection and session/artifact + # classification decode payload bytes, while this route has no + # derived tier to consume their result. Preserve the original + # bytes under the configured source identity and let the + # normal raw replay classify them once the index is available. + provider = fallback_provider + source_name = provider.value + try: + if heartbeat is not None: + heartbeat( + "full_blob_copy", + current_path=path, + source_payload_read_bytes=source_payload_read_bytes, + ) + raw_id, blob_size = blob_store.write_from_path( + path, + heartbeat=_blob_copy_heartbeat( + heartbeat, + path=path, + source_payload_read_bytes=source_payload_read_bytes, + ), + ) + blob_publication_receipt_id = blob_store.receipt_id(raw_id) + except OSError: + failed.append(path) + continue + source_payload_read_bytes += blob_size + if heartbeat is not None: + heartbeat( + "full_blob_copy", + current_path=path, + source_payload_read_bytes=source_payload_read_bytes, + ) elif ( origin_artifact_rule is None and not is_jsonl_source_path(str(path)) diff --git a/tests/unit/sources/test_live_batch_support.py b/tests/unit/sources/test_live_batch_support.py index 906d1e09c1..75be205fac 100644 --- a/tests/unit/sources/test_live_batch_support.py +++ b/tests/unit/sources/test_live_batch_support.py @@ -400,6 +400,8 @@ def test_full_ingest_acquires_but_does_not_parse_when_derived_tier_degraded( b'{"type":"response_item","payload":{"type":"message","id":"message-0","role":"user",' b'"content":[{"type":"input_text","text":"zero"}]}}\n' ) + json_path = root / "degraded-full.json" + json_path.write_bytes(b'{"mapping":{"root":{"message":{"author":{"role":"user"}}}}}') index_db = tmp_path / "index.db" processor = LiveBatchProcessor( cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=index_db))), @@ -418,16 +420,28 @@ def test_full_ingest_acquires_but_does_not_parse_when_derived_tier_degraded( "polylogue.sources.live.batch._parse_payload_as_session_artifact", lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("must not decode source-only evidence")), ) + monkeypatch.setattr( + "polylogue.sources.live.batch._jsonl_provider_and_session_artifact", + lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("must not classify source-only JSONL")), + ) + monkeypatch.setattr( + "polylogue.sources.live.batch.has_decoded_session_evidence", + lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("must not inspect source-only JSON evidence")), + ) + monkeypatch.setattr( + "polylogue.sources.live.batch._detect_provider_from_raw_bytes", + lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("must not detect source-only provider")), + ) try: - result = processor._ingest_full_paths_sync([path], source_name="codex") + result = processor._ingest_full_paths_sync([path, json_path], source_name="codex") finally: clear_degraded() - assert result.succeeded == [path] + assert result.succeeded == [path, json_path] assert result.failed == [] - parsed_at_ms, parse_error = _raw_parse_state(tmp_path) - assert parsed_at_ms is None - assert parse_error is None + with sqlite3.connect(tmp_path / "source.db") as conn: + raw_states = conn.execute("SELECT parsed_at_ms, parse_error FROM raw_sessions ORDER BY source_path").fetchall() + assert raw_states == [(None, None), (None, None)] def test_full_ingest_acquires_when_index_is_genuinely_semantic_distance_stale( From 4cebae566480f2689ee7828507e078db6a7c24f4 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 09:46:01 +0200 Subject: [PATCH 30/65] fix(daemon): pin generation through FTS closure --- polylogue/browser_capture/receiver.py | 11 ++- polylogue/daemon/cli.py | 44 +++++------ polylogue/product/raw_authority.py | 16 ++++ polylogue/sources/live/batch.py | 15 +++- tests/unit/browser_capture/test_receiver.py | 13 ++++ tests/unit/daemon/test_daemon_cli.py | 74 +++++++++++++++++++ tests/unit/product/test_raw_authority.py | 15 ++++ tests/unit/sources/test_live_batch_support.py | 15 +++- 8 files changed, 173 insertions(+), 30 deletions(-) diff --git a/polylogue/browser_capture/receiver.py b/polylogue/browser_capture/receiver.py index b1e912175c..fbada1bbc8 100644 --- a/polylogue/browser_capture/receiver.py +++ b/polylogue/browser_capture/receiver.py @@ -39,7 +39,7 @@ browser_capture_receiver_token_path, browser_capture_spool_root, ) -from polylogue.storage.archive_identity import resolve_active_index_path +from polylogue.storage.archive_identity import ArchiveLocationError, resolve_active_index_path from polylogue.storage.introspection import table_exists as _table_exists logger = get_logger(__name__) @@ -446,7 +446,14 @@ def _lookup_index_archive_state( provider: str, provider_session_id: str, ) -> _IndexArchiveLookup: - conn = _open_readonly_sqlite(resolve_active_index_path(archive_root)) + try: + index_path = resolve_active_index_path(archive_root) + except ArchiveLocationError: + # Archive state is a best-effort capture acknowledgement. A malformed + # active-generation pointer must not turn a receiver GET into a 500 or + # make us consult the conventional shadow index instead. + return _IndexArchiveLookup() + conn = _open_readonly_sqlite(index_path) if conn is None: return _IndexArchiveLookup() try: diff --git a/polylogue/daemon/cli.py b/polylogue/daemon/cli.py index 7d227028ed..2617f0d7fe 100644 --- a/polylogue/daemon/cli.py +++ b/polylogue/daemon/cli.py @@ -1309,17 +1309,18 @@ def _drain_raw_materialization_once( "raw authority: auto-resolved %d stale-plan blocker(s) before raw materialization", auto_resolved, ) - try: - result = raw_authority.repair_materialization( - config, - dry_run=False, - raw_artifact_limit=limit, - max_payload_bytes=_RAW_MATERIALIZATION_DAEMON_BLOB_LIMIT_BYTES, - prefetch_cache=prefetch_cache, - max_pass_seconds=_RAW_MATERIALIZATION_MAX_PASS_SECONDS, - ) - finally: - _close_raw_materialization_fts(config.current_db_path()) + with raw_authority.materialization_generation_lease(config) as index_db: + try: + result = raw_authority.repair_materialization( + config, + dry_run=False, + raw_artifact_limit=limit, + max_payload_bytes=_RAW_MATERIALIZATION_DAEMON_BLOB_LIMIT_BYTES, + prefetch_cache=prefetch_cache, + max_pass_seconds=_RAW_MATERIALIZATION_MAX_PASS_SECONDS, + ) + finally: + _close_raw_materialization_fts(index_db) _emit_raw_materialization_pass(result) frontier_repaired = _converge_raw_authority_frontier(config, limit=min(limit, 8)) if not result.success: @@ -1378,16 +1379,17 @@ def _run_raw_materialization_whale_pass_once(*, raw_artifact_id: str, max_payloa archive = archive_root() config = Config(archive_root=archive, render_root=render_root(), sources=[]) - try: - result = raw_authority.repair_materialization( - config, - dry_run=False, - raw_artifact_limit=1, - max_payload_bytes=max_payload_bytes, - raw_artifact_id=raw_artifact_id, - ) - finally: - _close_raw_materialization_fts(config.current_db_path()) + with raw_authority.materialization_generation_lease(config) as index_db: + try: + result = raw_authority.repair_materialization( + config, + dry_run=False, + raw_artifact_limit=1, + max_payload_bytes=max_payload_bytes, + raw_artifact_id=raw_artifact_id, + ) + finally: + _close_raw_materialization_fts(index_db) _emit_raw_materialization_pass(result) if not result.success: logger.warning("raw materialization: whale pass for %s incomplete: %s", raw_artifact_id, result.detail) diff --git a/polylogue/product/raw_authority.py b/polylogue/product/raw_authority.py index a865e6ca7f..bc61b2022a 100644 --- a/polylogue/product/raw_authority.py +++ b/polylogue/product/raw_authority.py @@ -7,6 +7,8 @@ from __future__ import annotations +import contextlib +from collections.abc import Iterator from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING, Any, Final @@ -119,6 +121,19 @@ def auto_resolve_stale_plan_blockers(config: Config) -> int: return _auto_resolve(config.archive_root) +@contextlib.contextmanager +def materialization_generation_lease(config: Config) -> Iterator[Path]: + """Pin one active index generation through a replay-adjacent closure.""" + from polylogue.storage.index_generation import ActiveWriterLease + + lease = ActiveWriterLease(config.archive_root) + lease.acquire() + try: + yield config.current_db_path() + finally: + lease.close() + + def repair_materialization( config: Config, *, @@ -222,6 +237,7 @@ def list_blockers(archive_root: Path, *, limit: int = 100, offset: int = 0) -> J "apply_frontier", "inspect_frontier", "list_blockers", + "materialization_generation_lease", "read_census", "read_detail", "recover_interrupted_frontier", diff --git a/polylogue/sources/live/batch.py b/polylogue/sources/live/batch.py index 8b22bf8d4b..195e2c99fc 100644 --- a/polylogue/sources/live/batch.py +++ b/polylogue/sources/live/batch.py @@ -2505,9 +2505,18 @@ def _ingest_full_records_archive( fallback_id = Path(record.source_path).stem blob_hash = record.blob_hash or record.raw_id acquired_at_ms = _iso_to_epoch_ms(record.acquired_at) - artifact_classification = _declared_non_session_artifact_classification( - provider, - record.source_path, + # Source-only acquisition deliberately has no decoded + # evidence with which to confirm or override a path + # classification. Keep every such raw pending instead of + # giving a filename-only fact/sidecar rule terminal + # authority that a recovered derived tier could not undo. + artifact_classification = ( + None + if source_only + else _declared_non_session_artifact_classification( + provider, + record.source_path, + ) ) session_evidence = False if artifact_classification is not None and not source_only: diff --git a/tests/unit/browser_capture/test_receiver.py b/tests/unit/browser_capture/test_receiver.py index cea57e9fbb..2271ea2082 100644 --- a/tests/unit/browser_capture/test_receiver.py +++ b/tests/unit/browser_capture/test_receiver.py @@ -703,6 +703,19 @@ def test_receiver_archive_state_reports_missing_without_spool_or_archive(tmp_pat assert Path(state.artifact_ref).is_absolute() is False +def test_receiver_archive_state_tolerates_invalid_active_index_pointer(tmp_path: Path) -> None: + envelope = BrowserCaptureEnvelope.model_validate(_payload()) + write_capture_envelope(envelope, spool_path=tmp_path) + (tmp_path / ".index-active-pointer").write_text("not-an-index.db\n", encoding="utf-8") + + state = BrowserCaptureArchiveStatePayload.model_validate( + existing_capture_state("chatgpt", "conv-123", spool_path=tmp_path, archive_root=tmp_path) + ) + + assert state.state == "spooled_only" + assert state.indexed_session_exists is False + + def test_receiver_archive_state_requires_indexed_messages(tmp_path: Path) -> None: envelope = BrowserCaptureEnvelope.model_validate(_payload()) write_capture_envelope(envelope, spool_path=tmp_path) diff --git a/tests/unit/daemon/test_daemon_cli.py b/tests/unit/daemon/test_daemon_cli.py index 28c4b59da0..373be4c20b 100644 --- a/tests/unit/daemon/test_daemon_cli.py +++ b/tests/unit/daemon/test_daemon_cli.py @@ -1083,6 +1083,80 @@ def cancel_repair(*_args: object, **_kwargs: object) -> object: assert closed == [active_index] +@pytest.mark.parametrize("whale", [False, True]) +def test_raw_materialization_holds_pinned_generation_lease_through_fts_closure( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + whale: bool, +) -> None: + """FTS closure must finish under the same promotion-excluding lease as replay.""" + from polylogue.daemon import cli as daemon_cli + + archive = tmp_path / "archive" + active_index = tmp_path / "generations" / "active" / "index.db" + active_index.parent.mkdir(parents=True) + active_index.touch() + archive.mkdir() + (archive / ".index-active-pointer").write_text(str(active_index), encoding="utf-8") + lease_events: list[str] = [] + held = 0 + + @contextlib.contextmanager + def fake_generation_lease(_config: Config) -> Any: + nonlocal held + held += 1 + lease_events.append("acquire") + try: + yield active_index + finally: + lease_events.append("close") + held -= 1 + + class FakeRestoreResult: + restored_count = 0 + + result = SimpleNamespace( + success=True, + repaired_count=1, + detail="repaired", + metrics={"raw_materialization_remaining_candidate_count": 0}, + ) + closed: list[Path] = [] + + monkeypatch.setattr("polylogue.paths.archive_root", lambda: archive) + monkeypatch.setattr("polylogue.paths.render_root", lambda: tmp_path / "render") + monkeypatch.setattr("polylogue.readiness.capability.raw_frontier_source_selection_block_reason", lambda _root: None) + monkeypatch.setattr( + "polylogue.storage.blob_integrity.restore_direct_blob_reference_debt", + lambda *_args, **_kwargs: FakeRestoreResult(), + ) + monkeypatch.setattr("polylogue.product.raw_authority.recover_interrupted_frontier", lambda _config: ()) + monkeypatch.setattr("polylogue.product.raw_authority.auto_resolve_stale_plan_blockers", lambda _config: 0) + monkeypatch.setattr("polylogue.product.raw_authority.repair_materialization", lambda *_args, **_kwargs: result) + monkeypatch.setattr("polylogue.product.raw_authority.materialization_generation_lease", fake_generation_lease) + monkeypatch.setattr(daemon_cli, "_emit_raw_materialization_pass", lambda _result: None) + monkeypatch.setattr(daemon_cli, "_converge_raw_authority_frontier", lambda _config, **_kwargs: 0) + + def close_fts(index_db: Path) -> None: + assert held == 1 + closed.append(index_db) + lease_events.append("fts") + + monkeypatch.setattr(daemon_cli, "_close_raw_materialization_fts", close_fts) + + if whale: + assert ( + daemon_cli._run_raw_materialization_whale_pass_once(raw_artifact_id="raw-whale", max_payload_bytes=123) + is result + ) + else: + assert daemon_cli._drain_raw_materialization_once().repaired_sessions == 1 + + assert closed == [active_index] + assert lease_events == ["acquire", "fts", "close"] + assert held == 0 + + def test_raw_materialization_fts_failure_records_durable_debt( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, diff --git a/tests/unit/product/test_raw_authority.py b/tests/unit/product/test_raw_authority.py index 4574213629..efedba2483 100644 --- a/tests/unit/product/test_raw_authority.py +++ b/tests/unit/product/test_raw_authority.py @@ -7,6 +7,7 @@ from polylogue.config import Config from polylogue.product import raw_authority +from polylogue.storage.index_generation import RebuildLease, RebuildLeaseUnavailableError from polylogue.storage.raw_authority import raw_authority_detail_query_handle from polylogue.storage.raw_reconciler import RawAuthorityFrontierApplyReport @@ -89,6 +90,20 @@ def test_frontier_apply_report_rejects_incoherent_counts() -> None: ) +def test_materialization_generation_lease_pins_active_index_and_excludes_promotion(tmp_path: Path) -> None: + active_index = tmp_path / "generations" / "active" / "index.db" + active_index.parent.mkdir(parents=True) + active_index.touch() + (tmp_path / ".index-active-pointer").write_text(str(active_index), encoding="utf-8") + config = Config(archive_root=tmp_path, render_root=tmp_path / "render", sources=[]) + + with raw_authority.materialization_generation_lease(config) as index_db: + assert index_db == active_index + with pytest.raises(RebuildLeaseUnavailableError): + with RebuildLease(tmp_path): + pass + + @pytest.mark.parametrize( ("selected_plan_ids", "preview_census_id", "outcome_plan_id", "message"), [ diff --git a/tests/unit/sources/test_live_batch_support.py b/tests/unit/sources/test_live_batch_support.py index 75be205fac..173c994fc8 100644 --- a/tests/unit/sources/test_live_batch_support.py +++ b/tests/unit/sources/test_live_batch_support.py @@ -402,10 +402,13 @@ def test_full_ingest_acquires_but_does_not_parse_when_derived_tier_degraded( ) json_path = root / "degraded-full.json" json_path.write_bytes(b'{"mapping":{"root":{"message":{"author":{"role":"user"}}}}}') + classified_path = root / "subagents" / "worker" / "agent-degraded.meta.json" + classified_path.parent.mkdir(parents=True) + classified_path.write_bytes(b'{"mapping":{"root":{"message":{"author":{"role":"user"}}}}}') index_db = tmp_path / "index.db" processor = LiveBatchProcessor( cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=index_db))), - (WatchSource(name="codex", root=root),), + (WatchSource(name="claude-code", root=root),), cursor=CursorStore(index_db), parser_fingerprint="test-parser", ) @@ -433,15 +436,19 @@ def test_full_ingest_acquires_but_does_not_parse_when_derived_tier_degraded( lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("must not detect source-only provider")), ) try: - result = processor._ingest_full_paths_sync([path, json_path], source_name="codex") + result = processor._ingest_full_paths_sync([path, json_path, classified_path], source_name="claude-code") finally: clear_degraded() - assert result.succeeded == [path, json_path] + assert result.succeeded == [path, json_path, classified_path] assert result.failed == [] with sqlite3.connect(tmp_path / "source.db") as conn: raw_states = conn.execute("SELECT parsed_at_ms, parse_error FROM raw_sessions ORDER BY source_path").fetchall() - assert raw_states == [(None, None), (None, None)] + artifact_rows = conn.execute( + "SELECT COUNT(*) FROM raw_artifacts WHERE source_path = ?", (str(classified_path),) + ).fetchone() + assert raw_states == [(None, None), (None, None), (None, None)] + assert artifact_rows == (0,) def test_full_ingest_acquires_when_index_is_genuinely_semantic_distance_stale( From 35abd21def8637df0dc4c5ee33892994ad34d26f Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 10:25:56 +0200 Subject: [PATCH 31/65] fix(ingest): preserve source authority across outages --- polylogue/browser_capture/receiver.py | 19 ++-- polylogue/core/raw_state.py | 28 ++++++ polylogue/daemon/cli.py | 22 +++-- polylogue/daemon/provenance.py | 17 +++- polylogue/schemas/sampling_db.py | 12 ++- polylogue/sources/live/batch.py | 95 +++++++++++++++++-- polylogue/sources/revision_backfill.py | 28 ++++-- .../sources/source_acquisition_components.py | 24 +++++ polylogue/storage/raw_retention.py | 5 +- polylogue/storage/repair.py | 7 +- polylogue/storage/sqlite/raw_state_update.py | 26 ++++- tests/unit/browser_capture/test_receiver.py | 22 ++++- tests/unit/core/test_sampling.py | 29 ++++++ tests/unit/daemon/test_daemon_cli.py | 47 +++++---- tests/unit/daemon/test_provenance_endpoint.py | 20 +++- tests/unit/sources/test_live_batch_support.py | 83 ++++++++++++++++ tests/unit/sources/test_revision_backfill.py | 17 ++++ tests/unit/storage/test_parse_tracking.py | 42 ++++++++ tests/unit/storage/test_repair.py | 9 +- 19 files changed, 481 insertions(+), 71 deletions(-) create mode 100644 polylogue/core/raw_state.py diff --git a/polylogue/browser_capture/receiver.py b/polylogue/browser_capture/receiver.py index fbada1bbc8..b94ffcfce6 100644 --- a/polylogue/browser_capture/receiver.py +++ b/polylogue/browser_capture/receiver.py @@ -31,6 +31,7 @@ from polylogue.core.hashing import hash_text_short from polylogue.core.json import JSONDecodeError, dumps_bytes from polylogue.core.json import loads as json_loads +from polylogue.core.raw_state import raw_state_authority from polylogue.core.timestamps import parse_timestamp from polylogue.logging import get_logger from polylogue.paths import archive_root as default_archive_root @@ -405,28 +406,26 @@ def _lookup_raw_archive_state( validation_status = ( str(row["validation_status"]) if "validation_status" in row_keys and row["validation_status"] else None ) - validation_is_current = ( - "parsed_at_ms" not in row_keys - or row["parsed_at_ms"] is None - or ( - "validated_at_ms" in row_keys - and row["validated_at_ms"] is not None - and row["validated_at_ms"] > row["parsed_at_ms"] - ) + validation_authority = raw_state_authority( + row["parsed_at_ms"] if "parsed_at_ms" in row_keys else None, + row["validated_at_ms"] if "validated_at_ms" in row_keys else None, ) if isinstance(parse_error, str) and parse_error: latest_failure = parse_error failure_source = "raw_parse" - elif validation_is_current and isinstance(validation_error, str) and validation_error: + elif validation_authority == "validation" and isinstance(validation_error, str) and validation_error: latest_failure = validation_error failure_source = "raw_validation" elif ( - validation_is_current + validation_authority == "validation" and validation_status is not None and validation_status not in {"passed", "valid", "ok"} ): latest_failure = validation_status failure_source = "raw_validation" + elif validation_authority == "ambiguous" and validation_status not in {None, "passed", "valid", "ok"}: + latest_failure = "raw validation and parse timestamps are indeterminate" + failure_source = "raw_state_order" return _RawArchiveLookup( raw_row_exists=True, raw_id=str(row["raw_id"]) if "raw_id" in row_keys and row["raw_id"] is not None else None, diff --git a/polylogue/core/raw_state.py b/polylogue/core/raw_state.py new file mode 100644 index 0000000000..132d2f8ee6 --- /dev/null +++ b/polylogue/core/raw_state.py @@ -0,0 +1,28 @@ +"""Ordering authority for durable raw parse and validation transitions.""" + +from __future__ import annotations + +from typing import Literal, TypeAlias + +RawStateAuthority: TypeAlias = Literal["parse", "validation", "ambiguous"] + + +def raw_state_authority( + parsed_at_ms: int | None, + validated_at_ms: int | None, +) -> RawStateAuthority: + """Return the proven terminal transition, never assigning equal times. + + New writes make opposing transitions strictly monotonic. Existing rows can + predate that invariant, so an equal non-null pair remains explicitly + indeterminate rather than being silently attributed to either stage. + """ + if parsed_at_ms is None: + return "validation" + if validated_at_ms is None: + return "parse" + if parsed_at_ms > validated_at_ms: + return "parse" + if validated_at_ms > parsed_at_ms: + return "validation" + return "ambiguous" diff --git a/polylogue/daemon/cli.py b/polylogue/daemon/cli.py index 2617f0d7fe..6a57b41040 100644 --- a/polylogue/daemon/cli.py +++ b/polylogue/daemon/cli.py @@ -1320,7 +1320,7 @@ def _drain_raw_materialization_once( max_pass_seconds=_RAW_MATERIALIZATION_MAX_PASS_SECONDS, ) finally: - _close_raw_materialization_fts(index_db) + _close_raw_materialization_fts(index_db, ops_db_path=config.archive_root / "ops.db") _emit_raw_materialization_pass(result) frontier_repaired = _converge_raw_authority_frontier(config, limit=min(limit, 8)) if not result.success: @@ -1389,7 +1389,7 @@ def _run_raw_materialization_whale_pass_once(*, raw_artifact_id: str, max_payloa raw_artifact_id=raw_artifact_id, ) finally: - _close_raw_materialization_fts(index_db) + _close_raw_materialization_fts(index_db, ops_db_path=config.archive_root / "ops.db") _emit_raw_materialization_pass(result) if not result.success: logger.warning("raw materialization: whale pass for %s incomplete: %s", raw_artifact_id, result.detail) @@ -1608,7 +1608,7 @@ def _emit_raw_materialization_pass(result: Any) -> None: ) -def _close_raw_materialization_fts(index_db: Path) -> None: +def _close_raw_materialization_fts(index_db: Path, *, ops_db_path: Path) -> None: """Return message search to ready or leave explicit retryable debt. Large raw replay batches deliberately suspend FTS triggers and may skip @@ -1622,7 +1622,9 @@ def _close_raw_materialization_fts(index_db: Path) -> None: try: needs_repair = _raw_materialization_fts_needs_repair(index_db) except Exception as exc: - _record_raw_materialization_fts_debt(index_db, f"FTS readiness probe failed after raw materialization: {exc}") + _record_raw_materialization_fts_debt( + index_db, ops_db_path=ops_db_path, error=f"FTS readiness probe failed after raw materialization: {exc}" + ) return if not needs_repair: return @@ -1634,14 +1636,15 @@ def _close_raw_materialization_fts(index_db: Path) -> None: # this closure retryable instead of masking the initiating failure. _record_raw_materialization_fts_debt( index_db, - f"FTS repair failed after raw materialization: {type(exc).__name__}: {exc}", + ops_db_path=ops_db_path, + error=f"FTS repair failed after raw materialization: {type(exc).__name__}: {exc}", ) return if repaired: try: from polylogue.sources.live.cursor import CursorStore - CursorStore(index_db).clear_convergence_debt( + CursorStore(index_db, ops_db_path=ops_db_path).clear_convergence_debt( subject_type="fts_surface", subject_id="messages_fts", stage="fts", @@ -1651,15 +1654,16 @@ def _close_raw_materialization_fts(index_db: Path) -> None: return _record_raw_materialization_fts_debt( index_db, - "raw materialization exited without restoring message FTS readiness", + ops_db_path=ops_db_path, + error="raw materialization exited without restoring message FTS readiness", ) -def _record_raw_materialization_fts_debt(index_db: Path, error: str) -> None: +def _record_raw_materialization_fts_debt(index_db: Path, *, ops_db_path: Path, error: str) -> None: from polylogue.sources.live.cursor import CursorStore try: - CursorStore(index_db).record_convergence_debt( + CursorStore(index_db, ops_db_path=ops_db_path).record_convergence_debt( stage="fts", subject_type="fts_surface", subject_id="messages_fts", diff --git a/polylogue/daemon/provenance.py b/polylogue/daemon/provenance.py index 4a350ae83f..08220adec4 100644 --- a/polylogue/daemon/provenance.py +++ b/polylogue/daemon/provenance.py @@ -34,6 +34,7 @@ from pathlib import Path from typing import Final +from polylogue.core.raw_state import raw_state_authority from polylogue.logging import get_logger from polylogue.paths import archive_root from polylogue.storage.archive_identity import resolve_active_index_path @@ -62,8 +63,10 @@ class ProvenanceRow: acquired_at: str | None file_mtime: str | None parsed_at: str | None + parsed_at_ms: int | None parse_error: str | None validated_at: str | None + validated_at_ms: int | None validation_status: str | None validation_error: str | None @@ -185,8 +188,10 @@ def _fetch_archive_provenance_row( acquired_at=_iso_from_epoch_ms(row["acquired_at_ms"]), file_mtime=_iso_from_epoch_ms(row["file_mtime_ms"]), parsed_at=_iso_from_epoch_ms(row["parsed_at_ms"]), + parsed_at_ms=(int(row["parsed_at_ms"]) if row["parsed_at_ms"] is not None else None), parse_error=(str(row["parse_error"]) if row["parse_error"] is not None else None), validated_at=_iso_from_epoch_ms(row["validated_at_ms"]), + validated_at_ms=(int(row["validated_at_ms"]) if row["validated_at_ms"] is not None else None), validation_status=(str(row["validation_status"]) if row["validation_status"] is not None else None), validation_error=(str(row["validation_error"]) if row["validation_error"] is not None else None), ) @@ -253,8 +258,10 @@ def fetch_provenance_row(session_id: str) -> ProvenanceRow | None: acquired_at=(str(row["acquired_at"]) if row["acquired_at"] is not None else None), file_mtime=(str(row["file_mtime"]) if row["file_mtime"] is not None else None), parsed_at=(str(row["parsed_at"]) if row["parsed_at"] is not None else None), + parsed_at_ms=None, parse_error=(str(row["parse_error"]) if row["parse_error"] is not None else None), validated_at=(str(row["validated_at"]) if row["validated_at"] is not None else None), + validated_at_ms=None, validation_status=(str(row["validation_status"]) if row["validation_status"] is not None else None), validation_error=(str(row["validation_error"]) if row["validation_error"] is not None else None), ) @@ -272,10 +279,12 @@ def _quarantine_state(row: ProvenanceRow) -> tuple[bool, str | None]: return True, "no_raw_artifact" if row.parse_error: return True, "parse_error" - if row.validation_status == "failed" and ( - row.parsed_at is None or row.validated_at is None or row.validated_at > row.parsed_at - ): - return True, "validation_failed" + if row.validation_status == "failed": + validation_authority = raw_state_authority(row.parsed_at_ms, row.validated_at_ms) + if validation_authority == "validation": + return True, "validation_failed" + if validation_authority == "ambiguous": + return True, "validation_parse_order_ambiguous" return False, None diff --git a/polylogue/schemas/sampling_db.py b/polylogue/schemas/sampling_db.py index d8fac0cd7e..9c01e68f2d 100644 --- a/polylogue/schemas/sampling_db.py +++ b/polylogue/schemas/sampling_db.py @@ -22,6 +22,7 @@ canonical_runtime_provider, canonical_schema_provider, ) +from polylogue.core.raw_state import raw_state_authority from polylogue.core.sources import origin_from_provider, provider_from_origin from polylogue.logging import get_logger from polylogue.paths import db_path as index_db_path @@ -372,14 +373,17 @@ def _iter_schema_units_from_db( ) continue - if row.validation_status == "failed" and ( - row.parsed_at_ms is None or row.validated_at_ms is None or row.validated_at_ms > row.parsed_at_ms - ): + validation_authority = raw_state_authority(row.parsed_at_ms, row.validated_at_ms) + if row.validation_status == "failed" and validation_authority != "parse": _record_terminal( terminal_recorder, row, status="quarantined", - reason="source_validation_failed", + reason=( + "source_validation_failed" + if validation_authority == "validation" + else "source_validation_parse_order_ambiguous" + ), ) continue diff --git a/polylogue/sources/live/batch.py b/polylogue/sources/live/batch.py index 195e2c99fc..b3ca89ee51 100644 --- a/polylogue/sources/live/batch.py +++ b/polylogue/sources/live/batch.py @@ -146,6 +146,7 @@ _DETECTION_PREFIX_SIZE, ZipEntryReadContext, iter_zip_entry_raw_data, + stream_preserved_zip_entry_raw_data, ) from polylogue.sources.source_parsing import has_decoded_session_evidence from polylogue.sources.sqlite_snapshot import ( @@ -1904,12 +1905,20 @@ def _ingest_full_paths_sync( ) if path.suffix.lower() == ".zip": file_mtime = datetime.fromtimestamp(stat.st_mtime_ns / 1_000_000_000, UTC).isoformat() - zip_records, zip_bytes = self._extract_zip_member_records( - path, - blob_store=blob_store, - fallback_provider=fallback_provider, - file_mtime=file_mtime, - ) + if source_only: + zip_records, zip_bytes = self._extract_source_only_zip_member_records( + path, + blob_store=blob_store, + fallback_provider=fallback_provider, + file_mtime=file_mtime, + ) + else: + zip_records, zip_bytes = self._extract_zip_member_records( + path, + blob_store=blob_store, + fallback_provider=fallback_provider, + file_mtime=file_mtime, + ) if not zip_records: self._mark_excluded_cursor(path, stat, source_name=fallback_provider.value) continue @@ -1962,7 +1971,9 @@ def _ingest_full_paths_sync( current_path=path, source_payload_read_bytes=source_payload_read_bytes, ) - elif path.name in _CODEX_STATE_DB_NAMES and codex_state.is_in_scope_codex_sqlite_path(path): + elif path.name in _CODEX_STATE_DB_NAMES and ( + source_only or codex_state.is_in_scope_codex_sqlite_path(path) + ): # polylogue-0jf4: acquire live Codex SQLite state the same # way Hermes acquires its state.db -- a consistent # backup/snapshot (never a raw read of a possibly-live-locked @@ -1970,6 +1981,9 @@ def _ingest_full_paths_sync( # gate keeps this cheap for the vast majority of ~/.codex # traffic (JSONL rollouts); ``is_in_scope_codex_sqlite_path`` # then re-confirms the table shape before trusting the name. + # Source-only acquisition intentionally skips that structural + # decode: a mid-write or future-schema state snapshot is still + # durable authority to replay once the derived tier returns. provider = Provider.CODEX source_name = provider.value try: @@ -3343,6 +3357,73 @@ def _extract_zip_member_records( return [], 0 return records, total_bytes + def _extract_source_only_zip_member_records( + self, + path: Path, + *, + blob_store: BlobStore, + fallback_provider: Provider, + file_mtime: str, + ) -> tuple[list[tuple[str, RawSessionRecord]], int]: + """Acquire admitted ZIP members without interpreting their bytes. + + A derived-tier outage does not authorize the source tier to infer a + provider, parse JSON, or classify a member. It does still enforce the + ordinary ZIP admission policy before streaming every retained member + under its exact ``:`` coordinate. + """ + source = Source(name=fallback_provider.value, path=path.parent) + acquired_at = datetime.now(UTC).isoformat() + records: list[tuple[str, RawSessionRecord]] = [] + total_bytes = 0 + validator = _ZipEntryValidator(fallback_provider, cursor_state=None, zip_path=path) + try: + with zipfile.ZipFile(path) as zf: + for source_index, info in enumerate(validator.filter_entries(zf.infolist())): + if info.file_size == 0: + continue + try: + raw_data = stream_preserved_zip_entry_raw_data( + zf, + ZipEntryReadContext( + source=source, + zip_path=path, + entry=info, + file_mtime=file_mtime, + provider_hint=fallback_provider, + blob_store=blob_store, + ), + provider_hint=fallback_provider, + source_index=source_index, + ) + except ZipBombError as exc: + logger.warning("Skipping ZIP member %s in %s: %s", info.filename, path, exc) + continue + if raw_data.blob_hash is None: + continue + total_bytes += raw_data.blob_size or 0 + records.append( + ( + raw_data.blob_hash, + RawSessionRecord( + raw_id=raw_data.blob_hash, + payload_provider=fallback_provider, + capture_mode=fallback_provider, + source_name=fallback_provider.value, + source_path=raw_data.source_path, + source_index=source_index, + blob_size=raw_data.blob_size or 0, + blob_publication_receipt_id=raw_data.blob_publication_receipt_id, + acquired_at=acquired_at, + file_mtime=raw_data.file_mtime, + ), + ) + ) + except (zipfile.BadZipFile, OSError) as exc: + logger.warning("Failed to expand inbox ZIP %s: %s", path, exc) + return [], 0 + return records, total_bytes + @staticmethod def _sniff_zip_provider( zf: zipfile.ZipFile, diff --git a/polylogue/sources/revision_backfill.py b/polylogue/sources/revision_backfill.py index 209b9ceb83..6facfd5812 100644 --- a/polylogue/sources/revision_backfill.py +++ b/polylogue/sources/revision_backfill.py @@ -46,6 +46,7 @@ ) from polylogue.sources.decoders import _iter_json_stream from polylogue.sources.dispatch import ( + is_jsonl_source_path, is_stream_record_provider, parse_payload, parse_stream_payload, @@ -3017,9 +3018,11 @@ def _declared_non_session_artifact_classification( these; this replay engine (used by ``polylogue ops reset --index`` / ``devtools`` rebuild-index) is a SEPARATE parse chokepoint that did not, and would silently recreate exactly the ``.meta`` phantom sessions - that fix is meant to eliminate on every future rebuild. Same check, same - rule table, so a declared fact artifact can never become a session - through either entry point. + that fix is meant to eliminate on every future rebuild. A positive JSONL + session proof is the one deliberate exception, matching the live route: + a source-only outage may retain bytes before it can inspect a path that + normally carries fact evidence, and recovery must not make that filename + permanently override later decoded session authority. polylogue-9ykn: a path-declared rule is only half of the live path's gate. ``pipeline/services/ingest_worker.py`` also runs every sampled @@ -3044,7 +3047,7 @@ def _declared_non_session_artifact_classification( from polylogue.archive.artifact_taxonomy import classify_artifact rule = artifact_rule_for_path(provider, source_path) - if rule is not None and rule.parse_policy != "session": + if rule is not None and rule.parse_policy != "session" and not sample: classification = classify_artifact([], provider=provider, source_path=source_path) if not classification.parse_as_session: return classification @@ -3137,9 +3140,17 @@ def _parse_one_raw( return sessions source_name = Path(source_path).name fallback_id = fallback_id_override or Path(source_path).stem + rule = artifact_rule_for_path(provider, source_path) + declared_path_session_evidence = False + if rule is not None and rule.parse_policy != "session" and is_jsonl_source_path(source_path): + from polylogue.archive.raw_payload.decode import jsonl_session_artifact + + declared_path_session_evidence = jsonl_session_artifact(payload, provider=provider) is not None if is_stream_record_provider(source_path, str(provider)): records = list(_iter_json_stream(BytesIO(payload), source_name)) - if _is_declared_non_session_artifact(provider, source_path, sample=records[:64]): + if not declared_path_session_evidence and _is_declared_non_session_artifact( + provider, source_path, sample=records[:64] + ): return [] return parse_stream_payload( provider, @@ -3147,7 +3158,10 @@ def _parse_one_raw( fallback_id, source_path=source_path, ) - if _is_declared_non_session_artifact(provider, source_path): + records = list(_iter_json_stream(BytesIO(payload), source_name)) + if not declared_path_session_evidence and _is_declared_non_session_artifact( + provider, source_path, sample=records[:64] + ): return [] if provider is Provider.HERMES and looks_like_sqlite_bytes(payload): with _sqlite_payload_path(payload, payload_path, archive_root) as sqlite_path: @@ -3167,7 +3181,7 @@ def _parse_one_raw( ) return parse_payload( provider, - list(_iter_json_stream(BytesIO(payload), source_name)), + records, fallback_id, source_path=source_path, ) diff --git a/polylogue/sources/source_acquisition_components.py b/polylogue/sources/source_acquisition_components.py index f3280040ff..beeab04f8b 100644 --- a/polylogue/sources/source_acquisition_components.py +++ b/polylogue/sources/source_acquisition_components.py @@ -421,6 +421,28 @@ def _stream_preserved_zip_entry( *, provider_hint: Provider, ) -> RawSessionData: + return stream_preserved_zip_entry_raw_data( + zf, + context, + provider_hint=provider_hint, + ) + + +def stream_preserved_zip_entry_raw_data( + zf: zipfile.ZipFile, + context: ZipEntryReadContext, + *, + provider_hint: Provider, + source_index: int | None = None, +) -> RawSessionData: + """Durably stream one admitted ZIP member without decoding its content. + + The caller remains responsible for applying :class:`_ZipEntryValidator` + before this function. Keeping the bounded entry reader here means a + source-tier-only outage retains the same ZIP-bomb protection as ordinary + acquisition while deliberately avoiding provider detection, JSON decoding, + and artifact classification. + """ with _decoders.open_bounded_zip_entry(zf, context.entry) as handle: blob_hash, blob_size = stream_fileobj_to_blob( context.blob_store, @@ -446,6 +468,7 @@ def _stream_preserved_zip_entry( provider_hint=provider_hint, blob_hash=blob_hash, blob_size=blob_size, + source_index=source_index, blob_publication_receipt_id=publication_id, ) @@ -528,6 +551,7 @@ def iter_zip_entry_raw_data( "observe_acquisition", "raw_data_record", "read_plain_source_file", + "stream_preserved_zip_entry_raw_data", "stream_fileobj_to_blob", "stream_path_to_blob", ] diff --git a/polylogue/storage/raw_retention.py b/polylogue/storage/raw_retention.py index 8fc55218a0..5c7ec3ff20 100644 --- a/polylogue/storage/raw_retention.py +++ b/polylogue/storage/raw_retention.py @@ -1710,7 +1710,10 @@ def _terminal_artifact_paths(conn: sqlite3.Connection, source_paths: set[str]) - AND ( evidence_raw.parsed_at_ms IS NULL OR evidence_raw.validated_at_ms IS NULL - OR evidence_raw.validated_at_ms > evidence_raw.parsed_at_ms + -- A legacy tie has no proven winner; + -- retain it rather than deleting raw + -- authority based on an arbitrary side. + OR evidence_raw.validated_at_ms >= evidence_raw.parsed_at_ms ) ) ) diff --git a/polylogue/storage/repair.py b/polylogue/storage/repair.py index 0446393594..1c26b035db 100644 --- a/polylogue/storage/repair.py +++ b/polylogue/storage/repair.py @@ -3966,7 +3966,10 @@ def _raw_materialization_candidate_ids( AND ( r.parsed_at_ms IS NULL OR r.validated_at_ms IS NULL - OR r.validated_at_ms > r.parsed_at_ms + -- Equal legacy timestamps are indeterminate. Do not + -- replay and overwrite either authority until a new, + -- monotonic transition resolves the ambiguity. + OR r.validated_at_ms >= r.parsed_at_ms ) ) AND ( @@ -4004,7 +4007,7 @@ def _raw_materialization_candidate_ids( AND ( r.parsed_at_ms IS NULL OR r.validated_at_ms IS NULL - OR r.validated_at_ms > r.parsed_at_ms + OR r.validated_at_ms >= r.parsed_at_ms ) ) ) diff --git a/polylogue/storage/sqlite/raw_state_update.py b/polylogue/storage/sqlite/raw_state_update.py index c145503b4f..791d916462 100644 --- a/polylogue/storage/sqlite/raw_state_update.py +++ b/polylogue/storage/sqlite/raw_state_update.py @@ -18,9 +18,25 @@ def compile_raw_state_update( """Compile one typed mutation for either SQLite connection adapter.""" set_clauses: list[str] = [] params: list[object] = [] + parsed_at_ms = _timestamp_ms(state.parsed_at) if isinstance(state.parsed_at, str) else None + validation_transition = state.validation_status is not UNSET or state.validation_error is not UNSET if state.parsed_at is not UNSET: - set_clauses.append("parsed_at_ms = ?") - params.append(_timestamp_ms(state.parsed_at) if isinstance(state.parsed_at, str) else None) + if parsed_at_ms is None: + set_clauses.append("parsed_at_ms = ?") + params.append(None) + elif validation_transition: + # SQLite evaluates every SET expression from the old row. A + # combined update records validation first and parse second, so + # advance parse by two from either old transition (and one from + # this validation clock) to preserve that authority ordering even + # when wall time is equal or moves backward. + set_clauses.append( + "parsed_at_ms = MAX(?, ? + 1, COALESCE(parsed_at_ms + 2, ?), COALESCE(validated_at_ms + 2, ?))" + ) + params.extend((parsed_at_ms, now_ms, parsed_at_ms, parsed_at_ms)) + else: + set_clauses.append("parsed_at_ms = MAX(?, COALESCE(parsed_at_ms + 1, ?), COALESCE(validated_at_ms + 1, ?))") + params.extend((parsed_at_ms, parsed_at_ms, parsed_at_ms)) if state.parse_error is not UNSET: set_clauses.append("parse_error = ?") params.append(state.parse_error[:2000] if isinstance(state.parse_error, str) else state.parse_error) @@ -53,9 +69,9 @@ def compile_raw_state_update( warnings = state.detection_warnings set_clauses.append("detection_warnings_json = ?") params.append(json.dumps([warnings[:2000]]) if isinstance(warnings, str) and warnings else "[]") - if state.validation_status is not UNSET or state.validation_error is not UNSET: - set_clauses.append("validated_at_ms = ?") - params.append(now_ms) + if validation_transition: + set_clauses.append("validated_at_ms = MAX(?, COALESCE(validated_at_ms + 1, ?), COALESCE(parsed_at_ms + 1, ?))") + params.extend((now_ms, now_ms, now_ms)) return tuple(set_clauses), tuple(params) diff --git a/tests/unit/browser_capture/test_receiver.py b/tests/unit/browser_capture/test_receiver.py index 2271ea2082..e228ee7ec4 100644 --- a/tests/unit/browser_capture/test_receiver.py +++ b/tests/unit/browser_capture/test_receiver.py @@ -803,7 +803,7 @@ def test_receiver_uses_active_index_and_ignores_historical_validation_failure(tm tmp_path, validation_status="failed", parsed_at_ms=1, - validated_at_ms=1, + validated_at_ms=0, message_count=0, ) active_index = tmp_path / "generations" / "active" / "index.db" @@ -844,6 +844,26 @@ def test_receiver_surfaces_validation_failure_newer_than_parse(tmp_path: Path) - assert state.failure_source == "raw_validation" +def test_receiver_surfaces_indeterminate_raw_state_order_without_choosing_validation(tmp_path: Path) -> None: + envelope = BrowserCaptureEnvelope.model_validate(_payload()) + write_capture_envelope(envelope, spool_path=tmp_path) + _seed_browser_capture_archive( + tmp_path, + validation_status="failed", + parsed_at_ms=1, + validated_at_ms=1, + validation_error="equal-time failure", + ) + + state = BrowserCaptureArchiveStatePayload.model_validate( + existing_capture_state("chatgpt", "conv-123", spool_path=tmp_path, archive_root=tmp_path) + ) + + assert state.state == "failed" + assert state.latest_failure == "raw validation and parse timestamps are indeterminate" + assert state.failure_source == "raw_state_order" + + def test_receiver_echoes_safe_request_id_header(tmp_path: Path) -> None: with _running_receiver(tmp_path) as (host, port): conn = HTTPConnection(host, port) diff --git a/tests/unit/core/test_sampling.py b/tests/unit/core/test_sampling.py index 76ed5baf0b..d6be892135 100644 --- a/tests/unit/core/test_sampling.py +++ b/tests/unit/core/test_sampling.py @@ -328,6 +328,35 @@ def test_sampling_quarantines_validation_failure_newer_than_parse(self, tmp_path } ] + def test_sampling_records_equal_raw_transition_timestamps_as_indeterminate(self, tmp_path: Path) -> None: + db = _archive_index_db(tmp_path) + raw_id = _insert_raw_session( + db_path=db, + origin="claude-ai-export", + source_path="/tmp/equal-time.json", + raw_content=b'{"uuid":"equal-time","chat_messages":[]}', + ) + with sqlite3.connect(db.with_name("source.db")) as conn: + cursor = conn.execute( + "UPDATE raw_sessions SET parsed_at_ms = 1, validated_at_ms = 1, validation_status = 'failed' WHERE raw_id = ?", + (raw_id,), + ) + assert cursor.rowcount == 1 + conn.commit() + outcomes: list[dict[str, object]] = [] + + result = list( + iter_schema_units( + "claude-ai", + db_path=db, + full_corpus=True, + terminal_recorder=lambda **outcome: outcomes.append(outcome), + ) + ) + + assert result == [] + assert outcomes[0]["reason"] == "source_validation_parse_order_ambiguous" + def test_record_provider_sampling_streams_without_full_envelope( self, tmp_path: Path, diff --git a/tests/unit/daemon/test_daemon_cli.py b/tests/unit/daemon/test_daemon_cli.py index 373be4c20b..d78c43e1cb 100644 --- a/tests/unit/daemon/test_daemon_cli.py +++ b/tests/unit/daemon/test_daemon_cli.py @@ -664,7 +664,7 @@ def fake_repair(*_args: object, **_kwargs: object) -> object: return SimpleNamespace(success=True, repaired_count=1, detail="unexpected writer call") monkeypatch.setattr("polylogue.product.raw_authority.repair_materialization", fake_repair) - monkeypatch.setattr(daemon_cli, "_close_raw_materialization_fts", lambda _path: None) + monkeypatch.setattr(daemon_cli, "_close_raw_materialization_fts", lambda _path, *, ops_db_path: None) monkeypatch.setattr(daemon_cli, "_emit_raw_materialization_pass", lambda _result: None) with pytest.raises(RuntimeError, match="source-selection gate blocked"): @@ -1059,7 +1059,7 @@ def test_raw_materialization_closes_fts_on_cancellation( active_index.touch() archive.mkdir() (archive / ".index-active-pointer").write_text(str(active_index), encoding="utf-8") - closed: list[Path] = [] + closed: list[tuple[Path, Path]] = [] class FakeRestoreResult: restored_count = 0 @@ -1075,12 +1075,16 @@ def cancel_repair(*_args: object, **_kwargs: object) -> object: lambda *_args, **_kwargs: FakeRestoreResult(), ) monkeypatch.setattr("polylogue.storage.repair.repair_raw_materialization", cancel_repair) - monkeypatch.setattr(daemon_cli, "_close_raw_materialization_fts", closed.append) + monkeypatch.setattr( + daemon_cli, + "_close_raw_materialization_fts", + lambda index_db, *, ops_db_path: closed.append((index_db, ops_db_path)), + ) with pytest.raises(asyncio.CancelledError): daemon_cli._drain_raw_materialization_once() - assert closed == [active_index] + assert closed == [(active_index, archive / "ops.db")] @pytest.mark.parametrize("whale", [False, True]) @@ -1121,7 +1125,7 @@ class FakeRestoreResult: detail="repaired", metrics={"raw_materialization_remaining_candidate_count": 0}, ) - closed: list[Path] = [] + closed: list[tuple[Path, Path]] = [] monkeypatch.setattr("polylogue.paths.archive_root", lambda: archive) monkeypatch.setattr("polylogue.paths.render_root", lambda: tmp_path / "render") @@ -1137,9 +1141,9 @@ class FakeRestoreResult: monkeypatch.setattr(daemon_cli, "_emit_raw_materialization_pass", lambda _result: None) monkeypatch.setattr(daemon_cli, "_converge_raw_authority_frontier", lambda _config, **_kwargs: 0) - def close_fts(index_db: Path) -> None: + def close_fts(index_db: Path, *, ops_db_path: Path) -> None: assert held == 1 - closed.append(index_db) + closed.append((index_db, ops_db_path)) lease_events.append("fts") monkeypatch.setattr(daemon_cli, "_close_raw_materialization_fts", close_fts) @@ -1152,7 +1156,7 @@ def close_fts(index_db: Path) -> None: else: assert daemon_cli._drain_raw_materialization_once().repaired_sessions == 1 - assert closed == [active_index] + assert closed == [(active_index, archive / "ops.db")] assert lease_events == ["acquire", "fts", "close"] assert held == 0 @@ -1163,13 +1167,16 @@ def test_raw_materialization_fts_failure_records_durable_debt( ) -> None: from polylogue.daemon import cli as daemon_cli - index_db = tmp_path / "index.db" + index_db = tmp_path / "generations" / "active" / "index.db" + ops_db = tmp_path / "ops.db" + index_db.parent.mkdir(parents=True) index_db.touch() calls: list[tuple[str, str, str, str | None]] = [] class FakeCursor: - def __init__(self, db: Path) -> None: + def __init__(self, db: Path, *, ops_db_path: Path) -> None: assert db == index_db + assert ops_db_path == ops_db def clear_convergence_debt(self, **_kwargs: object) -> None: raise AssertionError("failed FTS repair must not clear debt") @@ -1188,7 +1195,7 @@ def record_convergence_debt( monkeypatch.setattr("polylogue.daemon.convergence_stages.repair_fts_surface", lambda *_args: False) monkeypatch.setattr("polylogue.sources.live.cursor.CursorStore", FakeCursor) - daemon_cli._close_raw_materialization_fts(index_db) + daemon_cli._close_raw_materialization_fts(index_db, ops_db_path=ops_db) assert calls == [ ( @@ -1206,13 +1213,16 @@ def test_raw_materialization_fts_success_clears_prior_debt( ) -> None: from polylogue.daemon import cli as daemon_cli - index_db = tmp_path / "index.db" + index_db = tmp_path / "generations" / "active" / "index.db" + ops_db = tmp_path / "ops.db" + index_db.parent.mkdir(parents=True) index_db.touch() cleared: list[dict[str, object]] = [] class FakeCursor: - def __init__(self, db: Path) -> None: + def __init__(self, db: Path, *, ops_db_path: Path) -> None: assert db == index_db + assert ops_db_path == ops_db def clear_convergence_debt(self, **kwargs: object) -> None: cleared.append(kwargs) @@ -1224,7 +1234,7 @@ def record_convergence_debt(self, **_kwargs: object) -> None: monkeypatch.setattr("polylogue.daemon.convergence_stages.repair_fts_surface", lambda *_args: True) monkeypatch.setattr("polylogue.sources.live.cursor.CursorStore", FakeCursor) - daemon_cli._close_raw_materialization_fts(index_db) + daemon_cli._close_raw_materialization_fts(index_db, ops_db_path=ops_db) assert cleared == [{"subject_type": "fts_surface", "subject_id": "messages_fts", "stage": "fts"}] @@ -1235,13 +1245,16 @@ def test_raw_materialization_fts_exception_becomes_explicit_debt( ) -> None: from polylogue.daemon import cli as daemon_cli - index_db = tmp_path / "index.db" + index_db = tmp_path / "generations" / "active" / "index.db" + ops_db = tmp_path / "ops.db" + index_db.parent.mkdir(parents=True) index_db.touch() errors: list[str | None] = [] class FakeCursor: - def __init__(self, db: Path) -> None: + def __init__(self, db: Path, *, ops_db_path: Path) -> None: assert db == index_db + assert ops_db_path == ops_db def record_convergence_debt(self, *, error: str | None = None, **_kwargs: object) -> None: errors.append(error) @@ -1253,7 +1266,7 @@ def record_convergence_debt(self, *, error: str | None = None, **_kwargs: object ) monkeypatch.setattr("polylogue.sources.live.cursor.CursorStore", FakeCursor) - daemon_cli._close_raw_materialization_fts(index_db) + daemon_cli._close_raw_materialization_fts(index_db, ops_db_path=ops_db) assert errors == ["FTS repair failed after raw materialization: RuntimeError: injected FTS failure"] diff --git a/tests/unit/daemon/test_provenance_endpoint.py b/tests/unit/daemon/test_provenance_endpoint.py index 8ed269cd2d..db59a92870 100644 --- a/tests/unit/daemon/test_provenance_endpoint.py +++ b/tests/unit/daemon/test_provenance_endpoint.py @@ -403,7 +403,7 @@ def test_historical_validation_failure_is_not_current_quarantine(self, workspace raw_id=raw_id, source_path="/tmp/x.json", blob_size=2, - validated_at_ms=1_767_225_602_000, + validated_at_ms=1_767_225_601_000, validation_status="failed", ) @@ -415,6 +415,24 @@ def test_historical_validation_failure_is_not_current_quarantine(self, workspace assert result["quarantined"] is False assert result["quarantine_reason"] is None + def test_equal_raw_transition_timestamps_surface_order_ambiguity(self, workspace_env: dict[str, Path]) -> None: + raw_id = _seed_raw_blob(b"{}") + session_id = _seed_archive_provenance( + session_id="c-ambiguous-validation", + raw_id=raw_id, + source_path="/tmp/x.json", + blob_size=2, + parsed_at_ms=1_767_225_602_000, + validated_at_ms=1_767_225_602_000, + validation_status="failed", + ) + + result = build_provenance_payload(session_id) + + assert result is not None + assert result["quarantined"] is True + assert result["quarantine_reason"] == "validation_parse_order_ambiguous" + def test_quarantine_surfaces_when_no_raw_artifact(self, workspace_env: dict[str, Path]) -> None: session_id = _seed_archive_provenance( session_id="c-orphan", diff --git a/tests/unit/sources/test_live_batch_support.py b/tests/unit/sources/test_live_batch_support.py index 173c994fc8..018b6930e0 100644 --- a/tests/unit/sources/test_live_batch_support.py +++ b/tests/unit/sources/test_live_batch_support.py @@ -5,6 +5,7 @@ import json import os import sqlite3 +import zipfile from dataclasses import replace from hashlib import sha256 from pathlib import Path @@ -451,6 +452,88 @@ def test_full_ingest_acquires_but_does_not_parse_when_derived_tier_degraded( assert artifact_rows == (0,) +def test_source_only_full_ingest_streams_admitted_zip_members_without_decoding( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The production full-ingest ZIP route must retain bytes before decode.""" + from polylogue.core.degraded import DegradedReason, clear_degraded, set_degraded + + root = tmp_path / "sessions" + root.mkdir() + bundle = root / "degraded.zip" + member_names = ("sessions/one.jsonl", "sessions/two.json") + with zipfile.ZipFile(bundle, "w") as zf: + zf.writestr(member_names[0], b'{"opaque":"first"}\n') + zf.writestr(member_names[1], b'{"opaque":"second"}') + index_db = tmp_path / "index.db" + processor = LiveBatchProcessor( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=index_db))), + (WatchSource(name="claude-code", root=root),), + cursor=CursorStore(index_db), + parser_fingerprint="test-parser", + ) + set_degraded(DegradedReason(code="schema_version_mismatch", message="index unavailable", derived_only=True)) + for target in ( + "polylogue.sources.live.batch.iter_zip_entry_raw_data", + "polylogue.sources.live.batch.LiveBatchProcessor._sniff_zip_provider", + "polylogue.sources.live.batch._detect_provider_from_raw_bytes", + "polylogue.sources.source_acquisition_components.iter_entry_payloads", + "polylogue.sources.source_acquisition_components.classify_artifact", + ): + monkeypatch.setattr( + target, lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("must not decode ZIP")) + ) + try: + result = processor._ingest_full_paths_sync([bundle], source_name="claude-code") + finally: + clear_degraded() + + assert result.succeeded == [bundle] + assert result.failed == [] + with sqlite3.connect(tmp_path / "source.db") as conn: + rows = conn.execute( + "SELECT source_path, source_index, parsed_at_ms, parse_error FROM raw_sessions ORDER BY source_index" + ).fetchall() + assert rows == [ + (f"{bundle}:{member_names[0]}", 0, None, None), + (f"{bundle}:{member_names[1]}", 1, None, None), + ] + + +def test_source_only_full_ingest_snapshots_unrecognized_codex_state_without_shape_probe( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A degraded source tier retains a valid but future-shaped Codex state DB.""" + from polylogue.core.degraded import DegradedReason, clear_degraded, set_degraded + + root = tmp_path / "codex" + root.mkdir() + state_db = root / "state_5.sqlite" + _write_plain_sqlite_db(state_db) + index_db = tmp_path / "index.db" + processor = LiveBatchProcessor( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=index_db))), + (WatchSource(name="codex", root=root),), + cursor=CursorStore(index_db), + parser_fingerprint="test-parser", + ) + set_degraded(DegradedReason(code="schema_version_mismatch", message="index unavailable", derived_only=True)) + monkeypatch.setattr( + "polylogue.sources.parsers.codex_state.is_in_scope_codex_sqlite_path", + lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("must not inspect source-only state schema")), + ) + try: + result = processor._ingest_full_paths_sync([state_db], source_name="codex") + finally: + clear_degraded() + + assert result.succeeded == [state_db] + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT source_path, parsed_at_ms FROM raw_sessions").fetchall() == [(str(state_db), None)] + + def test_full_ingest_acquires_when_index_is_genuinely_semantic_distance_stale( tmp_path: Path, ) -> None: diff --git a/tests/unit/sources/test_revision_backfill.py b/tests/unit/sources/test_revision_backfill.py index 7f90f41870..3d3a60ac8e 100644 --- a/tests/unit/sources/test_revision_backfill.py +++ b/tests/unit/sources/test_revision_backfill.py @@ -190,6 +190,23 @@ def test_parse_one_refuses_declared_fact_artifacts(tmp_path: Path, source_path_s assert sessions == [] +def test_parse_one_recovery_accepts_session_evidence_at_a_declared_fact_path(tmp_path: Path) -> None: + """Source-only raw recovery decodes evidence before assigning fact taxonomy.""" + source_path = tmp_path / ".claude" / "projects" / "proj" / "subagents" / "workflows" / "wf" / "journal.jsonl" + payload = ( + b'{"parentUuid":null,"type":"user","sessionId":"wf","message":{"role":"user","content":"recover me"},' + b'"uuid":"user-1","timestamp":"2025-01-01T00:00:00Z"}\n' + b'{"parentUuid":"user-1","type":"assistant","sessionId":"wf","message":{"role":"assistant",' + b'"content":[{"type":"text","text":"recovered"}]},"uuid":"assistant-1",' + b'"timestamp":"2025-01-01T00:00:01Z"}\n' + ) + + sessions = _parse_one(Provider.CLAUDE_CODE, payload, str(source_path)) + + assert len(sessions) == 1 + assert [message.text for message in sessions[0].messages] == ["recover me", "recovered"] + + def _relationship_index_jsonl_bytes(count: int = 8) -> bytes: """Bytes shaped like the real sinex analysis artifact from polylogue-9ykn (gvgi): a graph-edge index sitting under a watched Claude Code directory, diff --git a/tests/unit/storage/test_parse_tracking.py b/tests/unit/storage/test_parse_tracking.py index 35a5251de4..0479d37445 100644 --- a/tests/unit/storage/test_parse_tracking.py +++ b/tests/unit/storage/test_parse_tracking.py @@ -194,6 +194,48 @@ async def test_update_raw_state_truncates_error_fields(self, backend: SQLiteBack assert rec.validation_error is not None assert len(rec.validation_error) == 2000 + async def test_failed_validation_after_parse_advances_past_identical_or_backward_clock( + self, backend: SQLiteBackend, monkeypatch: pytest.MonkeyPatch + ) -> None: + """A failed revalidation cannot tie or precede the parse it supersedes.""" + from polylogue.storage.sqlite.queries import raw_state as raw_state_queries + + await self._save_raw(backend, raw_id="parse-then-failed-validation") + await backend.update_raw_state( + "parse-then-failed-validation", + state=RawSessionStateUpdate(parsed_at="1970-01-01T00:00:01Z"), + ) + monkeypatch.setattr(raw_state_queries, "_now_ms", lambda: 999) + await backend.mark_raw_validated("parse-then-failed-validation", status="failed", error="rejected") + + with sqlite3.connect(backend._source_db_path) as conn: + row = conn.execute( + "SELECT parsed_at_ms, validated_at_ms, validation_status FROM raw_sessions WHERE raw_id = ?", + ("parse-then-failed-validation",), + ).fetchone() + assert row == (1000, 1001, "failed") + + async def test_successful_parse_after_validation_advances_past_identical_or_backward_clock( + self, backend: SQLiteBackend, monkeypatch: pytest.MonkeyPatch + ) -> None: + """A later parse wins even if its injected wall clock is older.""" + from polylogue.storage.sqlite.queries import raw_state as raw_state_queries + + await self._save_raw(backend, raw_id="validation-then-parse") + monkeypatch.setattr(raw_state_queries, "_now_ms", lambda: 1000) + await backend.mark_raw_validated("validation-then-parse", status="failed", error="rejected") + await backend.update_raw_state( + "validation-then-parse", + state=RawSessionStateUpdate(parsed_at="1970-01-01T00:00:00.999Z", parse_error=None), + ) + + with sqlite3.connect(backend._source_db_path) as conn: + row = conn.execute( + "SELECT parsed_at_ms, validated_at_ms, validation_status FROM raw_sessions WHERE raw_id = ?", + ("validation-then-parse",), + ).fetchone() + assert row == (1001, 1000, "failed") + class TestMarkRawValidated: """Tests for mark_raw_validated backend method.""" diff --git a/tests/unit/storage/test_repair.py b/tests/unit/storage/test_repair.py index a59b4bf66d..3fec57866c 100644 --- a/tests/unit/storage/test_repair.py +++ b/tests/unit/storage/test_repair.py @@ -958,8 +958,11 @@ def test_raw_materialization_replays_successful_raw_with_historical_validation_f assert conn.execute("SELECT COUNT(*) FROM sessions WHERE raw_id = ?", (raw_id,)).fetchone() == (1,) -def test_raw_materialization_refuses_validation_failure_newer_than_parse(tmp_path: Path) -> None: - """A later validation failure remains current authority after an earlier parse.""" +@pytest.mark.parametrize("validation_offset", [0, 1]) +def test_raw_materialization_refuses_non_parse_authoritative_validation_failure( + tmp_path: Path, validation_offset: int +) -> None: + """A newer failure or legacy tie must not replay and overwrite raw authority.""" from polylogue.core.enums import Provider from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root @@ -989,7 +992,7 @@ def test_raw_materialization_refuses_validation_failure_newer_than_parse(tmp_pat SET validation_status = 'failed', validation_error = ?, validated_at_ms = ? WHERE raw_id = ? """, - ("strict validation rejected the later observation", parsed_at_ms + 1, raw_id), + ("strict validation rejected the later observation", parsed_at_ms + validation_offset, raw_id), ) conn.commit() From c1cd22b8896dccd01a429c7b83deeb9347e2b2cd Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 11:09:17 +0200 Subject: [PATCH 32/65] fix(ingest): replay retained source authority --- polylogue/schemas/validation/corpus.py | 25 ++- polylogue/sources/codex_state_evidence.py | 81 +++++++ polylogue/sources/live/batch.py | 128 ++++------- polylogue/sources/revision_backfill.py | 144 +++++++++++-- .../sources/source_acquisition_components.py | 20 ++ tests/unit/core/test_schema_validation.py | 31 +++ tests/unit/sources/test_live_batch_support.py | 198 ++++++++++++++++++ tests/unit/sources/test_revision_backfill.py | 17 ++ 8 files changed, 520 insertions(+), 124 deletions(-) create mode 100644 polylogue/sources/codex_state_evidence.py diff --git a/polylogue/schemas/validation/corpus.py b/polylogue/schemas/validation/corpus.py index 50f0137bd2..41e74d6e7d 100644 --- a/polylogue/schemas/validation/corpus.py +++ b/polylogue/schemas/validation/corpus.py @@ -10,11 +10,13 @@ from polylogue.archive.raw_payload import RawPayloadEnvelope, build_raw_payload_envelope from polylogue.core.common import format_malformed_jsonl_error as _format_malformed_jsonl_error -from polylogue.core.enums import Origin, Provider +from polylogue.core.enums import Origin, Provider, ValidationMode, ValidationStatus from polylogue.core.sources import origin_from_provider, provider_from_origin from polylogue.schemas.validator import SchemaValidator from polylogue.storage.blob_store import get_blob_store +from polylogue.storage.raw.models import RawSessionStateUpdate from polylogue.storage.sqlite.connection_profile import open_connection +from polylogue.storage.sqlite.raw_state_update import compile_raw_state_update from .models import ProviderSchemaVerification, SchemaVerificationReport from .requests import SchemaVerificationRequest, bounded_window @@ -201,17 +203,18 @@ def apply_quarantine_updates( """ validated_at_ms = int(datetime.now(tz=timezone.utc).timestamp() * 1000) for raw_id, reason, _provider, _payload_provider in updates: + set_clauses, params = compile_raw_state_update( + RawSessionStateUpdate( + validation_status=ValidationStatus.FAILED, + validation_error=reason, + validation_drift_count=0, + validation_mode=ValidationMode.STRICT, + ), + now_ms=validated_at_ms, + ) conn.execute( - """ - UPDATE raw_sessions - SET validation_status = 'failed', - validation_error = ?, - validation_drift_count = 0, - validation_mode = 'strict', - validated_at_ms = ? - WHERE raw_id = ? - """, - (reason, validated_at_ms, raw_id), + f"UPDATE raw_sessions SET {', '.join(set_clauses)} WHERE raw_id = ?", + (*params, raw_id), ) conn.execute( """ diff --git a/polylogue/sources/codex_state_evidence.py b/polylogue/sources/codex_state_evidence.py new file mode 100644 index 0000000000..fc78152527 --- /dev/null +++ b/polylogue/sources/codex_state_evidence.py @@ -0,0 +1,81 @@ +"""Durable session-linked evidence derived from retained Codex state.""" + +from __future__ import annotations + +from json import dumps as json_dumps +from typing import Any + +from polylogue.core.enums import Origin, Provider +from polylogue.sources.parsers import codex_state + + +def write_codex_thread_state_evidence( + archive: Any, + snapshot: codex_state.CodexStateSnapshot, + *, + source_path: str, + acquired_at_ms: int, +) -> None: + """Attach state-db thread metadata to existing Codex sessions. + + The state database is evidence about sessions, never a session itself. + Both live ingest and retained-raw replay call this writer so a + source-only acquisition is completed from the immutable snapshot when + the derived tier returns. + """ + from polylogue.storage.sqlite.archive_tiers.source_write import ArchiveHookEvent + + for thread in snapshot.threads: + payload: dict[str, object] = { + "thread_id": thread.thread_id, + "title": thread.title, + "cwd": thread.cwd, + "source": thread.source, + "model": thread.model, + "agent_nickname": thread.agent_nickname, + "agent_role": thread.agent_role, + "archived": thread.archived, + } + encoded = json_dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") + archive.write_hook_event( + provider=Provider.CODEX, + payload=encoded, + source_path=source_path, + acquired_at_ms=acquired_at_ms, + hook_event=ArchiveHookEvent( + hook_event_id=f"codex-thread-title:{thread.thread_id}", + origin=Origin.CODEX_SESSION, + source_path=source_path, + event_type="codex_thread_title", + payload=payload, + observed_at_ms=thread.updated_at_ms or acquired_at_ms, + native_id=f"{thread.thread_id}:codex_thread_title", + session_native_id=thread.thread_id, + ), + ) + for edge in snapshot.spawn_edges: + payload = { + "parent_thread_id": edge.parent_thread_id, + "child_thread_id": edge.child_thread_id, + "status": edge.status, + } + encoded = json_dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") + archive.write_hook_event( + provider=Provider.CODEX, + payload=encoded, + source_path=source_path, + acquired_at_ms=acquired_at_ms, + hook_event=ArchiveHookEvent( + hook_event_id=f"codex-thread-spawn-edge:{edge.parent_thread_id}:{edge.child_thread_id}", + origin=Origin.CODEX_SESSION, + source_path=source_path, + event_type="codex_thread_spawn_edge", + payload=payload, + observed_at_ms=acquired_at_ms, + native_id=f"{edge.parent_thread_id}:{edge.child_thread_id}:codex_thread_spawn_edge", + session_native_id=edge.parent_thread_id, + ), + ) + + +__all__ = ["write_codex_thread_state_evidence"] diff --git a/polylogue/sources/live/batch.py b/polylogue/sources/live/batch.py index b3ca89ee51..eb6056d836 100644 --- a/polylogue/sources/live/batch.py +++ b/polylogue/sources/live/batch.py @@ -66,6 +66,7 @@ success_disposition, ) from polylogue.pipeline.services.ingest_batch._models import _IngestBatchSummary +from polylogue.sources.codex_state_evidence import write_codex_thread_state_evidence from polylogue.sources.decoder_json import PartialJsonStreamError from polylogue.sources.decoder_zip import ZipBombError, open_bounded_zip_entry from polylogue.sources.decoders import JsonlDecodeError, _iter_json_stream, _ZipEntryValidator @@ -147,6 +148,7 @@ ZipEntryReadContext, iter_zip_entry_raw_data, stream_preserved_zip_entry_raw_data, + zip_member_raw_id, ) from polylogue.sources.source_parsing import has_decoded_session_evidence from polylogue.sources.sqlite_snapshot import ( @@ -276,82 +278,6 @@ def _hot_capture_prefix_is_proven( return fingerprint == expected_fingerprint and _file_observation(proof_start) == _file_observation(proof_end) -def _write_codex_thread_state_evidence( - archive: Any, - snapshot: codex_state.CodexStateSnapshot, - *, - source_path: str, - acquired_at_ms: int, -) -> None: - """Attach ``threads``/``thread_spawn_edges`` evidence to EXISTING sessions. - - polylogue-0jf4 acceptance criterion 3: threads.title and - thread_spawn_edges must reach the archive as typed evidence without ever - minting a session or session of their own -- the same hook-event - incident precedent as polylogue-31r1 (standalone hook-event ingestion - once inflated the archive from 18,391 to 83,286 sessions). Reuses - ``ArchiveStore.write_hook_event``/``raw_hook_events`` exactly as - ``sources/hooks.py`` does: a durable, session-scoped evidence row keyed - by ``session_native_id`` (here the Codex ``thread_id``), joined at read - time (``ArchiveStore.hook_event_summary_for_session``) rather than - materialized into ``index.db`` via a full session replace. No schema - change -- ``raw_hook_events.event_type`` is unconstrained TEXT. - """ - from polylogue.storage.sqlite.archive_tiers.source_write import ArchiveHookEvent - - for thread in snapshot.threads: - payload: dict[str, object] = { - "thread_id": thread.thread_id, - "title": thread.title, - "cwd": thread.cwd, - "source": thread.source, - "model": thread.model, - "agent_nickname": thread.agent_nickname, - "agent_role": thread.agent_role, - "archived": thread.archived, - } - encoded = json_dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") - archive.write_hook_event( - provider=Provider.CODEX, - payload=encoded, - source_path=source_path, - acquired_at_ms=acquired_at_ms, - hook_event=ArchiveHookEvent( - hook_event_id=f"codex-thread-title:{thread.thread_id}", - origin=Origin.CODEX_SESSION, - source_path=source_path, - event_type="codex_thread_title", - payload=payload, - observed_at_ms=thread.updated_at_ms or acquired_at_ms, - native_id=f"{thread.thread_id}:codex_thread_title", - session_native_id=thread.thread_id, - ), - ) - for edge in snapshot.spawn_edges: - edge_payload: dict[str, object] = { - "parent_thread_id": edge.parent_thread_id, - "child_thread_id": edge.child_thread_id, - "status": edge.status, - } - encoded = json_dumps(edge_payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") - archive.write_hook_event( - provider=Provider.CODEX, - payload=encoded, - source_path=source_path, - acquired_at_ms=acquired_at_ms, - hook_event=ArchiveHookEvent( - hook_event_id=f"codex-thread-spawn-edge:{edge.parent_thread_id}:{edge.child_thread_id}", - origin=Origin.CODEX_SESSION, - source_path=source_path, - event_type="codex_thread_spawn_edge", - payload=edge_payload, - observed_at_ms=acquired_at_ms, - native_id=f"{edge.parent_thread_id}:{edge.child_thread_id}:codex_thread_spawn_edge", - session_native_id=edge.parent_thread_id, - ), - ) - - def _is_json_stream_decode_error(error: BaseException) -> bool: return isinstance(error, (StdlibJSONDecodeError, UnicodeDecodeError, PartialJsonStreamError, JsonlDecodeError)) @@ -1935,9 +1861,16 @@ def _ingest_full_paths_sync( ingested.append(path) raw_byte_sizes[path] = stat.st_size continue - if hermes_state.looks_like_state_db_path( - path - ) or hermes_verification.looks_like_verification_evidence_db_path(path): + hermes_owned_sqlite_name = ( + source_only + and fallback_provider is Provider.HERMES + and path.name in {"state.db", "verification_evidence.db"} + ) + if ( + hermes_owned_sqlite_name + or hermes_state.looks_like_state_db_path(path) + or hermes_verification.looks_like_verification_evidence_db_path(path) + ): provider = Provider.HERMES source_name = provider.value try: @@ -2585,13 +2518,14 @@ def _ingest_full_records_archive( blob_size=record.blob_size, source_path=record.source_path, source_index=record.source_index or 0, - # A populated ``blob_hash`` field marks a - # sqlite-snapshot acquisition (Hermes or, per - # polylogue-0jf4, Codex state dbs), whose raw_id - # is a deterministic profile/path-scoped id - # distinct from the blob's own content hash -- - # every other provider's raw_id already IS the - # content hash, so passing it again is a no-op. + # A populated ``blob_hash`` field means this + # record already has a durable blob reference. + # SQLite snapshots and ZIP members both keep a + # raw id distinct from that blob address: the + # former is profile/path scoped, the latter is + # coordinate scoped. Preserve that identity at + # source admission rather than collapsing either + # kind back onto a shared content hash. raw_id=(record.raw_id if record.blob_hash is not None else None), acquired_at_ms=acquired_at_ms, blob_publication_receipt_id=record.blob_publication_receipt_id, @@ -2723,7 +2657,7 @@ def _ingest_full_records_archive( state_kind = codex_state.classify_codex_sqlite_path(state_path, immutable=True) if state_kind == "thread_state": state_snapshot = codex_state.parse_codex_state_db(state_path, immutable=True) - _write_codex_thread_state_evidence( + write_codex_thread_state_evidence( archive, state_snapshot, source_path=record.source_path, @@ -3329,11 +3263,17 @@ def _extract_zip_member_records( member_provider = raw_data.provider_hint or fallback_provider member_size = raw_data.blob_size or 0 total_bytes += member_size + member_raw_id = zip_member_raw_id( + raw_data.source_path, + raw_data.source_index or 0, + raw_data.blob_hash, + ) records.append( ( - raw_data.blob_hash, + member_raw_id, RawSessionRecord( - raw_id=raw_data.blob_hash, + raw_id=member_raw_id, + blob_hash=raw_data.blob_hash, payload_provider=member_provider, capture_mode=( fallback_provider @@ -3402,11 +3342,17 @@ def _extract_source_only_zip_member_records( if raw_data.blob_hash is None: continue total_bytes += raw_data.blob_size or 0 + member_raw_id = zip_member_raw_id( + raw_data.source_path, + source_index, + raw_data.blob_hash, + ) records.append( ( - raw_data.blob_hash, + member_raw_id, RawSessionRecord( - raw_id=raw_data.blob_hash, + raw_id=member_raw_id, + blob_hash=raw_data.blob_hash, payload_provider=fallback_provider, capture_mode=fallback_provider, source_name=fallback_provider.value, diff --git a/polylogue/sources/revision_backfill.py b/polylogue/sources/revision_backfill.py index 6facfd5812..e88937c3a2 100644 --- a/polylogue/sources/revision_backfill.py +++ b/polylogue/sources/revision_backfill.py @@ -44,8 +44,10 @@ parallel_threads_effective, resolve_revision_backfill_census_dispatch, ) +from polylogue.sources.codex_state_evidence import write_codex_thread_state_evidence from polylogue.sources.decoders import _iter_json_stream from polylogue.sources.dispatch import ( + detect_provider_from_raw_bytes_evidence, is_jsonl_source_path, is_stream_record_provider, parse_payload, @@ -53,9 +55,10 @@ require_positive_conversational_evidence, ) from polylogue.sources.origin_specs import artifact_rule_for_path -from polylogue.sources.parsers import antigravity, hermes_state, hermes_verification +from polylogue.sources.parsers import antigravity, codex_state, hermes_state, hermes_verification from polylogue.sources.parsers.base import ParsedSession from polylogue.sources.sqlite_snapshot import looks_like_sqlite_bytes +from polylogue.storage.raw.models import RawSessionStateUpdate from polylogue.storage.raw_authority import ( RAW_AUTHORITY_PARSER_FINGERPRINT, SUPERSEDED_MEMBERSHIP_FINGERPRINTS, @@ -66,6 +69,7 @@ FrozenSourceRemediationRequiredError, record_current_parser_source_census, ) +from polylogue.storage.sqlite.archive_tiers.source_write import apply_source_raw_state_update from polylogue.storage.sqlite.archive_tiers.write import PreparedSessionRows, prepare_session_rows _LOGGER = _polylogue_logging.get_logger(__name__) @@ -656,6 +660,18 @@ def apply_outcome( commit_unit() return sessions, payload_bytes, revision_kind = outcome + stored_provider, _blob_hash, _source_path, _stored_kind, _stored_size = archive.raw_revision_descriptor(raw_id) + if stored_provider is Provider.UNKNOWN and sessions: + # Acquisition deliberately did not decode an UNKNOWN source-only + # member. A successful replay now has durable shape evidence for + # its provider, so retain that result independently of the later + # index promotion outcome. + apply_source_raw_state_update( + archive._ensure_source_conn(), + raw_id, + state=RawSessionStateUpdate(payload_provider=Provider.from_string(sessions[0].source_name)), + manage_transaction=not batched, + ) state.classified += int(len(sessions) == 1) spill.add(raw_id, sessions, payload_bytes=payload_bytes) if len(sessions) == 1 and revision_kind is RawRevisionKind.UNKNOWN: @@ -730,6 +746,12 @@ def bind_byte_proven_older_member(raw_id: str, logical_key: str) -> None: census_selection = initial_selection while True: rows = archive.raw_membership_census_rows(census_selection) + for raw_id, _source_index, _terminal_non_session in rows: + if raw_id in state.censused or not _replay_retained_codex_state_evidence(archive, raw_id): + continue + state.scanned += 1 + state.censused.add(raw_id) + commit_unit() terminal_raw_ids = { raw_id for raw_id, _source_index, terminal_non_session in rows if terminal_non_session } @@ -1922,9 +1944,27 @@ def census_parse_worker( fallback_id_override = native_id if kind is RawRevisionKind.APPEND else None publisher = ArchiveBlobPublisher(Path(source_db_path_str), Path(blob_root_str)) try: + if provider is Provider.UNKNOWN: + payload = publisher.read_all(blob_hash) + provider, _evidence = detect_provider_from_raw_bytes_evidence(payload, Path(source_path).name, provider) + payload_path = None + if provider is Provider.HERMES: + candidate_path = publisher.blob_path(blob_hash) + payload_path = candidate_path if candidate_path.exists() else None + sessions = _parse_one( + provider, + payload, + source_path, + payload_path=payload_path, + archive_root=Path(blob_root_str).parent, + fallback_id_override=fallback_id_override, + ) + return raw_id, sessions, None if is_stream: - with publisher.open(blob_hash) as payload: - sessions = _parse_stream(provider, payload, source_path, fallback_id_override=fallback_id_override) + with publisher.open(blob_hash) as stream_payload: + sessions = _parse_stream( + provider, stream_payload, source_path, fallback_id_override=fallback_id_override + ) else: payload_path = None if provider is Provider.HERMES: @@ -2121,6 +2161,8 @@ def _enrich_retained_parse_results( continue provider, _blob_hash, source_path, _descriptor_kind, _size, _native_id = descriptors[raw_id] sessions, payload_bytes, kind = outcome + if sessions: + provider = Provider.from_string(sessions[0].source_name) results[raw_id] = ( _replay_safe_enrich_sessions( source_conn, @@ -2326,6 +2368,26 @@ def parse_retained_raw_sessions(archive: ArchiveStore, raw_id: str) -> list[Pars # the unchanged stem-based fallback -- their stored bytes still carry # the synthetic session_meta line that made this unnecessary for them. fallback_id_override = archive.raw_native_id(raw_id) if kind is RawRevisionKind.APPEND else None + if provider is Provider.UNKNOWN: + # Source-only acquisition deliberately retains unknown ZIP members + # without decoding them. Recovery is the first lawful point to + # inspect the durable bytes and resolve their parser, before deciding + # whether their filename is a stream route. + _provider, eager_payload, _source_path, _eager_kind = archive.raw_revision_material(raw_id) + provider, _evidence = detect_provider_from_raw_bytes_evidence( + eager_payload, + Path(source_path).name, + provider, + ) + payload_path = archive.blob_path_for_hash(blob_hash) if provider is Provider.HERMES else None + return _parse_one( + provider, + eager_payload, + source_path, + payload_path=payload_path, + archive_root=archive.archive_root, + fallback_id_override=fallback_id_override, + ) if is_stream_record_provider(source_path, str(provider)): with archive.open_raw_revision_material(raw_id) as (stream_provider, payload, stream_path, _stream_kind): return _parse_stream(stream_provider, payload, stream_path, fallback_id_override=fallback_id_override) @@ -2341,6 +2403,42 @@ def parse_retained_raw_sessions(archive: ArchiveStore, raw_id: str) -> list[Pars ) +def _replay_retained_codex_state_evidence(archive: ArchiveStore, raw_id: str) -> bool: + """Apply a retained, in-scope Codex state snapshot without minting a session. + + Source-only acquisition snapshots named Codex databases before it can + inspect their schema. Once recovery owns the derived tier, only a + recognized retained snapshot may become thread evidence. The parser + reads the immutable blob path, never the original mutable state DB. + """ + provider, blob_hash, source_path, _kind, _payload_size = archive.raw_revision_descriptor(raw_id) + if provider is not Provider.CODEX: + return False + state_path = archive.blob_path_for_hash(blob_hash) + if state_path is None: + return False + state_kind = codex_state.classify_codex_sqlite_path(state_path, immutable=True) + if state_kind not in codex_state.IN_SCOPE_KINDS: + return False + if state_kind == "thread_state": + write_codex_thread_state_evidence( + archive, + codex_state.parse_codex_state_db(state_path, immutable=True), + source_path=source_path, + acquired_at_ms=archive.raw_revision_acquired_at_ms(raw_id), + ) + archive.replace_raw_membership_census( + raw_id, + [], + parser_fingerprint=RAW_AUTHORITY_PARSER_FINGERPRINT, + censused_at_ms=0, + detail="retained Codex state evidence applied", + retire_full_revision_governance=True, + ) + archive.mark_raw_parse_succeeded(raw_id, provider=Provider.CODEX) + return True + + #: Lever-A prefetch-buffer budget clamp (estimated tree bytes) -- same #: adaptive formula as ``_ParsedSessionSpill``'s hot decoded cache (physical #: RAM / 16 within these bounds), but deliberately its own pair of constants: @@ -3140,6 +3238,22 @@ def _parse_one_raw( return sessions source_name = Path(source_path).name fallback_id = fallback_id_override or Path(source_path).stem + if provider is Provider.HERMES and looks_like_sqlite_bytes(payload): + with _sqlite_payload_path(payload, payload_path, archive_root) as sqlite_path: + if hermes_state.looks_like_state_db_path(sqlite_path, immutable=True): + return hermes_state.parse_state_db( + sqlite_path, + fallback_id=fallback_id, + profile_root=Path(source_path).parent, + immutable=True, + ) + if hermes_verification.looks_like_verification_evidence_db_path(sqlite_path, immutable=True): + return hermes_verification.parse_verification_evidence_db( + sqlite_path, + fallback_id=fallback_id, + profile_root=Path(source_path).parent, + immutable=True, + ) rule = artifact_rule_for_path(provider, source_path) declared_path_session_evidence = False if rule is not None and rule.parse_policy != "session" and is_jsonl_source_path(source_path): @@ -3163,22 +3277,6 @@ def _parse_one_raw( provider, source_path, sample=records[:64] ): return [] - if provider is Provider.HERMES and looks_like_sqlite_bytes(payload): - with _sqlite_payload_path(payload, payload_path, archive_root) as sqlite_path: - if hermes_state.looks_like_state_db_path(sqlite_path, immutable=True): - return hermes_state.parse_state_db( - sqlite_path, - fallback_id=fallback_id, - profile_root=Path(source_path).parent, - immutable=True, - ) - if hermes_verification.looks_like_verification_evidence_db_path(sqlite_path, immutable=True): - return hermes_verification.parse_verification_evidence_db( - sqlite_path, - fallback_id=fallback_id, - profile_root=Path(source_path).parent, - immutable=True, - ) return parse_payload( provider, records, @@ -3236,8 +3334,6 @@ def _parse_stream_raw( *, fallback_id_override: str | None = None, ) -> list[ParsedSession]: - if _is_declared_non_session_artifact(provider, source_path): - return [] source_name = Path(source_path).name fallback_id = fallback_id_override or Path(source_path).stem stream = _iter_json_stream(payload, source_name) @@ -3247,7 +3343,11 @@ def _parse_stream_raw( # ingest gate uses -- are materialized for content classification; the # rest of the stream is chained back on unread. sample = list(islice(stream, 64)) - if _is_declared_non_session_artifact(provider, source_path, sample=sample): + sample_sessions = parse_stream_payload(provider, sample, fallback_id, source_path=source_path) + sample_has_session_evidence = bool( + require_positive_conversational_evidence(sample_sessions, provider=provider, source_path=source_path) + ) + if not sample_has_session_evidence and _is_declared_non_session_artifact(provider, source_path, sample=sample): return [] return parse_stream_payload( provider, diff --git a/polylogue/sources/source_acquisition_components.py b/polylogue/sources/source_acquisition_components.py index beeab04f8b..b65e464a18 100644 --- a/polylogue/sources/source_acquisition_components.py +++ b/polylogue/sources/source_acquisition_components.py @@ -6,6 +6,7 @@ import zipfile from collections.abc import Callable, Iterable from dataclasses import dataclass, field +from hashlib import sha256 from pathlib import Path from typing import IO, TypeAlias @@ -28,6 +29,7 @@ _DETECTION_PREFIX_SIZE = 8192 # 8 KB — enough for provider detection _HEARTBEAT_INTERVAL_S = 5.0 +_ZIP_MEMBER_RAW_ID_DOMAIN = b"polylogue:zip-member-raw:v1\0" AcquisitionObservation: TypeAlias = JSONDocument ObservationCallback: TypeAlias = Callable[[AcquisitionObservation], None] @@ -35,6 +37,24 @@ CursorState: TypeAlias = CursorStatePayload +def zip_member_raw_id(source_path: str, source_index: int, blob_hash: str) -> str: + """Identify one ZIP coordinate without giving up blob-level deduplication. + + ZIP exports legitimately contain duplicate member bytes. The blob hash + remains their shared immutable storage address, while raw authority must + retain each admitted ``:`` coordinate independently. + ``source_index`` additionally distinguishes duplicate member names. + """ + digest = sha256() + digest.update(_ZIP_MEMBER_RAW_ID_DOMAIN) + digest.update(source_path.encode("utf-8", errors="surrogatepass")) + digest.update(b"\0") + digest.update(str(source_index).encode("utf-8")) + digest.update(b"\0") + digest.update(bytes.fromhex(blob_hash)) + return digest.hexdigest() + + @dataclass(frozen=True, slots=True) class SourceReadContext: """Common acquisition dependencies for one local source artifact.""" diff --git a/tests/unit/core/test_schema_validation.py b/tests/unit/core/test_schema_validation.py index d678c6aa2f..275ba1aa62 100644 --- a/tests/unit/core/test_schema_validation.py +++ b/tests/unit/core/test_schema_validation.py @@ -870,6 +870,37 @@ def test_verify_raw_corpus_quarantine_malformed_updates_validation_state(db_path assert isinstance(row["parse_error"], str) and "Malformed JSONL lines" in row["parse_error"] +def test_verify_raw_corpus_quarantine_advances_past_an_existing_parse_transition(db_path: Path) -> None: + """A wall clock behind the parse transition must not reverse raw-state authority.""" + raw_id = _insert_raw_record( + db_path=db_path, + raw_id="raw-codex-quarantine-order", + source_name="codex", + source_path="/tmp/quarantine-order.jsonl", + raw_content=( + b'{"type":"session_meta"}\nnot json at all\n{"type":"response_item","payload":{"type":"message"}}' + ), + ) + with sqlite3.connect(db_path.parent / "source.db") as conn: + conn.execute("UPDATE raw_sessions SET parsed_at_ms = 9999999999999 WHERE raw_id = ?", (raw_id,)) + conn.commit() + + from polylogue.schemas.validation.corpus import apply_quarantine_updates + + with sqlite3.connect(db_path.parent / "source.db") as conn: + apply_quarantine_updates( + conn, + updates=[(raw_id, "malformed retained JSONL", "codex", "codex")], + ) + + with sqlite3.connect(db_path.parent / "source.db") as conn: + parsed_at_ms, validated_at_ms, validation_status = conn.execute( + "SELECT parsed_at_ms, validated_at_ms, validation_status FROM raw_sessions WHERE raw_id = ?", (raw_id,) + ).fetchone() + assert validation_status == "failed" + assert validated_at_ms > parsed_at_ms + + def test_verify_raw_corpus_quarantine_empty_payload_updates_validation_state(db_path: Path) -> None: raw_id = _insert_raw_record( db_path=db_path, diff --git a/tests/unit/sources/test_live_batch_support.py b/tests/unit/sources/test_live_batch_support.py index 018b6930e0..d162dc8b01 100644 --- a/tests/unit/sources/test_live_batch_support.py +++ b/tests/unit/sources/test_live_batch_support.py @@ -53,6 +53,7 @@ ) from polylogue.sources.live.cursor import CursorStore from polylogue.sources.parsers.base import ParsedMessage, ParsedSession +from polylogue.sources.revision_backfill import backfill_historical_revision_evidence from polylogue.sources.source_parsing import has_decoded_session_evidence from polylogue.storage.blob_store import BlobStore from polylogue.storage.raw_authority import RAW_AUTHORITY_PARSER_FINGERPRINT @@ -501,6 +502,91 @@ def test_source_only_full_ingest_streams_admitted_zip_members_without_decoding( ] +def test_source_only_zip_replay_resolves_unknown_chatgpt_member_and_keeps_duplicate_coordinates( + tmp_path: Path, +) -> None: + """Recovery, not acquisition, resolves UNKNOWN ZIP bytes and replays each coordinate.""" + from polylogue.core.degraded import DegradedReason, clear_degraded, set_degraded + + root = tmp_path / "inbox" + root.mkdir() + bundle = root / "export.zip" + payload = json.dumps( + [ + { + "id": "zip-chatgpt", + "conversation_id": "zip-chatgpt", + "title": "ZIP recovery", + "create_time": 1_700_000_000, + "update_time": 1_700_000_001, + "current_node": "assistant-node", + "mapping": { + "user-node": { + "id": "user-node", + "parent": None, + "children": ["assistant-node"], + "message": { + "id": "user-message", + "author": {"role": "user"}, + "content": {"content_type": "text", "parts": ["recover ZIP"]}, + "create_time": 1_700_000_000, + }, + }, + "assistant-node": { + "id": "assistant-node", + "parent": "user-node", + "children": [], + "message": { + "id": "assistant-message", + "author": {"role": "assistant"}, + "content": {"content_type": "text", "parts": ["replayed"]}, + "create_time": 1_700_000_001, + }, + }, + }, + } + ], + sort_keys=True, + ).encode() + with zipfile.ZipFile(bundle, "w") as zf: + zf.writestr("first/conversations.json", payload) + zf.writestr("second/conversations.json", payload) + index_db = tmp_path / "index.db" + processor = LiveBatchProcessor( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=index_db))), + (WatchSource(name="unknown", root=root),), + cursor=CursorStore(index_db), + parser_fingerprint="test-parser", + ) + + set_degraded(DegradedReason(code="schema_version_mismatch", message="index unavailable", derived_only=True)) + try: + result = processor._ingest_full_paths_sync([bundle], source_name="unknown") + finally: + clear_degraded() + + assert result.succeeded == [bundle] + with sqlite3.connect(tmp_path / "source.db") as conn: + before_replay = conn.execute( + "SELECT raw_id, hex(blob_hash), source_path, source_index, origin FROM raw_sessions ORDER BY source_index" + ).fetchall() + assert len(before_replay) == 2 + assert len({row[0] for row in before_replay}) == 2 + assert len({row[1] for row in before_replay}) == 1 + assert [row[2:] for row in before_replay] == [ + (f"{bundle}:first/conversations.json", 0, "unknown-export"), + (f"{bundle}:second/conversations.json", 1, "unknown-export"), + ] + + replay = backfill_historical_revision_evidence(tmp_path) + + assert replay.replayed_logical_sources == 2 + with sqlite3.connect(tmp_path / "index.db") as conn: + assert conn.execute("SELECT native_id, message_count FROM sessions").fetchall() == [("zip-chatgpt", 2)] + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT COUNT(*) FROM raw_sessions WHERE origin = 'chatgpt-export'").fetchone() == (2,) + + def test_source_only_full_ingest_snapshots_unrecognized_codex_state_without_shape_probe( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -534,6 +620,118 @@ def test_source_only_full_ingest_snapshots_unrecognized_codex_state_without_shap assert conn.execute("SELECT source_path, parsed_at_ms FROM raw_sessions").fetchall() == [(str(state_db), None)] +def _write_codex_thread_state_db(path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with sqlite3.connect(path) as conn: + conn.executescript( + """ + CREATE TABLE threads ( + id TEXT PRIMARY KEY, + title TEXT NOT NULL, + cwd TEXT NOT NULL, + created_at_ms INTEGER NOT NULL, + updated_at_ms INTEGER NOT NULL, + source TEXT NOT NULL, + model TEXT, + agent_nickname TEXT, + agent_role TEXT, + archived INTEGER NOT NULL DEFAULT 0 + ); + CREATE TABLE thread_spawn_edges ( + parent_thread_id TEXT NOT NULL, + child_thread_id TEXT NOT NULL PRIMARY KEY, + status TEXT NOT NULL + ); + """ + ) + conn.execute( + "INSERT INTO threads VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ("codex-thread", "Recover retained state", "/work", 1, 2, "cli", "gpt-5", None, None, 0), + ) + conn.execute( + "INSERT INTO thread_spawn_edges VALUES (?, ?, ?)", + ("codex-thread", "codex-child", "closed"), + ) + + +def test_source_only_codex_state_recovery_replays_retained_thread_evidence(tmp_path: Path) -> None: + """Removing the replay effect leaves the durable state raw pending and title-less.""" + from polylogue.core.degraded import DegradedReason, clear_degraded, set_degraded + + root = tmp_path / "codex" + state_db = root / "state_5.sqlite" + _write_codex_thread_state_db(state_db) + index_db = tmp_path / "index.db" + processor = LiveBatchProcessor( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=index_db))), + (WatchSource(name="codex", root=root),), + cursor=CursorStore(index_db), + parser_fingerprint="test-parser", + ) + + set_degraded(DegradedReason(code="schema_version_mismatch", message="index unavailable", derived_only=True)) + try: + assert processor._ingest_full_paths_sync([state_db], source_name="codex").succeeded == [state_db] + finally: + clear_degraded() + + replay = backfill_historical_revision_evidence(tmp_path) + + assert replay.scanned == 1 + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT parsed_at_ms IS NOT NULL FROM raw_sessions").fetchone() == (1,) + assert conn.execute( + "SELECT hook_event_id, event_type FROM raw_hook_events ORDER BY hook_event_id" + ).fetchall() == [ + ("codex-thread-spawn-edge:codex-thread:codex-child", "codex_thread_spawn_edge"), + ("codex-thread-title:codex-thread", "codex_thread_title"), + ] + + +@pytest.mark.parametrize("state_name", ["state.db", "verification_evidence.db"]) +def test_source_only_hermes_named_sqlite_uses_consistent_backup_before_generic_capture( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + state_name: str, +) -> None: + """A direct file copy loses an uncheckpointed WAL row; the snapshot retains it.""" + from polylogue.core.degraded import DegradedReason, clear_degraded, set_degraded + + root = tmp_path / "hermes" + state_db = root / state_name + state_db.parent.mkdir(parents=True) + writer = sqlite3.connect(state_db) + writer.execute("PRAGMA journal_mode=WAL") + writer.execute("CREATE TABLE retained_wal_row (value TEXT NOT NULL)") + writer.commit() + writer.execute("INSERT INTO retained_wal_row VALUES ('must survive')") + writer.commit() + index_db = tmp_path / "index.db" + processor = LiveBatchProcessor( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=index_db))), + (WatchSource(name="hermes", root=root),), + cursor=CursorStore(index_db), + parser_fingerprint="test-parser", + ) + monkeypatch.setattr("polylogue.sources.parsers.hermes_state.looks_like_state_db_path", lambda *_a, **_k: False) + monkeypatch.setattr( + "polylogue.sources.parsers.hermes_verification.looks_like_verification_evidence_db_path", + lambda *_a, **_k: False, + ) + + set_degraded(DegradedReason(code="schema_version_mismatch", message="index unavailable", derived_only=True)) + try: + assert processor._ingest_full_paths_sync([state_db], source_name="hermes").succeeded == [state_db] + finally: + clear_degraded() + writer.close() + + with sqlite3.connect(tmp_path / "source.db") as conn: + blob_hash = str(conn.execute("SELECT hex(blob_hash) FROM raw_sessions").fetchone()[0]).lower() + with sqlite3.connect(BlobStore(tmp_path / "blob").blob_path(blob_hash)) as snapshot: + assert snapshot.execute("SELECT value FROM retained_wal_row").fetchall() == [("must survive",)] + + def test_full_ingest_acquires_when_index_is_genuinely_semantic_distance_stale( tmp_path: Path, ) -> None: diff --git a/tests/unit/sources/test_revision_backfill.py b/tests/unit/sources/test_revision_backfill.py index 3d3a60ac8e..c06214f0e9 100644 --- a/tests/unit/sources/test_revision_backfill.py +++ b/tests/unit/sources/test_revision_backfill.py @@ -207,6 +207,23 @@ def test_parse_one_recovery_accepts_session_evidence_at_a_declared_fact_path(tmp assert [message.text for message in sessions[0].messages] == ["recover me", "recovered"] +def test_parse_stream_recovery_accepts_session_evidence_at_a_declared_fact_path(tmp_path: Path) -> None: + """The streamed replay route must inspect fact-path records before refusing them.""" + source_path = tmp_path / ".claude" / "projects" / "proj" / "subagents" / "workflows" / "wf" / "journal.jsonl" + payload = BytesIO( + b'{"parentUuid":null,"type":"user","sessionId":"wf","message":{"role":"user","content":"recover me"},' + b'"uuid":"user-1","timestamp":"2025-01-01T00:00:00Z"}\n' + b'{"parentUuid":"user-1","type":"assistant","sessionId":"wf","message":{"role":"assistant",' + b'"content":[{"type":"text","text":"recovered"}]},"uuid":"assistant-1",' + b'"timestamp":"2025-01-01T00:00:01Z"}\n' + ) + + sessions = revision_backfill._parse_stream(Provider.CLAUDE_CODE, payload, str(source_path)) + + assert len(sessions) == 1 + assert [message.text for message in sessions[0].messages] == ["recover me", "recovered"] + + def _relationship_index_jsonl_bytes(count: int = 8) -> bytes: """Bytes shaped like the real sinex analysis artifact from polylogue-9ykn (gvgi): a graph-edge index sitting under a watched Claude Code directory, From 21b25ecd2f10d523b4b9fb09e5e1197d062c8d31 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 11:23:16 +0200 Subject: [PATCH 33/65] fix(repair): preserve raw authority contracts --- polylogue/config.py | 1 + polylogue/core/raw_state.py | 4 + polylogue/storage/repair.py | 63 ++++++++------ polylogue/storage/sqlite/raw_state_update.py | 2 + tests/unit/core/test_config.py | 5 ++ tests/unit/core/test_sampling.py | 5 +- tests/unit/sources/test_live_watcher.py | 5 +- tests/unit/storage/test_parse_tracking.py | 43 ++++++++-- tests/unit/storage/test_repair.py | 87 +++++++++++++++++++- 9 files changed, 179 insertions(+), 36 deletions(-) diff --git a/polylogue/config.py b/polylogue/config.py index 83c0253279..3991317a3e 100644 --- a/polylogue/config.py +++ b/polylogue/config.py @@ -103,6 +103,7 @@ class Config: # the defaults makes the spec honest about the optional surface. drive_config: DriveConfig | None = None index_config: IndexConfig | None = None + _db_path_explicit: bool = False embedding_model: str = "voyage-4-lite" embedding_dimension: int = 1024 judgment_automation_interval_s: int = 3600 diff --git a/polylogue/core/raw_state.py b/polylogue/core/raw_state.py index 132d2f8ee6..a02bf16022 100644 --- a/polylogue/core/raw_state.py +++ b/polylogue/core/raw_state.py @@ -16,6 +16,10 @@ def raw_state_authority( New writes make opposing transitions strictly monotonic. Existing rows can predate that invariant, so an equal non-null pair remains explicitly indeterminate rather than being silently attributed to either stage. + + A pair with both values ``None`` reports ``"validation"``. These legacy + rows carry no ordering evidence, so callers retain the stored validation + verdict instead of inventing parse authority. """ if parsed_at_ms is None: return "validation" diff --git a/polylogue/storage/repair.py b/polylogue/storage/repair.py index 1c26b035db..4d7edf7dcc 100644 --- a/polylogue/storage/repair.py +++ b/polylogue/storage/repair.py @@ -3794,6 +3794,18 @@ def _raw_artifact_coordinate_predicate(*, artifact_alias: str, raw_alias: str) - """ +def _failed_validation_overrides_parse_predicate(*, raw_alias: str) -> str: + """Return the durable state in which validation blocks raw replay.""" + return f""" + COALESCE({raw_alias}.validation_status, '') = 'failed' + AND ( + {raw_alias}.parsed_at_ms IS NULL + OR {raw_alias}.validated_at_ms IS NULL + OR {raw_alias}.validated_at_ms >= {raw_alias}.parsed_at_ms + ) + """ + + def _raw_materialization_candidate_ids( config: Config, *, @@ -3961,17 +3973,10 @@ def _raw_materialization_candidate_ids( -- successful parse records a durable parsed timestamp. Keep -- that historical diagnostic, but do not let it block an -- index reset from replaying successfully parsed raw bytes. - AND NOT ( - COALESCE(r.validation_status, '') = 'failed' - AND ( - r.parsed_at_ms IS NULL - OR r.validated_at_ms IS NULL - -- Equal legacy timestamps are indeterminate. Do not - -- replay and overwrite either authority until a new, - -- monotonic transition resolves the ambiguity. - OR r.validated_at_ms >= r.parsed_at_ms - ) - ) + -- Equal legacy timestamps are indeterminate. Do not replay + -- and overwrite either authority until a monotonic transition + -- resolves the ambiguity. + AND NOT ({_failed_validation_overrides_parse_predicate(raw_alias="r")}) AND ( r.parse_error IS NULL OR r.parse_error = 'OperationalError: database is locked' @@ -4002,14 +4007,7 @@ def _raw_materialization_candidate_ids( ) AND ( r.parse_error IS NOT NULL - OR ( - r.validation_status = 'failed' - AND ( - r.parsed_at_ms IS NULL - OR r.validated_at_ms IS NULL - OR r.validated_at_ms >= r.parsed_at_ms - ) - ) + OR ({_failed_validation_overrides_parse_predicate(raw_alias="r")}) ) ) AND NOT ( @@ -4727,8 +4725,7 @@ def _raw_replay_plan_outcome( SELECT 1 FROM raw_sessions WHERE raw_id IN ({placeholders}) - AND validation_status = 'failed' - AND parsed_at_ms IS NULL + AND ({_failed_validation_overrides_parse_predicate(raw_alias="raw_sessions")}) UNION ALL SELECT 1 FROM raw_sessions @@ -5926,10 +5923,18 @@ def repair_superseded_raw_snapshots(config: Config, dry_run: bool = False) -> Re if dry_run: return _repair_superseded_raw_snapshots(config, dry_run=True) - from polylogue.storage.index_generation import ActiveWriterLease + from polylogue.storage.index_generation import ActiveWriterLease, RebuildLeaseUnavailableError lease = ActiveWriterLease(_raw_materialization_archive_root(config)) - lease.acquire() + try: + lease.acquire() + except RebuildLeaseUnavailableError as exc: + return _repair_result( + "superseded_raw_snapshots", + repaired_count=0, + success=False, + detail=f"Skipped destructive raw cleanup: {exc}", + ) try: return _repair_superseded_raw_snapshots(config, dry_run=False) finally: @@ -6288,11 +6293,19 @@ def run() -> RepairResult: if dry_run: return run() - from polylogue.storage.index_generation import ActiveWriterLease + from polylogue.storage.index_generation import ActiveWriterLease, RebuildLeaseUnavailableError archive_root = _raw_materialization_archive_root(config) lease = ActiveWriterLease(archive_root) - lease.acquire() + try: + lease.acquire() + except RebuildLeaseUnavailableError as exc: + return _internal_derived_repair_result( + "raw_materialization", + repaired_count=0, + success=False, + detail=f"Skipped raw materialization while offline index rebuild owns archive: {exc}", + ) try: return run() finally: diff --git a/polylogue/storage/sqlite/raw_state_update.py b/polylogue/storage/sqlite/raw_state_update.py index 791d916462..7976289122 100644 --- a/polylogue/storage/sqlite/raw_state_update.py +++ b/polylogue/storage/sqlite/raw_state_update.py @@ -22,6 +22,8 @@ def compile_raw_state_update( validation_transition = state.validation_status is not UNSET or state.validation_error is not UNSET if state.parsed_at is not UNSET: if parsed_at_ms is None: + if isinstance(state.parsed_at, str): + raise ValueError(f"parsed_at must be a valid timestamp, got {state.parsed_at!r}") set_clauses.append("parsed_at_ms = ?") params.append(None) elif validation_transition: diff --git a/tests/unit/core/test_config.py b/tests/unit/core/test_config.py index 151e7613b9..6fea7a8081 100644 --- a/tests/unit/core/test_config.py +++ b/tests/unit/core/test_config.py @@ -9,6 +9,7 @@ import sys from io import StringIO from pathlib import Path +from unittest.mock import MagicMock import pytest @@ -47,6 +48,10 @@ def test_config_with_sources(self, tmp_path: Path) -> None: assert config.sources[0].name == "inbox" assert config.sources[1].name == "claude-code" + def test_config_mock_spec_exposes_explicit_database_tracking(self) -> None: + """Consumers cloning Config can inspect its explicit-path contract.""" + assert hasattr(MagicMock(spec=Config), "_db_path_explicit") + def test_config_db_path_default(self, workspace_env: dict[str, Path]) -> None: """db_path defaults to the resolved index.db database path.""" config = Config( diff --git a/tests/unit/core/test_sampling.py b/tests/unit/core/test_sampling.py index d6be892135..6976866cd7 100644 --- a/tests/unit/core/test_sampling.py +++ b/tests/unit/core/test_sampling.py @@ -283,7 +283,10 @@ def test_sampling_keeps_successfully_reparsed_historical_validation_failure(self ).encode(), ) with sqlite3.connect(db.with_name("source.db")) as conn: - conn.execute("UPDATE raw_sessions SET parsed_at_ms = 1, validated_at_ms = 0, validation_status = 'failed'") + cursor = conn.execute( + "UPDATE raw_sessions SET parsed_at_ms = 1, validated_at_ms = 0, validation_status = 'failed'" + ) + assert cursor.rowcount == 1 conn.commit() result = load_samples_from_db("claude-ai", db_path=db) diff --git a/tests/unit/sources/test_live_watcher.py b/tests/unit/sources/test_live_watcher.py index 85e4757f3d..6ec94beae0 100644 --- a/tests/unit/sources/test_live_watcher.py +++ b/tests/unit/sources/test_live_watcher.py @@ -3733,9 +3733,10 @@ async def drain() -> None: try: watcher._schedule_hook_spool_directory_retry(shard) task = watcher._hook_spool_directory_retry_tasks[shard.resolve()] - while not task.done(): + with contextlib.suppress(sqlite3.OperationalError): + await task + if watcher._hook_spool_directory_retry_tasks: await asyncio.sleep(0) - await asyncio.sleep(0) finally: parse_stage.shutdown() diff --git a/tests/unit/storage/test_parse_tracking.py b/tests/unit/storage/test_parse_tracking.py index 0479d37445..6aea9b4e89 100644 --- a/tests/unit/storage/test_parse_tracking.py +++ b/tests/unit/storage/test_parse_tracking.py @@ -194,8 +194,9 @@ async def test_update_raw_state_truncates_error_fields(self, backend: SQLiteBack assert rec.validation_error is not None assert len(rec.validation_error) == 2000 + @pytest.mark.parametrize("wall_clock_ms", [1000, 999]) async def test_failed_validation_after_parse_advances_past_identical_or_backward_clock( - self, backend: SQLiteBackend, monkeypatch: pytest.MonkeyPatch + self, backend: SQLiteBackend, monkeypatch: pytest.MonkeyPatch, wall_clock_ms: int ) -> None: """A failed revalidation cannot tie or precede the parse it supersedes.""" from polylogue.storage.sqlite.queries import raw_state as raw_state_queries @@ -205,7 +206,7 @@ async def test_failed_validation_after_parse_advances_past_identical_or_backward "parse-then-failed-validation", state=RawSessionStateUpdate(parsed_at="1970-01-01T00:00:01Z"), ) - monkeypatch.setattr(raw_state_queries, "_now_ms", lambda: 999) + monkeypatch.setattr(raw_state_queries, "_now_ms", lambda: wall_clock_ms) await backend.mark_raw_validated("parse-then-failed-validation", status="failed", error="rejected") with sqlite3.connect(backend._source_db_path) as conn: @@ -215,8 +216,19 @@ async def test_failed_validation_after_parse_advances_past_identical_or_backward ).fetchone() assert row == (1000, 1001, "failed") + @pytest.mark.parametrize( + ("parsed_at", "expected"), + [ + ("1970-01-01T00:00:01Z", (1001, 1000, "failed")), + ("1970-01-01T00:00:00.999Z", (1001, 1000, "failed")), + ], + ) async def test_successful_parse_after_validation_advances_past_identical_or_backward_clock( - self, backend: SQLiteBackend, monkeypatch: pytest.MonkeyPatch + self, + backend: SQLiteBackend, + monkeypatch: pytest.MonkeyPatch, + parsed_at: str, + expected: tuple[int, int, str], ) -> None: """A later parse wins even if its injected wall clock is older.""" from polylogue.storage.sqlite.queries import raw_state as raw_state_queries @@ -226,7 +238,7 @@ async def test_successful_parse_after_validation_advances_past_identical_or_back await backend.mark_raw_validated("validation-then-parse", status="failed", error="rejected") await backend.update_raw_state( "validation-then-parse", - state=RawSessionStateUpdate(parsed_at="1970-01-01T00:00:00.999Z", parse_error=None), + state=RawSessionStateUpdate(parsed_at=parsed_at, parse_error=None), ) with sqlite3.connect(backend._source_db_path) as conn: @@ -234,7 +246,28 @@ async def test_successful_parse_after_validation_advances_past_identical_or_back "SELECT parsed_at_ms, validated_at_ms, validation_status FROM raw_sessions WHERE raw_id = ?", ("validation-then-parse",), ).fetchone() - assert row == (1001, 1000, "failed") + assert row == expected + + async def test_malformed_parse_timestamp_cannot_clear_existing_parse_authority( + self, backend: SQLiteBackend + ) -> None: + await self._save_raw(backend, raw_id="malformed-parse-timestamp") + await backend.update_raw_state( + "malformed-parse-timestamp", + state=RawSessionStateUpdate(parsed_at="1970-01-01T00:00:01Z"), + ) + + with pytest.raises(ValueError, match="parsed_at must be a valid timestamp"): + await backend.update_raw_state( + "malformed-parse-timestamp", + state=RawSessionStateUpdate(parsed_at="not-a-timestamp"), + ) + + with sqlite3.connect(backend._source_db_path) as conn: + row = conn.execute( + "SELECT parsed_at_ms FROM raw_sessions WHERE raw_id = ?", ("malformed-parse-timestamp",) + ).fetchone() + assert row == (1000,) class TestMarkRawValidated: diff --git a/tests/unit/storage/test_repair.py b/tests/unit/storage/test_repair.py index 3fec57866c..d202ad0f3d 100644 --- a/tests/unit/storage/test_repair.py +++ b/tests/unit/storage/test_repair.py @@ -14,6 +14,7 @@ from polylogue.config import Config from polylogue.core.enums import ArtifactSupportStatus from polylogue.core.errors import RawCASFrontierError +from polylogue.core.json import json_document from polylogue.core.raw_failure_evidence import RawFailureEvidenceKind from polylogue.daemon.status import raw_failure_info_for_root from polylogue.maintenance.models import DerivedModelStatus @@ -24,7 +25,7 @@ from polylogue.storage.insights.session.repair_assessment import assess_session_insight_repairs from polylogue.storage.insights.session.runtime import SessionInsightCounts, SessionInsightStatusSnapshot from polylogue.storage.raw.models import RawSessionStateUpdate -from polylogue.storage.raw_authority import RawReplayPlan, RawReplayPlanOutcome +from polylogue.storage.raw_authority import RawReplayPlan, RawReplayPlanOutcome, RawReplayPlanStatus from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root, initialize_archive_database from polylogue.storage.sqlite.archive_tiers.source_write import ArchiveSourceArtifact, upsert_raw_artifact from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier @@ -91,6 +92,30 @@ def inspect_inner(*_args: object, **_kwargs: object) -> Any: pass +def test_raw_materialization_returns_a_typed_failure_while_rebuild_owns_archive(tmp_path: Path) -> None: + """A lease conflict cannot abort a caller aggregating repair results.""" + from polylogue.storage.index_generation import RebuildLease + + initialize_active_archive_root(tmp_path) + with RebuildLease(tmp_path): + result = repair_mod.repair_raw_materialization(_config(tmp_path)) + + assert result.success is False + assert "offline index rebuild owns archive" in result.detail + + +def test_raw_snapshot_cleanup_returns_a_typed_failure_while_rebuild_owns_archive(tmp_path: Path) -> None: + """Destructive raw cleanup reports a lease conflict through RepairResult.""" + from polylogue.storage.index_generation import RebuildLease + + initialize_active_archive_root(tmp_path) + with RebuildLease(tmp_path): + result = repair_mod.repair_superseded_raw_snapshots(_config(tmp_path)) + + assert result.success is False + assert "offline index rebuild owns archive" in result.detail + + def test_raw_materialization_reparses_legacy_indexed_raw_before_receipting(tmp_path: Path) -> None: """The daemon reopens legacy bytes instead of certifying old durable bindings.""" from polylogue.archive.message.roles import Role @@ -1004,6 +1029,62 @@ def test_raw_materialization_refuses_non_parse_authoritative_validation_failure( assert repair_mod.raw_materialization_replay_backlog(config)["candidate_count"] == 0 +def test_raw_replay_plan_marks_tied_validation_component_terminal(tmp_path: Path) -> None: + """A tied failed member cannot make an otherwise parsed component look executed.""" + from polylogue.core.enums import Provider + from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore + + initialize_active_archive_root(tmp_path) + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + parsed_raw_id = archive.write_raw_payload( + provider=Provider.CODEX, + payload=b'{"type":"session_meta","payload":{"id":"parsed-member"}}\n', + source_path="parsed-member.jsonl", + acquired_at_ms=1, + ) + tied_raw_id = archive.write_raw_payload( + provider=Provider.CODEX, + payload=b'{"type":"session_meta","payload":{"id":"tied-member"}}\n', + source_path="tied-member.jsonl", + acquired_at_ms=2, + ) + archive.finalize_raw_parse_state( + parsed_raw_id, + state=RawSessionStateUpdate(parsed_at="1970-01-01T00:00:00.001Z"), + ) + archive.finalize_raw_parse_state( + tied_raw_id, + state=RawSessionStateUpdate(parsed_at="1970-01-01T00:00:00.001Z"), + ) + + with sqlite3.connect(tmp_path / "source.db") as conn: + cursor = conn.execute( + """ + UPDATE raw_sessions + SET validation_status = 'failed', validation_error = ?, validated_at_ms = parsed_at_ms + WHERE raw_id = ? + """, + ("rejected at the same legacy millisecond", tied_raw_id), + ) + assert cursor.rowcount == 1 + conn.commit() + + plan = RawReplayPlan( + "raw-replay:tied-validation-component", + "0" * 64, + (parsed_raw_id, tied_raw_id), + ("codex:tied-validation-component",), + json_document({}), + json_document({}), + json_document({}), + ) + remaining = repair_mod.RawMaterializationCandidates(raw_ids=[], missing_blobs=0, already_parsed=0) + + outcome = repair_mod._raw_replay_plan_outcomes(tmp_path, tmp_path / "index.db", [plan], remaining=remaining)[0] + + assert outcome.status is RawReplayPlanStatus.TERMINAL + + @pytest.mark.parametrize("artifact_kind", ["deferred_hot_jsonl_capture", "deferred_claude_code_partial_jsonl"]) def test_raw_materialization_does_not_replay_hot_partial_capture(tmp_path: Path, artifact_kind: str) -> None: """Hot partial evidence stays deferred until a complete source observation arrives.""" @@ -2192,7 +2273,7 @@ def test_raw_materialization_reports_uncensused_append_fragments_as_pending_debt assert "persisted parser census" in targeted.detail with sqlite3.connect(tmp_path / "source.db") as source_conn: - source_conn.execute( + cursor = source_conn.execute( """ UPDATE raw_membership_census SET parser_fingerprint = 'test', status = 'failed', member_count = 0, @@ -2201,7 +2282,7 @@ def test_raw_materialization_reports_uncensused_append_fragments_as_pending_debt """, (raw_id,), ) - assert source_conn.total_changes == 1 + assert cursor.rowcount == 1 source_conn.commit() governed = repair_mod._raw_materialization_candidate_ids(config) From 08ba6a774154cce8781c858d0eab37436dbb016e Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 11:33:34 +0200 Subject: [PATCH 34/65] fix(replay): bound retained source recovery --- polylogue/sources/revision_backfill.py | 63 ++++++++++++++++---- polylogue/storage/repair.py | 2 + tests/unit/sources/test_revision_backfill.py | 46 ++++++++++++++ 3 files changed, 99 insertions(+), 12 deletions(-) diff --git a/polylogue/sources/revision_backfill.py b/polylogue/sources/revision_backfill.py index e88937c3a2..2773fb77a6 100644 --- a/polylogue/sources/revision_backfill.py +++ b/polylogue/sources/revision_backfill.py @@ -58,6 +58,7 @@ from polylogue.sources.parsers import antigravity, codex_state, hermes_state, hermes_verification from polylogue.sources.parsers.base import ParsedSession from polylogue.sources.sqlite_snapshot import looks_like_sqlite_bytes +from polylogue.storage.archive_identity import ArchiveLocation from polylogue.storage.raw.models import RawSessionStateUpdate from polylogue.storage.raw_authority import ( RAW_AUTHORITY_PARSER_FINGERPRINT, @@ -73,6 +74,7 @@ from polylogue.storage.sqlite.archive_tiers.write import PreparedSessionRows, prepare_session_rows _LOGGER = _polylogue_logging.get_logger(__name__) +_REPLAY_PROVIDER_DETECTION_PREFIX_BYTES: Final[int] = 8192 def _canonical_authority_logical_key(logical_key: str) -> str: @@ -746,7 +748,10 @@ def bind_byte_proven_older_member(raw_id: str, logical_key: str) -> None: census_selection = initial_selection while True: rows = archive.raw_membership_census_rows(census_selection) - for raw_id, _source_index, _terminal_non_session in rows: + for raw_id, _source_index, _terminal_non_session in sorted( + rows, + key=lambda row: (archive.raw_revision_acquired_at_ms(row[0]), row[1], row[0]), + ): if raw_id in state.censused or not _replay_retained_codex_state_evidence(archive, raw_id): continue state.scanned += 1 @@ -1176,7 +1181,11 @@ def validate_frozen_source_authority( archive_root, active_index_path=active_index_path, ) as archive, - _ParsedSessionSpill(archive_root, max_cached_payload_bytes=max_payload_bytes) as spill, + _ParsedSessionSpill( + archive_root, + index_path=active_index_path, + max_cached_payload_bytes=max_payload_bytes, + ) as spill, ): census = _load_frozen_revision_evidence( archive, @@ -1233,6 +1242,7 @@ def validate_frozen_source_authority( def census_historical_revision_evidence( archive_root: Path, *, + active_index_path: Path | None = None, selected_raw_ids: list[str] | None = None, max_payload_bytes: int | None = None, ingest_workers: int = 1, @@ -1248,7 +1258,11 @@ def census_historical_revision_evidence( """ with ( ArchiveStore.open_existing(archive_root, read_only=False) as archive, - _ParsedSessionSpill(archive_root, max_cached_payload_bytes=max_payload_bytes) as spill, + _ParsedSessionSpill( + archive_root, + index_path=active_index_path, + max_cached_payload_bytes=max_payload_bytes, + ) as spill, ): state = _census_historical_revision_evidence( archive, @@ -1370,6 +1384,7 @@ def visit(key: str) -> None: def backfill_historical_revision_evidence( archive_root: Path, *, + active_index_path: Path | None = None, selected_raw_ids: list[str] | None = None, owned_inactive_generation: tuple[str, str] | None = None, retention_observer: Callable[[int, int], None] | None = None, @@ -1508,7 +1523,11 @@ def backfill_historical_revision_evidence( prepare_pool = ThreadPoolExecutor(max_workers=1) if parallel_threads_effective() else None with ( archive_context as archive, - _ParsedSessionSpill(archive_root, max_cached_payload_bytes=spill_cache_bytes) as spill, + _ParsedSessionSpill( + archive_root, + index_path=active_index_path, + max_cached_payload_bytes=spill_cache_bytes, + ) as spill, prepare_pool if prepare_pool is not None else nullcontext(), ): census_started = time.perf_counter() @@ -1945,8 +1964,19 @@ def census_parse_worker( publisher = ArchiveBlobPublisher(Path(source_db_path_str), Path(blob_root_str)) try: if provider is Provider.UNKNOWN: + provider, _evidence = detect_provider_from_raw_bytes_evidence( + publisher.read_prefix(blob_hash, _REPLAY_PROVIDER_DETECTION_PREFIX_BYTES), + Path(source_path).name, + provider, + truncated_tail_ok=True, + ) + if is_stream_record_provider(source_path, str(provider)): + with publisher.open(blob_hash) as stream_payload: + sessions = _parse_stream( + provider, stream_payload, source_path, fallback_id_override=fallback_id_override + ) + return raw_id, sessions, None payload = publisher.read_all(blob_hash) - provider, _evidence = detect_provider_from_raw_bytes_evidence(payload, Path(source_path).name, provider) payload_path = None if provider is Provider.HERMES: candidate_path = publisher.blob_path(blob_hash) @@ -2373,12 +2403,15 @@ def parse_retained_raw_sessions(archive: ArchiveStore, raw_id: str) -> list[Pars # without decoding them. Recovery is the first lawful point to # inspect the durable bytes and resolve their parser, before deciding # whether their filename is a stream route. - _provider, eager_payload, _source_path, _eager_kind = archive.raw_revision_material(raw_id) + with archive.open_raw_revision_material(raw_id) as (_stream_provider, payload, _stream_path, _stream_kind): + detection_prefix = payload.read(_REPLAY_PROVIDER_DETECTION_PREFIX_BYTES) provider, _evidence = detect_provider_from_raw_bytes_evidence( - eager_payload, - Path(source_path).name, - provider, + detection_prefix, Path(source_path).name, provider, truncated_tail_ok=True ) + if is_stream_record_provider(source_path, str(provider)): + with archive.open_raw_revision_material(raw_id) as (_stream_provider, payload, stream_path, _stream_kind): + return _parse_stream(provider, payload, stream_path, fallback_id_override=fallback_id_override) + _provider, eager_payload, _source_path, _eager_kind = archive.raw_revision_material(raw_id) payload_path = archive.blob_path_for_hash(blob_hash) if provider is Provider.HERMES else None return _parse_one( provider, @@ -2885,14 +2918,20 @@ class _ParsedSessionSpill: #: existing crash-recovery contract for the sqlite-backed spill. _WHALE_CACHE_MAX_TREE_BYTES: Final[int] = 8 * 1024 * 1024 * 1024 - def __init__(self, archive_root: Path, *, max_cached_payload_bytes: int | None) -> None: + def __init__( + self, + archive_root: Path, + *, + index_path: Path | None = None, + max_cached_payload_bytes: int | None, + ) -> None: # Place the spill beside the RESOLVED index tier, not the archive # root: on deployments where the .db files are symlinks (e.g. root # SSD config dir -> NVMe data disk), a spill in archive_root would # put census churn on the wear-limited disk the symlinks exist to # protect. - index_path = archive_root / "index.db" - spill_dir = index_path.resolve().parent if index_path.exists() else archive_root + resolved_index_path = index_path or ArchiveLocation.resolve(archive_root).active_index_path + spill_dir = resolved_index_path.resolve().parent if resolved_index_path.exists() else archive_root fd, name = tempfile.mkstemp(prefix=".revision-census-", suffix=".sqlite", dir=spill_dir) os.close(fd) self.path = Path(name) diff --git a/polylogue/storage/repair.py b/polylogue/storage/repair.py index 4d7edf7dcc..9216bdde55 100644 --- a/polylogue/storage/repair.py +++ b/polylogue/storage/repair.py @@ -6487,6 +6487,7 @@ def _pass_deadline_exceeded() -> bool: try: census_historical_revision_evidence( archive_root, + active_index_path=index_db, selected_raw_ids=[seed], max_payload_bytes=max_payload_bytes, ingest_workers=ingest_workers, @@ -7009,6 +7010,7 @@ def _pass_deadline_exceeded() -> bool: try: part = backfill_historical_revision_evidence( archive_root, + active_index_path=index_db, selected_raw_ids=[raw_id], max_payload_bytes=max_payload_bytes, ingest_workers=ingest_workers, diff --git a/tests/unit/sources/test_revision_backfill.py b/tests/unit/sources/test_revision_backfill.py index c06214f0e9..7ba811cd0f 100644 --- a/tests/unit/sources/test_revision_backfill.py +++ b/tests/unit/sources/test_revision_backfill.py @@ -156,6 +156,52 @@ def test_parse_one_replays_single_session_state_db_bytes_via_temp_spill(tmp_path assert sessions[0].messages[0].text == "hi" +def test_unknown_retained_stream_replay_detects_from_prefix_without_eager_payload( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """UNKNOWN source-only JSONL reopens the blob as a stream after prefix detection.""" + initialize_active_archive_root(tmp_path) + payload = ( + b'{"type":"session_meta","payload":{"id":"unknown-stream","timestamp":"2026-06-01T00:00:00Z"}}\n' + b'{"type":"response_item","payload":{"type":"message","id":"m1","role":"user",' + b'"content":[{"type":"input_text","text":"prefix detected replay"}]}}\n' + ) + + def detect_from_prefix(raw_bytes: bytes, *_args: object, **_kwargs: object) -> tuple[Provider, str]: + assert raw_bytes == payload + return Provider.CODEX, "test prefix" + + monkeypatch.setattr(revision_backfill, "detect_provider_from_raw_bytes_evidence", detect_from_prefix) + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.UNKNOWN, + payload=payload, + source_path="unknown-member.jsonl", + acquired_at_ms=1, + ) + archive.raw_revision_material = lambda _raw_id: (_ for _ in ()).throw(AssertionError("eager payload read")) # type: ignore[method-assign] + sessions = revision_backfill.parse_retained_raw_sessions(archive, raw_id) + + assert [session.provider_session_id for session in sessions] == ["unknown-stream"] + + +def test_parsed_session_spill_uses_the_pinned_active_index_directory(tmp_path: Path) -> None: + """Repair spill churn follows the generation being repaired, not a shadow index.""" + archive_root = tmp_path / "archive" + archive_root.mkdir() + active_index = tmp_path / "external-generation" / "index.db" + active_index.parent.mkdir() + active_index.touch() + (archive_root / "index.db").touch() + + with revision_backfill._ParsedSessionSpill( + archive_root, + index_path=active_index, + max_cached_payload_bytes=None, + ) as spill: + assert spill.path.parent == active_index.parent + + @pytest.mark.parametrize( "source_path_suffix", [ From 4bbc47da6971a95d9abdda463316cf7035ed8d73 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 12:10:00 +0200 Subject: [PATCH 35/65] test(replay): type eager-read sentinel --- tests/unit/sources/test_revision_backfill.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/unit/sources/test_revision_backfill.py b/tests/unit/sources/test_revision_backfill.py index 7ba811cd0f..28cf9403f6 100644 --- a/tests/unit/sources/test_revision_backfill.py +++ b/tests/unit/sources/test_revision_backfill.py @@ -179,7 +179,11 @@ def detect_from_prefix(raw_bytes: bytes, *_args: object, **_kwargs: object) -> t source_path="unknown-member.jsonl", acquired_at_ms=1, ) - archive.raw_revision_material = lambda _raw_id: (_ for _ in ()).throw(AssertionError("eager payload read")) # type: ignore[method-assign] + + def reject_eager_material(_raw_id: str) -> tuple[Provider, bytes, str, RawRevisionKind]: + raise AssertionError("eager payload read") + + monkeypatch.setattr(archive, "raw_revision_material", reject_eager_material) sessions = revision_backfill.parse_retained_raw_sessions(archive, raw_id) assert [session.provider_session_id for session in sessions] == ["unknown-stream"] From b43242c9871f298ef8b908c11782048ec3d65778 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 15:29:14 +0200 Subject: [PATCH 36/65] fix: preserve retained raw replay authority (#3968) ## Summary Preserve retained raw replay authority across complete-document detection, long JSONL streams, re-observed Codex snapshots, invalid active pointers, and terminal source-only artifacts. ## Problem Exact-head review of #3952 found five replay paths where bounded inspection, first-acquisition ordering, or an unhandled storage error could lose materialized sessions or leave retained raw evidence indefinitely non-terminal. ## Solution - Retry UNKNOWN document detection from complete retained bytes after the bounded prefix misses, in both sequential and worker replay paths. - Stream declared JSONL artifacts to exhaustion before deciding whether conversational evidence is absent. - Order retained Codex snapshots by their latest durable raw observation receipt. - Project malformed or unreadable active index pointers as unavailable raw retention instead of raising. - Persist confirmed non-session artifacts and their successful parser census through the existing raw-artifact and source-state paths. ## Verification - `devtools test tests/unit/sources/test_revision_backfill.py::test_unknown_retained_document_replays_after_complete_payload_detection tests/unit/sources/test_revision_backfill.py::test_backfill_scans_declared_stream_past_non_session_prefix tests/unit/sources/test_revision_backfill.py::test_backfill_replays_codex_state_by_latest_raw_observation tests/unit/sources/test_revision_backfill.py::test_backfill_terminalizes_source_only_declared_artifact tests/unit/storage/test_raw_retention.py::test_raw_frontier_integrity_projection_reports_malformed_active_pointer`: `5 passed in 13.74s`. - `ruff format --check` and `ruff check` on the changed production and test files: passed. - `mypy`: `Success: no issues found in 2743 source files`. - `devtools test tests/unit/scenarios/test_codex_804_live_proof.py::test_sanitized_codex_804_revision_recovery_proof`: fails before its checkpoint seam because one retained raw lacks a complete current-parser census. This PR does not alter that frozen-census precondition. ## Scope No Bead records are assigned or mutated by this exact-head review lane. --- polylogue/sources/revision_backfill.py | 184 +++++++- polylogue/storage/raw_retention.py | 33 +- polylogue/storage/repair.py | 13 + .../storage/sqlite/archive_tiers/archive.py | 12 +- .../archive_tiers/revision_governance.py | 57 ++- .../sqlite/archive_tiers/source_write.py | 45 +- tests/unit/sources/test_revision_backfill.py | 437 ++++++++++++++++++ .../test_archive_tiers_source_write.py | 50 ++ tests/unit/storage/test_raw_retention.py | 49 ++ tests/unit/storage/test_repair.py | 20 +- 10 files changed, 855 insertions(+), 45 deletions(-) diff --git a/polylogue/sources/revision_backfill.py b/polylogue/sources/revision_backfill.py index 2773fb77a6..a0f3af1ea1 100644 --- a/polylogue/sources/revision_backfill.py +++ b/polylogue/sources/revision_backfill.py @@ -14,7 +14,6 @@ from contextlib import closing, contextmanager, nullcontext from dataclasses import dataclass, field from io import BytesIO -from itertools import chain, islice from pathlib import Path from types import TracebackType from typing import BinaryIO, Final, Literal, cast @@ -59,6 +58,7 @@ from polylogue.sources.parsers.base import ParsedSession from polylogue.sources.sqlite_snapshot import looks_like_sqlite_bytes from polylogue.storage.archive_identity import ArchiveLocation +from polylogue.storage.artifacts.inspection import artifact_observation_id from polylogue.storage.raw.models import RawSessionStateUpdate from polylogue.storage.raw_authority import ( RAW_AUTHORITY_PARSER_FINGERPRINT, @@ -68,9 +68,14 @@ from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore from polylogue.storage.sqlite.archive_tiers.revision_governance import ( FrozenSourceRemediationRequiredError, + _raw_parse_success_state, record_current_parser_source_census, ) -from polylogue.storage.sqlite.archive_tiers.source_write import apply_source_raw_state_update +from polylogue.storage.sqlite.archive_tiers.source_write import ( + ArchiveSourceArtifact, + apply_source_raw_state_update, + upsert_raw_artifact, +) from polylogue.storage.sqlite.archive_tiers.write import PreparedSessionRows, prepare_session_rows _LOGGER = _polylogue_logging.get_logger(__name__) @@ -674,6 +679,54 @@ def apply_outcome( state=RawSessionStateUpdate(payload_provider=Provider.from_string(sessions[0].source_name)), manage_transaction=not batched, ) + if not sessions: + provider = _detected_provider_for_empty_replay( + archive, + raw_id, + stored_provider=stored_provider, + source_path=_source_path, + ) + # A terminal artifact makes this raw ineligible for future census + # work. Its artifact carrier, parse state, and both census receipts + # must therefore become durable as one source-tier transaction. + # Batches retain that transaction until their existing commit + # boundary instead of forcing one SQLite commit per empty raw. + transaction = nullcontext() if batched else archive._ensure_source_conn() + with transaction: + if stored_provider is Provider.UNKNOWN and provider is not Provider.UNKNOWN: + apply_source_raw_state_update( + archive._ensure_source_conn(), + raw_id, + state=RawSessionStateUpdate(payload_provider=provider), + manage_transaction=False, + ) + terminalized = _persist_terminal_non_session_artifact( + archive, + raw_id, + provider=provider, + source_path=_source_path, + source_index=source_index, + manage_transaction=False, + ) + if provider is not Provider.UNKNOWN: + archive.replace_raw_membership_census( + raw_id, + [], + parser_fingerprint=RAW_AUTHORITY_PARSER_FINGERPRINT, + censused_at_ms=0, + retire_full_revision_governance=revision_kind is not RawRevisionKind.UNKNOWN, + manage_transaction=False, + ) + if not terminalized: + apply_source_raw_state_update( + archive._ensure_source_conn(), + raw_id, + state=_raw_parse_success_state(provider), + manage_transaction=False, + ) + if provider is not Provider.UNKNOWN: + commit_unit() + return state.classified += int(len(sessions) == 1) spill.add(raw_id, sessions, payload_bytes=payload_bytes) if len(sessions) == 1 and revision_kind is RawRevisionKind.UNKNOWN: @@ -748,9 +801,9 @@ def bind_byte_proven_older_member(raw_id: str, logical_key: str) -> None: census_selection = initial_selection while True: rows = archive.raw_membership_census_rows(census_selection) - for raw_id, _source_index, _terminal_non_session in sorted( + for raw_id, _source_index, _terminal_non_session, _raw_rowid in sorted( rows, - key=lambda row: (archive.raw_revision_acquired_at_ms(row[0]), row[1], row[0]), + key=lambda row: archive.raw_revision_observation_order(row[0]), ): if raw_id in state.censused or not _replay_retained_codex_state_evidence(archive, raw_id): continue @@ -758,14 +811,14 @@ def bind_byte_proven_older_member(raw_id: str, logical_key: str) -> None: state.censused.add(raw_id) commit_unit() terminal_raw_ids = { - raw_id for raw_id, _source_index, terminal_non_session in rows if terminal_non_session + raw_id for raw_id, _source_index, terminal_non_session, _raw_rowid in rows if terminal_non_session } for raw_id in terminal_raw_ids - state.censused: state.scanned += 1 state.censused.add(raw_id) pending_rows = [ (raw_id, source_index) - for raw_id, source_index, terminal_non_session in rows + for raw_id, source_index, terminal_non_session, _raw_rowid in rows if raw_id not in state.censused and not terminal_non_session ] if max_payload_bytes is not None: @@ -830,7 +883,7 @@ def bind_byte_proven_older_member(raw_id: str, logical_key: str) -> None: break census_selection = expanded except BaseException: - if batched and pending_commits > 0: + if batched: archive.rollback() raise if batched and pending_commits > 0: @@ -858,7 +911,7 @@ def _load_frozen_revision_evidence( rows = archive.raw_membership_census_rows(expanded_raw_ids if selected_raw_ids is not None else None) if max_payload_bytes is not None: payload_sizes = archive.raw_payload_sizes( - [raw_id for raw_id, _source_index, terminal_non_session in rows if not terminal_non_session] + [raw_id for raw_id, _source_index, terminal_non_session, _raw_rowid in rows if not terminal_non_session] ) total_payload_bytes = sum(payload_sizes.values()) oversized = [raw_id for raw_id, size in payload_sizes.items() if size > max_payload_bytes] @@ -867,7 +920,9 @@ def _load_frozen_revision_evidence( sorted(oversized or payload_sizes), max_payload_bytes, total_payload_bytes ) parseable_raw_ids = [ - raw_id for raw_id, source_index, terminal_non_session in rows if source_index >= 0 and not terminal_non_session + raw_id + for raw_id, source_index, terminal_non_session, _raw_rowid in rows + if source_index >= 0 and not terminal_non_session ] parsed_outcomes = _parse_retained_raws( archive, @@ -876,7 +931,7 @@ def _load_frozen_revision_evidence( prefetch_cache=prefetch_cache, ) state = _RevisionCensusState(0, 0, 0, set(), {}, {}) - for raw_id, source_index, terminal_non_session in rows: + for raw_id, source_index, terminal_non_session, _raw_rowid in rows: state.scanned += 1 state.censused.add(raw_id) if terminal_non_session: @@ -1977,6 +2032,12 @@ def census_parse_worker( ) return raw_id, sessions, None payload = publisher.read_all(blob_hash) + if provider is Provider.UNKNOWN: + provider, _evidence = detect_provider_from_raw_bytes_evidence( + payload, + Path(source_path).name, + provider, + ) payload_path = None if provider is Provider.HERMES: candidate_path = publisher.blob_path(blob_hash) @@ -2412,6 +2473,12 @@ def parse_retained_raw_sessions(archive: ArchiveStore, raw_id: str) -> list[Pars with archive.open_raw_revision_material(raw_id) as (_stream_provider, payload, stream_path, _stream_kind): return _parse_stream(provider, payload, stream_path, fallback_id_override=fallback_id_override) _provider, eager_payload, _source_path, _eager_kind = archive.raw_revision_material(raw_id) + if provider is Provider.UNKNOWN: + provider, _evidence = detect_provider_from_raw_bytes_evidence( + eager_payload, + Path(source_path).name, + provider, + ) payload_path = archive.blob_path_for_hash(blob_hash) if provider is Provider.HERMES else None return _parse_one( provider, @@ -2458,7 +2525,7 @@ def _replay_retained_codex_state_evidence(archive: ArchiveStore, raw_id: str) -> archive, codex_state.parse_codex_state_db(state_path, immutable=True), source_path=source_path, - acquired_at_ms=archive.raw_revision_acquired_at_ms(raw_id), + acquired_at_ms=archive.raw_revision_observed_at_ms(raw_id), ) archive.replace_raw_membership_census( raw_id, @@ -3212,6 +3279,87 @@ def _declared_non_session_artifact_classification( return classification if not classification.parse_as_session else None +def _detected_provider_for_empty_replay( + archive: ArchiveStore, + raw_id: str, + *, + stored_provider: Provider, + source_path: str, +) -> Provider: + """Resolve a provider before terminalizing an empty retained replay.""" + if stored_provider is not Provider.UNKNOWN: + return stored_provider + with archive.open_raw_revision_material(raw_id) as (_provider, payload, _path, _kind): + provider, _evidence = detect_provider_from_raw_bytes_evidence( + payload.read(_REPLAY_PROVIDER_DETECTION_PREFIX_BYTES), + Path(source_path).name, + stored_provider, + truncated_tail_ok=True, + ) + if provider is not Provider.UNKNOWN: + return provider + _provider, full_payload, _path, _kind = archive.raw_revision_material(raw_id) + provider, _evidence = detect_provider_from_raw_bytes_evidence( + full_payload, + Path(source_path).name, + stored_provider, + ) + return provider + + +def _persist_terminal_non_session_artifact( + archive: ArchiveStore, + raw_id: str, + *, + provider: Provider, + source_path: str, + source_index: int, + manage_transaction: bool, +) -> bool: + """Record replay-confirmed source-only artifact authority once. + + Replay reaches this function only after the real parser has consumed the + complete stream and produced no conversational session. The terminal + receipt therefore follows that one authoritative parse result instead of + reclassifying the raw through a second, weaker JSONL shape scan. + """ + if provider is Provider.UNKNOWN: + return False + classification = _declared_non_session_artifact_classification(provider, source_path) + if classification is None: + return False + origin = origin_from_provider(provider) + observed_at_ms = archive.raw_revision_observed_at_ms(raw_id) + upsert_raw_artifact( + archive._ensure_source_conn(), + raw_id, + ArchiveSourceArtifact( + artifact_id=artifact_observation_id( + source_name=origin.value, + source_path=source_path, + source_index=source_index, + ), + origin=origin, + source_path=source_path, + source_index=source_index, + artifact_kind=classification.cohort, + classification_reason=classification.reason, + parse_as_session=False, + schema_eligible=classification.schema_eligible, + first_observed_at_ms=observed_at_ms, + last_observed_at_ms=observed_at_ms, + ), + manage_transaction=manage_transaction, + ) + apply_source_raw_state_update( + archive._ensure_source_conn(), + raw_id, + state=_raw_parse_success_state(provider), + manage_transaction=manage_transaction, + ) + return True + + def _is_declared_non_session_artifact( provider: Provider, source_path: str, @@ -3376,21 +3524,9 @@ def _parse_stream_raw( source_name = Path(source_path).name fallback_id = fallback_id_override or Path(source_path).stem stream = _iter_json_stream(payload, source_name) - # Multi-GiB Claude Code JSONL must stay memory-bounded (module docstring: - # "a memory-bounded streaming path exists for multi-GiB Claude Code - # JSONL"), so only the first 64 records -- the same sample bound the live - # ingest gate uses -- are materialized for content classification; the - # rest of the stream is chained back on unread. - sample = list(islice(stream, 64)) - sample_sessions = parse_stream_payload(provider, sample, fallback_id, source_path=source_path) - sample_has_session_evidence = bool( - require_positive_conversational_evidence(sample_sessions, provider=provider, source_path=source_path) - ) - if not sample_has_session_evidence and _is_declared_non_session_artifact(provider, source_path, sample=sample): - return [] return parse_stream_payload( provider, - chain(sample, stream), + stream, fallback_id, source_path=source_path, ) diff --git a/polylogue/storage/raw_retention.py b/polylogue/storage/raw_retention.py index 5c7ec3ff20..c251ad8930 100644 --- a/polylogue/storage/raw_retention.py +++ b/polylogue/storage/raw_retention.py @@ -12,7 +12,7 @@ from polylogue.core.raw_failure_evidence import RAW_FAILURE_EVIDENCE_KINDS, RawFailureEvidenceKind from polylogue.logging import get_logger -from polylogue.storage.archive_identity import resolve_active_index_path +from polylogue.storage.archive_identity import ArchiveLocationError, resolve_active_index_path from polylogue.storage.blob_store import BlobStore, get_blob_store from polylogue.storage.introspection import column_exists as _column_exists from polylogue.storage.introspection import table_exists as _table_exists @@ -1184,7 +1184,16 @@ def raw_frontier_integrity_projection( raw_materialization_readiness, sample_limit=sample_limit, ) - index_db_path = resolve_active_index_path(archive_root) + try: + index_db_path = resolve_active_index_path(archive_root) + except ArchiveLocationError as exc: + return unknown_raw_frontier_integrity_projection( + f"active index pointer unavailable: {exc}", + missing_source_raw_status=missing_status, + missing_source_raw_count=missing_count, + missing_source_raw_samples=missing_samples, + missing_source_raw_reason=missing_reason, + ) source_db_path = archive_root / "source.db" ops_db_path = archive_root / "ops.db" snapshot = _unavailable_frontier_integrity_snapshot(f"source tier is unavailable: {source_db_path}") @@ -1234,7 +1243,14 @@ def raw_frontier_integrity_projection( ) -def unknown_raw_frontier_integrity_projection(reason: str) -> RawFrontierIntegrityProjection: +def unknown_raw_frontier_integrity_projection( + reason: str, + *, + missing_source_raw_status: RawFrontierIntegrityStatus = "unknown", + missing_source_raw_count: int = 0, + missing_source_raw_samples: tuple[Mapping[str, object], ...] = (), + missing_source_raw_reason: str | None = None, +) -> RawFrontierIntegrityProjection: """Return the canonical explicit-unknown projection for an unavailable read. Cache and presentation adapters use this instead of inventing partial @@ -1243,18 +1259,19 @@ def unknown_raw_frontier_integrity_projection(reason: str) -> RawFrontierIntegri """ snapshot = _unavailable_frontier_integrity_snapshot(reason) + statuses = (snapshot.broken_head_status, missing_source_raw_status, snapshot.cursor_ahead_status) return RawFrontierIntegrityProjection( available=False, - overall_status="unknown", + overall_status=combine_raw_frontier_integrity_statuses(*statuses), broken_head_status=snapshot.broken_head_status, broken_head_count=snapshot.broken_head_count, broken_head_checked_count=snapshot.broken_head_checked_count, broken_head_samples=snapshot.broken_head_samples, broken_head_reason=snapshot.broken_head_reason, - missing_source_raw_status="unknown", - missing_source_raw_count=0, - missing_source_raw_samples=(), - missing_source_raw_reason=reason, + missing_source_raw_status=missing_source_raw_status, + missing_source_raw_count=missing_source_raw_count, + missing_source_raw_samples=missing_source_raw_samples, + missing_source_raw_reason=reason if missing_source_raw_reason is None else missing_source_raw_reason, cursor_ahead_status=snapshot.cursor_ahead_status, cursor_ahead_count=snapshot.cursor_ahead_count, cursor_ahead_checked_count=snapshot.cursor_ahead_checked_count, diff --git a/polylogue/storage/repair.py b/polylogue/storage/repair.py index 9216bdde55..08d814f006 100644 --- a/polylogue/storage/repair.py +++ b/polylogue/storage/repair.py @@ -61,6 +61,7 @@ count_unclassified_message_type_sync, ) from polylogue.storage.raw_authority import ( + RAW_AUTHORITY_PARSER_FINGERPRINT, RAW_REPLAY_NO_PROGRESS_REASON, SUPERSEDED_MEMBERSHIP_FINGERPRINTS, RawAuthorityCensusReceipt, @@ -3930,6 +3931,15 @@ def _raw_materialization_candidate_ids( AND (m.decision IS NULL OR m.decision IN ('ambiguous', 'deferred')) ) ) AS membership_authority_complete + , EXISTS ( + SELECT 1 + FROM raw_membership_census AS c + WHERE c.raw_id = r.raw_id + AND c.parser_fingerprint = ? + AND c.status = 'non_session' + AND r.parsed_at_ms IS NOT NULL + AND r.parse_error IS NULL + ) AS membership_non_session_terminal , EXISTS ( SELECT 1 FROM raw_membership_census AS c @@ -4023,6 +4033,7 @@ def _raw_materialization_candidate_ids( [ *sorted(RAW_FAILURE_REPLAY_AUTHORITY_EVIDENCE_KINDS), RAW_FAILURE_DEFERRED_SUPPORT_STATUS, + RAW_AUTHORITY_PARSER_FINGERPRINT, BYTE_AUTHORITY_CENSUS_DETAIL, BYTE_AUTHORITY_CENSUS_DETAIL, *sorted(RAW_FAILURE_REPLAY_AUTHORITY_EVIDENCE_KINDS), @@ -4072,6 +4083,8 @@ def _raw_materialization_candidate_ids( continue if _raw_materialized_by_source_path_native(materialized_aliases, row): continue + if bool(row["membership_non_session_terminal"]): + continue if _raw_materialization_parsed_non_session_artifact(archive_root, row): continue blob_hash = row["blob_hash"].hex() if isinstance(row["blob_hash"], bytes) else str(row["blob_hash"]) diff --git a/polylogue/storage/sqlite/archive_tiers/archive.py b/polylogue/storage/sqlite/archive_tiers/archive.py index aacae2f72a..96ed8c1286 100644 --- a/polylogue/storage/sqlite/archive_tiers/archive.py +++ b/polylogue/storage/sqlite/archive_tiers/archive.py @@ -216,6 +216,8 @@ raw_revision_descriptor, raw_revision_head_raw_id, raw_revision_material, + raw_revision_observation_order, + raw_revision_observed_at_ms, raw_revision_rebuild_selection, raw_revision_replay_adoptable, raw_revision_replay_plan, @@ -2699,7 +2701,9 @@ def raw_revision_rebuild_selection( ) -> tuple[tuple[tuple[str, int], ...], tuple[str, ...]]: return raw_revision_rebuild_selection(self, raw_ids) - def raw_membership_census_rows(self, raw_ids: Sequence[str] | None = None) -> tuple[tuple[str, int, bool], ...]: + def raw_membership_census_rows( + self, raw_ids: Sequence[str] | None = None + ) -> tuple[tuple[str, int, bool, int], ...]: return raw_membership_census_rows(self, raw_ids) def raw_payload_sizes(self, raw_ids: Sequence[str]) -> dict[str, int]: @@ -2762,6 +2766,12 @@ def raw_membership_raw_ids( def raw_revision_acquired_at_ms(self, raw_id: str) -> int: return raw_revision_acquired_at_ms(self, raw_id) + def raw_revision_observed_at_ms(self, raw_id: str) -> int: + return raw_revision_observed_at_ms(self, raw_id) + + def raw_revision_observation_order(self, raw_id: str) -> tuple[int, int]: + return raw_revision_observation_order(self, raw_id) + def raw_membership_rebuild_raw_ids(self, logical_source_key: str) -> tuple[str, ...]: return raw_membership_rebuild_raw_ids(self, logical_source_key) diff --git a/polylogue/storage/sqlite/archive_tiers/revision_governance.py b/polylogue/storage/sqlite/archive_tiers/revision_governance.py index ba32d52a0d..75b8f28304 100644 --- a/polylogue/storage/sqlite/archive_tiers/revision_governance.py +++ b/polylogue/storage/sqlite/archive_tiers/revision_governance.py @@ -1709,25 +1709,38 @@ def raw_revision_rebuild_selection( def raw_membership_census_rows( store: RawRevisionGovernanceHost, raw_ids: Sequence[str] | None = None -) -> tuple[tuple[str, int, bool], ...]: +) -> tuple[tuple[str, int, bool, int], ...]: """Return retained raws and whether durable evidence says they are non-sessions.""" conn = store._ensure_source_conn() columns = """ r.raw_id, r.source_index, - EXISTS(SELECT 1 FROM raw_artifacts AS a WHERE a.raw_id = r.raw_id AND a.parse_as_session = 0) + ( + EXISTS(SELECT 1 FROM raw_artifacts AS a WHERE a.raw_id = r.raw_id AND a.parse_as_session = 0) + OR EXISTS( + SELECT 1 FROM raw_membership_census AS c + WHERE c.raw_id = r.raw_id + AND c.parser_fingerprint = ? + AND c.status = 'non_session' + AND r.parsed_at_ms IS NOT NULL + AND r.parse_error IS NULL + ) + ), + r.rowid """ if raw_ids is None: - rows = conn.execute(f"SELECT {columns} FROM raw_sessions AS r ORDER BY r.raw_id").fetchall() + rows = conn.execute( + f"SELECT {columns} FROM raw_sessions AS r ORDER BY r.raw_id", (RAW_AUTHORITY_PARSER_FINGERPRINT,) + ).fetchall() elif raw_ids: placeholders = ",".join("?" for _ in raw_ids) rows = conn.execute( f"SELECT {columns} FROM raw_sessions AS r WHERE r.raw_id IN ({placeholders}) ORDER BY r.raw_id", - tuple(raw_ids), + (RAW_AUTHORITY_PARSER_FINGERPRINT, *raw_ids), ).fetchall() else: rows = [] - return tuple((str(row[0]), int(row[1]), bool(row[2])) for row in rows) + return tuple((str(row[0]), int(row[1]), bool(row[2]), int(row[3])) for row in rows) def raw_payload_sizes(store: RawRevisionGovernanceHost, raw_ids: Sequence[str]) -> dict[str, int]: @@ -1767,8 +1780,6 @@ def replace_raw_membership_census( ).fetchone() if revision is None: raise RuntimeError(f"membership census raw is missing: {raw_id}") - if revision[0] is not None and str(revision[1]) != RawRevisionKind.FULL.value: - raise RuntimeError("only self-contained full raws can move to membership governance") dependent = conn.execute( """ SELECT 1 FROM raw_sessions @@ -2192,6 +2203,38 @@ def raw_revision_acquired_at_ms(store: RawRevisionGovernanceHost, raw_id: str) - return int(row[0]) +def raw_revision_observed_at_ms(store: RawRevisionGovernanceHost, raw_id: str) -> int: + """Return the latest durable observation receipt for a retained raw. + + ``raw_sessions.acquired_at_ms`` is deliberately immutable because the raw + id is content-derived. Re-observing identical bytes refreshes the + ``blob_refs`` raw-payload receipt instead, which is the ordering authority + for replaying mutable state snapshots. + """ + return raw_revision_observation_order(store, raw_id)[0] + + +def raw_revision_observation_order(store: RawRevisionGovernanceHost, raw_id: str) -> tuple[int, int]: + """Return the latest observation timestamp and its durable receipt order.""" + conn = store._ensure_source_conn() + row = conn.execute( + """ + SELECT acquired_at_ms, rowid + FROM blob_refs + WHERE ref_id = ? AND ref_type = 'raw_payload' + ORDER BY acquired_at_ms DESC, rowid DESC + LIMIT 1 + """, + (raw_id,), + ).fetchone() + if row is not None: + return int(row[0]), int(row[1]) + row = conn.execute("SELECT acquired_at_ms, rowid FROM raw_sessions WHERE raw_id = ?", (raw_id,)).fetchone() + if row is None: + raise KeyError(f"unknown raw revision {raw_id}") + return int(row[0]), int(row[1]) + + def raw_membership_rebuild_raw_ids(store: RawRevisionGovernanceHost, logical_source_key: str) -> tuple[str, ...]: """Return census candidates excluding quarantined full rows with another authority key.""" rows = ( diff --git a/polylogue/storage/sqlite/archive_tiers/source_write.py b/polylogue/storage/sqlite/archive_tiers/source_write.py index d5886488ae..fdbe3137aa 100644 --- a/polylogue/storage/sqlite/archive_tiers/source_write.py +++ b/polylogue/storage/sqlite/archive_tiers/source_write.py @@ -1273,9 +1273,9 @@ def upsert_raw_artifact( """ failure_kind = _is_raw_failure_artifact_kind(artifact.artifact_kind) coordinate_predicate = ( - "raw_id = ? AND origin = ? AND source_path = ? AND source_index = ?" + "a.raw_id = ? AND a.origin = ? AND a.source_path = ? AND a.source_index = ?" if failure_kind - else "origin = ? AND source_path = ? AND source_index = ? AND artifact_kind NOT IN (" + else "a.origin = ? AND a.source_path = ? AND a.source_index = ? AND a.artifact_kind NOT IN (" + ", ".join("?" for _ in RAW_FAILURE_EVIDENCE_KINDS) + ")" ) @@ -1292,13 +1292,50 @@ def upsert_raw_artifact( with conn if manage_transaction else nullcontext(): existing = conn.execute( f""" - SELECT artifact_id - FROM raw_artifacts + SELECT a.artifact_id, a.raw_id, a.first_observed_at_ms, a.last_observed_at_ms, r.rowid + FROM raw_artifacts AS a + JOIN raw_sessions AS r ON r.raw_id = a.raw_id WHERE {coordinate_predicate} """, coordinate_params, ).fetchone() if existing is not None: + # One coordinate has one authority carrier. A delayed census of + # stale retained bytes must not replace a carrier observed later. + if str(existing[1]) != raw_id: + incoming_row = conn.execute( + """ + SELECT acquired_at_ms, rowid FROM blob_refs + WHERE ref_id = ? AND ref_type = 'raw_payload' + ORDER BY acquired_at_ms DESC, rowid DESC LIMIT 1 + """, + (raw_id,), + ).fetchone() + if incoming_row is None: + incoming_row = conn.execute( + "SELECT acquired_at_ms, rowid FROM raw_sessions WHERE raw_id = ?", (raw_id,) + ).fetchone() + if incoming_row is None: + raise KeyError(raw_id) + existing_observation = conn.execute( + """ + SELECT acquired_at_ms, rowid FROM blob_refs + WHERE ref_id = ? AND ref_type = 'raw_payload' + ORDER BY acquired_at_ms DESC, rowid DESC LIMIT 1 + """, + (str(existing[1]),), + ).fetchone() + existing_order = ( + (int(existing_observation[0]), int(existing_observation[1])) + if existing_observation is not None + else (int(existing[3]), int(existing[4])) + ) + if existing_order >= (int(incoming_row[0]), int(incoming_row[1])): + conn.execute( + "UPDATE raw_artifacts SET first_observed_at_ms = MIN(first_observed_at_ms, ?) WHERE artifact_id = ?", + (artifact.first_observed_at_ms, str(existing[0])), + ) + return artifact = replace(artifact, artifact_id=str(existing[0])) _insert_artifact(conn, raw_id, artifact) diff --git a/tests/unit/sources/test_revision_backfill.py b/tests/unit/sources/test_revision_backfill.py index 28cf9403f6..983d21d361 100644 --- a/tests/unit/sources/test_revision_backfill.py +++ b/tests/unit/sources/test_revision_backfill.py @@ -189,6 +189,30 @@ def reject_eager_material(_raw_id: str) -> tuple[Provider, bytes, str, RawRevisi assert [session.provider_session_id for session in sessions] == ["unknown-stream"] +def test_unknown_retained_document_replays_after_complete_payload_detection(tmp_path: Path) -> None: + """A complete ChatGPT document must retry UNKNOWN prefix detection. + + The raw is intentionally a source-only UNKNOWN ``conversations.json`` + whose first complete array item is larger than the replay detection + prefix. This drives the historical replay chokepoint against a real + archive, rather than testing the detector in isolation. + """ + initialize_active_archive_root(tmp_path) + payload = _bundle(_chatgpt_session("large-document", "x" * 9_000)) + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + archive.write_raw_payload( + provider=Provider.UNKNOWN, + payload=payload, + source_path="export/conversations.json", + acquired_at_ms=1, + ) + + backfill_historical_revision_evidence(tmp_path) + + with sqlite3.connect(tmp_path / "index.db") as conn: + assert conn.execute("SELECT session_id FROM sessions").fetchall() == [("chatgpt-export:large-document",)] + + def test_parsed_session_spill_uses_the_pinned_active_index_directory(tmp_path: Path) -> None: """Repair spill churn follows the generation being repaired, not a shadow index.""" archive_root = tmp_path / "archive" @@ -274,6 +298,37 @@ def test_parse_stream_recovery_accepts_session_evidence_at_a_declared_fact_path( assert [message.text for message in sessions[0].messages] == ["recover me", "recovered"] +def test_backfill_scans_declared_stream_past_non_session_prefix(tmp_path: Path) -> None: + """Later Claude records outrank an arbitrarily long fact-artifact prefix. + + The production backfill route must not turn the first 64 non-session + records into permanent artifact authority when later records prove a + session. The archive assertion fails if replay rejects that bounded + prefix before parsing the rest of the retained JSONL. + """ + initialize_active_archive_root(tmp_path) + source_path = str(tmp_path / ".claude" / "projects" / "proj" / "subagents" / "workflows" / "wf" / "journal.jsonl") + payload = _relationship_index_jsonl_bytes(64) + ( + b'{"parentUuid":null,"type":"user","sessionId":"late-session","message":{"role":"user","content":"late evidence"},' + b'"uuid":"late-user","timestamp":"2025-01-01T00:00:00Z"}\n' + b'{"parentUuid":"late-user","type":"assistant","sessionId":"late-session","message":{"role":"assistant",' + b'"content":[{"type":"text","text":"late reply"}]},"uuid":"late-assistant",' + b'"timestamp":"2025-01-01T00:00:01Z"}\n' + ) + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + archive.write_raw_payload( + provider=Provider.CLAUDE_CODE, + payload=payload, + source_path=source_path, + acquired_at_ms=1, + ) + + backfill_historical_revision_evidence(tmp_path) + + with sqlite3.connect(tmp_path / "index.db") as conn: + assert conn.execute("SELECT session_id FROM sessions").fetchall() == [("claude-code-session:late-session",)] + + def _relationship_index_jsonl_bytes(count: int = 8) -> bytes: """Bytes shaped like the real sinex analysis artifact from polylogue-9ykn (gvgi): a graph-edge index sitting under a watched Claude Code directory, @@ -434,6 +489,388 @@ def test_historical_backfill_streams_codex_raw_without_eager_blob_read( assert result.replayed_logical_sources == 1 +def _codex_thread_state_snapshot_bytes(tmp_path: Path, title: str) -> bytes: + state_path = tmp_path / f"{title}.sqlite" + with sqlite3.connect(state_path) as conn: + conn.executescript( + """ + CREATE TABLE threads ( + id TEXT PRIMARY KEY, title TEXT, cwd TEXT, created_at_ms INTEGER, + updated_at_ms INTEGER, source TEXT, model TEXT, agent_nickname TEXT, + agent_role TEXT, archived INTEGER + ); + CREATE TABLE thread_spawn_edges ( + parent_thread_id TEXT, child_thread_id TEXT, status TEXT + ); + """ + ) + conn.execute( + "INSERT INTO threads VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ("codex-state-thread", title, "/work", 1, 1, "cli", None, None, None, 0), + ) + conn.commit() + return state_path.read_bytes() + + +def test_backfill_replays_codex_state_by_latest_raw_observation(tmp_path: Path) -> None: + """A retained A -> B -> A state sequence leaves A's title current. + + Reacquiring A reuses its content-derived raw id, so this proves replay + orders its snapshot application by the latest durable raw-payload receipt, + not ``raw_sessions.acquired_at_ms`` from A's first observation. + """ + initialize_active_archive_root(tmp_path) + source_path = str(tmp_path / "codex" / "state_5.sqlite") + snapshot_a = _codex_thread_state_snapshot_bytes(tmp_path, "title A") + snapshot_b = _codex_thread_state_snapshot_bytes(tmp_path, "title B") + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + archive.write_raw_payload( + provider=Provider.CODEX, payload=snapshot_a, source_path=source_path, acquired_at_ms=1 + ) + archive.write_raw_payload( + provider=Provider.CODEX, payload=snapshot_b, source_path=source_path, acquired_at_ms=1 + ) + archive.write_raw_payload( + provider=Provider.CODEX, payload=snapshot_a, source_path=source_path, acquired_at_ms=1 + ) + + census_historical_revision_evidence(tmp_path) + + with sqlite3.connect(tmp_path / "source.db") as conn: + payload_json = conn.execute( + "SELECT payload_json FROM raw_hook_events WHERE hook_event_id = 'codex-thread-title:codex-state-thread'" + ).fetchone() + assert payload_json is not None + assert json.loads(str(payload_json[0]))["title"] == "title A" + + +def test_backfill_replays_equal_time_codex_state_by_raw_acquisition_order(tmp_path: Path) -> None: + """Equal-time Codex snapshots retain the later raw insertion as authority.""" + initialize_active_archive_root(tmp_path) + source_path = str(tmp_path / "codex" / "state_5.sqlite") + older_snapshot = _codex_thread_state_snapshot_bytes(tmp_path, "older title") + newer_snapshot = _codex_thread_state_snapshot_bytes(tmp_path, "newer title") + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + archive.write_raw_payload( + provider=Provider.CODEX, + payload=older_snapshot, + source_path=source_path, + acquired_at_ms=1, + raw_id="z-older-state", + ) + archive.write_raw_payload( + provider=Provider.CODEX, + payload=newer_snapshot, + source_path=source_path, + acquired_at_ms=1, + raw_id="a-newer-state", + ) + + census_historical_revision_evidence(tmp_path) + + with sqlite3.connect(tmp_path / "source.db") as conn: + payload_json = conn.execute( + "SELECT payload_json FROM raw_hook_events WHERE hook_event_id = 'codex-thread-title:codex-state-thread'" + ).fetchone() + assert payload_json is not None + assert json.loads(str(payload_json[0]))["title"] == "newer title" + + +def test_backfill_terminalizes_source_only_declared_artifact(tmp_path: Path) -> None: + """Replay turns a decoded fact-sidecar raw into terminal source authority. + + This exercises the same retained-raw replay path as recovery: the + source-only raw starts pending, the parser confirms it is a workflow + artifact, and the source tier must retain both typed artifact evidence and + a successful parse receipt so it is not selected forever. + """ + initialize_active_archive_root(tmp_path) + source_path = str(tmp_path / ".claude" / "projects" / "proj" / "subagents" / "workflows" / "wf" / "journal.jsonl") + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.CLAUDE_CODE, + payload=b'{"contentKey":"workflow-artifact","agentId":"agent"}\n', + source_path=source_path, + acquired_at_ms=1, + ) + + backfill_historical_revision_evidence(tmp_path) + + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute( + "SELECT parsed_at_ms IS NOT NULL FROM raw_sessions WHERE raw_id = ?", (raw_id,) + ).fetchone() == (1,) + assert conn.execute("SELECT parse_as_session FROM raw_artifacts WHERE raw_id = ?", (raw_id,)).fetchone() == (0,) + assert conn.execute("SELECT status FROM raw_membership_census WHERE raw_id = ?", (raw_id,)).fetchone() == ( + "non_session", + ) + assert conn.execute( + "SELECT status, logical_keys_json FROM raw_authority_parser_census WHERE raw_id = ?", (raw_id,) + ).fetchone() == ("complete", "[]") + + +def test_backfill_terminalizes_detected_unknown_empty_artifact(tmp_path: Path) -> None: + """Detected provider evidence must survive an empty retained replay.""" + initialize_active_archive_root(tmp_path) + source_path = str(tmp_path / ".claude" / "projects" / "proj" / "subagents" / "workflows" / "wf" / "journal.jsonl") + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.UNKNOWN, + payload=( + b'{"type":"file-history-snapshot","messageId":"history-message",' + b'"sessionId":"history-only-session","snapshot":{"trackedFileBackups":{}}}\n' + ), + source_path=source_path, + acquired_at_ms=1, + ) + + backfill_historical_revision_evidence(tmp_path) + + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute( + "SELECT origin, parsed_at_ms IS NOT NULL FROM raw_sessions WHERE raw_id = ?", (raw_id,) + ).fetchone() == ( + "claude-code-session", + 1, + ) + assert conn.execute("SELECT parse_as_session FROM raw_artifacts WHERE raw_id = ?", (raw_id,)).fetchone() == (0,) + assert conn.execute( + "SELECT status, logical_keys_json FROM raw_authority_parser_census WHERE raw_id = ?", (raw_id,) + ).fetchone() == ("complete", "[]") + + +def test_backfill_persists_detected_provider_for_empty_ordinary_session_path(tmp_path: Path) -> None: + """Provider detection survives even when a session path is not terminalized.""" + initialize_active_archive_root(tmp_path) + source_path = str(tmp_path / ".claude" / "projects" / "proj" / "history-only-session.jsonl") + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.UNKNOWN, + payload=( + b'{"type":"file-history-snapshot","messageId":"history-message",' + b'"sessionId":"history-only-session","snapshot":{"trackedFileBackups":{}}}\n' + ), + source_path=source_path, + acquired_at_ms=1, + ) + + census_historical_revision_evidence(tmp_path) + + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute( + "SELECT origin, parsed_at_ms IS NOT NULL FROM raw_sessions WHERE raw_id = ?", (raw_id,) + ).fetchone() == ("claude-code-session", 1) + assert conn.execute("SELECT COUNT(*) FROM raw_artifacts WHERE raw_id = ?", (raw_id,)).fetchone() == (0,) + assert conn.execute("SELECT status FROM raw_membership_census WHERE raw_id = ?", (raw_id,)).fetchone() == ( + "non_session", + ) + + with ArchiveStore.open_existing(tmp_path, read_only=True) as archive: + assert archive.raw_membership_census_rows([raw_id])[0][2] + + +def test_backfill_leaves_undetected_empty_raw_replayable(tmp_path: Path) -> None: + """An unknown shape is not terminal merely because it produced no sessions.""" + initialize_active_archive_root(tmp_path) + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.UNKNOWN, + payload=b'{"future_provider_shape":true}\n', + source_path=str(tmp_path / "future.jsonl"), + acquired_at_ms=1, + ) + + census_historical_revision_evidence(tmp_path) + + with ArchiveStore.open_existing(tmp_path, read_only=True) as archive: + assert not archive.raw_membership_census_rows([raw_id])[0][2] + + +def test_backfill_retires_stale_revision_governance_for_empty_replay(tmp_path: Path) -> None: + """A current zero-session parse cannot remain in a stale full-revision plan.""" + initialize_active_archive_root(tmp_path) + source_path = str(tmp_path / ".claude" / "projects" / "proj" / "history-only-session.jsonl") + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.CLAUDE_CODE, + payload=b'{"type":"file-history-snapshot","sessionId":"history-only","snapshot":{}}\n', + source_path=source_path, + acquired_at_ms=1, + ) + archive.bind_raw_revision( + raw_id, + RawRevisionEnvelope( + logical_source_key="claude-code-session:stale-session", + kind=RawRevisionKind.FULL, + source_revision=raw_id, + acquisition_generation=0, + authority=RawRevisionAuthority.QUARANTINED, + ), + ) + + census_historical_revision_evidence(tmp_path) + + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute( + "SELECT logical_source_key, revision_kind, revision_authority FROM raw_sessions WHERE raw_id = ?", (raw_id,) + ).fetchone() == (None, "unknown", "quarantined") + + +def test_terminal_artifact_receipts_roll_back_together(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """A failed terminal census cannot expose only its artifact carrier.""" + initialize_active_archive_root(tmp_path) + source_path = str(tmp_path / ".claude" / "projects" / "proj" / "subagents" / "workflows" / "wf" / "journal.jsonl") + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.CLAUDE_CODE, + payload=b'{"contentKey":"workflow-artifact","agentId":"agent"}\n', + source_path=source_path, + acquired_at_ms=1, + ) + + def fail_census(*_args: object, **_kwargs: object) -> None: + raise RuntimeError("injected terminal census failure") + + monkeypatch.setattr(ArchiveStore, "replace_raw_membership_census", fail_census) + with pytest.raises(RuntimeError, match="injected terminal census failure"): + backfill_historical_revision_evidence(tmp_path) + + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT COUNT(*) FROM raw_artifacts WHERE raw_id = ?", (raw_id,)).fetchone() == (0,) + assert conn.execute("SELECT parsed_at_ms FROM raw_sessions WHERE raw_id = ?", (raw_id,)).fetchone() == (None,) + assert conn.execute( + "SELECT COUNT(*) FROM raw_authority_parser_census WHERE raw_id = ?", (raw_id,) + ).fetchone() == (0,) + + +def test_batched_terminal_artifact_receipts_roll_back_together(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Batched empty outcomes retain one transaction through their batch boundary.""" + initialize_active_archive_root(tmp_path) + raw_ids: list[str] = [] + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + for index in range(2): + raw_ids.append( + archive.write_raw_payload( + provider=Provider.CLAUDE_CODE, + payload=f'{{"contentKey":"workflow-{index}","agentId":"agent"}}\n'.encode(), + source_path=str( + tmp_path + / ".claude" + / "projects" + / "proj" + / "subagents" + / "workflows" + / f"wf-{index}" + / "journal.jsonl" + ), + acquired_at_ms=index + 1, + ) + ) + + original_replace = ArchiveStore.replace_raw_membership_census + calls = 0 + + def fail_second_census( + self: ArchiveStore, + raw_id: str, + sessions: list[ParsedSession] | None, + *, + parser_fingerprint: str, + censused_at_ms: int, + detail: str = "", + retire_full_revision_governance: bool = False, + manage_transaction: bool = True, + ) -> None: + nonlocal calls + calls += 1 + if calls == 2: + raise RuntimeError("injected second terminal census failure") + original_replace( + self, + raw_id, + sessions, + parser_fingerprint=parser_fingerprint, + censused_at_ms=censused_at_ms, + detail=detail, + retire_full_revision_governance=retire_full_revision_governance, + manage_transaction=manage_transaction, + ) + + monkeypatch.setattr(ArchiveStore, "replace_raw_membership_census", fail_second_census) + with pytest.raises(RuntimeError, match="injected second terminal census failure"): + census_historical_revision_evidence(tmp_path, commit_batch_size=2) + + with sqlite3.connect(tmp_path / "source.db") as conn: + placeholders = ", ".join("?" for _ in raw_ids) + assert conn.execute( + f"SELECT COUNT(*) FROM raw_artifacts WHERE raw_id IN ({placeholders})", raw_ids + ).fetchone() == (0,) + assert conn.execute( + f"SELECT COUNT(*) FROM raw_authority_parser_census WHERE raw_id IN ({placeholders})", raw_ids + ).fetchone() == (0,) + + +def test_backfill_preserves_latest_terminal_artifact_observation(tmp_path: Path) -> None: + """A delayed older replay cannot replace a newer coordinate carrier.""" + initialize_active_archive_root(tmp_path) + source_path = str(tmp_path / ".claude" / "projects" / "proj" / "subagents" / "workflows" / "wf" / "journal.jsonl") + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + older_raw_id = archive.write_raw_payload( + provider=Provider.CLAUDE_CODE, + payload=b'{"contentKey":"workflow-artifact","agentId":"old"}\n', + source_path=source_path, + acquired_at_ms=1, + raw_id="z-older-artifact", + ) + newer_raw_id = archive.write_raw_payload( + provider=Provider.CLAUDE_CODE, + payload=b'{"contentKey":"workflow-artifact","agentId":"new"}\n', + source_path=source_path, + acquired_at_ms=2, + raw_id="a-newer-artifact", + ) + + backfill_historical_revision_evidence(tmp_path) + + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT raw_id, last_observed_at_ms FROM raw_artifacts").fetchone() == (newer_raw_id, 2) + assert older_raw_id > newer_raw_id + assert conn.execute( + "SELECT COUNT(*) FROM raw_sessions WHERE raw_id IN (?, ?) AND parsed_at_ms IS NOT NULL", + (older_raw_id, newer_raw_id), + ).fetchone() == (2,) + + with ArchiveStore.open_existing(tmp_path, read_only=True) as archive: + assert all(row[2] for row in archive.raw_membership_census_rows([older_raw_id, newer_raw_id])) + + +def test_backfill_uses_raw_observation_order_for_equal_time_artifacts(tmp_path: Path) -> None: + """Equal observation times use the durable raw insertion order, not raw-id order.""" + initialize_active_archive_root(tmp_path) + source_path = str(tmp_path / ".claude" / "projects" / "proj" / "subagents" / "workflows" / "wf" / "journal.jsonl") + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + older_raw_id = archive.write_raw_payload( + provider=Provider.CLAUDE_CODE, + payload=b'{"contentKey":"workflow-artifact","agentId":"old"}\n', + source_path=source_path, + acquired_at_ms=1, + raw_id="a-older-artifact", + ) + newer_raw_id = archive.write_raw_payload( + provider=Provider.CLAUDE_CODE, + payload=b'{"contentKey":"workflow-artifact","agentId":"new"}\n', + source_path=source_path, + acquired_at_ms=1, + raw_id="z-newer-artifact", + ) + + backfill_historical_revision_evidence(tmp_path) + + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT raw_id, last_observed_at_ms FROM raw_artifacts").fetchone() == (newer_raw_id, 1) + assert older_raw_id < newer_raw_id + + def test_historical_backfill_selects_prefix_newest_independent_of_acquisition_order(tmp_path: Path) -> None: initialize_active_archive_root(tmp_path) baseline = ( diff --git a/tests/unit/storage/test_archive_tiers_source_write.py b/tests/unit/storage/test_archive_tiers_source_write.py index 0cafa8418e..7223e8902f 100644 --- a/tests/unit/storage/test_archive_tiers_source_write.py +++ b/tests/unit/storage/test_archive_tiers_source_write.py @@ -276,6 +276,56 @@ def test_source_artifact_upsert_keeps_coordinate_deduplication_and_raw_failure_f assert tuple(ordinary) == ("ordinary-coordinate", raw_ids[0], "session_export") +def test_source_artifact_upsert_refreshes_current_equal_time_carrier(tmp_path: Path) -> None: + """The current raw may refine its own coordinate even at the same timestamp.""" + conn = _connect(tmp_path / "source.db") + raw_id = write_source_raw_session( + conn, + origin=Origin.CODEX_SESSION, + source_path="/tmp/current.jsonl", + source_index=0, + payload=b"current", + acquired_at_ms=1, + ) + upsert_raw_artifact( + conn, + raw_id, + ArchiveSourceArtifact( + artifact_id="deferred-current", + origin=Origin.CODEX_SESSION, + source_path="/tmp/current.jsonl", + source_index=0, + artifact_kind="deferred_cas_frontier", + classification_reason="deferred", + support_status=ArtifactSupportStatus.PARTIAL_DECODE, + ), + ) + upsert_raw_artifact( + conn, + raw_id, + ArchiveSourceArtifact( + artifact_id="terminal-current", + origin=Origin.CODEX_SESSION, + source_path="/tmp/current.jsonl", + source_index=0, + artifact_kind="terminal_corrupt_input", + classification_reason="corrupt", + support_status=ArtifactSupportStatus.DECODE_FAILED, + ), + ) + + row = conn.execute( + "SELECT artifact_id, artifact_kind, support_status, classification_reason FROM raw_artifacts" + ).fetchone() + assert row is not None + assert tuple(row) == ( + "deferred-current", + "terminal_corrupt_input", + ArtifactSupportStatus.DECODE_FAILED.value, + "corrupt", + ) + + def test_archive_tiers_source_writer_replays_hook_events_idempotently(tmp_path: Path) -> None: conn = _connect(tmp_path / "source.db") payload = b'{"kind":"session","messages":["hello"]}' diff --git a/tests/unit/storage/test_raw_retention.py b/tests/unit/storage/test_raw_retention.py index 3b6353c913..8d3712704c 100644 --- a/tests/unit/storage/test_raw_retention.py +++ b/tests/unit/storage/test_raw_retention.py @@ -38,6 +38,17 @@ def _write_blob(store: BlobStore, payload: bytes) -> tuple[str, int]: return store.write_from_bytes(payload) +def test_unavailable_frontier_preserves_empty_healthy_source_reason() -> None: + """A healthy source check must not inherit an unrelated pointer failure.""" + projection = raw_retention_mod.unknown_raw_frontier_integrity_projection( + "active index pointer unavailable", + missing_source_raw_status="healthy", + missing_source_raw_reason="", + ) + + assert projection.missing_source_raw_reason == "" + + def _ensure_archive_source_schema(conn: sqlite3.Connection) -> None: conn.execute( """CREATE TABLE raw_sessions ( @@ -2476,6 +2487,44 @@ def test_raw_frontier_integrity_projection_follows_active_index_pointer(tmp_path assert projection.available is True +def test_raw_frontier_integrity_projection_reports_malformed_active_pointer(tmp_path: Path) -> None: + """Status reads degrade to an unavailable projection when a pointer is invalid.""" + initialize_archive_database(tmp_path / "source.db", ArchiveTier.SOURCE) + initialize_archive_database(tmp_path / "ops.db", ArchiveTier.OPS) + (tmp_path / ".index-active-pointer").write_text("relative/index.db\n", encoding="utf-8") + + projection = raw_frontier_integrity_projection( + tmp_path, + {"available": True, "lost_source_evidence_count": 0}, + ) + + assert projection.available is False + assert projection.overall_status == "unknown" + assert "active index pointer" in projection.broken_head_reason + + +def test_raw_frontier_projection_retains_known_missing_source_violation_when_pointer_is_invalid(tmp_path: Path) -> None: + """An unavailable active pointer cannot erase known source-tier loss.""" + initialize_archive_database(tmp_path / "source.db", ArchiveTier.SOURCE) + initialize_archive_database(tmp_path / "ops.db", ArchiveTier.OPS) + (tmp_path / ".index-active-pointer").write_text("relative/index.db\n", encoding="utf-8") + + projection = raw_frontier_integrity_projection( + tmp_path, + { + "available": True, + "lost_source_evidence_count": 1, + "lost_source_evidence_samples": [{"session_id": "missing-session"}], + }, + ) + + assert projection.available is False + assert projection.overall_status == "violated" + assert projection.missing_source_raw_status == "violated" + assert projection.missing_source_raw_count == 1 + assert projection.missing_source_raw_samples == ({"session_id": "missing-session"},) + + @pytest.mark.parametrize("index_kind", ["missing", "malformed"]) def test_raw_frontier_integrity_snapshot_unavailable_index_tier_is_unknown_never_healthy( tmp_path: Path, diff --git a/tests/unit/storage/test_repair.py b/tests/unit/storage/test_repair.py index d202ad0f3d..feed4feeb6 100644 --- a/tests/unit/storage/test_repair.py +++ b/tests/unit/storage/test_repair.py @@ -12,7 +12,7 @@ import pytest from polylogue.config import Config -from polylogue.core.enums import ArtifactSupportStatus +from polylogue.core.enums import ArtifactSupportStatus, Provider from polylogue.core.errors import RawCASFrontierError from polylogue.core.json import json_document from polylogue.core.raw_failure_evidence import RawFailureEvidenceKind @@ -1639,6 +1639,24 @@ def test_raw_materialization_split_root_classifies_parsed_sidecar_from_routed_bl assert result.metrics["raw_materialization_candidate_count"] == 0.0 +def test_raw_materialization_skips_current_non_session_census(tmp_path: Path) -> None: + """A successful zero-session census settles an otherwise unknown sidecar shape.""" + from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore + + initialize_active_archive_root(tmp_path) + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.CLAUDE_CODE, + payload=b'{"type":"queue-operation","operation":"compact"}\n', + source_path=str(tmp_path / "ordinary.jsonl"), + acquired_at_ms=1, + ) + + census_historical_revision_evidence(tmp_path) + + assert raw_id not in repair_mod._raw_materialization_candidate_ids(_config(tmp_path)).raw_ids + + def test_superseded_raw_cleanup_protects_split_index_referenced_raw_ids(tmp_path: Path) -> None: config = _config(tmp_path) initialize_archive_database(tmp_path / "source.db", ArchiveTier.SOURCE) From 79a5b2db94e94b61716e43b1ad16b05fee163932 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 16:30:07 +0200 Subject: [PATCH 37/65] fix: retain source-only append authority Problem: derived-only append acquisition returned before preserving the append revision envelope, leaving replay without the resolved session identity and byte-contiguous parent. What changed: bind the existing APPEND envelope before source-only return and assert the production degraded route records the canonical key, parent, offsets, and authority. Ref #3952. --- polylogue/sources/live/append_ingest.py | 26 +++++++------------ tests/unit/sources/test_live_batch_support.py | 19 ++++++++++++++ 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/polylogue/sources/live/append_ingest.py b/polylogue/sources/live/append_ingest.py index 1ef5720512..8574526ed4 100644 --- a/polylogue/sources/live/append_ingest.py +++ b/polylogue/sources/live/append_ingest.py @@ -191,22 +191,6 @@ def _ingest_append_plans_archive( post_parse=True, ) _add_timing(timings, "append.source_raw_write", t0) - degraded = degraded_reason() - if degraded is not None and degraded.derived_only: - # polylogue-gbs02: the derived tier (index.db/ - # embeddings.db) is behind the running code, but - # source.db just durably got this append range -- - # stop here, before parsing or touching the stale - # derived tier. Treat as succeeded (not deferred): - # the acquire itself genuinely completed, so the - # cursor should advance normally rather than - # re-reading the same bytes on every tick. The raw - # row sits with parsed_at_ms=NULL exactly like any - # other not-yet-materialized raw, and ordinary - # convergence picks it up once the derived tier is - # current again -- no special resolution needed. - succeeded.append(plan) - continue t0 = time.perf_counter() # polylogue-u19l: prefer the resolved provider session # identity over the bare filename stem. For Codex this is @@ -283,6 +267,16 @@ def _ingest_append_plans_archive( authority=authority, ), ) + degraded = degraded_reason() + if degraded is not None and degraded.derived_only: + # Source-only acquisition must preserve the append + # chain before it returns. The delta usually has no + # session_meta record of its own, so replay needs this + # resolved session identity, predecessor, and byte + # offsets instead of falling back to the filename + # stem as a synthetic full revision. + succeeded.append(plan) + continue if authority is RawRevisionAuthority.QUARANTINED: deferred.append(plan) continue diff --git a/tests/unit/sources/test_live_batch_support.py b/tests/unit/sources/test_live_batch_support.py index d162dc8b01..2ea38cdb6a 100644 --- a/tests/unit/sources/test_live_batch_support.py +++ b/tests/unit/sources/test_live_batch_support.py @@ -292,6 +292,25 @@ def test_live_append_acquires_with_unreadable_active_pointer(tmp_path: Path) -> assert result.succeeded == [plan] assert result.failed == [] + with sqlite3.connect(tmp_path / "source.db") as conn: + append_row = conn.execute( + """ + SELECT logical_source_key, revision_kind, predecessor_raw_id, + baseline_raw_id, append_start_offset, append_end_offset, + revision_authority + FROM raw_sessions + WHERE source_index = -1 + """ + ).fetchone() + assert append_row is not None + assert append_row[:2] == ("codex:degraded-append", "append") + assert append_row[2] is not None + assert append_row[3] is not None + assert append_row[4:] == ( + plan.start_offset, + plan.last_complete_newline, + "byte_proven", + ) def test_derived_only_live_append_candidate_uses_source_acquisition(tmp_path: Path) -> None: From ade85d2dd4f20b4b62643a50f477a8f04931dcd5 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 17:07:49 +0200 Subject: [PATCH 38/65] fix: bind source-only append before parsing Problem: source-only append acquisition still required a successful parse before preserving its revision envelope, so runtime-only tails could not advance safely through a derived-tier outage. What changed: bind APPEND authority from the plan's durable identity and cursor evidence before parsing; retain the validated Claude identity as the plan sidecar hint. Ref #3952. --- polylogue/sources/live/append_ingest.py | 99 ++++++++++++------- polylogue/sources/live/batch.py | 2 +- tests/unit/sources/test_live_batch_support.py | 9 +- 3 files changed, 72 insertions(+), 38 deletions(-) diff --git a/polylogue/sources/live/append_ingest.py b/polylogue/sources/live/append_ingest.py index 8574526ed4..fb92962179 100644 --- a/polylogue/sources/live/append_ingest.py +++ b/polylogue/sources/live/append_ingest.py @@ -41,6 +41,48 @@ class _AppendIngestOwner(Protocol): _polylogue: Any +def _bind_append_revision( + archive: Any, + raw_id: str, + *, + provider: Provider, + session_id: str, + plan: _AppendPlan, +) -> tuple[str, RawRevisionAuthority]: + """Persist an APPEND envelope from the append plan's durable identity.""" + if plan.cursor_fingerprint is None: + raise ValueError("append payload did not prove cursor identity") + logical_source_key = f"{provider.value}:{session_id}" + parent = archive.raw_append_revision_parent( + logical_source_key, + plan.start_offset, + plan.cursor_fingerprint, + ) + predecessor_raw_id: str | None = None + baseline_raw_id: str | None = None + generation = archive.raw_full_revision_generation(logical_source_key) + authority = RawRevisionAuthority.QUARANTINED + if parent is not None: + predecessor_raw_id, baseline_raw_id, generation = parent + authority = RawRevisionAuthority.BYTE_PROVEN + archive.bind_raw_revision( + raw_id, + RawRevisionEnvelope( + logical_source_key=logical_source_key, + kind=RawRevisionKind.APPEND, + source_revision=append_source_revision(plan.cursor_fingerprint, plan.payload_hash), + acquisition_generation=generation, + predecessor_source_revision=plan.cursor_fingerprint, + predecessor_raw_id=predecessor_raw_id, + baseline_raw_id=baseline_raw_id, + append_start_offset=plan.start_offset, + append_end_offset=plan.last_complete_newline, + authority=authority, + ), + ) + return logical_source_key, authority + + def reset_transient_raw_parse_state( archive: Any, raw_id: str, @@ -191,6 +233,22 @@ def _ingest_append_plans_archive( post_parse=True, ) _add_timing(timings, "append.source_raw_write", t0) + degraded = degraded_reason() + if degraded is not None and degraded.derived_only: + if plan.native_id_hint is None: + raise ValueError("source-only append has no durable session identity") + _bind_append_revision( + archive, + raw_id, + provider=provider, + session_id=plan.native_id_hint, + plan=plan, + ) + # Keep the byte-contiguous append chain replayable + # even when this delta is runtime-only or cannot be + # parsed while the derived tier is unavailable. + succeeded.append(plan) + continue t0 = time.perf_counter() # polylogue-u19l: prefer the resolved provider session # identity over the bare filename stem. For Codex this is @@ -239,44 +297,13 @@ def _ingest_append_plans_archive( failed.append(plan) continue session = sessions[0] - logical_source_key = f"{provider.value}:{session.provider_session_id}" - parent = archive.raw_append_revision_parent( - logical_source_key, - plan.start_offset, - plan.cursor_fingerprint, - ) - predecessor_raw_id: str | None = None - baseline_raw_id: str | None = None - generation = archive.raw_full_revision_generation(logical_source_key) - authority = RawRevisionAuthority.QUARANTINED - if parent is not None: - predecessor_raw_id, baseline_raw_id, generation = parent - authority = RawRevisionAuthority.BYTE_PROVEN - archive.bind_raw_revision( + logical_source_key, authority = _bind_append_revision( + archive, raw_id, - RawRevisionEnvelope( - logical_source_key=logical_source_key, - kind=RawRevisionKind.APPEND, - source_revision=append_source_revision(plan.cursor_fingerprint, plan.payload_hash), - acquisition_generation=generation, - predecessor_source_revision=plan.cursor_fingerprint, - predecessor_raw_id=predecessor_raw_id, - baseline_raw_id=baseline_raw_id, - append_start_offset=plan.start_offset, - append_end_offset=plan.last_complete_newline, - authority=authority, - ), + provider=provider, + session_id=session.provider_session_id, + plan=plan, ) - degraded = degraded_reason() - if degraded is not None and degraded.derived_only: - # Source-only acquisition must preserve the append - # chain before it returns. The delta usually has no - # session_meta record of its own, so replay needs this - # resolved session identity, predecessor, and byte - # offsets instead of falling back to the filename - # stem as a synthetic full revision. - succeeded.append(plan) - continue if authority is RawRevisionAuthority.QUARANTINED: deferred.append(plan) continue diff --git a/polylogue/sources/live/batch.py b/polylogue/sources/live/batch.py index eb6056d836..fdb8603439 100644 --- a/polylogue/sources/live/batch.py +++ b/polylogue/sources/live/batch.py @@ -3811,7 +3811,7 @@ def _append_payload_for_provider( path, payload, existing_id=identity ): return None - return payload, None + return payload, identity def _existing_provider_session_id(self, path: Path, *, expected_origin: str) -> str | None: identity = self._existing_archive_session_native_id(path, expected_origin=expected_origin) diff --git a/tests/unit/sources/test_live_batch_support.py b/tests/unit/sources/test_live_batch_support.py index 2ea38cdb6a..18010f7d73 100644 --- a/tests/unit/sources/test_live_batch_support.py +++ b/tests/unit/sources/test_live_batch_support.py @@ -273,7 +273,10 @@ def test_live_append_replay_streams_retained_jsonl_raw( assert result.failed == [] -def test_live_append_acquires_with_unreadable_active_pointer(tmp_path: Path) -> None: +def test_live_append_acquires_with_unreadable_active_pointer( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: from polylogue.core.degraded import DegradedReason, clear_degraded, set_degraded _path, plan, owner, _processor = _seed_live_append_plan(tmp_path, native_id="degraded-append") @@ -285,6 +288,10 @@ def test_live_append_acquires_with_unreadable_active_pointer(tmp_path: Path) -> derived_only=True, ) ) + monkeypatch.setattr( + "polylogue.sources.dispatch.parse_stream_payload", + lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("source-only append must not parse")), + ) try: result = ingest_append_plans(cast(Any, owner), [plan]) finally: From b90b6f699bdc8219ecf7877c62ba20bf200b7c73 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 17:38:14 +0200 Subject: [PATCH 39/65] fix: preserve source-only append authority Problem: source-only append capture could classify terminal artifacts before binding their APPEND envelope, advance quarantined plans, and rekey retried Claude append bytes after an upgrade. What changed: bind source-only plans before classification, defer quarantined authority, and separate logical session identity from migration-stable raw acquisition identity. Add real-route regressions for all three failure modes. Ref #3952. --- polylogue/sources/live/append_ingest.py | 79 +++++---- polylogue/sources/live/batch.py | 19 ++- polylogue/sources/live/batch_support.py | 14 +- tests/unit/sources/test_live_batch_support.py | 155 ++++++++++++++++++ 4 files changed, 224 insertions(+), 43 deletions(-) diff --git a/polylogue/sources/live/append_ingest.py b/polylogue/sources/live/append_ingest.py index fb92962179..241b51262e 100644 --- a/polylogue/sources/live/append_ingest.py +++ b/polylogue/sources/live/append_ingest.py @@ -7,7 +7,7 @@ from datetime import UTC, datetime from io import BytesIO from pathlib import Path -from typing import Any, Protocol +from typing import Any, Protocol, cast from polylogue.archive.artifact_taxonomy import classify_artifact, classify_artifact_path from polylogue.archive.raw_payload.decode import _sample_jsonl_payload_with_detail, jsonl_session_artifact @@ -83,6 +83,28 @@ def _bind_append_revision( return logical_source_key, authority +def _write_append_raw_payload( + archive: Any, + *, + provider: Provider, + plan: _AppendPlan, + acquired_at_ms: int, +) -> str: + """Capture literal append bytes with their migration-stable raw identity.""" + return cast( + str, + archive.write_raw_payload( + provider=provider, + payload=plan.payload, + source_path=str(plan.path), + source_index=-1, + acquired_at_ms=acquired_at_ms, + native_id=plan.acquisition_native_id_hint, + post_parse=True, + ), + ) + + def reset_transient_raw_parse_state( archive: Any, raw_id: str, @@ -156,6 +178,30 @@ def _ingest_append_plans_archive( session_artifact = None try: provider = Provider.from_string(plan.source_name) + degraded = degraded_reason() + if degraded is not None and degraded.derived_only: + if plan.native_id_hint is None: + raise ValueError("source-only append has no durable session identity") + t0 = time.perf_counter() + raw_id = _write_append_raw_payload( + archive, + provider=provider, + plan=plan, + acquired_at_ms=acquired_at_ms, + ) + _add_timing(timings, "append.source_raw_write", t0) + _logical_source_key, authority = _bind_append_revision( + archive, + raw_id, + provider=provider, + session_id=plan.native_id_hint, + plan=plan, + ) + if authority is RawRevisionAuthority.QUARANTINED: + deferred.append(plan) + else: + succeeded.append(plan) + continue path_artifact = classify_artifact_path( str(plan.path), provider=provider, @@ -217,38 +263,13 @@ def _ingest_append_plans_archive( succeeded.append(plan) continue t0 = time.perf_counter() - raw_id = archive.write_raw_payload( + raw_id = _write_append_raw_payload( + archive, provider=provider, - payload=plan.payload, - source_path=str(plan.path), - source_index=-1, + plan=plan, acquired_at_ms=acquired_at_ms, - # polylogue-u19l: persist the resolved provider - # session identity as sidecar metadata instead of - # splicing a synthetic session_meta record into the - # hashed/stored payload (batch.py's - # _append_payload_for_provider), so the stored blob - # stays a literal slice of the live file. - native_id=plan.native_id_hint, - post_parse=True, ) _add_timing(timings, "append.source_raw_write", t0) - degraded = degraded_reason() - if degraded is not None and degraded.derived_only: - if plan.native_id_hint is None: - raise ValueError("source-only append has no durable session identity") - _bind_append_revision( - archive, - raw_id, - provider=provider, - session_id=plan.native_id_hint, - plan=plan, - ) - # Keep the byte-contiguous append chain replayable - # even when this delta is runtime-only or cannot be - # parsed while the derived tier is unavailable. - succeeded.append(plan) - continue t0 = time.perf_counter() # polylogue-u19l: prefer the resolved provider session # identity over the bare filename stem. For Codex this is diff --git a/polylogue/sources/live/batch.py b/polylogue/sources/live/batch.py index fdb8603439..85df5b2523 100644 --- a/polylogue/sources/live/batch.py +++ b/polylogue/sources/live/batch.py @@ -3721,7 +3721,7 @@ def _append_plan(self, path: Path, *, cursor: CursorRecord | None = None) -> _Ap append_result = self._append_payload_for_provider(path, self._source_name_for(path), complete_payload) if append_result is None: return None - append_payload, native_id_hint = append_result + append_payload, native_id_hint, acquisition_native_id_hint = append_result tail_hash = sha256(complete_payload).hexdigest() return _AppendPlan( path=path, @@ -3741,12 +3741,13 @@ def _append_plan(self, path: Path, *, cursor: CursorRecord | None = None) -> _Ap accepted_prefix_hash=accepted_prefix_hash, authority_bytes_read=last_complete_newline, native_id_hint=native_id_hint, + acquisition_native_id_hint=acquisition_native_id_hint, ) def _append_payload_for_provider( self, path: Path, source_name: str, payload: bytes - ) -> tuple[bytes, str | None] | None: - """Return the literal append payload plus an optional identity hint. + ) -> tuple[bytes, str | None, str | None] | None: + """Return literal bytes plus logical and acquisition identity hints. polylogue-u19l: this used to prepend a synthetic ``session_meta`` line ahead of ``payload`` for Codex before hashing/storing it, so the @@ -3759,8 +3760,8 @@ def _append_payload_for_provider( Now the identity is resolved here exactly as before, but returned as a sidecar hint instead of being spliced into the hashed bytes. - Callers persist it to ``raw_sessions.native_id`` (``_AppendPlan. - native_id_hint`` -> ``append_ingest.py``) and pass it back as the + Callers persist the Codex acquisition hint to + ``raw_sessions.native_id`` and pass the logical hint back as the parser's ``fallback_id`` at replay time (``revision_backfill.parse_retained_raw_sessions``), which is exactly equivalent for Codex: ``_parse_records`` only ever falls back to @@ -3806,12 +3807,16 @@ def _append_payload_for_provider( "identity recovered from archived session / prior session_meta " "line and carried as native_id_hint, not spliced into hashed bytes", ) - return payload, identity + return payload, identity, identity if provider is Provider.CLAUDE_CODE and not self._claude_code_tail_matches_existing_identity( path, payload, existing_id=identity ): return None - return payload, identity + # Claude append raws have historically used native_id=NULL. Its own + # records carry sessionId, so the resolved identity is needed for + # governance but must not change deterministic acquisition identity + # for a retry of pre-upgrade bytes. + return payload, identity, None def _existing_provider_session_id(self, path: Path, *, expected_origin: str) -> str | None: identity = self._existing_archive_session_native_id(path, expected_origin=expected_origin) diff --git a/polylogue/sources/live/batch_support.py b/polylogue/sources/live/batch_support.py index 8e3c7579ef..bed83e626a 100644 --- a/polylogue/sources/live/batch_support.py +++ b/polylogue/sources/live/batch_support.py @@ -133,14 +133,14 @@ class _AppendPlan: ctime_ns: int | None = None accepted_prefix_hash: str | None = None authority_bytes_read: int = 0 - # polylogue-u19l: the resolved provider session identity for this append, - # when the provider's own record stream cannot self-describe it (Codex - # append deltas have no ``session_meta`` record of their own). Carried as - # sidecar metadata -- persisted to ``raw_sessions.native_id`` and used to - # override the replay ``fallback_id`` -- instead of being injected into - # the hashed/stored payload bytes, so the stored blob stays a literal - # slice of the live file. ``None`` for providers/plans that don't need it. + # The resolved logical session identity used to bind this append and as a + # parser fallback when its own record stream cannot self-describe it. native_id_hint: str | None = None + # Acquisition identity is deliberately separate from logical identity. + # Codex append rows introduced this sidecar together with literal delta + # bytes. Claude append rows predate it with native_id=NULL, so retaining + # NULL keeps deterministic raw IDs stable across upgrades and retries. + acquisition_native_id_hint: str | None = None @dataclass(frozen=True, slots=True) diff --git a/tests/unit/sources/test_live_batch_support.py b/tests/unit/sources/test_live_batch_support.py index 18010f7d73..a27ba162d9 100644 --- a/tests/unit/sources/test_live_batch_support.py +++ b/tests/unit/sources/test_live_batch_support.py @@ -253,6 +253,36 @@ def _seed_live_append_plan( return path, plan, _append_owner(archive_root), processor +def _seed_claude_live_append_plan( + archive_root: Path, + *, + native_id: str, + append: bytes, +) -> tuple[Path, _AppendPlan, object, LiveBatchProcessor]: + root = archive_root / "claude-projects" + root.mkdir() + path = root / f"{native_id}.jsonl" + baseline = ( + f'{{"parentUuid":null,"type":"user","message":{{"role":"user","content":"zero"}},' + f'"uuid":"message-0","timestamp":"2026-06-02T00:00:00Z","sessionId":"{native_id}"}}\n' + ).encode() + path.write_bytes(baseline) + index_db = archive_root / "index.db" + processor = LiveBatchProcessor( + cast(Any, SimpleNamespace(archive_root=archive_root, backend=SimpleNamespace(db_path=index_db))), + (WatchSource(name="claude-code", root=root),), + cursor=CursorStore(index_db), + parser_fingerprint="test-parser", + ) + seeded = asyncio.run(processor.ingest_files([path], emit_event=False)) + assert seeded.succeeded_file_count == 1 + with path.open("ab") as handle: + handle.write(append) + plan = processor._append_plan(path) + assert isinstance(plan, _AppendPlan) + return path, plan, _append_owner(archive_root), processor + + def test_live_append_replay_streams_retained_jsonl_raw( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -320,6 +350,131 @@ def test_live_append_acquires_with_unreadable_active_pointer( ) +def test_source_only_file_history_append_binds_before_artifact_classification(tmp_path: Path) -> None: + from polylogue.core.degraded import DegradedReason, clear_degraded, set_degraded + + native_id = "source-only-history" + append = ( + f'{{"type":"file-history-snapshot","sessionId":"{native_id}",' + '"uuid":"history-1","snapshot":{},"timestamp":"2026-06-02T00:00:01Z"}\n' + ).encode() + _path, plan, owner, _processor = _seed_claude_live_append_plan( + tmp_path, + native_id=native_id, + append=append, + ) + assert plan.native_id_hint == native_id + assert plan.acquisition_native_id_hint is None + set_degraded( + DegradedReason( + code="schema_version_mismatch", + message="derived generation unavailable", + derived_only=True, + ) + ) + try: + result = ingest_append_plans(cast(Any, owner), [plan]) + finally: + clear_degraded() + + assert result.succeeded == [plan] + assert result.failed == [] + assert result.deferred == [] + with sqlite3.connect(tmp_path / "source.db") as conn: + append_row = conn.execute( + """ + SELECT logical_source_key, revision_kind, predecessor_raw_id, + baseline_raw_id, append_start_offset, append_end_offset, + revision_authority, native_id + FROM raw_sessions + WHERE source_index = -1 + """ + ).fetchone() + artifact_count = conn.execute("SELECT COUNT(*) FROM raw_artifacts").fetchone() + assert append_row is not None + assert append_row[:2] == (f"claude-code:{native_id}", "append") + assert append_row[2] is not None + assert append_row[3] is not None + assert append_row[4:] == ( + plan.start_offset, + plan.last_complete_newline, + "byte_proven", + None, + ) + assert artifact_count == (0,) + + +def test_source_only_quarantined_append_is_deferred(tmp_path: Path) -> None: + from polylogue.core.degraded import DegradedReason, clear_degraded, set_degraded + + path = tmp_path / "quarantined-source-only.jsonl" + payload = ( + b'{"type":"response_item","payload":{"type":"message","id":"message-1",' + b'"role":"assistant","content":[{"type":"output_text","text":"one"}]}}\n' + ) + path.write_bytes(payload) + plan = replace( + _append_plan(path, payload, payload_hash=sha256(payload).hexdigest()), + native_id_hint="quarantined-source-only", + acquisition_native_id_hint="quarantined-source-only", + ) + set_degraded( + DegradedReason( + code="schema_version_mismatch", + message="derived generation unavailable", + derived_only=True, + ) + ) + try: + result = ingest_append_plans(cast(Any, _append_owner(tmp_path)), [plan]) + finally: + clear_degraded() + + assert result.succeeded == [] + assert result.failed == [] + assert result.deferred == [plan] + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute( + "SELECT logical_source_key, revision_kind, revision_authority FROM raw_sessions" + ).fetchone() == ("codex:quarantined-source-only", "append", "quarantined") + + +def test_claude_append_retry_preserves_legacy_null_acquisition_identity(tmp_path: Path) -> None: + native_id = "claude-legacy-append" + append = ( + f'{{"parentUuid":"message-0","type":"assistant","message":{{"role":"assistant",' + f'"content":[{{"type":"text","text":"one"}}]}},"uuid":"message-1",' + f'"timestamp":"2026-06-02T00:00:01Z","sessionId":"{native_id}"}}\n' + ).encode() + _path, plan, owner, _processor = _seed_claude_live_append_plan( + tmp_path, + native_id=native_id, + append=append, + ) + assert plan.native_id_hint == native_id + assert plan.acquisition_native_id_hint is None + + legacy_plan = replace(plan, native_id_hint=None, acquisition_native_id_hint=None) + first = ingest_append_plans(cast(Any, owner), [legacy_plan]) + assert first.succeeded == [legacy_plan] + assert first.failed == [] + with sqlite3.connect(tmp_path / "source.db") as conn: + before_retry = conn.execute("SELECT raw_id, native_id FROM raw_sessions WHERE source_index = -1").fetchall() + assert len(before_retry) == 1 + assert before_retry[0][1] is None + + retry = ingest_append_plans(cast(Any, owner), [plan]) + + assert retry.succeeded == [plan] + assert retry.failed == [] + assert retry.deferred == [] + with sqlite3.connect(tmp_path / "source.db") as conn: + after_retry = conn.execute( + "SELECT raw_id, native_id, revision_authority FROM raw_sessions WHERE source_index = -1" + ).fetchall() + assert after_retry == [(before_retry[0][0], None, "byte_proven")] + + def test_derived_only_live_append_candidate_uses_source_acquisition(tmp_path: Path) -> None: """The managed batch route must not plan an index-backed append while derived-only.""" From da30080a5c2894a45f108cb8b1139cd954cb9b3a Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 18:05:56 +0200 Subject: [PATCH 40/65] fix: scan retained JSONL before terminal classification Problem: large Codex JSONL could be classified as a terminal non-session artifact from its first session_meta line, omitting the terminal raw from parser census and blocking retained revision recovery. What changed: confirm negative Claude and Codex prefix classifications with the existing memory-bounded rolling scanner. Add a large-stream regression that proves later conversational evidence remains authoritative. Ref #3952. --- polylogue/storage/artifacts/inspection.py | 62 ++++++++++++++----- .../storage/test_artifact_loss_surfacing.py | 31 +++++++++- 2 files changed, 75 insertions(+), 18 deletions(-) diff --git a/polylogue/storage/artifacts/inspection.py b/polylogue/storage/artifacts/inspection.py index 34edab172c..522be88eb7 100644 --- a/polylogue/storage/artifacts/inspection.py +++ b/polylogue/storage/artifacts/inspection.py @@ -8,8 +8,13 @@ from datetime import datetime, timezone from pathlib import Path -from polylogue.archive.artifact_taxonomy import ArtifactKind, classify_artifact_path -from polylogue.archive.raw_payload import JSONValue, RawPayloadEnvelope, build_raw_payload_envelope +from polylogue.archive.artifact_taxonomy import ArtifactClassification, ArtifactKind, classify_artifact_path +from polylogue.archive.raw_payload import ( + JSONValue, + RawPayloadEnvelope, + build_raw_payload_envelope, +) +from polylogue.archive.raw_payload.decode import jsonl_session_artifact from polylogue.core.enums import ArtifactSupportStatus, Provider from polylogue.schemas.observation import derive_bundle_scope, schema_cluster_id from polylogue.schemas.packages import SchemaResolution @@ -170,6 +175,21 @@ def _inspect_payload_envelope(record: RawSessionRecord, *, blob_store: BlobStore return envelope +def _complete_stream_session_artifact( + record: RawSessionRecord, + *, + provider: Provider, + blob_store: BlobStore, +) -> ArtifactClassification | None: + """Recover positive stream evidence hidden by bounded inspection.""" + if not _prefers_json_stream(record.source_path) or provider not in {Provider.CLAUDE_CODE, Provider.CODEX}: + return None + return jsonl_session_artifact( + blob_store.blob_path(_record_blob_ref(record)), + provider=provider, + ) + + def _sidecar_agent_type(payload: JSONValue) -> str | None: if isinstance(payload, dict): agent_type = payload.get("agentType") @@ -312,9 +332,9 @@ def _stream_loss_accounting( def inspect_raw_artifact(record: RawSessionRecord, *, blob_store: BlobStore | None = None) -> ArtifactObservationRecord: """Inspect one raw record into a durable artifact observation. - Uses only a small prefix of raw_content for classification — never - decodes the full payload. This keeps memory bounded regardless of - file size (a 1.5 GB JSONL file is classified from its first line). + Classification starts from a small prefix. If that prefix would refuse a + Claude or Codex JSONL stream, a memory-bounded rolling scan must confirm + that no later record supplies positive session evidence. """ resolved_blob_store = blob_store or get_blob_store() provider_hint = _normalize_payload_provider_hint(record) @@ -331,6 +351,16 @@ def inspect_raw_artifact(record: RawSessionRecord, *, blob_store: BlobStore | No try: envelope = _inspect_payload_envelope(record, blob_store=resolved_blob_store) payload_provider = envelope.provider + artifact = envelope.artifact + if not artifact.parse_as_session: + artifact = ( + _complete_stream_session_artifact( + record, + provider=payload_provider, + blob_store=resolved_blob_store, + ) + or artifact + ) resolution: SchemaResolution | None = None has_supported_resolution = False @@ -344,7 +374,7 @@ def inspect_raw_artifact(record: RawSessionRecord, *, blob_store: BlobStore | No blob_store=resolved_blob_store, ) - if envelope.artifact.parse_as_session and envelope.artifact.schema_eligible and malformed_jsonl_lines == 0: + if artifact.parse_as_session and artifact.schema_eligible and malformed_jsonl_lines == 0: resolution, has_supported_resolution = _resolve_payload_support( registry=registry, payload_provider=payload_provider, @@ -356,10 +386,10 @@ def inspect_raw_artifact(record: RawSessionRecord, *, blob_store: BlobStore | No resolution_reason = resolution.reason if resolution is not None else None support_status = _support_status( - parse_as_session=envelope.artifact.parse_as_session, - schema_eligible=envelope.artifact.schema_eligible, + parse_as_session=artifact.parse_as_session, + schema_eligible=artifact.schema_eligible, malformed_jsonl_lines=malformed_jsonl_lines, - artifact_kind=envelope.artifact.kind.value, + artifact_kind=artifact.kind.value, has_supported_resolution=has_supported_resolution, had_decode_error=False, partial_decode=partial_decode, @@ -374,23 +404,21 @@ def inspect_raw_artifact(record: RawSessionRecord, *, blob_store: BlobStore | No source_index=record.source_index, file_mtime=record.file_mtime, wire_format=envelope.wire_format, - artifact_kind=envelope.artifact.kind.value, - classification_reason=envelope.artifact.reason, - parse_as_session=envelope.artifact.parse_as_session, - schema_eligible=envelope.artifact.schema_eligible, + artifact_kind=artifact.kind.value, + classification_reason=artifact.reason, + parse_as_session=artifact.parse_as_session, + schema_eligible=artifact.schema_eligible, support_status=support_status, malformed_jsonl_lines=malformed_jsonl_lines, decode_error=None, bundle_scope=bundle_scope, - cohort_id=schema_cluster_id(envelope.payload, envelope.artifact.cohort), + cohort_id=schema_cluster_id(envelope.payload, artifact.cohort), resolved_package_version=resolved_package_version, resolved_element_kind=resolved_element_kind, resolution_reason=resolution_reason, link_group_key=_link_group_key(record.source_path), sidecar_agent_type=( - _sidecar_agent_type(envelope.payload) - if envelope.artifact.kind is ArtifactKind.AGENT_SIDECAR_META - else None + _sidecar_agent_type(envelope.payload) if artifact.kind is ArtifactKind.AGENT_SIDECAR_META else None ), first_observed_at=observed_at, last_observed_at=observed_at, diff --git a/tests/unit/storage/test_artifact_loss_surfacing.py b/tests/unit/storage/test_artifact_loss_surfacing.py index 5487c01c48..d90624be82 100644 --- a/tests/unit/storage/test_artifact_loss_surfacing.py +++ b/tests/unit/storage/test_artifact_loss_surfacing.py @@ -41,13 +41,14 @@ def _write_record( content: bytes, source_path: str, source_name: str = "claude-code", + provider: Provider = Provider.CLAUDE_CODE, ) -> RawSessionRecord: raw_id, blob_size = store.write_from_bytes(content) return RawSessionRecord( raw_id=raw_id, source_name=source_name, source_path=source_path, - payload_provider=Provider.CLAUDE_CODE, + payload_provider=provider, source_index=None, blob_size=blob_size, acquired_at="2026-01-01T00:00:00+00:00", @@ -103,3 +104,31 @@ def test_clean_large_jsonl_is_not_flagged(blob_store: BlobStore) -> None: ArtifactSupportStatus.PARTIAL_DECODE, ArtifactSupportStatus.DECODE_FAILED, } + + +def test_large_codex_stream_is_not_terminalized_from_session_meta_prefix(blob_store: BlobStore) -> None: + session_meta = b'{"type":"session_meta","payload":{"id":"large-codex"}}\n' + message = ( + b'{"type":"response_item","payload":{"type":"message","id":"message-1",' + b'"role":"user","content":[{"type":"input_text","text":"hello"}]}}\n' + ) + padding = ( + b'{"type":"response_item","payload":{"type":"token_count","padding":"' + + (b"x" * (_INSPECTION_PREFIX_BYTES * 2)) + + b'"}}\n' + ) + content = session_meta + message + padding + assert len(content) > _INSPECTION_PREFIX_BYTES + + record = _write_record( + blob_store, + content=content, + source_path="codex/large-session.jsonl", + source_name="codex", + provider=Provider.CODEX, + ) + observation = inspect_raw_artifact(record) + + assert observation.parse_as_session is True + assert observation.artifact_kind == "session_record_stream" + assert observation.classification_reason == "parser-supported Codex session record stream" From 95e9b797d95fc6ad5f1f85fce87cbf17b2ddd0de Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 18:23:22 +0200 Subject: [PATCH 41/65] fix: preserve stream exclusions during retained inspection Problem: retained JSONL inspection could miss session evidence when its first record exceeded the prefix, while the rolling fallback could also override definitive sidecar paths. What changed: recover decode-failed Claude and Codex streams through the complete rolling classifier and gate every rolling classification behind strong path authority. Add anti-vacuity regressions for a large first Codex record and transcript-shaped tool-result sidecar. Ref #3952. --- polylogue/storage/artifacts/inspection.py | 28 ++++++++- .../storage/test_artifact_loss_surfacing.py | 58 +++++++++++++++++-- 2 files changed, 79 insertions(+), 7 deletions(-) diff --git a/polylogue/storage/artifacts/inspection.py b/polylogue/storage/artifacts/inspection.py index 522be88eb7..7a2a820c72 100644 --- a/polylogue/storage/artifacts/inspection.py +++ b/polylogue/storage/artifacts/inspection.py @@ -8,7 +8,12 @@ from datetime import datetime, timezone from pathlib import Path -from polylogue.archive.artifact_taxonomy import ArtifactClassification, ArtifactKind, classify_artifact_path +from polylogue.archive.artifact_taxonomy import ( + ArtifactClassification, + ArtifactKind, + classify_artifact_path, + strong_path_classification, +) from polylogue.archive.raw_payload import ( JSONValue, RawPayloadEnvelope, @@ -184,6 +189,9 @@ def _complete_stream_session_artifact( """Recover positive stream evidence hidden by bounded inspection.""" if not _prefers_json_stream(record.source_path) or provider not in {Provider.CLAUDE_CODE, Provider.CODEX}: return None + path_artifact = strong_path_classification(record.source_path, provider=provider) + if path_artifact is not None and not path_artifact.parse_as_session: + return None return jsonl_session_artifact( blob_store.blob_path(_record_blob_ref(record)), provider=provider, @@ -349,7 +357,23 @@ def inspect_raw_artifact(record: RawSessionRecord, *, blob_store: BlobStore | No registry = _SCHEMA_REGISTRY try: - envelope = _inspect_payload_envelope(record, blob_store=resolved_blob_store) + try: + envelope = _inspect_payload_envelope(record, blob_store=resolved_blob_store) + except Exception: + stream_provider = Provider.from_string(provider_token) + recovered_artifact = _complete_stream_session_artifact( + record, + provider=stream_provider, + blob_store=resolved_blob_store, + ) + if recovered_artifact is None: + raise + envelope = RawPayloadEnvelope( + payload=[], + provider=stream_provider, + wire_format="jsonl", + artifact=recovered_artifact, + ) payload_provider = envelope.provider artifact = envelope.artifact if not artifact.parse_as_session: diff --git a/tests/unit/storage/test_artifact_loss_surfacing.py b/tests/unit/storage/test_artifact_loss_surfacing.py index d90624be82..f7cacee372 100644 --- a/tests/unit/storage/test_artifact_loss_surfacing.py +++ b/tests/unit/storage/test_artifact_loss_surfacing.py @@ -1,9 +1,8 @@ -"""Regression tests: artifact DECODE_FAILED/PARTIAL_DECODE covers the whole file (#1745). +"""Regression tests: artifact inspection covers the whole retained stream. -The artifact support status is derived from raw inspection. Inspection reads -only a 64 KB prefix to bound memory, so malformed JSONL content *past* the -prefix used to be invisible and the artifact was never flagged. These tests -assert that loss past the prefix is surfaced via a full-scan fallback. +Inspection starts from a 64 KB prefix to bound memory, then uses rolling stream +passes for whole-file loss accounting and positive session evidence. These +tests preserve both duties without weakening definitive sidecar exclusions. """ from __future__ import annotations @@ -132,3 +131,52 @@ def test_large_codex_stream_is_not_terminalized_from_session_meta_prefix(blob_st assert observation.parse_as_session is True assert observation.artifact_kind == "session_record_stream" assert observation.classification_reason == "parser-supported Codex session record stream" + + +def test_codex_stream_recovers_when_first_record_exceeds_inspection_prefix(blob_store: BlobStore) -> None: + session_meta = ( + b'{"type":"session_meta","payload":{"id":"large-first-record","base_instructions":{"text":"' + + (b"x" * (_INSPECTION_PREFIX_BYTES * 2)) + + b'"}}}\n' + ) + message = ( + b'{"type":"response_item","payload":{"type":"message","id":"message-1",' + b'"role":"user","content":[{"type":"input_text","text":"hello"}]}}\n' + ) + assert session_meta.find(b"\n") > _INSPECTION_PREFIX_BYTES + + record = _write_record( + blob_store, + content=session_meta + message, + source_path="codex/large-first-record.jsonl", + source_name="codex", + provider=Provider.CODEX, + ) + observation = inspect_raw_artifact(record) + + assert observation.parse_as_session is True + assert observation.artifact_kind == "session_record_stream" + assert observation.wire_format == "jsonl" + assert observation.decode_error is None + + +def test_rolling_scan_preserves_tool_result_sidecar_exclusion(blob_store: BlobStore) -> None: + content = ( + b'{"parentUuid":null,"type":"user","sessionId":"embedded",' + b'"message":{"role":"user","content":"copied transcript"},' + b'"uuid":"user-1","timestamp":"2026-01-01T00:00:00Z"}\n' + b'{"parentUuid":"user-1","type":"assistant","sessionId":"embedded",' + b'"message":{"role":"assistant","content":[{"type":"text","text":"copied reply"}]},' + b'"uuid":"assistant-1","timestamp":"2026-01-01T00:00:01Z"}\n' + ) + record = _write_record( + blob_store, + content=content, + source_path="projects/project/session/tool-results/copied-transcript.jsonl", + ) + + observation = inspect_raw_artifact(record) + + assert observation.parse_as_session is False + assert observation.schema_eligible is False + assert observation.artifact_kind == "tool_result_sidecar" From e843008b22af31eb84a25be37051a09bc862987e Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 18:49:01 +0200 Subject: [PATCH 42/65] fix: bound retained stream recovery records Problem: exact-head review found that recovery could allocate a whole giant line, discard the records proving session shape, and lose path-dependent artifact kinds. What changed: add a chunked record-cap scanner that returns its bounded evidence sample, thread source paths through classification, and persist truthful cohort/schema inputs. Add real-route regressions for bounded oversized-line recovery and subagent taxonomy. Ref #3952. --- polylogue/archive/raw_payload/decode.py | 110 ++++++++++++++++-- polylogue/storage/artifacts/inspection.py | 43 ++++--- .../storage/test_artifact_loss_surfacing.py | 72 ++++++++++++ 3 files changed, 201 insertions(+), 24 deletions(-) diff --git a/polylogue/archive/raw_payload/decode.py b/polylogue/archive/raw_payload/decode.py index 6333e65fb0..93fae108c2 100644 --- a/polylogue/archive/raw_payload/decode.py +++ b/polylogue/archive/raw_payload/decode.py @@ -3,6 +3,7 @@ from __future__ import annotations from collections import deque +from collections.abc import Iterator from dataclasses import dataclass from pathlib import Path from typing import IO, Literal, TypeAlias, cast @@ -79,6 +80,56 @@ class RawPayloadEnvelope: malformed_jsonl_detail: str | None = None +@dataclass(frozen=True) +class JSONLSessionArtifactScan: + """Bounded records that supplied a stream's positive session evidence.""" + + artifact: ArtifactClassification | None + sample: tuple[JSONValue, ...] = () + oversized_records: int = 0 + + +def _bounded_raw_lines( + stream: IO[bytes] | IO[str], + *, + max_record_bytes: int | None, +) -> Iterator[tuple[bytes | str | None, bool]]: + """Yield complete lines without allocating beyond an optional record cap. + + Oversized records are consumed in bounded chunks and represented as + ``(None, True)`` so callers can continue at the next newline. + """ + if max_record_bytes is None: + for raw_line in stream: + yield raw_line, False + return + if max_record_bytes < 1: + raise ValueError("max_record_bytes must be positive") + + read_size = max_record_bytes + 1 + while True: + raw_line = stream.readline(read_size) + if not raw_line: + return + has_newline = raw_line.endswith(b"\n") if isinstance(raw_line, bytes) else raw_line.endswith("\n") + if has_newline: + if len(raw_line) > max_record_bytes: + yield None, True + else: + yield raw_line, False + continue + if len(raw_line) <= max_record_bytes: + yield raw_line, False + return + + while raw_line: + has_newline = raw_line.endswith(b"\n") if isinstance(raw_line, bytes) else raw_line.endswith("\n") + if has_newline: + break + raw_line = stream.readline(read_size) + yield None, True + + def _decode_jsonl_payload( raw: Path | bytes | str, *, @@ -133,6 +184,7 @@ def _sample_jsonl_payload_with_detail( max_samples: int = 64, jsonl_dict_only: bool = False, scan_full: bool = True, + max_record_bytes: int | None = None, ) -> tuple[list[JSONValue], int, str | None]: """Collect a bounded sample of valid JSONL records. @@ -148,8 +200,15 @@ def _sample_jsonl_payload_with_detail( line_number = 0 with raw_line_stream(raw) as stream: - for raw_line in stream: + for raw_line, oversized in _bounded_raw_lines(stream, max_record_bytes=max_record_bytes): line_number += 1 + if oversized: + malformed_lines += 1 + if malformed_detail is None: + malformed_detail = f"line {line_number}: record exceeds inspection byte bound" + first_line = False + continue + assert raw_line is not None try: line = _decode_provider_utf8(raw_line) if isinstance(raw_line, bytes) else raw_line except UnicodeDecodeError as exc: @@ -183,22 +242,32 @@ def _sample_jsonl_payload_with_detail( return samples, malformed_lines, malformed_detail -def jsonl_session_artifact( +def scan_jsonl_session_artifact( raw: Path | bytes | str | IO[bytes] | IO[str], *, provider: Provider, jsonl_dict_only: bool = False, -) -> ArtifactClassification | None: - """Stream JSONL until one decoded record proves session eligibility. + source_path: str | Path | None = None, + max_record_bytes: int | None = None, +) -> JSONLSessionArtifactScan: + """Stream JSONL until bounded decoded records prove session eligibility. Terminal artifact admission must not let an arbitrary prefix of - non-conversational records hide a later session record. This retains a - rolling 32-record window, including for blob-backed multi-gigabyte JSONL. + non-conversational records hide a later session record. The rolling window + retains at most 32 decoded records. When ``max_record_bytes`` is supplied, + oversized records are discarded in chunks so a later record remains + inspectable without allocating the oversized line. """ records: deque[JSONValue] = deque(maxlen=32) first_line = True + oversized_records = 0 with raw_line_stream(raw) as stream: - for raw_line in stream: + for raw_line, oversized in _bounded_raw_lines(stream, max_record_bytes=max_record_bytes): + if oversized: + oversized_records += 1 + first_line = False + continue + assert raw_line is not None try: line = _decode_provider_utf8(raw_line) if isinstance(raw_line, bytes) else raw_line except UnicodeDecodeError: @@ -218,10 +287,29 @@ def jsonl_session_artifact( records.append(payload) window = list(records) for start in range(len(window)): - artifact = classify_artifact(window[start:], provider=provider) + sample = window[start:] + artifact = classify_artifact(sample, provider=provider, source_path=source_path) if artifact.parse_as_session: - return artifact - return None + return JSONLSessionArtifactScan( + artifact=artifact, + sample=tuple(sample), + oversized_records=oversized_records, + ) + return JSONLSessionArtifactScan(artifact=None, oversized_records=oversized_records) + + +def jsonl_session_artifact( + raw: Path | bytes | str | IO[bytes] | IO[str], + *, + provider: Provider, + jsonl_dict_only: bool = False, +) -> ArtifactClassification | None: + """Compatibility wrapper for callers that only need classification.""" + return scan_jsonl_session_artifact( + raw, + provider=provider, + jsonl_dict_only=jsonl_dict_only, + ).artifact def sample_jsonl_payload( @@ -482,8 +570,10 @@ def _hermes_sqlite_marker_payload( "JSONRecord", "JSONValue", "RawPayloadEnvelope", + "JSONLSessionArtifactScan", "WireFormat", "build_raw_payload_envelope", "jsonl_session_artifact", + "scan_jsonl_session_artifact", "sample_jsonl_payload", ] diff --git a/polylogue/storage/artifacts/inspection.py b/polylogue/storage/artifacts/inspection.py index 7a2a820c72..037a9dbcc3 100644 --- a/polylogue/storage/artifacts/inspection.py +++ b/polylogue/storage/artifacts/inspection.py @@ -9,7 +9,6 @@ from pathlib import Path from polylogue.archive.artifact_taxonomy import ( - ArtifactClassification, ArtifactKind, classify_artifact_path, strong_path_classification, @@ -19,7 +18,7 @@ RawPayloadEnvelope, build_raw_payload_envelope, ) -from polylogue.archive.raw_payload.decode import jsonl_session_artifact +from polylogue.archive.raw_payload.decode import JSONLSessionArtifactScan, scan_jsonl_session_artifact from polylogue.core.enums import ArtifactSupportStatus, Provider from polylogue.schemas.observation import derive_bundle_scope, schema_cluster_id from polylogue.schemas.packages import SchemaResolution @@ -185,16 +184,18 @@ def _complete_stream_session_artifact( *, provider: Provider, blob_store: BlobStore, -) -> ArtifactClassification | None: +) -> JSONLSessionArtifactScan | None: """Recover positive stream evidence hidden by bounded inspection.""" if not _prefers_json_stream(record.source_path) or provider not in {Provider.CLAUDE_CODE, Provider.CODEX}: return None path_artifact = strong_path_classification(record.source_path, provider=provider) if path_artifact is not None and not path_artifact.parse_as_session: return None - return jsonl_session_artifact( + return scan_jsonl_session_artifact( blob_store.blob_path(_record_blob_ref(record)), provider=provider, + source_path=record.source_path, + max_record_bytes=_INSPECTION_PREFIX_BYTES, ) @@ -297,6 +298,7 @@ def _full_scan_malformed_jsonl(record: RawSessionRecord, *, blob_store: BlobStor max_samples=1, jsonl_dict_only=False, scan_full=True, + max_record_bytes=_INSPECTION_PREFIX_BYTES, ) except ValueError: # No valid JSONL records at all — leave the decision to the prefix-based @@ -361,30 +363,43 @@ def inspect_raw_artifact(record: RawSessionRecord, *, blob_store: BlobStore | No envelope = _inspect_payload_envelope(record, blob_store=resolved_blob_store) except Exception: stream_provider = Provider.from_string(provider_token) - recovered_artifact = _complete_stream_session_artifact( + recovered_scan = _complete_stream_session_artifact( record, provider=stream_provider, blob_store=resolved_blob_store, ) - if recovered_artifact is None: + if recovered_scan is None or recovered_scan.artifact is None: raise envelope = RawPayloadEnvelope( - payload=[], + payload=list(recovered_scan.sample), provider=stream_provider, wire_format="jsonl", - artifact=recovered_artifact, + artifact=recovered_scan.artifact, + malformed_jsonl_lines=recovered_scan.oversized_records, + malformed_jsonl_detail=( + "one or more records exceeded the inspection byte bound" + if recovered_scan.oversized_records + else None + ), ) payload_provider = envelope.provider artifact = envelope.artifact if not artifact.parse_as_session: - artifact = ( - _complete_stream_session_artifact( - record, + recovered_scan = _complete_stream_session_artifact( + record, + provider=payload_provider, + blob_store=resolved_blob_store, + ) + if recovered_scan is not None and recovered_scan.artifact is not None: + envelope = RawPayloadEnvelope( + payload=list(recovered_scan.sample), provider=payload_provider, - blob_store=resolved_blob_store, + wire_format="jsonl", + artifact=recovered_scan.artifact, + malformed_jsonl_lines=envelope.malformed_jsonl_lines, + malformed_jsonl_detail=envelope.malformed_jsonl_detail, ) - or artifact - ) + artifact = envelope.artifact resolution: SchemaResolution | None = None has_supported_resolution = False diff --git a/tests/unit/storage/test_artifact_loss_surfacing.py b/tests/unit/storage/test_artifact_loss_surfacing.py index f7cacee372..f06b92b592 100644 --- a/tests/unit/storage/test_artifact_loss_surfacing.py +++ b/tests/unit/storage/test_artifact_loss_surfacing.py @@ -8,11 +8,15 @@ from __future__ import annotations from collections.abc import Iterator +from io import BytesIO from pathlib import Path import pytest +from polylogue.archive.raw_payload.decode import scan_jsonl_session_artifact from polylogue.core.enums import ArtifactSupportStatus, Provider +from polylogue.core.json import JSONValue +from polylogue.schemas.observation import schema_cluster_id from polylogue.storage.artifacts.inspection import ( _INSPECTION_PREFIX_BYTES, inspect_raw_artifact, @@ -158,6 +162,74 @@ def test_codex_stream_recovers_when_first_record_exceeds_inspection_prefix(blob_ assert observation.artifact_kind == "session_record_stream" assert observation.wire_format == "jsonl" assert observation.decode_error is None + expected_message: JSONValue = { + "type": "response_item", + "payload": { + "type": "message", + "id": "message-1", + "role": "user", + "content": [{"type": "input_text", "text": "hello"}], + }, + } + assert observation.cohort_id == schema_cluster_id([expected_message], "session_record_stream") + + +def test_recovery_reader_discards_oversized_record_in_bounded_chunks() -> None: + class BoundedReadlineStream(BytesIO): + def readline(self, size: int | None = -1, /) -> bytes: + assert isinstance(size, int) + assert 0 < size <= _INSPECTION_PREFIX_BYTES + 1 + return super().readline(size) + + oversized = b'{"ignored":"' + (b"x" * (_INSPECTION_PREFIX_BYTES * 3)) + b'"}\n' + message = ( + b'{"type":"response_item","payload":{"type":"message","id":"message-1",' + b'"role":"user","content":[{"type":"input_text","text":"hello"}]}}\n' + ) + + scan = scan_jsonl_session_artifact( + BoundedReadlineStream(oversized + message), + provider=Provider.CODEX, + source_path="codex/bounded.jsonl", + max_record_bytes=_INSPECTION_PREFIX_BYTES, + ) + + assert scan.artifact is not None + assert scan.artifact.parse_as_session is True + assert scan.oversized_records == 1 + assert len(scan.sample) == 1 + + +def test_recovered_stream_retains_subagent_artifact_kind(blob_store: BlobStore) -> None: + oversized = b'{"ignored":"' + (b"x" * (_INSPECTION_PREFIX_BYTES * 2)) + b'"}\n' + message = ( + b'{"parentUuid":null,"type":"user","sessionId":"agent-session",' + b'"message":{"role":"user","content":"hello"},' + b'"uuid":"user-1","timestamp":"2026-01-01T00:00:00Z"}\n' + ) + record = _write_record( + blob_store, + content=oversized + message, + source_path="projects/project/subagents/agent-abcd.jsonl", + ) + + observation = inspect_raw_artifact(record) + + assert observation.parse_as_session is True + assert observation.artifact_kind == "agent_transcript" + assert observation.cohort_id == schema_cluster_id( + [ + { + "parentUuid": None, + "type": "user", + "sessionId": "agent-session", + "message": {"role": "user", "content": "hello"}, + "uuid": "user-1", + "timestamp": "2026-01-01T00:00:00Z", + } + ], + "agent_transcript", + ) def test_rolling_scan_preserves_tool_result_sidecar_exclusion(blob_store: BlobStore) -> None: From c3b3303ba06b49d945814ac27ae31388d8d5333a Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 19:17:04 +0200 Subject: [PATCH 43/65] fix: retain retry authority for bounded acquisition --- polylogue/archive/raw_payload/decode.py | 6 +- polylogue/sources/live/batch.py | 14 +++- polylogue/storage/artifacts/inspection.py | 12 ++-- tests/unit/sources/test_live_batch_support.py | 65 +++++++++++++++++++ .../storage/test_artifact_loss_surfacing.py | 4 ++ 5 files changed, 88 insertions(+), 13 deletions(-) diff --git a/polylogue/archive/raw_payload/decode.py b/polylogue/archive/raw_payload/decode.py index 93fae108c2..f2241e91ce 100644 --- a/polylogue/archive/raw_payload/decode.py +++ b/polylogue/archive/raw_payload/decode.py @@ -191,6 +191,9 @@ def _sample_jsonl_payload_with_detail( This is intended for provider/artifact/schema resolution where full-record materialization is unnecessary. Set ``scan_full`` when malformed-line accounting must reflect the entire source, such as strict validation. + Records skipped because they exceed ``max_record_bytes`` are uninspected, + not malformed: the bound must not manufacture decode-loss evidence for a + syntactically valid stream. """ samples: list[JSONValue] = [] malformed_lines = 0 @@ -203,9 +206,6 @@ def _sample_jsonl_payload_with_detail( for raw_line, oversized in _bounded_raw_lines(stream, max_record_bytes=max_record_bytes): line_number += 1 if oversized: - malformed_lines += 1 - if malformed_detail is None: - malformed_detail = f"line {line_number}: record exceeds inspection byte bound" first_line = False continue assert raw_line is not None diff --git a/polylogue/sources/live/batch.py b/polylogue/sources/live/batch.py index 85df5b2523..e8c856f1d1 100644 --- a/polylogue/sources/live/batch.py +++ b/polylogue/sources/live/batch.py @@ -1832,12 +1832,16 @@ def _ingest_full_paths_sync( if path.suffix.lower() == ".zip": file_mtime = datetime.fromtimestamp(stat.st_mtime_ns / 1_000_000_000, UTC).isoformat() if source_only: - zip_records, zip_bytes = self._extract_source_only_zip_member_records( + source_only_zip = self._extract_source_only_zip_member_records( path, blob_store=blob_store, fallback_provider=fallback_provider, file_mtime=file_mtime, ) + if source_only_zip is None: + failed.append(path) + continue + zip_records, zip_bytes = source_only_zip else: zip_records, zip_bytes = self._extract_zip_member_records( path, @@ -3304,7 +3308,7 @@ def _extract_source_only_zip_member_records( blob_store: BlobStore, fallback_provider: Provider, file_mtime: str, - ) -> tuple[list[tuple[str, RawSessionRecord]], int]: + ) -> tuple[list[tuple[str, RawSessionRecord]], int] | None: """Acquire admitted ZIP members without interpreting their bytes. A derived-tier outage does not authorize the source tier to infer a @@ -3367,7 +3371,11 @@ def _extract_source_only_zip_member_records( ) except (zipfile.BadZipFile, OSError) as exc: logger.warning("Failed to expand inbox ZIP %s: %s", path, exc) - return [], 0 + # A transport/read failure is not evidence that the archive has no + # admissible members. Keep it distinct from a successful empty + # extraction so the caller records retryable failure state instead + # of permanently acknowledging this source coordinate as excluded. + return None return records, total_bytes @staticmethod diff --git a/polylogue/storage/artifacts/inspection.py b/polylogue/storage/artifacts/inspection.py index 037a9dbcc3..cbffe9ffb8 100644 --- a/polylogue/storage/artifacts/inspection.py +++ b/polylogue/storage/artifacts/inspection.py @@ -281,7 +281,9 @@ def _full_scan_malformed_jsonl(record: RawSessionRecord, *, blob_store: BlobStor The prefix-based classification only inspects the first 64 KB, so malformed content past the prefix never marks the artifact failed (#1745). This scan streams the whole blob line-by-line (never materializing it) so the - malformed-line count and decode status reflect the full artifact. + malformed-line count and decode status reflect the full artifact. Records + larger than the inspection bound are discarded in chunks but are not + counted as malformed: bounded inspection is not evidence of decode loss. Returns ``(malformed_lines, had_valid_records)``. ``had_valid_records`` is ``True`` when at least one line decoded successfully; the sampling helper @@ -375,12 +377,8 @@ def inspect_raw_artifact(record: RawSessionRecord, *, blob_store: BlobStore | No provider=stream_provider, wire_format="jsonl", artifact=recovered_scan.artifact, - malformed_jsonl_lines=recovered_scan.oversized_records, - malformed_jsonl_detail=( - "one or more records exceeded the inspection byte bound" - if recovered_scan.oversized_records - else None - ), + malformed_jsonl_lines=0, + malformed_jsonl_detail=None, ) payload_provider = envelope.provider artifact = envelope.artifact diff --git a/tests/unit/sources/test_live_batch_support.py b/tests/unit/sources/test_live_batch_support.py index a27ba162d9..eeb85e2170 100644 --- a/tests/unit/sources/test_live_batch_support.py +++ b/tests/unit/sources/test_live_batch_support.py @@ -29,6 +29,7 @@ from polylogue.pipeline.ids import session_content_hash, session_revision_projection from polylogue.sources.dispatch import parse_payload from polylogue.sources.live import LiveWatcher, WatchSource +from polylogue.sources.live import batch as live_batch from polylogue.sources.live.append_ingest import ingest_append_plans from polylogue.sources.live.batch import ( _MAX_APPEND_PLAN_PAYLOAD_BYTES, @@ -683,6 +684,70 @@ def test_source_only_full_ingest_streams_admitted_zip_members_without_decoding( ] +def test_source_only_zip_read_failure_remains_retryable_after_partial_copy( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The real source-only route must not exclude a transiently unreadable ZIP.""" + from polylogue.core.degraded import DegradedReason, clear_degraded, set_degraded + + root = tmp_path / "sessions" + root.mkdir() + bundle = root / "retry.zip" + member_names = ("sessions/one.jsonl", "sessions/two.jsonl") + with zipfile.ZipFile(bundle, "w") as zf: + zf.writestr(member_names[0], b'{"opaque":"first"}\n') + zf.writestr(member_names[1], b'{"opaque":"second"}\n') + index_db = tmp_path / "index.db" + cursor = CursorStore(index_db) + processor = LiveBatchProcessor( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=index_db))), + (WatchSource(name="claude-code", root=root),), + cursor=cursor, + parser_fingerprint="test-parser", + ) + original_stream = live_batch.stream_preserved_zip_entry_raw_data + calls = 0 + + def fail_after_first_copy(*args: Any, **kwargs: Any) -> Any: + nonlocal calls + calls += 1 + if calls == 2: + raise OSError("transient ZIP read failure") + return original_stream(*args, **kwargs) + + monkeypatch.setattr(live_batch, "stream_preserved_zip_entry_raw_data", fail_after_first_copy) + set_degraded(DegradedReason(code="schema_version_mismatch", message="index unavailable", derived_only=True)) + try: + failed = asyncio.run(processor.ingest_files([bundle], emit_event=False)) + + assert failed.succeeded_file_count == 0 + assert failed.failed_file_count == 1 + failed_cursor = cursor.get_record(bundle) + assert failed_cursor is not None + assert failed_cursor.failure_count == 1 + assert failed_cursor.excluded is False + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT COUNT(*) FROM raw_sessions").fetchone() == (0,) + + retried = asyncio.run(processor.ingest_files([bundle], emit_event=False)) + finally: + clear_degraded() + + assert retried.succeeded_file_count == 1 + assert retried.failed_file_count == 0 + recovered_cursor = cursor.get_record(bundle) + assert recovered_cursor is not None + assert recovered_cursor.failure_count == 0 + assert recovered_cursor.excluded is False + with sqlite3.connect(tmp_path / "source.db") as conn: + retained = conn.execute("SELECT source_path, source_index FROM raw_sessions ORDER BY source_index").fetchall() + assert retained == [ + (f"{bundle}:{member_names[0]}", 0), + (f"{bundle}:{member_names[1]}", 1), + ] + + def test_source_only_zip_replay_resolves_unknown_chatgpt_member_and_keeps_duplicate_coordinates( tmp_path: Path, ) -> None: diff --git a/tests/unit/storage/test_artifact_loss_surfacing.py b/tests/unit/storage/test_artifact_loss_surfacing.py index f06b92b592..e2726be5b8 100644 --- a/tests/unit/storage/test_artifact_loss_surfacing.py +++ b/tests/unit/storage/test_artifact_loss_surfacing.py @@ -162,6 +162,10 @@ def test_codex_stream_recovers_when_first_record_exceeds_inspection_prefix(blob_ assert observation.artifact_kind == "session_record_stream" assert observation.wire_format == "jsonl" assert observation.decode_error is None + assert observation.malformed_jsonl_lines == 0 + assert observation.support_status is ArtifactSupportStatus.SUPPORTED_PARSEABLE + assert observation.resolved_package_version == "v1" + assert observation.resolved_element_kind == "session_record_stream" expected_message: JSONValue = { "type": "response_item", "payload": { From 355cfd7b657b79a849c33a564a5ff1bf51fd68c3 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 19:22:12 +0200 Subject: [PATCH 44/65] test: patch ZIP acquisition through canonical export --- tests/unit/sources/test_live_batch_support.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/unit/sources/test_live_batch_support.py b/tests/unit/sources/test_live_batch_support.py index eeb85e2170..cbc34aeae0 100644 --- a/tests/unit/sources/test_live_batch_support.py +++ b/tests/unit/sources/test_live_batch_support.py @@ -29,7 +29,6 @@ from polylogue.pipeline.ids import session_content_hash, session_revision_projection from polylogue.sources.dispatch import parse_payload from polylogue.sources.live import LiveWatcher, WatchSource -from polylogue.sources.live import batch as live_batch from polylogue.sources.live.append_ingest import ingest_append_plans from polylogue.sources.live.batch import ( _MAX_APPEND_PLAN_PAYLOAD_BYTES, @@ -55,6 +54,7 @@ from polylogue.sources.live.cursor import CursorStore from polylogue.sources.parsers.base import ParsedMessage, ParsedSession from polylogue.sources.revision_backfill import backfill_historical_revision_evidence +from polylogue.sources.source_acquisition_components import stream_preserved_zip_entry_raw_data from polylogue.sources.source_parsing import has_decoded_session_evidence from polylogue.storage.blob_store import BlobStore from polylogue.storage.raw_authority import RAW_AUTHORITY_PARSER_FINGERPRINT @@ -706,7 +706,7 @@ def test_source_only_zip_read_failure_remains_retryable_after_partial_copy( cursor=cursor, parser_fingerprint="test-parser", ) - original_stream = live_batch.stream_preserved_zip_entry_raw_data + original_stream = stream_preserved_zip_entry_raw_data calls = 0 def fail_after_first_copy(*args: Any, **kwargs: Any) -> Any: @@ -716,7 +716,10 @@ def fail_after_first_copy(*args: Any, **kwargs: Any) -> Any: raise OSError("transient ZIP read failure") return original_stream(*args, **kwargs) - monkeypatch.setattr(live_batch, "stream_preserved_zip_entry_raw_data", fail_after_first_copy) + monkeypatch.setattr( + "polylogue.sources.live.batch.stream_preserved_zip_entry_raw_data", + fail_after_first_copy, + ) set_degraded(DegradedReason(code="schema_version_mismatch", message="index unavailable", derived_only=True)) try: failed = asyncio.run(processor.ingest_files([bundle], emit_event=False)) From d0add31cb047adcebae4b83026b6efec610f367c Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 19:42:43 +0200 Subject: [PATCH 45/65] fix: stabilize exact-head verification startup --- devtools/verify.py | 32 ++++++++++------ devtools/verify_runs.py | 35 ++++++++++++------ tests/unit/devtools/test_verify.py | 59 ++++++++++++++++++++++++++++++ 3 files changed, 103 insertions(+), 23 deletions(-) diff --git a/devtools/verify.py b/devtools/verify.py index da4a268652..f08979c1d3 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -3861,24 +3861,32 @@ def _main(argv: list[str] | None = None) -> int: final_checkout_fingerprint = worktree_fingerprint(ROOT) mutation_observation = finish_checkout_mutation_monitor(mutation_monitor) checkout_stable = True - if ( - changed_path_authority_failed - or head is None + checkout_fingerprint_unavailable = ( + head is None or final_head is None - or "unavailable" in {checkout_fingerprint, final_checkout_fingerprint} - or mutation_observation.unavailable - ): + or "unavailable" + in { + checkout_fingerprint, + final_checkout_fingerprint, + } + ) + if changed_path_authority_failed or checkout_fingerprint_unavailable or mutation_observation.unavailable: checkout_stable = False + diagnosis = ( + "testmon_changed_path_authority_unavailable" + if changed_path_authority_failed + else ( + "checkout_fingerprint_unavailable" + if checkout_fingerprint_unavailable + else "checkout_mutation_monitor_unavailable" + ) + ) step_results.append( { "name": "checkout stability", "duration_s": 0.0, "exit": 125, - "diagnosis": ( - "testmon_changed_path_authority_unavailable" - if changed_path_authority_failed - else "checkout_fingerprint_unavailable" - ), + "diagnosis": diagnosis, "initial_git_head": head, "final_git_head": final_head, "initial_worktree_fingerprint": checkout_fingerprint, @@ -3887,7 +3895,7 @@ def _main(argv: list[str] | None = None) -> int: ) if exit_code == 0: exit_code = 125 - sys.stderr.write("verify: checkout fingerprint unavailable; evidence is not exact-head.\n") + sys.stderr.write(f"verify: {diagnosis.replace('_', ' ')}; evidence is not exact-head.\n") elif final_head != head or mutation_observation.changed or final_checkout_fingerprint != checkout_fingerprint: checkout_stable = False step_results.append( diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index 30d46bfaf1..e5f57127ee 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -293,7 +293,20 @@ def start(self) -> None: self._unavailable = True self._ready.set() return - self._thread = threading.Thread(target=self._watch, name="checkout-mutation-monitor", daemon=True) + # Repository enumeration and Git authority discovery are synchronous + # preflight, not native watcher startup. Keeping them outside the + # backend deadline prevents a slow CI checkout from consuming the + # entire readiness budget before watchfiles can initialize. + watched_directories = self._watched_directories() + if self._unavailable: + self._ready.set() + return + self._thread = threading.Thread( + target=self._watch, + args=(watched_directories,), + name="checkout-mutation-monitor", + daemon=True, + ) self._thread.start() if not self._ready.wait(timeout=self._WATCH_START_TIMEOUT_S): with self._state_lock: @@ -320,11 +333,8 @@ def finish(self) -> CheckoutMutationObservation: observed_path=self._observed_path, ) - def _watch(self) -> None: + def _watch(self, watched_directories: Sequence[Path]) -> None: try: - watched_directories = self._watched_directories() - if self._unavailable: - return for changes in watchfiles.watch( *watched_directories, watch_filter=None, @@ -338,12 +348,15 @@ def _watch(self) -> None: recursive=False, ): # An empty timeout batch proves the backend initialized before - # a verification command starts, closing the startup race. - if not self._ready.is_set() and not self._directory_topology_is_stable(watched_directories): - with self._state_lock: - self._unavailable = True - return - self._ready.set() + # a verification command starts, closing the startup race. The + # active watcher protects the following topology recheck, so + # readiness need not wait for a second repository enumeration. + if not self._ready.is_set(): + self._ready.set() + if not self._directory_topology_is_stable(watched_directories): + with self._state_lock: + self._unavailable = True + return for _change, raw_path in changes: self._record_change(Path(raw_path)) if self._changed or self._unavailable: diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 829793d5d5..a4bf25fa32 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -2077,6 +2077,35 @@ def portable_watch(*paths: Path, **kwargs: object) -> object: assert observation == CheckoutMutationObservation(changed=True, unavailable=False, observed_path="tracked.py") +def test_checkout_mutation_monitor_prepares_paths_before_backend_startup_deadline( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + monitor = CheckoutMutationMonitor(tmp_path) + original_watched_directories = monitor._watched_directories + discovery_threads: list[threading.Thread] = [] + + def observed_discovery() -> list[Path]: + discovery_threads.append(threading.current_thread()) + return original_watched_directories() + + def portable_watch(*_paths: Path, **kwargs: object) -> object: + yield set() + stop_event = kwargs["stop_event"] + assert isinstance(stop_event, threading.Event) + stop_event.wait() + + monkeypatch.setattr(monitor, "_watched_directories", observed_discovery) + monkeypatch.setattr(watchfiles, "watch", portable_watch) + + monitor.start() + observation = monitor.finish() + + assert discovery_threads[0] is threading.main_thread() + assert observation == CheckoutMutationObservation(changed=False, unavailable=False) + + def test_checkout_mutation_monitor_rejects_source_topology_changed_during_startup( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -4939,6 +4968,36 @@ def test_verify_withholds_success_when_checkout_fingerprint_is_unavailable( assert checkout_step["final_worktree_fingerprint"] == fingerprints[1] +def test_verify_classifies_unavailable_mutation_monitor_separately( + capsys: pytest.CaptureFixture[str], +) -> None: + class _UnavailableMonitor: + def __init__(self, _root: Path) -> None: + pass + + def start(self) -> None: + pass + + def finish(self) -> CheckoutMutationObservation: + return CheckoutMutationObservation(changed=False, unavailable=True) + + with ( + patch("devtools.verify._run", return_value=(0, 0.01, {})), + patch("devtools.verify._git_head", return_value="head"), + patch("devtools.verify._save_history"), + patch("devtools.verify._stamp_head"), + patch("devtools.verify._notify"), + patch("devtools.verify.CheckoutMutationMonitor", _UnavailableMonitor), + patch("devtools.verify.worktree_fingerprint", return_value="stable"), + ): + rc = main(["--quick", "--json"]) + + assert rc == 125 + payload = json.loads(capsys.readouterr().out) + checkout_step = next(step for step in payload["steps"] if step["name"] == "checkout stability") + assert checkout_step["diagnosis"] == "checkout_mutation_monitor_unavailable" + + def test_verify_rejects_git_head_change_with_matching_worktree_fingerprints( capsys: pytest.CaptureFixture[str], ) -> None: From dd5ec5b5aee3cba1a1aa1bd482af34e26ffa158a Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 20:04:32 +0200 Subject: [PATCH 46/65] fix: retain oversized stream parse candidates --- polylogue/archive/raw_payload/decode.py | 18 ++++++++ .../storage/test_artifact_loss_surfacing.py | 45 +++++++++++++++++++ 2 files changed, 63 insertions(+) diff --git a/polylogue/archive/raw_payload/decode.py b/polylogue/archive/raw_payload/decode.py index f2241e91ce..54916300e5 100644 --- a/polylogue/archive/raw_payload/decode.py +++ b/polylogue/archive/raw_payload/decode.py @@ -13,6 +13,7 @@ ArtifactKind, classify_artifact, ) +from polylogue.archive.artifact_taxonomy.support import is_subagent_path from polylogue.archive.raw_payload.streams import raw_line_stream from polylogue.core.binary_signatures import detect_binary_signature from polylogue.core.enums import Provider @@ -295,6 +296,23 @@ def scan_jsonl_session_artifact( sample=tuple(sample), oversized_records=oversized_records, ) + if oversized_records and provider in {Provider.CLAUDE_CODE, Provider.CODEX}: + # A size-bounded inspection skip is unresolved evidence, not negative + # evidence. These providers have streaming parsers, so retain the raw + # as a parse candidate instead of allowing a weak path heuristic to + # terminalize a genuine session whose only record was oversized. + subagent = is_subagent_path(source_path) + return JSONLSessionArtifactScan( + artifact=ArtifactClassification( + provider=provider, + kind=ArtifactKind.AGENT_TRANSCRIPT if subagent else ArtifactKind.SESSION_RECORD_STREAM, + parse_as_session=True, + schema_eligible=False, + default_priority=90 if subagent else 120, + reason="uninspected oversized provider JSONL record retained for streaming parse", + ), + oversized_records=oversized_records, + ) return JSONLSessionArtifactScan(artifact=None, oversized_records=oversized_records) diff --git a/tests/unit/storage/test_artifact_loss_surfacing.py b/tests/unit/storage/test_artifact_loss_surfacing.py index e2726be5b8..3912ad8419 100644 --- a/tests/unit/storage/test_artifact_loss_surfacing.py +++ b/tests/unit/storage/test_artifact_loss_surfacing.py @@ -204,6 +204,51 @@ def readline(self, size: int | None = -1, /) -> bytes: assert len(scan.sample) == 1 +@pytest.mark.parametrize( + ("provider", "source_name"), + [ + pytest.param(Provider.CLAUDE_CODE, "claude-code", id="claude-code"), + pytest.param(Provider.CODEX, "codex", id="codex"), + ], +) +def test_single_oversized_provider_record_under_weak_path_remains_parse_candidate( + blob_store: BlobStore, + provider: Provider, + source_name: str, +) -> None: + if provider is Provider.CLAUDE_CODE: + content = ( + b'{"type":"user","uuid":"message-1","sessionId":"oversized-session",' + b'"parentUuid":null,"message":{"role":"user","content":"' + + (b"x" * (_INSPECTION_PREFIX_BYTES * 2)) + + b'"}}\n' + ) + else: + content = ( + b'{"type":"response_item","payload":{"type":"message","id":"message-1",' + b'"role":"user","content":[{"type":"input_text","text":"' + + (b"x" * (_INSPECTION_PREFIX_BYTES * 2)) + + b'"}]}}\n' + ) + assert content.find(b"\n") > _INSPECTION_PREFIX_BYTES + record = _write_record( + blob_store, + content=content, + source_path=f"{source_name}/analysis/re-homed-session.jsonl", + source_name=source_name, + provider=provider, + ) + + observation = inspect_raw_artifact(record) + + assert observation.parse_as_session is True + assert observation.schema_eligible is False + assert observation.artifact_kind == "session_record_stream" + assert observation.support_status is ArtifactSupportStatus.RECOGNIZED_UNPARSED + assert observation.malformed_jsonl_lines == 0 + assert observation.decode_error is None + + def test_recovered_stream_retains_subagent_artifact_kind(blob_store: BlobStore) -> None: oversized = b'{"ignored":"' + (b"x" * (_INSPECTION_PREFIX_BYTES * 2)) + b'"}\n' message = ( From b9567a630cc8ad3eef545c85efb46631ac41e21b Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 20:18:27 +0200 Subject: [PATCH 47/65] fix: isolate exact-head ref authority --- devtools/verify_runs.py | 31 +++++++++++++++-- tests/unit/devtools/test_verify.py | 55 ++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 3 deletions(-) diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index e5f57127ee..abb5f8fb09 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -277,12 +277,15 @@ def __init__(self, root: Path) -> None: self._unavailable = False self._stop = threading.Event() self._ready = threading.Event() + self._initialized = threading.Event() self._thread: threading.Thread | None = None self._state_lock = threading.Lock() self._tracked_paths: frozenset[Path] = frozenset() self._tracked_directories: frozenset[Path] = frozenset() self._ignored_roots: frozenset[Path] = frozenset() self._git_index_path: Path | None = None + self._git_current_ref_path: Path | None = None + self._git_current_ref_was_loose: bool | None = None self._git_authority_paths: dict[Path, str] = {} self._directory_topology_fingerprint: frozenset[str] | None = None @@ -292,6 +295,7 @@ def start(self) -> None: with self._state_lock: self._unavailable = True self._ready.set() + self._initialized.set() return # Repository enumeration and Git authority discovery are synchronous # preflight, not native watcher startup. Keeping them outside the @@ -300,6 +304,7 @@ def start(self) -> None: watched_directories = self._watched_directories() if self._unavailable: self._ready.set() + self._initialized.set() return self._thread = threading.Thread( target=self._watch, @@ -313,6 +318,12 @@ def start(self) -> None: self._unavailable = True self._stop.set() self._thread.join(timeout=self._WATCH_START_TIMEOUT_S) + return + # The one-second deadline proves only native backend startup. The + # protected topology recheck is ordinary repository discovery and may + # legitimately take longer on a cold CI checkout; complete it before + # the verification command can mutate the tree. + self._initialized.wait() def finish(self) -> CheckoutMutationObservation: """Stop monitoring only after the caller took its final fingerprint.""" @@ -349,14 +360,16 @@ def _watch(self, watched_directories: Sequence[Path]) -> None: ): # An empty timeout batch proves the backend initialized before # a verification command starts, closing the startup race. The - # active watcher protects the following topology recheck, so - # readiness need not wait for a second repository enumeration. + # active watcher protects the following topology recheck. The + # native-ready event has its own bounded startup deadline; + # ``start`` waits separately for repository discovery. if not self._ready.is_set(): self._ready.set() if not self._directory_topology_is_stable(watched_directories): with self._state_lock: self._unavailable = True return + self._initialized.set() for _change, raw_path in changes: self._record_change(Path(raw_path)) if self._changed or self._unavailable: @@ -369,6 +382,7 @@ def _watch(self, watched_directories: Sequence[Path]) -> None: self._unavailable = True finally: self._ready.set() + self._initialized.set() @classmethod def _polling_backend_requested(cls) -> bool: @@ -490,7 +504,10 @@ def _resolve_git_head_paths(self) -> dict[Path, str]: with self._state_lock: self._unavailable = True return paths - paths[Path(raw_ref_path)] = f".git/{symbolic_ref}" + self._git_current_ref_path = Path(raw_ref_path) + if self._git_current_ref_was_loose is None: + self._git_current_ref_was_loose = self._git_current_ref_path.exists() + paths[self._git_current_ref_path] = f".git/{symbolic_ref}" return paths def _git_command( @@ -540,6 +557,14 @@ def _record_change(self, candidate: Path) -> None: if not candidate.is_absolute(): candidate = self.root / candidate for authority_path, label in self._git_authority_paths.items(): + if label == ".git/packed-refs" and self._git_current_ref_was_loose is True: + # packed-refs is shared by linked worktrees. When this + # worktree's current branch has a loose ref, unrelated fetch + # maintenance cannot change its HEAD through the packed file. + # A real pack transition remains visible when the loose ref + # is removed or replaced. Preserve the startup state so a + # packed-to-loose transition cannot hide its own first event. + continue if candidate != authority_path and authority_path.is_relative_to(candidate): with self._state_lock: self._changed = True diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index a4bf25fa32..bcaa437cec 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -2288,6 +2288,61 @@ def test_checkout_mutation_monitor_observes_transient_head_ref_change(tmp_path: ) +def test_checkout_mutation_monitor_ignores_shared_packed_refs_when_current_ref_is_loose( + tmp_path: Path, +) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.email", "tests@example.invalid"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.name", "Polylogue Tests"], cwd=tmp_path, check=True) + tracked = tmp_path / "tracked.py" + tracked.write_text("value = 1\n", encoding="utf-8") + subprocess.run(["git", "add", "tracked.py"], cwd=tmp_path, check=True) + subprocess.run(["git", "commit", "-qm", "seed"], cwd=tmp_path, check=True) + subprocess.run(["git", "pack-refs", "--all", "--no-prune"], cwd=tmp_path, check=True) + + monitor = CheckoutMutationMonitor(tmp_path) + monitor._watched_directories() + monitor._record_change(tmp_path / ".git" / "packed-refs") + + assert monitor.finish() == CheckoutMutationObservation(changed=False, unavailable=False) + + +def test_checkout_mutation_monitor_watches_packed_refs_when_current_ref_is_packed( + tmp_path: Path, +) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.email", "tests@example.invalid"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.name", "Polylogue Tests"], cwd=tmp_path, check=True) + tracked = tmp_path / "tracked.py" + tracked.write_text("value = 1\n", encoding="utf-8") + subprocess.run(["git", "add", "tracked.py"], cwd=tmp_path, check=True) + subprocess.run(["git", "commit", "-qm", "seed"], cwd=tmp_path, check=True) + subprocess.run(["git", "pack-refs", "--all", "--prune"], cwd=tmp_path, check=True) + branch = subprocess.run( + ["git", "symbolic-ref", "HEAD"], cwd=tmp_path, check=True, capture_output=True, text=True + ).stdout.strip() + loose_ref = Path( + subprocess.run( + ["git", "rev-parse", "--path-format=absolute", "--git-path", branch], + cwd=tmp_path, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + ) + assert not loose_ref.exists() + + monitor = CheckoutMutationMonitor(tmp_path) + monitor._watched_directories() + monitor._record_change(tmp_path / ".git" / "packed-refs") + + assert monitor.finish() == CheckoutMutationObservation( + changed=True, + unavailable=False, + observed_path=".git/packed-refs", + ) + + @pytest.mark.uses_real_clock("waits for the filesystem watcher to witness a loose ref created from packed authority") def test_checkout_mutation_monitor_observes_packed_nested_branch_ref_change(tmp_path: Path) -> None: subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) From c9a3f64ea7cc637faad141dcfedc4a5f22656836 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 21:05:12 +0200 Subject: [PATCH 48/65] fix: close exact-head ingest authority gaps --- devtools/verify_runs.py | 31 +++++--- polylogue/daemon/cli.py | 58 +++++++++----- polylogue/product/raw_authority.py | 13 +++ polylogue/sources/live/batch.py | 42 +++++++--- polylogue/sources/revision_backfill.py | 79 +++++++++++++++---- .../sources/source_acquisition_components.py | 30 ++++++- polylogue/storage/repair.py | 18 +++-- tests/unit/daemon/test_daemon_cli.py | 64 +++++++++++++++ tests/unit/devtools/test_verify.py | 28 +++++++ tests/unit/sources/test_live_batch_support.py | 50 ++++++++++++ tests/unit/sources/test_revision_backfill.py | 56 +++++++++++-- 11 files changed, 390 insertions(+), 79 deletions(-) diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index abb5f8fb09..ec9d1bee52 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -486,6 +486,12 @@ def _resolve_git_head_paths(self) -> dict[Path, str]: self._unavailable = True return paths paths[Path(raw_head_path)] = ".git/HEAD" + if symbolic_result.returncode == 1: + # A detached checkout's complete revision authority is the + # worktree-specific HEAD file. packed-refs is shared by every + # linked worktree, so unrelated fetch/pack maintenance cannot + # mutate this checkout and must not invalidate its verification. + return paths packed_result = self._git_command(["rev-parse", "--path-format=absolute", "--git-path", "packed-refs"]) if packed_result is None: return paths @@ -495,19 +501,18 @@ def _resolve_git_head_paths(self) -> dict[Path, str]: self._unavailable = True return paths paths[Path(raw_packed_path)] = ".git/packed-refs" - if symbolic_result.returncode == 0: - ref_result = self._git_command(["rev-parse", "--path-format=absolute", "--git-path", symbolic_ref]) - if ref_result is None: - return paths - raw_ref_path = os.fsdecode(ref_result.stdout).strip() - if not raw_ref_path: - with self._state_lock: - self._unavailable = True - return paths - self._git_current_ref_path = Path(raw_ref_path) - if self._git_current_ref_was_loose is None: - self._git_current_ref_was_loose = self._git_current_ref_path.exists() - paths[self._git_current_ref_path] = f".git/{symbolic_ref}" + ref_result = self._git_command(["rev-parse", "--path-format=absolute", "--git-path", symbolic_ref]) + if ref_result is None: + return paths + raw_ref_path = os.fsdecode(ref_result.stdout).strip() + if not raw_ref_path: + with self._state_lock: + self._unavailable = True + return paths + self._git_current_ref_path = Path(raw_ref_path) + if self._git_current_ref_was_loose is None: + self._git_current_ref_was_loose = self._git_current_ref_path.exists() + paths[self._git_current_ref_path] = f".git/{symbolic_ref}" return paths def _git_command( diff --git a/polylogue/daemon/cli.py b/polylogue/daemon/cli.py index 6a57b41040..6e8d9db55a 100644 --- a/polylogue/daemon/cli.py +++ b/polylogue/daemon/cli.py @@ -1309,18 +1309,26 @@ def _drain_raw_materialization_once( "raw authority: auto-resolved %d stale-plan blocker(s) before raw materialization", auto_resolved, ) - with raw_authority.materialization_generation_lease(config) as index_db: + with contextlib.ExitStack() as lease_stack: try: - result = raw_authority.repair_materialization( - config, - dry_run=False, - raw_artifact_limit=limit, - max_payload_bytes=_RAW_MATERIALIZATION_DAEMON_BLOB_LIMIT_BYTES, - prefetch_cache=prefetch_cache, - max_pass_seconds=_RAW_MATERIALIZATION_MAX_PASS_SECONDS, - ) - finally: - _close_raw_materialization_fts(index_db, ops_db_path=config.archive_root / "ops.db") + index_db = lease_stack.enter_context(raw_authority.materialization_generation_lease(config)) + except Exception as exc: + refused_result = raw_authority.materialization_lease_refusal_result(exc) + if refused_result is None: + raise + result = refused_result + else: + try: + result = raw_authority.repair_materialization( + config, + dry_run=False, + raw_artifact_limit=limit, + max_payload_bytes=_RAW_MATERIALIZATION_DAEMON_BLOB_LIMIT_BYTES, + prefetch_cache=prefetch_cache, + max_pass_seconds=_RAW_MATERIALIZATION_MAX_PASS_SECONDS, + ) + finally: + _close_raw_materialization_fts(index_db, ops_db_path=config.archive_root / "ops.db") _emit_raw_materialization_pass(result) frontier_repaired = _converge_raw_authority_frontier(config, limit=min(limit, 8)) if not result.success: @@ -1379,17 +1387,25 @@ def _run_raw_materialization_whale_pass_once(*, raw_artifact_id: str, max_payloa archive = archive_root() config = Config(archive_root=archive, render_root=render_root(), sources=[]) - with raw_authority.materialization_generation_lease(config) as index_db: + with contextlib.ExitStack() as lease_stack: try: - result = raw_authority.repair_materialization( - config, - dry_run=False, - raw_artifact_limit=1, - max_payload_bytes=max_payload_bytes, - raw_artifact_id=raw_artifact_id, - ) - finally: - _close_raw_materialization_fts(index_db, ops_db_path=config.archive_root / "ops.db") + index_db = lease_stack.enter_context(raw_authority.materialization_generation_lease(config)) + except Exception as exc: + refused_result = raw_authority.materialization_lease_refusal_result(exc) + if refused_result is None: + raise + result = refused_result + else: + try: + result = raw_authority.repair_materialization( + config, + dry_run=False, + raw_artifact_limit=1, + max_payload_bytes=max_payload_bytes, + raw_artifact_id=raw_artifact_id, + ) + finally: + _close_raw_materialization_fts(index_db, ops_db_path=config.archive_root / "ops.db") _emit_raw_materialization_pass(result) if not result.success: logger.warning("raw materialization: whale pass for %s incomplete: %s", raw_artifact_id, result.detail) diff --git a/polylogue/product/raw_authority.py b/polylogue/product/raw_authority.py index bc61b2022a..02ad9dead3 100644 --- a/polylogue/product/raw_authority.py +++ b/polylogue/product/raw_authority.py @@ -19,6 +19,7 @@ if TYPE_CHECKING: from polylogue.sources.revision_backfill import RawParsePrefetchCache from polylogue.storage.raw_reconciler import RawAuthorityFrontierApplyReport, RawAuthorityFrontierCensus + from polylogue.storage.repair import RepairResult RAW_MATERIALIZATION_ORDINARY_BLOB_LIMIT_BYTES: Final = 64 * 1024 * 1024 @@ -134,6 +135,17 @@ def materialization_generation_lease(config: Config) -> Iterator[Path]: lease.close() +def materialization_lease_refusal_result(error: BaseException) -> RepairResult | None: + """Translate only a rebuild-lease refusal into raw repair's typed result.""" + from polylogue.storage.index_generation import RebuildLeaseUnavailableError + + if not isinstance(error, RebuildLeaseUnavailableError): + return None + from polylogue.storage.repair import raw_materialization_lease_refusal_result + + return raw_materialization_lease_refusal_result(error) + + def repair_materialization( config: Config, *, @@ -238,6 +250,7 @@ def list_blockers(archive_root: Path, *, limit: int = 100, offset: int = 0) -> J "inspect_frontier", "list_blockers", "materialization_generation_lease", + "materialization_lease_refusal_result", "read_census", "read_detail", "recover_interrupted_frontier", diff --git a/polylogue/sources/live/batch.py b/polylogue/sources/live/batch.py index e8c856f1d1..fd85aa360a 100644 --- a/polylogue/sources/live/batch.py +++ b/polylogue/sources/live/batch.py @@ -149,6 +149,7 @@ iter_zip_entry_raw_data, stream_preserved_zip_entry_raw_data, zip_member_raw_id, + zip_member_source_index, ) from polylogue.sources.source_parsing import has_decoded_session_evidence from polylogue.sources.sqlite_snapshot import ( @@ -3231,7 +3232,9 @@ def _extract_zip_member_records( ) try: with zipfile.ZipFile(path) as zf: - entries = list(validator.filter_entries(zf.infolist())) + central_directory = zf.infolist() + entry_ordinals = {id(info): ordinal for ordinal, info in enumerate(central_directory)} + entries = [(entry_ordinals[id(info)], info) for info in validator.filter_entries(central_directory)] # A GDPR/Takeout export ZIP dropped into a provider-agnostic # inbox (``fallback_provider is Provider.UNKNOWN``) still has # a real dominant provider -- it just isn't visible from any @@ -3246,8 +3249,10 @@ def _extract_zip_member_records( # provider (a per-provider watched directory) is left alone. zip_provider_hint = fallback_provider if fallback_provider is Provider.UNKNOWN: - zip_provider_hint = self._sniff_zip_provider(zf, entries) or fallback_provider - for info in entries: + zip_provider_hint = ( + self._sniff_zip_provider(zf, [info for _ordinal, info in entries]) or fallback_provider + ) + for entry_ordinal, info in entries: if info.file_size == 0: continue try: @@ -3267,10 +3272,16 @@ def _extract_zip_member_records( member_provider = raw_data.provider_hint or fallback_provider member_size = raw_data.blob_size or 0 total_bytes += member_size + split_index = raw_data.source_index if raw_data.source_index is not None else 0 + source_index = zip_member_source_index( + entry_ordinal=entry_ordinal, + split_index=split_index, + ) member_raw_id = zip_member_raw_id( - raw_data.source_path, - raw_data.source_index or 0, - raw_data.blob_hash, + source_path=raw_data.source_path, + entry_ordinal=entry_ordinal, + split_index=split_index, + blob_hash=raw_data.blob_hash, ) records.append( ( @@ -3286,7 +3297,7 @@ def _extract_zip_member_records( ), source_name=member_provider.value, source_path=raw_data.source_path, - source_index=raw_data.source_index or 0, + source_index=source_index, blob_size=member_size, blob_publication_receipt_id=raw_data.blob_publication_receipt_id, acquired_at=acquired_at, @@ -3323,9 +3334,17 @@ def _extract_source_only_zip_member_records( validator = _ZipEntryValidator(fallback_provider, cursor_state=None, zip_path=path) try: with zipfile.ZipFile(path) as zf: - for source_index, info in enumerate(validator.filter_entries(zf.infolist())): + central_directory = zf.infolist() + entry_ordinals = {id(info): ordinal for ordinal, info in enumerate(central_directory)} + for info in validator.filter_entries(central_directory): if info.file_size == 0: continue + entry_ordinal = entry_ordinals[id(info)] + split_index = 0 + source_index = zip_member_source_index( + entry_ordinal=entry_ordinal, + split_index=split_index, + ) try: raw_data = stream_preserved_zip_entry_raw_data( zf, @@ -3347,9 +3366,10 @@ def _extract_source_only_zip_member_records( continue total_bytes += raw_data.blob_size or 0 member_raw_id = zip_member_raw_id( - raw_data.source_path, - source_index, - raw_data.blob_hash, + source_path=raw_data.source_path, + entry_ordinal=entry_ordinal, + split_index=split_index, + blob_hash=raw_data.blob_hash, ) records.append( ( diff --git a/polylogue/sources/revision_backfill.py b/polylogue/sources/revision_backfill.py index a0f3af1ea1..c8654e5e27 100644 --- a/polylogue/sources/revision_backfill.py +++ b/polylogue/sources/revision_backfill.py @@ -82,6 +82,61 @@ _REPLAY_PROVIDER_DETECTION_PREFIX_BYTES: Final[int] = 8192 +def _detect_unknown_retained_provider( + payload: BinaryIO, + source_path: str, +) -> tuple[Provider, str]: + """Detect retained UNKNOWN bytes without eagerly materializing JSONL. + + A byte prefix can end inside the first physical JSONL record. For an + oversized record stream that makes a prefix-only detector inconclusive + even when a later bounded record identifies a streaming provider. Scan + complete records across the stream instead: each record is capped at the + same detection bound, oversized records are consumed in bounded chunks, + and the scan continues until positive provider evidence or EOF. + + Non-JSONL documents retain prefix detection here and their existing + complete-document retry at the caller. That eager retry is required for + document providers whose first complete value exceeds the prefix. + """ + stream_name = Path(source_path).name + if not is_jsonl_source_path(source_path): + return detect_provider_from_raw_bytes_evidence( + payload.read(_REPLAY_PROVIDER_DETECTION_PREFIX_BYTES), + stream_name, + Provider.UNKNOWN, + truncated_tail_ok=True, + ) + + # Local import avoids a source-dispatch import cycle during module load; + # this is the same bounded physical-record iterator used by raw-payload + # artifact inspection. + from polylogue.archive.raw_payload.decode import _bounded_raw_lines + + last_evidence = "no bounded JSONL record identified a provider; used fallback_provider" + for raw_line, oversized in _bounded_raw_lines( + payload, + max_record_bytes=_REPLAY_PROVIDER_DETECTION_PREFIX_BYTES, + ): + if oversized or raw_line is None: + continue + record_bytes = raw_line.encode("utf-8", errors="surrogatepass") if isinstance(raw_line, str) else raw_line + provider, last_evidence = detect_provider_from_raw_bytes_evidence( + record_bytes, + stream_name, + Provider.UNKNOWN, + ) + if provider is not Provider.UNKNOWN: + return provider, last_evidence + return Provider.UNKNOWN, last_evidence + + +def _require_resolved_jsonl_provider(provider: Provider, source_path: str) -> None: + """Refuse an eager fallback when bounded JSONL detection stayed unknown.""" + if provider is Provider.UNKNOWN and is_jsonl_source_path(source_path): + raise ValueError("retained UNKNOWN JSONL provider remained unresolved after bounded record scan") + + def _canonical_authority_logical_key(logical_key: str) -> str: """Normalize transitional provider and public-origin authority prefixes.""" prefix, separator, native_id = logical_key.partition(":") @@ -2019,18 +2074,15 @@ def census_parse_worker( publisher = ArchiveBlobPublisher(Path(source_db_path_str), Path(blob_root_str)) try: if provider is Provider.UNKNOWN: - provider, _evidence = detect_provider_from_raw_bytes_evidence( - publisher.read_prefix(blob_hash, _REPLAY_PROVIDER_DETECTION_PREFIX_BYTES), - Path(source_path).name, - provider, - truncated_tail_ok=True, - ) + with publisher.open(blob_hash) as detection_payload: + provider, _evidence = _detect_unknown_retained_provider(detection_payload, source_path) if is_stream_record_provider(source_path, str(provider)): with publisher.open(blob_hash) as stream_payload: sessions = _parse_stream( provider, stream_payload, source_path, fallback_id_override=fallback_id_override ) return raw_id, sessions, None + _require_resolved_jsonl_provider(provider, source_path) payload = publisher.read_all(blob_hash) if provider is Provider.UNKNOWN: provider, _evidence = detect_provider_from_raw_bytes_evidence( @@ -2465,13 +2517,11 @@ def parse_retained_raw_sessions(archive: ArchiveStore, raw_id: str) -> list[Pars # inspect the durable bytes and resolve their parser, before deciding # whether their filename is a stream route. with archive.open_raw_revision_material(raw_id) as (_stream_provider, payload, _stream_path, _stream_kind): - detection_prefix = payload.read(_REPLAY_PROVIDER_DETECTION_PREFIX_BYTES) - provider, _evidence = detect_provider_from_raw_bytes_evidence( - detection_prefix, Path(source_path).name, provider, truncated_tail_ok=True - ) + provider, _evidence = _detect_unknown_retained_provider(payload, source_path) if is_stream_record_provider(source_path, str(provider)): with archive.open_raw_revision_material(raw_id) as (_stream_provider, payload, stream_path, _stream_kind): return _parse_stream(provider, payload, stream_path, fallback_id_override=fallback_id_override) + _require_resolved_jsonl_provider(provider, source_path) _provider, eager_payload, _source_path, _eager_kind = archive.raw_revision_material(raw_id) if provider is Provider.UNKNOWN: provider, _evidence = detect_provider_from_raw_bytes_evidence( @@ -3290,14 +3340,11 @@ def _detected_provider_for_empty_replay( if stored_provider is not Provider.UNKNOWN: return stored_provider with archive.open_raw_revision_material(raw_id) as (_provider, payload, _path, _kind): - provider, _evidence = detect_provider_from_raw_bytes_evidence( - payload.read(_REPLAY_PROVIDER_DETECTION_PREFIX_BYTES), - Path(source_path).name, - stored_provider, - truncated_tail_ok=True, - ) + provider, _evidence = _detect_unknown_retained_provider(payload, source_path) if provider is not Provider.UNKNOWN: return provider + if is_jsonl_source_path(source_path): + return provider _provider, full_payload, _path, _kind = archive.raw_revision_material(raw_id) provider, _evidence = detect_provider_from_raw_bytes_evidence( full_payload, diff --git a/polylogue/sources/source_acquisition_components.py b/polylogue/sources/source_acquisition_components.py index b65e464a18..9a17c0cef8 100644 --- a/polylogue/sources/source_acquisition_components.py +++ b/polylogue/sources/source_acquisition_components.py @@ -29,7 +29,7 @@ _DETECTION_PREFIX_SIZE = 8192 # 8 KB — enough for provider detection _HEARTBEAT_INTERVAL_S = 5.0 -_ZIP_MEMBER_RAW_ID_DOMAIN = b"polylogue:zip-member-raw:v1\0" +_ZIP_MEMBER_RAW_ID_DOMAIN = b"polylogue:zip-member-raw:v2\0" AcquisitionObservation: TypeAlias = JSONDocument ObservationCallback: TypeAlias = Callable[[AcquisitionObservation], None] @@ -37,19 +37,41 @@ CursorState: TypeAlias = CursorStatePayload -def zip_member_raw_id(source_path: str, source_index: int, blob_hash: str) -> str: +def zip_member_source_index(*, entry_ordinal: int, split_index: int) -> int: + """Encode a ZIP entry/split coordinate into the persisted integer index. + + Cantor pairing is collision-free for all non-negative integer pairs, so + duplicate central-directory names remain distinct while multiple sessions + split from one member retain their own independent coordinate axis. + """ + if entry_ordinal < 0 or split_index < 0: + raise ValueError("ZIP entry ordinal and split index must be non-negative") + diagonal = entry_ordinal + split_index + return diagonal * (diagonal + 1) // 2 + split_index + + +def zip_member_raw_id( + *, + source_path: str, + entry_ordinal: int, + split_index: int, + blob_hash: str, +) -> str: """Identify one ZIP coordinate without giving up blob-level deduplication. ZIP exports legitimately contain duplicate member bytes. The blob hash remains their shared immutable storage address, while raw authority must retain each admitted ``:`` coordinate independently. - ``source_index`` additionally distinguishes duplicate member names. + The central-directory ordinal distinguishes duplicate member names and the + independent split index distinguishes sessions decoded from one member. """ digest = sha256() digest.update(_ZIP_MEMBER_RAW_ID_DOMAIN) digest.update(source_path.encode("utf-8", errors="surrogatepass")) digest.update(b"\0") - digest.update(str(source_index).encode("utf-8")) + digest.update(str(entry_ordinal).encode("utf-8")) + digest.update(b"\0") + digest.update(str(split_index).encode("utf-8")) digest.update(b"\0") digest.update(bytes.fromhex(blob_hash)) return digest.hexdigest() diff --git a/polylogue/storage/repair.py b/polylogue/storage/repair.py index 08d814f006..abdc334dfd 100644 --- a/polylogue/storage/repair.py +++ b/polylogue/storage/repair.py @@ -5613,6 +5613,16 @@ def _internal_derived_repair_result( ) +def raw_materialization_lease_refusal_result(error: BaseException) -> RepairResult: + """Translate active-generation lease refusal into the repair contract.""" + return _internal_derived_repair_result( + "raw_materialization", + repaired_count=0, + success=False, + detail=f"Skipped raw materialization while offline index rebuild owns archive: {error}", + ) + + def _archive_debt_status( target_name: str, *, @@ -6313,12 +6323,7 @@ def run() -> RepairResult: try: lease.acquire() except RebuildLeaseUnavailableError as exc: - return _internal_derived_repair_result( - "raw_materialization", - repaired_count=0, - success=False, - detail=f"Skipped raw materialization while offline index rebuild owns archive: {exc}", - ) + return raw_materialization_lease_refusal_result(exc) try: return run() finally: @@ -7474,6 +7479,7 @@ def run_selected_maintenance( "preview_superseded_raw_snapshots", "preview_message_type_backfill", "preview_session_insights", + "raw_materialization_lease_refusal_result", "raw_materialization_replay_backlog", "raw_materialization_scale_profile", "repair_empty_sessions", diff --git a/tests/unit/daemon/test_daemon_cli.py b/tests/unit/daemon/test_daemon_cli.py index d78c43e1cb..2ce35e070b 100644 --- a/tests/unit/daemon/test_daemon_cli.py +++ b/tests/unit/daemon/test_daemon_cli.py @@ -1161,6 +1161,70 @@ def close_fts(index_db: Path, *, ops_db_path: Path) -> None: assert held == 0 +@pytest.mark.parametrize("whale", [False, True]) +def test_raw_materialization_outer_lease_refusal_preserves_typed_result( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + whale: bool, +) -> None: + """Both daemon routes must emit the repair contract when pinning is refused.""" + from polylogue.daemon import cli as daemon_cli + from polylogue.storage.index_generation import ActiveWriterLease, RebuildLeaseUnavailableError + from polylogue.storage.repair import RepairResult + + archive = tmp_path / "archive" + archive.mkdir() + emitted: list[RepairResult] = [] + + class FakeRestoreResult: + restored_count = 0 + + def refuse_outer_lease(_lease: ActiveWriterLease) -> None: + raise RebuildLeaseUnavailableError("offline rebuild is active") + + def reject_repair(*_args: object, **_kwargs: object) -> None: + raise AssertionError("repair must not run when the outer generation pin is refused") + + monkeypatch.setattr("polylogue.paths.archive_root", lambda: archive) + monkeypatch.setattr("polylogue.paths.render_root", lambda: tmp_path / "render") + monkeypatch.setattr("polylogue.readiness.capability.raw_frontier_source_selection_block_reason", lambda _root: None) + monkeypatch.setattr( + "polylogue.storage.blob_integrity.restore_direct_blob_reference_debt", + lambda *_args, **_kwargs: FakeRestoreResult(), + ) + monkeypatch.setattr("polylogue.product.raw_authority.recover_interrupted_frontier", lambda _config: ()) + monkeypatch.setattr("polylogue.product.raw_authority.auto_resolve_stale_plan_blockers", lambda _config: 0) + monkeypatch.setattr("polylogue.product.raw_authority.repair_materialization", reject_repair) + monkeypatch.setattr(ActiveWriterLease, "acquire", refuse_outer_lease) + monkeypatch.setattr(daemon_cli, "_emit_raw_materialization_pass", emitted.append) + monkeypatch.setattr(daemon_cli, "_converge_raw_authority_frontier", lambda _config, **_kwargs: 0) + monkeypatch.setattr( + daemon_cli, + "_close_raw_materialization_fts", + lambda *_args, **_kwargs: pytest.fail("FTS closure requires an acquired generation pin"), + ) + + if whale: + returned = daemon_cli._run_raw_materialization_whale_pass_once( + raw_artifact_id="raw-whale", + max_payload_bytes=123, + ) + assert returned is emitted[0] + else: + counts = daemon_cli._drain_raw_materialization_once() + assert counts.repaired_sessions == 0 + + assert len(emitted) == 1 + result = emitted[0] + assert isinstance(result, RepairResult) + assert result.name == "raw_materialization" + assert result.success is False + assert result.repaired_count == 0 + assert result.detail == ( + "Skipped raw materialization while offline index rebuild owns archive: offline rebuild is active" + ) + + def test_raw_materialization_fts_failure_records_durable_debt( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index bcaa437cec..5558909a84 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -2343,6 +2343,34 @@ def test_checkout_mutation_monitor_watches_packed_refs_when_current_ref_is_packe ) +def test_checkout_mutation_monitor_ignores_shared_packed_refs_when_head_is_detached( + tmp_path: Path, +) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.email", "tests@example.invalid"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.name", "Polylogue Tests"], cwd=tmp_path, check=True) + tracked = tmp_path / "tracked.py" + tracked.write_text("value = 1\n", encoding="utf-8") + subprocess.run(["git", "add", "tracked.py"], cwd=tmp_path, check=True) + subprocess.run(["git", "commit", "-qm", "seed"], cwd=tmp_path, check=True) + subprocess.run(["git", "switch", "--detach", "--quiet"], cwd=tmp_path, check=True) + packed_refs = Path( + subprocess.run( + ["git", "rev-parse", "--path-format=absolute", "--git-path", "packed-refs"], + cwd=tmp_path, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + ) + + monitor = CheckoutMutationMonitor(tmp_path) + monitor._watched_directories() + monitor._record_change(packed_refs) + + assert monitor.finish() == CheckoutMutationObservation(changed=False, unavailable=False) + + @pytest.mark.uses_real_clock("waits for the filesystem watcher to witness a loose ref created from packed authority") def test_checkout_mutation_monitor_observes_packed_nested_branch_ref_change(tmp_path: Path) -> None: subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) diff --git a/tests/unit/sources/test_live_batch_support.py b/tests/unit/sources/test_live_batch_support.py index cbc34aeae0..9f02a0d64f 100644 --- a/tests/unit/sources/test_live_batch_support.py +++ b/tests/unit/sources/test_live_batch_support.py @@ -836,6 +836,56 @@ def test_source_only_zip_replay_resolves_unknown_chatgpt_member_and_keeps_duplic assert conn.execute("SELECT COUNT(*) FROM raw_sessions WHERE origin = 'chatgpt-export'").fetchone() == (2,) +def test_zip_duplicate_member_coordinates_match_normal_and_source_only_routes(tmp_path: Path) -> None: + """Central-directory ordinal and within-member split remain independent.""" + root = tmp_path / "inbox" + root.mkdir() + bundle = root / "duplicates.zip" + member_name = "sessions/duplicate.jsonl" + payload = ( + b'{"type":"session_meta","payload":{"id":"duplicate-coordinate"}}\n' + b'{"type":"response_item","payload":{"type":"message","role":"user",' + b'"content":[{"type":"input_text","text":"retained twice"}]}}\n' + ) + with zipfile.ZipFile(bundle, "w") as zf: + zf.writestr("ignored/readme.txt", b"not admitted") + zf.writestr(member_name, payload) + with pytest.warns(UserWarning, match="Duplicate name"): + zf.writestr(member_name, payload) + + index_db = tmp_path / "index.db" + processor = LiveBatchProcessor( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=index_db))), + (WatchSource(name="codex", root=root),), + cursor=CursorStore(index_db), + parser_fingerprint="test-parser", + ) + blob_store = BlobStore(tmp_path / "blob") + + normal_records, _normal_bytes = processor._extract_zip_member_records( + bundle, + blob_store=blob_store, + fallback_provider=Provider.CODEX, + file_mtime="2026-08-13T00:00:00+00:00", + ) + source_only_result = processor._extract_source_only_zip_member_records( + bundle, + blob_store=blob_store, + fallback_provider=Provider.CODEX, + file_mtime="2026-08-13T00:00:00+00:00", + ) + + assert source_only_result is not None + source_only_records, _source_only_bytes = source_only_result + normal_ids = [raw_id for raw_id, _record in normal_records] + source_only_ids = [raw_id for raw_id, _record in source_only_records] + assert len(normal_ids) == 2 + assert len(set(normal_ids)) == 2 + assert source_only_ids == normal_ids + assert [record.source_index for _raw_id, record in normal_records] == [1, 3] + assert [record.source_index for _raw_id, record in source_only_records] == [1, 3] + + def test_source_only_full_ingest_snapshots_unrecognized_codex_state_without_shape_probe( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/sources/test_revision_backfill.py b/tests/unit/sources/test_revision_backfill.py index 983d21d361..5dd5a12a7c 100644 --- a/tests/unit/sources/test_revision_backfill.py +++ b/tests/unit/sources/test_revision_backfill.py @@ -156,22 +156,17 @@ def test_parse_one_replays_single_session_state_db_bytes_via_temp_spill(tmp_path assert sessions[0].messages[0].text == "hi" -def test_unknown_retained_stream_replay_detects_from_prefix_without_eager_payload( +def test_unknown_retained_stream_replay_scans_past_oversized_first_record_without_eager_payload( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: - """UNKNOWN source-only JSONL reopens the blob as a stream after prefix detection.""" + """UNKNOWN JSONL scans past an oversized first record before streaming replay.""" initialize_active_archive_root(tmp_path) payload = ( + json.dumps({"opaque": "x" * 9_000}, sort_keys=True).encode() + b"\n" b'{"type":"session_meta","payload":{"id":"unknown-stream","timestamp":"2026-06-01T00:00:00Z"}}\n' b'{"type":"response_item","payload":{"type":"message","id":"m1","role":"user",' b'"content":[{"type":"input_text","text":"prefix detected replay"}]}}\n' ) - - def detect_from_prefix(raw_bytes: bytes, *_args: object, **_kwargs: object) -> tuple[Provider, str]: - assert raw_bytes == payload - return Provider.CODEX, "test prefix" - - monkeypatch.setattr(revision_backfill, "detect_provider_from_raw_bytes_evidence", detect_from_prefix) with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: raw_id = archive.write_raw_payload( provider=Provider.UNKNOWN, @@ -189,6 +184,51 @@ def reject_eager_material(_raw_id: str) -> tuple[Provider, bytes, str, RawRevisi assert [session.provider_session_id for session in sessions] == ["unknown-stream"] +def test_unknown_retained_stream_census_worker_scans_past_oversized_first_record( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """The production census worker must discover a later bounded JSONL record.""" + initialize_active_archive_root(tmp_path) + payload = ( + json.dumps({"opaque": "x" * 9_000}, sort_keys=True).encode() + + b"\n" + + b'{"type":"session_meta","payload":{"id":"unknown-worker","timestamp":"2026-06-01T00:00:00Z"}}\n' + + b'{"type":"response_item","payload":{"type":"message","id":"m1","role":"user",' + + b'"content":[{"type":"input_text","text":"worker replay"}]}}\n' + ) + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.UNKNOWN, + payload=payload, + source_path="unknown-worker.jsonl", + acquired_at_ms=1, + ) + provider, blob_hash, source_path, kind, _payload_size = archive.raw_revision_descriptor(raw_id) + + monkeypatch.setattr( + ArchiveBlobPublisher, + "read_all", + lambda *_args, **_kwargs: pytest.fail("UNKNOWN stream census must not eagerly read the blob"), + ) + same_raw_id, sessions, error = revision_backfill.census_parse_worker( + raw_id, + provider.value, + blob_hash, + source_path, + False, + str(tmp_path / "blob"), + str(tmp_path / "source.db"), + kind.value, + None, + ) + + assert same_raw_id == raw_id + assert error is None + assert sessions is not None + assert [session.provider_session_id for session in sessions] == ["unknown-worker"] + + def test_unknown_retained_document_replays_after_complete_payload_detection(tmp_path: Path) -> None: """A complete ChatGPT document must retry UNKNOWN prefix detection. From f80f07da892f86cb3eaa7f5a764d36f76ebe13af Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 21:18:37 +0200 Subject: [PATCH 49/65] fix: preserve unknown JSONL document fallback --- polylogue/sources/revision_backfill.py | 10 ---------- tests/unit/sources/test_revision_backfill.py | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/polylogue/sources/revision_backfill.py b/polylogue/sources/revision_backfill.py index c8654e5e27..0b73d5699e 100644 --- a/polylogue/sources/revision_backfill.py +++ b/polylogue/sources/revision_backfill.py @@ -131,12 +131,6 @@ def _detect_unknown_retained_provider( return Provider.UNKNOWN, last_evidence -def _require_resolved_jsonl_provider(provider: Provider, source_path: str) -> None: - """Refuse an eager fallback when bounded JSONL detection stayed unknown.""" - if provider is Provider.UNKNOWN and is_jsonl_source_path(source_path): - raise ValueError("retained UNKNOWN JSONL provider remained unresolved after bounded record scan") - - def _canonical_authority_logical_key(logical_key: str) -> str: """Normalize transitional provider and public-origin authority prefixes.""" prefix, separator, native_id = logical_key.partition(":") @@ -2082,7 +2076,6 @@ def census_parse_worker( provider, stream_payload, source_path, fallback_id_override=fallback_id_override ) return raw_id, sessions, None - _require_resolved_jsonl_provider(provider, source_path) payload = publisher.read_all(blob_hash) if provider is Provider.UNKNOWN: provider, _evidence = detect_provider_from_raw_bytes_evidence( @@ -2521,7 +2514,6 @@ def parse_retained_raw_sessions(archive: ArchiveStore, raw_id: str) -> list[Pars if is_stream_record_provider(source_path, str(provider)): with archive.open_raw_revision_material(raw_id) as (_stream_provider, payload, stream_path, _stream_kind): return _parse_stream(provider, payload, stream_path, fallback_id_override=fallback_id_override) - _require_resolved_jsonl_provider(provider, source_path) _provider, eager_payload, _source_path, _eager_kind = archive.raw_revision_material(raw_id) if provider is Provider.UNKNOWN: provider, _evidence = detect_provider_from_raw_bytes_evidence( @@ -3343,8 +3335,6 @@ def _detected_provider_for_empty_replay( provider, _evidence = _detect_unknown_retained_provider(payload, source_path) if provider is not Provider.UNKNOWN: return provider - if is_jsonl_source_path(source_path): - return provider _provider, full_payload, _path, _kind = archive.raw_revision_material(raw_id) provider, _evidence = detect_provider_from_raw_bytes_evidence( full_payload, diff --git a/tests/unit/sources/test_revision_backfill.py b/tests/unit/sources/test_revision_backfill.py index 5dd5a12a7c..66669e4d44 100644 --- a/tests/unit/sources/test_revision_backfill.py +++ b/tests/unit/sources/test_revision_backfill.py @@ -229,6 +229,22 @@ def test_unknown_retained_stream_census_worker_scans_past_oversized_first_record assert [session.provider_session_id for session in sessions] == ["unknown-worker"] +def test_unknown_retained_nonstream_jsonl_keeps_complete_payload_fallback(tmp_path: Path) -> None: + """A bounded scan must not remove eager replay for a non-stream provider.""" + initialize_active_archive_root(tmp_path) + payload = json.dumps(_chatgpt_session("large-jsonl-document", "x" * 9_000), sort_keys=True).encode() + b"\n" + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.UNKNOWN, + payload=payload, + source_path="large-chatgpt.jsonl", + acquired_at_ms=1, + ) + sessions = revision_backfill.parse_retained_raw_sessions(archive, raw_id) + + assert [session.provider_session_id for session in sessions] == ["large-jsonl-document"] + + def test_unknown_retained_document_replays_after_complete_payload_detection(tmp_path: Path) -> None: """A complete ChatGPT document must retry UNKNOWN prefix detection. From 5ca621db73b10b12fe1f699698ddf880ff0ec3bb Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 21:37:25 +0200 Subject: [PATCH 50/65] fix: bound oversized unknown replay detection --- polylogue/sources/revision_backfill.py | 112 ++++++++++++------- tests/unit/sources/test_revision_backfill.py | 43 ++++++- 2 files changed, 114 insertions(+), 41 deletions(-) diff --git a/polylogue/sources/revision_backfill.py b/polylogue/sources/revision_backfill.py index 0b73d5699e..dcb65c4c89 100644 --- a/polylogue/sources/revision_backfill.py +++ b/polylogue/sources/revision_backfill.py @@ -18,6 +18,9 @@ from types import TracebackType from typing import BinaryIO, Final, Literal, cast +import ijson +from ijson.common import ObjectBuilder + from polylogue import logging as _polylogue_logging from polylogue.archive.artifact_taxonomy.models import ArtifactClassification, ArtifactKind from polylogue.archive.ingest_flags import ( @@ -46,6 +49,7 @@ from polylogue.sources.codex_state_evidence import write_codex_thread_state_evidence from polylogue.sources.decoders import _iter_json_stream from polylogue.sources.dispatch import ( + detect_provider_evidence, detect_provider_from_raw_bytes_evidence, is_jsonl_source_path, is_stream_record_provider, @@ -82,6 +86,47 @@ _REPLAY_PROVIDER_DETECTION_PREFIX_BYTES: Final[int] = 8192 +def _detect_provider_from_bounded_prefix( + prefix: bytes, + stream_name: str, + *, + record_stream: bool, +) -> tuple[Provider, str]: + """Classify completed structure exposed by a bounded JSON prefix. + + The ordinary raw-byte detector remains the first authority. When the + prefix ends inside one oversized JSON value, ijson still emits every + completed key/value event before that truncated value. Reconstructing + only those completed events preserves structural provider evidence (for + example a Codex ``session_meta`` envelope or a Claude ``sessionId``) + without retaining or completing the oversized record. + """ + provider, evidence = detect_provider_from_raw_bytes_evidence( + prefix, + stream_name, + Provider.UNKNOWN, + truncated_tail_ok=True, + ) + if provider is not Provider.UNKNOWN: + return provider, evidence + + builder = ObjectBuilder() + try: + for event, value in ijson.basic_parse(BytesIO(prefix), use_float=True): + builder.event(event, value) + except ijson.JSONError: + # Premature EOF is expected for an oversized-record prefix. The + # builder retains only values whose lexical token completed inside + # the bound; an unfinished string/object contributes no guessed data. + pass + partial = builder.value + candidate: object = [partial] if record_stream and isinstance(partial, dict) else partial + detected, partial_evidence = detect_provider_evidence(candidate) + if detected is None: + return Provider.UNKNOWN, evidence + return detected, f"bounded partial JSON structure: {partial_evidence}" + + def _detect_unknown_retained_provider( payload: BinaryIO, source_path: str, @@ -101,36 +146,43 @@ def _detect_unknown_retained_provider( """ stream_name = Path(source_path).name if not is_jsonl_source_path(source_path): - return detect_provider_from_raw_bytes_evidence( + return _detect_provider_from_bounded_prefix( payload.read(_REPLAY_PROVIDER_DETECTION_PREFIX_BYTES), stream_name, - Provider.UNKNOWN, - truncated_tail_ok=True, + record_stream=False, ) - # Local import avoids a source-dispatch import cycle during module load; - # this is the same bounded physical-record iterator used by raw-payload - # artifact inspection. - from polylogue.archive.raw_payload.decode import _bounded_raw_lines - last_evidence = "no bounded JSONL record identified a provider; used fallback_provider" - for raw_line, oversized in _bounded_raw_lines( - payload, - max_record_bytes=_REPLAY_PROVIDER_DETECTION_PREFIX_BYTES, - ): - if oversized or raw_line is None: - continue - record_bytes = raw_line.encode("utf-8", errors="surrogatepass") if isinstance(raw_line, str) else raw_line - provider, last_evidence = detect_provider_from_raw_bytes_evidence( - record_bytes, - stream_name, - Provider.UNKNOWN, - ) + read_size = _REPLAY_PROVIDER_DETECTION_PREFIX_BYTES + 1 + while raw_line := payload.readline(read_size): + has_newline = raw_line.endswith(b"\n") + oversized = not has_newline and len(raw_line) > _REPLAY_PROVIDER_DETECTION_PREFIX_BYTES + bounded_record = raw_line[:_REPLAY_PROVIDER_DETECTION_PREFIX_BYTES] + if oversized: + provider, last_evidence = _detect_provider_from_bounded_prefix( + bounded_record, + stream_name, + record_stream=True, + ) + while raw_line and not raw_line.endswith(b"\n"): + raw_line = payload.readline(read_size) + else: + provider, last_evidence = detect_provider_from_raw_bytes_evidence( + bounded_record, + stream_name, + Provider.UNKNOWN, + ) if provider is not Provider.UNKNOWN: return provider, last_evidence return Provider.UNKNOWN, last_evidence +def _require_bounded_provider(provider: Provider, source_path: str) -> None: + """Refuse eager UNKNOWN replay after bounded structural detection.""" + if provider is Provider.UNKNOWN: + raise ValueError(f"retained UNKNOWN provider remained unresolved after bounded scan: {source_path}") + + def _canonical_authority_logical_key(logical_key: str) -> str: """Normalize transitional provider and public-origin authority prefixes.""" prefix, separator, native_id = logical_key.partition(":") @@ -2076,13 +2128,8 @@ def census_parse_worker( provider, stream_payload, source_path, fallback_id_override=fallback_id_override ) return raw_id, sessions, None + _require_bounded_provider(provider, source_path) payload = publisher.read_all(blob_hash) - if provider is Provider.UNKNOWN: - provider, _evidence = detect_provider_from_raw_bytes_evidence( - payload, - Path(source_path).name, - provider, - ) payload_path = None if provider is Provider.HERMES: candidate_path = publisher.blob_path(blob_hash) @@ -2514,13 +2561,8 @@ def parse_retained_raw_sessions(archive: ArchiveStore, raw_id: str) -> list[Pars if is_stream_record_provider(source_path, str(provider)): with archive.open_raw_revision_material(raw_id) as (_stream_provider, payload, stream_path, _stream_kind): return _parse_stream(provider, payload, stream_path, fallback_id_override=fallback_id_override) + _require_bounded_provider(provider, source_path) _provider, eager_payload, _source_path, _eager_kind = archive.raw_revision_material(raw_id) - if provider is Provider.UNKNOWN: - provider, _evidence = detect_provider_from_raw_bytes_evidence( - eager_payload, - Path(source_path).name, - provider, - ) payload_path = archive.blob_path_for_hash(blob_hash) if provider is Provider.HERMES else None return _parse_one( provider, @@ -3335,12 +3377,6 @@ def _detected_provider_for_empty_replay( provider, _evidence = _detect_unknown_retained_provider(payload, source_path) if provider is not Provider.UNKNOWN: return provider - _provider, full_payload, _path, _kind = archive.raw_revision_material(raw_id) - provider, _evidence = detect_provider_from_raw_bytes_evidence( - full_payload, - Path(source_path).name, - stored_provider, - ) return provider diff --git a/tests/unit/sources/test_revision_backfill.py b/tests/unit/sources/test_revision_backfill.py index 66669e4d44..8e220758bb 100644 --- a/tests/unit/sources/test_revision_backfill.py +++ b/tests/unit/sources/test_revision_backfill.py @@ -184,6 +184,39 @@ def reject_eager_material(_raw_id: str) -> tuple[Provider, bytes, str, RawRevisi assert [session.provider_session_id for session in sessions] == ["unknown-stream"] +def test_unknown_retained_oversized_provider_record_never_uses_eager_payload( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """The only provider-defining record may itself exceed the scan bound.""" + initialize_active_archive_root(tmp_path) + payload = ( + json.dumps( + { + "sessionId": "oversized-only-provider-record", + "uuid": "message-1", + "type": "user", + "message": {"role": "user", "content": [{"type": "text", "text": "x" * 9_000}]}, + } + ).encode() + + b"\n" + ) + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.UNKNOWN, + payload=payload, + source_path="oversized-only.jsonl", + acquired_at_ms=1, + ) + + def reject_eager_material(_raw_id: str) -> tuple[Provider, bytes, str, RawRevisionKind]: + raise AssertionError("eager payload read") + + monkeypatch.setattr(archive, "raw_revision_material", reject_eager_material) + sessions = revision_backfill.parse_retained_raw_sessions(archive, raw_id) + + assert [session.provider_session_id for session in sessions] == ["oversized-only-provider-record"] + + def test_unknown_retained_stream_census_worker_scans_past_oversized_first_record( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, @@ -230,9 +263,11 @@ def test_unknown_retained_stream_census_worker_scans_past_oversized_first_record def test_unknown_retained_nonstream_jsonl_keeps_complete_payload_fallback(tmp_path: Path) -> None: - """A bounded scan must not remove eager replay for a non-stream provider.""" + """Positive bounded document evidence may select eager non-stream replay.""" initialize_active_archive_root(tmp_path) - payload = json.dumps(_chatgpt_session("large-jsonl-document", "x" * 9_000), sort_keys=True).encode() + b"\n" + document = _chatgpt_session("large-jsonl-document", "bounded evidence") + document["padding"] = "x" * 9_000 + payload = json.dumps(document, sort_keys=True).encode() + b"\n" with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: raw_id = archive.write_raw_payload( provider=Provider.UNKNOWN, @@ -254,7 +289,9 @@ def test_unknown_retained_document_replays_after_complete_payload_detection(tmp_ archive, rather than testing the detector in isolation. """ initialize_active_archive_root(tmp_path) - payload = _bundle(_chatgpt_session("large-document", "x" * 9_000)) + document = _chatgpt_session("large-document", "bounded evidence") + document["padding"] = "x" * 9_000 + payload = _bundle(document) with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: archive.write_raw_payload( provider=Provider.UNKNOWN, From fbeb3f21f520f8adb679905545473bcedbc4aa47 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 22:00:13 +0200 Subject: [PATCH 51/65] fix: close retained authority review gaps --- polylogue/daemon/convergence_stages.py | 6 +- polylogue/product/raw_authority.py | 4 +- polylogue/sources/live/batch.py | 24 +++- polylogue/sources/revision_backfill.py | 5 +- polylogue/storage/raw_retention.py | 65 +++++++--- .../archive_tiers/revision_governance.py | 14 ++- .../storage/sqlite/archive_tiers/source.py | 3 +- .../sqlite/migrations/source/033.train.json | 72 +++++++++++ .../source/033_detected_raw_provider.sql | 10 ++ .../storage/sqlite/queries/mappers_archive.py | 11 +- polylogue/storage/sqlite/queries/raw_reads.py | 8 +- polylogue/storage/sqlite/queries/raw_state.py | 25 +++- .../storage/sqlite/queries/raw_writes.py | 14 +-- polylogue/storage/sqlite/raw_state_update.py | 5 +- tests/unit/daemon/test_raw_parse_recovery.py | 30 ++++- tests/unit/product/test_raw_authority.py | 23 ++++ tests/unit/sources/test_live_batch_support.py | 77 +++++++++++- tests/unit/sources/test_revision_backfill.py | 118 +++++++++++++++++- tests/unit/storage/test_raw_retention.py | 81 ++++++++++++ 19 files changed, 535 insertions(+), 60 deletions(-) create mode 100644 polylogue/storage/sqlite/migrations/source/033.train.json create mode 100644 polylogue/storage/sqlite/migrations/source/033_detected_raw_provider.sql diff --git a/polylogue/daemon/convergence_stages.py b/polylogue/daemon/convergence_stages.py index a2ce44d18e..095bf84567 100644 --- a/polylogue/daemon/convergence_stages.py +++ b/polylogue/daemon/convergence_stages.py @@ -928,7 +928,11 @@ def _raw_parse_recovery_pending_count(db_path: Path, path: Path, *, archive_root WHERE (r.source_path = ? OR r.source_path LIKE ?) AND NOT ( COALESCE(r.validation_status, '') = 'failed' - AND r.parsed_at_ms IS NULL + AND ( + r.parsed_at_ms IS NULL + OR r.validated_at_ms IS NULL + OR r.validated_at_ms >= r.parsed_at_ms + ) ) AND ( ( diff --git a/polylogue/product/raw_authority.py b/polylogue/product/raw_authority.py index 02ad9dead3..64bf438de3 100644 --- a/polylogue/product/raw_authority.py +++ b/polylogue/product/raw_authority.py @@ -13,7 +13,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, Final -from polylogue.config import Config +from polylogue.config import Config, active_archive_root from polylogue.core.json import JSONDocument if TYPE_CHECKING: @@ -127,7 +127,7 @@ def materialization_generation_lease(config: Config) -> Iterator[Path]: """Pin one active index generation through a replay-adjacent closure.""" from polylogue.storage.index_generation import ActiveWriterLease - lease = ActiveWriterLease(config.archive_root) + lease = ActiveWriterLease(active_archive_root(config)) lease.acquire() try: yield config.current_db_path() diff --git a/polylogue/sources/live/batch.py b/polylogue/sources/live/batch.py index fd85aa360a..34679c7997 100644 --- a/polylogue/sources/live/batch.py +++ b/polylogue/sources/live/batch.py @@ -1775,9 +1775,6 @@ def _ingest_full_paths_sync( pass_clock_started = pass_started if pass_started is not None else time.monotonic() archive_root = Path(getattr(self._polylogue, "archive_root", self._cursor._db_path.parent)) blob_root = archive_root / "blob" - from polylogue.storage.blob_publication import ArchiveBlobPublisher - - blob_store = ArchiveBlobPublisher(archive_root / "source.db", blob_root) raw_records: list[RawSessionRecord] = [] raw_by_id: dict[str, Path] = {} raw_byte_sizes: dict[Path, int] = {} @@ -1794,8 +1791,19 @@ def _ingest_full_paths_sync( acquisition_capture_mode = fallback_provider source_only = _source_tier_acquisition_required() + source_db = archive_root / "source.db" + if source_only and not source_db.is_file(): + logger.error("source-only acquisition refused because the durable source tier is missing: %s", source_db) + return _FullIngestResult( + succeeded=[], + failed=list(paths), + source_payload_read_bytes=0, + ) + from polylogue.storage.blob_publication import ArchiveBlobPublisher + + blob_store = ArchiveBlobPublisher(source_db, blob_root) archive_active = self._archive_active(archive_root) - archive_bootstrapped = not archive_active and (not source_only or not (archive_root / "source.db").exists()) + archive_bootstrapped = not archive_active and not source_only if archive_bootstrapped: initialize_archive_root(archive_root) archive_active = self._archive_active(archive_root) @@ -1972,6 +1980,14 @@ def _ingest_full_paths_sync( # derived tier to consume their result. Preserve the original # bytes under the configured source identity and let the # normal raw replay classify them once the index is available. + # Antigravity brain metadata is the one path whose parser also + # reads a mutable sibling artifact. Until the derived route can + # consume both contemporaneously, leave this observation + # pending instead of advancing a cursor backed by only half of + # its material. + if fallback_provider is Provider.ANTIGRAVITY and path.name.endswith(".metadata.json"): + failed.append(path) + continue provider = fallback_provider source_name = provider.value try: diff --git a/polylogue/sources/revision_backfill.py b/polylogue/sources/revision_backfill.py index dcb65c4c89..d5372f2e62 100644 --- a/polylogue/sources/revision_backfill.py +++ b/polylogue/sources/revision_backfill.py @@ -815,7 +815,7 @@ def apply_outcome( [], parser_fingerprint=RAW_AUTHORITY_PARSER_FINGERPRINT, censused_at_ms=0, - retire_full_revision_governance=revision_kind is not RawRevisionKind.UNKNOWN, + retire_full_revision_governance=revision_kind is RawRevisionKind.FULL, manage_transaction=False, ) if not terminalized: @@ -952,6 +952,7 @@ def bind_byte_proven_older_member(raw_id: str, logical_key: str) -> None: continue apply_outcome(raw_id, source_index, parsed_outcomes) if head_by_older: + source_index_by_raw_id = dict(pending_rows) head_to_key = { raw_id: key for key, raw_ids in state.provisional_full_raw_ids.items() for raw_id in raw_ids } @@ -976,7 +977,7 @@ def bind_byte_proven_older_member(raw_id: str, logical_key: str) -> None: if resolved_key is not None: bind_byte_proven_older_member(older_raw_id, resolved_key) else: - apply_outcome(older_raw_id, 0, fallback_outcomes) + apply_outcome(older_raw_id, source_index_by_raw_id[older_raw_id], fallback_outcomes) if census_selection is None: break expanded, _keys = archive.expand_raw_membership_selection(list(census_selection)) diff --git a/polylogue/storage/raw_retention.py b/polylogue/storage/raw_retention.py index c251ad8930..a6648ec0d6 100644 --- a/polylogue/storage/raw_retention.py +++ b/polylogue/storage/raw_retention.py @@ -441,6 +441,8 @@ def active_raw_retention_authority( protected_raw_ids=protected_ids, eligible_raw_ids=frozenset(eligible.difference(protected_ids)), ) + except sqlite3.Error as exc: + raise RawRetentionSafetyError(f"raw retention authority is unreadable: {exc}") from exc finally: conn.row_factory = original_row_factory @@ -1675,7 +1677,8 @@ def _terminal_artifact_paths(conn: sqlite3.Connection, source_paths: set[str]) - terminal_raw_failure_kinds = tuple(sorted(_TERMINAL_RAW_FAILURE_EVIDENCE_KINDS)) raw_failure_placeholders = ", ".join("?" for _ in raw_failure_kinds) terminal_raw_failure_placeholders = ", ".join("?" for _ in terminal_raw_failure_kinds) - path_batch_size = max(1, 500 - len(raw_failure_kinds) - len(terminal_raw_failure_kinds)) + # The path batch binds once for observation receipts and once for raw rows. + path_batch_size = max(1, (500 - len(raw_failure_kinds) - len(terminal_raw_failure_kinds)) // 2) pending = set(source_paths) while pending: batch = tuple(sorted(pending)[:path_batch_size]) @@ -1683,25 +1686,45 @@ def _terminal_artifact_paths(conn: sqlite3.Connection, source_paths: set[str]) - placeholders = ", ".join("?" for _ in batch) rows = conn.execute( f""" - WITH newest_per_coordinate AS ( + WITH latest_raw_observation AS ( + SELECT raw_id, acquired_at_ms, observation_rowid + FROM ( + SELECT + ref_id AS raw_id, + acquired_at_ms, + rowid AS observation_rowid, + ROW_NUMBER() OVER ( + PARTITION BY ref_id + ORDER BY acquired_at_ms DESC, rowid DESC + ) AS observation_rank + FROM blob_refs + WHERE ref_type = 'raw_payload' + AND source_path IN ({placeholders}) + ) + WHERE observation_rank = 1 + ), + newest_per_coordinate AS ( SELECT raw_id, source_path, origin, source_index, parse_error, validation_status, validated_at_ms, parsed_at_ms FROM ( SELECT - raw_id, - source_path, - origin, - source_index, - parse_error, - validation_status, - validated_at_ms, - parsed_at_ms, + raw.raw_id, + raw.source_path, + raw.origin, + raw.source_index, + raw.parse_error, + raw.validation_status, + raw.validated_at_ms, + raw.parsed_at_ms, ROW_NUMBER() OVER ( - PARTITION BY source_path, origin, source_index - ORDER BY acquired_at_ms DESC, rowid DESC + PARTITION BY raw.source_path, raw.origin, raw.source_index + ORDER BY + COALESCE(observation.acquired_at_ms, raw.acquired_at_ms) DESC, + COALESCE(observation.observation_rowid, raw.rowid) DESC ) AS coordinate_rank - FROM raw_sessions - WHERE source_path IN ({placeholders}) + FROM raw_sessions AS raw + LEFT JOIN latest_raw_observation AS observation ON observation.raw_id = raw.raw_id + WHERE raw.source_path IN ({placeholders}) ) WHERE coordinate_rank = 1 ), @@ -1736,9 +1759,17 @@ def _terminal_artifact_paths(conn: sqlite3.Connection, source_paths: set[str]) - ) ) ) + ), + terminal_evidence AS ( + SELECT raw_id FROM terminal_artifacts + UNION + SELECT evidence_raw.raw_id + FROM newest_per_coordinate AS evidence_raw + JOIN raw_membership_census AS census ON census.raw_id = evidence_raw.raw_id + WHERE census.status = 'non_session' ) SELECT DISTINCT terminal_raw.source_path - FROM terminal_artifacts AS artifact + FROM terminal_evidence AS artifact JOIN newest_per_coordinate AS terminal_raw ON terminal_raw.raw_id = artifact.raw_id WHERE NOT EXISTS ( SELECT 1 @@ -1746,12 +1777,12 @@ def _terminal_artifact_paths(conn: sqlite3.Connection, source_paths: set[str]) - WHERE coordinate.source_path = terminal_raw.source_path AND NOT EXISTS ( SELECT 1 - FROM terminal_artifacts AS current_artifact + FROM terminal_evidence AS current_artifact WHERE current_artifact.raw_id = coordinate.raw_id ) ) """, - (*batch, *raw_failure_kinds, *terminal_raw_failure_kinds), + (*batch, *batch, *raw_failure_kinds, *terminal_raw_failure_kinds), ).fetchall() result.update(str(row[0]) for row in rows) return result diff --git a/polylogue/storage/sqlite/archive_tiers/revision_governance.py b/polylogue/storage/sqlite/archive_tiers/revision_governance.py index 75b8f28304..40df0a0888 100644 --- a/polylogue/storage/sqlite/archive_tiers/revision_governance.py +++ b/polylogue/storage/sqlite/archive_tiers/revision_governance.py @@ -1516,7 +1516,7 @@ def raw_revision_descriptor( store._ensure_source_conn() .execute( """ - SELECT origin, capture_mode, lower(hex(blob_hash)), source_path, revision_kind, blob_size + SELECT origin, detected_provider, capture_mode, lower(hex(blob_hash)), source_path, revision_kind, blob_size FROM raw_sessions WHERE raw_id = ? """, (raw_id,), @@ -1526,11 +1526,15 @@ def raw_revision_descriptor( if row is None: raise KeyError(raw_id) return ( - provider_from_origin(Origin.from_string(str(row[0])), family_hint=row[1]), - str(row[2]), + ( + Provider.from_string(str(row[1])) + if row[1] is not None + else provider_from_origin(Origin.from_string(str(row[0])), family_hint=row[2]) + ), str(row[3]), - RawRevisionKind(str(row[4])), - int(row[5]), + str(row[4]), + RawRevisionKind(str(row[5])), + int(row[6]), ) diff --git a/polylogue/storage/sqlite/archive_tiers/source.py b/polylogue/storage/sqlite/archive_tiers/source.py index 344f567149..6e15ca02d7 100644 --- a/polylogue/storage/sqlite/archive_tiers/source.py +++ b/polylogue/storage/sqlite/archive_tiers/source.py @@ -22,7 +22,7 @@ from polylogue.storage.sqlite.archive_tiers.types import ProvenRevisionAuthority from polylogue.storage.sqlite.audit_continuity import AUDIT_CONTINUITY_GENESIS_HEAD_SHA256 -SOURCE_SCHEMA_VERSION = 32 +SOURCE_SCHEMA_VERSION = 33 SOURCE_DDL = f""" CREATE TABLE IF NOT EXISTS raw_sessions ( @@ -58,6 +58,7 @@ CHECK ({check("revision_authority", RawRevisionAuthority)}) ,revision_authority_evidence TEXT CHECK(revision_authority_evidence IS NULL OR revision_authority_evidence IN ('live_source_verification_v1')) + ,detected_provider TEXT CHECK ({nullable_check("detected_provider", Provider)}) ) STRICT; CREATE INDEX IF NOT EXISTS idx_raw_sessions_origin diff --git a/polylogue/storage/sqlite/migrations/source/033.train.json b/polylogue/storage/sqlite/migrations/source/033.train.json new file mode 100644 index 0000000000..b6b23c8864 --- /dev/null +++ b/polylogue/storage/sqlite/migrations/source/033.train.json @@ -0,0 +1,72 @@ +{ + "manifest_format": "polylogue.durable-change-train.v1", + "train_id": "train:source:v33", + "tier": "source", + "current_version": 32, + "target_version": 33, + "slot": 33, + "owner_ref": "github:pull/3952#discussion_r3775839929", + "migration": { + "tier": "source", + "target_version": 33, + "slot": 33, + "path": "033_detected_raw_provider.sql", + "owner_ref": "polylogue/storage/sqlite/migrations/source/033_detected_raw_provider.sql", + "sql_sha256": "0952a74b0a397a83f00ef45a618db1e084337cd1413409894b8811d234e9babc", + "requires_backup": true + }, + "riders": [ + { + "rider_id": "rider:detected-raw-provider", + "owner_ref": "github:pull/3952#discussion_r3775839929", + "schema_objects": ["column:raw_sessions.detected_provider"], + "runtime_consumers": [ + { + "consumer_id": "raw-state-update", + "production_ref": "polylogue.storage.sqlite.raw_state_update:compile_raw_state_update", + "behavior_proof_ref": "proof:source-v33:preserve-acquisition-origin", + "roles": ["write"] + }, + { + "consumer_id": "revision-provider-resolution", + "production_ref": "polylogue.storage.sqlite.archive_tiers.revision_governance:raw_revision_descriptor", + "behavior_proof_ref": "proof:source-v33:reuse-detected-provider", + "roles": ["read"] + }, + { + "consumer_id": "raw-record-hydration", + "production_ref": "polylogue.storage.sqlite.queries.mappers_archive:_row_to_raw_session", + "behavior_proof_ref": "proof:source-v33:split-acquisition-and-parser-identity", + "roles": ["read"] + } + ], + "behavior_proof_refs": [ + "proof:source-v33:preserve-acquisition-origin", + "proof:source-v33:reuse-detected-provider", + "proof:source-v33:split-acquisition-and-parser-identity" + ], + "after_rider_ids": [], + "trust_floor_exception_ref": null + } + ], + "ordering_constraints": [], + "drop_constraints": [], + "row_change_allowances": [], + "backup_plan_ref": "backup-profile:source-tier", + "state": "declared", + "revision": 0, + "declared_at_ms": 0, + "admitted_at_ms": null, + "admission_evidence_ref": null, + "fresh_ddl_parity": null, + "reservation": null, + "backup_authorization": null, + "pre_apply_evidence": null, + "apply_evidence": null, + "proof": null, + "failure": null, + "released_at_ms": null, + "release_evidence_ref": null, + "proof_refs": [], + "manifest_sha256": "587200f83f1b9f9ed677412c95910592f8cb2dcbc51c1e3dbfc6098319b5587e" +} diff --git a/polylogue/storage/sqlite/migrations/source/033_detected_raw_provider.sql b/polylogue/storage/sqlite/migrations/source/033_detected_raw_provider.sql new file mode 100644 index 0000000000..256f4bd532 --- /dev/null +++ b/polylogue/storage/sqlite/migrations/source/033_detected_raw_provider.sql @@ -0,0 +1,10 @@ +-- Parser classification is not acquisition identity. Retain the exact +-- provider-wire result separately so replay can reuse it without rewriting +-- raw_sessions.origin and breaking deterministic reacquisition. +ALTER TABLE raw_sessions ADD COLUMN detected_provider TEXT CHECK ( + (detected_provider IN ( + 'chatgpt', 'claude-ai', 'claude-design', 'claude-code', 'codex', + 'gemini', 'gemini-cli', 'hermes', 'antigravity', 'beads', 'grok', + 'drive', 'unknown' + ) OR detected_provider IS NULL) +); diff --git a/polylogue/storage/sqlite/queries/mappers_archive.py b/polylogue/storage/sqlite/queries/mappers_archive.py index 67423a6851..355d732195 100644 --- a/polylogue/storage/sqlite/queries/mappers_archive.py +++ b/polylogue/storage/sqlite/queries/mappers_archive.py @@ -180,14 +180,15 @@ def _row_to_raw_session(row: sqlite3.Row) -> RawSessionRecord: validation_mode = _row_text(row, "validation_mode") blob_hash_value = _row_get(row, "blob_hash") blob_hash = bytes(blob_hash_value).hex() if isinstance(blob_hash_value, (bytes, bytearray)) else None - # raw_sessions carries a single ``origin`` column (#1743). The in-memory - # record still exposes provider-wire ``source_name``/``payload_provider``; - # both project from the stored origin. + # Acquisition origin is immutable raw identity. A later parser may retain + # its exact provider separately without rewriting that identity. capture_mode = _row_text(row, "capture_mode") - provider = provider_from_origin( + detected_provider = _row_text(row, "detected_provider") + acquisition_provider = provider_from_origin( Origin.from_string(row["origin"]), family_hint=capture_mode, ) + provider = Provider.from_string(detected_provider) if detected_provider is not None else acquisition_provider logical_source_key = _row_text(row, "logical_source_key") source_revision = _row_text(row, "source_revision") generation = _row_int(row, "acquisition_generation") @@ -210,7 +211,7 @@ def _row_to_raw_session(row: sqlite3.Row) -> RawSessionRecord: blob_hash=blob_hash, payload_provider=provider, capture_mode=Provider.from_string(capture_mode) if capture_mode is not None else None, - source_name=provider.value, + source_name=acquisition_provider.value, source_path=row["source_path"], source_index=row["source_index"], blob_size=row["blob_size"], diff --git a/polylogue/storage/sqlite/queries/raw_reads.py b/polylogue/storage/sqlite/queries/raw_reads.py index 02242b643d..596197780a 100644 --- a/polylogue/storage/sqlite/queries/raw_reads.py +++ b/polylogue/storage/sqlite/queries/raw_reads.py @@ -243,6 +243,7 @@ async def get_raw_session_states(conn: aiosqlite.Connection, raw_ids: list[str]) SELECT raw_id, origin, + detected_provider, capture_mode, source_path, parsed_at_ms, @@ -256,7 +257,12 @@ async def get_raw_session_states(conn: aiosqlite.Connection, raw_ids: list[str]) rows = await cursor.fetchall() def _state(row: aiosqlite.Row) -> RawSessionState: - provider = provider_from_origin(Origin.from_string(row["origin"]), family_hint=row["capture_mode"]) + detected_provider = row["detected_provider"] + provider = ( + Provider.from_string(str(detected_provider)) + if detected_provider is not None + else provider_from_origin(Origin.from_string(row["origin"]), family_hint=row["capture_mode"]) + ) return RawSessionState( raw_id=row["raw_id"], source_name=provider.value, diff --git a/polylogue/storage/sqlite/queries/raw_state.py b/polylogue/storage/sqlite/queries/raw_state.py index 93f707ee69..c2b0a852da 100644 --- a/polylogue/storage/sqlite/queries/raw_state.py +++ b/polylogue/storage/sqlite/queries/raw_state.py @@ -12,9 +12,28 @@ from polylogue.storage.sqlite.connection import _build_source_scope_filter from polylogue.storage.sqlite.raw_state_update import compile_raw_state_update -# raw_sessions carries a single ``origin`` column (#1743). Provider-token -# filters translate the token to its canonical origin value before matching. -RAW_ORIGIN_FILTER_SQL = "origin" +# Raw filters follow parser-classified provider evidence when present while +# preserving immutable acquisition origin as the fallback. The CASE keeps the +# comparison in public Origin vocabulary even though detected_provider retains +# the exact provider-wire token (including the Gemini/Drive fiber). +RAW_ORIGIN_FILTER_SQL = """ +CASE detected_provider + WHEN 'chatgpt' THEN 'chatgpt-export' + WHEN 'claude-ai' THEN 'claude-ai-export' + WHEN 'claude-design' THEN 'claude-design-session' + WHEN 'claude-code' THEN 'claude-code-session' + WHEN 'codex' THEN 'codex-session' + WHEN 'gemini' THEN 'aistudio-drive' + WHEN 'drive' THEN 'aistudio-drive' + WHEN 'gemini-cli' THEN 'gemini-cli-session' + WHEN 'hermes' THEN 'hermes-session' + WHEN 'antigravity' THEN 'antigravity-session' + WHEN 'beads' THEN 'beads-issue' + WHEN 'grok' THEN 'grok-export' + WHEN 'unknown' THEN 'unknown-export' + ELSE origin +END +""".strip() def origin_filter_value(token: str) -> str: diff --git a/polylogue/storage/sqlite/queries/raw_writes.py b/polylogue/storage/sqlite/queries/raw_writes.py index 2025a31f0d..b685eace3d 100644 --- a/polylogue/storage/sqlite/queries/raw_writes.py +++ b/polylogue/storage/sqlite/queries/raw_writes.py @@ -17,12 +17,9 @@ async def save_raw_session( record: RawSessionRecord, transaction_depth: int, ) -> bool: - # payload_provider wins when the payload has been classified; otherwise fall - # back to the source_name token (#1743 collapses both onto origin). - if record.payload_provider is not None: - origin = origin_from_provider(record.payload_provider) - else: - origin = origin_from_provider(Provider.from_string(record.source_name or "unknown")) + acquisition_provider = Provider.from_string(record.source_name or "unknown") + origin = origin_from_provider(acquisition_provider) + detected_provider = record.payload_provider # Only the acquisition path can assert a capture mode. A hydrated legacy # row has ``None`` here even though its compatibility projection supplies # a canonical payload provider; writing that projection back must not turn @@ -40,17 +37,18 @@ async def save_raw_session( cursor = await conn.execute( """ INSERT OR IGNORE INTO raw_sessions ( - raw_id, origin, capture_mode, native_id, source_path, source_index, blob_hash, + raw_id, origin, detected_provider, capture_mode, native_id, source_path, source_index, blob_hash, blob_size, acquired_at_ms, file_mtime_ms, parsed_at_ms, parse_error, validated_at_ms, validation_status, validation_error, validation_drift_count, validation_mode, detection_warnings_json, logical_source_key, revision_kind, source_revision, predecessor_source_revision, predecessor_raw_id, baseline_raw_id, append_start_offset, append_end_offset, acquisition_generation, revision_authority, revision_authority_evidence - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( record.raw_id, origin.value, + detected_provider.value if detected_provider is not None else None, capture_mode.value if capture_mode is not None else None, None, record.source_path, diff --git a/polylogue/storage/sqlite/raw_state_update.py b/polylogue/storage/sqlite/raw_state_update.py index 7976289122..de858d7031 100644 --- a/polylogue/storage/sqlite/raw_state_update.py +++ b/polylogue/storage/sqlite/raw_state_update.py @@ -5,7 +5,6 @@ import json from polylogue.core.enums import Provider, ValidationMode, ValidationStatus -from polylogue.core.sources import origin_from_provider from polylogue.storage.raw.models import UNSET, RawSessionStateUpdate, _RawStateUnset from polylogue.storage.sqlite.archive_tiers.write import _timestamp_ms @@ -65,8 +64,8 @@ def compile_raw_state_update( provider = state.payload_provider elif isinstance(state.validation_provider, Provider): provider = state.validation_provider - set_clauses.append("origin = COALESCE(?, origin)") - params.append(origin_from_provider(provider).value if provider is not None else None) + set_clauses.append("detected_provider = COALESCE(?, detected_provider)") + params.append(provider.value if provider is not None else None) if state.detection_warnings is not UNSET: warnings = state.detection_warnings set_clauses.append("detection_warnings_json = ?") diff --git a/tests/unit/daemon/test_raw_parse_recovery.py b/tests/unit/daemon/test_raw_parse_recovery.py index 56087ce26b..be37836da6 100644 --- a/tests/unit/daemon/test_raw_parse_recovery.py +++ b/tests/unit/daemon/test_raw_parse_recovery.py @@ -320,7 +320,7 @@ def test_raw_parse_recovery_skips_validation_failed_cas_frontier_failure(tmp_pat def test_raw_parse_recovery_drains_previously_parsed_cas_frontier_failure(tmp_path: Path) -> None: - """CAS authority replays an unmaterialized raw even when parsing had completed.""" + """A stale validation failure does not suppress newer parse authority.""" initialize_active_archive_root(tmp_path) path = tmp_path / "previously-parsed-cas-frontier.json" raw_id = _write_stuck_raw(tmp_path, source_path=str(path)) @@ -338,7 +338,7 @@ def test_raw_parse_recovery_drains_previously_parsed_cas_frontier_failure(tmp_pa ) conn.execute( "UPDATE raw_sessions SET validation_status = 'failed', validated_at_ms = ? WHERE raw_id = ?", - (parsed_at_ms, raw_id), + (parsed_at_ms - 1, raw_id), ) assert conn.total_changes == 1 conn.commit() @@ -351,6 +351,32 @@ def test_raw_parse_recovery_drains_previously_parsed_cas_frontier_failure(tmp_pa assert _sessions_for_raw(tmp_path, raw_id) == [("conv-stuck", raw_id)] +def test_raw_parse_recovery_skips_current_validation_failure_after_prior_parse(tmp_path: Path) -> None: + """A current validation failure cannot leave CAS recovery permanently pending.""" + initialize_active_archive_root(tmp_path) + path = tmp_path / "current-validation-failed-cas-frontier.json" + raw_id = _write_stuck_raw(tmp_path, source_path=str(path)) + + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + archive.mark_raw_parse_succeeded(raw_id, provider=Provider.CHATGPT) + archive.mark_raw_parse_failed( + raw_id, + provider=Provider.CHATGPT, + error=RawCASFrontierError("frontier changed before current validation failure"), + ) + with sqlite3.connect(tmp_path / "source.db") as conn: + parsed_at_ms = int( + conn.execute("SELECT parsed_at_ms FROM raw_sessions WHERE raw_id = ?", (raw_id,)).fetchone()[0] + ) + conn.execute( + "UPDATE raw_sessions SET validation_status = 'failed', validated_at_ms = ? WHERE raw_id = ?", + (parsed_at_ms, raw_id), + ) + conn.commit() + + assert make_raw_parse_recovery_stage(tmp_path / "index.db").check(path) is False + + def test_raw_parse_recovery_source_open_failure_is_failed_and_retryable( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/tests/unit/product/test_raw_authority.py b/tests/unit/product/test_raw_authority.py index efedba2483..e6e47d18a9 100644 --- a/tests/unit/product/test_raw_authority.py +++ b/tests/unit/product/test_raw_authority.py @@ -104,6 +104,29 @@ def test_materialization_generation_lease_pins_active_index_and_excludes_promoti pass +def test_materialization_generation_lease_uses_explicit_split_root(tmp_path: Path) -> None: + configured_root = tmp_path / "configured" + active_root = tmp_path / "active" + configured_root.mkdir() + active_root.mkdir() + active_index = active_root / "index.db" + active_index.touch() + config = Config( + archive_root=configured_root, + render_root=tmp_path / "render", + sources=[], + db_path=active_index, + ) + + with raw_authority.materialization_generation_lease(config) as index_db: + assert index_db == active_index + with pytest.raises(RebuildLeaseUnavailableError): + with RebuildLease(active_root): + pass + with RebuildLease(configured_root): + pass + + @pytest.mark.parametrize( ("selected_plan_ids", "preview_census_id", "outcome_plan_id", "message"), [ diff --git a/tests/unit/sources/test_live_batch_support.py b/tests/unit/sources/test_live_batch_support.py index 9f02a0d64f..ae7c8d40bc 100644 --- a/tests/unit/sources/test_live_batch_support.py +++ b/tests/unit/sources/test_live_batch_support.py @@ -635,6 +635,72 @@ def test_full_ingest_acquires_but_does_not_parse_when_derived_tier_degraded( assert artifact_rows == (0,) +def test_source_only_full_ingest_refuses_missing_durable_source_tier(tmp_path: Path) -> None: + """An established archive cannot silently bootstrap over source.db loss.""" + from polylogue.core.degraded import DegradedReason, clear_degraded, set_degraded + + initialize_active_archive_root(tmp_path) + (tmp_path / "source.db").unlink() + root = tmp_path / "sessions" + root.mkdir() + path = root / "pending.jsonl" + path.write_text('{"opaque":"must remain pending"}\n', encoding="utf-8") + index_db = tmp_path / "index.db" + processor = LiveBatchProcessor( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=index_db))), + (WatchSource(name="claude-code", root=root),), + cursor=CursorStore(index_db), + parser_fingerprint="test-parser", + ) + + set_degraded(DegradedReason(code="schema_version_mismatch", message="index unavailable", derived_only=True)) + try: + result = processor._ingest_full_paths_sync([path], source_name="claude-code") + finally: + clear_degraded() + + assert result.succeeded == [] + assert result.failed == [path] + assert result.source_payload_read_bytes == 0 + assert not (tmp_path / "source.db").exists() + + +def test_source_only_antigravity_metadata_stays_pending_with_mutable_companion(tmp_path: Path) -> None: + """Cursor authority cannot cover metadata while omitting its sibling bytes.""" + from polylogue.core.degraded import DegradedReason, clear_degraded, set_degraded + + initialize_active_archive_root(tmp_path) + root = tmp_path / "antigravity" + metadata = root / "brain" / "work-session" / "plan.md.metadata.json" + metadata.parent.mkdir(parents=True) + metadata.write_text('{"summary":"plan"}', encoding="utf-8") + companion = metadata.with_name("plan.md") + companion.write_text("contemporaneous body", encoding="utf-8") + index_db = tmp_path / "index.db" + cursor = CursorStore(index_db) + processor = LiveBatchProcessor( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=index_db))), + (WatchSource(name="antigravity", root=root),), + cursor=cursor, + parser_fingerprint="test-parser", + ) + + set_degraded(DegradedReason(code="schema_version_mismatch", message="index unavailable", derived_only=True)) + try: + metrics = asyncio.run(processor.ingest_files([metadata], emit_event=False)) + finally: + clear_degraded() + + assert metrics.succeeded_file_count == 0 + assert metrics.failed_paths == [str(metadata)] + cursor_record = cursor.get_record(metadata) + assert cursor_record is not None + assert cursor_record.excluded is False + assert cursor_record.failure_count == 1 + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT COUNT(*) FROM raw_sessions").fetchone() == (0,) + + def test_source_only_full_ingest_streams_admitted_zip_members_without_decoding( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -642,6 +708,7 @@ def test_source_only_full_ingest_streams_admitted_zip_members_without_decoding( """The production full-ingest ZIP route must retain bytes before decode.""" from polylogue.core.degraded import DegradedReason, clear_degraded, set_degraded + initialize_active_archive_root(tmp_path) root = tmp_path / "sessions" root.mkdir() bundle = root / "degraded.zip" @@ -691,6 +758,7 @@ def test_source_only_zip_read_failure_remains_retryable_after_partial_copy( """The real source-only route must not exclude a transiently unreadable ZIP.""" from polylogue.core.degraded import DegradedReason, clear_degraded, set_degraded + initialize_active_archive_root(tmp_path) root = tmp_path / "sessions" root.mkdir() bundle = root / "retry.zip" @@ -757,6 +825,7 @@ def test_source_only_zip_replay_resolves_unknown_chatgpt_member_and_keeps_duplic """Recovery, not acquisition, resolves UNKNOWN ZIP bytes and replays each coordinate.""" from polylogue.core.degraded import DegradedReason, clear_degraded, set_degraded + initialize_active_archive_root(tmp_path) root = tmp_path / "inbox" root.mkdir() bundle = root / "export.zip" @@ -833,7 +902,10 @@ def test_source_only_zip_replay_resolves_unknown_chatgpt_member_and_keeps_duplic with sqlite3.connect(tmp_path / "index.db") as conn: assert conn.execute("SELECT native_id, message_count FROM sessions").fetchall() == [("zip-chatgpt", 2)] with sqlite3.connect(tmp_path / "source.db") as conn: - assert conn.execute("SELECT COUNT(*) FROM raw_sessions WHERE origin = 'chatgpt-export'").fetchone() == (2,) + assert conn.execute("SELECT origin, detected_provider FROM raw_sessions ORDER BY source_index").fetchall() == [ + ("unknown-export", "chatgpt"), + ("unknown-export", "chatgpt"), + ] def test_zip_duplicate_member_coordinates_match_normal_and_source_only_routes(tmp_path: Path) -> None: @@ -893,6 +965,7 @@ def test_source_only_full_ingest_snapshots_unrecognized_codex_state_without_shap """A degraded source tier retains a valid but future-shaped Codex state DB.""" from polylogue.core.degraded import DegradedReason, clear_degraded, set_degraded + initialize_active_archive_root(tmp_path) root = tmp_path / "codex" root.mkdir() state_db = root / "state_5.sqlite" @@ -957,6 +1030,7 @@ def test_source_only_codex_state_recovery_replays_retained_thread_evidence(tmp_p """Removing the replay effect leaves the durable state raw pending and title-less.""" from polylogue.core.degraded import DegradedReason, clear_degraded, set_degraded + initialize_active_archive_root(tmp_path) root = tmp_path / "codex" state_db = root / "state_5.sqlite" _write_codex_thread_state_db(state_db) @@ -996,6 +1070,7 @@ def test_source_only_hermes_named_sqlite_uses_consistent_backup_before_generic_c """A direct file copy loses an uncheckpointed WAL row; the snapshot retains it.""" from polylogue.core.degraded import DegradedReason, clear_degraded, set_degraded + initialize_active_archive_root(tmp_path) root = tmp_path / "hermes" state_db = root / state_name state_db.parent.mkdir(parents=True) diff --git a/tests/unit/sources/test_revision_backfill.py b/tests/unit/sources/test_revision_backfill.py index 8e220758bb..94fc804658 100644 --- a/tests/unit/sources/test_revision_backfill.py +++ b/tests/unit/sources/test_revision_backfill.py @@ -31,6 +31,7 @@ ) from polylogue.storage.blob_publication import ArchiveBlobPublisher from polylogue.storage.raw_authority import RAW_AUTHORITY_PARSER_FINGERPRINT +from polylogue.storage.raw_retention import RawRetentionAuthority, active_raw_retention_authority from polylogue.storage.sqlite.archive_tiers import revision_governance as archive_revision_governance from polylogue.storage.sqlite.archive_tiers import write as archive_tier_write from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore @@ -721,9 +722,10 @@ def test_backfill_terminalizes_detected_unknown_empty_artifact(tmp_path: Path) - with sqlite3.connect(tmp_path / "source.db") as conn: assert conn.execute( - "SELECT origin, parsed_at_ms IS NOT NULL FROM raw_sessions WHERE raw_id = ?", (raw_id,) + "SELECT origin, detected_provider, parsed_at_ms IS NOT NULL FROM raw_sessions WHERE raw_id = ?", (raw_id,) ).fetchone() == ( - "claude-code-session", + "unknown-export", + "claude-code", 1, ) assert conn.execute("SELECT parse_as_session FROM raw_artifacts WHERE raw_id = ?", (raw_id,)).fetchone() == (0,) @@ -733,7 +735,7 @@ def test_backfill_terminalizes_detected_unknown_empty_artifact(tmp_path: Path) - def test_backfill_persists_detected_provider_for_empty_ordinary_session_path(tmp_path: Path) -> None: - """Provider detection survives even when a session path is not terminalized.""" + """Empty replay retains parser identity without mutating acquisition identity.""" initialize_active_archive_root(tmp_path) source_path = str(tmp_path / ".claude" / "projects" / "proj" / "history-only-session.jsonl") with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: @@ -751,8 +753,8 @@ def test_backfill_persists_detected_provider_for_empty_ordinary_session_path(tmp with sqlite3.connect(tmp_path / "source.db") as conn: assert conn.execute( - "SELECT origin, parsed_at_ms IS NOT NULL FROM raw_sessions WHERE raw_id = ?", (raw_id,) - ).fetchone() == ("claude-code-session", 1) + "SELECT origin, detected_provider, parsed_at_ms IS NOT NULL FROM raw_sessions WHERE raw_id = ?", (raw_id,) + ).fetchone() == ("unknown-export", "claude-code", 1) assert conn.execute("SELECT COUNT(*) FROM raw_artifacts WHERE raw_id = ?", (raw_id,)).fetchone() == (0,) assert conn.execute("SELECT status FROM raw_membership_census WHERE raw_id = ?", (raw_id,)).fetchone() == ( "non_session", @@ -761,6 +763,29 @@ def test_backfill_persists_detected_provider_for_empty_ordinary_session_path(tmp with ArchiveStore.open_existing(tmp_path, read_only=True) as archive: assert archive.raw_membership_census_rows([raw_id])[0][2] + with sqlite3.connect(tmp_path / "source.db") as conn: + assert active_raw_retention_authority( + conn, + index_db_path=tmp_path / "index.db", + ) == RawRetentionAuthority( + protected_raw_ids=frozenset({raw_id}), + eligible_raw_ids=frozenset(), + ) + + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + assert ( + archive.write_raw_payload( + provider=Provider.UNKNOWN, + payload=( + b'{"type":"file-history-snapshot","messageId":"history-message",' + b'"sessionId":"history-only-session","snapshot":{"trackedFileBackups":{}}}\n' + ), + source_path=source_path, + acquired_at_ms=2, + ) + == raw_id + ) + def test_backfill_leaves_undetected_empty_raw_replayable(tmp_path: Path) -> None: """An unknown shape is not terminal merely because it produced no sessions.""" @@ -809,6 +834,89 @@ def test_backfill_retires_stale_revision_governance_for_empty_replay(tmp_path: P ).fetchone() == (None, "unknown", "quarantined") +def test_backfill_preserves_empty_append_revision_governance(tmp_path: Path) -> None: + """A terminal empty APPEND remains reconstructible through its byte envelope.""" + initialize_active_archive_root(tmp_path) + source_path = str(tmp_path / ".claude" / "projects" / "proj" / "append.jsonl") + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.CLAUDE_CODE, + payload=b'{"type":"file-history-snapshot","sessionId":"append-only","snapshot":{}}\n', + source_path=source_path, + source_index=0, + acquired_at_ms=1, + ) + archive.bind_raw_revision( + raw_id, + RawRevisionEnvelope( + logical_source_key="claude-code-session:append-only", + kind=RawRevisionKind.APPEND, + source_revision=raw_id, + predecessor_source_revision="predecessor-revision", + predecessor_raw_id="predecessor-raw", + baseline_raw_id="baseline-raw", + append_start_offset=10, + append_end_offset=20, + acquisition_generation=2, + authority=RawRevisionAuthority.BYTE_PROVEN, + ), + ) + + census_historical_revision_evidence(tmp_path) + + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute( + """ + SELECT logical_source_key, revision_kind, predecessor_source_revision, + predecessor_raw_id, baseline_raw_id, append_start_offset, + append_end_offset, acquisition_generation, revision_authority + FROM raw_sessions WHERE raw_id = ? + """, + (raw_id,), + ).fetchone() == ( + "claude-code-session:append-only", + "append", + "predecessor-revision", + "predecessor-raw", + "baseline-raw", + 10, + 20, + 2, + "byte_proven", + ) + assert conn.execute("SELECT status FROM raw_membership_census WHERE raw_id = ?", (raw_id,)).fetchone() == ( + "non_session", + ) + + +def test_backfill_fallback_terminalization_preserves_each_source_index(tmp_path: Path) -> None: + """A deferred byte-growth member keeps its own artifact coordinate.""" + initialize_active_archive_root(tmp_path) + source_path = str(tmp_path / ".claude" / "projects" / "proj" / "subagents" / "workflows" / "wf" / "journal.jsonl") + older_payload = b'{"contentKey":"older","agentId":"agent"}\n' + head_payload = older_payload + b'{"contentKey":"head","agentId":"agent"}\n' + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + archive.write_raw_payload( + provider=Provider.CLAUDE_CODE, + payload=older_payload, + source_path=source_path, + source_index=4, + acquired_at_ms=1, + ) + archive.write_raw_payload( + provider=Provider.CLAUDE_CODE, + payload=head_payload, + source_path=source_path, + source_index=9, + acquired_at_ms=2, + ) + + census_historical_revision_evidence(tmp_path) + + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT source_index FROM raw_artifacts ORDER BY source_index").fetchall() == [(4,), (9,)] + + def test_terminal_artifact_receipts_roll_back_together(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """A failed terminal census cannot expose only its artifact carrier.""" initialize_active_archive_root(tmp_path) diff --git a/tests/unit/storage/test_raw_retention.py b/tests/unit/storage/test_raw_retention.py index 8d3712704c..f514ae99c3 100644 --- a/tests/unit/storage/test_raw_retention.py +++ b/tests/unit/storage/test_raw_retention.py @@ -31,6 +31,11 @@ from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root, initialize_archive_database from polylogue.storage.sqlite.archive_tiers.ops_write import upsert_ingest_cursor +from polylogue.storage.sqlite.archive_tiers.source_write import ( + ArchiveSourceArtifact, + upsert_raw_artifact, + write_source_raw_session, +) from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier @@ -737,6 +742,66 @@ def test_current_terminal_artifact_authorizes_historical_raws_but_not_later_sess assert snapshot.cursor_authority_gap_samples[0].state == "source_raws_without_accepted_head" +def test_terminal_coordinate_uses_latest_repeated_raw_observation(tmp_path: Path) -> None: + """A→B→A ranks A's reacquisition receipt, not its first raw-row time.""" + source_db = tmp_path / "source.db" + index_db = tmp_path / "index.db" + source_path = tmp_path / "repeated.jsonl" + initialize_archive_database(source_db, ArchiveTier.SOURCE) + initialize_archive_database(index_db, ArchiveTier.INDEX) + with sqlite3.connect(source_db) as conn: + raw_a = write_source_raw_session( + conn, + origin="claude-code-session", + source_path=str(source_path), + source_index=0, + payload=b"session A", + acquired_at_ms=1, + ) + raw_b = write_source_raw_session( + conn, + origin="claude-code-session", + source_path=str(source_path), + source_index=0, + payload=b"terminal B", + acquired_at_ms=2, + ) + upsert_raw_artifact( + conn, + raw_b, + ArchiveSourceArtifact( + artifact_id="artifact-repeated-coordinate", + origin="claude-code-session", + source_path=str(source_path), + source_index=0, + artifact_kind="workflow_journal", + classification_reason="terminal B", + parse_as_session=False, + first_observed_at_ms=2, + last_observed_at_ms=2, + ), + ) + assert ( + write_source_raw_session( + conn, + origin="claude-code-session", + source_path=str(source_path), + source_index=0, + payload=b"session A", + acquired_at_ms=3, + ) + == raw_a + ) + assert conn.execute("SELECT acquired_at_ms FROM raw_sessions WHERE raw_id = ?", (raw_a,)).fetchone() == (1,) + assert conn.execute( + "SELECT acquired_at_ms FROM blob_refs WHERE ref_type = 'raw_payload' AND ref_id = ?", (raw_a,) + ).fetchone() == (3,) + + assert raw_retention_mod._terminal_artifact_paths(conn, {str(source_path)}) == set() + with pytest.raises(RawRetentionSafetyError, match="index has no raw authority"): + active_raw_retention_authority(conn, index_db_path=index_db) + + def test_terminal_cursor_exemption_requires_every_source_coordinate(tmp_path: Path) -> None: """A terminal sibling cannot hide an unheaded conversational coordinate.""" @@ -2580,6 +2645,22 @@ def test_raw_frontier_integrity_snapshot_unavailable_source_tier_is_unknown_neve assert "unreadable" in snapshot.broken_head_reason +def test_active_retention_translates_missing_source_authority_table(tmp_path: Path) -> None: + """Cleanup callers receive the typed fail-closed exception contract.""" + source_db = tmp_path / "source.db" + index_db = tmp_path / "index.db" + initialize_archive_database(index_db, ArchiveTier.INDEX) + with sqlite3.connect(source_db) as conn: + conn.execute("CREATE TABLE placeholder (id INTEGER PRIMARY KEY)") + conn.commit() + + with ( + sqlite3.connect(source_db) as conn, + pytest.raises(RawRetentionSafetyError, match="raw retention authority is unreadable"), + ): + active_raw_retention_authority(conn, index_db_path=index_db) + + def test_raw_frontier_integrity_snapshot_partial_source_schema_is_unknown_not_violated(tmp_path: Path) -> None: source_db = tmp_path / "source.db" index_db = tmp_path / "index.db" From 61259155cd08ad11f996292db97d4a66565398a1 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 23:06:23 +0200 Subject: [PATCH 52/65] fix: harden retained provider authority routes --- polylogue/schemas/sampling_db.py | 25 +- polylogue/sources/live/batch.py | 7 +- polylogue/sources/revision_backfill.py | 284 +++++++++++++++++- polylogue/storage/artifacts/inspection.py | 4 +- polylogue/storage/repair.py | 56 ++-- polylogue/storage/sqlite/queries/raw_state.py | 26 +- tests/unit/sources/test_artifact_taxonomy.py | 33 ++ tests/unit/sources/test_live_batch_support.py | 28 ++ tests/unit/sources/test_revision_backfill.py | 45 ++- tests/unit/storage/test_repair.py | 32 +- 10 files changed, 488 insertions(+), 52 deletions(-) diff --git a/polylogue/schemas/sampling_db.py b/polylogue/schemas/sampling_db.py index 9c01e68f2d..9c45235d60 100644 --- a/polylogue/schemas/sampling_db.py +++ b/polylogue/schemas/sampling_db.py @@ -40,6 +40,7 @@ from polylogue.storage.blob_store import get_blob_store from polylogue.storage.introspection import table_exists from polylogue.storage.sqlite.connection_profile import connection_context +from polylogue.storage.sqlite.queries.raw_state import raw_provider_origin_sql logger = get_logger(__name__) @@ -71,6 +72,7 @@ def _ms_to_iso(value: object) -> str | None: class _RawSessionRow: source_path: str | None origin: str + detected_provider: str | None raw_id: str blob_hash: bytes file_mtime_ms: int | None @@ -81,6 +83,8 @@ class _RawSessionRow: @property def provider_token(self) -> str: + if self.detected_provider is not None: + return Provider.from_string(self.detected_provider).value try: return provider_from_origin(Origin.from_string(self.origin)).value except (ValueError, KeyError): @@ -109,13 +113,14 @@ def _sample_origins_for_provider(source_name: Provider, config: ProviderConfig) def _sample_provider_where_clause(source_name: str | Provider) -> tuple[str, tuple[str, ...]]: provider = Provider.from_string(source_name) origin = origin_from_provider(provider).value - return "origin = ?", (origin,) + return f"{raw_provider_origin_sql()} = ?", (origin,) def _coerce_schema_row(row: sqlite3.Row) -> _RawSessionRow: return _RawSessionRow( source_path=row["source_path"], origin=str(row["origin"]), + detected_provider=(str(row["detected_provider"]) if row["detected_provider"] is not None else None), raw_id=str(row["raw_id"]), blob_hash=bytes(row["blob_hash"]) if row["blob_hash"] is not None else b"", file_mtime_ms=row["file_mtime_ms"], @@ -296,6 +301,7 @@ def _iter_schema_units_from_db( query_provider = config.db_source_name or source_name origins = _sample_origins_for_provider(Provider.from_string(query_provider), config) placeholders = ",".join("?" for _ in origins) + effective_origin = raw_provider_origin_sql(table_alias="raw_sessions") with connection_context(source_db_path) as conn: conn.row_factory = sqlite3.Row if logical_heads_only: @@ -307,25 +313,26 @@ def _iter_schema_units_from_db( query = f""" WITH heads AS ( SELECT - source_path, origin, raw_id, blob_hash, file_mtime_ms, acquired_at_ms, parsed_at_ms, + source_path, origin, detected_provider, raw_id, blob_hash, file_mtime_ms, + acquired_at_ms, parsed_at_ms, validated_at_ms, validation_status, ROW_NUMBER() OVER ( - PARTITION BY origin, {logical_cohort_expr} + PARTITION BY {effective_origin}, {logical_cohort_expr} ORDER BY acquired_at_ms DESC, raw_id DESC ) AS rn FROM raw_sessions - WHERE origin IN ({placeholders}) + WHERE {effective_origin} IN ({placeholders}) ) - SELECT source_path, origin, raw_id, blob_hash, file_mtime_ms, acquired_at_ms, parsed_at_ms, - validated_at_ms, validation_status + SELECT source_path, origin, detected_provider, raw_id, blob_hash, file_mtime_ms, + acquired_at_ms, parsed_at_ms, validated_at_ms, validation_status FROM heads WHERE rn = 1 """ else: query = f""" - SELECT source_path, origin, raw_id, blob_hash, file_mtime_ms, acquired_at_ms, parsed_at_ms, - validated_at_ms, validation_status + SELECT source_path, origin, detected_provider, raw_id, blob_hash, file_mtime_ms, + acquired_at_ms, parsed_at_ms, validated_at_ms, validation_status FROM raw_sessions - WHERE origin IN ({placeholders}) + WHERE {effective_origin} IN ({placeholders}) """ cursor = conn.execute(query, origins) batch_size = 1 if config.sample_granularity == "record" else 100 diff --git a/polylogue/sources/live/batch.py b/polylogue/sources/live/batch.py index 34679c7997..a2331d27ee 100644 --- a/polylogue/sources/live/batch.py +++ b/polylogue/sources/live/batch.py @@ -1879,6 +1879,9 @@ def _ingest_full_paths_sync( and fallback_provider is Provider.HERMES and path.name in {"state.db", "verification_evidence.db"} ) + codex_owned_sqlite_name = ( + source_only and fallback_provider is Provider.CODEX and path.name in _CODEX_STATE_DB_NAMES + ) if ( hermes_owned_sqlite_name or hermes_state.looks_like_state_db_path(path) @@ -1917,8 +1920,8 @@ def _ingest_full_paths_sync( current_path=path, source_payload_read_bytes=source_payload_read_bytes, ) - elif path.name in _CODEX_STATE_DB_NAMES and ( - source_only or codex_state.is_in_scope_codex_sqlite_path(path) + elif codex_owned_sqlite_name or ( + path.name in _CODEX_STATE_DB_NAMES and codex_state.is_in_scope_codex_sqlite_path(path) ): # polylogue-0jf4: acquire live Codex SQLite state the same # way Hermes acquires its state.db -- a consistent diff --git a/polylogue/sources/revision_backfill.py b/polylogue/sources/revision_backfill.py index d5372f2e62..c97f06570c 100644 --- a/polylogue/sources/revision_backfill.py +++ b/polylogue/sources/revision_backfill.py @@ -85,6 +85,278 @@ _LOGGER = _polylogue_logging.get_logger(__name__) _REPLAY_PROVIDER_DETECTION_PREFIX_BYTES: Final[int] = 8192 +_DOCUMENT_PROBE_ROOT_KEYS: Final[frozenset[str]] = frozenset( + { + "account_uuid", + "artifactType", + "cascadeId", + "chat_messages", + "chunkedPrompt", + "chunks", + "conversation_id", + "conversations", + "conversations_memory", + "create_time", + "current_node", + "cwd", + "id", + "kind", + "lastUpdated", + "last_updated", + "leafUuid", + "mapping", + "markdown", + "message", + "messages", + "parentUuid", + "payload", + "platform", + "polylogue_capture_kind", + "project", + "projectHash", + "project_memories", + "record_type", + "session", + "sessionId", + "session_id", + "session_start", + "shared_conversation_id", + "source", + "startTime", + "summary", + "type", + "updatedAt", + "uuid", + "version", + } +) +_DOCUMENT_PROBE_EXACT_STRING_KEYS: Final[frozenset[str]] = frozenset( + {"kind", "polylogue_capture_kind", "record_type", "role", "source", "type"} +) +_DOCUMENT_PROBE_CHUNK_CONTENT_KEYS: Final[frozenset[str]] = frozenset( + { + "codeExecutionResult", + "driveAudio", + "driveDocument", + "driveImage", + "driveVideo", + "errorMessage", + "executableCode", + "grounding", + "inlineFile", + "inlineImage", + "isThought", + "parts", + "text", + "youtubeVideo", + } +) + + +def _document_probe_value(key: str, event: str, value: object) -> object | None: + """Retain only detector-relevant scalar type/equality evidence.""" + if event == "string": + return str(value) if key in _DOCUMENT_PROBE_EXACT_STRING_KEYS else "present" + if event == "number": + return 0 + if event == "boolean": + return bool(value) + if event == "null": + return None + return None + + +@dataclass(slots=True) +class _StreamingDocumentProviderProbe: + """Bounded structural summary for one object in a JSON document. + + Cardinality is fixed by provider detector fields. Large scalar bodies, + unrelated keys, repeated messages, and repeated mapping nodes are never + retained; the ijson event stream can therefore continue to EOF without a + whole-document allocation or a scan-count cutoff. + """ + + payload: dict[str, object] = field(default_factory=dict) + mapping_seen: bool = False + mapping_valid: bool = True + mapping_node_open: bool = False + mapping_node_message: Literal["absent", "null", "map", "invalid"] = "absent" + mapping_node_author: bool = False + chat_message_item: dict[str, object] = field(default_factory=dict) + chat_message_matched: bool = False + first_message_item: dict[str, object] = field(default_factory=dict) + first_message_complete: bool = False + chunk_item: dict[str, object] = field(default_factory=dict) + chunk_matched: bool = False + conversation_item_has_conversation: bool = False + conversation_item_has_responses: bool = False + conversation_matched: bool = False + + def _set_root(self, key: str, event: str, value: object) -> None: + if key not in _DOCUMENT_PROBE_ROOT_KEYS: + return + if event == "start_map": + self.payload[key] = {} + elif event == "start_array": + self.payload[key] = [] + elif event in {"string", "number", "boolean", "null"}: + self.payload[key] = _document_probe_value(key, event, value) + + @staticmethod + def _set_item_value(item: dict[str, object], key: str, event: str, value: object) -> None: + if event == "start_map": + item[key] = {} + elif event == "start_array": + item[key] = [] + elif event in {"string", "number", "boolean", "null"}: + item[key] = _document_probe_value(key, event, value) + + def feed(self, prefix: str, event: str, value: object) -> None: + parts = prefix.split(".") if prefix else [] + if len(parts) == 1: + self._set_root(parts[0], event, value) + + if parts == ["session", "provider"] and event == "string": + session = self.payload.setdefault("session", {}) + if isinstance(session, dict): + session["provider"] = str(value) + + if len(parts) == 2 and parts[0] == "payload": + nested = self.payload.setdefault("payload", {}) + if isinstance(nested, dict): + self._set_item_value(nested, parts[1], event, value) + + if len(parts) == 2 and parts[0] == "mapping": + if event == "start_map": + self.mapping_seen = True + self.mapping_node_open = True + self.mapping_node_message = "absent" + self.mapping_node_author = False + elif event not in {"end_map", "map_key"}: + self.mapping_seen = True + self.mapping_valid = False + elif len(parts) == 3 and parts[0] == "mapping" and parts[2] == "message": + if event == "start_map": + self.mapping_node_message = "map" + elif event == "null": + self.mapping_node_message = "null" + elif event not in {"map_key", "end_map"}: + self.mapping_node_message = "invalid" + elif len(parts) == 4 and parts[0] == "mapping" and parts[2:] == ["message", "author"] and event == "start_map": + self.mapping_node_author = True + + if len(parts) == 2 and parts == ["chat_messages", "item"] and event == "start_map": + self.chat_message_item = {} + elif len(parts) == 3 and parts[:2] == ["chat_messages", "item"]: + self._set_item_value(self.chat_message_item, parts[2], event, value) + + if len(parts) == 2 and parts == ["messages", "item"] and event == "start_map": + if not self.first_message_complete: + self.first_message_item = {} + elif len(parts) == 3 and parts[:2] == ["messages", "item"] and not self.first_message_complete: + self._set_item_value(self.first_message_item, parts[2], event, value) + + chunk_root = parts[:2] == ["chunks", "item"] + chunk_nested = parts[:3] == ["chunkedPrompt", "chunks", "item"] + if (chunk_root and len(parts) == 2 or chunk_nested and len(parts) == 3) and event == "start_map": + self.chunk_item = {} + elif chunk_root and len(parts) == 3: + self._set_item_value(self.chunk_item, parts[2], event, value) + elif chunk_nested and len(parts) == 4: + self._set_item_value(self.chunk_item, parts[3], event, value) + + if parts == ["conversations", "item"] and event == "start_map": + self.conversation_item_has_conversation = False + self.conversation_item_has_responses = False + elif parts == ["conversations", "item", "conversation"] and event == "start_map": + self.conversation_item_has_conversation = True + elif parts == ["conversations", "item", "responses"] and event == "start_array": + self.conversation_item_has_responses = True + + if event != "end_map": + return + if len(parts) == 2 and parts[0] == "mapping" and self.mapping_node_open: + if ( + self.mapping_node_message == "map" and not self.mapping_node_author + ) or self.mapping_node_message == "invalid": + self.mapping_valid = False + self.mapping_node_open = False + elif parts == ["chat_messages", "item"]: + has_role = any(key in self.chat_message_item for key in ("sender", "role", "author")) + has_content = any(key in self.chat_message_item for key in ("text", "content")) + self.chat_message_matched |= has_role and has_content + elif parts == ["messages", "item"] and not self.first_message_complete: + self.first_message_complete = True + elif parts in (["chunks", "item"], ["chunkedPrompt", "chunks", "item"]): + role = self.chunk_item.get("role") or self.chunk_item.get("author") + self.chunk_matched |= isinstance(role, str) and any( + key in self.chunk_item for key in _DOCUMENT_PROBE_CHUNK_CONTENT_KEYS + ) + elif parts == ["conversations", "item"]: + self.conversation_matched |= ( + self.conversation_item_has_conversation and self.conversation_item_has_responses + ) + + def classify(self) -> tuple[Provider, str]: + if self.mapping_seen and self.mapping_valid: + self.payload["mapping"] = {"bounded-node": {"message": None}} + if self.chat_message_matched: + self.payload["chat_messages"] = [{"role": "present", "text": "present"}] + if self.first_message_complete: + self.payload["messages"] = [self.first_message_item] + if self.chunk_matched: + chunk = {"role": "present", "text": "present"} + if isinstance(self.payload.get("chunkedPrompt"), dict): + self.payload["chunkedPrompt"] = {"chunks": [chunk]} + else: + self.payload["chunks"] = [chunk] + if self.conversation_matched: + self.payload["conversations"] = [{"conversation": {}, "responses": []}] + provider, evidence = detect_provider_evidence(self.payload) + if provider is None: + return Provider.UNKNOWN, evidence + return provider, f"bounded streaming JSON structure: {evidence}" + + +def _detect_provider_from_bounded_document(payload: BinaryIO) -> tuple[Provider, str]: + """Scan every document object while retaining fixed structural evidence.""" + payload.seek(0) + probe: _StreamingDocumentProviderProbe | None = None + root_is_array = False + last_evidence = "no bounded document structure identified a provider; used fallback_provider" + try: + for prefix, event, value in ijson.parse(payload, use_float=True): + if prefix == "" and event == "start_array": + root_is_array = True + continue + if root_is_array: + if prefix == "item" and event == "start_map": + probe = _StreamingDocumentProviderProbe() + continue + if probe is None: + continue + if prefix == "item" and event == "end_map": + provider, last_evidence = probe.classify() + if provider is not Provider.UNKNOWN: + return provider, last_evidence + probe = None + continue + if prefix.startswith("item."): + probe.feed(prefix.removeprefix("item."), event, value) + continue + + if prefix == "" and event == "start_map": + probe = _StreamingDocumentProviderProbe() + continue + if probe is None: + continue + if prefix == "" and event == "end_map": + return probe.classify() + probe.feed(prefix, event, value) + except ijson.JSONError: + return Provider.UNKNOWN, last_evidence + return Provider.UNKNOWN, last_evidence + def _detect_provider_from_bounded_prefix( prefix: bytes, @@ -140,17 +412,21 @@ def _detect_unknown_retained_provider( same detection bound, oversized records are consumed in bounded chunks, and the scan continues until positive provider evidence or EOF. - Non-JSONL documents retain prefix detection here and their existing - complete-document retry at the caller. That eager retry is required for - document providers whose first complete value exceeds the prefix. + Non-JSONL documents first use the same prefix evidence, then continue a + bounded structural event scan through every document object. Eager replay + remains gated on a positive provider result from one of those bounded + passes; an unresolved UNKNOWN document is never materialized wholesale. """ stream_name = Path(source_path).name if not is_jsonl_source_path(source_path): - return _detect_provider_from_bounded_prefix( + provider, evidence = _detect_provider_from_bounded_prefix( payload.read(_REPLAY_PROVIDER_DETECTION_PREFIX_BYTES), stream_name, record_stream=False, ) + if provider is not Provider.UNKNOWN: + return provider, evidence + return _detect_provider_from_bounded_document(payload) last_evidence = "no bounded JSONL record identified a provider; used fallback_provider" read_size = _REPLAY_PROVIDER_DETECTION_PREFIX_BYTES + 1 diff --git a/polylogue/storage/artifacts/inspection.py b/polylogue/storage/artifacts/inspection.py index cbffe9ffb8..de36d02789 100644 --- a/polylogue/storage/artifacts/inspection.py +++ b/polylogue/storage/artifacts/inspection.py @@ -20,6 +20,7 @@ ) from polylogue.archive.raw_payload.decode import JSONLSessionArtifactScan, scan_jsonl_session_artifact from polylogue.core.enums import ArtifactSupportStatus, Provider +from polylogue.core.sources import origin_from_provider from polylogue.schemas.observation import derive_bundle_scope, schema_cluster_id from polylogue.schemas.packages import SchemaResolution from polylogue.schemas.runtime_registry import SchemaRegistry @@ -352,8 +353,9 @@ def inspect_raw_artifact(record: RawSessionRecord, *, blob_store: BlobStore | No provider_hint = _normalize_payload_provider_hint(record) provider_token = provider_hint or record.source_name or "" bundle_scope = derive_bundle_scope(provider_token, record.source_path) + observation_origin = origin_from_provider(Provider.from_string(provider_token)) observation_id = artifact_observation_id( - source_name=record.source_name, + source_name=observation_origin.value, source_path=record.source_path, source_index=record.source_index, ) diff --git a/polylogue/storage/repair.py b/polylogue/storage/repair.py index abdc334dfd..e6673ba001 100644 --- a/polylogue/storage/repair.py +++ b/polylogue/storage/repair.py @@ -84,6 +84,7 @@ validate_raw_replay_application_receipt, validate_raw_replay_plan, ) +from polylogue.storage.sqlite.queries.raw_state import raw_provider_origin_sql if TYPE_CHECKING: # ``revision_backfill`` imports ``ArchiveStore``, which (via @@ -3787,9 +3788,13 @@ def _raw_materialization_index_path(config: Config, archive_root: Path) -> Path: def _raw_artifact_coordinate_predicate(*, artifact_alias: str, raw_alias: str) -> str: """Correlate evidence with the exact failed artifact observation.""" + provider_origin = raw_provider_origin_sql(table_alias=raw_alias) return f""" AND {artifact_alias}.raw_id IS {raw_alias}.raw_id - AND {artifact_alias}.origin IS {raw_alias}.origin + AND ( + {artifact_alias}.origin IS {raw_alias}.origin + OR {artifact_alias}.origin IS ({provider_origin}) + ) AND {artifact_alias}.source_path IS {raw_alias}.source_path AND {artifact_alias}.source_index IS {raw_alias}.source_index """ @@ -3871,10 +3876,11 @@ def _raw_materialization_candidate_ids( if raw_artifact_id is not None: raw_filter = "AND r.raw_id = ?" params.append(raw_artifact_id) + effective_origin = raw_provider_origin_sql(table_alias="r") origin_filter = "" provider_origin = _raw_materialization_origin_from_provider(provider) if provider_origin is not None: - origin_filter += " AND r.origin = ?" + origin_filter += f" AND {effective_origin} = ?" params.append(provider_origin) if source_family is not None: origin_filter += " AND r.origin = ?" @@ -3887,7 +3893,8 @@ def _raw_materialization_candidate_ids( terminal_pair_placeholders = ", ".join("(?, ?)" for _ in RAW_FAILURE_TERMINAL_EVIDENCE_SUPPORT_STATUS_PAIRS) rows = conn.execute( f""" - SELECT r.raw_id, r.origin, r.native_id, r.source_path, r.blob_hash, r.blob_size, + SELECT r.raw_id, r.origin, {effective_origin} AS provider_origin, + r.native_id, r.source_path, r.blob_hash, r.blob_size, r.acquired_at_ms, r.parsed_at_ms, r.validated_at_ms, r.parse_error, ( @@ -3970,7 +3977,7 @@ def _raw_materialization_candidate_ids( LEFT JOIN index_tier.sessions AS s_by_raw ON s_by_raw.raw_id = r.raw_id LEFT JOIN index_tier.sessions AS s_by_native ON r.native_id IS NOT NULL - AND s_by_native.origin = r.origin + AND s_by_native.origin = {effective_origin} AND s_by_native.native_id = r.native_id LEFT JOIN raw_sessions AS existing_native_raw ON existing_native_raw.raw_id = s_by_native.raw_id @@ -4091,7 +4098,7 @@ def _raw_materialization_candidate_ids( if blob_store.exists(blob_hash): raw_id = str(row["raw_id"]) raw_ids.append(raw_id) - raw_origins[raw_id] = str(row["origin"] or "") + raw_origins[raw_id] = str(row["provider_origin"] or "") raw_source_paths[raw_id] = str(row["source_path"] or "") raw_acquired_at_ms[raw_id] = int(row["acquired_at_ms"] or 0) blob_size = row["blob_size"] @@ -4113,8 +4120,12 @@ def _raw_materialization_candidate_ids( for offset in range(0, len(expanded_raw_ids), 500): raw_id_chunk = expanded_raw_ids[offset : offset + 500] placeholders = ",".join("?" for _ in raw_id_chunk) + expanded_effective_origin = raw_provider_origin_sql(table_alias="raw_sessions") for row in conn.execute( - f"SELECT raw_id, blob_size, origin, source_path FROM raw_sessions WHERE raw_id IN ({placeholders})", + f""" + SELECT raw_id, blob_size, {expanded_effective_origin}, source_path + FROM raw_sessions WHERE raw_id IN ({placeholders}) + """, raw_id_chunk, ): rid = str(row[0]) @@ -4176,9 +4187,10 @@ def _raw_materialization_parser_census_candidates( if raw_artifact_id is not None: filters.append("r.raw_id = ?") params.append(raw_artifact_id) + effective_origin = raw_provider_origin_sql(table_alias="r") provider_origin = _raw_materialization_origin_from_provider(provider) if provider_origin is not None: - filters.append("r.origin = ?") + filters.append(f"{effective_origin} = ?") params.append(provider_origin) if source_family is not None: filters.append("r.origin = ?") @@ -4190,7 +4202,8 @@ def _raw_materialization_parser_census_candidates( where = f"WHERE {' AND '.join(filters)}" if filters else "" rows = conn.execute( f""" - SELECT r.raw_id, r.blob_size, r.origin, r.source_path, r.acquired_at_ms + SELECT r.raw_id, r.blob_size, {effective_origin} AS provider_origin, + r.source_path, r.acquired_at_ms FROM raw_sessions AS r {where} ORDER BY r.acquired_at_ms DESC, r.raw_id ASC @@ -4206,7 +4219,7 @@ def _raw_materialization_parser_census_candidates( raw_id = str(row["raw_id"]) raw_ids.append(raw_id) raw_blob_bytes[raw_id] = int(row["blob_size"] or 0) - raw_origins[raw_id] = str(row["origin"] or "") + raw_origins[raw_id] = str(row["provider_origin"] or "") raw_source_paths[raw_id] = str(row["source_path"] or "") raw_acquired_at_ms[raw_id] = int(row["acquired_at_ms"] or 0) from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore @@ -4219,8 +4232,12 @@ def _raw_materialization_parser_census_candidates( for offset in range(0, len(expanded_raw_ids), 500): raw_id_chunk = expanded_raw_ids[offset : offset + 500] placeholders = ",".join("?" for _ in raw_id_chunk) + expanded_effective_origin = raw_provider_origin_sql(table_alias="raw_sessions") for row in conn.execute( - f"SELECT raw_id, blob_size, origin, source_path FROM raw_sessions WHERE raw_id IN ({placeholders})", + f""" + SELECT raw_id, blob_size, {expanded_effective_origin}, source_path + FROM raw_sessions WHERE raw_id IN ({placeholders}) + """, raw_id_chunk, ): raw_id = str(row[0]) @@ -4316,18 +4333,23 @@ def raw_materialization_readonly_descriptors( placeholders = ",".join("?" for _ in raw_id_chunk) rows = conn.execute( f""" - SELECT raw_id, origin, capture_mode, lower(hex(blob_hash)), source_path, revision_kind, blob_size + SELECT raw_id, origin, detected_provider, capture_mode, + lower(hex(blob_hash)), source_path, revision_kind, blob_size FROM raw_sessions WHERE raw_id IN ({placeholders}) """, raw_id_chunk, ).fetchall() for row in rows: result[str(row[0])] = ( - provider_from_origin(Origin.from_string(str(row[1])), family_hint=row[2]), - str(row[3]), + ( + Provider.from_string(str(row[2])) + if row[2] is not None + else provider_from_origin(Origin.from_string(str(row[1])), family_hint=row[3]) + ), str(row[4]), - RawRevisionKind(str(row[5])), - int(row[6]), + str(row[5]), + RawRevisionKind(str(row[6])), + int(row[7]), ) return result @@ -5117,7 +5139,7 @@ def raw_materialization_scale_profile(config: Config) -> dict[str, object]: def _raw_materialized_by_source_path_native(materialized_aliases: set[tuple[str, str]], row: sqlite3.Row) -> bool: - origin = str(row["origin"] or "") + origin = str(row["provider_origin"] or "") if not origin: return False for native_id in _source_path_native_id_candidates(str(row["source_path"] or "")): @@ -5136,7 +5158,7 @@ def _raw_materialization_parsed_non_session_artifact(archive_root: Path, row: sq return ( parsed_non_session_artifact_reason( archive_root=archive_root, - origin=str(row["origin"] or ""), + origin=str(row["provider_origin"] or ""), source_path=str(row["source_path"] or ""), blob_hash=blob_hash, ) diff --git a/polylogue/storage/sqlite/queries/raw_state.py b/polylogue/storage/sqlite/queries/raw_state.py index c2b0a852da..ffc1c2b2bc 100644 --- a/polylogue/storage/sqlite/queries/raw_state.py +++ b/polylogue/storage/sqlite/queries/raw_state.py @@ -12,12 +12,20 @@ from polylogue.storage.sqlite.connection import _build_source_scope_filter from polylogue.storage.sqlite.raw_state_update import compile_raw_state_update -# Raw filters follow parser-classified provider evidence when present while -# preserving immutable acquisition origin as the fallback. The CASE keeps the -# comparison in public Origin vocabulary even though detected_provider retains -# the exact provider-wire token (including the Gemini/Drive fiber). -RAW_ORIGIN_FILTER_SQL = """ -CASE detected_provider + +def raw_provider_origin_sql(*, table_alias: str | None = None) -> str: + """Project parser-classified provider evidence into Origin vocabulary. + + Acquisition ``origin`` remains immutable raw identity. Provider-scoped + readers use this expression so a later positive parser classification is + visible without rewriting that identity. ``table_alias`` keeps the same + contract usable in joined repair and sampling queries. + """ + prefix = f"{table_alias}." if table_alias else "" + detected = f"{prefix}detected_provider" + origin = f"{prefix}origin" + return f""" +CASE {detected} WHEN 'chatgpt' THEN 'chatgpt-export' WHEN 'claude-ai' THEN 'claude-ai-export' WHEN 'claude-design' THEN 'claude-design-session' @@ -31,11 +39,14 @@ WHEN 'beads' THEN 'beads-issue' WHEN 'grok' THEN 'grok-export' WHEN 'unknown' THEN 'unknown-export' - ELSE origin + ELSE {origin} END """.strip() +RAW_ORIGIN_FILTER_SQL = raw_provider_origin_sql() + + def origin_filter_value(token: str) -> str: """Normalize a raw-wire token to the origin stored in ``raw_sessions``. @@ -236,6 +247,7 @@ async def reset_validation_status( "coerce_provider", "coerce_status", "origin_filter_value", + "raw_provider_origin_sql", "mark_raw_parsed", "mark_raw_validated", "reset_parse_status", diff --git a/tests/unit/sources/test_artifact_taxonomy.py b/tests/unit/sources/test_artifact_taxonomy.py index c1f2273acb..fdcb98f86b 100644 --- a/tests/unit/sources/test_artifact_taxonomy.py +++ b/tests/unit/sources/test_artifact_taxonomy.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import sqlite3 from pathlib import Path from polylogue.archive.artifact_taxonomy import ArtifactKind, classify_artifact, classify_artifact_path @@ -285,6 +286,38 @@ def record_terminal( ] +def test_schema_sampling_uses_detected_provider_for_unknown_acquisition(workspace_env: dict[str, Path]) -> None: + """Provider-scoped schema reads include source-only raws learned during replay.""" + archive_root = workspace_env["archive_root"] + payload = ( + b'{"type":"user","uuid":"message-1","sessionId":"learned-session",' + b'"parentUuid":null,"message":{"role":"user","content":"hello"}}\n' + ) + with ArchiveStore(archive_root) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.UNKNOWN, + payload=payload, + source_path="/captures/learned/session.jsonl", + acquired_at_ms=1, + ) + with sqlite3.connect(archive_root / "source.db") as conn: + conn.execute( + "UPDATE raw_sessions SET detected_provider = 'claude-code' WHERE raw_id = ?", + (raw_id,), + ) + + units = list( + _iter_schema_units_from_db( + Provider.CLAUDE_CODE, + db_path=archive_root / "index.db", + config=resolve_provider_config(Provider.CLAUDE_CODE), + ) + ) + + assert units + assert {unit.raw_id for unit in units} == {raw_id} + + def test_tool_result_sidecar_never_classifies_as_session_even_when_content_looks_like_one() -> None: """Regression for polylogue-omsw: a ``tool-results/`` sidecar must never become a session regardless of its content, only its path. diff --git a/tests/unit/sources/test_live_batch_support.py b/tests/unit/sources/test_live_batch_support.py index ae7c8d40bc..a6597043ff 100644 --- a/tests/unit/sources/test_live_batch_support.py +++ b/tests/unit/sources/test_live_batch_support.py @@ -992,6 +992,34 @@ def test_source_only_full_ingest_snapshots_unrecognized_codex_state_without_shap assert conn.execute("SELECT source_path, parsed_at_ms FROM raw_sessions").fetchall() == [(str(state_db), None)] +def test_source_only_foreign_sqlite_name_cannot_claim_codex_authority(tmp_path: Path) -> None: + """A foreign watch source cannot turn a filename into Codex authority.""" + from polylogue.core.degraded import DegradedReason, clear_degraded, set_degraded + + initialize_active_archive_root(tmp_path) + root = tmp_path / "inbox" + state_db = root / "state_5.sqlite" + _write_plain_sqlite_db(state_db) + index_db = tmp_path / "index.db" + processor = LiveBatchProcessor( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=index_db))), + (WatchSource(name="inbox", root=root, suffixes=(".sqlite",)),), + cursor=CursorStore(index_db), + parser_fingerprint="test-parser", + ) + + set_degraded(DegradedReason(code="schema_version_mismatch", message="index unavailable", derived_only=True)) + try: + result = processor._ingest_full_paths_sync([state_db], source_name="inbox") + finally: + clear_degraded() + + assert result.succeeded == [] + assert result.failed == [] + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT origin FROM raw_sessions").fetchall() == [] + + def _write_codex_thread_state_db(path: Path) -> None: path.parent.mkdir(parents=True, exist_ok=True) with sqlite3.connect(path) as conn: diff --git a/tests/unit/sources/test_revision_backfill.py b/tests/unit/sources/test_revision_backfill.py index 94fc804658..114d744483 100644 --- a/tests/unit/sources/test_revision_backfill.py +++ b/tests/unit/sources/test_revision_backfill.py @@ -29,13 +29,16 @@ backfill_historical_revision_evidence, census_historical_revision_evidence, ) +from polylogue.storage.artifacts.inspection import inspect_raw_artifact from polylogue.storage.blob_publication import ArchiveBlobPublisher +from polylogue.storage.blob_store import BlobStore from polylogue.storage.raw_authority import RAW_AUTHORITY_PARSER_FINGERPRINT from polylogue.storage.raw_retention import RawRetentionAuthority, active_raw_retention_authority from polylogue.storage.sqlite.archive_tiers import revision_governance as archive_revision_governance from polylogue.storage.sqlite.archive_tiers import write as archive_tier_write from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root +from polylogue.storage.sqlite.async_sqlite import SQLiteBackend from tests.infra.revision_backfill_benchmark import ( REVISION_CHAIN_SHAPE, WHALE_BEARING_SHAPE, @@ -281,18 +284,18 @@ def test_unknown_retained_nonstream_jsonl_keeps_complete_payload_fallback(tmp_pa assert [session.provider_session_id for session in sessions] == ["large-jsonl-document"] -def test_unknown_retained_document_replays_after_complete_payload_detection(tmp_path: Path) -> None: - """A complete ChatGPT document must retry UNKNOWN prefix detection. +def test_unknown_retained_document_scans_past_oversized_leading_value(tmp_path: Path) -> None: + """A complete ChatGPT document must scan beyond its bounded prefix. The raw is intentionally a source-only UNKNOWN ``conversations.json`` - whose first complete array item is larger than the replay detection - prefix. This drives the historical replay chokepoint against a real - archive, rather than testing the detector in isolation. + whose provider-defining fields follow an oversized leading value. This + drives the historical replay chokepoint against a real archive, rather + than testing the structural scanner in isolation. """ initialize_active_archive_root(tmp_path) - document = _chatgpt_session("large-document", "bounded evidence") - document["padding"] = "x" * 9_000 - payload = _bundle(document) + document = {"padding": "x" * 9_000, **_chatgpt_session("large-document", "bounded evidence")} + payload = json.dumps([document]).encode() + assert b'"mapping"' not in payload[: revision_backfill._REPLAY_PROVIDER_DETECTION_PREFIX_BYTES] with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: archive.write_raw_payload( provider=Provider.UNKNOWN, @@ -703,7 +706,8 @@ def test_backfill_terminalizes_source_only_declared_artifact(tmp_path: Path) -> ).fetchone() == ("complete", "[]") -def test_backfill_terminalizes_detected_unknown_empty_artifact(tmp_path: Path) -> None: +@pytest.mark.asyncio +async def test_backfill_terminalizes_detected_unknown_empty_artifact(tmp_path: Path) -> None: """Detected provider evidence must survive an empty retained replay.""" initialize_active_archive_root(tmp_path) source_path = str(tmp_path / ".claude" / "projects" / "proj" / "subagents" / "workflows" / "wf" / "journal.jsonl") @@ -733,6 +737,29 @@ def test_backfill_terminalizes_detected_unknown_empty_artifact(tmp_path: Path) - "SELECT status, logical_keys_json FROM raw_authority_parser_census WHERE raw_id = ?", (raw_id,) ).fetchone() == ("complete", "[]") + terminal_artifact_id = str( + conn.execute("SELECT artifact_id FROM raw_artifacts WHERE raw_id = ?", (raw_id,)).fetchone()[0] + ) + + backend = SQLiteBackend(db_path=tmp_path / "index.db") + try: + record = await backend.get_raw_session(raw_id) + assert record is not None + refreshed = inspect_raw_artifact(record, blob_store=BlobStore(tmp_path / "blob")) + assert refreshed.observation_id == terminal_artifact_id + assert await backend.save_artifact_observation(refreshed) is False + finally: + await backend.close() + + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute( + """ + SELECT COUNT(*) FROM raw_artifacts + WHERE origin = 'claude-code-session' AND source_path = ? AND source_index = 0 + """, + (source_path,), + ).fetchone() == (1,) + def test_backfill_persists_detected_provider_for_empty_ordinary_session_path(tmp_path: Path) -> None: """Empty replay retains parser identity without mutating acquisition identity.""" diff --git a/tests/unit/storage/test_repair.py b/tests/unit/storage/test_repair.py index feed4feeb6..abb29a001e 100644 --- a/tests/unit/storage/test_repair.py +++ b/tests/unit/storage/test_repair.py @@ -2990,6 +2990,7 @@ def test_raw_materialization_scope_filters_count_only_matching_raw_rows(tmp_path claude_raw_id, claude_size = blob_store.write_from_bytes(b'{"parentUuid":null,"sessionId":"claude-a"}') codex_raw_id, codex_size = blob_store.write_from_bytes(b'{"items":[]}') other_root_raw_id, other_root_size = blob_store.write_from_bytes(b'{"parentUuid":null,"sessionId":"claude-b"}') + learned_raw_id, learned_size = blob_store.write_from_bytes(b'{"parentUuid":null,"sessionId":"claude-learned"}') with sqlite3.connect(tmp_path / "source.db") as source_conn: source_conn.executemany( @@ -3031,6 +3032,22 @@ def test_raw_materialization_scope_filters_count_only_matching_raw_rows(tmp_path ), ), ) + source_conn.execute( + """ + INSERT INTO raw_sessions ( + raw_id, origin, detected_provider, native_id, source_path, source_index, + blob_hash, blob_size, acquired_at_ms + ) VALUES (?, 'unknown-export', 'claude-code', ?, ?, 0, ?, ?, ?) + """, + ( + learned_raw_id, + "claude-learned", + "/captures/claude/learned.jsonl", + bytes.fromhex(learned_raw_id), + learned_size, + 4, + ), + ) source_conn.commit() by_provider = repair_mod.repair_raw_materialization(config, dry_run=True, provider="claude-code") @@ -3040,9 +3057,18 @@ def test_raw_materialization_scope_filters_count_only_matching_raw_rows(tmp_path assert by_provider.repaired_count == 0 assert by_family.repaired_count == 0 assert by_root.repaired_count == 0 - assert by_provider.metrics["raw_materialization_candidate_count"] == 2.0 - assert by_provider.metrics["raw_materialization_total_blob_bytes"] == float(claude_size + other_root_size) - assert by_provider.metrics["raw_materialization_max_blob_bytes"] == float(max(claude_size, other_root_size)) + assert by_provider.metrics["raw_materialization_candidate_count"] == 3.0 + assert by_provider.metrics["raw_materialization_total_blob_bytes"] == float( + claude_size + other_root_size + learned_size + ) + assert by_provider.metrics["raw_materialization_max_blob_bytes"] == float( + max(claude_size, other_root_size, learned_size) + ) + census_candidates = repair_mod._raw_materialization_parser_census_candidates( + config, + provider="claude-code", + ) + assert set(census_candidates.raw_ids) == {claude_raw_id, other_root_raw_id, learned_raw_id} def test_raw_materialization_uses_authority_substrate_not_legacy_ingest_stage( From 6bef347fbcd9c7ee022e4bfab9ecdba762ce79a0 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 14 Aug 2026 00:09:55 +0200 Subject: [PATCH 53/65] fix: close retained authority review residuals Problem: Retained artifact ordering could compare rowids from unrelated tables, raw-materialization lease refusals could continue into unpinned frontier work, and ZIP recovery could not select duplicate central-directory entries from their durable coordinates. What changed: Use comparable receipt or legacy raw-session observation keys, decode ZIP entry ordinals independently from split indexes during source recovery, and stop the normal daemon route after a typed generation-pin refusal. Strengthen the affected production-route regressions, including repeated A to B to A observations and nonzero ZIP split recovery. Verification: Focused retained-authority batch: 14 passed in 341.02s. Strengthened ZIP recovery discriminator: 1 passed in 1.12s. devtools verify --quick: exit 0, run 20260813T220204Z-quick-2066087-b615da08. --- polylogue/core/raw_coordinates.py | 73 ++++++++++++ polylogue/daemon/cli.py | 13 ++- polylogue/sources/live/batch.py | 3 +- .../sources/source_acquisition_components.py | 43 ------- polylogue/storage/blob_integrity.py | 59 +++++++--- .../sqlite/archive_tiers/source_write.py | 42 ++++--- tests/unit/core/test_sampling.py | 10 +- tests/unit/daemon/test_daemon_cli.py | 6 +- tests/unit/sources/test_revision_backfill.py | 47 +++++++- tests/unit/storage/test_blob_integrity.py | 107 ++++++++++++++++++ tests/unit/storage/test_repair.py | 10 ++ 11 files changed, 334 insertions(+), 79 deletions(-) create mode 100644 polylogue/core/raw_coordinates.py diff --git a/polylogue/core/raw_coordinates.py b/polylogue/core/raw_coordinates.py new file mode 100644 index 0000000000..9d33b1c782 --- /dev/null +++ b/polylogue/core/raw_coordinates.py @@ -0,0 +1,73 @@ +"""Stable coordinates for raw payloads acquired from container members.""" + +from __future__ import annotations + +from hashlib import sha256 +from math import isqrt + +_ZIP_MEMBER_RAW_ID_DOMAIN = b"polylogue:zip-member-raw:v2\0" + + +def zip_member_source_index(*, entry_ordinal: int, split_index: int) -> int: + """Encode a ZIP entry ordinal and within-entry split index losslessly.""" + if entry_ordinal < 0 or split_index < 0: + raise ValueError("ZIP entry ordinal and split index must be non-negative") + diagonal = entry_ordinal + split_index + return diagonal * (diagonal + 1) // 2 + split_index + + +def zip_member_source_coordinate(source_index: int) -> tuple[int, int]: + """Recover the independent entry ordinal and split index from storage.""" + if source_index < 0: + raise ValueError("ZIP member source index must be non-negative") + diagonal = (isqrt(8 * source_index + 1) - 1) // 2 + diagonal_start = diagonal * (diagonal + 1) // 2 + split_index = source_index - diagonal_start + entry_ordinal = diagonal - split_index + return entry_ordinal, split_index + + +def zip_member_raw_id( + *, + source_path: str, + entry_ordinal: int, + split_index: int, + blob_hash: str, +) -> str: + """Identify one ZIP coordinate without giving up blob-level deduplication.""" + digest = sha256() + digest.update(_ZIP_MEMBER_RAW_ID_DOMAIN) + digest.update(source_path.encode("utf-8", errors="surrogatepass")) + digest.update(b"\0") + digest.update(str(entry_ordinal).encode("utf-8")) + digest.update(b"\0") + digest.update(str(split_index).encode("utf-8")) + digest.update(b"\0") + digest.update(bytes.fromhex(blob_hash)) + return digest.hexdigest() + + +def zip_member_identity_coordinate( + *, + raw_id: str, + source_path: str, + source_index: int, + blob_hash: str, +) -> tuple[int, int] | None: + """Decode a v2 raw identity, rejecting legacy or unrelated coordinates.""" + entry_ordinal, split_index = zip_member_source_coordinate(source_index) + expected_raw_id = zip_member_raw_id( + source_path=source_path, + entry_ordinal=entry_ordinal, + split_index=split_index, + blob_hash=blob_hash, + ) + return (entry_ordinal, split_index) if raw_id == expected_raw_id else None + + +__all__ = [ + "zip_member_identity_coordinate", + "zip_member_raw_id", + "zip_member_source_coordinate", + "zip_member_source_index", +] diff --git a/polylogue/daemon/cli.py b/polylogue/daemon/cli.py index 6e8d9db55a..610ed4cf12 100644 --- a/polylogue/daemon/cli.py +++ b/polylogue/daemon/cli.py @@ -1309,6 +1309,7 @@ def _drain_raw_materialization_once( "raw authority: auto-resolved %d stale-plan blocker(s) before raw materialization", auto_resolved, ) + generation_pin_refused = False with contextlib.ExitStack() as lease_stack: try: index_db = lease_stack.enter_context(raw_authority.materialization_generation_lease(config)) @@ -1317,6 +1318,7 @@ def _drain_raw_materialization_once( if refused_result is None: raise result = refused_result + generation_pin_refused = True else: try: result = raw_authority.repair_materialization( @@ -1330,13 +1332,22 @@ def _drain_raw_materialization_once( finally: _close_raw_materialization_fts(index_db, ops_db_path=config.archive_root / "ops.db") _emit_raw_materialization_pass(result) - frontier_repaired = _converge_raw_authority_frontier(config, limit=min(limit, 8)) if not result.success: logger.warning("raw materialization: bounded convergence incomplete: %s", result.detail) metrics = dict(getattr(result, "metrics", {})) remaining = int(metrics.get("raw_materialization_remaining_candidate_count", 0)) if remaining == 0: remaining = int(metrics.get("raw_materialization_census_incomplete_raw_count", 0)) + if generation_pin_refused: + return raw_authority.RawMaterializationCounts( + repaired_sessions=result.repaired_count, + executed_plans=0, + remaining_candidates=remaining, + censused_components=int(metrics.get("raw_materialization_census_components_attempted", 0)), + candidate_count=int(metrics.get("raw_materialization_candidate_count", 0)), + pending_blob_bytes=int(metrics.get("raw_materialization_total_blob_bytes", 0)), + ) + frontier_repaired = _converge_raw_authority_frontier(config, limit=min(limit, 8)) return raw_authority.RawMaterializationCounts( repaired_sessions=result.repaired_count, executed_plans=frontier_repaired, diff --git a/polylogue/sources/live/batch.py b/polylogue/sources/live/batch.py index a2331d27ee..2e732abcc5 100644 --- a/polylogue/sources/live/batch.py +++ b/polylogue/sources/live/batch.py @@ -51,6 +51,7 @@ read_peak_rss_self_mb, ) from polylogue.core.provider_identity import canonical_acquisition_provider +from polylogue.core.raw_coordinates import zip_member_raw_id, zip_member_source_index from polylogue.core.raw_failure_evidence import ( RAW_FAILURE_EVIDENCE_KINDS, RAW_FAILURE_LIFECYCLE_EVIDENCE_SUPPORT_STATUS_PAIRS, @@ -148,8 +149,6 @@ ZipEntryReadContext, iter_zip_entry_raw_data, stream_preserved_zip_entry_raw_data, - zip_member_raw_id, - zip_member_source_index, ) from polylogue.sources.source_parsing import has_decoded_session_evidence from polylogue.sources.sqlite_snapshot import ( diff --git a/polylogue/sources/source_acquisition_components.py b/polylogue/sources/source_acquisition_components.py index 9a17c0cef8..4e19fd903d 100644 --- a/polylogue/sources/source_acquisition_components.py +++ b/polylogue/sources/source_acquisition_components.py @@ -6,7 +6,6 @@ import zipfile from collections.abc import Callable, Iterable from dataclasses import dataclass, field -from hashlib import sha256 from pathlib import Path from typing import IO, TypeAlias @@ -29,54 +28,12 @@ _DETECTION_PREFIX_SIZE = 8192 # 8 KB — enough for provider detection _HEARTBEAT_INTERVAL_S = 5.0 -_ZIP_MEMBER_RAW_ID_DOMAIN = b"polylogue:zip-member-raw:v2\0" - AcquisitionObservation: TypeAlias = JSONDocument ObservationCallback: TypeAlias = Callable[[AcquisitionObservation], None] StatusCallback: TypeAlias = Callable[[str], None] CursorState: TypeAlias = CursorStatePayload -def zip_member_source_index(*, entry_ordinal: int, split_index: int) -> int: - """Encode a ZIP entry/split coordinate into the persisted integer index. - - Cantor pairing is collision-free for all non-negative integer pairs, so - duplicate central-directory names remain distinct while multiple sessions - split from one member retain their own independent coordinate axis. - """ - if entry_ordinal < 0 or split_index < 0: - raise ValueError("ZIP entry ordinal and split index must be non-negative") - diagonal = entry_ordinal + split_index - return diagonal * (diagonal + 1) // 2 + split_index - - -def zip_member_raw_id( - *, - source_path: str, - entry_ordinal: int, - split_index: int, - blob_hash: str, -) -> str: - """Identify one ZIP coordinate without giving up blob-level deduplication. - - ZIP exports legitimately contain duplicate member bytes. The blob hash - remains their shared immutable storage address, while raw authority must - retain each admitted ``:`` coordinate independently. - The central-directory ordinal distinguishes duplicate member names and the - independent split index distinguishes sessions decoded from one member. - """ - digest = sha256() - digest.update(_ZIP_MEMBER_RAW_ID_DOMAIN) - digest.update(source_path.encode("utf-8", errors="surrogatepass")) - digest.update(b"\0") - digest.update(str(entry_ordinal).encode("utf-8")) - digest.update(b"\0") - digest.update(str(split_index).encode("utf-8")) - digest.update(b"\0") - digest.update(bytes.fromhex(blob_hash)) - return digest.hexdigest() - - @dataclass(frozen=True, slots=True) class SourceReadContext: """Common acquisition dependencies for one local source artifact.""" diff --git a/polylogue/storage/blob_integrity.py b/polylogue/storage/blob_integrity.py index 7afc537708..108f248353 100644 --- a/polylogue/storage/blob_integrity.py +++ b/polylogue/storage/blob_integrity.py @@ -33,6 +33,7 @@ from polylogue.core.json import JSONDecodeError as CoreJSONDecodeError from polylogue.core.json import dumps_bytes as json_dumps_bytes from polylogue.core.json import loads as json_loads +from polylogue.core.raw_coordinates import zip_member_identity_coordinate from polylogue.logging import get_logger from polylogue.storage.blob_store import BlobNamespaceEntry, BlobStore from polylogue.storage.introspection import column_exists as _column_exists @@ -1119,6 +1120,8 @@ def _current_raw_payload_bytes( source_path: str, source_index: int | None, *, + raw_id: str | None = None, + blob_hash: str | None = None, source_bytes_cache: dict[str, bytes] | None = None, decoded_payload_cache: dict[str, object] | None = None, ) -> tuple[bytes | None, str | None]: @@ -1129,14 +1132,36 @@ def _current_raw_payload_bytes( zip_path, member = split if not zip_path.exists(): return None, "source_missing" + entry_ordinal: int | None = None + split_index = source_index + if raw_id is not None and blob_hash is not None and source_index is not None: + coordinate = zip_member_identity_coordinate( + raw_id=raw_id, + source_path=source_path, + source_index=source_index, + blob_hash=blob_hash, + ) + if coordinate is not None: + entry_ordinal, split_index = coordinate + cache_key = source_path if entry_ordinal is None else f"{source_path}\0{entry_ordinal}" try: - if source_bytes_cache is not None and source_path in source_bytes_cache: - member_bytes = source_bytes_cache[source_path] + if source_bytes_cache is not None and cache_key in source_bytes_cache: + member_bytes = source_bytes_cache[cache_key] else: with zipfile.ZipFile(zip_path) as archive: - matching = [info for info in archive.infolist() if info.filename == member] + central_directory = archive.infolist() + if entry_ordinal is None: + matching = [info for info in central_directory if info.filename == member] + elif entry_ordinal >= len(central_directory): + return None, "container_coordinate_mismatch" + else: + coordinated = central_directory[entry_ordinal] + matching = [coordinated] if coordinated.filename == member else [] if len(matching) != 1: - return None, "ambiguous_container_member" + reason = ( + "ambiguous_container_member" if entry_ordinal is None else "container_coordinate_mismatch" + ) + return None, reason admitted = list( ZipAdmission(zip_path=zip_path).filter_entries(matching, allowed_suffixes=ZIP_JSON_SUFFIXES) ) @@ -1145,32 +1170,34 @@ def _current_raw_payload_bytes( with open_bounded_zip_entry(archive, admitted[0]) as handle: member_bytes = handle.read(MAX_UNCOMPRESSED_SIZE + 1) if source_bytes_cache is not None: - source_bytes_cache[source_path] = member_bytes + source_bytes_cache[cache_key] = member_bytes except KeyError: return None, "source_missing" except ZipBombError: return None, "container_member_rejected" - if source_index is None: + if split_index is None: return None, "source_index_missing" + if blob_hash is not None and hashlib.sha256(member_bytes).hexdigest() == blob_hash: + return member_bytes, None try: - if decoded_payload_cache is not None and source_path in decoded_payload_cache: - decoded_payload = decoded_payload_cache[source_path] + if decoded_payload_cache is not None and cache_key in decoded_payload_cache: + decoded_payload = decoded_payload_cache[cache_key] if isinstance(decoded_payload, list): - payload = decoded_payload[int(source_index)] - elif int(source_index) == 0: + payload = decoded_payload[int(split_index)] + elif int(split_index) == 0: payload = decoded_payload else: raise IndexError("non-array JSON payload only supports source_index 0") else: if member.endswith(".jsonl"): - payload = _jsonl_payload_at_index(member_bytes, int(source_index)) + payload = _jsonl_payload_at_index(member_bytes, int(split_index)) else: decoded_payload = json_loads(member_bytes) if decoded_payload_cache is not None: - decoded_payload_cache[source_path] = decoded_payload + decoded_payload_cache[cache_key] = decoded_payload if isinstance(decoded_payload, list): - payload = decoded_payload[int(source_index)] - elif int(source_index) == 0: + payload = decoded_payload[int(split_index)] + elif int(split_index) == 0: payload = decoded_payload else: raise IndexError("non-array JSON payload only supports source_index 0") @@ -1519,6 +1546,8 @@ def replace_raw_backed_blob_reference_debt_from_source( payload_bytes, reason = _current_raw_payload_bytes( source_path, int(row["source_index"]) if row.get("source_index") is not None else None, + raw_id=raw_id, + blob_hash=old_blob_hash, source_bytes_cache=source_bytes_cache, decoded_payload_cache=decoded_payload_cache, ) @@ -1611,6 +1640,8 @@ def replace_raw_backed_blob_reference_debt_from_source( payload_bytes, _reason = _current_raw_payload_bytes( source_path, int(row["source_index"]) if row.get("source_index") is not None else None, + raw_id=str(row["raw_id"]), + blob_hash=str(row.get("blob_hash") or ""), source_bytes_cache=apply_source_bytes_cache, decoded_payload_cache=apply_decoded_payload_cache, ) diff --git a/polylogue/storage/sqlite/archive_tiers/source_write.py b/polylogue/storage/sqlite/archive_tiers/source_write.py index fdbe3137aa..c087546809 100644 --- a/polylogue/storage/sqlite/archive_tiers/source_write.py +++ b/polylogue/storage/sqlite/archive_tiers/source_write.py @@ -1292,9 +1292,8 @@ def upsert_raw_artifact( with conn if manage_transaction else nullcontext(): existing = conn.execute( f""" - SELECT a.artifact_id, a.raw_id, a.first_observed_at_ms, a.last_observed_at_ms, r.rowid + SELECT a.artifact_id, a.raw_id FROM raw_artifacts AS a - JOIN raw_sessions AS r ON r.raw_id = a.raw_id WHERE {coordinate_predicate} """, coordinate_params, @@ -1303,7 +1302,7 @@ def upsert_raw_artifact( # One coordinate has one authority carrier. A delayed census of # stale retained bytes must not replace a carrier observed later. if str(existing[1]) != raw_id: - incoming_row = conn.execute( + incoming_receipt = conn.execute( """ SELECT acquired_at_ms, rowid FROM blob_refs WHERE ref_id = ? AND ref_type = 'raw_payload' @@ -1311,13 +1310,7 @@ def upsert_raw_artifact( """, (raw_id,), ).fetchone() - if incoming_row is None: - incoming_row = conn.execute( - "SELECT acquired_at_ms, rowid FROM raw_sessions WHERE raw_id = ?", (raw_id,) - ).fetchone() - if incoming_row is None: - raise KeyError(raw_id) - existing_observation = conn.execute( + existing_receipt = conn.execute( """ SELECT acquired_at_ms, rowid FROM blob_refs WHERE ref_id = ? AND ref_type = 'raw_payload' @@ -1325,12 +1318,29 @@ def upsert_raw_artifact( """, (str(existing[1]),), ).fetchone() - existing_order = ( - (int(existing_observation[0]), int(existing_observation[1])) - if existing_observation is not None - else (int(existing[3]), int(existing[4])) - ) - if existing_order >= (int(incoming_row[0]), int(incoming_row[1])): + if (incoming_receipt is None) != (existing_receipt is None): + raise RuntimeError( + "cannot compare artifact observation order across incompatible raw-payload receipt coverage" + ) + if incoming_receipt is None: + incoming_observation = conn.execute( + "SELECT acquired_at_ms, rowid FROM raw_sessions WHERE raw_id = ?", + (raw_id,), + ).fetchone() + existing_observation = conn.execute( + "SELECT acquired_at_ms, rowid FROM raw_sessions WHERE raw_id = ?", + (str(existing[1]),), + ).fetchone() + else: + incoming_observation = incoming_receipt + existing_observation = existing_receipt + if incoming_observation is None: + raise KeyError(raw_id) + if existing_observation is None: + raise KeyError(str(existing[1])) + existing_order = (int(existing_observation[0]), int(existing_observation[1])) + incoming_order = (int(incoming_observation[0]), int(incoming_observation[1])) + if existing_order >= incoming_order: conn.execute( "UPDATE raw_artifacts SET first_observed_at_ms = MIN(first_observed_at_ms, ?) WHERE artifact_id = ?", (artifact.first_observed_at_ms, str(existing[0])), diff --git a/tests/unit/core/test_sampling.py b/tests/unit/core/test_sampling.py index 6976866cd7..2b61480f84 100644 --- a/tests/unit/core/test_sampling.py +++ b/tests/unit/core/test_sampling.py @@ -358,7 +358,15 @@ def test_sampling_records_equal_raw_transition_timestamps_as_indeterminate(self, ) assert result == [] - assert outcomes[0]["reason"] == "source_validation_parse_order_ambiguous" + assert outcomes == [ + { + "raw_id": raw_id, + "status": "quarantined", + "artifact_kind": None, + "source_path": "/tmp/equal-time.json", + "reason": "source_validation_parse_order_ambiguous", + } + ] def test_record_provider_sampling_streams_without_full_envelope( self, diff --git a/tests/unit/daemon/test_daemon_cli.py b/tests/unit/daemon/test_daemon_cli.py index 2ce35e070b..d79ceb32cc 100644 --- a/tests/unit/daemon/test_daemon_cli.py +++ b/tests/unit/daemon/test_daemon_cli.py @@ -1197,7 +1197,11 @@ def reject_repair(*_args: object, **_kwargs: object) -> None: monkeypatch.setattr("polylogue.product.raw_authority.repair_materialization", reject_repair) monkeypatch.setattr(ActiveWriterLease, "acquire", refuse_outer_lease) monkeypatch.setattr(daemon_cli, "_emit_raw_materialization_pass", emitted.append) - monkeypatch.setattr(daemon_cli, "_converge_raw_authority_frontier", lambda _config, **_kwargs: 0) + monkeypatch.setattr( + daemon_cli, + "_converge_raw_authority_frontier", + lambda _config, **_kwargs: pytest.fail("frontier convergence requires an acquired generation pin"), + ) monkeypatch.setattr( daemon_cli, "_close_raw_materialization_fts", diff --git a/tests/unit/sources/test_revision_backfill.py b/tests/unit/sources/test_revision_backfill.py index 114d744483..ff72ee8945 100644 --- a/tests/unit/sources/test_revision_backfill.py +++ b/tests/unit/sources/test_revision_backfill.py @@ -1061,6 +1061,7 @@ def test_backfill_preserves_latest_terminal_artifact_observation(tmp_path: Path) backfill_historical_revision_evidence(tmp_path) with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT COUNT(*) FROM raw_artifacts").fetchone() == (1,) assert conn.execute("SELECT raw_id, last_observed_at_ms FROM raw_artifacts").fetchone() == (newer_raw_id, 2) assert older_raw_id > newer_raw_id assert conn.execute( @@ -1073,7 +1074,7 @@ def test_backfill_preserves_latest_terminal_artifact_observation(tmp_path: Path) def test_backfill_uses_raw_observation_order_for_equal_time_artifacts(tmp_path: Path) -> None: - """Equal observation times use the durable raw insertion order, not raw-id order.""" + """Legacy receipt-free observations use raw insertion order, not raw-id order.""" initialize_active_archive_root(tmp_path) source_path = str(tmp_path / ".claude" / "projects" / "proj" / "subagents" / "workflows" / "wf" / "journal.jsonl") with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: @@ -1092,13 +1093,57 @@ def test_backfill_uses_raw_observation_order_for_equal_time_artifacts(tmp_path: raw_id="z-newer-artifact", ) + with sqlite3.connect(tmp_path / "source.db") as conn: + # Legacy rows can lack receipts entirely. The fallback must compare + # both observations through raw_sessions, never one rowid per table. + conn.execute("DELETE FROM blob_refs WHERE ref_id IN (?, ?)", (older_raw_id, newer_raw_id)) + conn.commit() + backfill_historical_revision_evidence(tmp_path) with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT COUNT(*) FROM raw_artifacts").fetchone() == (1,) assert conn.execute("SELECT raw_id, last_observed_at_ms FROM raw_artifacts").fetchone() == (newer_raw_id, 1) assert older_raw_id < newer_raw_id +def test_backfill_preserves_latest_repeated_artifact_observation(tmp_path: Path) -> None: + """A -> B -> A reacquisition restores A as the coordinate authority.""" + initialize_active_archive_root(tmp_path) + source_path = str(tmp_path / ".claude" / "projects" / "proj" / "subagents" / "workflows" / "wf" / "journal.jsonl") + payload_a = b'{"contentKey":"workflow-artifact","agentId":"a"}\n' + payload_b = b'{"contentKey":"workflow-artifact","agentId":"b"}\n' + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_a = archive.write_raw_payload( + provider=Provider.CLAUDE_CODE, + payload=payload_a, + source_path=source_path, + acquired_at_ms=1, + ) + raw_b = archive.write_raw_payload( + provider=Provider.CLAUDE_CODE, + payload=payload_b, + source_path=source_path, + acquired_at_ms=2, + ) + assert ( + archive.write_raw_payload( + provider=Provider.CLAUDE_CODE, + payload=payload_a, + source_path=source_path, + acquired_at_ms=3, + ) + == raw_a + ) + + backfill_historical_revision_evidence(tmp_path) + + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT COUNT(*) FROM raw_artifacts").fetchone() == (1,) + assert conn.execute("SELECT raw_id, last_observed_at_ms FROM raw_artifacts").fetchone() == (raw_a, 3) + assert raw_a != raw_b + + def test_historical_backfill_selects_prefix_newest_independent_of_acquisition_order(tmp_path: Path) -> None: initialize_active_archive_root(tmp_path) baseline = ( diff --git a/tests/unit/storage/test_blob_integrity.py b/tests/unit/storage/test_blob_integrity.py index a59be45beb..a906fe004f 100644 --- a/tests/unit/storage/test_blob_integrity.py +++ b/tests/unit/storage/test_blob_integrity.py @@ -14,6 +14,7 @@ from polylogue.archive import zip_admission from polylogue.archive.message.roles import Role from polylogue.core.enums import BlockType, Provider +from polylogue.core.raw_coordinates import zip_member_raw_id, zip_member_source_index from polylogue.sources.parsers.base import ParsedAttachment, ParsedContentBlock, ParsedMessage, ParsedSession from polylogue.storage import blob_integrity from polylogue.storage.blob_gc import run_blob_gc_report @@ -1021,6 +1022,112 @@ def fail_open(*args: object, **kwargs: object) -> object: assert reason == "ambiguous_container_member" +def test_blob_recovery_uses_v2_entry_ordinal_without_consuming_split_index(tmp_path: Path) -> None: + """Durable live-ZIP identities reacquire the exact duplicate-name entry.""" + source_db = tmp_path / "source.db" + store = BlobStore(tmp_path / "blob") + zip_source = tmp_path / "duplicate-v2.zip" + member = "sessions/duplicate.json" + member_payloads = ( + b'[{"member":"first-zero"},{"member":"first-one"}]', + b'[{"member":"second-zero"},{"member":"second-one"}]', + ) + selected_payloads = (b'{"member":"first-one"}', b'{"member":"second-one"}') + split_index = 1 + with zipfile.ZipFile(zip_source, "w") as archive: + archive.writestr(member, member_payloads[0]) + with pytest.warns(UserWarning, match="Duplicate name"): + archive.writestr(member, member_payloads[1]) + + source_path = f"{zip_source}:{member}" + hashes = tuple(hashlib.sha256(payload).hexdigest() for payload in selected_payloads) + coordinates = tuple( + zip_member_source_index(entry_ordinal=ordinal, split_index=split_index) + for ordinal in range(len(member_payloads)) + ) + raw_ids = tuple( + zip_member_raw_id( + source_path=source_path, + entry_ordinal=ordinal, + split_index=split_index, + blob_hash=hashes[ordinal], + ) + for ordinal in range(len(member_payloads)) + ) + with sqlite3.connect(source_db) as conn: + conn.executescript( + """ + CREATE TABLE raw_sessions ( + raw_id TEXT PRIMARY KEY, + origin TEXT, + native_id TEXT, + source_path TEXT, + source_index INTEGER, + blob_hash BLOB, + blob_size INTEGER NOT NULL, + acquired_at_ms INTEGER, + file_mtime_ms INTEGER + ); + CREATE TABLE blob_refs ( + blob_hash BLOB NOT NULL, + ref_id TEXT NOT NULL, + ref_type TEXT NOT NULL, + source_path TEXT, + size_bytes INTEGER NOT NULL, + acquired_at_ms INTEGER NOT NULL, + PRIMARY KEY(blob_hash, ref_type, ref_id) + ); + CREATE TABLE blob_publication_reservations ( + publication_id TEXT PRIMARY KEY, + blob_hash BLOB NOT NULL, + size_bytes INTEGER NOT NULL, + publisher_id TEXT NOT NULL, + reserved_at_ms INTEGER NOT NULL + ); + """ + ) + conn.executemany( + """ + INSERT INTO raw_sessions ( + raw_id, origin, native_id, source_path, source_index, blob_hash, + blob_size, acquired_at_ms, file_mtime_ms + ) VALUES (?, 'codex-session', NULL, ?, ?, ?, ?, 1, 1) + """, + [ + (raw_id, source_path, source_index, bytes.fromhex(blob_hash), len(payload)) + for raw_id, source_index, blob_hash, payload in zip( + raw_ids, coordinates, hashes, selected_payloads, strict=True + ) + ], + ) + conn.executemany( + """ + INSERT INTO blob_refs (blob_hash, ref_id, ref_type, source_path, size_bytes, acquired_at_ms) + VALUES (?, ?, 'raw_payload', ?, ?, 1) + """, + [ + (bytes.fromhex(blob_hash), raw_id, source_path, len(payload)) + for raw_id, blob_hash, payload in zip(raw_ids, hashes, selected_payloads, strict=True) + ], + ) + + report = replace_raw_backed_blob_reference_debt_from_source( + source_db, + store=store, + dry_run=False, + manifest_path=tmp_path / "duplicate-v2-replacement.jsonl", + ) + + assert report.replaced_rows == 2 + assert report.written_blobs == 2 + assert all(store.exists(blob_hash) for blob_hash in hashes) + assert tuple(store.read_all(blob_hash) for blob_hash in hashes) == selected_payloads + with sqlite3.connect(source_db) as conn: + assert conn.execute( + "SELECT raw_id, lower(hex(blob_hash)), source_index FROM raw_sessions ORDER BY source_index" + ).fetchall() == list(zip(raw_ids, hashes, coordinates, strict=True)) + + def test_blob_recovery_rejects_oversized_container_member_before_open( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/storage/test_repair.py b/tests/unit/storage/test_repair.py index abb29a001e..c82b2b9b8c 100644 --- a/tests/unit/storage/test_repair.py +++ b/tests/unit/storage/test_repair.py @@ -972,6 +972,10 @@ def test_raw_materialization_replays_successful_raw_with_historical_validation_f active_index = tmp_path / "generations" / "active" / "index.db" initialize_archive_database(active_index, ArchiveTier.INDEX) (tmp_path / ".index-active-pointer").write_text(f"{active_index}\n", encoding="utf-8") + with sqlite3.connect(tmp_path / "index.db") as conn: + shadow_applications_before = conn.execute( + "SELECT COUNT(*) FROM raw_revision_applications WHERE raw_id = ?", (raw_id,) + ).fetchone() replay = repair_mod.repair_raw_materialization(_config(tmp_path)) @@ -981,6 +985,10 @@ def test_raw_materialization_replays_successful_raw_with_historical_validation_f assert conn.execute("SELECT COUNT(*) FROM sessions WHERE raw_id = ?", (raw_id,)).fetchone() == (1,) with sqlite3.connect(tmp_path / "index.db") as conn: assert conn.execute("SELECT COUNT(*) FROM sessions WHERE raw_id = ?", (raw_id,)).fetchone() == (1,) + assert ( + conn.execute("SELECT COUNT(*) FROM raw_revision_applications WHERE raw_id = ?", (raw_id,)).fetchone() + == shadow_applications_before + ) @pytest.mark.parametrize("validation_offset", [0, 1]) @@ -1652,6 +1660,8 @@ def test_raw_materialization_skips_current_non_session_census(tmp_path: Path) -> acquired_at_ms=1, ) + assert raw_id in repair_mod._raw_materialization_candidate_ids(_config(tmp_path)).raw_ids + census_historical_revision_evidence(tmp_path) assert raw_id not in repair_mod._raw_materialization_candidate_ids(_config(tmp_path)).raw_ids From 553a05dca2524b233ef5b28dd1d6586de9320e5d Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 14 Aug 2026 00:47:00 +0200 Subject: [PATCH 54/65] fix: close retained replay review gaps Bound retained document scalar tokens before structural provider detection and preserve whole-sequence detector tightness. Carry Codex state snapshots through frozen validation as non-session evidence, enforce replay budgets before state parsing, and refresh configured roots during periodic catch-up. --- polylogue/sources/live/watcher.py | 12 +- polylogue/sources/revision_backfill.py | 209 +++++++++++++++--- tests/unit/sources/test_live_batch_support.py | 11 +- tests/unit/sources/test_live_watcher.py | 36 +++ tests/unit/sources/test_revision_backfill.py | 96 ++++++++ 5 files changed, 327 insertions(+), 37 deletions(-) diff --git a/polylogue/sources/live/watcher.py b/polylogue/sources/live/watcher.py index dee13fd265..4c8143f8bc 100644 --- a/polylogue/sources/live/watcher.py +++ b/polylogue/sources/live/watcher.py @@ -317,6 +317,10 @@ async def _run_writer_sync( def catch_up_complete(self) -> asyncio.Event: return self._catch_up_complete + def _existing_source_roots(self) -> list[Path]: + """Return configured roots that exist at the instant of a scan.""" + return [source.root for source in self._sources if source.exists()] + async def run(self) -> None: # Hook commands create their first pending envelope lazily. Ensure the # nested root exists before ``awatch`` snapshots its roots, otherwise a @@ -324,7 +328,7 @@ async def run(self) -> None: for source in self._sources: if source.name == "hooks": source.root.mkdir(parents=True, exist_ok=True) - roots = [s.root for s in self._sources if s.exists()] + roots = self._existing_source_roots() if not roots: logger.warning("live.watcher: no source roots exist; nothing to watch") self._catch_up_complete.set() @@ -406,14 +410,16 @@ def cancel_pending(self) -> None: self._cancel_periodic_catch_up() self._cancel_hook_spool_directory_retries() - async def _periodic_catch_up(self, roots: list[Path]) -> None: + async def _periodic_catch_up(self, _initial_roots: list[Path]) -> None: delay_s = _PERIODIC_CATCH_UP_INTERVAL_S while not self._stop.is_set(): await asyncio.sleep(delay_s) if self._stop.is_set(): return try: - await self._catch_up(roots) + roots = self._existing_source_roots() + if roots: + await self._catch_up(roots) except sqlite3.OperationalError as exc: if not _is_database_locked(exc): raise diff --git a/polylogue/sources/revision_backfill.py b/polylogue/sources/revision_backfill.py index c97f06570c..365ebcd6f9 100644 --- a/polylogue/sources/revision_backfill.py +++ b/polylogue/sources/revision_backfill.py @@ -9,7 +9,7 @@ import threading import time from collections import OrderedDict -from collections.abc import Callable, Iterator, Sequence +from collections.abc import Callable, Iterator, Sequence, Set from concurrent.futures import Future, ThreadPoolExecutor from contextlib import closing, contextmanager, nullcontext from dataclasses import dataclass, field @@ -166,6 +166,95 @@ def _document_probe_value(key: str, event: str, value: object) -> object | None: return None +class _ScalarBoundedJSONReader: + """Stream JSON while capping every scalar token before ijson sees it.""" + + def __init__(self, payload: BinaryIO) -> None: + self._payload = payload + self._output = bytearray() + self._eof = False + self._in_string = False + self._string_bytes = 0 + self._escape = bytearray() + self._escape_target = 0 + self._utf8_remaining = 0 + self._emit_utf8 = False + self._in_number = False + + def read(self, size: int = -1) -> bytes: + if size == 0: + return b"" + if size < 0: + chunks: list[bytes] = [] + while chunk := self.read(_REPLAY_PROVIDER_DETECTION_PREFIX_BYTES): + chunks.append(chunk) + return b"".join(chunks) + while len(self._output) < size and not self._eof: + chunk = self._payload.read(_REPLAY_PROVIDER_DETECTION_PREFIX_BYTES) + if not chunk: + self._eof = True + break + self._filter(chunk) + result = bytes(self._output[:size]) + del self._output[:size] + return result + + def _filter(self, chunk: bytes) -> None: + for byte in chunk: + if self._in_string: + self._filter_string_byte(byte) + continue + if self._in_number: + if byte in b"0123456789.eE+-": + continue + self._in_number = False + if byte == ord('"'): + self._output.append(byte) + self._in_string = True + self._string_bytes = 0 + elif byte in b"-0123456789": + self._output.extend(b"0") + self._in_number = True + else: + self._output.append(byte) + + def _filter_string_byte(self, byte: int) -> None: + if self._escape: + self._escape.append(byte) + if len(self._escape) == 2: + self._escape_target = 6 if byte == ord("u") else 2 + if len(self._escape) == self._escape_target: + if self._string_bytes + len(self._escape) <= _REPLAY_PROVIDER_DETECTION_PREFIX_BYTES: + self._output.extend(self._escape) + self._string_bytes += len(self._escape) + self._escape.clear() + self._escape_target = 0 + return + if self._utf8_remaining: + if self._emit_utf8: + self._output.append(byte) + self._utf8_remaining -= 1 + return + if byte == ord("\\"): + self._escape.append(byte) + return + if byte == ord('"'): + self._output.append(byte) + self._in_string = False + return + if byte < 0x80: + if self._string_bytes < _REPLAY_PROVIDER_DETECTION_PREFIX_BYTES: + self._output.append(byte) + self._string_bytes += 1 + return + utf8_bytes = 2 if byte < 0xE0 else 3 if byte < 0xF0 else 4 if byte < 0xF8 else 1 + self._utf8_remaining = utf8_bytes - 1 + self._emit_utf8 = self._string_bytes + utf8_bytes <= _REPLAY_PROVIDER_DETECTION_PREFIX_BYTES + if self._emit_utf8: + self._output.append(byte) + self._string_bytes += utf8_bytes + + @dataclass(slots=True) class _StreamingDocumentProviderProbe: """Bounded structural summary for one object in a JSON document. @@ -297,9 +386,9 @@ def feed(self, prefix: str, event: str, value: object) -> None: self.conversation_item_has_conversation and self.conversation_item_has_responses ) - def classify(self) -> tuple[Provider, str]: + def classify(self, *, sequence_item: bool = False) -> tuple[Provider, str]: if self.mapping_seen and self.mapping_valid: - self.payload["mapping"] = {"bounded-node": {"message": None}} + self.payload["mapping"] = {"bounded-node": {"id": "bounded-node", "message": None}} if self.chat_message_matched: self.payload["chat_messages"] = [{"role": "present", "text": "present"}] if self.first_message_complete: @@ -312,7 +401,8 @@ def classify(self) -> tuple[Provider, str]: self.payload["chunks"] = [chunk] if self.conversation_matched: self.payload["conversations"] = [{"conversation": {}, "responses": []}] - provider, evidence = detect_provider_evidence(self.payload) + candidate: object = [self.payload] if sequence_item else self.payload + provider, evidence = detect_provider_evidence(candidate) if provider is None: return Provider.UNKNOWN, evidence return provider, f"bounded streaming JSON structure: {evidence}" @@ -321,11 +411,12 @@ def classify(self) -> tuple[Provider, str]: def _detect_provider_from_bounded_document(payload: BinaryIO) -> tuple[Provider, str]: """Scan every document object while retaining fixed structural evidence.""" payload.seek(0) + bounded_payload = _ScalarBoundedJSONReader(payload) probe: _StreamingDocumentProviderProbe | None = None root_is_array = False last_evidence = "no bounded document structure identified a provider; used fallback_provider" try: - for prefix, event, value in ijson.parse(payload, use_float=True): + for prefix, event, value in ijson.parse(bounded_payload, use_float=True): if prefix == "" and event == "start_array": root_is_array = True continue @@ -336,7 +427,7 @@ def _detect_provider_from_bounded_document(payload: BinaryIO) -> tuple[Provider, if probe is None: continue if prefix == "item" and event == "end_map": - provider, last_evidence = probe.classify() + provider, last_evidence = probe.classify(sequence_item=True) if provider is not Provider.UNKNOWN: return provider, last_evidence probe = None @@ -589,6 +680,7 @@ class _RevisionCensusState: censused: set[str] membership_candidates: dict[str, set[str]] provisional_full_raw_ids: dict[str, set[str]] + transient_non_session_raw_ids: set[str] @dataclass(slots=True) @@ -992,7 +1084,7 @@ def _census_historical_revision_evidence( replay) still independently re-derives byte-provenness from raw bytes for every raw. """ - state = _RevisionCensusState(0, 0, 0, set(), {}, {}) + state = _RevisionCensusState(0, 0, 0, set(), {}, {}, set()) batch_size = commit_batch_size if commit_batch_size is not None and commit_batch_size > 0 else None batched = batch_size is not None pending_commits = 0 @@ -1178,6 +1270,21 @@ def bind_byte_proven_older_member(raw_id: str, logical_key: str) -> None: census_selection = initial_selection while True: rows = archive.raw_membership_census_rows(census_selection) + if max_payload_bytes is not None: + payload_sizes = archive.raw_payload_sizes( + [ + raw_id + for raw_id, _source_index, terminal_non_session, _raw_rowid in rows + if raw_id not in state.censused and not terminal_non_session + ] + ) + total_payload_bytes = sum(payload_sizes.values()) + oversized = [raw_id for raw_id, size in payload_sizes.items() if size > max_payload_bytes] + if oversized or total_payload_bytes > max_payload_bytes: + blocked_ids = oversized or list(payload_sizes) + raise RawRevisionReplayResourceBlockedError( + sorted(blocked_ids), max_payload_bytes, total_payload_bytes + ) for raw_id, _source_index, _terminal_non_session, _raw_rowid in sorted( rows, key=lambda row: archive.raw_revision_observation_order(row[0]), @@ -1186,6 +1293,7 @@ def bind_byte_proven_older_member(raw_id: str, logical_key: str) -> None: continue state.scanned += 1 state.censused.add(raw_id) + state.transient_non_session_raw_ids.add(raw_id) commit_unit() terminal_raw_ids = { raw_id for raw_id, _source_index, terminal_non_session, _raw_rowid in rows if terminal_non_session @@ -1198,15 +1306,6 @@ def bind_byte_proven_older_member(raw_id: str, logical_key: str) -> None: for raw_id, source_index, terminal_non_session, _raw_rowid in rows if raw_id not in state.censused and not terminal_non_session ] - if max_payload_bytes is not None: - payload_sizes = archive.raw_payload_sizes([raw_id for raw_id, _index in pending_rows]) - total_payload_bytes = sum(payload_sizes.values()) - oversized = [raw_id for raw_id, size in payload_sizes.items() if size > max_payload_bytes] - if oversized or total_payload_bytes > max_payload_bytes: - blocked_ids = oversized or list(payload_sizes) - raise RawRevisionReplayResourceBlockedError( - sorted(blocked_ids), max_payload_bytes, total_payload_bytes - ) # Parse is read-only blob->ParsedSession decode and authority-neutral; # spread it across a process pool when there is more than one raw to # parse. Archive writes below stay in fixed `pending_rows` order @@ -1282,10 +1381,6 @@ def _load_frozen_revision_evidence( expanded_raw_ids, _logical_keys = archive.expand_raw_membership_selection(selected_raw_ids) if selected_raw_ids is not None: expanded_raw_ids = _expand_frozen_revision_link_selection(archive.archive_root, expanded_raw_ids) - recorded_logical_keys = require_current_parser_source_census( - archive.archive_root, - selected_raw_ids=expanded_raw_ids if selected_raw_ids is not None else None, - ) rows = archive.raw_membership_census_rows(expanded_raw_ids if selected_raw_ids is not None else None) if max_payload_bytes is not None: payload_sizes = archive.raw_payload_sizes( @@ -1297,10 +1392,20 @@ def _load_frozen_revision_evidence( raise RawRevisionReplayResourceBlockedError( sorted(oversized or payload_sizes), max_payload_bytes, total_payload_bytes ) + frozen_codex_state_raw_ids = frozenset( + raw_id + for raw_id, _source_index, terminal_non_session, _raw_rowid in rows + if not terminal_non_session and _retained_codex_state_descriptor(archive, raw_id) is not None + ) + recorded_logical_keys = require_current_parser_source_census( + archive.archive_root, + selected_raw_ids=expanded_raw_ids if selected_raw_ids is not None else None, + transient_non_session_raw_ids=frozen_codex_state_raw_ids, + ) parseable_raw_ids = [ raw_id for raw_id, source_index, terminal_non_session, _raw_rowid in rows - if source_index >= 0 and not terminal_non_session + if source_index >= 0 and not terminal_non_session and raw_id not in frozen_codex_state_raw_ids ] parsed_outcomes = _parse_retained_raws( archive, @@ -1308,11 +1413,11 @@ def _load_frozen_revision_evidence( ingest_workers=ingest_workers, prefetch_cache=prefetch_cache, ) - state = _RevisionCensusState(0, 0, 0, set(), {}, {}) + state = _RevisionCensusState(0, 0, 0, set(), {}, {}, set(frozen_codex_state_raw_ids)) for raw_id, source_index, terminal_non_session, _raw_rowid in rows: state.scanned += 1 state.censused.add(raw_id) - if terminal_non_session: + if terminal_non_session or raw_id in frozen_codex_state_raw_ids: continue if source_index < 0: state.quarantined += 1 @@ -1353,6 +1458,7 @@ def require_current_parser_source_census( archive_root: Path, *, selected_raw_ids: Sequence[str] | None = None, + transient_non_session_raw_ids: Set[str] = frozenset(), ) -> dict[str, tuple[str, ...]]: """Require phase-2 parser receipts before allocating an index candidate.""" stale_raw_ids: list[str] = [] @@ -1380,6 +1486,9 @@ def require_current_parser_source_census( ) for raw_id_value, fingerprint, status, logical_keys_json in rows: raw_id = str(raw_id_value) + if raw_id in transient_non_session_raw_ids: + recorded_logical_keys[raw_id] = () + continue if fingerprint != RAW_AUTHORITY_PARSER_FINGERPRINT or status != "complete": stale_raw_ids.append(raw_id) continue @@ -1415,6 +1524,7 @@ def require_current_parser_source_census( ) for raw_id_value, typed_key, revision_kind, membership_key, typed_non_session in rows: raw_id = str(raw_id_value) + typed_non_session = bool(typed_non_session) or raw_id in transient_non_session_raw_ids existing_typed, existing_kind, memberships, existing_non_session = durable_bindings.get( raw_id, (typed_key, revision_kind, [], bool(typed_non_session)) ) @@ -1589,6 +1699,7 @@ def require_current_parser_source_census( """, authority_params, ) + if str(row[0]) not in transient_non_session_raw_ids ) if unresolved_raw_ids: sample = ", ".join(unresolved_raw_ids[:5]) @@ -1599,6 +1710,27 @@ def require_current_parser_source_census( return recorded_logical_keys +def _logical_keys_for_raw_ids(archive: ArchiveStore, raw_ids: Set[str]) -> set[str]: + """Read typed logical keys for an arbitrary-size raw selection.""" + keys: set[str] = set() + ordered_raw_ids = sorted(raw_ids) + conn = archive._ensure_source_conn() + for offset in range(0, len(ordered_raw_ids), 500): + chunk = ordered_raw_ids[offset : offset + 500] + placeholders = ",".join("?" for _ in chunk) + keys.update( + str(row[0]) + for row in conn.execute( + f""" + SELECT DISTINCT logical_source_key FROM raw_sessions + WHERE raw_id IN ({placeholders}) AND logical_source_key IS NOT NULL + """, + chunk, + ) + ) + return keys + + def validate_frozen_source_authority( archive_root: Path, *, @@ -1629,11 +1761,15 @@ def validate_frozen_source_authority( prefetch_cache=prefetch_cache, ) _unclassified, logical_keys = archive.raw_revision_rebuild_selection(selected_raw_ids) + transient_non_session_keys = _logical_keys_for_raw_ids( + archive, + census.transient_non_session_raw_ids, + ) _membership_raw_ids, persisted_membership_keys = archive.expand_raw_membership_selection(selected_raw_ids) membership_keys = {*persisted_membership_keys, *census.membership_candidates} byte_replayed_keys: set[str] = set() - for logical_key in sorted(logical_keys): + for logical_key in sorted(set(logical_keys) - transient_non_session_keys): plan = archive.classify_raw_revision_cohort_for_frozen_candidate(logical_key) if not plan.accepted_raw_ids: convertible = archive.convertible_full_revision_raw_ids(logical_key) @@ -2864,6 +3000,20 @@ def parse_retained_raw_sessions(archive: ArchiveStore, raw_id: str) -> list[Pars ) +def _retained_codex_state_descriptor(archive: ArchiveStore, raw_id: str) -> tuple[Path, str, str] | None: + """Identify one immutable retained Codex state snapshot without mutating it.""" + provider, blob_hash, source_path, _kind, _payload_size = archive.raw_revision_descriptor(raw_id) + if provider is not Provider.CODEX: + return None + state_path = archive.blob_path_for_hash(blob_hash) + if state_path is None: + return None + state_kind = codex_state.classify_codex_sqlite_path(state_path, immutable=True) + if state_kind not in codex_state.IN_SCOPE_KINDS: + return None + return state_path, source_path, state_kind + + def _replay_retained_codex_state_evidence(archive: ArchiveStore, raw_id: str) -> bool: """Apply a retained, in-scope Codex state snapshot without minting a session. @@ -2872,15 +3022,10 @@ def _replay_retained_codex_state_evidence(archive: ArchiveStore, raw_id: str) -> recognized retained snapshot may become thread evidence. The parser reads the immutable blob path, never the original mutable state DB. """ - provider, blob_hash, source_path, _kind, _payload_size = archive.raw_revision_descriptor(raw_id) - if provider is not Provider.CODEX: - return False - state_path = archive.blob_path_for_hash(blob_hash) - if state_path is None: - return False - state_kind = codex_state.classify_codex_sqlite_path(state_path, immutable=True) - if state_kind not in codex_state.IN_SCOPE_KINDS: + descriptor = _retained_codex_state_descriptor(archive, raw_id) + if descriptor is None: return False + state_path, source_path, state_kind = descriptor if state_kind == "thread_state": write_codex_thread_state_evidence( archive, diff --git a/tests/unit/sources/test_live_batch_support.py b/tests/unit/sources/test_live_batch_support.py index a6597043ff..19ff2c06c9 100644 --- a/tests/unit/sources/test_live_batch_support.py +++ b/tests/unit/sources/test_live_batch_support.py @@ -53,7 +53,10 @@ ) from polylogue.sources.live.cursor import CursorStore from polylogue.sources.parsers.base import ParsedMessage, ParsedSession -from polylogue.sources.revision_backfill import backfill_historical_revision_evidence +from polylogue.sources.revision_backfill import ( + backfill_historical_revision_evidence, + validate_frozen_source_authority, +) from polylogue.sources.source_acquisition_components import stream_preserved_zip_entry_raw_data from polylogue.sources.source_parsing import has_decoded_session_evidence from polylogue.storage.blob_store import BlobStore @@ -1055,7 +1058,7 @@ def _write_codex_thread_state_db(path: Path) -> None: def test_source_only_codex_state_recovery_replays_retained_thread_evidence(tmp_path: Path) -> None: - """Removing the replay effect leaves the durable state raw pending and title-less.""" + """Frozen validation admits state as non-session before mutable replay.""" from polylogue.core.degraded import DegradedReason, clear_degraded, set_degraded initialize_active_archive_root(tmp_path) @@ -1076,6 +1079,10 @@ def test_source_only_codex_state_recovery_replays_retained_thread_evidence(tmp_p finally: clear_degraded() + source_before = sha256((tmp_path / "source.db").read_bytes()).hexdigest() + validate_frozen_source_authority(tmp_path) + assert sha256((tmp_path / "source.db").read_bytes()).hexdigest() == source_before + replay = backfill_historical_revision_evidence(tmp_path) assert replay.scanned == 1 diff --git a/tests/unit/sources/test_live_watcher.py b/tests/unit/sources/test_live_watcher.py index 6ec94beae0..23d0aba0db 100644 --- a/tests/unit/sources/test_live_watcher.py +++ b/tests/unit/sources/test_live_watcher.py @@ -3920,6 +3920,42 @@ async def _drive() -> None: asyncio.run(_drive()) +def test_periodic_catch_up_adds_configured_nested_root_created_after_start( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """A late nested root remains recoverable after its add event is missed.""" + outer = tmp_path / "sources" + nested = outer / "late-codex" + outer.mkdir() + watcher, parse_sources = _make_watcher( + tmp_path, + outer, + sources=( + WatchSource(name="outer", root=outer, suffixes=(".jsonl",)), + WatchSource(name="nested", root=nested, suffixes=(".jsonl",)), + ), + ) + monkeypatch.setattr(live_watcher, "_PERIODIC_CATCH_UP_INTERVAL_S", 0.02) + + async def _drive() -> None: + task = asyncio.create_task(watcher._periodic_catch_up([outer])) + await asyncio.sleep(0.03) + nested.mkdir() + (nested / "missed.jsonl").write_text('{"type":"session_meta","payload":{"id":"late"}}\n') + for _ in range(60): + if parse_sources.await_count >= 1: + break + await asyncio.sleep(0.05) + watcher.stop() + task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await task + assert parse_sources.await_count >= 1 + + asyncio.run(_drive()) + + def test_periodic_catch_up_backs_off_after_each_reconciliation_pass( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, diff --git a/tests/unit/sources/test_revision_backfill.py b/tests/unit/sources/test_revision_backfill.py index ff72ee8945..2226c67ccc 100644 --- a/tests/unit/sources/test_revision_backfill.py +++ b/tests/unit/sources/test_revision_backfill.py @@ -7,6 +7,7 @@ from pathlib import Path from typing import Any +import ijson import pytest from polylogue.archive.ingest_flags import ( @@ -310,6 +311,78 @@ def test_unknown_retained_document_scans_past_oversized_leading_value(tmp_path: assert conn.execute("SELECT session_id FROM sessions").fetchall() == [("chatgpt-export:large-document",)] +def test_unknown_retained_document_caps_oversized_scalar_before_structural_scan( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """The retained-document route never hands a whole giant scalar to ijson.""" + initialize_active_archive_root(tmp_path) + payload = json.dumps({"padding": "x" * 128_000, "metadata": {"shape": "unknown"}}).encode() + observed_string_bytes: list[int] = [] + original_parse = ijson.parse + + def guarded_parse(*args: object, **kwargs: object) -> Any: + for prefix, event, value in original_parse(*args, **kwargs): + if event == "string": + observed_string_bytes.append(len(str(value).encode())) + assert observed_string_bytes[-1] <= revision_backfill._REPLAY_PROVIDER_DETECTION_PREFIX_BYTES + yield prefix, event, value + + monkeypatch.setattr(ijson, "parse", guarded_parse) + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.UNKNOWN, + payload=payload, + source_path="export/unknown-document.json", + acquired_at_ms=1, + ) + + def reject_eager_material(_raw_id: str) -> tuple[Provider, bytes, str, RawRevisionKind]: + raise AssertionError("unclassified document must not use eager payload materialization") + + monkeypatch.setattr(archive, "raw_revision_material", reject_eager_material) + with pytest.raises(ValueError, match="remained unresolved after bounded scan"): + revision_backfill.parse_retained_raw_sessions(archive, raw_id) + + assert max(observed_string_bytes) == revision_backfill._REPLAY_PROVIDER_DETECTION_PREFIX_BYTES + + +def test_unknown_retained_array_ignores_fragment_only_mapping_before_real_provider(tmp_path: Path) -> None: + """An unrelated mapping fragment cannot claim a whole document sequence.""" + initialize_active_archive_root(tmp_path) + payload = json.dumps( + [ + {"mapping": {"foreign-node": {"message": None}}, "metadata": "not a conversation"}, + { + "uuid": "later-claude-provider", + "name": "Later Claude provider", + "chat_messages": [ + { + "uuid": "claude-message", + "sender": "human", + "text": "real provider evidence", + "created_at": "2026-08-13T00:00:00Z", + } + ], + }, + ] + ).encode() + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + archive.write_raw_payload( + provider=Provider.UNKNOWN, + payload=payload, + source_path="export/unknown-array.json", + acquired_at_ms=1, + ) + + backfill_historical_revision_evidence(tmp_path) + + with sqlite3.connect(tmp_path / "index.db") as conn: + assert conn.execute("SELECT session_id FROM sessions").fetchall() == [ + ("claude-ai-export:later-claude-provider",) + ] + + def test_parsed_session_spill_uses_the_pinned_active_index_directory(tmp_path: Path) -> None: """Repair spill churn follows the generation being repaired, not a shadow index.""" archive_root = tmp_path / "archive" @@ -609,6 +682,29 @@ def _codex_thread_state_snapshot_bytes(tmp_path: Path, title: str) -> bytes: return state_path.read_bytes() +def test_codex_state_replay_applies_payload_budget_before_sqlite_parse(tmp_path: Path) -> None: + """A bounded census defers a state snapshot before it can write evidence.""" + initialize_active_archive_root(tmp_path) + payload = _codex_thread_state_snapshot_bytes(tmp_path, "oversized state") + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.CODEX, + payload=payload, + source_path=str(tmp_path / "codex" / "state_5.sqlite"), + acquired_at_ms=1, + ) + + with pytest.raises(revision_backfill.RawRevisionReplayResourceBlockedError) as blocked: + census_historical_revision_evidence(tmp_path, max_payload_bytes=1) + + assert blocked.value.raw_ids == (raw_id,) + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute( + "SELECT parsed_at_ms, parse_error FROM raw_sessions WHERE raw_id = ?", (raw_id,) + ).fetchone() == (None, None) + assert conn.execute("SELECT COUNT(*) FROM raw_hook_events").fetchone() == (0,) + + def test_backfill_replays_codex_state_by_latest_raw_observation(tmp_path: Path) -> None: """A retained A -> B -> A state sequence leaves A's title current. From b7057ebf94617bb8bee1cb31ec96df65fb564d24 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 14 Aug 2026 02:03:21 +0200 Subject: [PATCH 55/65] fix: preserve provider and ZIP acquisition authority Unknown acquisition and parser classification had drifted across live ZIP replay, corpus verification, and blob reconstruction. Oversized NDJSON sampling could also allocate a complete physical record before streaming admission. Keep acquisition provider separate from detected provider, persist independent ZIP entry and split coordinates in source v33, reuse those coordinates after blob replacement, and cap live JSONL record sampling. Inventory the new durable writer and cover the production routes with anti-vacuous regressions. --- docs/plans/layering.yaml | 2 +- polylogue/archive/raw_payload/decode.py | 4 + polylogue/schemas/validation/corpus.py | 33 +++--- polylogue/sources/live/batch.py | 63 ++++++++--- polylogue/sources/live/batch_support.py | 27 ++--- polylogue/storage/artifacts/inspection.py | 8 +- polylogue/storage/blob_integrity.py | 68 ++++++++++-- .../storage/sqlite/archive_tiers/archive.py | 18 ++++ .../storage/sqlite/archive_tiers/source.py | 7 ++ .../sqlite/archive_tiers/source_write.py | 35 ++++++ .../sqlite/migrations/source/033.train.json | 29 ++++- .../source/033_detected_raw_provider.sql | 11 ++ tests/unit/core/test_schema_validation.py | 47 ++++++++ tests/unit/sources/test_live_batch_support.py | 79 ++++++++++++++ tests/unit/storage/test_blob_integrity.py | 101 +++++++++--------- 15 files changed, 429 insertions(+), 103 deletions(-) diff --git a/docs/plans/layering.yaml b/docs/plans/layering.yaml index 2aad7cee72..22c0907fba 100644 --- a/docs/plans/layering.yaml +++ b/docs/plans/layering.yaml @@ -60,7 +60,7 @@ writer_modules: interruption: atomic entrypoints: [apply_source_raw_state_update, bind_source_raw_revision, record_capture_mode_observation, - record_excised_blob_hash, write_history_sidecar, + record_excised_blob_hash, record_raw_container_coordinate, write_history_sidecar, delete_source_hook_event, write_source_blob_refs, write_source_hook_event, write_source_raw_session, write_source_raw_session_blob_ref, upsert_raw_artifact] - path: polylogue/storage/sqlite/archive_tiers/write.py diff --git a/polylogue/archive/raw_payload/decode.py b/polylogue/archive/raw_payload/decode.py index 54916300e5..59d45987ac 100644 --- a/polylogue/archive/raw_payload/decode.py +++ b/polylogue/archive/raw_payload/decode.py @@ -90,6 +90,9 @@ class JSONLSessionArtifactScan: oversized_records: int = 0 +JSONL_RECORD_INSPECTION_BYTES = 64 * 1024 + + def _bounded_raw_lines( stream: IO[bytes] | IO[str], *, @@ -589,6 +592,7 @@ def _hermes_sqlite_marker_payload( "JSONValue", "RawPayloadEnvelope", "JSONLSessionArtifactScan", + "JSONL_RECORD_INSPECTION_BYTES", "WireFormat", "build_raw_payload_envelope", "jsonl_session_artifact", diff --git a/polylogue/schemas/validation/corpus.py b/polylogue/schemas/validation/corpus.py index 41e74d6e7d..b39b01f2c2 100644 --- a/polylogue/schemas/validation/corpus.py +++ b/polylogue/schemas/validation/corpus.py @@ -33,23 +33,30 @@ def verification_provider_clause(providers: list[str]) -> tuple[str, tuple[str, ...]]: - """Build a `raw_sessions.origin` filter for requested providers. + """Build a detected-provider-aware filter for requested providers. - raw rows carry a single ``origin`` token rather than the - legacy ``payload_provider`` / ``source_name`` pair. Each requested - provider token is mapped to its archive origin via - :func:`origin_from_provider`; the row matches when its ``origin`` is in - that set. + Parser classification outranks acquisition origin once present. Rows that + have not been classified retain the origin fallback used by older source + schemas and ordinary provider-owned acquisition. """ - origins = [origin_from_provider(Provider.from_string(p)).value for p in providers] - placeholders = ",".join("?" for _ in origins) - clause = f"origin IN ({placeholders})" - return clause, tuple(origins) + provider_tokens = [Provider.from_string(provider).value for provider in providers] + origins = [origin_from_provider(Provider.from_string(provider)).value for provider in providers] + provider_placeholders = ",".join("?" for _ in provider_tokens) + origin_placeholders = ",".join("?" for _ in origins) + clause = ( + f"(detected_provider IN ({provider_placeholders}) OR " + f"(detected_provider IS NULL AND origin IN ({origin_placeholders})))" + ) + return clause, (*provider_tokens, *origins) def _row_payload_data(row: sqlite3.Row) -> VerificationRow: - origin = str(row["origin"]) - provider = provider_from_origin(Origin.from_string(origin), family_hint=Provider.DRIVE).value + detected_provider = row["detected_provider"] + if detected_provider is not None: + provider = Provider.from_string(str(detected_provider)).value + else: + origin = str(row["origin"]) + provider = provider_from_origin(Origin.from_string(origin), family_hint=Provider.DRIVE).value return ( str(row["raw_id"]), provider, @@ -97,7 +104,7 @@ def rows() -> Iterator[sqlite3.Row]: return last_rowid = row[0] - base_query = "SELECT rowid, raw_id, origin, source_path, blob_hash FROM raw_sessions " + base_query = "SELECT rowid, raw_id, origin, detected_provider, source_path, blob_hash FROM raw_sessions " records_fetched = 0 while True: if bounded_limit is not None: diff --git a/polylogue/sources/live/batch.py b/polylogue/sources/live/batch.py index 2e732abcc5..dbfaf3d9c2 100644 --- a/polylogue/sources/live/batch.py +++ b/polylogue/sources/live/batch.py @@ -51,7 +51,11 @@ read_peak_rss_self_mb, ) from polylogue.core.provider_identity import canonical_acquisition_provider -from polylogue.core.raw_coordinates import zip_member_raw_id, zip_member_source_index +from polylogue.core.raw_coordinates import ( + zip_member_identity_coordinate, + zip_member_raw_id, + zip_member_source_index, +) from polylogue.core.raw_failure_evidence import ( RAW_FAILURE_EVIDENCE_KINDS, RAW_FAILURE_LIFECYCLE_EVIDENCE_SUPPORT_STATUS_PAIRS, @@ -392,6 +396,32 @@ def _blob_jsonl_has_session_evidence( return False +def _record_zip_container_coordinate( + archive: Any, + record: RawSessionRecord, + *, + source_raw_id: str, + blob_hash: str, +) -> None: + if record.source_index is None: + return + coordinate = zip_member_identity_coordinate( + raw_id=source_raw_id, + source_path=record.source_path, + source_index=record.source_index, + blob_hash=blob_hash, + ) + if coordinate is None: + return + entry_ordinal, split_index = coordinate + archive.record_raw_container_coordinate( + source_raw_id, + coordinate_format="zip-v2", + entry_ordinal=entry_ordinal, + split_index=split_index, + ) + + def _live_parse_stage_candidates(paths: list[Path], *, fallback_provider: Provider) -> list[LiveParseCandidate]: """Select and read eligible files for off-writer-hold pre-parse (polylogue-wf8a). @@ -2247,9 +2277,7 @@ def _ingest_full_paths_sync( raw_id=raw_id, blob_hash=(blob_hash if acquired_via_sqlite_snapshot and blob_hash is not None else None), payload_provider=provider, - capture_mode=( - acquisition_capture_mode if acquisition_capture_mode is not Provider.UNKNOWN else provider - ), + capture_mode=acquisition_capture_mode, source_name=source_name, source_path=( str(original_sqlite_source_path(path) or path) if path in raw_source_revisions else str(path) @@ -2470,6 +2498,7 @@ def _ingest_full_records_archive( record_timings: dict[str, float] = {} t0 = time.perf_counter() provider = record.payload_provider or Provider.from_string(record.source_name) + acquisition_provider = record.capture_mode or provider payload = raw_payloads.get(record.raw_id) source_name = Path(record.source_path).name fallback_id = Path(record.source_path).stem @@ -2508,7 +2537,7 @@ def _ingest_full_records_archive( explicit_raw_id = record.raw_id if record.blob_hash is not None else None if payload is None: source_raw_id = archive.admit_raw_artifact_blob_ref( - provider=provider, + provider=acquisition_provider, blob_hash_hex=blob_hash, blob_size=record.blob_size, source_path=record.source_path, @@ -2520,7 +2549,7 @@ def _ingest_full_records_archive( ).raw_id else: source_raw_id = archive.admit_raw_artifact_payload( - provider=provider, + provider=acquisition_provider, payload=payload, source_path=record.source_path, source_index=record.source_index or 0, @@ -2529,13 +2558,19 @@ def _ingest_full_records_archive( classification=artifact_classification, blob_publication_receipt_id=record.blob_publication_receipt_id, ).raw_id + _record_zip_container_coordinate( + archive, + record, + source_raw_id=source_raw_id, + blob_hash=blob_hash, + ) result.raw_ids[record.raw_id] = source_raw_id _accumulate_stage_timings(result.stage_timings_s, record_timings) continue source_write_started = time.perf_counter() if payload is None: source_raw_id = archive.write_raw_blob_ref( - provider=provider, + provider=acquisition_provider, capture_mode=record.capture_mode, blob_hash_hex=blob_hash, blob_size=record.blob_size, @@ -2557,7 +2592,7 @@ def _ingest_full_records_archive( source_write_name = "full.source_raw_blob_ref_write" else: source_raw_id = archive.write_raw_payload( - provider=provider, + provider=acquisition_provider, capture_mode=record.capture_mode, payload=payload, source_path=record.source_path, @@ -2567,6 +2602,12 @@ def _ingest_full_records_archive( post_parse=True, ) source_write_name = "full.source_raw_write" + _record_zip_container_coordinate( + archive, + record, + source_raw_id=source_raw_id, + blob_hash=blob_hash, + ) record_timings[source_write_name] = time.perf_counter() - source_write_started degraded = degraded_reason() if degraded is not None and degraded.derived_only: @@ -3308,11 +3349,7 @@ def _extract_zip_member_records( raw_id=member_raw_id, blob_hash=raw_data.blob_hash, payload_provider=member_provider, - capture_mode=( - fallback_provider - if fallback_provider is not Provider.UNKNOWN - else member_provider - ), + capture_mode=fallback_provider, source_name=member_provider.value, source_path=raw_data.source_path, source_index=source_index, diff --git a/polylogue/sources/live/batch_support.py b/polylogue/sources/live/batch_support.py index bed83e626a..ff7a7f8942 100644 --- a/polylogue/sources/live/batch_support.py +++ b/polylogue/sources/live/batch_support.py @@ -17,7 +17,11 @@ classify_artifact_path, strong_path_classification, ) -from polylogue.archive.raw_payload.decode import jsonl_session_artifact +from polylogue.archive.raw_payload.decode import ( + JSONL_RECORD_INSPECTION_BYTES, + _sample_jsonl_payload_with_detail, + jsonl_session_artifact, +) from polylogue.core.enums import Provider from polylogue.core.json import JSONDecodeError, JSONValue from polylogue.core.json import loads as json_loads @@ -540,18 +544,15 @@ def _browser_capture_provider_from_path(path: Path) -> Provider | None: def _jsonl_sample_from_path(path: Path, *, max_records: int = 32) -> list[JSONValue]: - records: list[JSONValue] = [] - with path.open("rb") as handle: - for line in handle: - if len(records) >= max_records: - break - raw = line.strip() - if not raw: - continue - try: - records.append(json_loads(raw)) - except JSONDecodeError: - continue + try: + records, _malformed_lines, _malformed_detail = _sample_jsonl_payload_with_detail( + path, + max_samples=max_records, + scan_full=False, + max_record_bytes=JSONL_RECORD_INSPECTION_BYTES, + ) + except ValueError: + return [] return records diff --git a/polylogue/storage/artifacts/inspection.py b/polylogue/storage/artifacts/inspection.py index de36d02789..c74adb2a94 100644 --- a/polylogue/storage/artifacts/inspection.py +++ b/polylogue/storage/artifacts/inspection.py @@ -18,7 +18,11 @@ RawPayloadEnvelope, build_raw_payload_envelope, ) -from polylogue.archive.raw_payload.decode import JSONLSessionArtifactScan, scan_jsonl_session_artifact +from polylogue.archive.raw_payload.decode import ( + JSONL_RECORD_INSPECTION_BYTES, + JSONLSessionArtifactScan, + scan_jsonl_session_artifact, +) from polylogue.core.enums import ArtifactSupportStatus, Provider from polylogue.core.sources import origin_from_provider from polylogue.schemas.observation import derive_bundle_scope, schema_cluster_id @@ -240,7 +244,7 @@ def _support_status( return ArtifactSupportStatus.UNSUPPORTED_PARSEABLE -_INSPECTION_PREFIX_BYTES = 64 * 1024 # 64 KB — enough to classify any format +_INSPECTION_PREFIX_BYTES = JSONL_RECORD_INSPECTION_BYTES _FULL_JSON_INSPECTION_MAX_BYTES = 8 * 1024 * 1024 # 8 MB — bounded fallback for large JSON documents diff --git a/polylogue/storage/blob_integrity.py b/polylogue/storage/blob_integrity.py index 108f248353..466d043270 100644 --- a/polylogue/storage/blob_integrity.py +++ b/polylogue/storage/blob_integrity.py @@ -1056,18 +1056,31 @@ def _missing_raw_backed_blob_rows(conn: sqlite3.Connection) -> list[dict[str, An blob_size_column = "blob_size" if _column_exists(conn, "raw_sessions", "blob_size") else "NULL" acquired_at_ms_column = "acquired_at_ms" if _column_exists(conn, "raw_sessions", "acquired_at_ms") else "NULL" file_mtime_ms_column = "file_mtime_ms" if _column_exists(conn, "raw_sessions", "file_mtime_ms") else "NULL" + has_container_coordinates = _table_exists(conn, "raw_container_coordinates") + coordinate_join = ( + "LEFT JOIN raw_container_coordinates coordinate ON coordinate.raw_id = raw_sessions.raw_id" + if has_container_coordinates + else "" + ) + coordinate_format_column = "coordinate.coordinate_format" if has_container_coordinates else "NULL" + entry_ordinal_column = "coordinate.entry_ordinal" if has_container_coordinates else "NULL" + split_index_column = "coordinate.split_index" if has_container_coordinates else "NULL" rows = conn.execute( f""" SELECT lower(hex(blob_hash)) AS blob_hash, - raw_id, + raw_sessions.raw_id AS raw_id, {origin_column} AS origin, {native_id_column} AS native_id, {source_path_column} AS source_path, {source_index_column} AS source_index, {blob_size_column} AS expected_size_bytes, {acquired_at_ms_column} AS acquired_at_ms, - {file_mtime_ms_column} AS file_mtime_ms + {file_mtime_ms_column} AS file_mtime_ms, + {coordinate_format_column} AS coordinate_format, + {entry_ordinal_column} AS entry_ordinal, + {split_index_column} AS split_index FROM raw_sessions + {coordinate_join} WHERE blob_hash IS NOT NULL ORDER BY origin, source_path, source_index, raw_id """ @@ -1122,6 +1135,7 @@ def _current_raw_payload_bytes( *, raw_id: str | None = None, blob_hash: str | None = None, + zip_coordinate: tuple[int, int] | None = None, source_bytes_cache: dict[str, bytes] | None = None, decoded_payload_cache: dict[str, object] | None = None, ) -> tuple[bytes | None, str | None]: @@ -1132,9 +1146,9 @@ def _current_raw_payload_bytes( zip_path, member = split if not zip_path.exists(): return None, "source_missing" - entry_ordinal: int | None = None - split_index = source_index - if raw_id is not None and blob_hash is not None and source_index is not None: + entry_ordinal: int | None = zip_coordinate[0] if zip_coordinate is not None else None + split_index = zip_coordinate[1] if zip_coordinate is not None else source_index + if zip_coordinate is None and raw_id is not None and blob_hash is not None and source_index is not None: coordinate = zip_member_identity_coordinate( raw_id=raw_id, source_path=source_path, @@ -1214,6 +1228,26 @@ def _current_raw_payload_bytes( return None, f"error:{exc}" +def _raw_zip_coordinate(row: dict[str, Any]) -> tuple[int, int] | None: + if row.get("coordinate_format") == "zip-v2": + entry_ordinal = row.get("entry_ordinal") + split_index = row.get("split_index") + if entry_ordinal is not None and split_index is not None: + return int(entry_ordinal), int(split_index) + source_path = _optional_str(row.get("source_path")) + source_index = row.get("source_index") + raw_id = str(row.get("raw_id") or "") + blob_hash = str(row.get("blob_hash") or "") + if not source_path or not _path_is_container_member(source_path) or source_index is None: + return None + return zip_member_identity_coordinate( + raw_id=raw_id, + source_path=source_path, + source_index=int(source_index), + blob_hash=blob_hash, + ) + + def _delete_blob_refs_for_raw_id(conn: sqlite3.Connection, raw_id: str) -> None: ref_id_column = "ref_id" if _column_exists(conn, "blob_refs", "ref_id") else "raw_id" conn.execute(f"DELETE FROM blob_refs WHERE {ref_id_column} = ?", (raw_id,)) @@ -1507,7 +1541,7 @@ def replace_raw_backed_blob_reference_debt_from_source( manifest_rows: list[dict[str, object]] = [] by_origin: Counter[str] = Counter() by_source_shape: Counter[str] = Counter() - candidate_updates: list[tuple[dict[str, Any], str, int, int | None, int]] = [] + candidate_updates: list[tuple[dict[str, Any], str, int, int | None, int, tuple[int, int] | None]] = [] skipped_existing_blob = 0 skipped_no_source_path = 0 skipped_source_missing = 0 @@ -1542,12 +1576,14 @@ def replace_raw_backed_blob_reference_debt_from_source( ) continue + zip_coordinate = _raw_zip_coordinate(row) try: payload_bytes, reason = _current_raw_payload_bytes( source_path, int(row["source_index"]) if row.get("source_index") is not None else None, raw_id=raw_id, blob_hash=old_blob_hash, + zip_coordinate=zip_coordinate, source_bytes_cache=source_bytes_cache, decoded_payload_cache=decoded_payload_cache, ) @@ -1603,7 +1639,7 @@ def replace_raw_backed_blob_reference_debt_from_source( "new_equals_old": new_blob_hash == old_blob_hash, } manifest_rows.append(manifest_row) - candidate_updates.append((row, new_blob_hash, new_blob_size, file_mtime_ms, acquired_at_ms)) + candidate_updates.append((row, new_blob_hash, new_blob_size, file_mtime_ms, acquired_at_ms, zip_coordinate)) if len(samples) < max(0, sample_size): samples.append( BlobReferenceSourceReplaceSample( @@ -1633,7 +1669,7 @@ def replace_raw_backed_blob_reference_debt_from_source( apply_decoded_payload_cache: dict[str, object] = {} publisher = ArchiveBlobPublisher(source_db, blob_store.root, store=blob_store) publication_receipts: list[str | None] = [] - for row, new_blob_hash, _new_blob_size, _file_mtime_ms, _acquired_at_ms in candidate_updates: + for row, new_blob_hash, _new_blob_size, _file_mtime_ms, _acquired_at_ms, zip_coordinate in candidate_updates: receipt_id: str | None = None existed_before_publication = blob_store.exists(new_blob_hash) source_path = str(row["source_path"]) @@ -1642,6 +1678,7 @@ def replace_raw_backed_blob_reference_debt_from_source( int(row["source_index"]) if row.get("source_index") is not None else None, raw_id=str(row["raw_id"]), blob_hash=str(row.get("blob_hash") or ""), + zip_coordinate=zip_coordinate, source_bytes_cache=apply_source_bytes_cache, decoded_payload_cache=apply_decoded_payload_cache, ) @@ -1662,12 +1699,27 @@ def replace_raw_backed_blob_reference_debt_from_source( new_blob_size, file_mtime_ms, acquired_at_ms, + zip_coordinate, ), publication_receipt_id in zip(candidate_updates, publication_receipts, strict=True): raw_id = str(row["raw_id"]) source_path = str(row["source_path"]) if not blob_store.exists(new_blob_hash): skipped_error += 1 continue + if zip_coordinate is not None and _table_exists(conn, "raw_container_coordinates"): + entry_ordinal, split_index = zip_coordinate + from polylogue.storage.sqlite.archive_tiers.source_write import ( + record_raw_container_coordinate, + ) + + record_raw_container_coordinate( + conn, + raw_id, + coordinate_format="zip-v2", + entry_ordinal=entry_ordinal, + split_index=split_index, + manage_transaction=False, + ) _delete_blob_refs_for_raw_id(conn, raw_id) _update_raw_session_blob_ref( conn, diff --git a/polylogue/storage/sqlite/archive_tiers/archive.py b/polylogue/storage/sqlite/archive_tiers/archive.py index 96ed8c1286..7180bb4914 100644 --- a/polylogue/storage/sqlite/archive_tiers/archive.py +++ b/polylogue/storage/sqlite/archive_tiers/archive.py @@ -242,6 +242,7 @@ deterministic_blob_hash, deterministic_raw_session_id, list_hook_events, + record_raw_container_coordinate, write_source_hook_event, ) from polylogue.storage.sqlite.archive_tiers.types import ( @@ -2461,6 +2462,23 @@ def write_raw_blob_ref( post_parse=post_parse, ) + def record_raw_container_coordinate( + self, + raw_id: str, + *, + coordinate_format: Literal["zip-v2"], + entry_ordinal: int, + split_index: int, + ) -> None: + self._require_writable("record source.db container coordinate") + record_raw_container_coordinate( + self._ensure_source_conn(), + raw_id, + coordinate_format=coordinate_format, + entry_ordinal=entry_ordinal, + split_index=split_index, + ) + def admit_raw_artifact_payload( self, *, diff --git a/polylogue/storage/sqlite/archive_tiers/source.py b/polylogue/storage/sqlite/archive_tiers/source.py index 6e15ca02d7..dccde7e983 100644 --- a/polylogue/storage/sqlite/archive_tiers/source.py +++ b/polylogue/storage/sqlite/archive_tiers/source.py @@ -61,6 +61,13 @@ ,detected_provider TEXT CHECK ({nullable_check("detected_provider", Provider)}) ) STRICT; +CREATE TABLE IF NOT EXISTS raw_container_coordinates ( + raw_id TEXT PRIMARY KEY REFERENCES raw_sessions(raw_id) ON DELETE CASCADE, + coordinate_format TEXT NOT NULL CHECK(coordinate_format = 'zip-v2'), + entry_ordinal INTEGER NOT NULL CHECK(entry_ordinal >= 0), + split_index INTEGER NOT NULL CHECK(split_index >= 0) +) STRICT; + CREATE INDEX IF NOT EXISTS idx_raw_sessions_origin ON raw_sessions(origin); diff --git a/polylogue/storage/sqlite/archive_tiers/source_write.py b/polylogue/storage/sqlite/archive_tiers/source_write.py index c087546809..6383596040 100644 --- a/polylogue/storage/sqlite/archive_tiers/source_write.py +++ b/polylogue/storage/sqlite/archive_tiers/source_write.py @@ -280,6 +280,40 @@ def record_capture_mode_observation( ) +def record_raw_container_coordinate( + conn: sqlite3.Connection, + raw_id: str, + *, + coordinate_format: Literal["zip-v2"], + entry_ordinal: int, + split_index: int, + manage_transaction: bool = True, +) -> None: + """Persist one content-independent container coordinate for a raw row.""" + if entry_ordinal < 0 or split_index < 0: + raise ValueError("container entry ordinal and split index must be non-negative") + with conn if manage_transaction else nullcontext(): + conn.execute( + """ + INSERT OR IGNORE INTO raw_container_coordinates ( + raw_id, coordinate_format, entry_ordinal, split_index + ) VALUES (?, ?, ?, ?) + """, + (raw_id, coordinate_format, entry_ordinal, split_index), + ) + stored = conn.execute( + """ + SELECT coordinate_format, entry_ordinal, split_index + FROM raw_container_coordinates + WHERE raw_id = ? + """, + (raw_id,), + ).fetchone() + expected = (coordinate_format, entry_ordinal, split_index) + if stored is None or tuple(stored) != expected: + raise ValueError(f"raw container coordinate changed for {raw_id}") + + def read_capture_mode_resolution(conn: sqlite3.Connection, raw_id: str) -> CaptureModeResolution: """Read every acquisition mode ever observed for ``raw_id``, explicitly ambiguous or not. @@ -1463,6 +1497,7 @@ def _enum_value(value: object) -> str | None: "read_raw_artifact", "read_archive_raw_session_envelope", "record_capture_mode_observation", + "record_raw_container_coordinate", "record_excised_blob_hash", "pending_raw_logical_source_key", "upsert_raw_artifact", diff --git a/polylogue/storage/sqlite/migrations/source/033.train.json b/polylogue/storage/sqlite/migrations/source/033.train.json index b6b23c8864..49e77188f6 100644 --- a/polylogue/storage/sqlite/migrations/source/033.train.json +++ b/polylogue/storage/sqlite/migrations/source/033.train.json @@ -12,7 +12,7 @@ "slot": 33, "path": "033_detected_raw_provider.sql", "owner_ref": "polylogue/storage/sqlite/migrations/source/033_detected_raw_provider.sql", - "sql_sha256": "0952a74b0a397a83f00ef45a618db1e084337cd1413409894b8811d234e9babc", + "sql_sha256": "8823c3de62eed0e93c7254222e24577442e146d6a76f8f1c810316b521ae6a81", "requires_backup": true }, "riders": [ @@ -47,6 +47,31 @@ ], "after_rider_ids": [], "trust_floor_exception_ref": null + }, + { + "rider_id": "rider:raw-container-coordinate", + "owner_ref": "github:pull/3952#discussion_r3779604611", + "schema_objects": ["table:raw_container_coordinates"], + "runtime_consumers": [ + { + "consumer_id": "live-zip-coordinate-write", + "production_ref": "polylogue.sources.live.batch:_record_zip_container_coordinate", + "behavior_proof_ref": "proof:source-v33:persist-zip-coordinate", + "roles": ["write"] + }, + { + "consumer_id": "raw-blob-source-replacement", + "production_ref": "polylogue.storage.blob_integrity:replace_raw_backed_blob_reference_debt_from_source", + "behavior_proof_ref": "proof:source-v33:reuse-zip-coordinate-after-replacement", + "roles": ["read", "write"] + } + ], + "behavior_proof_refs": [ + "proof:source-v33:persist-zip-coordinate", + "proof:source-v33:reuse-zip-coordinate-after-replacement" + ], + "after_rider_ids": [], + "trust_floor_exception_ref": null } ], "ordering_constraints": [], @@ -68,5 +93,5 @@ "released_at_ms": null, "release_evidence_ref": null, "proof_refs": [], - "manifest_sha256": "587200f83f1b9f9ed677412c95910592f8cb2dcbc51c1e3dbfc6098319b5587e" + "manifest_sha256": "7d5d512f5932e6d0a49bdaea48d7a54052116c180d0f24c4dfcbc84244f3381a" } diff --git a/polylogue/storage/sqlite/migrations/source/033_detected_raw_provider.sql b/polylogue/storage/sqlite/migrations/source/033_detected_raw_provider.sql index 256f4bd532..8c892914f8 100644 --- a/polylogue/storage/sqlite/migrations/source/033_detected_raw_provider.sql +++ b/polylogue/storage/sqlite/migrations/source/033_detected_raw_provider.sql @@ -8,3 +8,14 @@ ALTER TABLE raw_sessions ADD COLUMN detected_provider TEXT CHECK ( 'drive', 'unknown' ) OR detected_provider IS NULL) ); + +-- ZIP member raw ids bind the content hash while source_index stores a paired +-- central-directory ordinal and within-member split index. Source replacement +-- may legitimately change the row's blob hash without changing raw_id, so the +-- coordinate format must remain independently durable for later recovery. +CREATE TABLE raw_container_coordinates ( + raw_id TEXT PRIMARY KEY REFERENCES raw_sessions(raw_id) ON DELETE CASCADE, + coordinate_format TEXT NOT NULL CHECK(coordinate_format = 'zip-v2'), + entry_ordinal INTEGER NOT NULL CHECK(entry_ordinal >= 0), + split_index INTEGER NOT NULL CHECK(split_index >= 0) +) STRICT; diff --git a/tests/unit/core/test_schema_validation.py b/tests/unit/core/test_schema_validation.py index 275ba1aa62..f196b5690f 100644 --- a/tests/unit/core/test_schema_validation.py +++ b/tests/unit/core/test_schema_validation.py @@ -748,6 +748,53 @@ def validate(self, _sample: object) -> ValidationResult: assert stats.decode_errors == 0 +def test_verify_raw_corpus_filters_and_parses_by_detected_provider( + db_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """An UNKNOWN acquisition classified as Codex remains in the Codex corpus.""" + + class _AlwaysValidValidator: + provider = "codex" + + def validation_samples(self, payload: object, max_samples: int = 16) -> list[object]: + del max_samples + return [payload] + + def validate(self, _sample: object) -> ValidationResult: + return ValidationResult(is_valid=True) + + selected_providers: list[str] = [] + + def validator_for_payload(provider: str, *_args: object, **_kwargs: object) -> _AlwaysValidValidator: + selected_providers.append(provider) + return _AlwaysValidValidator() + + monkeypatch.setattr("polylogue.schemas.validation.corpus.SchemaValidator.for_payload", validator_for_payload) + raw_id = _insert_raw_record( + db_path=db_path, + raw_id="raw-unknown-codex", + source_name="unknown", + source_path="/tmp/learned-codex.jsonl", + raw_content=( + b'{"type":"session_meta","payload":{"id":"learned-codex"}}\n' + b'{"type":"response_item","payload":{"type":"message","role":"user",' + b'"content":[{"type":"input_text","text":"retained"}]}}\n' + ), + ) + with sqlite3.connect(db_path.parent / "source.db") as conn: + conn.execute("UPDATE raw_sessions SET detected_provider = 'codex' WHERE raw_id = ?", (raw_id,)) + + report = verify_raw_corpus( + db_path=db_path, + request=SchemaVerificationRequest(providers=["codex"], max_samples=16), + ) + + assert report.total_records == 1 + assert report.providers["codex"].valid_records == 1 + assert selected_providers == ["codex"] + + def test_verify_raw_corpus_counts_missing_schema_as_skipped(db_path: Path) -> None: _insert_raw_record( db_path=db_path, diff --git a/tests/unit/sources/test_live_batch_support.py b/tests/unit/sources/test_live_batch_support.py index 19ff2c06c9..13369a5936 100644 --- a/tests/unit/sources/test_live_batch_support.py +++ b/tests/unit/sources/test_live_batch_support.py @@ -754,6 +754,53 @@ def test_source_only_full_ingest_streams_admitted_zip_members_without_decoding( ] +def test_source_only_full_ingest_bounds_oversized_ndjson_sampling( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The production NDJSON route reaches streaming retention before eager decode.""" + from polylogue.core.degraded import DegradedReason, clear_degraded, set_degraded + + initialize_active_archive_root(tmp_path) + root = tmp_path / "inbox" + root.mkdir() + source = root / "oversized.ndjson" + payload = ( + json.dumps( + { + "type": "session_meta", + "payload": {"id": "oversized-record", "padding": "x" * 128_000}, + } + ).encode() + + b"\n" + ) + source.write_bytes(payload) + processor = LiveBatchProcessor( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=tmp_path / "index.db"))), + (WatchSource(name="inbox", root=root, suffixes=(".ndjson",)),), + cursor=CursorStore(tmp_path / "index.db"), + parser_fingerprint="test-parser", + ) + monkeypatch.setattr("polylogue.sources.live.batch._STREAMING_FULL_INGEST_BYTES", 1) + monkeypatch.setattr( + "polylogue.sources.live.batch_support.json_loads", + lambda _raw: (_ for _ in ()).throw(AssertionError("sampling must not decode an oversized physical record")), + ) + + set_degraded(DegradedReason(code="schema_version_mismatch", message="index unavailable", derived_only=True)) + try: + result = processor._ingest_full_paths_sync([source], source_name="inbox") + finally: + clear_degraded() + + assert result.succeeded == [source] + assert result.failed == [] + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT origin, blob_size, parsed_at_ms, parse_error FROM raw_sessions").fetchall() == [ + ("unknown-export", len(payload), None, None) + ] + + def test_source_only_zip_read_failure_remains_retryable_after_partial_copy( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -898,6 +945,14 @@ def test_source_only_zip_replay_resolves_unknown_chatgpt_member_and_keeps_duplic (f"{bundle}:first/conversations.json", 0, "unknown-export"), (f"{bundle}:second/conversations.json", 1, "unknown-export"), ] + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute( + "SELECT raw_id, coordinate_format, entry_ordinal, split_index " + "FROM raw_container_coordinates ORDER BY entry_ordinal" + ).fetchall() == [ + (before_replay[0][0], "zip-v2", 0, 0), + (before_replay[1][0], "zip-v2", 1, 0), + ] replay = backfill_historical_revision_evidence(tmp_path) @@ -909,6 +964,30 @@ def test_source_only_zip_replay_resolves_unknown_chatgpt_member_and_keeps_duplic ("unknown-export", "chatgpt"), ("unknown-export", "chatgpt"), ] + assert conn.execute( + "SELECT raw_id, coordinate_format, entry_ordinal, split_index " + "FROM raw_container_coordinates ORDER BY entry_ordinal" + ).fetchall() == [ + (before_replay[0][0], "zip-v2", 0, 0), + (before_replay[1][0], "zip-v2", 1, 0), + ] + + reobserved = processor._ingest_full_paths_sync([bundle], source_name="unknown") + + assert reobserved.succeeded == [bundle] + assert reobserved.failed == [] + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT origin, detected_provider FROM raw_sessions ORDER BY source_index").fetchall() == [ + ("unknown-export", "chatgpt"), + ("unknown-export", "chatgpt"), + ] + assert conn.execute( + "SELECT raw_id, coordinate_format, entry_ordinal, split_index " + "FROM raw_container_coordinates ORDER BY entry_ordinal" + ).fetchall() == [ + (before_replay[0][0], "zip-v2", 0, 0), + (before_replay[1][0], "zip-v2", 1, 0), + ] def test_zip_duplicate_member_coordinates_match_normal_and_source_only_routes(tmp_path: Path) -> None: diff --git a/tests/unit/storage/test_blob_integrity.py b/tests/unit/storage/test_blob_integrity.py index a906fe004f..5083039bc2 100644 --- a/tests/unit/storage/test_blob_integrity.py +++ b/tests/unit/storage/test_blob_integrity.py @@ -1023,69 +1023,44 @@ def fail_open(*args: object, **kwargs: object) -> object: def test_blob_recovery_uses_v2_entry_ordinal_without_consuming_split_index(tmp_path: Path) -> None: - """Durable live-ZIP identities reacquire the exact duplicate-name entry.""" + """ZIP coordinates survive one replacement and authorize the next recovery.""" + initialize_active_archive_root(tmp_path) source_db = tmp_path / "source.db" store = BlobStore(tmp_path / "blob") zip_source = tmp_path / "duplicate-v2.zip" member = "sessions/duplicate.json" - member_payloads = ( - b'[{"member":"first-zero"},{"member":"first-one"}]', - b'[{"member":"second-zero"},{"member":"second-one"}]', + old_selected_payloads = (b'{"member":"old-first-one"}', b'{"member":"old-second-one"}') + current_member_payloads = ( + b'[{"member":"current-first-zero"},{"member":"current-first-one"}]', + b'[{"member":"current-second-zero"},{"member":"current-second-one"}]', + ) + current_selected_payloads = ( + b'{"member":"current-first-one"}', + b'{"member":"current-second-one"}', ) - selected_payloads = (b'{"member":"first-one"}', b'{"member":"second-one"}') split_index = 1 with zipfile.ZipFile(zip_source, "w") as archive: - archive.writestr(member, member_payloads[0]) + archive.writestr(member, current_member_payloads[0]) with pytest.warns(UserWarning, match="Duplicate name"): - archive.writestr(member, member_payloads[1]) + archive.writestr(member, current_member_payloads[1]) source_path = f"{zip_source}:{member}" - hashes = tuple(hashlib.sha256(payload).hexdigest() for payload in selected_payloads) + old_hashes = tuple(hashlib.sha256(payload).hexdigest() for payload in old_selected_payloads) + current_hashes = tuple(hashlib.sha256(payload).hexdigest() for payload in current_selected_payloads) coordinates = tuple( zip_member_source_index(entry_ordinal=ordinal, split_index=split_index) - for ordinal in range(len(member_payloads)) + for ordinal in range(len(current_member_payloads)) ) raw_ids = tuple( zip_member_raw_id( source_path=source_path, entry_ordinal=ordinal, split_index=split_index, - blob_hash=hashes[ordinal], + blob_hash=old_hashes[ordinal], ) - for ordinal in range(len(member_payloads)) + for ordinal in range(len(current_member_payloads)) ) with sqlite3.connect(source_db) as conn: - conn.executescript( - """ - CREATE TABLE raw_sessions ( - raw_id TEXT PRIMARY KEY, - origin TEXT, - native_id TEXT, - source_path TEXT, - source_index INTEGER, - blob_hash BLOB, - blob_size INTEGER NOT NULL, - acquired_at_ms INTEGER, - file_mtime_ms INTEGER - ); - CREATE TABLE blob_refs ( - blob_hash BLOB NOT NULL, - ref_id TEXT NOT NULL, - ref_type TEXT NOT NULL, - source_path TEXT, - size_bytes INTEGER NOT NULL, - acquired_at_ms INTEGER NOT NULL, - PRIMARY KEY(blob_hash, ref_type, ref_id) - ); - CREATE TABLE blob_publication_reservations ( - publication_id TEXT PRIMARY KEY, - blob_hash BLOB NOT NULL, - size_bytes INTEGER NOT NULL, - publisher_id TEXT NOT NULL, - reserved_at_ms INTEGER NOT NULL - ); - """ - ) conn.executemany( """ INSERT INTO raw_sessions ( @@ -1096,7 +1071,7 @@ def test_blob_recovery_uses_v2_entry_ordinal_without_consuming_split_index(tmp_p [ (raw_id, source_path, source_index, bytes.fromhex(blob_hash), len(payload)) for raw_id, source_index, blob_hash, payload in zip( - raw_ids, coordinates, hashes, selected_payloads, strict=True + raw_ids, coordinates, old_hashes, old_selected_payloads, strict=True ) ], ) @@ -1107,25 +1082,49 @@ def test_blob_recovery_uses_v2_entry_ordinal_without_consuming_split_index(tmp_p """, [ (bytes.fromhex(blob_hash), raw_id, source_path, len(payload)) - for raw_id, blob_hash, payload in zip(raw_ids, hashes, selected_payloads, strict=True) + for raw_id, blob_hash, payload in zip(raw_ids, old_hashes, old_selected_payloads, strict=True) ], ) - report = replace_raw_backed_blob_reference_debt_from_source( + first = replace_raw_backed_blob_reference_debt_from_source( source_db, store=store, dry_run=False, - manifest_path=tmp_path / "duplicate-v2-replacement.jsonl", + manifest_path=tmp_path / "duplicate-v2-first-replacement.jsonl", ) - assert report.replaced_rows == 2 - assert report.written_blobs == 2 - assert all(store.exists(blob_hash) for blob_hash in hashes) - assert tuple(store.read_all(blob_hash) for blob_hash in hashes) == selected_payloads + assert first.replaced_rows == 2 + assert first.written_blobs == 2 + assert all(store.exists(blob_hash) for blob_hash in current_hashes) + with sqlite3.connect(source_db) as conn: + assert conn.execute( + "SELECT raw_id, lower(hex(blob_hash)), source_index FROM raw_sessions ORDER BY source_index" + ).fetchall() == list(zip(raw_ids, current_hashes, coordinates, strict=True)) + assert conn.execute( + "SELECT raw_id, coordinate_format, entry_ordinal, split_index " + "FROM raw_container_coordinates ORDER BY entry_ordinal" + ).fetchall() == [ + (raw_ids[0], "zip-v2", 0, split_index), + (raw_ids[1], "zip-v2", 1, split_index), + ] + + for blob_hash in current_hashes: + store.blob_path(blob_hash).unlink() + + second = replace_raw_backed_blob_reference_debt_from_source( + source_db, + store=store, + dry_run=False, + manifest_path=tmp_path / "duplicate-v2-second-replacement.jsonl", + ) + + assert second.replaced_rows == 2 + assert second.written_blobs == 2 + assert tuple(store.read_all(blob_hash) for blob_hash in current_hashes) == current_selected_payloads with sqlite3.connect(source_db) as conn: assert conn.execute( "SELECT raw_id, lower(hex(blob_hash)), source_index FROM raw_sessions ORDER BY source_index" - ).fetchall() == list(zip(raw_ids, hashes, coordinates, strict=True)) + ).fetchall() == list(zip(raw_ids, current_hashes, coordinates, strict=True)) def test_blob_recovery_rejects_oversized_container_member_before_open( From 118005dbe10b3778b73813b8c6c819e0344f0ed7 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 14 Aug 2026 02:27:36 +0200 Subject: [PATCH 56/65] fix: harden retained replay review paths Return directly after a generation-lease refusal, bound unknown retained JSONL detection, and keep terminal Codex state snapshots out of frozen session parsing. Ref #3952. --- polylogue/daemon/cli.py | 25 +++-- polylogue/sources/revision_backfill.py | 23 +++- tests/unit/daemon/test_raw_parse_recovery.py | 32 +++++- tests/unit/sources/test_live_watcher.py | 39 +++++++ tests/unit/sources/test_revision_backfill.py | 109 ++++++++++++++++++- 5 files changed, 211 insertions(+), 17 deletions(-) diff --git a/polylogue/daemon/cli.py b/polylogue/daemon/cli.py index 610ed4cf12..dceea1dca3 100644 --- a/polylogue/daemon/cli.py +++ b/polylogue/daemon/cli.py @@ -1331,26 +1331,29 @@ def _drain_raw_materialization_once( ) finally: _close_raw_materialization_fts(index_db, ops_db_path=config.archive_root / "ops.db") + if generation_pin_refused: + _emit_raw_materialization_pass(result) + if not result.success: + logger.warning("raw materialization: bounded convergence incomplete: %s", result.detail) + return _raw_materialization_counts(result) _emit_raw_materialization_pass(result) if not result.success: logger.warning("raw materialization: bounded convergence incomplete: %s", result.detail) + frontier_repaired = _converge_raw_authority_frontier(config, limit=min(limit, 8)) + return _raw_materialization_counts(result, executed_plans=frontier_repaired) + + +def _raw_materialization_counts(result: Any, *, executed_plans: int = 0) -> RawMaterializationCounts: + """Project one typed raw-materialization result into daemon scheduling counts.""" + from polylogue.product import raw_authority + metrics = dict(getattr(result, "metrics", {})) remaining = int(metrics.get("raw_materialization_remaining_candidate_count", 0)) if remaining == 0: remaining = int(metrics.get("raw_materialization_census_incomplete_raw_count", 0)) - if generation_pin_refused: - return raw_authority.RawMaterializationCounts( - repaired_sessions=result.repaired_count, - executed_plans=0, - remaining_candidates=remaining, - censused_components=int(metrics.get("raw_materialization_census_components_attempted", 0)), - candidate_count=int(metrics.get("raw_materialization_candidate_count", 0)), - pending_blob_bytes=int(metrics.get("raw_materialization_total_blob_bytes", 0)), - ) - frontier_repaired = _converge_raw_authority_frontier(config, limit=min(limit, 8)) return raw_authority.RawMaterializationCounts( repaired_sessions=result.repaired_count, - executed_plans=frontier_repaired, + executed_plans=executed_plans, remaining_candidates=remaining, censused_components=int(metrics.get("raw_materialization_census_components_attempted", 0)), candidate_count=int(metrics.get("raw_materialization_candidate_count", 0)), diff --git a/polylogue/sources/revision_backfill.py b/polylogue/sources/revision_backfill.py index 365ebcd6f9..0f6a435996 100644 --- a/polylogue/sources/revision_backfill.py +++ b/polylogue/sources/revision_backfill.py @@ -84,6 +84,7 @@ _LOGGER = _polylogue_logging.get_logger(__name__) _REPLAY_PROVIDER_DETECTION_PREFIX_BYTES: Final[int] = 8192 +_REPLAY_PROVIDER_DETECTION_MAX_SCAN_BYTES: Final[int] = 64 * 1024 _DOCUMENT_PROBE_ROOT_KEYS: Final[frozenset[str]] = frozenset( { @@ -521,7 +522,19 @@ def _detect_unknown_retained_provider( last_evidence = "no bounded JSONL record identified a provider; used fallback_provider" read_size = _REPLAY_PROVIDER_DETECTION_PREFIX_BYTES + 1 - while raw_line := payload.readline(read_size): + + scanned_bytes = 0 + + def read_bounded_line() -> bytes: + nonlocal scanned_bytes + remaining_bytes = _REPLAY_PROVIDER_DETECTION_MAX_SCAN_BYTES - scanned_bytes + if remaining_bytes <= 0: + return b"" + raw_line = payload.readline(min(read_size, remaining_bytes)) + scanned_bytes += len(raw_line) + return raw_line + + while raw_line := read_bounded_line(): has_newline = raw_line.endswith(b"\n") oversized = not has_newline and len(raw_line) > _REPLAY_PROVIDER_DETECTION_PREFIX_BYTES bounded_record = raw_line[:_REPLAY_PROVIDER_DETECTION_PREFIX_BYTES] @@ -532,7 +545,9 @@ def _detect_unknown_retained_provider( record_stream=True, ) while raw_line and not raw_line.endswith(b"\n"): - raw_line = payload.readline(read_size) + raw_line = read_bounded_line() + if not raw_line: + return Provider.UNKNOWN, "bounded JSONL provider scan exhausted; used fallback_provider" else: provider, last_evidence = detect_provider_from_raw_bytes_evidence( bounded_record, @@ -1394,8 +1409,8 @@ def _load_frozen_revision_evidence( ) frozen_codex_state_raw_ids = frozenset( raw_id - for raw_id, _source_index, terminal_non_session, _raw_rowid in rows - if not terminal_non_session and _retained_codex_state_descriptor(archive, raw_id) is not None + for raw_id, _source_index, _terminal_non_session, _raw_rowid in rows + if _retained_codex_state_descriptor(archive, raw_id) is not None ) recorded_logical_keys = require_current_parser_source_census( archive.archive_root, diff --git a/tests/unit/daemon/test_raw_parse_recovery.py b/tests/unit/daemon/test_raw_parse_recovery.py index be37836da6..ca13453932 100644 --- a/tests/unit/daemon/test_raw_parse_recovery.py +++ b/tests/unit/daemon/test_raw_parse_recovery.py @@ -28,13 +28,14 @@ import pytest -from polylogue.core.enums import Provider +from polylogue.core.enums import Provider, ValidationStatus from polylogue.core.errors import RawCASFrontierError from polylogue.core.raw_failure_evidence import RawFailureEvidenceKind from polylogue.daemon.convergence import DaemonConverger, StageState from polylogue.daemon.convergence_stages import make_raw_parse_recovery_stage from polylogue.sources.live.cursor import CursorStore from polylogue.storage.archive_identity import archive_file_set_root +from polylogue.storage.raw.models import RawSessionStateUpdate from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root @@ -351,6 +352,35 @@ def test_raw_parse_recovery_drains_previously_parsed_cas_frontier_failure(tmp_pa assert _sessions_for_raw(tmp_path, raw_id) == [("conv-stuck", raw_id)] +def test_raw_parse_recovery_uses_monotonic_parse_state_after_failed_validation(tmp_path: Path) -> None: + """The probe and repair route agree when a later parse supersedes validation.""" + initialize_active_archive_root(tmp_path) + path = tmp_path / "monotonic-validation-recovery.json" + raw_id = _write_stuck_raw(tmp_path, source_path=str(path)) + + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + archive.finalize_raw_parse_state( + raw_id, + state=RawSessionStateUpdate( + parsed_at="1970-01-01T00:00:00.001Z", + validation_status=ValidationStatus.FAILED, + validation_error="older validation failure", + ), + ) + archive.mark_raw_parse_failed( + raw_id, + provider=Provider.CHATGPT, + error=RawCASFrontierError("retry after the later parser state"), + ) + + stage = make_raw_parse_recovery_stage(tmp_path / "index.db") + + assert stage.check(path) is True + assert stage.execute(path) is True + assert stage.check(path) is False + assert _sessions_for_raw(tmp_path, raw_id) == [("conv-stuck", raw_id)] + + def test_raw_parse_recovery_skips_current_validation_failure_after_prior_parse(tmp_path: Path) -> None: """A current validation failure cannot leave CAS recovery permanently pending.""" initialize_active_archive_root(tmp_path) diff --git a/tests/unit/sources/test_live_watcher.py b/tests/unit/sources/test_live_watcher.py index 23d0aba0db..df38579db7 100644 --- a/tests/unit/sources/test_live_watcher.py +++ b/tests/unit/sources/test_live_watcher.py @@ -3956,6 +3956,45 @@ async def _drive() -> None: asyncio.run(_drive()) +def test_watcher_run_periodically_rediscovers_nested_root_created_after_start( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """The daemon route refreshes configured roots after its initial watch snapshot.""" + outer = tmp_path / "sources" + nested = outer / "late-codex" + outer.mkdir() + watcher, parse_sources = _make_watcher( + tmp_path, + outer, + sources=( + WatchSource(name="outer", root=outer, suffixes=(".jsonl",)), + WatchSource(name="nested", root=nested, suffixes=(".jsonl",)), + ), + ) + monkeypatch.setattr(live_watcher, "_PERIODIC_CATCH_UP_INTERVAL_S", 0.02) + + async def wait_for_stop(_roots: list[Path]) -> None: + await watcher._stop.wait() + + monkeypatch.setattr(watcher, "_watch_changes", wait_for_stop) + + async def _drive() -> None: + task = asyncio.create_task(watcher.run()) + await asyncio.wait_for(watcher.catch_up_complete.wait(), timeout=1.0) + nested.mkdir() + (nested / "missed.jsonl").write_text('{"type":"session_meta","payload":{"id":"late"}}\n') + for _ in range(60): + if parse_sources.await_count >= 1: + break + await asyncio.sleep(0.05) + watcher.stop() + await asyncio.wait_for(task, timeout=1.0) + assert parse_sources.await_count >= 1 + + asyncio.run(_drive()) + + def test_periodic_catch_up_backs_off_after_each_reconciliation_pass( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, diff --git a/tests/unit/sources/test_revision_backfill.py b/tests/unit/sources/test_revision_backfill.py index 2226c67ccc..91e7e50f8b 100644 --- a/tests/unit/sources/test_revision_backfill.py +++ b/tests/unit/sources/test_revision_backfill.py @@ -3,9 +3,11 @@ import json import sqlite3 import time +from collections.abc import Iterator +from contextlib import contextmanager from io import BytesIO from pathlib import Path -from typing import Any +from typing import Any, BinaryIO import ijson import pytest @@ -21,6 +23,7 @@ from polylogue.sources import revision_backfill from polylogue.sources.decoders import _iter_json_stream from polylogue.sources.dispatch import parse_payload +from polylogue.sources.parsers import codex_state from polylogue.sources.parsers.base import ParsedSession from polylogue.sources.revision_backfill import ( RawParsePrefetchCache, @@ -29,6 +32,7 @@ _parse_one, backfill_historical_revision_evidence, census_historical_revision_evidence, + validate_frozen_source_authority, ) from polylogue.storage.artifacts.inspection import inspect_raw_artifact from polylogue.storage.blob_publication import ArchiveBlobPublisher @@ -222,6 +226,109 @@ def reject_eager_material(_raw_id: str) -> tuple[Provider, bytes, str, RawRevisi assert [session.provider_session_id for session in sessions] == ["oversized-only-provider-record"] +def test_unknown_retained_jsonl_detection_caps_total_scan_before_typed_failure( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """An unidentifiable retained JSONL blob stops at the detection envelope.""" + initialize_active_archive_root(tmp_path) + payload = (b'{"opaque":"' + b"x" * 9_000 + b'"}\n') * 32 + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.UNKNOWN, + payload=payload, + source_path="opaque.jsonl", + acquired_at_ms=1, + ) + read_bytes = 0 + original_open = archive.open_raw_revision_material + + class CountingReader: + def __init__(self, wrapped: BinaryIO) -> None: + self._wrapped = wrapped + + def read(self, size: int = -1) -> bytes: + nonlocal read_bytes + chunk = self._wrapped.read(size) + read_bytes += len(chunk) + return chunk + + def readline(self, size: int = -1) -> bytes: + nonlocal read_bytes + chunk = self._wrapped.readline(size) + read_bytes += len(chunk) + return chunk + + @contextmanager + def tracked_open(requested_raw_id: str) -> Iterator[tuple[Provider, CountingReader, str, RawRevisionKind]]: + with original_open(requested_raw_id) as (provider, stream, source_path, kind): + yield provider, CountingReader(stream), source_path, kind + + monkeypatch.setattr(archive, "open_raw_revision_material", tracked_open) + monkeypatch.setattr( + archive, + "raw_revision_material", + lambda *_args, **_kwargs: pytest.fail("unidentified JSONL must not fall through to eager blob loading"), + ) + + with pytest.raises(ValueError, match="retained UNKNOWN provider remained unresolved"): + revision_backfill.parse_retained_raw_sessions(archive, raw_id) + + assert read_bytes <= revision_backfill._REPLAY_PROVIDER_DETECTION_MAX_SCAN_BYTES + + +def test_frozen_source_validation_treats_codex_state_as_non_session_evidence( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """Frozen validation must route Codex state SQLite past the JSON parser.""" + initialize_active_archive_root(tmp_path) + payload = _codex_thread_state_snapshot_bytes(tmp_path, "frozen state") + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + archive.write_raw_payload( + provider=Provider.CODEX, + payload=payload, + source_path=str(tmp_path / "codex" / "state_5.sqlite"), + acquired_at_ms=1, + ) + + census_historical_revision_evidence(tmp_path) + parsed_raw_ids: list[str] = [] + + def record_parse_dispatch(_archive: ArchiveStore, raw_ids: list[str], **_kwargs: object) -> dict[object, object]: + parsed_raw_ids.extend(raw_ids) + return {} + + monkeypatch.setattr(revision_backfill, "_parse_retained_raws", record_parse_dispatch) + + validate_frozen_source_authority(tmp_path) + assert parsed_raw_ids == [] + + +def test_frozen_codex_state_budget_blocks_before_sqlite_classification( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """Frozen-source validation rejects an oversized state snapshot before opening it.""" + initialize_active_archive_root(tmp_path) + payload = _codex_thread_state_snapshot_bytes(tmp_path, "frozen oversized state") + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.CODEX, + payload=payload, + source_path=str(tmp_path / "codex" / "state_5.sqlite"), + acquired_at_ms=1, + ) + + monkeypatch.setattr( + codex_state, + "classify_codex_sqlite_path", + lambda *_args, **_kwargs: pytest.fail("payload budget must block before Codex SQLite classification"), + ) + + with pytest.raises(revision_backfill.RawRevisionReplayResourceBlockedError) as blocked: + validate_frozen_source_authority(tmp_path, max_payload_bytes=1) + + assert blocked.value.raw_ids == (raw_id,) + + def test_unknown_retained_stream_census_worker_scans_past_oversized_first_record( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, From ec151aab942ea2f3ee16538c9a14635ac7b690c0 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 14 Aug 2026 02:56:41 +0200 Subject: [PATCH 57/65] fix: hold generation authority across raw recovery --- polylogue/daemon/cli.py | 31 ++++++++-------- polylogue/sources/revision_backfill.py | 7 ++++ tests/unit/daemon/test_daemon_cli.py | 38 ++++++++++++++++---- tests/unit/sources/test_revision_backfill.py | 4 +-- 4 files changed, 56 insertions(+), 24 deletions(-) diff --git a/polylogue/daemon/cli.py b/polylogue/daemon/cli.py index dceea1dca3..a65f1f1111 100644 --- a/polylogue/daemon/cli.py +++ b/polylogue/daemon/cli.py @@ -1294,22 +1294,8 @@ def _drain_raw_materialization_once( render_root=render_root(), sources=[], ) - if recover: - raw_authority.recover_interrupted_frontier(config) - # polylogue-d7im: a stale-plan blocker requires no operator judgment (it - # is a pure TOCTOU race between a census and its apply, already - # recomputed unattended in the crash-recovery path above) but, left - # unresolved, unresolved_raw_replay_blockers makes repair_materialization - # below fail closed for the WHOLE archive, not just the affected raw. - # Clear these automatically before every pass instead of waiting for a - # manual raw-authority-blocker-resolve invocation. - auto_resolved = raw_authority.auto_resolve_stale_plan_blockers(config) - if auto_resolved: - logger.info( - "raw authority: auto-resolved %d stale-plan blocker(s) before raw materialization", - auto_resolved, - ) generation_pin_refused = False + frontier_repaired = 0 with contextlib.ExitStack() as lease_stack: try: index_db = lease_stack.enter_context(raw_authority.materialization_generation_lease(config)) @@ -1320,6 +1306,19 @@ def _drain_raw_materialization_once( result = refused_result generation_pin_refused = True else: + if recover: + raw_authority.recover_interrupted_frontier(config) + # polylogue-d7im: a stale-plan blocker requires no operator + # judgment. Recovery, stale-plan resolution, repair, FTS closure, + # and frontier apply all consume the selected index generation, + # so the one promotion-excluding lease must cover the complete + # sequence rather than only the middle repair call. + auto_resolved = raw_authority.auto_resolve_stale_plan_blockers(config) + if auto_resolved: + logger.info( + "raw authority: auto-resolved %d stale-plan blocker(s) before raw materialization", + auto_resolved, + ) try: result = raw_authority.repair_materialization( config, @@ -1331,6 +1330,7 @@ def _drain_raw_materialization_once( ) finally: _close_raw_materialization_fts(index_db, ops_db_path=config.archive_root / "ops.db") + frontier_repaired = _converge_raw_authority_frontier(config, limit=min(limit, 8)) if generation_pin_refused: _emit_raw_materialization_pass(result) if not result.success: @@ -1339,7 +1339,6 @@ def _drain_raw_materialization_once( _emit_raw_materialization_pass(result) if not result.success: logger.warning("raw materialization: bounded convergence incomplete: %s", result.detail) - frontier_repaired = _converge_raw_authority_frontier(config, limit=min(limit, 8)) return _raw_materialization_counts(result, executed_plans=frontier_repaired) diff --git a/polylogue/sources/revision_backfill.py b/polylogue/sources/revision_backfill.py index 0f6a435996..40128750a2 100644 --- a/polylogue/sources/revision_backfill.py +++ b/polylogue/sources/revision_backfill.py @@ -544,6 +544,13 @@ def read_bounded_line() -> bytes: stream_name, record_stream=True, ) + # Provider authority comes from the bounded structural prefix; + # draining the rest of an oversized physical record is needed + # only when that prefix was inconclusive. In particular, do not + # replace positive evidence with UNKNOWN merely because the + # record itself extends beyond the total scan envelope. + if provider is not Provider.UNKNOWN: + return provider, last_evidence while raw_line and not raw_line.endswith(b"\n"): raw_line = read_bounded_line() if not raw_line: diff --git a/tests/unit/daemon/test_daemon_cli.py b/tests/unit/daemon/test_daemon_cli.py index d79ceb32cc..17d9ca8081 100644 --- a/tests/unit/daemon/test_daemon_cli.py +++ b/tests/unit/daemon/test_daemon_cli.py @@ -1134,12 +1134,29 @@ class FakeRestoreResult: "polylogue.storage.blob_integrity.restore_direct_blob_reference_debt", lambda *_args, **_kwargs: FakeRestoreResult(), ) - monkeypatch.setattr("polylogue.product.raw_authority.recover_interrupted_frontier", lambda _config: ()) - monkeypatch.setattr("polylogue.product.raw_authority.auto_resolve_stale_plan_blockers", lambda _config: 0) + + def recover_frontier(_config: Config) -> tuple[()]: + assert held == 1 + lease_events.append("recover") + return () + + def resolve_stale(_config: Config) -> int: + assert held == 1 + lease_events.append("stale") + return 0 + + monkeypatch.setattr("polylogue.product.raw_authority.recover_interrupted_frontier", recover_frontier) + monkeypatch.setattr("polylogue.product.raw_authority.auto_resolve_stale_plan_blockers", resolve_stale) monkeypatch.setattr("polylogue.product.raw_authority.repair_materialization", lambda *_args, **_kwargs: result) monkeypatch.setattr("polylogue.product.raw_authority.materialization_generation_lease", fake_generation_lease) monkeypatch.setattr(daemon_cli, "_emit_raw_materialization_pass", lambda _result: None) - monkeypatch.setattr(daemon_cli, "_converge_raw_authority_frontier", lambda _config, **_kwargs: 0) + + def converge_frontier(_config: Config, **_kwargs: object) -> int: + assert held == 1 + lease_events.append("frontier") + return 0 + + monkeypatch.setattr(daemon_cli, "_converge_raw_authority_frontier", converge_frontier) def close_fts(index_db: Path, *, ops_db_path: Path) -> None: assert held == 1 @@ -1157,7 +1174,10 @@ def close_fts(index_db: Path, *, ops_db_path: Path) -> None: assert daemon_cli._drain_raw_materialization_once().repaired_sessions == 1 assert closed == [(active_index, archive / "ops.db")] - assert lease_events == ["acquire", "fts", "close"] + expected_events = ( + ["acquire", "fts", "close"] if whale else ["acquire", "recover", "stale", "fts", "frontier", "close"] + ) + assert lease_events == expected_events assert held == 0 @@ -1192,8 +1212,14 @@ def reject_repair(*_args: object, **_kwargs: object) -> None: "polylogue.storage.blob_integrity.restore_direct_blob_reference_debt", lambda *_args, **_kwargs: FakeRestoreResult(), ) - monkeypatch.setattr("polylogue.product.raw_authority.recover_interrupted_frontier", lambda _config: ()) - monkeypatch.setattr("polylogue.product.raw_authority.auto_resolve_stale_plan_blockers", lambda _config: 0) + monkeypatch.setattr( + "polylogue.product.raw_authority.recover_interrupted_frontier", + lambda _config: pytest.fail("frontier recovery requires an acquired generation pin"), + ) + monkeypatch.setattr( + "polylogue.product.raw_authority.auto_resolve_stale_plan_blockers", + lambda _config: pytest.fail("stale-plan recovery requires an acquired generation pin"), + ) monkeypatch.setattr("polylogue.product.raw_authority.repair_materialization", reject_repair) monkeypatch.setattr(ActiveWriterLease, "acquire", refuse_outer_lease) monkeypatch.setattr(daemon_cli, "_emit_raw_materialization_pass", emitted.append) diff --git a/tests/unit/sources/test_revision_backfill.py b/tests/unit/sources/test_revision_backfill.py index 91e7e50f8b..23282ca9f6 100644 --- a/tests/unit/sources/test_revision_backfill.py +++ b/tests/unit/sources/test_revision_backfill.py @@ -196,7 +196,7 @@ def reject_eager_material(_raw_id: str) -> tuple[Provider, bytes, str, RawRevisi def test_unknown_retained_oversized_provider_record_never_uses_eager_payload( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: - """The only provider-defining record may itself exceed the scan bound.""" + """Positive prefix evidence survives a record larger than the total scan cap.""" initialize_active_archive_root(tmp_path) payload = ( json.dumps( @@ -204,7 +204,7 @@ def test_unknown_retained_oversized_provider_record_never_uses_eager_payload( "sessionId": "oversized-only-provider-record", "uuid": "message-1", "type": "user", - "message": {"role": "user", "content": [{"type": "text", "text": "x" * 9_000}]}, + "message": {"role": "user", "content": [{"type": "text", "text": "x" * 80_000}]}, } ).encode() + b"\n" From 76533937a9890bcbe8e42c502679c6ef43a10aef Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 14 Aug 2026 03:10:33 +0200 Subject: [PATCH 58/65] fix: prevent raw writes before generation lease --- polylogue/daemon/cli.py | 23 +++++++++++------------ tests/unit/daemon/test_daemon_cli.py | 17 +++++++++++------ 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/polylogue/daemon/cli.py b/polylogue/daemon/cli.py index a65f1f1111..fb5d94c393 100644 --- a/polylogue/daemon/cli.py +++ b/polylogue/daemon/cli.py @@ -1277,18 +1277,6 @@ def _drain_raw_materialization_once( from polylogue.storage.blob_integrity import restore_direct_blob_reference_debt archive = archive_root() - restored = restore_direct_blob_reference_debt( - archive / "source.db", - dry_run=False, - max_count=_BLOB_REFERENCE_RESTORE_CONVERGENCE_BATCH_LIMIT, - sample_size=0, - ) - if restored.restored_count: - logger.info( - "blob references: restored %d direct source blob(s) before raw materialization", - restored.restored_count, - ) - config = Config( archive_root=archive, render_root=render_root(), @@ -1306,6 +1294,17 @@ def _drain_raw_materialization_once( result = refused_result generation_pin_refused = True else: + restored = restore_direct_blob_reference_debt( + archive / "source.db", + dry_run=False, + max_count=_BLOB_REFERENCE_RESTORE_CONVERGENCE_BATCH_LIMIT, + sample_size=0, + ) + if restored.restored_count: + logger.info( + "blob references: restored %d direct source blob(s) before raw materialization", + restored.restored_count, + ) if recover: raw_authority.recover_interrupted_frontier(config) # polylogue-d7im: a stale-plan blocker requires no operator diff --git a/tests/unit/daemon/test_daemon_cli.py b/tests/unit/daemon/test_daemon_cli.py index 17d9ca8081..24e29fa0af 100644 --- a/tests/unit/daemon/test_daemon_cli.py +++ b/tests/unit/daemon/test_daemon_cli.py @@ -1119,6 +1119,11 @@ def fake_generation_lease(_config: Config) -> Any: class FakeRestoreResult: restored_count = 0 + def restore_debt(*_args: object, **_kwargs: object) -> FakeRestoreResult: + assert held == 1 + lease_events.append("restore") + return FakeRestoreResult() + result = SimpleNamespace( success=True, repaired_count=1, @@ -1132,7 +1137,7 @@ class FakeRestoreResult: monkeypatch.setattr("polylogue.readiness.capability.raw_frontier_source_selection_block_reason", lambda _root: None) monkeypatch.setattr( "polylogue.storage.blob_integrity.restore_direct_blob_reference_debt", - lambda *_args, **_kwargs: FakeRestoreResult(), + restore_debt, ) def recover_frontier(_config: Config) -> tuple[()]: @@ -1175,7 +1180,7 @@ def close_fts(index_db: Path, *, ops_db_path: Path) -> None: assert closed == [(active_index, archive / "ops.db")] expected_events = ( - ["acquire", "fts", "close"] if whale else ["acquire", "recover", "stale", "fts", "frontier", "close"] + ["acquire", "fts", "close"] if whale else ["acquire", "restore", "recover", "stale", "fts", "frontier", "close"] ) assert lease_events == expected_events assert held == 0 @@ -1196,12 +1201,12 @@ def test_raw_materialization_outer_lease_refusal_preserves_typed_result( archive.mkdir() emitted: list[RepairResult] = [] - class FakeRestoreResult: - restored_count = 0 - def refuse_outer_lease(_lease: ActiveWriterLease) -> None: raise RebuildLeaseUnavailableError("offline rebuild is active") + def reject_restore(*_args: object, **_kwargs: object) -> None: + raise AssertionError("blob-reference restoration requires an acquired generation pin") + def reject_repair(*_args: object, **_kwargs: object) -> None: raise AssertionError("repair must not run when the outer generation pin is refused") @@ -1210,7 +1215,7 @@ def reject_repair(*_args: object, **_kwargs: object) -> None: monkeypatch.setattr("polylogue.readiness.capability.raw_frontier_source_selection_block_reason", lambda _root: None) monkeypatch.setattr( "polylogue.storage.blob_integrity.restore_direct_blob_reference_debt", - lambda *_args, **_kwargs: FakeRestoreResult(), + reject_restore, ) monkeypatch.setattr( "polylogue.product.raw_authority.recover_interrupted_frontier", From c0cd770799f04da8ac392f05cfeba9ce6c55f7c6 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 14 Aug 2026 03:23:15 +0200 Subject: [PATCH 59/65] fix: hold rebuild exclusion for daemon lifetime --- polylogue/daemon/cli.py | 60 ++++++++++++++++++++++++++++ tests/unit/daemon/test_daemon_cli.py | 46 +++++++++++++++++++++ 2 files changed, 106 insertions(+) diff --git a/polylogue/daemon/cli.py b/polylogue/daemon/cli.py index fb5d94c393..fa1e72d084 100644 --- a/polylogue/daemon/cli.py +++ b/polylogue/daemon/cli.py @@ -2144,6 +2144,66 @@ async def run_daemon_services( api_port: int = 8766, api_auth_token: str | None = None, api_allow_no_auth: bool = False, +) -> None: + """Run the daemon while excluding every offline index rebuild. + + The lease is intentionally process-lifetime authority rather than a + per-maintenance-call guard. Startup readiness, reservation recovery, + live acquisition, and periodic convergence all mutate source or index + state; an offline rebuild must therefore refuse the daemon before any of + those routes can run, and the daemon must prevent a rebuild from starting + until its writer coordinator has drained. + """ + from polylogue.paths import archive_root + from polylogue.storage.index_generation import ActiveWriterLease + + archive_root_path = Path(archive_root()) + archive_root_path.mkdir(mode=0o700, parents=True, exist_ok=True) + active_writer_lease = ActiveWriterLease(archive_root_path) + active_writer_lease.acquire() + try: + await _run_daemon_services_under_active_writer_lease( + sources=sources, + debounce_s=debounce_s, + enable_watch=enable_watch, + enable_source_catchup=enable_source_catchup, + enable_browser_capture=enable_browser_capture, + browser_capture_host=browser_capture_host, + browser_capture_port=browser_capture_port, + browser_capture_spool_path=browser_capture_spool_path, + browser_capture_allow_remote=browser_capture_allow_remote, + browser_capture_auth_token=browser_capture_auth_token, + browser_capture_allow_no_auth=browser_capture_allow_no_auth, + browser_capture_extra_origins=browser_capture_extra_origins, + enable_api=enable_api, + api_host=api_host, + api_port=api_port, + api_auth_token=api_auth_token, + api_allow_no_auth=api_allow_no_auth, + ) + finally: + active_writer_lease.close() + + +async def _run_daemon_services_under_active_writer_lease( + *, + sources: tuple[WatchSource, ...], + debounce_s: float, + enable_watch: bool, + enable_source_catchup: bool = True, + enable_browser_capture: bool, + browser_capture_host: str, + browser_capture_port: int, + browser_capture_spool_path: Path | None, + browser_capture_allow_remote: bool = False, + browser_capture_auth_token: str | None = None, + browser_capture_allow_no_auth: bool = False, + browser_capture_extra_origins: tuple[str, ...] = (), + enable_api: bool = False, + api_host: str = "127.0.0.1", + api_port: int = 8766, + api_auth_token: str | None = None, + api_allow_no_auth: bool = False, ) -> None: """Run configured daemon components until interrupted.""" from polylogue.daemon import process_start as _process_start diff --git a/tests/unit/daemon/test_daemon_cli.py b/tests/unit/daemon/test_daemon_cli.py index 24e29fa0af..e4747e0291 100644 --- a/tests/unit/daemon/test_daemon_cli.py +++ b/tests/unit/daemon/test_daemon_cli.py @@ -3556,6 +3556,52 @@ def test_reconcile_blob_publications_clears_terminal_receipts_at_startup( assert conn.execute("SELECT COUNT(*) FROM blob_publication_reservations").fetchone()[0] == 0 +def test_daemon_rebuild_lease_refusal_precedes_startup_blob_reconciliation( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """An offline rebuild must refuse the daemon before durable startup writes.""" + from polylogue.daemon import cli as daemon_cli + from polylogue.storage.blob_publication import ArchiveBlobPublisher + from polylogue.storage.blob_store import BlobStore + from polylogue.storage.index_generation import RebuildLease, RebuildLeaseUnavailableError + from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root + + archive_root_path = tmp_path / "archive" + initialize_active_archive_root(archive_root_path) + monkeypatch.setenv("POLYLOGUE_ARCHIVE_ROOT", str(archive_root_path)) + + source_db = archive_root_path / "source.db" + publisher = ArchiveBlobPublisher(source_db, BlobStore(archive_root_path / "blob").root) + publisher.write_from_bytes(b"startup-rebuild-refusal") + publisher.flush() + with sqlite3.connect(source_db) as conn: + assert conn.execute("SELECT COUNT(*) FROM blob_publication_reservations").fetchone()[0] == 1 + + with ( + RebuildLease(archive_root_path), + pytest.raises( + RebuildLeaseUnavailableError, + match="offline index rebuild owns archive", + ), + ): + asyncio.run( + daemon_cli.run_daemon_services( + sources=(), + debounce_s=1.0, + enable_watch=False, + enable_browser_capture=False, + browser_capture_host="127.0.0.1", + browser_capture_port=8765, + browser_capture_spool_path=None, + ) + ) + + with sqlite3.connect(source_db) as conn: + assert conn.execute("SELECT COUNT(*) FROM blob_publication_reservations").fetchone()[0] == 1 + assert not (archive_root_path / "daemon.pid").exists() + + def test_run_daemon_services_stops_live_watcher_on_failure() -> None: from polylogue.daemon import cli as daemon_cli From 4984774bcc3df941c79578d4ac6f0122e02e7663 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 14 Aug 2026 03:28:41 +0200 Subject: [PATCH 60/65] refactor: route daemon lease through product boundary --- polylogue/daemon/cli.py | 8 ++------ polylogue/product/raw_authority.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/polylogue/daemon/cli.py b/polylogue/daemon/cli.py index fa1e72d084..099af2dee3 100644 --- a/polylogue/daemon/cli.py +++ b/polylogue/daemon/cli.py @@ -2155,13 +2155,11 @@ async def run_daemon_services( until its writer coordinator has drained. """ from polylogue.paths import archive_root - from polylogue.storage.index_generation import ActiveWriterLease + from polylogue.product.raw_authority import archive_writer_rebuild_exclusion archive_root_path = Path(archive_root()) archive_root_path.mkdir(mode=0o700, parents=True, exist_ok=True) - active_writer_lease = ActiveWriterLease(archive_root_path) - active_writer_lease.acquire() - try: + with archive_writer_rebuild_exclusion(archive_root_path): await _run_daemon_services_under_active_writer_lease( sources=sources, debounce_s=debounce_s, @@ -2181,8 +2179,6 @@ async def run_daemon_services( api_auth_token=api_auth_token, api_allow_no_auth=api_allow_no_auth, ) - finally: - active_writer_lease.close() async def _run_daemon_services_under_active_writer_lease( diff --git a/polylogue/product/raw_authority.py b/polylogue/product/raw_authority.py index 64bf438de3..dbe844ad21 100644 --- a/polylogue/product/raw_authority.py +++ b/polylogue/product/raw_authority.py @@ -135,6 +135,19 @@ def materialization_generation_lease(config: Config) -> Iterator[Path]: lease.close() +@contextlib.contextmanager +def archive_writer_rebuild_exclusion(archive_root: Path) -> Iterator[None]: + """Exclude an offline rebuild for the complete lifetime of an archive writer.""" + from polylogue.storage.index_generation import ActiveWriterLease + + lease = ActiveWriterLease(archive_root) + lease.acquire() + try: + yield + finally: + lease.close() + + def materialization_lease_refusal_result(error: BaseException) -> RepairResult | None: """Translate only a rebuild-lease refusal into raw repair's typed result.""" from polylogue.storage.index_generation import RebuildLeaseUnavailableError @@ -247,6 +260,7 @@ def list_blockers(archive_root: Path, *, limit: int = 100, offset: int = 0) -> J __all__ = [ "RawMaterializationCounts", "apply_frontier", + "archive_writer_rebuild_exclusion", "inspect_frontier", "list_blockers", "materialization_generation_lease", From 7e58dd94e0c5dae005ad6777ba9e3845af239464 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 14 Aug 2026 03:44:08 +0200 Subject: [PATCH 61/65] fix: retain rebuild exclusion through writer drain --- polylogue/daemon/cli.py | 66 +++++++++++++---- polylogue/product/raw_authority.py | 45 ++++++++++-- tests/unit/daemon/test_daemon_cli.py | 106 +++++++++++++++++++++++---- 3 files changed, 179 insertions(+), 38 deletions(-) diff --git a/polylogue/daemon/cli.py b/polylogue/daemon/cli.py index 099af2dee3..be0053bd8f 100644 --- a/polylogue/daemon/cli.py +++ b/polylogue/daemon/cli.py @@ -79,7 +79,7 @@ from polylogue.config import Config from polylogue.daemon.lifecycle import DaemonLifecycle from polylogue.daemon.parse_prefetch import DaemonParseStage - from polylogue.product.raw_authority import RawMaterializationCounts + from polylogue.product.raw_authority import ArchiveWriterRebuildExclusion, RawMaterializationCounts from polylogue.sources.revision_backfill import RawParsePrefetchCache from polylogue.storage.blob_publication import BlobPublicationReconciliation @@ -2103,26 +2103,50 @@ async def _emit_daemon_lifecycle_event( logger.warning("daemon: failed to emit lifecycle event %s", phase, exc_info=True) +def _retain_rebuild_exclusion_for_undrained_writer( + rebuild_exclusion: ArchiveWriterRebuildExclusion, + *, + writer_drained: bool, +) -> None: + """Transfer rebuild exclusion to process lifetime after a drain timeout.""" + if not writer_drained: + rebuild_exclusion.retain_until_process_exit() + + async def run_live_watcher( *, sources: tuple[WatchSource, ...], debounce_s: float, ) -> None: from polylogue.daemon.events import emit_catch_up_cycle + from polylogue.paths import archive_root + from polylogue.product.raw_authority import archive_writer_rebuild_exclusion - async with Polylogue() as polylogue: - watcher = LiveWatcher( - polylogue, - sources, - debounce_s=debounce_s, - event_emitter=_emit_live_batch_event, - catch_up_event_emitter=emit_catch_up_cycle, - write_coordinator=daemon_write_coordinator(), - ) + archive_root_path = Path(archive_root()) + archive_root_path.mkdir(mode=0o700, parents=True, exist_ok=True) + with archive_writer_rebuild_exclusion(archive_root_path) as rebuild_exclusion: + coordinator = daemon_write_coordinator() + watcher: LiveWatcher | None = None try: - await watcher.run() - except KeyboardInterrupt: - watcher.stop() + async with Polylogue() as polylogue: + watcher = LiveWatcher( + polylogue, + sources, + debounce_s=debounce_s, + event_emitter=_emit_live_batch_event, + catch_up_event_emitter=emit_catch_up_cycle, + write_coordinator=coordinator, + ) + with contextlib.suppress(KeyboardInterrupt): + await watcher.run() + finally: + if watcher is not None: + watcher.stop() + writer_drained = await coordinator.shutdown(timeout=5.0) + _retain_rebuild_exclusion_for_undrained_writer( + rebuild_exclusion, + writer_drained=writer_drained, + ) async def run_daemon_services( @@ -2159,8 +2183,9 @@ async def run_daemon_services( archive_root_path = Path(archive_root()) archive_root_path.mkdir(mode=0o700, parents=True, exist_ok=True) - with archive_writer_rebuild_exclusion(archive_root_path): + with archive_writer_rebuild_exclusion(archive_root_path) as rebuild_exclusion: await _run_daemon_services_under_active_writer_lease( + rebuild_exclusion=rebuild_exclusion, sources=sources, debounce_s=debounce_s, enable_watch=enable_watch, @@ -2183,6 +2208,7 @@ async def run_daemon_services( async def _run_daemon_services_under_active_writer_lease( *, + rebuild_exclusion: ArchiveWriterRebuildExclusion, sources: tuple[WatchSource, ...], debounce_s: float, enable_watch: bool, @@ -2408,6 +2434,10 @@ async def _run_daemon_services_under_active_writer_lease( with contextlib.suppress(Exception): await write_coordinator.run_sync("daemon.lifecycle.stop", lifecycle.stop, exit_kind="error") writer_drained = await write_coordinator.shutdown(timeout=5.0) + _retain_rebuild_exclusion_for_undrained_writer( + rebuild_exclusion, + writer_drained=writer_drained, + ) _release_pidfile_after_writer_drain(pidfile_fd, writer_drained=writer_drained) if writer_drained: archive_owner.release() @@ -2443,6 +2473,10 @@ async def _run_daemon_services_under_active_writer_lease( with contextlib.suppress(Exception): await write_coordinator.run_sync("daemon.lifecycle.stop", lifecycle.stop, exit_kind="error") writer_drained = await write_coordinator.shutdown(timeout=5.0) + _retain_rebuild_exclusion_for_undrained_writer( + rebuild_exclusion, + writer_drained=writer_drained, + ) _release_pidfile_after_writer_drain(pidfile_fd, writer_drained=writer_drained) if writer_drained: archive_owner.release() @@ -2804,6 +2838,10 @@ async def _run_daemon_services_under_active_writer_lease( logger.warning("daemon: could not persist final lifecycle stop", exc_info=True) writer_drained = await write_coordinator.shutdown(timeout=5.0) + _retain_rebuild_exclusion_for_undrained_writer( + rebuild_exclusion, + writer_drained=writer_drained, + ) pidfile_fd = _release_pidfile_after_writer_drain(pidfile_fd, writer_drained=writer_drained) finally: if server is not None: diff --git a/polylogue/product/raw_authority.py b/polylogue/product/raw_authority.py index dbe844ad21..cae001979a 100644 --- a/polylogue/product/raw_authority.py +++ b/polylogue/product/raw_authority.py @@ -135,17 +135,45 @@ def materialization_generation_lease(config: Config) -> Iterator[Path]: lease.close() -@contextlib.contextmanager -def archive_writer_rebuild_exclusion(archive_root: Path) -> Iterator[None]: - """Exclude an offline rebuild for the complete lifetime of an archive writer.""" - from polylogue.storage.index_generation import ActiveWriterLease +class ArchiveWriterRebuildExclusion: + """Product authority preventing an offline rebuild from overlapping a writer.""" - lease = ActiveWriterLease(archive_root) - lease.acquire() + def __init__(self, archive_root: Path) -> None: + from polylogue.storage.index_generation import ActiveWriterLease + + self._lease = ActiveWriterLease(archive_root) + self._retained_until_process_exit = False + self._lease.acquire() + + def retain_until_process_exit(self) -> None: + """Keep exclusion when a writer cannot be proven drained. + + The raw file descriptor deliberately remains open and is reclaimed by + the OS at process exit. Releasing it after a bounded shutdown timeout + would let an offline rebuild overlap the admitted writer that caused + that timeout. + """ + self._retained_until_process_exit = True + + def release(self) -> None: + """Release exclusion after every admitted writer is proven drained.""" + self._lease.close() + self._retained_until_process_exit = False + + def release_if_safe(self) -> None: + """Release unless shutdown transferred authority to process lifetime.""" + if not self._retained_until_process_exit: + self.release() + + +@contextlib.contextmanager +def archive_writer_rebuild_exclusion(archive_root: Path) -> Iterator[ArchiveWriterRebuildExclusion]: + """Acquire process-lifetime-capable rebuild exclusion for an archive writer.""" + exclusion = ArchiveWriterRebuildExclusion(archive_root) try: - yield + yield exclusion finally: - lease.close() + exclusion.release_if_safe() def materialization_lease_refusal_result(error: BaseException) -> RepairResult | None: @@ -258,6 +286,7 @@ def list_blockers(archive_root: Path, *, limit: int = 100, offset: int = 0) -> J __all__ = [ + "ArchiveWriterRebuildExclusion", "RawMaterializationCounts", "apply_frontier", "archive_writer_rebuild_exclusion", diff --git a/tests/unit/daemon/test_daemon_cli.py b/tests/unit/daemon/test_daemon_cli.py index e4747e0291..b46997133b 100644 --- a/tests/unit/daemon/test_daemon_cli.py +++ b/tests/unit/daemon/test_daemon_cli.py @@ -3,6 +3,7 @@ import asyncio import contextlib import functools +import hashlib import inspect import os import sqlite3 @@ -2421,7 +2422,10 @@ def test_explicit_browser_capture_root_uses_spool_override_classifier(tmp_path: ) -def test_run_live_watcher_stops_on_keyboard_interrupt() -> None: +def test_run_live_watcher_stops_on_keyboard_interrupt( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: from polylogue.daemon import cli as daemon_cli class FakePolylogue: @@ -2432,6 +2436,12 @@ async def __aexit__(self, *exc: object) -> None: return None stopped: list[bool] = [] + shutdown_timeouts: list[float] = [] + + class Coordinator: + async def shutdown(self, *, timeout: float) -> bool: + shutdown_timeouts.append(timeout) + return True class FakeWatcher: stopped = False @@ -2447,14 +2457,43 @@ def stop(self) -> None: stopped.append(self.stopped) sources = (WatchSource(name="codex", root=Path("/tmp/codex")),) + monkeypatch.setattr("polylogue.paths.archive_root", lambda: tmp_path / "archive") with ( patch.object(daemon_cli, "Polylogue", FakePolylogue), patch.object(daemon_cli, "LiveWatcher", FakeWatcher), + patch.object(daemon_cli, "daemon_write_coordinator", return_value=Coordinator()), ): asyncio.run(daemon_cli.run_live_watcher(sources=sources, debounce_s=1.0)) assert stopped == [True] + assert shutdown_timeouts == [5.0] + + +def test_run_live_watcher_refuses_before_entry_while_rebuild_lease_is_held( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + from polylogue.daemon import cli as daemon_cli + from polylogue.storage.index_generation import RebuildLease, RebuildLeaseUnavailableError + + archive_root_path = tmp_path / "archive" + archive_root_path.mkdir() + monkeypatch.setattr("polylogue.paths.archive_root", lambda: archive_root_path) + + class ForbiddenPolylogue: + def __init__(self) -> None: + raise AssertionError("standalone watcher entered archive before rebuild refusal") + + monkeypatch.setattr(daemon_cli, "Polylogue", ForbiddenPolylogue) + with ( + RebuildLease(archive_root_path), + pytest.raises( + RebuildLeaseUnavailableError, + match="offline index rebuild owns archive", + ), + ): + asyncio.run(daemon_cli.run_live_watcher(sources=(), debounce_s=1.0)) def test_ensure_fts_startup_readiness_skips_old_non_blocks_shape( @@ -3578,24 +3617,37 @@ def test_daemon_rebuild_lease_refusal_precedes_startup_blob_reconciliation( with sqlite3.connect(source_db) as conn: assert conn.execute("SELECT COUNT(*) FROM blob_publication_reservations").fetchone()[0] == 1 - with ( - RebuildLease(archive_root_path), - pytest.raises( + def archive_digest() -> str: + digest = hashlib.sha256() + for path in sorted(archive_root_path.rglob("*")): + if not path.is_file(): + continue + relative = path.relative_to(archive_root_path).as_posix().encode() + payload = path.read_bytes() + digest.update(len(relative).to_bytes(8, "big")) + digest.update(relative) + digest.update(len(payload).to_bytes(8, "big")) + digest.update(payload) + return digest.hexdigest() + + with RebuildLease(archive_root_path): + before = archive_digest() + with pytest.raises( RebuildLeaseUnavailableError, match="offline index rebuild owns archive", - ), - ): - asyncio.run( - daemon_cli.run_daemon_services( - sources=(), - debounce_s=1.0, - enable_watch=False, - enable_browser_capture=False, - browser_capture_host="127.0.0.1", - browser_capture_port=8765, - browser_capture_spool_path=None, + ): + asyncio.run( + daemon_cli.run_daemon_services( + sources=(), + debounce_s=1.0, + enable_watch=False, + enable_browser_capture=False, + browser_capture_host="127.0.0.1", + browser_capture_port=8765, + browser_capture_spool_path=None, + ) ) - ) + assert archive_digest() == before with sqlite3.connect(source_db) as conn: assert conn.execute("SELECT COUNT(*) FROM blob_publication_reservations").fetchone()[0] == 1 @@ -3920,6 +3972,28 @@ def test_pidfile_remains_locked_until_admitted_writers_are_drained( os.close(successor_fd) +def test_rebuild_exclusion_survives_an_undrained_writer_timeout(tmp_path: Path) -> None: + from polylogue.daemon import cli as daemon_cli + from polylogue.product.raw_authority import archive_writer_rebuild_exclusion + from polylogue.storage.index_generation import RebuildLease, RebuildLeaseUnavailableError + + archive_root_path = tmp_path / "archive" + archive_root_path.mkdir() + with archive_writer_rebuild_exclusion(archive_root_path) as exclusion: + daemon_cli._retain_rebuild_exclusion_for_undrained_writer( + exclusion, + writer_drained=False, + ) + + with pytest.raises(RebuildLeaseUnavailableError, match="index rebuild lease is already held"): + with RebuildLease(archive_root_path): + pass + + exclusion.release() + with RebuildLease(archive_root_path): + pass + + def test_shutdown_lifecycle_event_is_bounded_when_writer_gate_is_stuck(tmp_path: Path) -> None: from polylogue.daemon import cli as daemon_cli From 8dbb6352ec32103ddd01a0801b2114e2df38bfc2 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 14 Aug 2026 03:56:53 +0200 Subject: [PATCH 62/65] fix: retain rebuild exclusion on drain cancellation --- polylogue/daemon/cli.py | 43 ++++++++++++++------ tests/unit/daemon/test_daemon_cli.py | 59 ++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 12 deletions(-) diff --git a/polylogue/daemon/cli.py b/polylogue/daemon/cli.py index be0053bd8f..7c0ce68595 100644 --- a/polylogue/daemon/cli.py +++ b/polylogue/daemon/cli.py @@ -2113,6 +2113,25 @@ def _retain_rebuild_exclusion_for_undrained_writer( rebuild_exclusion.retain_until_process_exit() +async def _shutdown_writer_coordinator_with_rebuild_exclusion( + coordinator: DaemonWriteCoordinator, + rebuild_exclusion: ArchiveWriterRebuildExclusion, + *, + timeout: float, +) -> bool: + """Drain writers or retain rebuild exclusion when drain cannot be proven.""" + try: + writer_drained = await coordinator.shutdown(timeout=timeout) + except BaseException: + rebuild_exclusion.retain_until_process_exit() + raise + _retain_rebuild_exclusion_for_undrained_writer( + rebuild_exclusion, + writer_drained=writer_drained, + ) + return writer_drained + + async def run_live_watcher( *, sources: tuple[WatchSource, ...], @@ -2142,10 +2161,10 @@ async def run_live_watcher( finally: if watcher is not None: watcher.stop() - writer_drained = await coordinator.shutdown(timeout=5.0) - _retain_rebuild_exclusion_for_undrained_writer( + await _shutdown_writer_coordinator_with_rebuild_exclusion( + coordinator, rebuild_exclusion, - writer_drained=writer_drained, + timeout=5.0, ) @@ -2433,10 +2452,10 @@ async def _run_daemon_services_under_active_writer_lease( if lifecycle is not None: with contextlib.suppress(Exception): await write_coordinator.run_sync("daemon.lifecycle.stop", lifecycle.stop, exit_kind="error") - writer_drained = await write_coordinator.shutdown(timeout=5.0) - _retain_rebuild_exclusion_for_undrained_writer( + writer_drained = await _shutdown_writer_coordinator_with_rebuild_exclusion( + write_coordinator, rebuild_exclusion, - writer_drained=writer_drained, + timeout=5.0, ) _release_pidfile_after_writer_drain(pidfile_fd, writer_drained=writer_drained) if writer_drained: @@ -2472,10 +2491,10 @@ async def _run_daemon_services_under_active_writer_lease( if lifecycle is not None: with contextlib.suppress(Exception): await write_coordinator.run_sync("daemon.lifecycle.stop", lifecycle.stop, exit_kind="error") - writer_drained = await write_coordinator.shutdown(timeout=5.0) - _retain_rebuild_exclusion_for_undrained_writer( + writer_drained = await _shutdown_writer_coordinator_with_rebuild_exclusion( + write_coordinator, rebuild_exclusion, - writer_drained=writer_drained, + timeout=5.0, ) _release_pidfile_after_writer_drain(pidfile_fd, writer_drained=writer_drained) if writer_drained: @@ -2837,10 +2856,10 @@ async def _run_daemon_services_under_active_writer_lease( except Exception: logger.warning("daemon: could not persist final lifecycle stop", exc_info=True) - writer_drained = await write_coordinator.shutdown(timeout=5.0) - _retain_rebuild_exclusion_for_undrained_writer( + writer_drained = await _shutdown_writer_coordinator_with_rebuild_exclusion( + write_coordinator, rebuild_exclusion, - writer_drained=writer_drained, + timeout=5.0, ) pidfile_fd = _release_pidfile_after_writer_drain(pidfile_fd, writer_drained=writer_drained) finally: diff --git a/tests/unit/daemon/test_daemon_cli.py b/tests/unit/daemon/test_daemon_cli.py index b46997133b..f75c4dccaa 100644 --- a/tests/unit/daemon/test_daemon_cli.py +++ b/tests/unit/daemon/test_daemon_cli.py @@ -2496,6 +2496,65 @@ def __init__(self) -> None: asyncio.run(daemon_cli.run_live_watcher(sources=(), debounce_s=1.0)) +def test_live_watcher_cancellation_during_drain_retains_rebuild_exclusion( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + from polylogue.daemon import cli as daemon_cli + from polylogue.storage.index_generation import RebuildLease, RebuildLeaseUnavailableError + + archive_root_path = tmp_path / "archive" + archive_root_path.mkdir() + monkeypatch.setattr("polylogue.paths.archive_root", lambda: archive_root_path) + + class FakePolylogue: + async def __aenter__(self) -> object: + return object() + + async def __aexit__(self, *exc: object) -> None: + return None + + class FakeWatcher: + def __init__(self, *_args: object, **_kwargs: object) -> None: + pass + + async def run(self) -> None: + return None + + def stop(self) -> None: + return None + + class BlockingCoordinator: + def __init__(self) -> None: + self.shutdown_started = asyncio.Event() + + async def shutdown(self, *, timeout: float) -> bool: + assert timeout == 5.0 + self.shutdown_started.set() + await asyncio.Event().wait() + return False + + coordinator = BlockingCoordinator() + + async def exercise() -> None: + with ( + patch.object(daemon_cli, "Polylogue", FakePolylogue), + patch.object(daemon_cli, "LiveWatcher", FakeWatcher), + patch.object(daemon_cli, "daemon_write_coordinator", return_value=coordinator), + ): + task = asyncio.create_task(daemon_cli.run_live_watcher(sources=(), debounce_s=1.0)) + await coordinator.shutdown_started.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + asyncio.run(exercise()) + + with pytest.raises(RebuildLeaseUnavailableError, match="index rebuild lease is already held"): + with RebuildLease(archive_root_path): + pass + + def test_ensure_fts_startup_readiness_skips_old_non_blocks_shape( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, From ed804c8fd729e2aacc72ebd98309506f6f63357e Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 14 Aug 2026 04:14:00 +0200 Subject: [PATCH 63/65] fix: retain rebuild exclusion across cleanup failures --- polylogue/daemon/cli.py | 9 ++++ tests/unit/daemon/test_daemon_cli.py | 78 ++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+) diff --git a/polylogue/daemon/cli.py b/polylogue/daemon/cli.py index 7c0ce68595..0b4031de6e 100644 --- a/polylogue/daemon/cli.py +++ b/polylogue/daemon/cli.py @@ -2863,6 +2863,15 @@ async def _run_daemon_services_under_active_writer_lease( ) pidfile_fd = _release_pidfile_after_writer_drain(pidfile_fd, writer_drained=writer_drained) finally: + # Any exception or repeated cancellation before coordinator + # shutdown leaves writer drain unproven. The outer product + # context must not interpret that control-flow escape as a safe + # release: keep rebuild exclusion until process exit unless the + # coordinator returned an affirmative drain result. + _retain_rebuild_exclusion_for_undrained_writer( + rebuild_exclusion, + writer_drained=writer_drained, + ) if server is not None: with contextlib.suppress(Exception): server.server_close() diff --git a/tests/unit/daemon/test_daemon_cli.py b/tests/unit/daemon/test_daemon_cli.py index f75c4dccaa..f40c3631f7 100644 --- a/tests/unit/daemon/test_daemon_cli.py +++ b/tests/unit/daemon/test_daemon_cli.py @@ -10,6 +10,7 @@ import stat import threading import time +from collections.abc import Iterator from pathlib import Path from types import SimpleNamespace from typing import Any, cast @@ -3759,6 +3760,83 @@ def stop(self) -> None: assert stopped == [True] +def test_daemon_cleanup_failure_retains_rebuild_exclusion_until_process_exit( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Cleanup errors before coordinator shutdown must never reopen rebuilds.""" + from polylogue.daemon import cli as daemon_cli + from polylogue.product import raw_authority + from polylogue.storage.index_generation import RebuildLease, RebuildLeaseUnavailableError + + async def noop() -> None: + return None + + class FakePolylogue: + async def __aenter__(self) -> object: + return object() + + async def __aexit__(self, *exc: object) -> None: + return None + + class FakeWatcher: + def __init__(self, *_args: object, **_kwargs: object) -> None: + pass + + async def run(self) -> None: + raise RuntimeError("watch stopped") + + def stop(self) -> None: + return None + + captured: list[raw_authority.ArchiveWriterRebuildExclusion] = [] + archive_roots: list[Path] = [] + real_exclusion = raw_authority.archive_writer_rebuild_exclusion + + @contextlib.contextmanager + def capture_exclusion(archive_root: Path) -> Iterator[raw_authority.ArchiveWriterRebuildExclusion]: + archive_roots.append(archive_root) + with real_exclusion(archive_root) as exclusion: + captured.append(exclusion) + yield exclusion + + def fail_shutdown_marker() -> None: + raise RuntimeError("shutdown marker failed") + + monkeypatch.setattr(raw_authority, "archive_writer_rebuild_exclusion", capture_exclusion) + with ( + patch.object(daemon_cli, "Polylogue", FakePolylogue), + patch.object(daemon_cli, "LiveWatcher", FakeWatcher), + patch.object(daemon_cli, "_reconcile_blob_publications", noop), + patch.object( + daemon_cli, + "_mark_interrupted_live_ingest_attempts_on_shutdown", + fail_shutdown_marker, + ), + pytest.raises(RuntimeError, match="shutdown marker failed"), + ): + asyncio.run( + daemon_cli.run_daemon_services( + sources=(WatchSource(name="codex", root=Path("/tmp/codex")),), + debounce_s=1.0, + enable_watch=True, + enable_browser_capture=False, + browser_capture_host="127.0.0.1", + browser_capture_port=8765, + browser_capture_spool_path=None, + ) + ) + + assert len(captured) == 1 + assert len(archive_roots) == 1 + with pytest.raises(RebuildLeaseUnavailableError, match="index rebuild lease is already held"): + with RebuildLease(archive_roots[0]): + pass + + captured[0].release() + with RebuildLease(archive_roots[0]): + pass + + def test_lifecycle_heartbeat_runs_without_index_stats(monkeypatch: pytest.MonkeyPatch) -> None: """The degraded daemon heartbeat must not depend on index.db existing.""" from polylogue.daemon import cli as daemon_cli From 58a687b26eb27a5b64a5a77ae1fb6114f7e6bef0 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 14 Aug 2026 04:34:20 +0200 Subject: [PATCH 64/65] fix: scan oversized retained JSONL records structurally Problem: Provider detection stopped at the first 8192 bytes of an oversized JSONL record, so a valid discriminator after a large preceding field was treated as UNKNOWN. What changed: Stream each retained JSONL record through the existing scalar-bounded structural probe in 4096-byte chunks, sharing the hard 64 KiB total scan envelope. Add a Codex late-session_meta regression and malformed huge-input budget coverage. Alternatives rejected: Eager whole-record parsing and unbounded scanning would violate the retained-replay resource contract. Compatibility: Positive evidence remains routed through the existing provider detectors. Evidence outside the envelope remains UNKNOWN. --- polylogue/sources/revision_backfill.py | 137 ++++++++++++------- tests/unit/sources/test_revision_backfill.py | 75 ++++++++++ 2 files changed, 166 insertions(+), 46 deletions(-) diff --git a/polylogue/sources/revision_backfill.py b/polylogue/sources/revision_backfill.py index 40128750a2..2760490aa4 100644 --- a/polylogue/sources/revision_backfill.py +++ b/polylogue/sources/revision_backfill.py @@ -16,7 +16,7 @@ from io import BytesIO from pathlib import Path from types import TracebackType -from typing import BinaryIO, Final, Literal, cast +from typing import BinaryIO, Final, Literal, Protocol, cast import ijson from ijson.common import ObjectBuilder @@ -85,6 +85,20 @@ _LOGGER = _polylogue_logging.get_logger(__name__) _REPLAY_PROVIDER_DETECTION_PREFIX_BYTES: Final[int] = 8192 _REPLAY_PROVIDER_DETECTION_MAX_SCAN_BYTES: Final[int] = 64 * 1024 +_REPLAY_PROVIDER_DETECTION_READ_CHUNK_BYTES: Final[int] = 4096 + + +class _ReadableBinary(Protocol): + def read(self, size: int = -1) -> bytes: ... + + +class _LineReadableBinary(_ReadableBinary, Protocol): + def readline(self, size: int = -1) -> bytes: ... + + +class _SeekableReadableBinary(_LineReadableBinary, Protocol): + def seek(self, offset: int, whence: int = 0) -> int: ... + _DOCUMENT_PROBE_ROOT_KEYS: Final[frozenset[str]] = frozenset( { @@ -170,7 +184,7 @@ def _document_probe_value(key: str, event: str, value: object) -> object | None: class _ScalarBoundedJSONReader: """Stream JSON while capping every scalar token before ijson sees it.""" - def __init__(self, payload: BinaryIO) -> None: + def __init__(self, payload: _ReadableBinary) -> None: self._payload = payload self._output = bytearray() self._eof = False @@ -256,6 +270,36 @@ def _filter_string_byte(self, byte: int) -> None: self._string_bytes += utf8_bytes +class _BoundedJSONLRecordReader: + """Expose one JSONL record without exceeding the shared scan budget.""" + + def __init__(self, payload: _LineReadableBinary, byte_budget: int) -> None: + self._payload = payload + self._remaining = byte_budget + self.bytes_read = 0 + self._done = False + + def read(self, size: int = -1) -> bytes: + if size == 0 or self._done: + return b"" + if self._remaining <= 0: + self._done = True + return b"" + read_size = _REPLAY_PROVIDER_DETECTION_READ_CHUNK_BYTES if size < 0 else size + read_size = min(read_size, _REPLAY_PROVIDER_DETECTION_READ_CHUNK_BYTES, self._remaining) + chunk = self._payload.readline(read_size) + self.bytes_read += len(chunk) + self._remaining -= len(chunk) + if not chunk or chunk.endswith(b"\n") or self._remaining <= 0: + self._done = True + return chunk + + def drain(self) -> None: + """Consume this physical record, subject to the remaining scan budget.""" + while not self._done: + self.read(_REPLAY_PROVIDER_DETECTION_READ_CHUNK_BYTES) + + @dataclass(slots=True) class _StreamingDocumentProviderProbe: """Bounded structural summary for one object in a JSON document. @@ -409,7 +453,7 @@ def classify(self, *, sequence_item: bool = False) -> tuple[Provider, str]: return provider, f"bounded streaming JSON structure: {evidence}" -def _detect_provider_from_bounded_document(payload: BinaryIO) -> tuple[Provider, str]: +def _detect_provider_from_bounded_document(payload: _SeekableReadableBinary) -> tuple[Provider, str]: """Scan every document object while retaining fixed structural evidence.""" payload.seek(0) bounded_payload = _ScalarBoundedJSONReader(payload) @@ -491,18 +535,45 @@ def _detect_provider_from_bounded_prefix( return detected, f"bounded partial JSON structure: {partial_evidence}" +def _detect_provider_from_bounded_record( + record: _BoundedJSONLRecordReader, +) -> tuple[Provider, str]: + """Classify a streamed JSONL record from bounded structural evidence.""" + bounded_record = _ScalarBoundedJSONReader(record) + probe = _StreamingDocumentProviderProbe() + last_evidence = "no bounded JSONL record structure identified a provider" + try: + for prefix, event, value in ijson.parse(bounded_record, use_float=True): + if prefix == "": + if event == "start_map": + continue + if event == "end_map": + provider, last_evidence = probe.classify(sequence_item=True) + if provider is not Provider.UNKNOWN: + return provider, last_evidence + continue + elif prefix: + probe.feed(prefix, event, value) + except ijson.JSONError: + # A scan-budget cutoff or malformed JSON is expected for retained + # unknown bytes. Completed structural events are still valid evidence; + # an incomplete tail contributes nothing. + pass + provider, last_evidence = probe.classify(sequence_item=True) + return provider, last_evidence + + def _detect_unknown_retained_provider( - payload: BinaryIO, + payload: _SeekableReadableBinary, source_path: str, ) -> tuple[Provider, str]: """Detect retained UNKNOWN bytes without eagerly materializing JSONL. A byte prefix can end inside the first physical JSONL record. For an oversized record stream that makes a prefix-only detector inconclusive - even when a later bounded record identifies a streaming provider. Scan - complete records across the stream instead: each record is capped at the - same detection bound, oversized records are consumed in bounded chunks, - and the scan continues until positive provider evidence or EOF. + even when a later structural key in that record identifies a streaming + provider. Stream each record through the structural probe in bounded + chunks, stopping at positive evidence, the total scan envelope, or EOF. Non-JSONL documents first use the same prefix evidence, then continue a bounded structural event scan through every document object. Eager replay @@ -521,48 +592,22 @@ def _detect_unknown_retained_provider( return _detect_provider_from_bounded_document(payload) last_evidence = "no bounded JSONL record identified a provider; used fallback_provider" - read_size = _REPLAY_PROVIDER_DETECTION_PREFIX_BYTES + 1 - scanned_bytes = 0 - def read_bounded_line() -> bytes: - nonlocal scanned_bytes - remaining_bytes = _REPLAY_PROVIDER_DETECTION_MAX_SCAN_BYTES - scanned_bytes - if remaining_bytes <= 0: - return b"" - raw_line = payload.readline(min(read_size, remaining_bytes)) - scanned_bytes += len(raw_line) - return raw_line - - while raw_line := read_bounded_line(): - has_newline = raw_line.endswith(b"\n") - oversized = not has_newline and len(raw_line) > _REPLAY_PROVIDER_DETECTION_PREFIX_BYTES - bounded_record = raw_line[:_REPLAY_PROVIDER_DETECTION_PREFIX_BYTES] - if oversized: - provider, last_evidence = _detect_provider_from_bounded_prefix( - bounded_record, - stream_name, - record_stream=True, - ) - # Provider authority comes from the bounded structural prefix; - # draining the rest of an oversized physical record is needed - # only when that prefix was inconclusive. In particular, do not - # replace positive evidence with UNKNOWN merely because the - # record itself extends beyond the total scan envelope. - if provider is not Provider.UNKNOWN: - return provider, last_evidence - while raw_line and not raw_line.endswith(b"\n"): - raw_line = read_bounded_line() - if not raw_line: - return Provider.UNKNOWN, "bounded JSONL provider scan exhausted; used fallback_provider" - else: - provider, last_evidence = detect_provider_from_raw_bytes_evidence( - bounded_record, - stream_name, - Provider.UNKNOWN, - ) + while scanned_bytes < _REPLAY_PROVIDER_DETECTION_MAX_SCAN_BYTES: + record = _BoundedJSONLRecordReader( + payload, + _REPLAY_PROVIDER_DETECTION_MAX_SCAN_BYTES - scanned_bytes, + ) + provider, last_evidence = _detect_provider_from_bounded_record(record) + parsed_bytes = record.bytes_read + scanned_bytes += parsed_bytes if provider is not Provider.UNKNOWN: return provider, last_evidence + record.drain() + scanned_bytes += record.bytes_read - parsed_bytes + if record.bytes_read == 0: + break return Provider.UNKNOWN, last_evidence diff --git a/tests/unit/sources/test_revision_backfill.py b/tests/unit/sources/test_revision_backfill.py index 23282ca9f6..76ba01cfc3 100644 --- a/tests/unit/sources/test_revision_backfill.py +++ b/tests/unit/sources/test_revision_backfill.py @@ -193,6 +193,81 @@ def reject_eager_material(_raw_id: str) -> tuple[Provider, bytes, str, RawRevisi assert [session.provider_session_id for session in sessions] == ["unknown-stream"] +def test_unknown_retained_codex_record_scans_provider_key_past_8k_padding( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """A late Codex discriminator in one oversized record remains visible.""" + initialize_active_archive_root(tmp_path) + late_session_meta = json.dumps( + { + "padding": "x" * (revision_backfill._REPLAY_PROVIDER_DETECTION_PREFIX_BYTES + 512), + "type": "session_meta", + "payload": {"id": "late-codex", "timestamp": "2026-06-01T00:00:00Z"}, + }, + separators=(",", ":"), + ).encode() + payload = ( + late_session_meta + + b"\n" + + b'{"type":"response_item","payload":{"type":"message","id":"m1","role":"user",' + + b'"content":[{"type":"input_text","text":"late discriminator"}]}}\n' + ) + assert ( + b'"type":"session_meta"' not in late_session_meta[: revision_backfill._REPLAY_PROVIDER_DETECTION_PREFIX_BYTES] + ) + + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.UNKNOWN, + payload=payload, + source_path="late-codex.jsonl", + acquired_at_ms=1, + ) + + monkeypatch.setattr( + archive, + "raw_revision_material", + lambda *_args, **_kwargs: pytest.fail("late Codex evidence must select the streaming route"), + ) + sessions = revision_backfill.parse_retained_raw_sessions(archive, raw_id) + + assert [session.provider_session_id for session in sessions] == ["late-codex"] + + +def test_unknown_retained_malformed_huge_record_stays_unknown_inside_total_budget() -> None: + """Malformed data cannot make a discriminator beyond the scan envelope authoritative.""" + + class CountingReader: + def __init__(self, payload: bytes) -> None: + self._payload = BytesIO(payload) + self.bytes_read = 0 + + def readline(self, size: int = -1) -> bytes: + chunk = self._payload.readline(size) + self.bytes_read += len(chunk) + return chunk + + def read(self, size: int = -1) -> bytes: + chunk = self._payload.read(size) + self.bytes_read += len(chunk) + return chunk + + def seek(self, offset: int, whence: int = 0) -> int: + return self._payload.seek(offset, whence) + + payload = ( + b'{"padding":"' + + b"x" * (revision_backfill._REPLAY_PROVIDER_DETECTION_MAX_SCAN_BYTES + 16_384) + + b'","type":"session_meta","payload":{"id":"outside-budget"}' + ) + reader = CountingReader(payload) + + provider, _evidence = revision_backfill._detect_unknown_retained_provider(reader, "huge.jsonl") + + assert provider is Provider.UNKNOWN + assert reader.bytes_read <= revision_backfill._REPLAY_PROVIDER_DETECTION_MAX_SCAN_BYTES + + def test_unknown_retained_oversized_provider_record_never_uses_eager_payload( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: From 7412c09f9b8516c07a51ce8464e6de3c174a9642 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 14 Aug 2026 04:53:19 +0200 Subject: [PATCH 65/65] fix: drain standalone watcher after stop failure --- polylogue/daemon/cli.py | 16 +++---- tests/unit/daemon/test_daemon_cli.py | 63 ++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 7 deletions(-) diff --git a/polylogue/daemon/cli.py b/polylogue/daemon/cli.py index 0b4031de6e..f849bccac5 100644 --- a/polylogue/daemon/cli.py +++ b/polylogue/daemon/cli.py @@ -2159,13 +2159,15 @@ async def run_live_watcher( with contextlib.suppress(KeyboardInterrupt): await watcher.run() finally: - if watcher is not None: - watcher.stop() - await _shutdown_writer_coordinator_with_rebuild_exclusion( - coordinator, - rebuild_exclusion, - timeout=5.0, - ) + try: + if watcher is not None: + watcher.stop() + finally: + await _shutdown_writer_coordinator_with_rebuild_exclusion( + coordinator, + rebuild_exclusion, + timeout=5.0, + ) async def run_daemon_services( diff --git a/tests/unit/daemon/test_daemon_cli.py b/tests/unit/daemon/test_daemon_cli.py index f40c3631f7..691a317843 100644 --- a/tests/unit/daemon/test_daemon_cli.py +++ b/tests/unit/daemon/test_daemon_cli.py @@ -2556,6 +2556,69 @@ async def exercise() -> None: pass +def test_live_watcher_stop_failure_still_retains_undrained_rebuild_exclusion( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """A watcher stop exception cannot bypass coordinator drain authority.""" + from polylogue.daemon import cli as daemon_cli + from polylogue.product import raw_authority + from polylogue.storage.index_generation import RebuildLease, RebuildLeaseUnavailableError + + archive_root_path = tmp_path / "archive" + archive_root_path.mkdir() + monkeypatch.setattr("polylogue.paths.archive_root", lambda: archive_root_path) + + class FakePolylogue: + async def __aenter__(self) -> object: + return object() + + async def __aexit__(self, *exc: object) -> None: + return None + + class FailingStopWatcher: + def __init__(self, *_args: object, **_kwargs: object) -> None: + pass + + async def run(self) -> None: + return None + + def stop(self) -> None: + raise RuntimeError("watcher stop failed") + + class UndrainedCoordinator: + async def shutdown(self, *, timeout: float) -> bool: + assert timeout == 5.0 + return False + + captured: list[raw_authority.ArchiveWriterRebuildExclusion] = [] + real_exclusion = raw_authority.archive_writer_rebuild_exclusion + + @contextlib.contextmanager + def capture_exclusion(root: Path) -> Iterator[raw_authority.ArchiveWriterRebuildExclusion]: + with real_exclusion(root) as exclusion: + captured.append(exclusion) + yield exclusion + + monkeypatch.setattr(raw_authority, "archive_writer_rebuild_exclusion", capture_exclusion) + with ( + patch.object(daemon_cli, "Polylogue", FakePolylogue), + patch.object(daemon_cli, "LiveWatcher", FailingStopWatcher), + patch.object(daemon_cli, "daemon_write_coordinator", return_value=UndrainedCoordinator()), + pytest.raises(RuntimeError, match="watcher stop failed"), + ): + asyncio.run(daemon_cli.run_live_watcher(sources=(), debounce_s=1.0)) + + assert len(captured) == 1 + with pytest.raises(RebuildLeaseUnavailableError, match="index rebuild lease is already held"): + with RebuildLease(archive_root_path): + pass + + captured[0].release() + with RebuildLease(archive_root_path): + pass + + def test_ensure_fts_startup_readiness_skips_old_non_blocks_shape( tmp_path: Path, monkeypatch: pytest.MonkeyPatch,