diff --git a/devtools/verify.py b/devtools/verify.py index da4a268652..f08979c1d3 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -3861,24 +3861,32 @@ def _main(argv: list[str] | None = None) -> int: final_checkout_fingerprint = worktree_fingerprint(ROOT) mutation_observation = finish_checkout_mutation_monitor(mutation_monitor) checkout_stable = True - if ( - changed_path_authority_failed - or head is None + checkout_fingerprint_unavailable = ( + head is None or final_head is None - or "unavailable" in {checkout_fingerprint, final_checkout_fingerprint} - or mutation_observation.unavailable - ): + or "unavailable" + in { + checkout_fingerprint, + final_checkout_fingerprint, + } + ) + if changed_path_authority_failed or checkout_fingerprint_unavailable or mutation_observation.unavailable: checkout_stable = False + diagnosis = ( + "testmon_changed_path_authority_unavailable" + if changed_path_authority_failed + else ( + "checkout_fingerprint_unavailable" + if checkout_fingerprint_unavailable + else "checkout_mutation_monitor_unavailable" + ) + ) step_results.append( { "name": "checkout stability", "duration_s": 0.0, "exit": 125, - "diagnosis": ( - "testmon_changed_path_authority_unavailable" - if changed_path_authority_failed - else "checkout_fingerprint_unavailable" - ), + "diagnosis": diagnosis, "initial_git_head": head, "final_git_head": final_head, "initial_worktree_fingerprint": checkout_fingerprint, @@ -3887,7 +3895,7 @@ def _main(argv: list[str] | None = None) -> int: ) if exit_code == 0: exit_code = 125 - sys.stderr.write("verify: checkout fingerprint unavailable; evidence is not exact-head.\n") + sys.stderr.write(f"verify: {diagnosis.replace('_', ' ')}; evidence is not exact-head.\n") elif final_head != head or mutation_observation.changed or final_checkout_fingerprint != checkout_fingerprint: checkout_stable = False step_results.append( diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index 30d46bfaf1..ec9d1bee52 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -277,12 +277,15 @@ def __init__(self, root: Path) -> None: self._unavailable = False self._stop = threading.Event() self._ready = threading.Event() + self._initialized = threading.Event() self._thread: threading.Thread | None = None self._state_lock = threading.Lock() self._tracked_paths: frozenset[Path] = frozenset() self._tracked_directories: frozenset[Path] = frozenset() self._ignored_roots: frozenset[Path] = frozenset() self._git_index_path: Path | None = None + self._git_current_ref_path: Path | None = None + self._git_current_ref_was_loose: bool | None = None self._git_authority_paths: dict[Path, str] = {} self._directory_topology_fingerprint: frozenset[str] | None = None @@ -292,14 +295,35 @@ def start(self) -> None: with self._state_lock: self._unavailable = True self._ready.set() + self._initialized.set() return - self._thread = threading.Thread(target=self._watch, name="checkout-mutation-monitor", daemon=True) + # Repository enumeration and Git authority discovery are synchronous + # preflight, not native watcher startup. Keeping them outside the + # backend deadline prevents a slow CI checkout from consuming the + # entire readiness budget before watchfiles can initialize. + watched_directories = self._watched_directories() + if self._unavailable: + self._ready.set() + self._initialized.set() + return + self._thread = threading.Thread( + target=self._watch, + args=(watched_directories,), + name="checkout-mutation-monitor", + daemon=True, + ) self._thread.start() if not self._ready.wait(timeout=self._WATCH_START_TIMEOUT_S): with self._state_lock: self._unavailable = True self._stop.set() self._thread.join(timeout=self._WATCH_START_TIMEOUT_S) + return + # The one-second deadline proves only native backend startup. The + # protected topology recheck is ordinary repository discovery and may + # legitimately take longer on a cold CI checkout; complete it before + # the verification command can mutate the tree. + self._initialized.wait() def finish(self) -> CheckoutMutationObservation: """Stop monitoring only after the caller took its final fingerprint.""" @@ -320,11 +344,8 @@ def finish(self) -> CheckoutMutationObservation: observed_path=self._observed_path, ) - def _watch(self) -> None: + def _watch(self, watched_directories: Sequence[Path]) -> None: try: - watched_directories = self._watched_directories() - if self._unavailable: - return for changes in watchfiles.watch( *watched_directories, watch_filter=None, @@ -338,12 +359,17 @@ def _watch(self) -> None: recursive=False, ): # An empty timeout batch proves the backend initialized before - # a verification command starts, closing the startup race. - if not self._ready.is_set() and not self._directory_topology_is_stable(watched_directories): - with self._state_lock: - self._unavailable = True - return - self._ready.set() + # a verification command starts, closing the startup race. The + # active watcher protects the following topology recheck. The + # native-ready event has its own bounded startup deadline; + # ``start`` waits separately for repository discovery. + if not self._ready.is_set(): + self._ready.set() + if not self._directory_topology_is_stable(watched_directories): + with self._state_lock: + self._unavailable = True + return + self._initialized.set() for _change, raw_path in changes: self._record_change(Path(raw_path)) if self._changed or self._unavailable: @@ -356,6 +382,7 @@ def _watch(self) -> None: self._unavailable = True finally: self._ready.set() + self._initialized.set() @classmethod def _polling_backend_requested(cls) -> bool: @@ -459,6 +486,12 @@ def _resolve_git_head_paths(self) -> dict[Path, str]: self._unavailable = True return paths paths[Path(raw_head_path)] = ".git/HEAD" + if symbolic_result.returncode == 1: + # A detached checkout's complete revision authority is the + # worktree-specific HEAD file. packed-refs is shared by every + # linked worktree, so unrelated fetch/pack maintenance cannot + # mutate this checkout and must not invalidate its verification. + return paths packed_result = self._git_command(["rev-parse", "--path-format=absolute", "--git-path", "packed-refs"]) if packed_result is None: return paths @@ -468,16 +501,18 @@ def _resolve_git_head_paths(self) -> dict[Path, str]: self._unavailable = True return paths paths[Path(raw_packed_path)] = ".git/packed-refs" - if symbolic_result.returncode == 0: - ref_result = self._git_command(["rev-parse", "--path-format=absolute", "--git-path", symbolic_ref]) - if ref_result is None: - return paths - raw_ref_path = os.fsdecode(ref_result.stdout).strip() - if not raw_ref_path: - with self._state_lock: - self._unavailable = True - return paths - paths[Path(raw_ref_path)] = f".git/{symbolic_ref}" + ref_result = self._git_command(["rev-parse", "--path-format=absolute", "--git-path", symbolic_ref]) + if ref_result is None: + return paths + raw_ref_path = os.fsdecode(ref_result.stdout).strip() + if not raw_ref_path: + with self._state_lock: + self._unavailable = True + return paths + self._git_current_ref_path = Path(raw_ref_path) + if self._git_current_ref_was_loose is None: + self._git_current_ref_was_loose = self._git_current_ref_path.exists() + paths[self._git_current_ref_path] = f".git/{symbolic_ref}" return paths def _git_command( @@ -527,6 +562,14 @@ def _record_change(self, candidate: Path) -> None: if not candidate.is_absolute(): candidate = self.root / candidate for authority_path, label in self._git_authority_paths.items(): + if label == ".git/packed-refs" and self._git_current_ref_was_loose is True: + # packed-refs is shared by linked worktrees. When this + # worktree's current branch has a loose ref, unrelated fetch + # maintenance cannot change its HEAD through the packed file. + # A real pack transition remains visible when the loose ref + # is removed or replaced. Preserve the startup state so a + # packed-to-loose transition cannot hide its own first event. + continue if candidate != authority_path and authority_path.is_relative_to(candidate): with self._state_lock: self._changed = True diff --git a/docs/plans/classifier-fingerprints.json b/docs/plans/classifier-fingerprints.json index ddde05b6e5..3439772203 100644 --- a/docs/plans/classifier-fingerprints.json +++ b/docs/plans/classifier-fingerprints.json @@ -10,11 +10,11 @@ } }, "polylogue/archive/artifact_taxonomy/runtime.py:classify_artifact_path": { - "fingerprint": "10f54d1827fdca585985cf3180e54c27098de26121530076e51a26cef7b2de24", + "fingerprint": "32b6f26516b4cc9ed0342c262e492fca469aaca9fb47a66e4ca1c76b7f58987a", "covered_by": { "kind": "acknowledged_safe", - "reason": "Baseline snapshot at classifier-fingerprint gate introduction (polylogue-gucv).", - "ref": "polylogue-gucv" + "reason": "Strong-only admission helper preserves this classifier output; no archived payload classification changes.", + "ref": "#3952" } }, "polylogue/archive/artifact_taxonomy/support.py:looks_like_beads_interaction": { diff --git a/docs/plans/layering.yaml b/docs/plans/layering.yaml index 2aad7cee72..22c0907fba 100644 --- a/docs/plans/layering.yaml +++ b/docs/plans/layering.yaml @@ -60,7 +60,7 @@ writer_modules: interruption: atomic entrypoints: [apply_source_raw_state_update, bind_source_raw_revision, record_capture_mode_observation, - record_excised_blob_hash, write_history_sidecar, + record_excised_blob_hash, record_raw_container_coordinate, write_history_sidecar, delete_source_hook_event, write_source_blob_refs, write_source_hook_event, write_source_raw_session, write_source_raw_session_blob_ref, upsert_raw_artifact] - path: polylogue/storage/sqlite/archive_tiers/write.py diff --git a/polylogue/archive/artifact_taxonomy/__init__.py b/polylogue/archive/artifact_taxonomy/__init__.py index ec44d19408..79396181e9 100644 --- a/polylogue/archive/artifact_taxonomy/__init__.py +++ b/polylogue/archive/artifact_taxonomy/__init__.py @@ -8,11 +8,16 @@ from __future__ import annotations from polylogue.archive.artifact_taxonomy.models import ArtifactClassification, ArtifactKind -from polylogue.archive.artifact_taxonomy.runtime import classify_artifact, classify_artifact_path +from polylogue.archive.artifact_taxonomy.runtime import ( + classify_artifact, + classify_artifact_path, + strong_path_classification, +) __all__ = [ "ArtifactClassification", "ArtifactKind", "classify_artifact", "classify_artifact_path", + "strong_path_classification", ] diff --git a/polylogue/archive/artifact_taxonomy/runtime.py b/polylogue/archive/artifact_taxonomy/runtime.py index 95499d9375..ddf2f38b47 100644 --- a/polylogue/archive/artifact_taxonomy/runtime.py +++ b/polylogue/archive/artifact_taxonomy/runtime.py @@ -102,6 +102,21 @@ def classify_artifact_path( """ if weak := _self_generated_artifact_dir_classification(source_path, provider=provider): return weak + return strong_path_classification(source_path, provider=provider) + + +def strong_path_classification( + source_path: str | Path | None, + *, + provider: str | Provider, +) -> ArtifactClassification | None: + """Classify only definitive path rules. + + Live admission uses this before deciding whether a payload may enter a + bounded streaming route. The weak ``analysis/`` location heuristic is + deliberately excluded there because it must yield to bounded payload + evidence or the streaming policy. + """ return _classify_artifact_path_strong(source_path, provider=provider) diff --git a/polylogue/archive/raw_payload/decode.py b/polylogue/archive/raw_payload/decode.py index 6333e65fb0..59d45987ac 100644 --- a/polylogue/archive/raw_payload/decode.py +++ b/polylogue/archive/raw_payload/decode.py @@ -3,6 +3,7 @@ from __future__ import annotations from collections import deque +from collections.abc import Iterator from dataclasses import dataclass from pathlib import Path from typing import IO, Literal, TypeAlias, cast @@ -12,6 +13,7 @@ ArtifactKind, classify_artifact, ) +from polylogue.archive.artifact_taxonomy.support import is_subagent_path from polylogue.archive.raw_payload.streams import raw_line_stream from polylogue.core.binary_signatures import detect_binary_signature from polylogue.core.enums import Provider @@ -79,6 +81,59 @@ class RawPayloadEnvelope: malformed_jsonl_detail: str | None = None +@dataclass(frozen=True) +class JSONLSessionArtifactScan: + """Bounded records that supplied a stream's positive session evidence.""" + + artifact: ArtifactClassification | None + sample: tuple[JSONValue, ...] = () + oversized_records: int = 0 + + +JSONL_RECORD_INSPECTION_BYTES = 64 * 1024 + + +def _bounded_raw_lines( + stream: IO[bytes] | IO[str], + *, + max_record_bytes: int | None, +) -> Iterator[tuple[bytes | str | None, bool]]: + """Yield complete lines without allocating beyond an optional record cap. + + Oversized records are consumed in bounded chunks and represented as + ``(None, True)`` so callers can continue at the next newline. + """ + if max_record_bytes is None: + for raw_line in stream: + yield raw_line, False + return + if max_record_bytes < 1: + raise ValueError("max_record_bytes must be positive") + + read_size = max_record_bytes + 1 + while True: + raw_line = stream.readline(read_size) + if not raw_line: + return + has_newline = raw_line.endswith(b"\n") if isinstance(raw_line, bytes) else raw_line.endswith("\n") + if has_newline: + if len(raw_line) > max_record_bytes: + yield None, True + else: + yield raw_line, False + continue + if len(raw_line) <= max_record_bytes: + yield raw_line, False + return + + while raw_line: + has_newline = raw_line.endswith(b"\n") if isinstance(raw_line, bytes) else raw_line.endswith("\n") + if has_newline: + break + raw_line = stream.readline(read_size) + yield None, True + + def _decode_jsonl_payload( raw: Path | bytes | str, *, @@ -133,12 +188,16 @@ def _sample_jsonl_payload_with_detail( max_samples: int = 64, jsonl_dict_only: bool = False, scan_full: bool = True, + max_record_bytes: int | None = None, ) -> tuple[list[JSONValue], int, str | None]: """Collect a bounded sample of valid JSONL records. This is intended for provider/artifact/schema resolution where full-record materialization is unnecessary. Set ``scan_full`` when malformed-line accounting must reflect the entire source, such as strict validation. + Records skipped because they exceed ``max_record_bytes`` are uninspected, + not malformed: the bound must not manufacture decode-loss evidence for a + syntactically valid stream. """ samples: list[JSONValue] = [] malformed_lines = 0 @@ -148,8 +207,12 @@ def _sample_jsonl_payload_with_detail( line_number = 0 with raw_line_stream(raw) as stream: - for raw_line in stream: + for raw_line, oversized in _bounded_raw_lines(stream, max_record_bytes=max_record_bytes): line_number += 1 + if oversized: + first_line = False + continue + assert raw_line is not None try: line = _decode_provider_utf8(raw_line) if isinstance(raw_line, bytes) else raw_line except UnicodeDecodeError as exc: @@ -183,22 +246,32 @@ def _sample_jsonl_payload_with_detail( return samples, malformed_lines, malformed_detail -def jsonl_session_artifact( +def scan_jsonl_session_artifact( raw: Path | bytes | str | IO[bytes] | IO[str], *, provider: Provider, jsonl_dict_only: bool = False, -) -> ArtifactClassification | None: - """Stream JSONL until one decoded record proves session eligibility. + source_path: str | Path | None = None, + max_record_bytes: int | None = None, +) -> JSONLSessionArtifactScan: + """Stream JSONL until bounded decoded records prove session eligibility. Terminal artifact admission must not let an arbitrary prefix of - non-conversational records hide a later session record. This retains a - rolling 32-record window, including for blob-backed multi-gigabyte JSONL. + non-conversational records hide a later session record. The rolling window + retains at most 32 decoded records. When ``max_record_bytes`` is supplied, + oversized records are discarded in chunks so a later record remains + inspectable without allocating the oversized line. """ records: deque[JSONValue] = deque(maxlen=32) first_line = True + oversized_records = 0 with raw_line_stream(raw) as stream: - for raw_line in stream: + for raw_line, oversized in _bounded_raw_lines(stream, max_record_bytes=max_record_bytes): + if oversized: + oversized_records += 1 + first_line = False + continue + assert raw_line is not None try: line = _decode_provider_utf8(raw_line) if isinstance(raw_line, bytes) else raw_line except UnicodeDecodeError: @@ -218,10 +291,46 @@ def jsonl_session_artifact( records.append(payload) window = list(records) for start in range(len(window)): - artifact = classify_artifact(window[start:], provider=provider) + sample = window[start:] + artifact = classify_artifact(sample, provider=provider, source_path=source_path) if artifact.parse_as_session: - return artifact - return None + return JSONLSessionArtifactScan( + artifact=artifact, + sample=tuple(sample), + oversized_records=oversized_records, + ) + if oversized_records and provider in {Provider.CLAUDE_CODE, Provider.CODEX}: + # A size-bounded inspection skip is unresolved evidence, not negative + # evidence. These providers have streaming parsers, so retain the raw + # as a parse candidate instead of allowing a weak path heuristic to + # terminalize a genuine session whose only record was oversized. + subagent = is_subagent_path(source_path) + return JSONLSessionArtifactScan( + artifact=ArtifactClassification( + provider=provider, + kind=ArtifactKind.AGENT_TRANSCRIPT if subagent else ArtifactKind.SESSION_RECORD_STREAM, + parse_as_session=True, + schema_eligible=False, + default_priority=90 if subagent else 120, + reason="uninspected oversized provider JSONL record retained for streaming parse", + ), + oversized_records=oversized_records, + ) + return JSONLSessionArtifactScan(artifact=None, oversized_records=oversized_records) + + +def jsonl_session_artifact( + raw: Path | bytes | str | IO[bytes] | IO[str], + *, + provider: Provider, + jsonl_dict_only: bool = False, +) -> ArtifactClassification | None: + """Compatibility wrapper for callers that only need classification.""" + return scan_jsonl_session_artifact( + raw, + provider=provider, + jsonl_dict_only=jsonl_dict_only, + ).artifact def sample_jsonl_payload( @@ -482,8 +591,11 @@ def _hermes_sqlite_marker_payload( "JSONRecord", "JSONValue", "RawPayloadEnvelope", + "JSONLSessionArtifactScan", + "JSONL_RECORD_INSPECTION_BYTES", "WireFormat", "build_raw_payload_envelope", "jsonl_session_artifact", + "scan_jsonl_session_artifact", "sample_jsonl_payload", ] diff --git a/polylogue/browser_capture/receiver.py b/polylogue/browser_capture/receiver.py index aa96933acc..b94ffcfce6 100644 --- a/polylogue/browser_capture/receiver.py +++ b/polylogue/browser_capture/receiver.py @@ -31,6 +31,7 @@ from polylogue.core.hashing import hash_text_short from polylogue.core.json import JSONDecodeError, dumps_bytes from polylogue.core.json import loads as json_loads +from polylogue.core.raw_state import raw_state_authority from polylogue.core.timestamps import parse_timestamp from polylogue.logging import get_logger from polylogue.paths import archive_root as default_archive_root @@ -39,6 +40,7 @@ browser_capture_receiver_token_path, browser_capture_spool_root, ) +from polylogue.storage.archive_identity import ArchiveLocationError, resolve_active_index_path from polylogue.storage.introspection import table_exists as _table_exists logger = get_logger(__name__) @@ -370,7 +372,7 @@ def _lookup_raw_archive_state( return _RawArchiveLookup() columns = _columns(conn, "raw_sessions") select = ["raw_id"] if "raw_id" in columns else [] - for optional in ("parse_error", "validation_error", "validation_status"): + for optional in ("parse_error", "validation_error", "validation_status", "parsed_at_ms", "validated_at_ms"): if optional in columns: select.append(optional) if not select: @@ -404,15 +406,26 @@ def _lookup_raw_archive_state( validation_status = ( str(row["validation_status"]) if "validation_status" in row_keys and row["validation_status"] else None ) + validation_authority = raw_state_authority( + row["parsed_at_ms"] if "parsed_at_ms" in row_keys else None, + row["validated_at_ms"] if "validated_at_ms" in row_keys else None, + ) if isinstance(parse_error, str) and parse_error: latest_failure = parse_error failure_source = "raw_parse" - elif isinstance(validation_error, str) and validation_error: + elif validation_authority == "validation" and isinstance(validation_error, str) and validation_error: latest_failure = validation_error failure_source = "raw_validation" - elif validation_status is not None and validation_status not in {"passed", "valid", "ok"}: + elif ( + validation_authority == "validation" + and validation_status is not None + and validation_status not in {"passed", "valid", "ok"} + ): latest_failure = validation_status failure_source = "raw_validation" + elif validation_authority == "ambiguous" and validation_status not in {None, "passed", "valid", "ok"}: + latest_failure = "raw validation and parse timestamps are indeterminate" + failure_source = "raw_state_order" return _RawArchiveLookup( raw_row_exists=True, raw_id=str(row["raw_id"]) if "raw_id" in row_keys and row["raw_id"] is not None else None, @@ -432,7 +445,14 @@ def _lookup_index_archive_state( provider: str, provider_session_id: str, ) -> _IndexArchiveLookup: - conn = _open_readonly_sqlite(archive_root / "index.db") + try: + index_path = resolve_active_index_path(archive_root) + except ArchiveLocationError: + # Archive state is a best-effort capture acknowledgement. A malformed + # active-generation pointer must not turn a receiver GET into a 500 or + # make us consult the conventional shadow index instead. + return _IndexArchiveLookup() + conn = _open_readonly_sqlite(index_path) if conn is None: return _IndexArchiveLookup() try: diff --git a/polylogue/config.py b/polylogue/config.py index cf215e96f1..3991317a3e 100644 --- a/polylogue/config.py +++ b/polylogue/config.py @@ -103,6 +103,7 @@ class Config: # the defaults makes the spec honest about the optional surface. drive_config: DriveConfig | None = None index_config: IndexConfig | None = None + _db_path_explicit: bool = False embedding_model: str = "voyage-4-lite" embedding_dimension: int = 1024 judgment_automation_interval_s: int = 3600 @@ -122,6 +123,7 @@ def __init__( self.archive_root = archive_root self.render_root = render_root self.sources = sources + self._db_path_explicit = db_path is not None self.db_path = db_path if db_path is not None else resolve_active_index_path(archive_root) self.drive_config = drive_config self.index_config = index_config @@ -137,6 +139,13 @@ def __init__( if isinstance(judgment_automation_interval_s, bool) or not isinstance(judgment_automation_interval_s, int): raise ConfigError("Config.judgment_automation_interval_s must be an integer") + def current_db_path(self) -> Path: + """Resolve the current generation unless the caller pinned an override.""" + + if self._db_path_explicit: + return self.db_path + return resolve_active_index_path(self.archive_root) + def __eq__(self, other: object) -> bool: if not isinstance(other, Config): return NotImplemented @@ -166,7 +175,7 @@ def with_sources(self, sources: list[Source]) -> Config: archive_root=self.archive_root, render_root=self.render_root, sources=sources, - db_path=self.db_path, + db_path=self.db_path if self._db_path_explicit else None, drive_config=self.drive_config, index_config=self.index_config, embedding_model=self.embedding_model, diff --git a/polylogue/core/raw_coordinates.py b/polylogue/core/raw_coordinates.py new file mode 100644 index 0000000000..9d33b1c782 --- /dev/null +++ b/polylogue/core/raw_coordinates.py @@ -0,0 +1,73 @@ +"""Stable coordinates for raw payloads acquired from container members.""" + +from __future__ import annotations + +from hashlib import sha256 +from math import isqrt + +_ZIP_MEMBER_RAW_ID_DOMAIN = b"polylogue:zip-member-raw:v2\0" + + +def zip_member_source_index(*, entry_ordinal: int, split_index: int) -> int: + """Encode a ZIP entry ordinal and within-entry split index losslessly.""" + if entry_ordinal < 0 or split_index < 0: + raise ValueError("ZIP entry ordinal and split index must be non-negative") + diagonal = entry_ordinal + split_index + return diagonal * (diagonal + 1) // 2 + split_index + + +def zip_member_source_coordinate(source_index: int) -> tuple[int, int]: + """Recover the independent entry ordinal and split index from storage.""" + if source_index < 0: + raise ValueError("ZIP member source index must be non-negative") + diagonal = (isqrt(8 * source_index + 1) - 1) // 2 + diagonal_start = diagonal * (diagonal + 1) // 2 + split_index = source_index - diagonal_start + entry_ordinal = diagonal - split_index + return entry_ordinal, split_index + + +def zip_member_raw_id( + *, + source_path: str, + entry_ordinal: int, + split_index: int, + blob_hash: str, +) -> str: + """Identify one ZIP coordinate without giving up blob-level deduplication.""" + digest = sha256() + digest.update(_ZIP_MEMBER_RAW_ID_DOMAIN) + digest.update(source_path.encode("utf-8", errors="surrogatepass")) + digest.update(b"\0") + digest.update(str(entry_ordinal).encode("utf-8")) + digest.update(b"\0") + digest.update(str(split_index).encode("utf-8")) + digest.update(b"\0") + digest.update(bytes.fromhex(blob_hash)) + return digest.hexdigest() + + +def zip_member_identity_coordinate( + *, + raw_id: str, + source_path: str, + source_index: int, + blob_hash: str, +) -> tuple[int, int] | None: + """Decode a v2 raw identity, rejecting legacy or unrelated coordinates.""" + entry_ordinal, split_index = zip_member_source_coordinate(source_index) + expected_raw_id = zip_member_raw_id( + source_path=source_path, + entry_ordinal=entry_ordinal, + split_index=split_index, + blob_hash=blob_hash, + ) + return (entry_ordinal, split_index) if raw_id == expected_raw_id else None + + +__all__ = [ + "zip_member_identity_coordinate", + "zip_member_raw_id", + "zip_member_source_coordinate", + "zip_member_source_index", +] diff --git a/polylogue/core/raw_state.py b/polylogue/core/raw_state.py new file mode 100644 index 0000000000..a02bf16022 --- /dev/null +++ b/polylogue/core/raw_state.py @@ -0,0 +1,32 @@ +"""Ordering authority for durable raw parse and validation transitions.""" + +from __future__ import annotations + +from typing import Literal, TypeAlias + +RawStateAuthority: TypeAlias = Literal["parse", "validation", "ambiguous"] + + +def raw_state_authority( + parsed_at_ms: int | None, + validated_at_ms: int | None, +) -> RawStateAuthority: + """Return the proven terminal transition, never assigning equal times. + + New writes make opposing transitions strictly monotonic. Existing rows can + predate that invariant, so an equal non-null pair remains explicitly + indeterminate rather than being silently attributed to either stage. + + A pair with both values ``None`` reports ``"validation"``. These legacy + rows carry no ordering evidence, so callers retain the stored validation + verdict instead of inventing parse authority. + """ + if parsed_at_ms is None: + return "validation" + if validated_at_ms is None: + return "parse" + if parsed_at_ms > validated_at_ms: + return "parse" + if validated_at_ms > parsed_at_ms: + return "validation" + return "ambiguous" diff --git a/polylogue/daemon/cli.py b/polylogue/daemon/cli.py index 753bab6703..f849bccac5 100644 --- a/polylogue/daemon/cli.py +++ b/polylogue/daemon/cli.py @@ -79,7 +79,7 @@ from polylogue.config import Config from polylogue.daemon.lifecycle import DaemonLifecycle from polylogue.daemon.parse_prefetch import DaemonParseStage - from polylogue.product.raw_authority import RawMaterializationCounts + from polylogue.product.raw_authority import ArchiveWriterRebuildExclusion, RawMaterializationCounts from polylogue.sources.revision_backfill import RawParsePrefetchCache from polylogue.storage.blob_publication import BlobPublicationReconciliation @@ -1277,60 +1277,81 @@ def _drain_raw_materialization_once( from polylogue.storage.blob_integrity import restore_direct_blob_reference_debt archive = archive_root() - restored = restore_direct_blob_reference_debt( - archive / "source.db", - dry_run=False, - max_count=_BLOB_REFERENCE_RESTORE_CONVERGENCE_BATCH_LIMIT, - sample_size=0, - ) - if restored.restored_count: - logger.info( - "blob references: restored %d direct source blob(s) before raw materialization", - restored.restored_count, - ) - config = Config( archive_root=archive, render_root=render_root(), sources=[], ) - if recover: - raw_authority.recover_interrupted_frontier(config) - # polylogue-d7im: a stale-plan blocker requires no operator judgment (it - # is a pure TOCTOU race between a census and its apply, already - # recomputed unattended in the crash-recovery path above) but, left - # unresolved, unresolved_raw_replay_blockers makes repair_materialization - # below fail closed for the WHOLE archive, not just the affected raw. - # Clear these automatically before every pass instead of waiting for a - # manual raw-authority-blocker-resolve invocation. - auto_resolved = raw_authority.auto_resolve_stale_plan_blockers(config) - if auto_resolved: - logger.info( - "raw authority: auto-resolved %d stale-plan blocker(s) before raw materialization", - auto_resolved, - ) - try: - result = raw_authority.repair_materialization( - config, - dry_run=False, - raw_artifact_limit=limit, - max_payload_bytes=_RAW_MATERIALIZATION_DAEMON_BLOB_LIMIT_BYTES, - prefetch_cache=prefetch_cache, - max_pass_seconds=_RAW_MATERIALIZATION_MAX_PASS_SECONDS, - ) - finally: - _close_raw_materialization_fts(config.archive_root / "index.db") + generation_pin_refused = False + frontier_repaired = 0 + with contextlib.ExitStack() as lease_stack: + try: + index_db = lease_stack.enter_context(raw_authority.materialization_generation_lease(config)) + except Exception as exc: + refused_result = raw_authority.materialization_lease_refusal_result(exc) + if refused_result is None: + raise + result = refused_result + generation_pin_refused = True + else: + restored = restore_direct_blob_reference_debt( + archive / "source.db", + dry_run=False, + max_count=_BLOB_REFERENCE_RESTORE_CONVERGENCE_BATCH_LIMIT, + sample_size=0, + ) + if restored.restored_count: + logger.info( + "blob references: restored %d direct source blob(s) before raw materialization", + restored.restored_count, + ) + if recover: + raw_authority.recover_interrupted_frontier(config) + # polylogue-d7im: a stale-plan blocker requires no operator + # judgment. Recovery, stale-plan resolution, repair, FTS closure, + # and frontier apply all consume the selected index generation, + # so the one promotion-excluding lease must cover the complete + # sequence rather than only the middle repair call. + auto_resolved = raw_authority.auto_resolve_stale_plan_blockers(config) + if auto_resolved: + logger.info( + "raw authority: auto-resolved %d stale-plan blocker(s) before raw materialization", + auto_resolved, + ) + try: + result = raw_authority.repair_materialization( + config, + dry_run=False, + raw_artifact_limit=limit, + max_payload_bytes=_RAW_MATERIALIZATION_DAEMON_BLOB_LIMIT_BYTES, + prefetch_cache=prefetch_cache, + max_pass_seconds=_RAW_MATERIALIZATION_MAX_PASS_SECONDS, + ) + finally: + _close_raw_materialization_fts(index_db, ops_db_path=config.archive_root / "ops.db") + frontier_repaired = _converge_raw_authority_frontier(config, limit=min(limit, 8)) + if generation_pin_refused: + _emit_raw_materialization_pass(result) + if not result.success: + logger.warning("raw materialization: bounded convergence incomplete: %s", result.detail) + return _raw_materialization_counts(result) _emit_raw_materialization_pass(result) - frontier_repaired = _converge_raw_authority_frontier(config, limit=min(limit, 8)) if not result.success: logger.warning("raw materialization: bounded convergence incomplete: %s", result.detail) + return _raw_materialization_counts(result, executed_plans=frontier_repaired) + + +def _raw_materialization_counts(result: Any, *, executed_plans: int = 0) -> RawMaterializationCounts: + """Project one typed raw-materialization result into daemon scheduling counts.""" + from polylogue.product import raw_authority + metrics = dict(getattr(result, "metrics", {})) remaining = int(metrics.get("raw_materialization_remaining_candidate_count", 0)) if remaining == 0: remaining = int(metrics.get("raw_materialization_census_incomplete_raw_count", 0)) return raw_authority.RawMaterializationCounts( repaired_sessions=result.repaired_count, - executed_plans=frontier_repaired, + executed_plans=executed_plans, remaining_candidates=remaining, censused_components=int(metrics.get("raw_materialization_census_components_attempted", 0)), candidate_count=int(metrics.get("raw_materialization_candidate_count", 0)), @@ -1378,16 +1399,25 @@ def _run_raw_materialization_whale_pass_once(*, raw_artifact_id: str, max_payloa archive = archive_root() config = Config(archive_root=archive, render_root=render_root(), sources=[]) - try: - result = raw_authority.repair_materialization( - config, - dry_run=False, - raw_artifact_limit=1, - max_payload_bytes=max_payload_bytes, - raw_artifact_id=raw_artifact_id, - ) - finally: - _close_raw_materialization_fts(config.archive_root / "index.db") + with contextlib.ExitStack() as lease_stack: + try: + index_db = lease_stack.enter_context(raw_authority.materialization_generation_lease(config)) + except Exception as exc: + refused_result = raw_authority.materialization_lease_refusal_result(exc) + if refused_result is None: + raise + result = refused_result + else: + try: + result = raw_authority.repair_materialization( + config, + dry_run=False, + raw_artifact_limit=1, + max_payload_bytes=max_payload_bytes, + raw_artifact_id=raw_artifact_id, + ) + finally: + _close_raw_materialization_fts(index_db, ops_db_path=config.archive_root / "ops.db") _emit_raw_materialization_pass(result) if not result.success: logger.warning("raw materialization: whale pass for %s incomplete: %s", raw_artifact_id, result.detail) @@ -1606,7 +1636,7 @@ def _emit_raw_materialization_pass(result: Any) -> None: ) -def _close_raw_materialization_fts(index_db: Path) -> None: +def _close_raw_materialization_fts(index_db: Path, *, ops_db_path: Path) -> None: """Return message search to ready or leave explicit retryable debt. Large raw replay batches deliberately suspend FTS triggers and may skip @@ -1620,7 +1650,9 @@ def _close_raw_materialization_fts(index_db: Path) -> None: try: needs_repair = _raw_materialization_fts_needs_repair(index_db) except Exception as exc: - _record_raw_materialization_fts_debt(index_db, f"FTS readiness probe failed after raw materialization: {exc}") + _record_raw_materialization_fts_debt( + index_db, ops_db_path=ops_db_path, error=f"FTS readiness probe failed after raw materialization: {exc}" + ) return if not needs_repair: return @@ -1632,14 +1664,15 @@ def _close_raw_materialization_fts(index_db: Path) -> None: # this closure retryable instead of masking the initiating failure. _record_raw_materialization_fts_debt( index_db, - f"FTS repair failed after raw materialization: {type(exc).__name__}: {exc}", + ops_db_path=ops_db_path, + error=f"FTS repair failed after raw materialization: {type(exc).__name__}: {exc}", ) return if repaired: try: from polylogue.sources.live.cursor import CursorStore - CursorStore(index_db).clear_convergence_debt( + CursorStore(index_db, ops_db_path=ops_db_path).clear_convergence_debt( subject_type="fts_surface", subject_id="messages_fts", stage="fts", @@ -1649,15 +1682,16 @@ def _close_raw_materialization_fts(index_db: Path) -> None: return _record_raw_materialization_fts_debt( index_db, - "raw materialization exited without restoring message FTS readiness", + ops_db_path=ops_db_path, + error="raw materialization exited without restoring message FTS readiness", ) -def _record_raw_materialization_fts_debt(index_db: Path, error: str) -> None: +def _record_raw_materialization_fts_debt(index_db: Path, *, ops_db_path: Path, error: str) -> None: from polylogue.sources.live.cursor import CursorStore try: - CursorStore(index_db).record_convergence_debt( + CursorStore(index_db, ops_db_path=ops_db_path).record_convergence_debt( stage="fts", subject_type="fts_surface", subject_id="messages_fts", @@ -2069,26 +2103,71 @@ async def _emit_daemon_lifecycle_event( logger.warning("daemon: failed to emit lifecycle event %s", phase, exc_info=True) +def _retain_rebuild_exclusion_for_undrained_writer( + rebuild_exclusion: ArchiveWriterRebuildExclusion, + *, + writer_drained: bool, +) -> None: + """Transfer rebuild exclusion to process lifetime after a drain timeout.""" + if not writer_drained: + rebuild_exclusion.retain_until_process_exit() + + +async def _shutdown_writer_coordinator_with_rebuild_exclusion( + coordinator: DaemonWriteCoordinator, + rebuild_exclusion: ArchiveWriterRebuildExclusion, + *, + timeout: float, +) -> bool: + """Drain writers or retain rebuild exclusion when drain cannot be proven.""" + try: + writer_drained = await coordinator.shutdown(timeout=timeout) + except BaseException: + rebuild_exclusion.retain_until_process_exit() + raise + _retain_rebuild_exclusion_for_undrained_writer( + rebuild_exclusion, + writer_drained=writer_drained, + ) + return writer_drained + + async def run_live_watcher( *, sources: tuple[WatchSource, ...], debounce_s: float, ) -> None: from polylogue.daemon.events import emit_catch_up_cycle + from polylogue.paths import archive_root + from polylogue.product.raw_authority import archive_writer_rebuild_exclusion - async with Polylogue() as polylogue: - watcher = LiveWatcher( - polylogue, - sources, - debounce_s=debounce_s, - event_emitter=_emit_live_batch_event, - catch_up_event_emitter=emit_catch_up_cycle, - write_coordinator=daemon_write_coordinator(), - ) + archive_root_path = Path(archive_root()) + archive_root_path.mkdir(mode=0o700, parents=True, exist_ok=True) + with archive_writer_rebuild_exclusion(archive_root_path) as rebuild_exclusion: + coordinator = daemon_write_coordinator() + watcher: LiveWatcher | None = None try: - await watcher.run() - except KeyboardInterrupt: - watcher.stop() + async with Polylogue() as polylogue: + watcher = LiveWatcher( + polylogue, + sources, + debounce_s=debounce_s, + event_emitter=_emit_live_batch_event, + catch_up_event_emitter=emit_catch_up_cycle, + write_coordinator=coordinator, + ) + with contextlib.suppress(KeyboardInterrupt): + await watcher.run() + finally: + try: + if watcher is not None: + watcher.stop() + finally: + await _shutdown_writer_coordinator_with_rebuild_exclusion( + coordinator, + rebuild_exclusion, + timeout=5.0, + ) async def run_daemon_services( @@ -2110,6 +2189,64 @@ async def run_daemon_services( api_port: int = 8766, api_auth_token: str | None = None, api_allow_no_auth: bool = False, +) -> None: + """Run the daemon while excluding every offline index rebuild. + + The lease is intentionally process-lifetime authority rather than a + per-maintenance-call guard. Startup readiness, reservation recovery, + live acquisition, and periodic convergence all mutate source or index + state; an offline rebuild must therefore refuse the daemon before any of + those routes can run, and the daemon must prevent a rebuild from starting + until its writer coordinator has drained. + """ + from polylogue.paths import archive_root + from polylogue.product.raw_authority import archive_writer_rebuild_exclusion + + archive_root_path = Path(archive_root()) + archive_root_path.mkdir(mode=0o700, parents=True, exist_ok=True) + with archive_writer_rebuild_exclusion(archive_root_path) as rebuild_exclusion: + await _run_daemon_services_under_active_writer_lease( + rebuild_exclusion=rebuild_exclusion, + sources=sources, + debounce_s=debounce_s, + enable_watch=enable_watch, + enable_source_catchup=enable_source_catchup, + enable_browser_capture=enable_browser_capture, + browser_capture_host=browser_capture_host, + browser_capture_port=browser_capture_port, + browser_capture_spool_path=browser_capture_spool_path, + browser_capture_allow_remote=browser_capture_allow_remote, + browser_capture_auth_token=browser_capture_auth_token, + browser_capture_allow_no_auth=browser_capture_allow_no_auth, + browser_capture_extra_origins=browser_capture_extra_origins, + enable_api=enable_api, + api_host=api_host, + api_port=api_port, + api_auth_token=api_auth_token, + api_allow_no_auth=api_allow_no_auth, + ) + + +async def _run_daemon_services_under_active_writer_lease( + *, + rebuild_exclusion: ArchiveWriterRebuildExclusion, + sources: tuple[WatchSource, ...], + debounce_s: float, + enable_watch: bool, + enable_source_catchup: bool = True, + enable_browser_capture: bool, + browser_capture_host: str, + browser_capture_port: int, + browser_capture_spool_path: Path | None, + browser_capture_allow_remote: bool = False, + browser_capture_auth_token: str | None = None, + browser_capture_allow_no_auth: bool = False, + browser_capture_extra_origins: tuple[str, ...] = (), + enable_api: bool = False, + api_host: str = "127.0.0.1", + api_port: int = 8766, + api_auth_token: str | None = None, + api_allow_no_auth: bool = False, ) -> None: """Run configured daemon components until interrupted.""" from polylogue.daemon import process_start as _process_start @@ -2317,7 +2454,11 @@ async def run_daemon_services( if lifecycle is not None: with contextlib.suppress(Exception): await write_coordinator.run_sync("daemon.lifecycle.stop", lifecycle.stop, exit_kind="error") - writer_drained = await write_coordinator.shutdown(timeout=5.0) + writer_drained = await _shutdown_writer_coordinator_with_rebuild_exclusion( + write_coordinator, + rebuild_exclusion, + timeout=5.0, + ) _release_pidfile_after_writer_drain(pidfile_fd, writer_drained=writer_drained) if writer_drained: archive_owner.release() @@ -2352,7 +2493,11 @@ async def run_daemon_services( if lifecycle is not None: with contextlib.suppress(Exception): await write_coordinator.run_sync("daemon.lifecycle.stop", lifecycle.stop, exit_kind="error") - writer_drained = await write_coordinator.shutdown(timeout=5.0) + writer_drained = await _shutdown_writer_coordinator_with_rebuild_exclusion( + write_coordinator, + rebuild_exclusion, + timeout=5.0, + ) _release_pidfile_after_writer_drain(pidfile_fd, writer_drained=writer_drained) if writer_drained: archive_owner.release() @@ -2713,9 +2858,22 @@ async def run_daemon_services( except Exception: logger.warning("daemon: could not persist final lifecycle stop", exc_info=True) - writer_drained = await write_coordinator.shutdown(timeout=5.0) + writer_drained = await _shutdown_writer_coordinator_with_rebuild_exclusion( + write_coordinator, + rebuild_exclusion, + timeout=5.0, + ) pidfile_fd = _release_pidfile_after_writer_drain(pidfile_fd, writer_drained=writer_drained) finally: + # Any exception or repeated cancellation before coordinator + # shutdown leaves writer drain unproven. The outer product + # context must not interpret that control-flow escape as a safe + # release: keep rebuild exclusion until process exit unless the + # coordinator returned an affirmative drain result. + _retain_rebuild_exclusion_for_undrained_writer( + rebuild_exclusion, + writer_drained=writer_drained, + ) if server is not None: with contextlib.suppress(Exception): server.server_close() diff --git a/polylogue/daemon/convergence_stages.py b/polylogue/daemon/convergence_stages.py index a511de244c..095bf84567 100644 --- a/polylogue/daemon/convergence_stages.py +++ b/polylogue/daemon/convergence_stages.py @@ -926,7 +926,14 @@ def _raw_parse_recovery_pending_count(db_path: Path, path: Path, *, archive_root FROM raw_sessions AS r {materialized_join} WHERE (r.source_path = ? OR r.source_path LIKE ?) - AND COALESCE(r.validation_status, '') != 'failed' + AND NOT ( + COALESCE(r.validation_status, '') = 'failed' + AND ( + r.parsed_at_ms IS NULL + OR r.validated_at_ms IS NULL + OR r.validated_at_ms >= r.parsed_at_ms + ) + ) AND ( ( r.parsed_at_ms IS NULL diff --git a/polylogue/daemon/provenance.py b/polylogue/daemon/provenance.py index 407a6dca9d..08220adec4 100644 --- a/polylogue/daemon/provenance.py +++ b/polylogue/daemon/provenance.py @@ -34,6 +34,7 @@ from pathlib import Path from typing import Final +from polylogue.core.raw_state import raw_state_authority from polylogue.logging import get_logger from polylogue.paths import archive_root from polylogue.storage.archive_identity import resolve_active_index_path @@ -62,8 +63,10 @@ class ProvenanceRow: acquired_at: str | None file_mtime: str | None parsed_at: str | None + parsed_at_ms: int | None parse_error: str | None validated_at: str | None + validated_at_ms: int | None validation_status: str | None validation_error: str | None @@ -185,8 +188,10 @@ def _fetch_archive_provenance_row( acquired_at=_iso_from_epoch_ms(row["acquired_at_ms"]), file_mtime=_iso_from_epoch_ms(row["file_mtime_ms"]), parsed_at=_iso_from_epoch_ms(row["parsed_at_ms"]), + parsed_at_ms=(int(row["parsed_at_ms"]) if row["parsed_at_ms"] is not None else None), parse_error=(str(row["parse_error"]) if row["parse_error"] is not None else None), validated_at=_iso_from_epoch_ms(row["validated_at_ms"]), + validated_at_ms=(int(row["validated_at_ms"]) if row["validated_at_ms"] is not None else None), validation_status=(str(row["validation_status"]) if row["validation_status"] is not None else None), validation_error=(str(row["validation_error"]) if row["validation_error"] is not None else None), ) @@ -253,8 +258,10 @@ def fetch_provenance_row(session_id: str) -> ProvenanceRow | None: acquired_at=(str(row["acquired_at"]) if row["acquired_at"] is not None else None), file_mtime=(str(row["file_mtime"]) if row["file_mtime"] is not None else None), parsed_at=(str(row["parsed_at"]) if row["parsed_at"] is not None else None), + parsed_at_ms=None, parse_error=(str(row["parse_error"]) if row["parse_error"] is not None else None), validated_at=(str(row["validated_at"]) if row["validated_at"] is not None else None), + validated_at_ms=None, validation_status=(str(row["validation_status"]) if row["validation_status"] is not None else None), validation_error=(str(row["validation_error"]) if row["validation_error"] is not None else None), ) @@ -273,7 +280,11 @@ def _quarantine_state(row: ProvenanceRow) -> tuple[bool, str | None]: if row.parse_error: return True, "parse_error" if row.validation_status == "failed": - return True, "validation_failed" + validation_authority = raw_state_authority(row.parsed_at_ms, row.validated_at_ms) + if validation_authority == "validation": + return True, "validation_failed" + if validation_authority == "ambiguous": + return True, "validation_parse_order_ambiguous" return False, None diff --git a/polylogue/product/raw_authority.py b/polylogue/product/raw_authority.py index a865e6ca7f..cae001979a 100644 --- a/polylogue/product/raw_authority.py +++ b/polylogue/product/raw_authority.py @@ -7,16 +7,19 @@ from __future__ import annotations +import contextlib +from collections.abc import Iterator from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING, Any, Final -from polylogue.config import Config +from polylogue.config import Config, active_archive_root from polylogue.core.json import JSONDocument if TYPE_CHECKING: from polylogue.sources.revision_backfill import RawParsePrefetchCache from polylogue.storage.raw_reconciler import RawAuthorityFrontierApplyReport, RawAuthorityFrontierCensus + from polylogue.storage.repair import RepairResult RAW_MATERIALIZATION_ORDINARY_BLOB_LIMIT_BYTES: Final = 64 * 1024 * 1024 @@ -119,6 +122,71 @@ def auto_resolve_stale_plan_blockers(config: Config) -> int: return _auto_resolve(config.archive_root) +@contextlib.contextmanager +def materialization_generation_lease(config: Config) -> Iterator[Path]: + """Pin one active index generation through a replay-adjacent closure.""" + from polylogue.storage.index_generation import ActiveWriterLease + + lease = ActiveWriterLease(active_archive_root(config)) + lease.acquire() + try: + yield config.current_db_path() + finally: + lease.close() + + +class ArchiveWriterRebuildExclusion: + """Product authority preventing an offline rebuild from overlapping a writer.""" + + def __init__(self, archive_root: Path) -> None: + from polylogue.storage.index_generation import ActiveWriterLease + + self._lease = ActiveWriterLease(archive_root) + self._retained_until_process_exit = False + self._lease.acquire() + + def retain_until_process_exit(self) -> None: + """Keep exclusion when a writer cannot be proven drained. + + The raw file descriptor deliberately remains open and is reclaimed by + the OS at process exit. Releasing it after a bounded shutdown timeout + would let an offline rebuild overlap the admitted writer that caused + that timeout. + """ + self._retained_until_process_exit = True + + def release(self) -> None: + """Release exclusion after every admitted writer is proven drained.""" + self._lease.close() + self._retained_until_process_exit = False + + def release_if_safe(self) -> None: + """Release unless shutdown transferred authority to process lifetime.""" + if not self._retained_until_process_exit: + self.release() + + +@contextlib.contextmanager +def archive_writer_rebuild_exclusion(archive_root: Path) -> Iterator[ArchiveWriterRebuildExclusion]: + """Acquire process-lifetime-capable rebuild exclusion for an archive writer.""" + exclusion = ArchiveWriterRebuildExclusion(archive_root) + try: + yield exclusion + finally: + exclusion.release_if_safe() + + +def materialization_lease_refusal_result(error: BaseException) -> RepairResult | None: + """Translate only a rebuild-lease refusal into raw repair's typed result.""" + from polylogue.storage.index_generation import RebuildLeaseUnavailableError + + if not isinstance(error, RebuildLeaseUnavailableError): + return None + from polylogue.storage.repair import raw_materialization_lease_refusal_result + + return raw_materialization_lease_refusal_result(error) + + def repair_materialization( config: Config, *, @@ -218,10 +286,14 @@ def list_blockers(archive_root: Path, *, limit: int = 100, offset: int = 0) -> J __all__ = [ + "ArchiveWriterRebuildExclusion", "RawMaterializationCounts", "apply_frontier", + "archive_writer_rebuild_exclusion", "inspect_frontier", "list_blockers", + "materialization_generation_lease", + "materialization_lease_refusal_result", "read_census", "read_detail", "recover_interrupted_frontier", diff --git a/polylogue/schemas/sampling_db.py b/polylogue/schemas/sampling_db.py index f072789dfb..9c45235d60 100644 --- a/polylogue/schemas/sampling_db.py +++ b/polylogue/schemas/sampling_db.py @@ -22,6 +22,7 @@ canonical_runtime_provider, canonical_schema_provider, ) +from polylogue.core.raw_state import raw_state_authority from polylogue.core.sources import origin_from_provider, provider_from_origin from polylogue.logging import get_logger from polylogue.paths import db_path as index_db_path @@ -39,6 +40,7 @@ from polylogue.storage.blob_store import get_blob_store from polylogue.storage.introspection import table_exists from polylogue.storage.sqlite.connection_profile import connection_context +from polylogue.storage.sqlite.queries.raw_state import raw_provider_origin_sql logger = get_logger(__name__) @@ -70,14 +72,19 @@ def _ms_to_iso(value: object) -> str | None: class _RawSessionRow: source_path: str | None origin: str + detected_provider: str | None raw_id: str blob_hash: bytes file_mtime_ms: int | None acquired_at_ms: int | None + parsed_at_ms: int | None + validated_at_ms: int | None validation_status: str | None @property def provider_token(self) -> str: + if self.detected_provider is not None: + return Provider.from_string(self.detected_provider).value try: return provider_from_origin(Origin.from_string(self.origin)).value except (ValueError, KeyError): @@ -106,17 +113,20 @@ def _sample_origins_for_provider(source_name: Provider, config: ProviderConfig) def _sample_provider_where_clause(source_name: str | Provider) -> tuple[str, tuple[str, ...]]: provider = Provider.from_string(source_name) origin = origin_from_provider(provider).value - return "origin = ?", (origin,) + return f"{raw_provider_origin_sql()} = ?", (origin,) def _coerce_schema_row(row: sqlite3.Row) -> _RawSessionRow: return _RawSessionRow( source_path=row["source_path"], origin=str(row["origin"]), + detected_provider=(str(row["detected_provider"]) if row["detected_provider"] is not None else None), raw_id=str(row["raw_id"]), blob_hash=bytes(row["blob_hash"]) if row["blob_hash"] is not None else b"", file_mtime_ms=row["file_mtime_ms"], acquired_at_ms=row["acquired_at_ms"], + parsed_at_ms=row["parsed_at_ms"], + validated_at_ms=row["validated_at_ms"], validation_status=row["validation_status"], ) @@ -291,6 +301,7 @@ def _iter_schema_units_from_db( query_provider = config.db_source_name or source_name origins = _sample_origins_for_provider(Provider.from_string(query_provider), config) placeholders = ",".join("?" for _ in origins) + effective_origin = raw_provider_origin_sql(table_alias="raw_sessions") with connection_context(source_db_path) as conn: conn.row_factory = sqlite3.Row if logical_heads_only: @@ -302,24 +313,26 @@ def _iter_schema_units_from_db( query = f""" WITH heads AS ( SELECT - source_path, origin, raw_id, blob_hash, file_mtime_ms, acquired_at_ms, - validation_status, + source_path, origin, detected_provider, raw_id, blob_hash, file_mtime_ms, + acquired_at_ms, parsed_at_ms, + validated_at_ms, validation_status, ROW_NUMBER() OVER ( - PARTITION BY origin, {logical_cohort_expr} + PARTITION BY {effective_origin}, {logical_cohort_expr} ORDER BY acquired_at_ms DESC, raw_id DESC ) AS rn FROM raw_sessions - WHERE origin IN ({placeholders}) + WHERE {effective_origin} IN ({placeholders}) ) - SELECT source_path, origin, raw_id, blob_hash, file_mtime_ms, acquired_at_ms, validation_status + SELECT source_path, origin, detected_provider, raw_id, blob_hash, file_mtime_ms, + acquired_at_ms, parsed_at_ms, validated_at_ms, validation_status FROM heads WHERE rn = 1 """ else: query = f""" - SELECT source_path, origin, raw_id, blob_hash, file_mtime_ms, acquired_at_ms, - validation_status + SELECT source_path, origin, detected_provider, raw_id, blob_hash, file_mtime_ms, + acquired_at_ms, parsed_at_ms, validated_at_ms, validation_status FROM raw_sessions - WHERE origin IN ({placeholders}) + WHERE {effective_origin} IN ({placeholders}) """ cursor = conn.execute(query, origins) batch_size = 1 if config.sample_granularity == "record" else 100 @@ -367,12 +380,17 @@ def _iter_schema_units_from_db( ) continue - if row.validation_status == "failed": + validation_authority = raw_state_authority(row.parsed_at_ms, row.validated_at_ms) + if row.validation_status == "failed" and validation_authority != "parse": _record_terminal( terminal_recorder, row, status="quarantined", - reason="source_validation_failed", + reason=( + "source_validation_failed" + if validation_authority == "validation" + else "source_validation_parse_order_ambiguous" + ), ) continue diff --git a/polylogue/schemas/validation/corpus.py b/polylogue/schemas/validation/corpus.py index 50f0137bd2..b39b01f2c2 100644 --- a/polylogue/schemas/validation/corpus.py +++ b/polylogue/schemas/validation/corpus.py @@ -10,11 +10,13 @@ from polylogue.archive.raw_payload import RawPayloadEnvelope, build_raw_payload_envelope from polylogue.core.common import format_malformed_jsonl_error as _format_malformed_jsonl_error -from polylogue.core.enums import Origin, Provider +from polylogue.core.enums import Origin, Provider, ValidationMode, ValidationStatus from polylogue.core.sources import origin_from_provider, provider_from_origin from polylogue.schemas.validator import SchemaValidator from polylogue.storage.blob_store import get_blob_store +from polylogue.storage.raw.models import RawSessionStateUpdate from polylogue.storage.sqlite.connection_profile import open_connection +from polylogue.storage.sqlite.raw_state_update import compile_raw_state_update from .models import ProviderSchemaVerification, SchemaVerificationReport from .requests import SchemaVerificationRequest, bounded_window @@ -31,23 +33,30 @@ def verification_provider_clause(providers: list[str]) -> tuple[str, tuple[str, ...]]: - """Build a `raw_sessions.origin` filter for requested providers. + """Build a detected-provider-aware filter for requested providers. - raw rows carry a single ``origin`` token rather than the - legacy ``payload_provider`` / ``source_name`` pair. Each requested - provider token is mapped to its archive origin via - :func:`origin_from_provider`; the row matches when its ``origin`` is in - that set. + Parser classification outranks acquisition origin once present. Rows that + have not been classified retain the origin fallback used by older source + schemas and ordinary provider-owned acquisition. """ - origins = [origin_from_provider(Provider.from_string(p)).value for p in providers] - placeholders = ",".join("?" for _ in origins) - clause = f"origin IN ({placeholders})" - return clause, tuple(origins) + provider_tokens = [Provider.from_string(provider).value for provider in providers] + origins = [origin_from_provider(Provider.from_string(provider)).value for provider in providers] + provider_placeholders = ",".join("?" for _ in provider_tokens) + origin_placeholders = ",".join("?" for _ in origins) + clause = ( + f"(detected_provider IN ({provider_placeholders}) OR " + f"(detected_provider IS NULL AND origin IN ({origin_placeholders})))" + ) + return clause, (*provider_tokens, *origins) def _row_payload_data(row: sqlite3.Row) -> VerificationRow: - origin = str(row["origin"]) - provider = provider_from_origin(Origin.from_string(origin), family_hint=Provider.DRIVE).value + detected_provider = row["detected_provider"] + if detected_provider is not None: + provider = Provider.from_string(str(detected_provider)).value + else: + origin = str(row["origin"]) + provider = provider_from_origin(Origin.from_string(origin), family_hint=Provider.DRIVE).value return ( str(row["raw_id"]), provider, @@ -95,7 +104,7 @@ def rows() -> Iterator[sqlite3.Row]: return last_rowid = row[0] - base_query = "SELECT rowid, raw_id, origin, source_path, blob_hash FROM raw_sessions " + base_query = "SELECT rowid, raw_id, origin, detected_provider, source_path, blob_hash FROM raw_sessions " records_fetched = 0 while True: if bounded_limit is not None: @@ -201,17 +210,18 @@ def apply_quarantine_updates( """ validated_at_ms = int(datetime.now(tz=timezone.utc).timestamp() * 1000) for raw_id, reason, _provider, _payload_provider in updates: + set_clauses, params = compile_raw_state_update( + RawSessionStateUpdate( + validation_status=ValidationStatus.FAILED, + validation_error=reason, + validation_drift_count=0, + validation_mode=ValidationMode.STRICT, + ), + now_ms=validated_at_ms, + ) conn.execute( - """ - UPDATE raw_sessions - SET validation_status = 'failed', - validation_error = ?, - validation_drift_count = 0, - validation_mode = 'strict', - validated_at_ms = ? - WHERE raw_id = ? - """, - (reason, validated_at_ms, raw_id), + f"UPDATE raw_sessions SET {', '.join(set_clauses)} WHERE raw_id = ?", + (*params, raw_id), ) conn.execute( """ diff --git a/polylogue/sources/codex_state_evidence.py b/polylogue/sources/codex_state_evidence.py new file mode 100644 index 0000000000..fc78152527 --- /dev/null +++ b/polylogue/sources/codex_state_evidence.py @@ -0,0 +1,81 @@ +"""Durable session-linked evidence derived from retained Codex state.""" + +from __future__ import annotations + +from json import dumps as json_dumps +from typing import Any + +from polylogue.core.enums import Origin, Provider +from polylogue.sources.parsers import codex_state + + +def write_codex_thread_state_evidence( + archive: Any, + snapshot: codex_state.CodexStateSnapshot, + *, + source_path: str, + acquired_at_ms: int, +) -> None: + """Attach state-db thread metadata to existing Codex sessions. + + The state database is evidence about sessions, never a session itself. + Both live ingest and retained-raw replay call this writer so a + source-only acquisition is completed from the immutable snapshot when + the derived tier returns. + """ + from polylogue.storage.sqlite.archive_tiers.source_write import ArchiveHookEvent + + for thread in snapshot.threads: + payload: dict[str, object] = { + "thread_id": thread.thread_id, + "title": thread.title, + "cwd": thread.cwd, + "source": thread.source, + "model": thread.model, + "agent_nickname": thread.agent_nickname, + "agent_role": thread.agent_role, + "archived": thread.archived, + } + encoded = json_dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") + archive.write_hook_event( + provider=Provider.CODEX, + payload=encoded, + source_path=source_path, + acquired_at_ms=acquired_at_ms, + hook_event=ArchiveHookEvent( + hook_event_id=f"codex-thread-title:{thread.thread_id}", + origin=Origin.CODEX_SESSION, + source_path=source_path, + event_type="codex_thread_title", + payload=payload, + observed_at_ms=thread.updated_at_ms or acquired_at_ms, + native_id=f"{thread.thread_id}:codex_thread_title", + session_native_id=thread.thread_id, + ), + ) + for edge in snapshot.spawn_edges: + payload = { + "parent_thread_id": edge.parent_thread_id, + "child_thread_id": edge.child_thread_id, + "status": edge.status, + } + encoded = json_dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") + archive.write_hook_event( + provider=Provider.CODEX, + payload=encoded, + source_path=source_path, + acquired_at_ms=acquired_at_ms, + hook_event=ArchiveHookEvent( + hook_event_id=f"codex-thread-spawn-edge:{edge.parent_thread_id}:{edge.child_thread_id}", + origin=Origin.CODEX_SESSION, + source_path=source_path, + event_type="codex_thread_spawn_edge", + payload=payload, + observed_at_ms=acquired_at_ms, + native_id=f"{edge.parent_thread_id}:{edge.child_thread_id}:codex_thread_spawn_edge", + session_native_id=edge.parent_thread_id, + ), + ) + + +__all__ = ["write_codex_thread_state_evidence"] diff --git a/polylogue/sources/hooks.py b/polylogue/sources/hooks.py index 4cffd7182a..88b916b643 100644 --- a/polylogue/sources/hooks.py +++ b/polylogue/sources/hooks.py @@ -297,8 +297,16 @@ def drain_hook_event_spool( acknowledged = 0 failed = 0 try: - initialize_active_archive_root(archive_root) - store = ArchiveStore.open_existing(archive_root, read_only=False) + # Import after this module has initialized: ``sources.live.__init__`` + # exposes the watcher, and the watcher imports this spool module. + from polylogue.sources.live.archive_open import ( + _open_archive_for_live_write, + _source_tier_acquisition_required, + ) + + if not _source_tier_acquisition_required(): + initialize_active_archive_root(archive_root) + store = _open_archive_for_live_write(archive_root) except (OSError, sqlite3.Error, ValueError): logger.warning("hook spool drain could not open the archive; all events remain pending", exc_info=True) return HookSpoolDrainResult( diff --git a/polylogue/sources/live/append_ingest.py b/polylogue/sources/live/append_ingest.py index 91fcb4e231..241b51262e 100644 --- a/polylogue/sources/live/append_ingest.py +++ b/polylogue/sources/live/append_ingest.py @@ -7,7 +7,7 @@ from datetime import UTC, datetime from io import BytesIO from pathlib import Path -from typing import Any, Protocol +from typing import Any, Protocol, cast from polylogue.archive.artifact_taxonomy import classify_artifact, classify_artifact_path from polylogue.archive.raw_payload.decode import _sample_jsonl_payload_with_detail, jsonl_session_artifact @@ -20,10 +20,11 @@ from polylogue.core.degraded import degraded_reason from polylogue.core.enums import Provider from polylogue.logging import get_logger -from polylogue.sources.live.archive_open import _open_archive_for_live_write +from polylogue.sources.live.archive_open import _open_archive_for_live_write, _source_tier_acquisition_required from polylogue.sources.live.batch_support import _AppendPlan, _AppendResult from polylogue.sources.live.cursor import CursorStore from polylogue.sources.live.sqlite_locking import is_transient_sqlite_lock +from polylogue.storage.archive_identity import resolve_active_index_path from polylogue.storage.raw.models import RawSessionStateUpdate from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root from polylogue.storage.sqlite.archive_tiers.raw_admission import RawAdmissionArm @@ -40,6 +41,70 @@ class _AppendIngestOwner(Protocol): _polylogue: Any +def _bind_append_revision( + archive: Any, + raw_id: str, + *, + provider: Provider, + session_id: str, + plan: _AppendPlan, +) -> tuple[str, RawRevisionAuthority]: + """Persist an APPEND envelope from the append plan's durable identity.""" + if plan.cursor_fingerprint is None: + raise ValueError("append payload did not prove cursor identity") + logical_source_key = f"{provider.value}:{session_id}" + parent = archive.raw_append_revision_parent( + logical_source_key, + plan.start_offset, + plan.cursor_fingerprint, + ) + predecessor_raw_id: str | None = None + baseline_raw_id: str | None = None + generation = archive.raw_full_revision_generation(logical_source_key) + authority = RawRevisionAuthority.QUARANTINED + if parent is not None: + predecessor_raw_id, baseline_raw_id, generation = parent + authority = RawRevisionAuthority.BYTE_PROVEN + archive.bind_raw_revision( + raw_id, + RawRevisionEnvelope( + logical_source_key=logical_source_key, + kind=RawRevisionKind.APPEND, + source_revision=append_source_revision(plan.cursor_fingerprint, plan.payload_hash), + acquisition_generation=generation, + predecessor_source_revision=plan.cursor_fingerprint, + predecessor_raw_id=predecessor_raw_id, + baseline_raw_id=baseline_raw_id, + append_start_offset=plan.start_offset, + append_end_offset=plan.last_complete_newline, + authority=authority, + ), + ) + return logical_source_key, authority + + +def _write_append_raw_payload( + archive: Any, + *, + provider: Provider, + plan: _AppendPlan, + acquired_at_ms: int, +) -> str: + """Capture literal append bytes with their migration-stable raw identity.""" + return cast( + str, + archive.write_raw_payload( + provider=provider, + payload=plan.payload, + source_path=str(plan.path), + source_index=-1, + acquired_at_ms=acquired_at_ms, + native_id=plan.acquisition_native_id_hint, + post_parse=True, + ), + ) + + def reset_transient_raw_parse_state( archive: Any, raw_id: str, @@ -72,9 +137,12 @@ def _ingest_append_plans_archive( archive_root: Path, ) -> _AppendResult: timings: dict[str, float] = {} - index_db = archive_root / "index.db" source_db = archive_root / "source.db" - if not index_db.exists() or not source_db.exists(): + source_only = _source_tier_acquisition_required() + archive_missing = not source_db.exists() + if not source_only: + archive_missing = archive_missing or not resolve_active_index_path(archive_root).exists() + if archive_missing: t0 = time.perf_counter() initialize_active_archive_root(archive_root) _add_timing(timings, "append.archive_init", t0) @@ -110,6 +178,30 @@ def _ingest_append_plans_archive( session_artifact = None try: provider = Provider.from_string(plan.source_name) + degraded = degraded_reason() + if degraded is not None and degraded.derived_only: + if plan.native_id_hint is None: + raise ValueError("source-only append has no durable session identity") + t0 = time.perf_counter() + raw_id = _write_append_raw_payload( + archive, + provider=provider, + plan=plan, + acquired_at_ms=acquired_at_ms, + ) + _add_timing(timings, "append.source_raw_write", t0) + _logical_source_key, authority = _bind_append_revision( + archive, + raw_id, + provider=provider, + session_id=plan.native_id_hint, + plan=plan, + ) + if authority is RawRevisionAuthority.QUARANTINED: + deferred.append(plan) + else: + succeeded.append(plan) + continue path_artifact = classify_artifact_path( str(plan.path), provider=provider, @@ -171,38 +263,13 @@ def _ingest_append_plans_archive( succeeded.append(plan) continue t0 = time.perf_counter() - raw_id = archive.write_raw_payload( + raw_id = _write_append_raw_payload( + archive, provider=provider, - payload=plan.payload, - source_path=str(plan.path), - source_index=-1, + plan=plan, acquired_at_ms=acquired_at_ms, - # polylogue-u19l: persist the resolved provider - # session identity as sidecar metadata instead of - # splicing a synthetic session_meta record into the - # hashed/stored payload (batch.py's - # _append_payload_for_provider), so the stored blob - # stays a literal slice of the live file. - native_id=plan.native_id_hint, - post_parse=True, ) _add_timing(timings, "append.source_raw_write", t0) - degraded = degraded_reason() - if degraded is not None and degraded.derived_only: - # polylogue-gbs02: the derived tier (index.db/ - # embeddings.db) is behind the running code, but - # source.db just durably got this append range -- - # stop here, before parsing or touching the stale - # derived tier. Treat as succeeded (not deferred): - # the acquire itself genuinely completed, so the - # cursor should advance normally rather than - # re-reading the same bytes on every tick. The raw - # row sits with parsed_at_ms=NULL exactly like any - # other not-yet-materialized raw, and ordinary - # convergence picks it up once the derived tier is - # current again -- no special resolution needed. - succeeded.append(plan) - continue t0 = time.perf_counter() # polylogue-u19l: prefer the resolved provider session # identity over the bare filename stem. For Codex this is @@ -251,33 +318,12 @@ def _ingest_append_plans_archive( failed.append(plan) continue session = sessions[0] - logical_source_key = f"{provider.value}:{session.provider_session_id}" - parent = archive.raw_append_revision_parent( - logical_source_key, - plan.start_offset, - plan.cursor_fingerprint, - ) - predecessor_raw_id: str | None = None - baseline_raw_id: str | None = None - generation = archive.raw_full_revision_generation(logical_source_key) - authority = RawRevisionAuthority.QUARANTINED - if parent is not None: - predecessor_raw_id, baseline_raw_id, generation = parent - authority = RawRevisionAuthority.BYTE_PROVEN - archive.bind_raw_revision( + logical_source_key, authority = _bind_append_revision( + archive, raw_id, - RawRevisionEnvelope( - logical_source_key=logical_source_key, - kind=RawRevisionKind.APPEND, - source_revision=append_source_revision(plan.cursor_fingerprint, plan.payload_hash), - acquisition_generation=generation, - predecessor_source_revision=plan.cursor_fingerprint, - predecessor_raw_id=predecessor_raw_id, - baseline_raw_id=baseline_raw_id, - append_start_offset=plan.start_offset, - append_end_offset=plan.last_complete_newline, - authority=authority, - ), + provider=provider, + session_id=session.provider_session_id, + plan=plan, ) if authority is RawRevisionAuthority.QUARANTINED: deferred.append(plan) diff --git a/polylogue/sources/live/archive_open.py b/polylogue/sources/live/archive_open.py index 379eb36738..1eedfefc02 100644 --- a/polylogue/sources/live/archive_open.py +++ b/polylogue/sources/live/archive_open.py @@ -19,6 +19,13 @@ from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore +def _source_tier_acquisition_required() -> bool: + """Return whether live ingest must avoid every derived-tier read.""" + + reason = degraded_reason() + return reason is not None and reason.derived_only + + def _open_archive_for_live_write(archive_root: Path) -> ArchiveStore: """Open the archive for a live ingest write pass. @@ -29,7 +36,6 @@ def _open_archive_for_live_write(archive_root: Path) -> ArchiveStore: nothing beyond raw admission is reached. Otherwise returns the ordinary full writer, preserving its all-tier validation exactly. """ - reason = degraded_reason() - if reason is not None and reason.derived_only: + if _source_tier_acquisition_required(): return ArchiveStore.open_source_tier_acquisition(archive_root) return ArchiveStore.open_existing(archive_root, read_only=False) diff --git a/polylogue/sources/live/batch.py b/polylogue/sources/live/batch.py index b7ed29f790..dbfaf3d9c2 100644 --- a/polylogue/sources/live/batch.py +++ b/polylogue/sources/live/batch.py @@ -20,6 +20,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, Literal, ParamSpec, TypeVar, cast +from polylogue.archive.artifact_taxonomy import ArtifactKind, classify_artifact_path, strong_path_classification from polylogue.archive.ingest_flags import ( COMPACT_BROWSER_CAPTURE_INGEST_FLAG, DOM_FALLBACK_INGEST_FLAG, @@ -50,6 +51,11 @@ read_peak_rss_self_mb, ) from polylogue.core.provider_identity import canonical_acquisition_provider +from polylogue.core.raw_coordinates import ( + zip_member_identity_coordinate, + zip_member_raw_id, + zip_member_source_index, +) from polylogue.core.raw_failure_evidence import ( RAW_FAILURE_EVIDENCE_KINDS, RAW_FAILURE_LIFECYCLE_EVIDENCE_SUPPORT_STATUS_PAIRS, @@ -65,18 +71,20 @@ success_disposition, ) from polylogue.pipeline.services.ingest_batch._models import _IngestBatchSummary +from polylogue.sources.codex_state_evidence import write_codex_thread_state_evidence from polylogue.sources.decoder_json import PartialJsonStreamError from polylogue.sources.decoder_zip import ZipBombError, open_bounded_zip_entry from polylogue.sources.decoders import JsonlDecodeError, _iter_json_stream, _ZipEntryValidator from polylogue.sources.dispatch import ( _detect_provider_from_raw_bytes, + is_jsonl_source_path, is_stream_record_provider, parse_payload, parse_stream_payload, require_positive_conversational_evidence, ) from polylogue.sources.live.append_ingest import ingest_append_plans, reset_transient_raw_parse_state -from polylogue.sources.live.archive_open import _open_archive_for_live_write +from polylogue.sources.live.archive_open import _open_archive_for_live_write, _source_tier_acquisition_required from polylogue.sources.live.batch_observability import ( record_attempt_progress, ) @@ -131,6 +139,7 @@ from polylogue.sources.live.deferred_cursor import record_deferred_append_cursor from polylogue.sources.live.metrics import LiveBatchMetrics, LiveFullIngestAggregate from polylogue.sources.live.parse_prefetch import LiveParseCandidate, LiveParseStage +from polylogue.sources.live.source_selection import deepest_source_for_path from polylogue.sources.live.sqlite_locking import is_transient_sqlite_lock from polylogue.sources.origin_specs import artifact_rule_for_path from polylogue.sources.parsers import codex_state, hermes_state, hermes_verification @@ -143,7 +152,9 @@ _DETECTION_PREFIX_SIZE, ZipEntryReadContext, iter_zip_entry_raw_data, + stream_preserved_zip_entry_raw_data, ) +from polylogue.sources.source_parsing import has_decoded_session_evidence from polylogue.sources.sqlite_snapshot import ( codex_state_raw_id, hermes_profile_raw_id, @@ -156,11 +167,13 @@ from polylogue.storage.sqlite.archive_tiers.archive import ActiveByteRevisionChainError from polylogue.storage.sqlite.archive_tiers.bootstrap import ( ARCHIVE_TIER_SPECS, + archive_tier_spec, ) from polylogue.storage.sqlite.archive_tiers.bootstrap import ( initialize_active_archive_root as initialize_archive_root, ) from polylogue.storage.sqlite.archive_tiers.source_write import ContentExcisedError +from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier if TYPE_CHECKING: from polylogue.api import Polylogue @@ -269,82 +282,6 @@ def _hot_capture_prefix_is_proven( return fingerprint == expected_fingerprint and _file_observation(proof_start) == _file_observation(proof_end) -def _write_codex_thread_state_evidence( - archive: Any, - snapshot: codex_state.CodexStateSnapshot, - *, - source_path: str, - acquired_at_ms: int, -) -> None: - """Attach ``threads``/``thread_spawn_edges`` evidence to EXISTING sessions. - - polylogue-0jf4 acceptance criterion 3: threads.title and - thread_spawn_edges must reach the archive as typed evidence without ever - minting a session or session of their own -- the same hook-event - incident precedent as polylogue-31r1 (standalone hook-event ingestion - once inflated the archive from 18,391 to 83,286 sessions). Reuses - ``ArchiveStore.write_hook_event``/``raw_hook_events`` exactly as - ``sources/hooks.py`` does: a durable, session-scoped evidence row keyed - by ``session_native_id`` (here the Codex ``thread_id``), joined at read - time (``ArchiveStore.hook_event_summary_for_session``) rather than - materialized into ``index.db`` via a full session replace. No schema - change -- ``raw_hook_events.event_type`` is unconstrained TEXT. - """ - from polylogue.storage.sqlite.archive_tiers.source_write import ArchiveHookEvent - - for thread in snapshot.threads: - payload: dict[str, object] = { - "thread_id": thread.thread_id, - "title": thread.title, - "cwd": thread.cwd, - "source": thread.source, - "model": thread.model, - "agent_nickname": thread.agent_nickname, - "agent_role": thread.agent_role, - "archived": thread.archived, - } - encoded = json_dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") - archive.write_hook_event( - provider=Provider.CODEX, - payload=encoded, - source_path=source_path, - acquired_at_ms=acquired_at_ms, - hook_event=ArchiveHookEvent( - hook_event_id=f"codex-thread-title:{thread.thread_id}", - origin=Origin.CODEX_SESSION, - source_path=source_path, - event_type="codex_thread_title", - payload=payload, - observed_at_ms=thread.updated_at_ms or acquired_at_ms, - native_id=f"{thread.thread_id}:codex_thread_title", - session_native_id=thread.thread_id, - ), - ) - for edge in snapshot.spawn_edges: - edge_payload: dict[str, object] = { - "parent_thread_id": edge.parent_thread_id, - "child_thread_id": edge.child_thread_id, - "status": edge.status, - } - encoded = json_dumps(edge_payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") - archive.write_hook_event( - provider=Provider.CODEX, - payload=encoded, - source_path=source_path, - acquired_at_ms=acquired_at_ms, - hook_event=ArchiveHookEvent( - hook_event_id=f"codex-thread-spawn-edge:{edge.parent_thread_id}:{edge.child_thread_id}", - origin=Origin.CODEX_SESSION, - source_path=source_path, - event_type="codex_thread_spawn_edge", - payload=edge_payload, - observed_at_ms=acquired_at_ms, - native_id=f"{edge.parent_thread_id}:{edge.child_thread_id}:codex_thread_spawn_edge", - session_native_id=edge.parent_thread_id, - ), - ) - - def _is_json_stream_decode_error(error: BaseException) -> bool: return isinstance(error, (StdlibJSONDecodeError, UnicodeDecodeError, PartialJsonStreamError, JsonlDecodeError)) @@ -451,7 +388,7 @@ def _blob_jsonl_has_session_evidence( provider: Provider, source_path: str, ) -> bool: - if Path(source_path).suffix.lower() != ".jsonl": + if not is_jsonl_source_path(source_path): return False try: return jsonl_session_artifact(blob_store.blob_path(blob_hash), provider=provider) is not None @@ -459,10 +396,36 @@ def _blob_jsonl_has_session_evidence( return False +def _record_zip_container_coordinate( + archive: Any, + record: RawSessionRecord, + *, + source_raw_id: str, + blob_hash: str, +) -> None: + if record.source_index is None: + return + coordinate = zip_member_identity_coordinate( + raw_id=source_raw_id, + source_path=record.source_path, + source_index=record.source_index, + blob_hash=blob_hash, + ) + if coordinate is None: + return + entry_ordinal, split_index = coordinate + archive.record_raw_container_coordinate( + source_raw_id, + coordinate_format="zip-v2", + entry_ordinal=entry_ordinal, + split_index=split_index, + ) + + def _live_parse_stage_candidates(paths: list[Path], *, fallback_provider: Provider) -> list[LiveParseCandidate]: """Select and read eligible files for off-writer-hold pre-parse (polylogue-wf8a). - Deliberately narrow scope: only plain ``.jsonl`` provider-session files + Deliberately narrow scope: only JSONL/NDJSON provider-session files below ``_STREAMING_FULL_INGEST_BYTES`` are eligible -- exactly the branch at lines ~1377-1425 of ``_ingest_full_paths_sync`` that reads the whole payload into memory and later parses it via ``parse_payload``/ @@ -476,7 +439,7 @@ def _live_parse_stage_candidates(paths: list[Path], *, fallback_provider: Provid """ candidates: list[LiveParseCandidate] = [] for path in paths: - if path.suffix.lower() != ".jsonl": + if not is_jsonl_source_path(str(path)): continue try: stat = path.stat() @@ -514,8 +477,7 @@ def _captured_jsonl_ends_at_record_boundary( blob_hash: str, blob_size: int, ) -> bool: - path = Path(source_path) - if not required or path.suffix.lower() not in {".jsonl", ".ndjson"}: + if not required or not is_jsonl_source_path(source_path): return True if blob_size <= 0: # A zero-byte capture has zero records -- none complete, none @@ -639,6 +601,12 @@ def cursor_authority_block_reason(self) -> str | None: exist. Once they do, the readiness proof is fail-closed and shared with raw convergence, recovery, and reindex. """ + if _source_tier_acquisition_required(): + # The derived tier is explicitly unavailable in this mode. Raw + # admission establishes source authority without consulting or + # mutating it; trying to resolve the active index pointer here + # would defeat the acquire-only route before it can run. + return None archive_root = Path(getattr(self._polylogue, "archive_root", self._cursor._db_path.parent)) if ( not (archive_root / "source.db").is_file() @@ -891,6 +859,13 @@ async def flush_append_plans() -> None: if authorization is not None and authorization.force_full_ingest: full_paths.append(path) continue + if _source_tier_acquisition_required(): + # Append planning and replay both consult the active index to + # prove lineage. In acquire-only mode the derived tier is the + # unavailable component, so capture the complete source + # observation through the source-only full route instead. + full_paths.append(path) + continue if is_fully_degraded(): full_paths.append(path) continue @@ -1109,7 +1084,8 @@ async def flush_append_plans() -> None: ) if self._last_cursor_write_stale: stale_cursor_write_count += 1 - self._record_convergence_outcome(path, debt_by_source_path.get(path, ())) + if not _source_tier_acquisition_required(): + self._record_convergence_outcome(path, debt_by_source_path.get(path, ())) for path in full_result.failed: failed_paths.append(str(path)) cursor_fingerprint_read_bytes += self._record_failed_cursor(path) @@ -1138,7 +1114,7 @@ async def flush_append_plans() -> None: stage_payload=summary_stage_payload, ) - if succeeded_paths: + if succeeded_paths and not _source_tier_acquisition_required(): await self._run_sync( "watcher.live_ingest.raw_compaction", self._compact_superseded_raw_snapshots, @@ -1742,13 +1718,9 @@ def _current_parser_fingerprint(self) -> str: return self._parser_fingerprint def _source_name_for(self, path: Path) -> str: - resolved = path.resolve() - for source in self._sources: - try: - if resolved.is_relative_to(source.root.resolve()): - return str(source.name) - except OSError: - continue + source = deepest_source_for_path(path, self._sources) + if source is not None: + return str(source.name) return path.parent.name def _can_ingest_appends_directly(self) -> bool: @@ -1765,7 +1737,7 @@ async def _ingest_full_paths( max_pass_seconds: float | None = None, pass_started: float | None = None, ) -> _FullIngestResult: - if self._parse_stage is not None: + if self._parse_stage is not None and not _source_tier_acquisition_required(): # polylogue-wf8a: pre-parse eligible candidates BEFORE ever # asking the write coordinator for the writer hold below -- # identical sequencing guarantee to ``DaemonParseStage.warm`` @@ -1832,9 +1804,6 @@ def _ingest_full_paths_sync( pass_clock_started = pass_started if pass_started is not None else time.monotonic() archive_root = Path(getattr(self._polylogue, "archive_root", self._cursor._db_path.parent)) blob_root = archive_root / "blob" - from polylogue.storage.blob_publication import ArchiveBlobPublisher - - blob_store = ArchiveBlobPublisher(archive_root / "source.db", blob_root) raw_records: list[RawSessionRecord] = [] raw_by_id: dict[str, Path] = {} raw_byte_sizes: dict[Path, int] = {} @@ -1850,7 +1819,20 @@ def _ingest_full_paths_sync( fallback_provider = Provider.from_string(canonical_acquisition_provider(source_name, source_name=source_name)) acquisition_capture_mode = fallback_provider - archive_bootstrapped = not self._archive_active(archive_root) + source_only = _source_tier_acquisition_required() + source_db = archive_root / "source.db" + if source_only and not source_db.is_file(): + logger.error("source-only acquisition refused because the durable source tier is missing: %s", source_db) + return _FullIngestResult( + succeeded=[], + failed=list(paths), + source_payload_read_bytes=0, + ) + from polylogue.storage.blob_publication import ArchiveBlobPublisher + + blob_store = ArchiveBlobPublisher(source_db, blob_root) + archive_active = self._archive_active(archive_root) + archive_bootstrapped = not archive_active and not source_only if archive_bootstrapped: initialize_archive_root(archive_root) archive_active = self._archive_active(archive_root) @@ -1877,6 +1859,8 @@ def _ingest_full_paths_sync( continue captured_file_observations[path] = _file_observation(stat) origin_artifact_rule = artifact_rule_for_path(fallback_provider, str(path)) + path_artifact = classify_artifact_path(path, provider=fallback_provider) + strong_path_artifact = strong_path_classification(path, provider=fallback_provider) if heartbeat is not None: heartbeat( "full_file_scan", @@ -1885,12 +1869,24 @@ def _ingest_full_paths_sync( ) if path.suffix.lower() == ".zip": file_mtime = datetime.fromtimestamp(stat.st_mtime_ns / 1_000_000_000, UTC).isoformat() - zip_records, zip_bytes = self._extract_zip_member_records( - path, - blob_store=blob_store, - fallback_provider=fallback_provider, - file_mtime=file_mtime, - ) + if source_only: + source_only_zip = self._extract_source_only_zip_member_records( + path, + blob_store=blob_store, + fallback_provider=fallback_provider, + file_mtime=file_mtime, + ) + if source_only_zip is None: + failed.append(path) + continue + zip_records, zip_bytes = source_only_zip + else: + zip_records, zip_bytes = self._extract_zip_member_records( + path, + blob_store=blob_store, + fallback_provider=fallback_provider, + file_mtime=file_mtime, + ) if not zip_records: self._mark_excluded_cursor(path, stat, source_name=fallback_provider.value) continue @@ -1907,9 +1903,19 @@ def _ingest_full_paths_sync( ingested.append(path) raw_byte_sizes[path] = stat.st_size continue - if hermes_state.looks_like_state_db_path( - path - ) or hermes_verification.looks_like_verification_evidence_db_path(path): + hermes_owned_sqlite_name = ( + source_only + and fallback_provider is Provider.HERMES + and path.name in {"state.db", "verification_evidence.db"} + ) + codex_owned_sqlite_name = ( + source_only and fallback_provider is Provider.CODEX and path.name in _CODEX_STATE_DB_NAMES + ) + if ( + hermes_owned_sqlite_name + or hermes_state.looks_like_state_db_path(path) + or hermes_verification.looks_like_verification_evidence_db_path(path) + ): provider = Provider.HERMES source_name = provider.value try: @@ -1943,7 +1949,9 @@ def _ingest_full_paths_sync( current_path=path, source_payload_read_bytes=source_payload_read_bytes, ) - elif path.name in _CODEX_STATE_DB_NAMES and codex_state.is_in_scope_codex_sqlite_path(path): + elif codex_owned_sqlite_name or ( + path.name in _CODEX_STATE_DB_NAMES and codex_state.is_in_scope_codex_sqlite_path(path) + ): # polylogue-0jf4: acquire live Codex SQLite state the same # way Hermes acquires its state.db -- a consistent # backup/snapshot (never a raw read of a possibly-live-locked @@ -1951,6 +1959,9 @@ def _ingest_full_paths_sync( # gate keeps this cheap for the vast majority of ~/.codex # traffic (JSONL rollouts); ``is_in_scope_codex_sqlite_path`` # then re-confirms the table shape before trusting the name. + # Source-only acquisition intentionally skips that structural + # decode: a mid-write or future-schema state snapshot is still + # durable authority to replay once the derived tier returns. provider = Provider.CODEX source_name = provider.value try: @@ -1994,6 +2005,73 @@ def _ingest_full_paths_sync( # the bytes as a generic session artifact. self._mark_excluded_cursor(path, stat, source_name=fallback_provider.value) continue + elif source_only: + # A derived-only outage must not turn durable acquisition into + # an ad hoc parse pass. Provider detection and session/artifact + # classification decode payload bytes, while this route has no + # derived tier to consume their result. Preserve the original + # bytes under the configured source identity and let the + # normal raw replay classify them once the index is available. + # Antigravity brain metadata is the one path whose parser also + # reads a mutable sibling artifact. Until the derived route can + # consume both contemporaneously, leave this observation + # pending instead of advancing a cursor backed by only half of + # its material. + if fallback_provider is Provider.ANTIGRAVITY and path.name.endswith(".metadata.json"): + failed.append(path) + continue + provider = fallback_provider + source_name = provider.value + try: + if heartbeat is not None: + heartbeat( + "full_blob_copy", + current_path=path, + source_payload_read_bytes=source_payload_read_bytes, + ) + raw_id, blob_size = blob_store.write_from_path( + path, + heartbeat=_blob_copy_heartbeat( + heartbeat, + path=path, + source_payload_read_bytes=source_payload_read_bytes, + ), + ) + blob_publication_receipt_id = blob_store.receipt_id(raw_id) + except OSError: + failed.append(path) + continue + source_payload_read_bytes += blob_size + if heartbeat is not None: + heartbeat( + "full_blob_copy", + current_path=path, + source_payload_read_bytes=source_payload_read_bytes, + ) + elif ( + origin_artifact_rule is None + and not is_jsonl_source_path(str(path)) + and path_artifact is not None + and not path_artifact.parse_as_session + and stat.st_size < _STREAMING_FULL_INGEST_BYTES + # An unknown JSON payload under the weak ``analysis/`` path + # heuristic must reach the generic JSON route. That route + # retains raw bytes and records a typed terminal outcome for + # malformed or empty input. Strong named sidecars such as + # ``sessions-index.json`` remain excluded before acquisition. + and not ( + fallback_provider is Provider.UNKNOWN + and path.suffix.lower() == ".json" + and path_artifact.kind is ArtifactKind.METADATA_DOCUMENT + and (strong_path_artifact is None or strong_path_artifact.parse_as_session) + ) + and not has_decoded_session_evidence(path, provider=fallback_provider) + ): + # Keep path-only metadata out of the generic JSON fallback, + # but let real decoded session evidence outrank a stale or + # overbroad filename rule just as offline source parsing does. + self._mark_excluded_cursor(path, stat, source_name=fallback_provider.value) + continue elif origin_artifact_rule is not None and origin_artifact_rule.parse_policy != "session": provider = fallback_provider source_name = provider.value @@ -2034,9 +2112,14 @@ def _ingest_full_paths_sync( current_path=path, source_payload_read_bytes=source_payload_read_bytes, ) - elif path.suffix.lower() == ".jsonl": + elif is_jsonl_source_path(str(path)): provider, parse_as_session = _jsonl_provider_and_session_artifact(path, fallback_provider) source_name = provider.value + # An unknown JSONL cannot be safely excluded from acquire: the + # strict parse route persists typed terminal evidence for empty + # and malformed exports. Known-provider sidecars remain + # cursor-excluded here because their classification is already + # authoritative. if not parse_as_session and provider is not Provider.UNKNOWN: self._mark_excluded_cursor(path, stat, source_name=source_name) continue @@ -2194,9 +2277,7 @@ def _ingest_full_paths_sync( raw_id=raw_id, blob_hash=(blob_hash if acquired_via_sqlite_snapshot and blob_hash is not None else None), payload_provider=provider, - capture_mode=( - acquisition_capture_mode if acquisition_capture_mode is not Provider.UNKNOWN else provider - ), + capture_mode=acquisition_capture_mode, source_name=source_name, source_path=( str(original_sqlite_source_path(path) or path) if path in raw_source_revisions else str(path) @@ -2207,7 +2288,7 @@ def _ingest_full_paths_sync( acquired_at=datetime.now(UTC).isoformat(), file_mtime=datetime.fromtimestamp(stat.st_mtime_ns / 1_000_000_000, UTC).isoformat(), captured_source_revision=raw_source_revisions.get(path, raw_id), - requires_complete_record_boundary=path.suffix.lower() in {".jsonl", ".ndjson"}, + requires_complete_record_boundary=is_jsonl_source_path(str(path)), ) ) raw_source_revisions.setdefault(path, raw_id) @@ -2317,6 +2398,8 @@ def _ingest_full_paths_sync( return result def _archive_active(self, archive_root: Path) -> bool: + if _source_tier_acquisition_required(): + return (archive_root / "source.db").exists() and (archive_root / "user.db").exists() return ( ArchiveLocation.resolve(archive_root).active_index_path.exists() and (archive_root / "source.db").exists() ) @@ -2328,14 +2411,26 @@ def _archive_storage_probe_payload( archive_active: bool, archive_bootstrapped: bool, ) -> dict[str, object]: - tier_paths = { - spec.tier.value: ( - ArchiveLocation.resolve(archive_root).active_index_path - if spec.tier.value == "index" - else archive_root / spec.filename - ) - for spec in ARCHIVE_TIER_SPECS.values() - } + if _source_tier_acquisition_required(): + tier_paths = { + tier.value: archive_root / archive_tier_spec(tier).filename + for tier in (ArchiveTier.SOURCE, ArchiveTier.USER) + } + storage_route = "archive_source_acquisition" + storage_tiers = ",".join(tier_paths) + storage_write_tiers = ArchiveTier.SOURCE.value + else: + tier_paths = { + spec.tier.value: ( + ArchiveLocation.resolve(archive_root).active_index_path + if spec.tier.value == "index" + else archive_root / spec.filename + ) + for spec in ARCHIVE_TIER_SPECS.values() + } + storage_route = "archive_full" + storage_tiers = _ARCHIVE_RUNTIME_TIERS + storage_write_tiers = _ARCHIVE_NATIVE_WRITE_TIERS present = [tier for tier, path in tier_paths.items() if path.exists()] missing = [tier for tier, path in tier_paths.items() if not path.exists()] user_versions: dict[str, int | None] = {} @@ -2352,9 +2447,9 @@ def _archive_storage_probe_payload( except sqlite3.Error: user_versions[tier] = -1 return { - "storage_route": "archive_full", - "storage_tiers": _ARCHIVE_RUNTIME_TIERS, - "storage_write_tiers": _ARCHIVE_NATIVE_WRITE_TIERS, + "storage_route": storage_route, + "storage_tiers": storage_tiers, + "storage_write_tiers": storage_write_tiers, "archive_active": archive_active, "archive_bootstrapped": archive_bootstrapped, "archive_present_tiers": ",".join(present), @@ -2377,6 +2472,7 @@ def _ingest_full_records_archive( result = _ArchiveFullWriteResult() pass_clock_started = pass_started if pass_started is not None else time.monotonic() with _open_archive_for_live_write(archive_root) as archive: + source_only = _source_tier_acquisition_required() for record_index, record in enumerate(records): # polylogue-11cg9: a single logical session write cannot be # split mid-transaction (it must remain atomic), so the @@ -2402,34 +2498,46 @@ def _ingest_full_records_archive( record_timings: dict[str, float] = {} t0 = time.perf_counter() provider = record.payload_provider or Provider.from_string(record.source_name) + acquisition_provider = record.capture_mode or provider payload = raw_payloads.get(record.raw_id) source_name = Path(record.source_path).name fallback_id = Path(record.source_path).stem blob_hash = record.blob_hash or record.raw_id acquired_at_ms = _iso_to_epoch_ms(record.acquired_at) - artifact_classification = _declared_non_session_artifact_classification( - provider, - record.source_path, - ) - session_evidence = ( - _blob_jsonl_has_session_evidence( - blob_store, - blob_hash, - provider=provider, - source_path=record.source_path, - ) - if payload is None - else _parse_payload_as_session_artifact( - Path(record.source_path), - provider=provider, - payload=payload, + # Source-only acquisition deliberately has no decoded + # evidence with which to confirm or override a path + # classification. Keep every such raw pending instead of + # giving a filename-only fact/sidecar rule terminal + # authority that a recovered derived tier could not undo. + artifact_classification = ( + None + if source_only + else _declared_non_session_artifact_classification( + provider, + record.source_path, ) ) + session_evidence = False + if artifact_classification is not None and not source_only: + session_evidence = ( + _blob_jsonl_has_session_evidence( + blob_store, + blob_hash, + provider=provider, + source_path=record.source_path, + ) + if payload is None + else _parse_payload_as_session_artifact( + Path(record.source_path), + provider=provider, + payload=payload, + ) + ) if artifact_classification is not None and not session_evidence: explicit_raw_id = record.raw_id if record.blob_hash is not None else None if payload is None: source_raw_id = archive.admit_raw_artifact_blob_ref( - provider=provider, + provider=acquisition_provider, blob_hash_hex=blob_hash, blob_size=record.blob_size, source_path=record.source_path, @@ -2441,7 +2549,7 @@ def _ingest_full_records_archive( ).raw_id else: source_raw_id = archive.admit_raw_artifact_payload( - provider=provider, + provider=acquisition_provider, payload=payload, source_path=record.source_path, source_index=record.source_index or 0, @@ -2450,25 +2558,32 @@ def _ingest_full_records_archive( classification=artifact_classification, blob_publication_receipt_id=record.blob_publication_receipt_id, ).raw_id + _record_zip_container_coordinate( + archive, + record, + source_raw_id=source_raw_id, + blob_hash=blob_hash, + ) result.raw_ids[record.raw_id] = source_raw_id _accumulate_stage_timings(result.stage_timings_s, record_timings) continue source_write_started = time.perf_counter() if payload is None: source_raw_id = archive.write_raw_blob_ref( - provider=provider, + provider=acquisition_provider, capture_mode=record.capture_mode, blob_hash_hex=blob_hash, blob_size=record.blob_size, source_path=record.source_path, source_index=record.source_index or 0, - # A populated ``blob_hash`` field marks a - # sqlite-snapshot acquisition (Hermes or, per - # polylogue-0jf4, Codex state dbs), whose raw_id - # is a deterministic profile/path-scoped id - # distinct from the blob's own content hash -- - # every other provider's raw_id already IS the - # content hash, so passing it again is a no-op. + # A populated ``blob_hash`` field means this + # record already has a durable blob reference. + # SQLite snapshots and ZIP members both keep a + # raw id distinct from that blob address: the + # former is profile/path scoped, the latter is + # coordinate scoped. Preserve that identity at + # source admission rather than collapsing either + # kind back onto a shared content hash. raw_id=(record.raw_id if record.blob_hash is not None else None), acquired_at_ms=acquired_at_ms, blob_publication_receipt_id=record.blob_publication_receipt_id, @@ -2477,7 +2592,7 @@ def _ingest_full_records_archive( source_write_name = "full.source_raw_blob_ref_write" else: source_raw_id = archive.write_raw_payload( - provider=provider, + provider=acquisition_provider, capture_mode=record.capture_mode, payload=payload, source_path=record.source_path, @@ -2487,6 +2602,12 @@ def _ingest_full_records_archive( post_parse=True, ) source_write_name = "full.source_raw_write" + _record_zip_container_coordinate( + archive, + record, + source_raw_id=source_raw_id, + blob_hash=blob_hash, + ) record_timings[source_write_name] = time.perf_counter() - source_write_started degraded = degraded_reason() if degraded is not None and degraded.derived_only: @@ -2600,7 +2721,7 @@ def _ingest_full_records_archive( state_kind = codex_state.classify_codex_sqlite_path(state_path, immutable=True) if state_kind == "thread_state": state_snapshot = codex_state.parse_codex_state_db(state_path, immutable=True) - _write_codex_thread_state_evidence( + write_codex_thread_state_evidence( archive, state_snapshot, source_path=record.source_path, @@ -3170,7 +3291,9 @@ def _extract_zip_member_records( ) try: with zipfile.ZipFile(path) as zf: - entries = list(validator.filter_entries(zf.infolist())) + central_directory = zf.infolist() + entry_ordinals = {id(info): ordinal for ordinal, info in enumerate(central_directory)} + entries = [(entry_ordinals[id(info)], info) for info in validator.filter_entries(central_directory)] # A GDPR/Takeout export ZIP dropped into a provider-agnostic # inbox (``fallback_provider is Provider.UNKNOWN``) still has # a real dominant provider -- it just isn't visible from any @@ -3185,8 +3308,10 @@ def _extract_zip_member_records( # provider (a per-provider watched directory) is left alone. zip_provider_hint = fallback_provider if fallback_provider is Provider.UNKNOWN: - zip_provider_hint = self._sniff_zip_provider(zf, entries) or fallback_provider - for info in entries: + zip_provider_hint = ( + self._sniff_zip_provider(zf, [info for _ordinal, info in entries]) or fallback_provider + ) + for entry_ordinal, info in entries: if info.file_size == 0: continue try: @@ -3206,20 +3331,28 @@ def _extract_zip_member_records( member_provider = raw_data.provider_hint or fallback_provider member_size = raw_data.blob_size or 0 total_bytes += member_size + split_index = raw_data.source_index if raw_data.source_index is not None else 0 + source_index = zip_member_source_index( + entry_ordinal=entry_ordinal, + split_index=split_index, + ) + member_raw_id = zip_member_raw_id( + source_path=raw_data.source_path, + entry_ordinal=entry_ordinal, + split_index=split_index, + blob_hash=raw_data.blob_hash, + ) records.append( ( - raw_data.blob_hash, + member_raw_id, RawSessionRecord( - raw_id=raw_data.blob_hash, + raw_id=member_raw_id, + blob_hash=raw_data.blob_hash, payload_provider=member_provider, - capture_mode=( - fallback_provider - if fallback_provider is not Provider.UNKNOWN - else member_provider - ), + capture_mode=fallback_provider, source_name=member_provider.value, source_path=raw_data.source_path, - source_index=raw_data.source_index or 0, + source_index=source_index, blob_size=member_size, blob_publication_receipt_id=raw_data.blob_publication_receipt_id, acquired_at=acquired_at, @@ -3234,6 +3367,92 @@ def _extract_zip_member_records( return [], 0 return records, total_bytes + def _extract_source_only_zip_member_records( + self, + path: Path, + *, + blob_store: BlobStore, + fallback_provider: Provider, + file_mtime: str, + ) -> tuple[list[tuple[str, RawSessionRecord]], int] | None: + """Acquire admitted ZIP members without interpreting their bytes. + + A derived-tier outage does not authorize the source tier to infer a + provider, parse JSON, or classify a member. It does still enforce the + ordinary ZIP admission policy before streaming every retained member + under its exact ``:`` coordinate. + """ + source = Source(name=fallback_provider.value, path=path.parent) + acquired_at = datetime.now(UTC).isoformat() + records: list[tuple[str, RawSessionRecord]] = [] + total_bytes = 0 + validator = _ZipEntryValidator(fallback_provider, cursor_state=None, zip_path=path) + try: + with zipfile.ZipFile(path) as zf: + central_directory = zf.infolist() + entry_ordinals = {id(info): ordinal for ordinal, info in enumerate(central_directory)} + for info in validator.filter_entries(central_directory): + if info.file_size == 0: + continue + entry_ordinal = entry_ordinals[id(info)] + split_index = 0 + source_index = zip_member_source_index( + entry_ordinal=entry_ordinal, + split_index=split_index, + ) + try: + raw_data = stream_preserved_zip_entry_raw_data( + zf, + ZipEntryReadContext( + source=source, + zip_path=path, + entry=info, + file_mtime=file_mtime, + provider_hint=fallback_provider, + blob_store=blob_store, + ), + provider_hint=fallback_provider, + source_index=source_index, + ) + except ZipBombError as exc: + logger.warning("Skipping ZIP member %s in %s: %s", info.filename, path, exc) + continue + if raw_data.blob_hash is None: + continue + total_bytes += raw_data.blob_size or 0 + member_raw_id = zip_member_raw_id( + source_path=raw_data.source_path, + entry_ordinal=entry_ordinal, + split_index=split_index, + blob_hash=raw_data.blob_hash, + ) + records.append( + ( + member_raw_id, + RawSessionRecord( + raw_id=member_raw_id, + blob_hash=raw_data.blob_hash, + payload_provider=fallback_provider, + capture_mode=fallback_provider, + source_name=fallback_provider.value, + source_path=raw_data.source_path, + source_index=source_index, + blob_size=raw_data.blob_size or 0, + blob_publication_receipt_id=raw_data.blob_publication_receipt_id, + acquired_at=acquired_at, + file_mtime=raw_data.file_mtime, + ), + ) + ) + except (zipfile.BadZipFile, OSError) as exc: + logger.warning("Failed to expand inbox ZIP %s: %s", path, exc) + # A transport/read failure is not evidence that the archive has no + # admissible members. Keep it distinct from a successful empty + # extraction so the caller records retryable failure state instead + # of permanently acknowledging this source coordinate as excluded. + return None + return records, total_bytes + @staticmethod def _sniff_zip_provider( zf: zipfile.ZipFile, @@ -3585,7 +3804,7 @@ def _append_plan(self, path: Path, *, cursor: CursorRecord | None = None) -> _Ap append_result = self._append_payload_for_provider(path, self._source_name_for(path), complete_payload) if append_result is None: return None - append_payload, native_id_hint = append_result + append_payload, native_id_hint, acquisition_native_id_hint = append_result tail_hash = sha256(complete_payload).hexdigest() return _AppendPlan( path=path, @@ -3605,12 +3824,13 @@ def _append_plan(self, path: Path, *, cursor: CursorRecord | None = None) -> _Ap accepted_prefix_hash=accepted_prefix_hash, authority_bytes_read=last_complete_newline, native_id_hint=native_id_hint, + acquisition_native_id_hint=acquisition_native_id_hint, ) def _append_payload_for_provider( self, path: Path, source_name: str, payload: bytes - ) -> tuple[bytes, str | None] | None: - """Return the literal append payload plus an optional identity hint. + ) -> tuple[bytes, str | None, str | None] | None: + """Return literal bytes plus logical and acquisition identity hints. polylogue-u19l: this used to prepend a synthetic ``session_meta`` line ahead of ``payload`` for Codex before hashing/storing it, so the @@ -3623,8 +3843,8 @@ def _append_payload_for_provider( Now the identity is resolved here exactly as before, but returned as a sidecar hint instead of being spliced into the hashed bytes. - Callers persist it to ``raw_sessions.native_id`` (``_AppendPlan. - native_id_hint`` -> ``append_ingest.py``) and pass it back as the + Callers persist the Codex acquisition hint to + ``raw_sessions.native_id`` and pass the logical hint back as the parser's ``fallback_id`` at replay time (``revision_backfill.parse_retained_raw_sessions``), which is exactly equivalent for Codex: ``_parse_records`` only ever falls back to @@ -3670,12 +3890,16 @@ def _append_payload_for_provider( "identity recovered from archived session / prior session_meta " "line and carried as native_id_hint, not spliced into hashed bytes", ) - return payload, identity + return payload, identity, identity if provider is Provider.CLAUDE_CODE and not self._claude_code_tail_matches_existing_identity( path, payload, existing_id=identity ): return None - return payload, None + # Claude append raws have historically used native_id=NULL. Its own + # records carry sessionId, so the resolved identity is needed for + # governance but must not change deterministic acquisition identity + # for a retry of pre-upgrade bytes. + return payload, identity, None def _existing_provider_session_id(self, path: Path, *, expected_origin: str) -> str | None: identity = self._existing_archive_session_native_id(path, expected_origin=expected_origin) @@ -3833,8 +4057,9 @@ def _ingest_append_plans(self, plans: list[_AppendPlan]) -> _AppendResult: return ingest_append_plans(self, plans) def _compact_superseded_raw_snapshots(self, paths: list[Path]) -> None: - if not paths: + if not paths or _source_tier_acquisition_required(): return + from polylogue.storage.index_generation import ActiveWriterLease from polylogue.storage.raw_retention import ( RawRetentionSafetyError, active_raw_retention_authority, @@ -3843,24 +4068,33 @@ def _compact_superseded_raw_snapshots(self, paths: list[Path]) -> None: archive_root = Path(getattr(self._polylogue, "archive_root", self._cursor._db_path.parent)) source_db = archive_root / "source.db" - index_db = ArchiveLocation.resolve(archive_root).active_index_path if not source_db.exists(): return - with closing(sqlite3.connect(source_db)) as conn, conn: - conn.row_factory = sqlite3.Row - try: - retention_authority = active_raw_retention_authority(conn, index_db_path=index_db) - except RawRetentionSafetyError as exc: - logger.warning("live.watcher: skipped unsafe raw snapshot compaction: %s", exc) - return - result = compact_paths_superseded_raw_snapshots( - conn, - paths, - limit_per_path=25, - min_acquired_at=self._raw_compaction_min_acquired_at, - protected_raw_ids=retention_authority.protected_raw_ids, - eligible_raw_ids=retention_authority.eligible_raw_ids, - ) + lease = ActiveWriterLease(archive_root) + lease.acquire() + try: + index_db = ArchiveLocation.resolve(archive_root).active_index_path + with closing(sqlite3.connect(source_db)) as conn, conn: + conn.row_factory = sqlite3.Row + try: + retention_authority = active_raw_retention_authority( + conn, + index_db_path=index_db, + terminal_source_paths=paths, + ) + except RawRetentionSafetyError as exc: + logger.warning("live.watcher: skipped unsafe raw snapshot compaction: %s", exc) + return + result = compact_paths_superseded_raw_snapshots( + conn, + paths, + limit_per_path=25, + min_acquired_at=self._raw_compaction_min_acquired_at, + protected_raw_ids=retention_authority.protected_raw_ids, + eligible_raw_ids=retention_authority.eligible_raw_ids, + ) + finally: + lease.close() if result.errors: logger.warning("live.watcher: raw snapshot compaction errors: %s", "; ".join(result.errors[:3])) diff --git a/polylogue/sources/live/batch_observability.py b/polylogue/sources/live/batch_observability.py index 6f907e6d35..8f90699fe8 100644 --- a/polylogue/sources/live/batch_observability.py +++ b/polylogue/sources/live/batch_observability.py @@ -17,6 +17,7 @@ read_peak_rss_children_mb, read_peak_rss_self_mb, ) +from polylogue.storage.archive_identity import resolve_active_index_path def record_attempt_progress( @@ -126,7 +127,7 @@ def session_ids_for_source_path(path: Path, *, archive_root: Path | None = None) def _schema_archive_session_ids_for_source_path(archive_root: Path, path: Path) -> tuple[str, ...]: - index_db = archive_root / "index.db" + index_db = resolve_active_index_path(archive_root) source_db = archive_root / "source.db" if not index_db.exists() or not source_db.exists(): return () diff --git a/polylogue/sources/live/batch_support.py b/polylogue/sources/live/batch_support.py index cd4ff8efe7..ff7a7f8942 100644 --- a/polylogue/sources/live/batch_support.py +++ b/polylogue/sources/live/batch_support.py @@ -12,13 +12,21 @@ import ijson -from polylogue.archive.artifact_taxonomy import classify_artifact, classify_artifact_path -from polylogue.archive.raw_payload.decode import jsonl_session_artifact +from polylogue.archive.artifact_taxonomy import ( + classify_artifact, + classify_artifact_path, + strong_path_classification, +) +from polylogue.archive.raw_payload.decode import ( + JSONL_RECORD_INSPECTION_BYTES, + _sample_jsonl_payload_with_detail, + jsonl_session_artifact, +) from polylogue.core.enums import Provider from polylogue.core.json import JSONDecodeError, JSONValue from polylogue.core.json import loads as json_loads from polylogue.pipeline.services.ingest_batch._core import _select_ingest_worker_count -from polylogue.sources.dispatch import _detect_provider_from_raw_bytes, detect_provider +from polylogue.sources.dispatch import _detect_provider_from_raw_bytes, detect_provider, is_jsonl_source_path from polylogue.sources.parsers import hermes_state, hermes_verification from polylogue.storage.runtime import RawSessionRecord @@ -129,14 +137,14 @@ class _AppendPlan: ctime_ns: int | None = None accepted_prefix_hash: str | None = None authority_bytes_read: int = 0 - # polylogue-u19l: the resolved provider session identity for this append, - # when the provider's own record stream cannot self-describe it (Codex - # append deltas have no ``session_meta`` record of their own). Carried as - # sidecar metadata -- persisted to ``raw_sessions.native_id`` and used to - # override the replay ``fallback_id`` -- instead of being injected into - # the hashed/stored payload bytes, so the stored blob stays a literal - # slice of the live file. ``None`` for providers/plans that don't need it. + # The resolved logical session identity used to bind this append and as a + # parser fallback when its own record stream cannot self-describe it. native_id_hint: str | None = None + # Acquisition identity is deliberately separate from logical identity. + # Codex append rows introduced this sidecar together with literal delta + # bytes. Claude append rows predate it with native_id=NULL, so retaining + # NULL keeps deterministic raw IDs stable across upgrades and retries. + acquisition_native_id_hint: str | None = None @dataclass(frozen=True, slots=True) @@ -536,18 +544,15 @@ def _browser_capture_provider_from_path(path: Path) -> Provider | None: def _jsonl_sample_from_path(path: Path, *, max_records: int = 32) -> list[JSONValue]: - records: list[JSONValue] = [] - with path.open("rb") as handle: - for line in handle: - if len(records) >= max_records: - break - raw = line.strip() - if not raw: - continue - try: - records.append(json_loads(raw)) - except JSONDecodeError: - continue + try: + records, _malformed_lines, _malformed_detail = _sample_jsonl_payload_with_detail( + path, + max_samples=max_records, + scan_full=False, + max_record_bytes=JSONL_RECORD_INSPECTION_BYTES, + ) + except ValueError: + return [] return records @@ -556,7 +561,7 @@ def _detect_provider_from_path_sample(path: Path, fallback_provider: Provider) - path ): return Provider.HERMES - if path.suffix.lower() == ".jsonl": + if is_jsonl_source_path(str(path)): records = _jsonl_sample_from_path(path) if records: return detect_provider(records) or fallback_provider @@ -591,12 +596,12 @@ def _parse_path_as_session_artifact(path: Path, *, provider: Provider) -> bool: or hermes_verification.looks_like_verification_evidence_db_path(path) ): return True - if path.suffix.lower() == ".jsonl": + if is_jsonl_source_path(str(path)): if jsonl_session_artifact(path, provider=provider) is not None: return True path_classification = classify_artifact_path(path, provider=provider) return path_classification.parse_as_session if path_classification is not None else False - path_classification = classify_artifact_path(path, provider=provider) + path_classification = strong_path_classification(path, provider=provider) if path_classification is not None: return path_classification.parse_as_session if _path_size(path) > _STREAMING_FULL_INGEST_BYTES: @@ -638,12 +643,12 @@ def _parse_payload_as_session_artifact(path: Path, *, provider: Provider, payloa return hermes_state.looks_like_state_db_path( path ) or hermes_verification.looks_like_verification_evidence_db_path(path) - if path.suffix.lower() == ".jsonl": + if is_jsonl_source_path(str(path)): if jsonl_session_artifact(payload, provider=provider) is not None: return True path_classification = classify_artifact_path(path, provider=provider) return path_classification.parse_as_session if path_classification is not None else False - path_classification = classify_artifact_path(path, provider=provider) + path_classification = strong_path_classification(path, provider=provider) if path_classification is not None: return path_classification.parse_as_session try: diff --git a/polylogue/sources/live/convergence_debt_retry.py b/polylogue/sources/live/convergence_debt_retry.py index cab92122e2..22f4730d2c 100644 --- a/polylogue/sources/live/convergence_debt_retry.py +++ b/polylogue/sources/live/convergence_debt_retry.py @@ -6,6 +6,7 @@ from datetime import UTC, datetime from pathlib import Path +from polylogue.storage.archive_identity import resolve_active_index_path from polylogue.storage.introspection import table_exists as _table_exists _HOT_INSIGHT_DEFERRED = "insights deferred until source quiet" @@ -69,7 +70,7 @@ def convergence_debt_source_path( def _archive_convergence_debt_source_path_from_root(archive_root: Path, session_id: str) -> Path | None: - index_db = archive_root / "index.db" + index_db = resolve_active_index_path(archive_root) source_db = archive_root / "source.db" if not index_db.exists() or not source_db.exists(): return None diff --git a/polylogue/sources/live/source_selection.py b/polylogue/sources/live/source_selection.py new file mode 100644 index 0000000000..b24ca87fc6 --- /dev/null +++ b/polylogue/sources/live/source_selection.py @@ -0,0 +1,36 @@ +"""Deterministic ownership for overlapping live-source roots.""" + +from __future__ import annotations + +from collections.abc import Iterable +from pathlib import Path +from typing import Protocol, TypeVar + + +class RootedSource(Protocol): + @property + def root(self) -> Path: ... + + +SourceT = TypeVar("SourceT", bound=RootedSource) + + +def deepest_source_for_path(path: Path, sources: Iterable[SourceT]) -> SourceT | None: + """Return the most-specific configured source owning ``path``.""" + + try: + resolved = path.resolve() + except OSError: + return None + matches: list[tuple[int, SourceT]] = [] + for source in sources: + try: + source_root = source.root.resolve() + if resolved.is_relative_to(source_root): + matches.append((len(source_root.parts), source)) + except (OSError, ValueError): + continue + return max(matches, key=lambda match: match[0])[1] if matches else None + + +__all__ = ["deepest_source_for_path"] diff --git a/polylogue/sources/live/watcher.py b/polylogue/sources/live/watcher.py index a57ac556ef..4c8143f8bc 100644 --- a/polylogue/sources/live/watcher.py +++ b/polylogue/sources/live/watcher.py @@ -31,6 +31,7 @@ from polylogue.logging import get_logger from polylogue.sources.hooks import drain_hook_event_spool, hook_spool_root, pending_hook_spool_dir from polylogue.sources.live.acquisition_log import log_unclaimed_file +from polylogue.sources.live.archive_open import _source_tier_acquisition_required from polylogue.sources.live.batch import ( CursorAuthorityBlockedError, LiveBatchEventEmitter, @@ -51,7 +52,9 @@ from polylogue.sources.live.deferred_cursor import record_deferred_append_cursor from polylogue.sources.live.metrics import LiveBatchMetrics from polylogue.sources.live.parse_prefetch import LiveParseStage +from polylogue.sources.live.source_selection import deepest_source_for_path from polylogue.sources.sqlite_snapshot import is_sqlite_path, sqlite_database_for_sidecar, sqlite_source_revision +from polylogue.storage.archive_identity import ArchiveLocationError, resolve_active_index_path if TYPE_CHECKING: from polylogue.api import Polylogue @@ -61,6 +64,13 @@ # One bounded writer hold per hook-spool drain batch; the drain loops until # the backlog is gone, releasing the writer between batches. _HOOK_SPOOL_DRAIN_BATCH_LIMIT = 250 +# A hook creates a day-shard directory before atomically publishing its first +# envelope. An added-directory event can therefore precede the child-file +# event that a recursive watcher is about to install. Poll only that new shard +# until its first envelope is visible, rather than leaving it to periodic +# catch-up or relying on a scheduler-dependent fixed grace period. +_HOOK_SPOOL_DIRECTORY_RETRY_POLL_S = 0.05 +_HOOK_SPOOL_DIRECTORY_RETRY_MAX_SECONDS = 5.0 # A catch-up writer owns the only archive writer for the whole chunk. The # former 50-file/64-MiB envelope held it for 14+ minutes on the real archive, # starving fresh watcher events. Keep historical convergence fair by @@ -262,6 +272,7 @@ def __init__( self._drain_task: asyncio.Task[None] | None = None self._failed_retry_task: asyncio.Task[None] | None = None self._periodic_catch_up_task: asyncio.Task[None] | None = None + self._hook_spool_directory_retry_tasks: dict[Path, asyncio.Task[None]] = {} self._failed_retry_deadline: float | None = None self._last_enqueue_at = 0.0 self._last_batch_at: float = 0.0 @@ -306,6 +317,10 @@ async def _run_writer_sync( def catch_up_complete(self) -> asyncio.Event: return self._catch_up_complete + def _existing_source_roots(self) -> list[Path]: + """Return configured roots that exist at the instant of a scan.""" + return [source.root for source in self._sources if source.exists()] + async def run(self) -> None: # Hook commands create their first pending envelope lazily. Ensure the # nested root exists before ``awatch`` snapshots its roots, otherwise a @@ -313,7 +328,7 @@ async def run(self) -> None: for source in self._sources: if source.name == "hooks": source.root.mkdir(parents=True, exist_ok=True) - roots = [s.root for s in self._sources if s.exists()] + roots = self._existing_source_roots() if not roots: logger.warning("live.watcher: no source roots exist; nothing to watch") self._catch_up_complete.set() @@ -341,6 +356,7 @@ async def run(self) -> None: with suppress(asyncio.CancelledError): await watch_task self._cancel_periodic_catch_up() + self._cancel_hook_spool_directory_retries() async def _watch_changes(self, roots: list[Path]) -> None: from watchfiles import Change, awatch @@ -354,7 +370,19 @@ async def _watch_changes(self, roots: list[Path]) -> None: for change, raw_path in changes: if change is Change.deleted: continue - path = self._canonical_watch_path(Path(raw_path)) + observed_path = Path(raw_path) + if change is Change.added and observed_path.is_dir(): + if self._is_hook_spool_path(observed_path): + needs_first_envelope_retry = self._is_hook_spool_shard_directory( + observed_path + ) and not self._hook_spool_directory_has_envelope(observed_path) + await self._drain_hook_spool() + if needs_first_envelope_retry: + self._schedule_hook_spool_directory_retry(observed_path) + continue + self._enqueue_added_directory(observed_path) + continue + path = self._canonical_watch_path(observed_path) if path is None: continue if not self._source_accepts(path): @@ -368,6 +396,7 @@ def stop(self) -> None: self._stop.set() self._cancel_failed_retry_task() self._cancel_periodic_catch_up() + self._cancel_hook_spool_directory_retries() if self._parse_stage is not None and self._owns_parse_stage: self._parse_stage.shutdown() @@ -379,15 +408,18 @@ def cancel_pending(self) -> None: self._pending_scheduled = False self._cancel_failed_retry_task() self._cancel_periodic_catch_up() + self._cancel_hook_spool_directory_retries() - async def _periodic_catch_up(self, roots: list[Path]) -> None: + async def _periodic_catch_up(self, _initial_roots: list[Path]) -> None: delay_s = _PERIODIC_CATCH_UP_INTERVAL_S while not self._stop.is_set(): await asyncio.sleep(delay_s) if self._stop.is_set(): return try: - await self._catch_up(roots) + roots = self._existing_source_roots() + if roots: + await self._catch_up(roots) except sqlite3.OperationalError as exc: if not _is_database_locked(exc): raise @@ -404,6 +436,67 @@ def _cancel_periodic_catch_up(self) -> None: task.cancel() self._periodic_catch_up_task = None + def _schedule_hook_spool_directory_retry(self, directory: Path) -> None: + """Drain a newly added hook shard when its first envelope appears. + + The initial drain above handles an already-published envelope. This + task covers the narrow event-ordering race where the directory arrives + first and the recursive watcher misses the first child notification. + It does not block subsequent watcher events or start a source-tree + catch-up scan. + """ + + directory = directory.resolve() + existing = self._hook_spool_directory_retry_tasks.get(directory) + if existing is not None and not existing.done(): + return + task = asyncio.create_task(self._retry_hook_spool_directory_until_populated(directory)) + self._hook_spool_directory_retry_tasks[directory] = task + + def discard_completed_task(completed: asyncio.Task[None]) -> None: + if self._hook_spool_directory_retry_tasks.get(directory) is completed: + self._hook_spool_directory_retry_tasks.pop(directory, None) + if completed.cancelled(): + return + try: + completed.result() + except Exception: + logger.exception("live.watcher: hook spool directory retry failed for %s", directory) + + task.add_done_callback(discard_completed_task) + + async def _retry_hook_spool_directory_until_populated(self, directory: Path) -> None: + """Wait for an added shard's first envelope until it is acknowledged.""" + + deadline = asyncio.get_running_loop().time() + _HOOK_SPOOL_DIRECTORY_RETRY_MAX_SECONDS + delay_s = _HOOK_SPOOL_DIRECTORY_RETRY_POLL_S + while not self._stop.is_set() and asyncio.get_running_loop().time() < deadline: + try: + if not directory.exists(): + return + if any(directory.glob("*.json")): + await self._drain_hook_spool() + if not any(directory.glob("*.json")): + return + except sqlite3.OperationalError as exc: + # The normal periodic catch-up route retries transient source + # tier contention. A just-created shard must get the same + # treatment instead of letting this narrow event-ordering + # recovery task die before its envelope is acknowledged. + if not _is_database_locked(exc): + raise + logger.warning("live.watcher: archive busy while draining new hook shard; will retry") + except OSError: + return + await asyncio.sleep(delay_s) + delay_s = min(delay_s * 2, 0.5) + + def _cancel_hook_spool_directory_retries(self) -> None: + for task in tuple(self._hook_spool_directory_retry_tasks.values()): + if not task.done(): + task.cancel() + self._hook_spool_directory_retry_tasks.clear() + # ------------------------------------------------------------------ # Catch-up: batch all changed files # ------------------------------------------------------------------ @@ -620,6 +713,8 @@ def _scan_catch_up_candidates(self, roots: list[Path]) -> tuple[CandidateSourceF ] for filename in filenames: path = Path(directory) / filename + if deepest_source_for_path(path, self._sources) is not source: + continue if not source.accepts(path): # Unclaimed-file sweep (mission item 2): a file this # source's own root walk reached but whose suffix no @@ -1201,9 +1296,20 @@ def _archived_cursor_reconciliation_scope(self) -> Iterator[None]: cached connection would keep reading a replaced index.db inode across a blue-green generation swap. """ + if _source_tier_acquisition_required(): + self._archived_cursor_conns = None + self._archived_cursor_index_untrusted = False + yield + return archive_root = Path(getattr(self._polylogue, "archive_root", self._cursor._db_path.parent)) source_db = archive_root / "source.db" - index_db = archive_root / "index.db" + try: + index_db = resolve_active_index_path(archive_root) + except (ArchiveLocationError, OSError, UnicodeError): + self._archived_cursor_conns = None + self._archived_cursor_index_untrusted = False + yield + return conns: tuple[sqlite3.Connection, sqlite3.Connection] | None = None if source_db.exists() and index_db.exists(): try: @@ -1330,7 +1436,7 @@ def _cursor_skip_corroborated_by_index(self, path: Path) -> bool: return self._path_corroborated_by_index(path, source_conn=shared[0], index_conn=shared[1]) archive_root = Path(getattr(self._polylogue, "archive_root", self._cursor._db_path.parent)) source_db = archive_root / "source.db" - index_db = archive_root / "index.db" + index_db = resolve_active_index_path(archive_root) if not source_db.exists() or not index_db.exists(): return True with ( @@ -1338,7 +1444,7 @@ def _cursor_skip_corroborated_by_index(self, path: Path) -> bool: closing(sqlite3.connect(f"file:{index_db}?mode=ro", uri=True, timeout=1.0)) as index_conn, ): return self._path_corroborated_by_index(path, source_conn=source_conn, index_conn=index_conn) - except sqlite3.Error: + except (ArchiveLocationError, OSError, UnicodeError, sqlite3.Error): # Cannot prove absence on a transient DB error -- don't force a # spurious re-ingest of an otherwise-healthy cursor. return True @@ -1359,6 +1465,11 @@ def _reconcile_archived_cursor_outcome( archived prefix so catch-up can take the append path instead of parsing the whole active JSONL again. """ + if _source_tier_acquisition_required(): + # Derived corroboration is inapplicable in acquire-only mode. + # Force a fresh source observation instead of deferring on an + # index that this mode is explicitly forbidden to read. + return _ArchivedCursorReconciliation.INCOMPATIBLE shared = self._archived_cursor_conns archive_root = Path(getattr(self._polylogue, "archive_root", self._cursor._db_path.parent)) try: @@ -1366,7 +1477,7 @@ def _reconcile_archived_cursor_outcome( row = self._archived_cursor_row(path, source_conn=shared[0], index_conn=shared[1]) else: source_db = archive_root / "source.db" - index_db = archive_root / "index.db" + index_db = resolve_active_index_path(archive_root) if not source_db.exists() or not index_db.exists(): return _ArchivedCursorReconciliation.UNAVAILABLE with ( @@ -1374,7 +1485,7 @@ def _reconcile_archived_cursor_outcome( closing(sqlite3.connect(f"file:{index_db}?mode=ro", uri=True, timeout=1.0)) as index_conn, ): row = self._archived_cursor_row(path, source_conn=source_conn, index_conn=index_conn) - except sqlite3.Error: + except (ArchiveLocationError, OSError, UnicodeError, sqlite3.Error): return _ArchivedCursorReconciliation.UNAVAILABLE if row is None: return _ArchivedCursorReconciliation.INCOMPATIBLE @@ -1517,35 +1628,44 @@ async def _run_coordinated(self, actor: str, operation: Callable[[], Awaitable[N await operation() def _source_name_for(self, path: Path) -> str: - resolved = path.resolve() - for source in self._sources: - try: - if resolved.is_relative_to(source.root.resolve()): - return source.name - except OSError: - continue + source = deepest_source_for_path(path, self._sources) + if source is not None: + return source.name return path.parent.name def _source_accepts(self, path: Path) -> bool: - resolved = path.resolve() + source = deepest_source_for_path(path, self._sources) + return source.accepts(path) if source is not None else False + + def _is_hook_spool_path(self, path: Path) -> bool: for source in self._sources: + if source.name != "hooks": + continue try: - if resolved.is_relative_to(source.root.resolve()): - return source.accepts(path) + return path.resolve().is_relative_to(source.root.resolve()) except OSError: - continue - return path.suffix == ".jsonl" + return False + return False + + def _is_hook_spool_shard_directory(self, path: Path) -> bool: + """Return whether ``path`` is a direct day shard beneath ``pending``.""" - def _is_hook_spool_path(self, path: Path) -> bool: for source in self._sources: if source.name != "hooks": continue try: - return path.resolve().is_relative_to(source.root.resolve()) + return path.resolve().parent == source.root.resolve() except OSError: return False return False + @staticmethod + def _hook_spool_directory_has_envelope(directory: Path) -> bool: + try: + return next(directory.glob("*.json"), None) is not None + except OSError: + return False + def _hook_spool_root(self) -> Path: """Return the root paired with this watcher's hook source.""" @@ -1574,6 +1694,48 @@ def _canonical_watch_path(self, path: Path) -> Path | None: return database return None + def _source_for_directory(self, path: Path) -> WatchSource | None: + """Return the watched source owning a non-ignored directory.""" + + source = deepest_source_for_path(path, self._sources) + if source is None: + return None + try: + relative = path.resolve().relative_to(source.root.resolve()) + except (OSError, ValueError): + return None + return None if any(source.ignores_directory(Path(part)) for part in relative.parts) else source + + def _directory_is_watch_relevant(self, path: Path) -> bool: + """Return whether a directory is owned or leads to a configured source root.""" + + if self._source_for_directory(path) is not None: + return True + try: + resolved = path.resolve() + except OSError: + return False + for source in self._sources: + try: + if source.root.resolve().is_relative_to(resolved): + return True + except (OSError, ValueError): + continue + return False + + def _enqueue_added_directory(self, directory: Path) -> None: + """Cover files created before a recursive watcher installs its new sub-watch.""" + + if not self._directory_is_watch_relevant(directory): + return + for parent, dir_names, file_names in os.walk(directory): + dir_names[:] = [name for name in dir_names if self._directory_is_watch_relevant(Path(parent) / name)] + for name in file_names: + candidate = Path(parent) / name + canonical = self._canonical_watch_path(candidate) + if canonical is not None: + self._enqueue(canonical) + def _watch_filter(self, _change: object, path: str) -> bool: """Accept configured source files under hidden canonical roots. @@ -1583,7 +1745,10 @@ def _watch_filter(self, _change: object, path: str) -> bool: writes. This filter keeps the project's own source/suffix predicate as the gate instead. """ - return self._canonical_watch_path(Path(path)) is not None + observed_path = Path(path) + return self._canonical_watch_path(observed_path) is not None or ( + observed_path.is_dir() and self._directory_is_watch_relevant(observed_path) + ) def _interleave_by_source(candidates: list[CandidateSourceFile]) -> list[CandidateSourceFile]: @@ -1729,7 +1894,14 @@ def _cursor_db_path(polylogue: Polylogue) -> Path: def _is_database_locked(exc: sqlite3.OperationalError) -> bool: - return "database is locked" in str(exc).lower() + error_code = getattr(exc, "sqlite_errorcode", None) + if error_code in {sqlite3.SQLITE_BUSY, sqlite3.SQLITE_LOCKED}: + return True + message = str(exc).lower() + return any( + locked_message in message + for locked_message in ("database is locked", "database table is locked", "database schema is locked") + ) def _cursor_age_exceeds(cursor: CursorRecord, min_age_s: float) -> bool: diff --git a/polylogue/sources/revision_backfill.py b/polylogue/sources/revision_backfill.py index 209b9ceb83..2760490aa4 100644 --- a/polylogue/sources/revision_backfill.py +++ b/polylogue/sources/revision_backfill.py @@ -9,15 +9,17 @@ import threading import time from collections import OrderedDict -from collections.abc import Callable, Iterator, Sequence +from collections.abc import Callable, Iterator, Sequence, Set from concurrent.futures import Future, ThreadPoolExecutor from contextlib import closing, contextmanager, nullcontext from dataclasses import dataclass, field from io import BytesIO -from itertools import chain, islice from pathlib import Path from types import TracebackType -from typing import BinaryIO, Final, Literal, cast +from typing import BinaryIO, Final, Literal, Protocol, cast + +import ijson +from ijson.common import ObjectBuilder from polylogue import logging as _polylogue_logging from polylogue.archive.artifact_taxonomy.models import ArtifactClassification, ArtifactKind @@ -44,17 +46,24 @@ parallel_threads_effective, resolve_revision_backfill_census_dispatch, ) +from polylogue.sources.codex_state_evidence import write_codex_thread_state_evidence from polylogue.sources.decoders import _iter_json_stream from polylogue.sources.dispatch import ( + detect_provider_evidence, + detect_provider_from_raw_bytes_evidence, + is_jsonl_source_path, is_stream_record_provider, parse_payload, parse_stream_payload, require_positive_conversational_evidence, ) from polylogue.sources.origin_specs import artifact_rule_for_path -from polylogue.sources.parsers import antigravity, hermes_state, hermes_verification +from polylogue.sources.parsers import antigravity, codex_state, hermes_state, hermes_verification from polylogue.sources.parsers.base import ParsedSession from polylogue.sources.sqlite_snapshot import looks_like_sqlite_bytes +from polylogue.storage.archive_identity import ArchiveLocation +from polylogue.storage.artifacts.inspection import artifact_observation_id +from polylogue.storage.raw.models import RawSessionStateUpdate from polylogue.storage.raw_authority import ( RAW_AUTHORITY_PARSER_FINGERPRINT, SUPERSEDED_MEMBERSHIP_FINGERPRINTS, @@ -63,11 +72,549 @@ from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore from polylogue.storage.sqlite.archive_tiers.revision_governance import ( FrozenSourceRemediationRequiredError, + _raw_parse_success_state, record_current_parser_source_census, ) +from polylogue.storage.sqlite.archive_tiers.source_write import ( + ArchiveSourceArtifact, + apply_source_raw_state_update, + upsert_raw_artifact, +) from polylogue.storage.sqlite.archive_tiers.write import PreparedSessionRows, prepare_session_rows _LOGGER = _polylogue_logging.get_logger(__name__) +_REPLAY_PROVIDER_DETECTION_PREFIX_BYTES: Final[int] = 8192 +_REPLAY_PROVIDER_DETECTION_MAX_SCAN_BYTES: Final[int] = 64 * 1024 +_REPLAY_PROVIDER_DETECTION_READ_CHUNK_BYTES: Final[int] = 4096 + + +class _ReadableBinary(Protocol): + def read(self, size: int = -1) -> bytes: ... + + +class _LineReadableBinary(_ReadableBinary, Protocol): + def readline(self, size: int = -1) -> bytes: ... + + +class _SeekableReadableBinary(_LineReadableBinary, Protocol): + def seek(self, offset: int, whence: int = 0) -> int: ... + + +_DOCUMENT_PROBE_ROOT_KEYS: Final[frozenset[str]] = frozenset( + { + "account_uuid", + "artifactType", + "cascadeId", + "chat_messages", + "chunkedPrompt", + "chunks", + "conversation_id", + "conversations", + "conversations_memory", + "create_time", + "current_node", + "cwd", + "id", + "kind", + "lastUpdated", + "last_updated", + "leafUuid", + "mapping", + "markdown", + "message", + "messages", + "parentUuid", + "payload", + "platform", + "polylogue_capture_kind", + "project", + "projectHash", + "project_memories", + "record_type", + "session", + "sessionId", + "session_id", + "session_start", + "shared_conversation_id", + "source", + "startTime", + "summary", + "type", + "updatedAt", + "uuid", + "version", + } +) +_DOCUMENT_PROBE_EXACT_STRING_KEYS: Final[frozenset[str]] = frozenset( + {"kind", "polylogue_capture_kind", "record_type", "role", "source", "type"} +) +_DOCUMENT_PROBE_CHUNK_CONTENT_KEYS: Final[frozenset[str]] = frozenset( + { + "codeExecutionResult", + "driveAudio", + "driveDocument", + "driveImage", + "driveVideo", + "errorMessage", + "executableCode", + "grounding", + "inlineFile", + "inlineImage", + "isThought", + "parts", + "text", + "youtubeVideo", + } +) + + +def _document_probe_value(key: str, event: str, value: object) -> object | None: + """Retain only detector-relevant scalar type/equality evidence.""" + if event == "string": + return str(value) if key in _DOCUMENT_PROBE_EXACT_STRING_KEYS else "present" + if event == "number": + return 0 + if event == "boolean": + return bool(value) + if event == "null": + return None + return None + + +class _ScalarBoundedJSONReader: + """Stream JSON while capping every scalar token before ijson sees it.""" + + def __init__(self, payload: _ReadableBinary) -> None: + self._payload = payload + self._output = bytearray() + self._eof = False + self._in_string = False + self._string_bytes = 0 + self._escape = bytearray() + self._escape_target = 0 + self._utf8_remaining = 0 + self._emit_utf8 = False + self._in_number = False + + def read(self, size: int = -1) -> bytes: + if size == 0: + return b"" + if size < 0: + chunks: list[bytes] = [] + while chunk := self.read(_REPLAY_PROVIDER_DETECTION_PREFIX_BYTES): + chunks.append(chunk) + return b"".join(chunks) + while len(self._output) < size and not self._eof: + chunk = self._payload.read(_REPLAY_PROVIDER_DETECTION_PREFIX_BYTES) + if not chunk: + self._eof = True + break + self._filter(chunk) + result = bytes(self._output[:size]) + del self._output[:size] + return result + + def _filter(self, chunk: bytes) -> None: + for byte in chunk: + if self._in_string: + self._filter_string_byte(byte) + continue + if self._in_number: + if byte in b"0123456789.eE+-": + continue + self._in_number = False + if byte == ord('"'): + self._output.append(byte) + self._in_string = True + self._string_bytes = 0 + elif byte in b"-0123456789": + self._output.extend(b"0") + self._in_number = True + else: + self._output.append(byte) + + def _filter_string_byte(self, byte: int) -> None: + if self._escape: + self._escape.append(byte) + if len(self._escape) == 2: + self._escape_target = 6 if byte == ord("u") else 2 + if len(self._escape) == self._escape_target: + if self._string_bytes + len(self._escape) <= _REPLAY_PROVIDER_DETECTION_PREFIX_BYTES: + self._output.extend(self._escape) + self._string_bytes += len(self._escape) + self._escape.clear() + self._escape_target = 0 + return + if self._utf8_remaining: + if self._emit_utf8: + self._output.append(byte) + self._utf8_remaining -= 1 + return + if byte == ord("\\"): + self._escape.append(byte) + return + if byte == ord('"'): + self._output.append(byte) + self._in_string = False + return + if byte < 0x80: + if self._string_bytes < _REPLAY_PROVIDER_DETECTION_PREFIX_BYTES: + self._output.append(byte) + self._string_bytes += 1 + return + utf8_bytes = 2 if byte < 0xE0 else 3 if byte < 0xF0 else 4 if byte < 0xF8 else 1 + self._utf8_remaining = utf8_bytes - 1 + self._emit_utf8 = self._string_bytes + utf8_bytes <= _REPLAY_PROVIDER_DETECTION_PREFIX_BYTES + if self._emit_utf8: + self._output.append(byte) + self._string_bytes += utf8_bytes + + +class _BoundedJSONLRecordReader: + """Expose one JSONL record without exceeding the shared scan budget.""" + + def __init__(self, payload: _LineReadableBinary, byte_budget: int) -> None: + self._payload = payload + self._remaining = byte_budget + self.bytes_read = 0 + self._done = False + + def read(self, size: int = -1) -> bytes: + if size == 0 or self._done: + return b"" + if self._remaining <= 0: + self._done = True + return b"" + read_size = _REPLAY_PROVIDER_DETECTION_READ_CHUNK_BYTES if size < 0 else size + read_size = min(read_size, _REPLAY_PROVIDER_DETECTION_READ_CHUNK_BYTES, self._remaining) + chunk = self._payload.readline(read_size) + self.bytes_read += len(chunk) + self._remaining -= len(chunk) + if not chunk or chunk.endswith(b"\n") or self._remaining <= 0: + self._done = True + return chunk + + def drain(self) -> None: + """Consume this physical record, subject to the remaining scan budget.""" + while not self._done: + self.read(_REPLAY_PROVIDER_DETECTION_READ_CHUNK_BYTES) + + +@dataclass(slots=True) +class _StreamingDocumentProviderProbe: + """Bounded structural summary for one object in a JSON document. + + Cardinality is fixed by provider detector fields. Large scalar bodies, + unrelated keys, repeated messages, and repeated mapping nodes are never + retained; the ijson event stream can therefore continue to EOF without a + whole-document allocation or a scan-count cutoff. + """ + + payload: dict[str, object] = field(default_factory=dict) + mapping_seen: bool = False + mapping_valid: bool = True + mapping_node_open: bool = False + mapping_node_message: Literal["absent", "null", "map", "invalid"] = "absent" + mapping_node_author: bool = False + chat_message_item: dict[str, object] = field(default_factory=dict) + chat_message_matched: bool = False + first_message_item: dict[str, object] = field(default_factory=dict) + first_message_complete: bool = False + chunk_item: dict[str, object] = field(default_factory=dict) + chunk_matched: bool = False + conversation_item_has_conversation: bool = False + conversation_item_has_responses: bool = False + conversation_matched: bool = False + + def _set_root(self, key: str, event: str, value: object) -> None: + if key not in _DOCUMENT_PROBE_ROOT_KEYS: + return + if event == "start_map": + self.payload[key] = {} + elif event == "start_array": + self.payload[key] = [] + elif event in {"string", "number", "boolean", "null"}: + self.payload[key] = _document_probe_value(key, event, value) + + @staticmethod + def _set_item_value(item: dict[str, object], key: str, event: str, value: object) -> None: + if event == "start_map": + item[key] = {} + elif event == "start_array": + item[key] = [] + elif event in {"string", "number", "boolean", "null"}: + item[key] = _document_probe_value(key, event, value) + + def feed(self, prefix: str, event: str, value: object) -> None: + parts = prefix.split(".") if prefix else [] + if len(parts) == 1: + self._set_root(parts[0], event, value) + + if parts == ["session", "provider"] and event == "string": + session = self.payload.setdefault("session", {}) + if isinstance(session, dict): + session["provider"] = str(value) + + if len(parts) == 2 and parts[0] == "payload": + nested = self.payload.setdefault("payload", {}) + if isinstance(nested, dict): + self._set_item_value(nested, parts[1], event, value) + + if len(parts) == 2 and parts[0] == "mapping": + if event == "start_map": + self.mapping_seen = True + self.mapping_node_open = True + self.mapping_node_message = "absent" + self.mapping_node_author = False + elif event not in {"end_map", "map_key"}: + self.mapping_seen = True + self.mapping_valid = False + elif len(parts) == 3 and parts[0] == "mapping" and parts[2] == "message": + if event == "start_map": + self.mapping_node_message = "map" + elif event == "null": + self.mapping_node_message = "null" + elif event not in {"map_key", "end_map"}: + self.mapping_node_message = "invalid" + elif len(parts) == 4 and parts[0] == "mapping" and parts[2:] == ["message", "author"] and event == "start_map": + self.mapping_node_author = True + + if len(parts) == 2 and parts == ["chat_messages", "item"] and event == "start_map": + self.chat_message_item = {} + elif len(parts) == 3 and parts[:2] == ["chat_messages", "item"]: + self._set_item_value(self.chat_message_item, parts[2], event, value) + + if len(parts) == 2 and parts == ["messages", "item"] and event == "start_map": + if not self.first_message_complete: + self.first_message_item = {} + elif len(parts) == 3 and parts[:2] == ["messages", "item"] and not self.first_message_complete: + self._set_item_value(self.first_message_item, parts[2], event, value) + + chunk_root = parts[:2] == ["chunks", "item"] + chunk_nested = parts[:3] == ["chunkedPrompt", "chunks", "item"] + if (chunk_root and len(parts) == 2 or chunk_nested and len(parts) == 3) and event == "start_map": + self.chunk_item = {} + elif chunk_root and len(parts) == 3: + self._set_item_value(self.chunk_item, parts[2], event, value) + elif chunk_nested and len(parts) == 4: + self._set_item_value(self.chunk_item, parts[3], event, value) + + if parts == ["conversations", "item"] and event == "start_map": + self.conversation_item_has_conversation = False + self.conversation_item_has_responses = False + elif parts == ["conversations", "item", "conversation"] and event == "start_map": + self.conversation_item_has_conversation = True + elif parts == ["conversations", "item", "responses"] and event == "start_array": + self.conversation_item_has_responses = True + + if event != "end_map": + return + if len(parts) == 2 and parts[0] == "mapping" and self.mapping_node_open: + if ( + self.mapping_node_message == "map" and not self.mapping_node_author + ) or self.mapping_node_message == "invalid": + self.mapping_valid = False + self.mapping_node_open = False + elif parts == ["chat_messages", "item"]: + has_role = any(key in self.chat_message_item for key in ("sender", "role", "author")) + has_content = any(key in self.chat_message_item for key in ("text", "content")) + self.chat_message_matched |= has_role and has_content + elif parts == ["messages", "item"] and not self.first_message_complete: + self.first_message_complete = True + elif parts in (["chunks", "item"], ["chunkedPrompt", "chunks", "item"]): + role = self.chunk_item.get("role") or self.chunk_item.get("author") + self.chunk_matched |= isinstance(role, str) and any( + key in self.chunk_item for key in _DOCUMENT_PROBE_CHUNK_CONTENT_KEYS + ) + elif parts == ["conversations", "item"]: + self.conversation_matched |= ( + self.conversation_item_has_conversation and self.conversation_item_has_responses + ) + + def classify(self, *, sequence_item: bool = False) -> tuple[Provider, str]: + if self.mapping_seen and self.mapping_valid: + self.payload["mapping"] = {"bounded-node": {"id": "bounded-node", "message": None}} + if self.chat_message_matched: + self.payload["chat_messages"] = [{"role": "present", "text": "present"}] + if self.first_message_complete: + self.payload["messages"] = [self.first_message_item] + if self.chunk_matched: + chunk = {"role": "present", "text": "present"} + if isinstance(self.payload.get("chunkedPrompt"), dict): + self.payload["chunkedPrompt"] = {"chunks": [chunk]} + else: + self.payload["chunks"] = [chunk] + if self.conversation_matched: + self.payload["conversations"] = [{"conversation": {}, "responses": []}] + candidate: object = [self.payload] if sequence_item else self.payload + provider, evidence = detect_provider_evidence(candidate) + if provider is None: + return Provider.UNKNOWN, evidence + return provider, f"bounded streaming JSON structure: {evidence}" + + +def _detect_provider_from_bounded_document(payload: _SeekableReadableBinary) -> tuple[Provider, str]: + """Scan every document object while retaining fixed structural evidence.""" + payload.seek(0) + bounded_payload = _ScalarBoundedJSONReader(payload) + probe: _StreamingDocumentProviderProbe | None = None + root_is_array = False + last_evidence = "no bounded document structure identified a provider; used fallback_provider" + try: + for prefix, event, value in ijson.parse(bounded_payload, use_float=True): + if prefix == "" and event == "start_array": + root_is_array = True + continue + if root_is_array: + if prefix == "item" and event == "start_map": + probe = _StreamingDocumentProviderProbe() + continue + if probe is None: + continue + if prefix == "item" and event == "end_map": + provider, last_evidence = probe.classify(sequence_item=True) + if provider is not Provider.UNKNOWN: + return provider, last_evidence + probe = None + continue + if prefix.startswith("item."): + probe.feed(prefix.removeprefix("item."), event, value) + continue + + if prefix == "" and event == "start_map": + probe = _StreamingDocumentProviderProbe() + continue + if probe is None: + continue + if prefix == "" and event == "end_map": + return probe.classify() + probe.feed(prefix, event, value) + except ijson.JSONError: + return Provider.UNKNOWN, last_evidence + return Provider.UNKNOWN, last_evidence + + +def _detect_provider_from_bounded_prefix( + prefix: bytes, + stream_name: str, + *, + record_stream: bool, +) -> tuple[Provider, str]: + """Classify completed structure exposed by a bounded JSON prefix. + + The ordinary raw-byte detector remains the first authority. When the + prefix ends inside one oversized JSON value, ijson still emits every + completed key/value event before that truncated value. Reconstructing + only those completed events preserves structural provider evidence (for + example a Codex ``session_meta`` envelope or a Claude ``sessionId``) + without retaining or completing the oversized record. + """ + provider, evidence = detect_provider_from_raw_bytes_evidence( + prefix, + stream_name, + Provider.UNKNOWN, + truncated_tail_ok=True, + ) + if provider is not Provider.UNKNOWN: + return provider, evidence + + builder = ObjectBuilder() + try: + for event, value in ijson.basic_parse(BytesIO(prefix), use_float=True): + builder.event(event, value) + except ijson.JSONError: + # Premature EOF is expected for an oversized-record prefix. The + # builder retains only values whose lexical token completed inside + # the bound; an unfinished string/object contributes no guessed data. + pass + partial = builder.value + candidate: object = [partial] if record_stream and isinstance(partial, dict) else partial + detected, partial_evidence = detect_provider_evidence(candidate) + if detected is None: + return Provider.UNKNOWN, evidence + return detected, f"bounded partial JSON structure: {partial_evidence}" + + +def _detect_provider_from_bounded_record( + record: _BoundedJSONLRecordReader, +) -> tuple[Provider, str]: + """Classify a streamed JSONL record from bounded structural evidence.""" + bounded_record = _ScalarBoundedJSONReader(record) + probe = _StreamingDocumentProviderProbe() + last_evidence = "no bounded JSONL record structure identified a provider" + try: + for prefix, event, value in ijson.parse(bounded_record, use_float=True): + if prefix == "": + if event == "start_map": + continue + if event == "end_map": + provider, last_evidence = probe.classify(sequence_item=True) + if provider is not Provider.UNKNOWN: + return provider, last_evidence + continue + elif prefix: + probe.feed(prefix, event, value) + except ijson.JSONError: + # A scan-budget cutoff or malformed JSON is expected for retained + # unknown bytes. Completed structural events are still valid evidence; + # an incomplete tail contributes nothing. + pass + provider, last_evidence = probe.classify(sequence_item=True) + return provider, last_evidence + + +def _detect_unknown_retained_provider( + payload: _SeekableReadableBinary, + source_path: str, +) -> tuple[Provider, str]: + """Detect retained UNKNOWN bytes without eagerly materializing JSONL. + + A byte prefix can end inside the first physical JSONL record. For an + oversized record stream that makes a prefix-only detector inconclusive + even when a later structural key in that record identifies a streaming + provider. Stream each record through the structural probe in bounded + chunks, stopping at positive evidence, the total scan envelope, or EOF. + + Non-JSONL documents first use the same prefix evidence, then continue a + bounded structural event scan through every document object. Eager replay + remains gated on a positive provider result from one of those bounded + passes; an unresolved UNKNOWN document is never materialized wholesale. + """ + stream_name = Path(source_path).name + if not is_jsonl_source_path(source_path): + provider, evidence = _detect_provider_from_bounded_prefix( + payload.read(_REPLAY_PROVIDER_DETECTION_PREFIX_BYTES), + stream_name, + record_stream=False, + ) + if provider is not Provider.UNKNOWN: + return provider, evidence + return _detect_provider_from_bounded_document(payload) + + last_evidence = "no bounded JSONL record identified a provider; used fallback_provider" + scanned_bytes = 0 + + while scanned_bytes < _REPLAY_PROVIDER_DETECTION_MAX_SCAN_BYTES: + record = _BoundedJSONLRecordReader( + payload, + _REPLAY_PROVIDER_DETECTION_MAX_SCAN_BYTES - scanned_bytes, + ) + provider, last_evidence = _detect_provider_from_bounded_record(record) + parsed_bytes = record.bytes_read + scanned_bytes += parsed_bytes + if provider is not Provider.UNKNOWN: + return provider, last_evidence + record.drain() + scanned_bytes += record.bytes_read - parsed_bytes + if record.bytes_read == 0: + break + return Provider.UNKNOWN, last_evidence + + +def _require_bounded_provider(provider: Provider, source_path: str) -> None: + """Refuse eager UNKNOWN replay after bounded structural detection.""" + if provider is Provider.UNKNOWN: + raise ValueError(f"retained UNKNOWN provider remained unresolved after bounded scan: {source_path}") def _canonical_authority_logical_key(logical_key: str) -> str: @@ -200,6 +747,7 @@ class _RevisionCensusState: censused: set[str] membership_candidates: dict[str, set[str]] provisional_full_raw_ids: dict[str, set[str]] + transient_non_session_raw_ids: set[str] @dataclass(slots=True) @@ -603,7 +1151,7 @@ def _census_historical_revision_evidence( replay) still independently re-derives byte-provenness from raw bytes for every raw. """ - state = _RevisionCensusState(0, 0, 0, set(), {}, {}) + state = _RevisionCensusState(0, 0, 0, set(), {}, {}, set()) batch_size = commit_batch_size if commit_batch_size is not None and commit_batch_size > 0 else None batched = batch_size is not None pending_commits = 0 @@ -655,6 +1203,66 @@ def apply_outcome( commit_unit() return sessions, payload_bytes, revision_kind = outcome + stored_provider, _blob_hash, _source_path, _stored_kind, _stored_size = archive.raw_revision_descriptor(raw_id) + if stored_provider is Provider.UNKNOWN and sessions: + # Acquisition deliberately did not decode an UNKNOWN source-only + # member. A successful replay now has durable shape evidence for + # its provider, so retain that result independently of the later + # index promotion outcome. + apply_source_raw_state_update( + archive._ensure_source_conn(), + raw_id, + state=RawSessionStateUpdate(payload_provider=Provider.from_string(sessions[0].source_name)), + manage_transaction=not batched, + ) + if not sessions: + provider = _detected_provider_for_empty_replay( + archive, + raw_id, + stored_provider=stored_provider, + source_path=_source_path, + ) + # A terminal artifact makes this raw ineligible for future census + # work. Its artifact carrier, parse state, and both census receipts + # must therefore become durable as one source-tier transaction. + # Batches retain that transaction until their existing commit + # boundary instead of forcing one SQLite commit per empty raw. + transaction = nullcontext() if batched else archive._ensure_source_conn() + with transaction: + if stored_provider is Provider.UNKNOWN and provider is not Provider.UNKNOWN: + apply_source_raw_state_update( + archive._ensure_source_conn(), + raw_id, + state=RawSessionStateUpdate(payload_provider=provider), + manage_transaction=False, + ) + terminalized = _persist_terminal_non_session_artifact( + archive, + raw_id, + provider=provider, + source_path=_source_path, + source_index=source_index, + manage_transaction=False, + ) + if provider is not Provider.UNKNOWN: + archive.replace_raw_membership_census( + raw_id, + [], + parser_fingerprint=RAW_AUTHORITY_PARSER_FINGERPRINT, + censused_at_ms=0, + retire_full_revision_governance=revision_kind is RawRevisionKind.FULL, + manage_transaction=False, + ) + if not terminalized: + apply_source_raw_state_update( + archive._ensure_source_conn(), + raw_id, + state=_raw_parse_success_state(provider), + manage_transaction=False, + ) + if provider is not Provider.UNKNOWN: + commit_unit() + return state.classified += int(len(sessions) == 1) spill.add(raw_id, sessions, payload_bytes=payload_bytes) if len(sessions) == 1 and revision_kind is RawRevisionKind.UNKNOWN: @@ -729,26 +1337,42 @@ def bind_byte_proven_older_member(raw_id: str, logical_key: str) -> None: census_selection = initial_selection while True: rows = archive.raw_membership_census_rows(census_selection) + if max_payload_bytes is not None: + payload_sizes = archive.raw_payload_sizes( + [ + raw_id + for raw_id, _source_index, terminal_non_session, _raw_rowid in rows + if raw_id not in state.censused and not terminal_non_session + ] + ) + total_payload_bytes = sum(payload_sizes.values()) + oversized = [raw_id for raw_id, size in payload_sizes.items() if size > max_payload_bytes] + if oversized or total_payload_bytes > max_payload_bytes: + blocked_ids = oversized or list(payload_sizes) + raise RawRevisionReplayResourceBlockedError( + sorted(blocked_ids), max_payload_bytes, total_payload_bytes + ) + for raw_id, _source_index, _terminal_non_session, _raw_rowid in sorted( + rows, + key=lambda row: archive.raw_revision_observation_order(row[0]), + ): + if raw_id in state.censused or not _replay_retained_codex_state_evidence(archive, raw_id): + continue + state.scanned += 1 + state.censused.add(raw_id) + state.transient_non_session_raw_ids.add(raw_id) + commit_unit() terminal_raw_ids = { - raw_id for raw_id, _source_index, terminal_non_session in rows if terminal_non_session + raw_id for raw_id, _source_index, terminal_non_session, _raw_rowid in rows if terminal_non_session } for raw_id in terminal_raw_ids - state.censused: state.scanned += 1 state.censused.add(raw_id) pending_rows = [ (raw_id, source_index) - for raw_id, source_index, terminal_non_session in rows + for raw_id, source_index, terminal_non_session, _raw_rowid in rows if raw_id not in state.censused and not terminal_non_session ] - if max_payload_bytes is not None: - payload_sizes = archive.raw_payload_sizes([raw_id for raw_id, _index in pending_rows]) - total_payload_bytes = sum(payload_sizes.values()) - oversized = [raw_id for raw_id, size in payload_sizes.items() if size > max_payload_bytes] - if oversized or total_payload_bytes > max_payload_bytes: - blocked_ids = oversized or list(payload_sizes) - raise RawRevisionReplayResourceBlockedError( - sorted(blocked_ids), max_payload_bytes, total_payload_bytes - ) # Parse is read-only blob->ParsedSession decode and authority-neutral; # spread it across a process pool when there is more than one raw to # parse. Archive writes below stay in fixed `pending_rows` order @@ -770,6 +1394,7 @@ def bind_byte_proven_older_member(raw_id: str, logical_key: str) -> None: continue apply_outcome(raw_id, source_index, parsed_outcomes) if head_by_older: + source_index_by_raw_id = dict(pending_rows) head_to_key = { raw_id: key for key, raw_ids in state.provisional_full_raw_ids.items() for raw_id in raw_ids } @@ -794,7 +1419,7 @@ def bind_byte_proven_older_member(raw_id: str, logical_key: str) -> None: if resolved_key is not None: bind_byte_proven_older_member(older_raw_id, resolved_key) else: - apply_outcome(older_raw_id, 0, fallback_outcomes) + apply_outcome(older_raw_id, source_index_by_raw_id[older_raw_id], fallback_outcomes) if census_selection is None: break expanded, _keys = archive.expand_raw_membership_selection(list(census_selection)) @@ -802,7 +1427,7 @@ def bind_byte_proven_older_member(raw_id: str, logical_key: str) -> None: break census_selection = expanded except BaseException: - if batched and pending_commits > 0: + if batched: archive.rollback() raise if batched and pending_commits > 0: @@ -823,14 +1448,10 @@ def _load_frozen_revision_evidence( expanded_raw_ids, _logical_keys = archive.expand_raw_membership_selection(selected_raw_ids) if selected_raw_ids is not None: expanded_raw_ids = _expand_frozen_revision_link_selection(archive.archive_root, expanded_raw_ids) - recorded_logical_keys = require_current_parser_source_census( - archive.archive_root, - selected_raw_ids=expanded_raw_ids if selected_raw_ids is not None else None, - ) rows = archive.raw_membership_census_rows(expanded_raw_ids if selected_raw_ids is not None else None) if max_payload_bytes is not None: payload_sizes = archive.raw_payload_sizes( - [raw_id for raw_id, _source_index, terminal_non_session in rows if not terminal_non_session] + [raw_id for raw_id, _source_index, terminal_non_session, _raw_rowid in rows if not terminal_non_session] ) total_payload_bytes = sum(payload_sizes.values()) oversized = [raw_id for raw_id, size in payload_sizes.items() if size > max_payload_bytes] @@ -838,8 +1459,20 @@ def _load_frozen_revision_evidence( raise RawRevisionReplayResourceBlockedError( sorted(oversized or payload_sizes), max_payload_bytes, total_payload_bytes ) + frozen_codex_state_raw_ids = frozenset( + raw_id + for raw_id, _source_index, _terminal_non_session, _raw_rowid in rows + if _retained_codex_state_descriptor(archive, raw_id) is not None + ) + recorded_logical_keys = require_current_parser_source_census( + archive.archive_root, + selected_raw_ids=expanded_raw_ids if selected_raw_ids is not None else None, + transient_non_session_raw_ids=frozen_codex_state_raw_ids, + ) parseable_raw_ids = [ - raw_id for raw_id, source_index, terminal_non_session in rows if source_index >= 0 and not terminal_non_session + raw_id + for raw_id, source_index, terminal_non_session, _raw_rowid in rows + if source_index >= 0 and not terminal_non_session and raw_id not in frozen_codex_state_raw_ids ] parsed_outcomes = _parse_retained_raws( archive, @@ -847,11 +1480,11 @@ def _load_frozen_revision_evidence( ingest_workers=ingest_workers, prefetch_cache=prefetch_cache, ) - state = _RevisionCensusState(0, 0, 0, set(), {}, {}) - for raw_id, source_index, terminal_non_session in rows: + state = _RevisionCensusState(0, 0, 0, set(), {}, {}, set(frozen_codex_state_raw_ids)) + for raw_id, source_index, terminal_non_session, _raw_rowid in rows: state.scanned += 1 state.censused.add(raw_id) - if terminal_non_session: + if terminal_non_session or raw_id in frozen_codex_state_raw_ids: continue if source_index < 0: state.quarantined += 1 @@ -892,6 +1525,7 @@ def require_current_parser_source_census( archive_root: Path, *, selected_raw_ids: Sequence[str] | None = None, + transient_non_session_raw_ids: Set[str] = frozenset(), ) -> dict[str, tuple[str, ...]]: """Require phase-2 parser receipts before allocating an index candidate.""" stale_raw_ids: list[str] = [] @@ -919,6 +1553,9 @@ def require_current_parser_source_census( ) for raw_id_value, fingerprint, status, logical_keys_json in rows: raw_id = str(raw_id_value) + if raw_id in transient_non_session_raw_ids: + recorded_logical_keys[raw_id] = () + continue if fingerprint != RAW_AUTHORITY_PARSER_FINGERPRINT or status != "complete": stale_raw_ids.append(raw_id) continue @@ -954,6 +1591,7 @@ def require_current_parser_source_census( ) for raw_id_value, typed_key, revision_kind, membership_key, typed_non_session in rows: raw_id = str(raw_id_value) + typed_non_session = bool(typed_non_session) or raw_id in transient_non_session_raw_ids existing_typed, existing_kind, memberships, existing_non_session = durable_bindings.get( raw_id, (typed_key, revision_kind, [], bool(typed_non_session)) ) @@ -1128,6 +1766,7 @@ def require_current_parser_source_census( """, authority_params, ) + if str(row[0]) not in transient_non_session_raw_ids ) if unresolved_raw_ids: sample = ", ".join(unresolved_raw_ids[:5]) @@ -1138,6 +1777,27 @@ def require_current_parser_source_census( return recorded_logical_keys +def _logical_keys_for_raw_ids(archive: ArchiveStore, raw_ids: Set[str]) -> set[str]: + """Read typed logical keys for an arbitrary-size raw selection.""" + keys: set[str] = set() + ordered_raw_ids = sorted(raw_ids) + conn = archive._ensure_source_conn() + for offset in range(0, len(ordered_raw_ids), 500): + chunk = ordered_raw_ids[offset : offset + 500] + placeholders = ",".join("?" for _ in chunk) + keys.update( + str(row[0]) + for row in conn.execute( + f""" + SELECT DISTINCT logical_source_key FROM raw_sessions + WHERE raw_id IN ({placeholders}) AND logical_source_key IS NOT NULL + """, + chunk, + ) + ) + return keys + + def validate_frozen_source_authority( archive_root: Path, *, @@ -1153,7 +1813,11 @@ def validate_frozen_source_authority( archive_root, active_index_path=active_index_path, ) as archive, - _ParsedSessionSpill(archive_root, max_cached_payload_bytes=max_payload_bytes) as spill, + _ParsedSessionSpill( + archive_root, + index_path=active_index_path, + max_cached_payload_bytes=max_payload_bytes, + ) as spill, ): census = _load_frozen_revision_evidence( archive, @@ -1164,11 +1828,15 @@ def validate_frozen_source_authority( prefetch_cache=prefetch_cache, ) _unclassified, logical_keys = archive.raw_revision_rebuild_selection(selected_raw_ids) + transient_non_session_keys = _logical_keys_for_raw_ids( + archive, + census.transient_non_session_raw_ids, + ) _membership_raw_ids, persisted_membership_keys = archive.expand_raw_membership_selection(selected_raw_ids) membership_keys = {*persisted_membership_keys, *census.membership_candidates} byte_replayed_keys: set[str] = set() - for logical_key in sorted(logical_keys): + for logical_key in sorted(set(logical_keys) - transient_non_session_keys): plan = archive.classify_raw_revision_cohort_for_frozen_candidate(logical_key) if not plan.accepted_raw_ids: convertible = archive.convertible_full_revision_raw_ids(logical_key) @@ -1210,6 +1878,7 @@ def validate_frozen_source_authority( def census_historical_revision_evidence( archive_root: Path, *, + active_index_path: Path | None = None, selected_raw_ids: list[str] | None = None, max_payload_bytes: int | None = None, ingest_workers: int = 1, @@ -1225,7 +1894,11 @@ def census_historical_revision_evidence( """ with ( ArchiveStore.open_existing(archive_root, read_only=False) as archive, - _ParsedSessionSpill(archive_root, max_cached_payload_bytes=max_payload_bytes) as spill, + _ParsedSessionSpill( + archive_root, + index_path=active_index_path, + max_cached_payload_bytes=max_payload_bytes, + ) as spill, ): state = _census_historical_revision_evidence( archive, @@ -1347,6 +2020,7 @@ def visit(key: str) -> None: def backfill_historical_revision_evidence( archive_root: Path, *, + active_index_path: Path | None = None, selected_raw_ids: list[str] | None = None, owned_inactive_generation: tuple[str, str] | None = None, retention_observer: Callable[[int, int], None] | None = None, @@ -1485,7 +2159,11 @@ def backfill_historical_revision_evidence( prepare_pool = ThreadPoolExecutor(max_workers=1) if parallel_threads_effective() else None with ( archive_context as archive, - _ParsedSessionSpill(archive_root, max_cached_payload_bytes=spill_cache_bytes) as spill, + _ParsedSessionSpill( + archive_root, + index_path=active_index_path, + max_cached_payload_bytes=spill_cache_bytes, + ) as spill, prepare_pool if prepare_pool is not None else nullcontext(), ): census_started = time.perf_counter() @@ -1921,9 +2599,35 @@ def census_parse_worker( fallback_id_override = native_id if kind is RawRevisionKind.APPEND else None publisher = ArchiveBlobPublisher(Path(source_db_path_str), Path(blob_root_str)) try: + if provider is Provider.UNKNOWN: + with publisher.open(blob_hash) as detection_payload: + provider, _evidence = _detect_unknown_retained_provider(detection_payload, source_path) + if is_stream_record_provider(source_path, str(provider)): + with publisher.open(blob_hash) as stream_payload: + sessions = _parse_stream( + provider, stream_payload, source_path, fallback_id_override=fallback_id_override + ) + return raw_id, sessions, None + _require_bounded_provider(provider, source_path) + payload = publisher.read_all(blob_hash) + payload_path = None + if provider is Provider.HERMES: + candidate_path = publisher.blob_path(blob_hash) + payload_path = candidate_path if candidate_path.exists() else None + sessions = _parse_one( + provider, + payload, + source_path, + payload_path=payload_path, + archive_root=Path(blob_root_str).parent, + fallback_id_override=fallback_id_override, + ) + return raw_id, sessions, None if is_stream: - with publisher.open(blob_hash) as payload: - sessions = _parse_stream(provider, payload, source_path, fallback_id_override=fallback_id_override) + with publisher.open(blob_hash) as stream_payload: + sessions = _parse_stream( + provider, stream_payload, source_path, fallback_id_override=fallback_id_override + ) else: payload_path = None if provider is Provider.HERMES: @@ -2120,6 +2824,8 @@ def _enrich_retained_parse_results( continue provider, _blob_hash, source_path, _descriptor_kind, _size, _native_id = descriptors[raw_id] sessions, payload_bytes, kind = outcome + if sessions: + provider = Provider.from_string(sessions[0].source_name) results[raw_id] = ( _replay_safe_enrich_sessions( source_conn, @@ -2325,6 +3031,27 @@ def parse_retained_raw_sessions(archive: ArchiveStore, raw_id: str) -> list[Pars # the unchanged stem-based fallback -- their stored bytes still carry # the synthetic session_meta line that made this unnecessary for them. fallback_id_override = archive.raw_native_id(raw_id) if kind is RawRevisionKind.APPEND else None + if provider is Provider.UNKNOWN: + # Source-only acquisition deliberately retains unknown ZIP members + # without decoding them. Recovery is the first lawful point to + # inspect the durable bytes and resolve their parser, before deciding + # whether their filename is a stream route. + with archive.open_raw_revision_material(raw_id) as (_stream_provider, payload, _stream_path, _stream_kind): + provider, _evidence = _detect_unknown_retained_provider(payload, source_path) + if is_stream_record_provider(source_path, str(provider)): + with archive.open_raw_revision_material(raw_id) as (_stream_provider, payload, stream_path, _stream_kind): + return _parse_stream(provider, payload, stream_path, fallback_id_override=fallback_id_override) + _require_bounded_provider(provider, source_path) + _provider, eager_payload, _source_path, _eager_kind = archive.raw_revision_material(raw_id) + payload_path = archive.blob_path_for_hash(blob_hash) if provider is Provider.HERMES else None + return _parse_one( + provider, + eager_payload, + source_path, + payload_path=payload_path, + archive_root=archive.archive_root, + fallback_id_override=fallback_id_override, + ) if is_stream_record_provider(source_path, str(provider)): with archive.open_raw_revision_material(raw_id) as (stream_provider, payload, stream_path, _stream_kind): return _parse_stream(stream_provider, payload, stream_path, fallback_id_override=fallback_id_override) @@ -2340,6 +3067,51 @@ def parse_retained_raw_sessions(archive: ArchiveStore, raw_id: str) -> list[Pars ) +def _retained_codex_state_descriptor(archive: ArchiveStore, raw_id: str) -> tuple[Path, str, str] | None: + """Identify one immutable retained Codex state snapshot without mutating it.""" + provider, blob_hash, source_path, _kind, _payload_size = archive.raw_revision_descriptor(raw_id) + if provider is not Provider.CODEX: + return None + state_path = archive.blob_path_for_hash(blob_hash) + if state_path is None: + return None + state_kind = codex_state.classify_codex_sqlite_path(state_path, immutable=True) + if state_kind not in codex_state.IN_SCOPE_KINDS: + return None + return state_path, source_path, state_kind + + +def _replay_retained_codex_state_evidence(archive: ArchiveStore, raw_id: str) -> bool: + """Apply a retained, in-scope Codex state snapshot without minting a session. + + Source-only acquisition snapshots named Codex databases before it can + inspect their schema. Once recovery owns the derived tier, only a + recognized retained snapshot may become thread evidence. The parser + reads the immutable blob path, never the original mutable state DB. + """ + descriptor = _retained_codex_state_descriptor(archive, raw_id) + if descriptor is None: + return False + state_path, source_path, state_kind = descriptor + if state_kind == "thread_state": + write_codex_thread_state_evidence( + archive, + codex_state.parse_codex_state_db(state_path, immutable=True), + source_path=source_path, + acquired_at_ms=archive.raw_revision_observed_at_ms(raw_id), + ) + archive.replace_raw_membership_census( + raw_id, + [], + parser_fingerprint=RAW_AUTHORITY_PARSER_FINGERPRINT, + censused_at_ms=0, + detail="retained Codex state evidence applied", + retire_full_revision_governance=True, + ) + archive.mark_raw_parse_succeeded(raw_id, provider=Provider.CODEX) + return True + + #: Lever-A prefetch-buffer budget clamp (estimated tree bytes) -- same #: adaptive formula as ``_ParsedSessionSpill``'s hot decoded cache (physical #: RAM / 16 within these bounds), but deliberately its own pair of constants: @@ -2786,14 +3558,20 @@ class _ParsedSessionSpill: #: existing crash-recovery contract for the sqlite-backed spill. _WHALE_CACHE_MAX_TREE_BYTES: Final[int] = 8 * 1024 * 1024 * 1024 - def __init__(self, archive_root: Path, *, max_cached_payload_bytes: int | None) -> None: + def __init__( + self, + archive_root: Path, + *, + index_path: Path | None = None, + max_cached_payload_bytes: int | None, + ) -> None: # Place the spill beside the RESOLVED index tier, not the archive # root: on deployments where the .db files are symlinks (e.g. root # SSD config dir -> NVMe data disk), a spill in archive_root would # put census churn on the wear-limited disk the symlinks exist to # protect. - index_path = archive_root / "index.db" - spill_dir = index_path.resolve().parent if index_path.exists() else archive_root + resolved_index_path = index_path or ArchiveLocation.resolve(archive_root).active_index_path + spill_dir = resolved_index_path.resolve().parent if resolved_index_path.exists() else archive_root fd, name = tempfile.mkstemp(prefix=".revision-census-", suffix=".sqlite", dir=spill_dir) os.close(fd) self.path = Path(name) @@ -3017,9 +3795,11 @@ def _declared_non_session_artifact_classification( these; this replay engine (used by ``polylogue ops reset --index`` / ``devtools`` rebuild-index) is a SEPARATE parse chokepoint that did not, and would silently recreate exactly the ``.meta`` phantom sessions - that fix is meant to eliminate on every future rebuild. Same check, same - rule table, so a declared fact artifact can never become a session - through either entry point. + that fix is meant to eliminate on every future rebuild. A positive JSONL + session proof is the one deliberate exception, matching the live route: + a source-only outage may retain bytes before it can inspect a path that + normally carries fact evidence, and recovery must not make that filename + permanently override later decoded session authority. polylogue-9ykn: a path-declared rule is only half of the live path's gate. ``pipeline/services/ingest_worker.py`` also runs every sampled @@ -3044,7 +3824,7 @@ def _declared_non_session_artifact_classification( from polylogue.archive.artifact_taxonomy import classify_artifact rule = artifact_rule_for_path(provider, source_path) - if rule is not None and rule.parse_policy != "session": + if rule is not None and rule.parse_policy != "session" and not sample: classification = classify_artifact([], provider=provider, source_path=source_path) if not classification.parse_as_session: return classification @@ -3072,6 +3852,76 @@ def _declared_non_session_artifact_classification( return classification if not classification.parse_as_session else None +def _detected_provider_for_empty_replay( + archive: ArchiveStore, + raw_id: str, + *, + stored_provider: Provider, + source_path: str, +) -> Provider: + """Resolve a provider before terminalizing an empty retained replay.""" + if stored_provider is not Provider.UNKNOWN: + return stored_provider + with archive.open_raw_revision_material(raw_id) as (_provider, payload, _path, _kind): + provider, _evidence = _detect_unknown_retained_provider(payload, source_path) + if provider is not Provider.UNKNOWN: + return provider + return provider + + +def _persist_terminal_non_session_artifact( + archive: ArchiveStore, + raw_id: str, + *, + provider: Provider, + source_path: str, + source_index: int, + manage_transaction: bool, +) -> bool: + """Record replay-confirmed source-only artifact authority once. + + Replay reaches this function only after the real parser has consumed the + complete stream and produced no conversational session. The terminal + receipt therefore follows that one authoritative parse result instead of + reclassifying the raw through a second, weaker JSONL shape scan. + """ + if provider is Provider.UNKNOWN: + return False + classification = _declared_non_session_artifact_classification(provider, source_path) + if classification is None: + return False + origin = origin_from_provider(provider) + observed_at_ms = archive.raw_revision_observed_at_ms(raw_id) + upsert_raw_artifact( + archive._ensure_source_conn(), + raw_id, + ArchiveSourceArtifact( + artifact_id=artifact_observation_id( + source_name=origin.value, + source_path=source_path, + source_index=source_index, + ), + origin=origin, + source_path=source_path, + source_index=source_index, + artifact_kind=classification.cohort, + classification_reason=classification.reason, + parse_as_session=False, + schema_eligible=classification.schema_eligible, + first_observed_at_ms=observed_at_ms, + last_observed_at_ms=observed_at_ms, + ), + manage_transaction=manage_transaction, + ) + apply_source_raw_state_update( + archive._ensure_source_conn(), + raw_id, + state=_raw_parse_success_state(provider), + manage_transaction=manage_transaction, + ) + return True + + def _is_declared_non_session_artifact( provider: Provider, source_path: str, @@ -3137,18 +3987,6 @@ def _parse_one_raw( return sessions source_name = Path(source_path).name fallback_id = fallback_id_override or Path(source_path).stem - if is_stream_record_provider(source_path, str(provider)): - records = list(_iter_json_stream(BytesIO(payload), source_name)) - if _is_declared_non_session_artifact(provider, source_path, sample=records[:64]): - return [] - return parse_stream_payload( - provider, - records, - fallback_id, - source_path=source_path, - ) - if _is_declared_non_session_artifact(provider, source_path): - return [] if provider is Provider.HERMES and looks_like_sqlite_bytes(payload): with _sqlite_payload_path(payload, payload_path, archive_root) as sqlite_path: if hermes_state.looks_like_state_db_path(sqlite_path, immutable=True): @@ -3165,9 +4003,32 @@ def _parse_one_raw( profile_root=Path(source_path).parent, immutable=True, ) + rule = artifact_rule_for_path(provider, source_path) + declared_path_session_evidence = False + if rule is not None and rule.parse_policy != "session" and is_jsonl_source_path(source_path): + from polylogue.archive.raw_payload.decode import jsonl_session_artifact + + declared_path_session_evidence = jsonl_session_artifact(payload, provider=provider) is not None + if is_stream_record_provider(source_path, str(provider)): + records = list(_iter_json_stream(BytesIO(payload), source_name)) + if not declared_path_session_evidence and _is_declared_non_session_artifact( + provider, source_path, sample=records[:64] + ): + return [] + return parse_stream_payload( + provider, + records, + fallback_id, + source_path=source_path, + ) + records = list(_iter_json_stream(BytesIO(payload), source_name)) + if not declared_path_session_evidence and _is_declared_non_session_artifact( + provider, source_path, sample=records[:64] + ): + return [] return parse_payload( provider, - list(_iter_json_stream(BytesIO(payload), source_name)), + records, fallback_id, source_path=source_path, ) @@ -3222,22 +4083,12 @@ def _parse_stream_raw( *, fallback_id_override: str | None = None, ) -> list[ParsedSession]: - if _is_declared_non_session_artifact(provider, source_path): - return [] source_name = Path(source_path).name fallback_id = fallback_id_override or Path(source_path).stem stream = _iter_json_stream(payload, source_name) - # Multi-GiB Claude Code JSONL must stay memory-bounded (module docstring: - # "a memory-bounded streaming path exists for multi-GiB Claude Code - # JSONL"), so only the first 64 records -- the same sample bound the live - # ingest gate uses -- are materialized for content classification; the - # rest of the stream is chained back on unread. - sample = list(islice(stream, 64)) - if _is_declared_non_session_artifact(provider, source_path, sample=sample): - return [] return parse_stream_payload( provider, - chain(sample, stream), + stream, fallback_id, source_path=source_path, ) diff --git a/polylogue/sources/source_acquisition_components.py b/polylogue/sources/source_acquisition_components.py index f3280040ff..4e19fd903d 100644 --- a/polylogue/sources/source_acquisition_components.py +++ b/polylogue/sources/source_acquisition_components.py @@ -28,7 +28,6 @@ _DETECTION_PREFIX_SIZE = 8192 # 8 KB — enough for provider detection _HEARTBEAT_INTERVAL_S = 5.0 - AcquisitionObservation: TypeAlias = JSONDocument ObservationCallback: TypeAlias = Callable[[AcquisitionObservation], None] StatusCallback: TypeAlias = Callable[[str], None] @@ -421,6 +420,28 @@ def _stream_preserved_zip_entry( *, provider_hint: Provider, ) -> RawSessionData: + return stream_preserved_zip_entry_raw_data( + zf, + context, + provider_hint=provider_hint, + ) + + +def stream_preserved_zip_entry_raw_data( + zf: zipfile.ZipFile, + context: ZipEntryReadContext, + *, + provider_hint: Provider, + source_index: int | None = None, +) -> RawSessionData: + """Durably stream one admitted ZIP member without decoding its content. + + The caller remains responsible for applying :class:`_ZipEntryValidator` + before this function. Keeping the bounded entry reader here means a + source-tier-only outage retains the same ZIP-bomb protection as ordinary + acquisition while deliberately avoiding provider detection, JSON decoding, + and artifact classification. + """ with _decoders.open_bounded_zip_entry(zf, context.entry) as handle: blob_hash, blob_size = stream_fileobj_to_blob( context.blob_store, @@ -446,6 +467,7 @@ def _stream_preserved_zip_entry( provider_hint=provider_hint, blob_hash=blob_hash, blob_size=blob_size, + source_index=source_index, blob_publication_receipt_id=publication_id, ) @@ -528,6 +550,7 @@ def iter_zip_entry_raw_data( "observe_acquisition", "raw_data_record", "read_plain_source_file", + "stream_preserved_zip_entry_raw_data", "stream_fileobj_to_blob", "stream_path_to_blob", ] diff --git a/polylogue/sources/source_parsing.py b/polylogue/sources/source_parsing.py index 3214b4e589..93d580c78f 100644 --- a/polylogue/sources/source_parsing.py +++ b/polylogue/sources/source_parsing.py @@ -22,6 +22,7 @@ from .cursor import _log_source_iteration_summary, _ParseContext, _record_cursor_failure from .decoders import _process_zip from .dispatch import GROUP_PROVIDERS as _GROUP_PROVIDERS +from .dispatch import is_jsonl_source_path from .emitter import _SessionEmitter from .parsers import antigravity, hermes_state, hermes_verification from .parsers.base import ParsedSession, RawSessionData @@ -35,7 +36,7 @@ def has_decoded_session_evidence(path: Path, *, provider: Provider) -> bool: """Return whether decoded JSON content outranks a non-session path rule.""" - if path.suffix.lower() == ".jsonl": + if is_jsonl_source_path(str(path)): return jsonl_session_artifact(path, provider=provider) is not None if path.suffix.lower() != ".json": diff --git a/polylogue/storage/artifacts/inspection.py b/polylogue/storage/artifacts/inspection.py index 34edab172c..c74adb2a94 100644 --- a/polylogue/storage/artifacts/inspection.py +++ b/polylogue/storage/artifacts/inspection.py @@ -8,9 +8,23 @@ from datetime import datetime, timezone from pathlib import Path -from polylogue.archive.artifact_taxonomy import ArtifactKind, classify_artifact_path -from polylogue.archive.raw_payload import JSONValue, RawPayloadEnvelope, build_raw_payload_envelope +from polylogue.archive.artifact_taxonomy import ( + ArtifactKind, + classify_artifact_path, + strong_path_classification, +) +from polylogue.archive.raw_payload import ( + JSONValue, + RawPayloadEnvelope, + build_raw_payload_envelope, +) +from polylogue.archive.raw_payload.decode import ( + JSONL_RECORD_INSPECTION_BYTES, + JSONLSessionArtifactScan, + scan_jsonl_session_artifact, +) from polylogue.core.enums import ArtifactSupportStatus, Provider +from polylogue.core.sources import origin_from_provider from polylogue.schemas.observation import derive_bundle_scope, schema_cluster_id from polylogue.schemas.packages import SchemaResolution from polylogue.schemas.runtime_registry import SchemaRegistry @@ -170,6 +184,26 @@ def _inspect_payload_envelope(record: RawSessionRecord, *, blob_store: BlobStore return envelope +def _complete_stream_session_artifact( + record: RawSessionRecord, + *, + provider: Provider, + blob_store: BlobStore, +) -> JSONLSessionArtifactScan | None: + """Recover positive stream evidence hidden by bounded inspection.""" + if not _prefers_json_stream(record.source_path) or provider not in {Provider.CLAUDE_CODE, Provider.CODEX}: + return None + path_artifact = strong_path_classification(record.source_path, provider=provider) + if path_artifact is not None and not path_artifact.parse_as_session: + return None + return scan_jsonl_session_artifact( + blob_store.blob_path(_record_blob_ref(record)), + provider=provider, + source_path=record.source_path, + max_record_bytes=_INSPECTION_PREFIX_BYTES, + ) + + def _sidecar_agent_type(payload: JSONValue) -> str | None: if isinstance(payload, dict): agent_type = payload.get("agentType") @@ -210,7 +244,7 @@ def _support_status( return ArtifactSupportStatus.UNSUPPORTED_PARSEABLE -_INSPECTION_PREFIX_BYTES = 64 * 1024 # 64 KB — enough to classify any format +_INSPECTION_PREFIX_BYTES = JSONL_RECORD_INSPECTION_BYTES _FULL_JSON_INSPECTION_MAX_BYTES = 8 * 1024 * 1024 # 8 MB — bounded fallback for large JSON documents @@ -252,7 +286,9 @@ def _full_scan_malformed_jsonl(record: RawSessionRecord, *, blob_store: BlobStor The prefix-based classification only inspects the first 64 KB, so malformed content past the prefix never marks the artifact failed (#1745). This scan streams the whole blob line-by-line (never materializing it) so the - malformed-line count and decode status reflect the full artifact. + malformed-line count and decode status reflect the full artifact. Records + larger than the inspection bound are discarded in chunks but are not + counted as malformed: bounded inspection is not evidence of decode loss. Returns ``(malformed_lines, had_valid_records)``. ``had_valid_records`` is ``True`` when at least one line decoded successfully; the sampling helper @@ -269,6 +305,7 @@ def _full_scan_malformed_jsonl(record: RawSessionRecord, *, blob_store: BlobStor max_samples=1, jsonl_dict_only=False, scan_full=True, + max_record_bytes=_INSPECTION_PREFIX_BYTES, ) except ValueError: # No valid JSONL records at all — leave the decision to the prefix-based @@ -312,16 +349,17 @@ def _stream_loss_accounting( def inspect_raw_artifact(record: RawSessionRecord, *, blob_store: BlobStore | None = None) -> ArtifactObservationRecord: """Inspect one raw record into a durable artifact observation. - Uses only a small prefix of raw_content for classification — never - decodes the full payload. This keeps memory bounded regardless of - file size (a 1.5 GB JSONL file is classified from its first line). + Classification starts from a small prefix. If that prefix would refuse a + Claude or Codex JSONL stream, a memory-bounded rolling scan must confirm + that no later record supplies positive session evidence. """ resolved_blob_store = blob_store or get_blob_store() provider_hint = _normalize_payload_provider_hint(record) provider_token = provider_hint or record.source_name or "" bundle_scope = derive_bundle_scope(provider_token, record.source_path) + observation_origin = origin_from_provider(Provider.from_string(provider_token)) observation_id = artifact_observation_id( - source_name=record.source_name, + source_name=observation_origin.value, source_path=record.source_path, source_index=record.source_index, ) @@ -329,8 +367,43 @@ def inspect_raw_artifact(record: RawSessionRecord, *, blob_store: BlobStore | No registry = _SCHEMA_REGISTRY try: - envelope = _inspect_payload_envelope(record, blob_store=resolved_blob_store) + try: + envelope = _inspect_payload_envelope(record, blob_store=resolved_blob_store) + except Exception: + stream_provider = Provider.from_string(provider_token) + recovered_scan = _complete_stream_session_artifact( + record, + provider=stream_provider, + blob_store=resolved_blob_store, + ) + if recovered_scan is None or recovered_scan.artifact is None: + raise + envelope = RawPayloadEnvelope( + payload=list(recovered_scan.sample), + provider=stream_provider, + wire_format="jsonl", + artifact=recovered_scan.artifact, + malformed_jsonl_lines=0, + malformed_jsonl_detail=None, + ) payload_provider = envelope.provider + artifact = envelope.artifact + if not artifact.parse_as_session: + recovered_scan = _complete_stream_session_artifact( + record, + provider=payload_provider, + blob_store=resolved_blob_store, + ) + if recovered_scan is not None and recovered_scan.artifact is not None: + envelope = RawPayloadEnvelope( + payload=list(recovered_scan.sample), + provider=payload_provider, + wire_format="jsonl", + artifact=recovered_scan.artifact, + malformed_jsonl_lines=envelope.malformed_jsonl_lines, + malformed_jsonl_detail=envelope.malformed_jsonl_detail, + ) + artifact = envelope.artifact resolution: SchemaResolution | None = None has_supported_resolution = False @@ -344,7 +417,7 @@ def inspect_raw_artifact(record: RawSessionRecord, *, blob_store: BlobStore | No blob_store=resolved_blob_store, ) - if envelope.artifact.parse_as_session and envelope.artifact.schema_eligible and malformed_jsonl_lines == 0: + if artifact.parse_as_session and artifact.schema_eligible and malformed_jsonl_lines == 0: resolution, has_supported_resolution = _resolve_payload_support( registry=registry, payload_provider=payload_provider, @@ -356,10 +429,10 @@ def inspect_raw_artifact(record: RawSessionRecord, *, blob_store: BlobStore | No resolution_reason = resolution.reason if resolution is not None else None support_status = _support_status( - parse_as_session=envelope.artifact.parse_as_session, - schema_eligible=envelope.artifact.schema_eligible, + parse_as_session=artifact.parse_as_session, + schema_eligible=artifact.schema_eligible, malformed_jsonl_lines=malformed_jsonl_lines, - artifact_kind=envelope.artifact.kind.value, + artifact_kind=artifact.kind.value, has_supported_resolution=has_supported_resolution, had_decode_error=False, partial_decode=partial_decode, @@ -374,23 +447,21 @@ def inspect_raw_artifact(record: RawSessionRecord, *, blob_store: BlobStore | No source_index=record.source_index, file_mtime=record.file_mtime, wire_format=envelope.wire_format, - artifact_kind=envelope.artifact.kind.value, - classification_reason=envelope.artifact.reason, - parse_as_session=envelope.artifact.parse_as_session, - schema_eligible=envelope.artifact.schema_eligible, + artifact_kind=artifact.kind.value, + classification_reason=artifact.reason, + parse_as_session=artifact.parse_as_session, + schema_eligible=artifact.schema_eligible, support_status=support_status, malformed_jsonl_lines=malformed_jsonl_lines, decode_error=None, bundle_scope=bundle_scope, - cohort_id=schema_cluster_id(envelope.payload, envelope.artifact.cohort), + cohort_id=schema_cluster_id(envelope.payload, artifact.cohort), resolved_package_version=resolved_package_version, resolved_element_kind=resolved_element_kind, resolution_reason=resolution_reason, link_group_key=_link_group_key(record.source_path), sidecar_agent_type=( - _sidecar_agent_type(envelope.payload) - if envelope.artifact.kind is ArtifactKind.AGENT_SIDECAR_META - else None + _sidecar_agent_type(envelope.payload) if artifact.kind is ArtifactKind.AGENT_SIDECAR_META else None ), first_observed_at=observed_at, last_observed_at=observed_at, diff --git a/polylogue/storage/blob_integrity.py b/polylogue/storage/blob_integrity.py index 7afc537708..466d043270 100644 --- a/polylogue/storage/blob_integrity.py +++ b/polylogue/storage/blob_integrity.py @@ -33,6 +33,7 @@ from polylogue.core.json import JSONDecodeError as CoreJSONDecodeError from polylogue.core.json import dumps_bytes as json_dumps_bytes from polylogue.core.json import loads as json_loads +from polylogue.core.raw_coordinates import zip_member_identity_coordinate from polylogue.logging import get_logger from polylogue.storage.blob_store import BlobNamespaceEntry, BlobStore from polylogue.storage.introspection import column_exists as _column_exists @@ -1055,18 +1056,31 @@ def _missing_raw_backed_blob_rows(conn: sqlite3.Connection) -> list[dict[str, An blob_size_column = "blob_size" if _column_exists(conn, "raw_sessions", "blob_size") else "NULL" acquired_at_ms_column = "acquired_at_ms" if _column_exists(conn, "raw_sessions", "acquired_at_ms") else "NULL" file_mtime_ms_column = "file_mtime_ms" if _column_exists(conn, "raw_sessions", "file_mtime_ms") else "NULL" + has_container_coordinates = _table_exists(conn, "raw_container_coordinates") + coordinate_join = ( + "LEFT JOIN raw_container_coordinates coordinate ON coordinate.raw_id = raw_sessions.raw_id" + if has_container_coordinates + else "" + ) + coordinate_format_column = "coordinate.coordinate_format" if has_container_coordinates else "NULL" + entry_ordinal_column = "coordinate.entry_ordinal" if has_container_coordinates else "NULL" + split_index_column = "coordinate.split_index" if has_container_coordinates else "NULL" rows = conn.execute( f""" SELECT lower(hex(blob_hash)) AS blob_hash, - raw_id, + raw_sessions.raw_id AS raw_id, {origin_column} AS origin, {native_id_column} AS native_id, {source_path_column} AS source_path, {source_index_column} AS source_index, {blob_size_column} AS expected_size_bytes, {acquired_at_ms_column} AS acquired_at_ms, - {file_mtime_ms_column} AS file_mtime_ms + {file_mtime_ms_column} AS file_mtime_ms, + {coordinate_format_column} AS coordinate_format, + {entry_ordinal_column} AS entry_ordinal, + {split_index_column} AS split_index FROM raw_sessions + {coordinate_join} WHERE blob_hash IS NOT NULL ORDER BY origin, source_path, source_index, raw_id """ @@ -1119,6 +1133,9 @@ def _current_raw_payload_bytes( source_path: str, source_index: int | None, *, + raw_id: str | None = None, + blob_hash: str | None = None, + zip_coordinate: tuple[int, int] | None = None, source_bytes_cache: dict[str, bytes] | None = None, decoded_payload_cache: dict[str, object] | None = None, ) -> tuple[bytes | None, str | None]: @@ -1129,14 +1146,36 @@ def _current_raw_payload_bytes( zip_path, member = split if not zip_path.exists(): return None, "source_missing" + entry_ordinal: int | None = zip_coordinate[0] if zip_coordinate is not None else None + split_index = zip_coordinate[1] if zip_coordinate is not None else source_index + if zip_coordinate is None and raw_id is not None and blob_hash is not None and source_index is not None: + coordinate = zip_member_identity_coordinate( + raw_id=raw_id, + source_path=source_path, + source_index=source_index, + blob_hash=blob_hash, + ) + if coordinate is not None: + entry_ordinal, split_index = coordinate + cache_key = source_path if entry_ordinal is None else f"{source_path}\0{entry_ordinal}" try: - if source_bytes_cache is not None and source_path in source_bytes_cache: - member_bytes = source_bytes_cache[source_path] + if source_bytes_cache is not None and cache_key in source_bytes_cache: + member_bytes = source_bytes_cache[cache_key] else: with zipfile.ZipFile(zip_path) as archive: - matching = [info for info in archive.infolist() if info.filename == member] + central_directory = archive.infolist() + if entry_ordinal is None: + matching = [info for info in central_directory if info.filename == member] + elif entry_ordinal >= len(central_directory): + return None, "container_coordinate_mismatch" + else: + coordinated = central_directory[entry_ordinal] + matching = [coordinated] if coordinated.filename == member else [] if len(matching) != 1: - return None, "ambiguous_container_member" + reason = ( + "ambiguous_container_member" if entry_ordinal is None else "container_coordinate_mismatch" + ) + return None, reason admitted = list( ZipAdmission(zip_path=zip_path).filter_entries(matching, allowed_suffixes=ZIP_JSON_SUFFIXES) ) @@ -1145,32 +1184,34 @@ def _current_raw_payload_bytes( with open_bounded_zip_entry(archive, admitted[0]) as handle: member_bytes = handle.read(MAX_UNCOMPRESSED_SIZE + 1) if source_bytes_cache is not None: - source_bytes_cache[source_path] = member_bytes + source_bytes_cache[cache_key] = member_bytes except KeyError: return None, "source_missing" except ZipBombError: return None, "container_member_rejected" - if source_index is None: + if split_index is None: return None, "source_index_missing" + if blob_hash is not None and hashlib.sha256(member_bytes).hexdigest() == blob_hash: + return member_bytes, None try: - if decoded_payload_cache is not None and source_path in decoded_payload_cache: - decoded_payload = decoded_payload_cache[source_path] + if decoded_payload_cache is not None and cache_key in decoded_payload_cache: + decoded_payload = decoded_payload_cache[cache_key] if isinstance(decoded_payload, list): - payload = decoded_payload[int(source_index)] - elif int(source_index) == 0: + payload = decoded_payload[int(split_index)] + elif int(split_index) == 0: payload = decoded_payload else: raise IndexError("non-array JSON payload only supports source_index 0") else: if member.endswith(".jsonl"): - payload = _jsonl_payload_at_index(member_bytes, int(source_index)) + payload = _jsonl_payload_at_index(member_bytes, int(split_index)) else: decoded_payload = json_loads(member_bytes) if decoded_payload_cache is not None: - decoded_payload_cache[source_path] = decoded_payload + decoded_payload_cache[cache_key] = decoded_payload if isinstance(decoded_payload, list): - payload = decoded_payload[int(source_index)] - elif int(source_index) == 0: + payload = decoded_payload[int(split_index)] + elif int(split_index) == 0: payload = decoded_payload else: raise IndexError("non-array JSON payload only supports source_index 0") @@ -1187,6 +1228,26 @@ def _current_raw_payload_bytes( return None, f"error:{exc}" +def _raw_zip_coordinate(row: dict[str, Any]) -> tuple[int, int] | None: + if row.get("coordinate_format") == "zip-v2": + entry_ordinal = row.get("entry_ordinal") + split_index = row.get("split_index") + if entry_ordinal is not None and split_index is not None: + return int(entry_ordinal), int(split_index) + source_path = _optional_str(row.get("source_path")) + source_index = row.get("source_index") + raw_id = str(row.get("raw_id") or "") + blob_hash = str(row.get("blob_hash") or "") + if not source_path or not _path_is_container_member(source_path) or source_index is None: + return None + return zip_member_identity_coordinate( + raw_id=raw_id, + source_path=source_path, + source_index=int(source_index), + blob_hash=blob_hash, + ) + + def _delete_blob_refs_for_raw_id(conn: sqlite3.Connection, raw_id: str) -> None: ref_id_column = "ref_id" if _column_exists(conn, "blob_refs", "ref_id") else "raw_id" conn.execute(f"DELETE FROM blob_refs WHERE {ref_id_column} = ?", (raw_id,)) @@ -1480,7 +1541,7 @@ def replace_raw_backed_blob_reference_debt_from_source( manifest_rows: list[dict[str, object]] = [] by_origin: Counter[str] = Counter() by_source_shape: Counter[str] = Counter() - candidate_updates: list[tuple[dict[str, Any], str, int, int | None, int]] = [] + candidate_updates: list[tuple[dict[str, Any], str, int, int | None, int, tuple[int, int] | None]] = [] skipped_existing_blob = 0 skipped_no_source_path = 0 skipped_source_missing = 0 @@ -1515,10 +1576,14 @@ def replace_raw_backed_blob_reference_debt_from_source( ) continue + zip_coordinate = _raw_zip_coordinate(row) try: payload_bytes, reason = _current_raw_payload_bytes( source_path, int(row["source_index"]) if row.get("source_index") is not None else None, + raw_id=raw_id, + blob_hash=old_blob_hash, + zip_coordinate=zip_coordinate, source_bytes_cache=source_bytes_cache, decoded_payload_cache=decoded_payload_cache, ) @@ -1574,7 +1639,7 @@ def replace_raw_backed_blob_reference_debt_from_source( "new_equals_old": new_blob_hash == old_blob_hash, } manifest_rows.append(manifest_row) - candidate_updates.append((row, new_blob_hash, new_blob_size, file_mtime_ms, acquired_at_ms)) + candidate_updates.append((row, new_blob_hash, new_blob_size, file_mtime_ms, acquired_at_ms, zip_coordinate)) if len(samples) < max(0, sample_size): samples.append( BlobReferenceSourceReplaceSample( @@ -1604,13 +1669,16 @@ def replace_raw_backed_blob_reference_debt_from_source( apply_decoded_payload_cache: dict[str, object] = {} publisher = ArchiveBlobPublisher(source_db, blob_store.root, store=blob_store) publication_receipts: list[str | None] = [] - for row, new_blob_hash, _new_blob_size, _file_mtime_ms, _acquired_at_ms in candidate_updates: + for row, new_blob_hash, _new_blob_size, _file_mtime_ms, _acquired_at_ms, zip_coordinate in candidate_updates: receipt_id: str | None = None existed_before_publication = blob_store.exists(new_blob_hash) source_path = str(row["source_path"]) payload_bytes, _reason = _current_raw_payload_bytes( source_path, int(row["source_index"]) if row.get("source_index") is not None else None, + raw_id=str(row["raw_id"]), + blob_hash=str(row.get("blob_hash") or ""), + zip_coordinate=zip_coordinate, source_bytes_cache=apply_source_bytes_cache, decoded_payload_cache=apply_decoded_payload_cache, ) @@ -1631,12 +1699,27 @@ def replace_raw_backed_blob_reference_debt_from_source( new_blob_size, file_mtime_ms, acquired_at_ms, + zip_coordinate, ), publication_receipt_id in zip(candidate_updates, publication_receipts, strict=True): raw_id = str(row["raw_id"]) source_path = str(row["source_path"]) if not blob_store.exists(new_blob_hash): skipped_error += 1 continue + if zip_coordinate is not None and _table_exists(conn, "raw_container_coordinates"): + entry_ordinal, split_index = zip_coordinate + from polylogue.storage.sqlite.archive_tiers.source_write import ( + record_raw_container_coordinate, + ) + + record_raw_container_coordinate( + conn, + raw_id, + coordinate_format="zip-v2", + entry_ordinal=entry_ordinal, + split_index=split_index, + manage_transaction=False, + ) _delete_blob_refs_for_raw_id(conn, raw_id) _update_raw_session_blob_ref( conn, diff --git a/polylogue/storage/raw_authority.py b/polylogue/storage/raw_authority.py index 899f4535c0..873838eee9 100644 --- a/polylogue/storage/raw_authority.py +++ b/polylogue/storage/raw_authority.py @@ -744,11 +744,20 @@ def build_raw_replay_plan(conn: sqlite3.Connection, input_raw_ids: Sequence[str] ) -def build_raw_replay_plans(archive_root: Path, components: Sequence[tuple[str, ...]]) -> tuple[RawReplayPlan, ...]: +def build_raw_replay_plans( + archive_root: Path, + components: Sequence[tuple[str, ...]], + *, + index_db_path: Path | None = None, +) -> tuple[RawReplayPlan, ...]: if not components: return () + if index_db_path is None: + from polylogue.storage.archive_identity import resolve_active_index_path + + index_db_path = resolve_active_index_path(archive_root) with closing(sqlite3.connect(f"file:{archive_root / 'source.db'}?mode=ro", uri=True)) as conn: - conn.execute("ATTACH DATABASE ? AS index_tier", (str(archive_root / "index.db"),)) + conn.execute("ATTACH DATABASE ? AS index_tier", (str(index_db_path),)) return tuple(build_raw_replay_plan(conn, component) for component in components) @@ -1275,19 +1284,37 @@ def record_raw_authority_census( ) -def validate_raw_replay_plan(archive_root: Path, plan: RawReplayPlan) -> tuple[bool, JSONDocument]: +def validate_raw_replay_plan( + archive_root: Path, + plan: RawReplayPlan, + *, + index_db_path: Path | None = None, +) -> tuple[bool, JSONDocument]: try: - observed = build_raw_replay_plans(archive_root, (plan.input_raw_ids,))[0] + observed = build_raw_replay_plans( + archive_root, + (plan.input_raw_ids,), + index_db_path=index_db_path, + )[0] except Exception as exc: logger.warning("raw replay plan validation could not rebuild %s", plan.plan_id, exc_info=True) return False, json_document({"error": f"{type(exc).__name__}: {exc}"}) return observed == plan, observed.to_dict() -def raw_replay_application_receipt(archive_root: Path, plan: RawReplayPlan) -> JSONDocument: +def raw_replay_application_receipt( + archive_root: Path, + plan: RawReplayPlan, + *, + index_db_path: Path | None = None, +) -> JSONDocument: + if index_db_path is None: + from polylogue.storage.archive_identity import resolve_active_index_path + + index_db_path = resolve_active_index_path(archive_root) marks = ",".join("?" for _ in plan.input_raw_ids) with closing(sqlite3.connect(f"file:{archive_root / 'source.db'}?mode=ro", uri=True)) as conn: - conn.execute("ATTACH DATABASE ? AS index_tier", (str(archive_root / "index.db"),)) + conn.execute("ATTACH DATABASE ? AS index_tier", (str(index_db_path),)) source = _rows( conn, f""" @@ -1349,6 +1376,7 @@ def raw_replay_application_receipt(archive_root: Path, plan: RawReplayPlan) -> J return json_document( { "schema": "polylogue.raw-replay-application-receipt.v2", + "index_db_path": str(index_db_path), "source_rows": source, "membership_rows": memberships, "application_rows": applications, @@ -1735,6 +1763,8 @@ def finalize_raw_authority_census( def recover_interrupted_raw_authority_censuses( archive_root: Path, + *, + index_db_path: Path | None = None, ) -> tuple[tuple[str, JSONDocument], ...]: """Reconcile unfinished apply censuses from durable postconditions.""" source_db = archive_root / "source.db" @@ -1771,7 +1801,7 @@ def recover_interrupted_raw_authority_censuses( for row in rows: census_id = str(row["census_id"]) plan = _raw_replay_plan_from_row(row) - receipt = raw_replay_application_receipt(archive_root, plan) + receipt = raw_replay_application_receipt(archive_root, plan, index_db_path=index_db_path) valid_receipt, problems = validate_raw_replay_application_receipt(plan, receipt) if valid_receipt: outcome = RawReplayPlanOutcome( @@ -1784,7 +1814,7 @@ def recover_interrupted_raw_authority_censuses( ) record_raw_replay_outcome(archive_root, census_id, outcome) continue - valid_plan, observed = validate_raw_replay_plan(archive_root, plan) + valid_plan, observed = validate_raw_replay_plan(archive_root, plan, index_db_path=index_db_path) if not valid_plan: reject_stale_raw_replay_plan(archive_root, census_id, plan, observed) else: diff --git a/polylogue/storage/raw_reconciler.py b/polylogue/storage/raw_reconciler.py index 0fb12f9c08..e4a7450533 100644 --- a/polylogue/storage/raw_reconciler.py +++ b/polylogue/storage/raw_reconciler.py @@ -630,6 +630,7 @@ def _item( def _classify_frontier( conn: sqlite3.Connection, blob_store: BlobStore, + index_db: Path, row: dict[str, object], strategy_override: _StrategyOverride | None, ) -> RawAuthorityFrontierItem: @@ -671,7 +672,7 @@ def _classify_frontier( if len(duplicate_siblings) != 1: raise RuntimeError(f"duplicate alias classification is not injective for {raw_id}") - with closing(sqlite3.connect(f"file:{blob_store.root.parent / 'index.db'}?mode=ro", uri=True)) as proof_conn: + with closing(sqlite3.connect(f"file:{index_db}?mode=ro", uri=True)) as proof_conn: proof_conn.row_factory = sqlite3.Row proof_conn.execute( "ATTACH DATABASE ? AS source", @@ -782,6 +783,8 @@ def _classify_frontier( def _strategy_overrides( config: Config, rows: list[dict[str, object]], + *, + index_db_path: Path, ) -> dict[str, _StrategyOverride]: """Ask legacy incident inspectors for proofs, never for plan identity.""" from polylogue.storage.repair import ( @@ -799,7 +802,11 @@ def _strategy_overrides( } ) for browser_chunk in _chunks(browser_ids): - browser_items = inspect_browser_capture_origin_mismatches(config, browser_chunk) + browser_items = inspect_browser_capture_origin_mismatches( + config, + browser_chunk, + index_db_path=index_db_path, + ) for browser_item in browser_items: if browser_item.status in {"eligible", "already_repaired"}: overrides[browser_item.raw_id] = _StrategyOverride( @@ -809,7 +816,11 @@ def _strategy_overrides( witness=_browser_strategy_witness(browser_item), input_raw_ids=_browser_strategy_raw_ids(browser_item), ) - conflicts = inspect_browser_canonical_authority_conflicts(config, browser_chunk) + conflicts = inspect_browser_canonical_authority_conflicts( + config, + browser_chunk, + index_db_path=index_db_path, + ) for conflict_item in conflicts.items: if conflict_item.raw_id in overrides: continue @@ -860,7 +871,11 @@ def _strategy_overrides( } ) for quarantine_chunk in _chunks(quarantine_pairs, size=100): - quarantine_items = inspect_quarantined_accepted_raws(config, quarantine_chunk) + quarantine_items = inspect_quarantined_accepted_raws( + config, + quarantine_chunk, + index_db_path=index_db_path, + ) for (raw_id, logical_source_key), quarantine_item in zip(quarantine_chunk, quarantine_items, strict=True): if quarantine_item.status in {"eligible", "already_repaired"}: overrides[_quarantine_override_key(raw_id, logical_source_key)] = _StrategyOverride( @@ -1194,14 +1209,14 @@ def _plan(item: RawAuthorityFrontierItem) -> RawReplayPlan: def _frontier_items(config: Config) -> tuple[tuple[RawAuthorityFrontierItem, ...], int, int]: root = _archive_root(config) source_db = root / "source.db" - index_db = root / "index.db" + index_db = config.current_db_path() if not source_db.is_file() or not index_db.is_file(): raise RuntimeError("raw authority frontier census requires initialized source and index tiers") with closing(sqlite3.connect(source_db)) as conn, conn: conn.row_factory = sqlite3.Row conn.execute("ATTACH DATABASE ? AS index_tier", (str(index_db),)) head_rows = _frontier_rows(conn) - overrides = _strategy_overrides(config, head_rows) + overrides = _strategy_overrides(config, head_rows, index_db_path=index_db) def _override_for(row: dict[str, object]) -> _StrategyOverride | None: raw_id = str(row["accepted_raw_id"]) @@ -1216,7 +1231,9 @@ def _override_for(row: dict[str, object]) -> _StrategyOverride | None: # through this same connection; the outer ``conn`` context manager commits # those writes on clean exit (or rolls back on exception), so a receipt is # never durably recorded for bytes this pass didn't finish inspecting. - head_items = [_classify_frontier(conn, BlobStore(root / "blob"), row, _override_for(row)) for row in head_rows] + head_items = [ + _classify_frontier(conn, BlobStore(root / "blob"), index_db, row, _override_for(row)) for row in head_rows + ] superseded_items = _terminal_superseded_items(conn) all_items = _apply_judgment_dispositions(config, (*head_items, *superseded_items)) return ( @@ -1352,8 +1369,6 @@ def _apply_strategy( root = _archive_root(config) source_db = root / "source.db" - index_db = root / "index.db" - if item.actuator is RawAuthorityActuator.RESOLVE_CONFLICT: conflict = item.strategy_witness.get("conflict") judgment = item.strategy_witness.get("judgment") @@ -1362,7 +1377,7 @@ def _apply_strategy( evidence = conflict.get("evidence") if not isinstance(evidence, dict) or judgment.get("disposition") != "retain_canonical_authority": raise RuntimeError("conflict-resolution strategy is not explicitly authorized") - with RebuildLease(root), closing(sqlite3.connect(f"file:{index_db}?mode=rw", uri=True)) as conn: + with RebuildLease(root), closing(sqlite3.connect(f"file:{config.current_db_path()}?mode=rw", uri=True)) as conn: conn.row_factory = sqlite3.Row conn.execute("PRAGMA foreign_keys = ON") conn.execute("ATTACH DATABASE ? AS source", (f"file:{source_db}?mode=ro",)) @@ -1387,7 +1402,7 @@ def _apply_strategy( if item.logical_source_key is None: raise RuntimeError("duplicate-alias plan is missing the logical source key it was proven against") logical_source_key = item.logical_source_key - with RebuildLease(root), closing(sqlite3.connect(f"file:{index_db}?mode=rw", uri=True)) as conn: + with RebuildLease(root), closing(sqlite3.connect(f"file:{config.current_db_path()}?mode=rw", uri=True)) as conn: conn.row_factory = sqlite3.Row conn.execute("PRAGMA foreign_keys = ON") conn.execute("ATTACH DATABASE ? AS source", (f"file:{source_db}?mode=ro",)) @@ -1454,7 +1469,7 @@ def _apply_strategy( from polylogue.storage.blob_publication import exclude_archive_blob_publishers with RebuildLease(root), exclude_archive_blob_publishers(source_db): - with closing(sqlite3.connect(f"file:{index_db}?mode=ro", uri=True)) as proof_conn: + with closing(sqlite3.connect(f"file:{config.current_db_path()}?mode=ro", uri=True)) as proof_conn: proof_conn.execute("ATTACH DATABASE ? AS source", (str(source_db),)) preview = _inspect_browser_capture_origin_strategy(root, item.raw_id, conn=proof_conn) if _browser_strategy_witness(preview) != item.strategy_witness: @@ -1475,7 +1490,7 @@ def _apply_strategy( pass elif preview.status != "already_repaired": raise RuntimeError(f"browser-origin strategy lost its exact proof: {preview.reason}") - with closing(sqlite3.connect(f"file:{index_db}?mode=rw", uri=True)) as conn: + with closing(sqlite3.connect(f"file:{config.current_db_path()}?mode=rw", uri=True)) as conn: conn.execute("PRAGMA foreign_keys = ON") conn.execute("ATTACH DATABASE ? AS source", (str(source_db),)) conn.execute("BEGIN IMMEDIATE") @@ -1506,7 +1521,7 @@ def _apply_strategy( logical_source_key = item.logical_source_key with RebuildLease(root), closing(sqlite3.connect(f"file:{source_db}?mode=rw", uri=True)) as source_conn: source_conn.execute("PRAGMA foreign_keys = ON") - _attach_repair_index(source_conn, index_db) + _attach_repair_index(source_conn, config.current_db_path()) source_conn.execute("BEGIN IMMEDIATE") try: # Apply-side stays fail-closed: an authorized plan whose single diff --git a/polylogue/storage/raw_retention.py b/polylogue/storage/raw_retention.py index 830c5afaa1..a6648ec0d6 100644 --- a/polylogue/storage/raw_retention.py +++ b/polylogue/storage/raw_retention.py @@ -10,13 +10,19 @@ from pathlib import Path from typing import Literal +from polylogue.core.raw_failure_evidence import RAW_FAILURE_EVIDENCE_KINDS, RawFailureEvidenceKind from polylogue.logging import get_logger +from polylogue.storage.archive_identity import ArchiveLocationError, resolve_active_index_path from polylogue.storage.blob_store import BlobStore, get_blob_store from polylogue.storage.introspection import column_exists as _column_exists from polylogue.storage.introspection import table_exists as _table_exists logger = get_logger(__name__) +_TERMINAL_RAW_FAILURE_EVIDENCE_KINDS = frozenset( + kind.value for kind in RawFailureEvidenceKind if kind.lifecycle == "terminal" +) + _V1_RAW_CANDIDATE_SQL = """ WITH ranked AS ( SELECT @@ -375,6 +381,7 @@ def active_raw_retention_authority( conn: sqlite3.Connection, *, index_db_path: Path, + terminal_source_paths: Iterable[Path] | None = None, ) -> RawRetentionAuthority: """Return current protection plus explicitly authorized deletion rows. @@ -383,6 +390,9 @@ def active_raw_retention_authority( immutable ``superseded`` receipt tied to the current head authorizes raw deletion. Callers must serialize this read with source deletion under the daemon's single-writer contract, or stop the daemon for manual cleanup. + ``terminal_source_paths`` scopes terminal-artifact protection only for a + deletion operation constrained to those same physical paths; callers that + may delete archive-wide must leave it unset. """ original_row_factory = conn.row_factory conn.row_factory = sqlite3.Row @@ -391,15 +401,33 @@ def active_raw_retention_authority( seeds = set(session_raw_ids) seeds.update(head.accepted_raw_id for head in heads) if not seeds: - if conn.execute("SELECT 1 FROM raw_sessions LIMIT 1").fetchone() is not None: + all_raw_ids = frozenset(str(row[0]) for row in conn.execute("SELECT raw_id FROM raw_sessions").fetchall()) + terminal_artifact_raw_ids = _terminal_artifact_raw_ids(conn) + if all_raw_ids and all_raw_ids.issubset(terminal_artifact_raw_ids): + return RawRetentionAuthority(protected_raw_ids=all_raw_ids, eligible_raw_ids=frozenset()) + if all_raw_ids: raise RawRetentionSafetyError("source tier contains raw evidence but index has no raw authority") return RawRetentionAuthority(protected_raw_ids=frozenset(), eligible_raw_ids=frozenset()) + terminal_artifact_raw_ids = _terminal_artifact_raw_ids(conn, source_paths=terminal_source_paths) authority_raw_ids = seeds.union(receipt.raw_id for receipt in eligible_receipts) rows_by_id = _raw_revision_rows(conn, authority_raw_ids) protected: set[str] = set() + byte_head_raw_ids = {head.accepted_raw_id for head in heads if head.accepted_frontier_kind == "byte"} + semantic_only_raw_ids = { + head.accepted_raw_id for head in heads if head.accepted_frontier_kind != "byte" + }.difference(byte_head_raw_ids) + # A semantic membership head is accepted authority for retention, but + # it is deliberately not a byte-predecessor proof. Keep it protected + # without reinterpreting it as one. + protected.update(semantic_only_raw_ids) + protected.update(terminal_artifact_raw_ids) for seed_raw_id in sorted(session_raw_ids): + if seed_raw_id in semantic_only_raw_ids: + continue protected.update(_validate_active_revision_chain(rows_by_id, seed_raw_id)) for head in heads: + if head.accepted_raw_id in semantic_only_raw_ids: + continue row = rows_by_id[head.accepted_raw_id] if head.accepted_frontier_kind == "byte": _validate_byte_head(row, head) @@ -413,6 +441,8 @@ def active_raw_retention_authority( protected_raw_ids=protected_ids, eligible_raw_ids=frozenset(eligible.difference(protected_ids)), ) + except sqlite3.Error as exc: + raise RawRetentionSafetyError(f"raw retention authority is unreadable: {exc}") from exc finally: conn.row_factory = original_row_factory @@ -1156,7 +1186,16 @@ def raw_frontier_integrity_projection( raw_materialization_readiness, sample_limit=sample_limit, ) - index_db_path = archive_root / "index.db" + try: + index_db_path = resolve_active_index_path(archive_root) + except ArchiveLocationError as exc: + return unknown_raw_frontier_integrity_projection( + f"active index pointer unavailable: {exc}", + missing_source_raw_status=missing_status, + missing_source_raw_count=missing_count, + missing_source_raw_samples=missing_samples, + missing_source_raw_reason=missing_reason, + ) source_db_path = archive_root / "source.db" ops_db_path = archive_root / "ops.db" snapshot = _unavailable_frontier_integrity_snapshot(f"source tier is unavailable: {source_db_path}") @@ -1206,7 +1245,14 @@ def raw_frontier_integrity_projection( ) -def unknown_raw_frontier_integrity_projection(reason: str) -> RawFrontierIntegrityProjection: +def unknown_raw_frontier_integrity_projection( + reason: str, + *, + missing_source_raw_status: RawFrontierIntegrityStatus = "unknown", + missing_source_raw_count: int = 0, + missing_source_raw_samples: tuple[Mapping[str, object], ...] = (), + missing_source_raw_reason: str | None = None, +) -> RawFrontierIntegrityProjection: """Return the canonical explicit-unknown projection for an unavailable read. Cache and presentation adapters use this instead of inventing partial @@ -1215,18 +1261,19 @@ def unknown_raw_frontier_integrity_projection(reason: str) -> RawFrontierIntegri """ snapshot = _unavailable_frontier_integrity_snapshot(reason) + statuses = (snapshot.broken_head_status, missing_source_raw_status, snapshot.cursor_ahead_status) return RawFrontierIntegrityProjection( available=False, - overall_status="unknown", + overall_status=combine_raw_frontier_integrity_statuses(*statuses), broken_head_status=snapshot.broken_head_status, broken_head_count=snapshot.broken_head_count, broken_head_checked_count=snapshot.broken_head_checked_count, broken_head_samples=snapshot.broken_head_samples, broken_head_reason=snapshot.broken_head_reason, - missing_source_raw_status="unknown", - missing_source_raw_count=0, - missing_source_raw_samples=(), - missing_source_raw_reason=reason, + missing_source_raw_status=missing_source_raw_status, + missing_source_raw_count=missing_source_raw_count, + missing_source_raw_samples=missing_source_raw_samples, + missing_source_raw_reason=reason if missing_source_raw_reason is None else missing_source_raw_reason, cursor_ahead_status=snapshot.cursor_ahead_status, cursor_ahead_count=snapshot.cursor_ahead_count, cursor_ahead_checked_count=snapshot.cursor_ahead_checked_count, @@ -1369,6 +1416,15 @@ def _check_broken_active_chains( for head in heads: heads_by_raw_id.setdefault(head.accepted_raw_id, []).append(head) seed_raw_ids = set(session_raw_ids).union(heads_by_raw_id) + # Membership-governed snapshots carry a semantic head, not a byte + # predecessor chain. They remain active source authority and must be + # retained, but applying byte-chain validation to them turns a normal + # membership snapshot into a false broken-head violation. A raw selected + # by both regimes remains byte-validated. + byte_head_raw_ids = {head.accepted_raw_id for head in heads if head.accepted_frontier_kind == "byte"} + semantic_only_raw_ids = { + head.accepted_raw_id for head in heads if head.accepted_frontier_kind != "byte" + }.difference(byte_head_raw_ids) try: rows_by_id = _raw_revision_rows(conn, seed_raw_ids, allow_missing=True) except _RawRevisionAuthorityUnavailableError as exc: @@ -1388,6 +1444,8 @@ def _check_broken_active_chains( try: if row is None: raise RawRetentionSafetyError(f"active index raw is missing from source tier: {seed_raw_id}") + if seed_raw_id in semantic_only_raw_ids: + continue for head in seed_heads: if head.accepted_frontier_kind == "byte": _validate_byte_head(row, head) @@ -1479,6 +1537,11 @@ def _check_cursor_ahead_of_accepted( except sqlite3.Error as exc: logger.warning("raw frontier integrity: cursor source path lookup failed: %s", exc) return "unknown", 0, 0, 0, 0, (), 0, (), f"cursor source path lookup failed: {exc}" + try: + terminal_artifact_paths = _terminal_artifact_paths(conn, set(cursor_map)) + except sqlite3.Error as exc: + logger.warning("raw frontier integrity: terminal artifact authority lookup failed: %s", exc) + return "unknown", 0, 0, 0, 0, (), 0, (), f"terminal artifact authority is unreadable: {exc}" for path, cursor in cursor_map.items(): cursor_offset = cursor.byte_offset if cursor.is_deferred: @@ -1501,7 +1564,7 @@ def _check_cursor_ahead_of_accepted( if not comparable_heads: # A path governed exclusively by membership authority has no # comparable byte frontier and is intentionally out of scope. - if path in all_head_paths: + if path in all_head_paths or path in terminal_artifact_paths: continue gap_count += 1 if len(gaps) < sample_limit: @@ -1593,6 +1656,167 @@ def _source_paths_for_paths(conn: sqlite3.Connection, source_paths: set[str]) -> return result +def _terminal_artifact_paths(conn: sqlite3.Connection, source_paths: set[str]) -> set[str]: + """Return paths whose every current source coordinate is terminal evidence. + + A full-route cursor can legitimately advance over a workflow/fact artifact + that has no session head. ``raw_artifacts.parse_as_session = 0`` is the + source-tier terminal authority for that case. Ordinary artifact upserts + retain the source coordinate's latest receipt while ``raw_sessions`` + retains its historical acquisition evidence, so authority attaches to each + coordinate's newest raw observation rather than requiring a duplicate + receipt on every historical raw. A failure-kind carrier remains authority + only while that raw's current parse or validation state is failed; a later + successful reparse makes the retained carrier historical evidence. Every + ``(origin, source_index)`` member of a physical path must be terminal before + the cursor path is exempt. + """ + + result: set[str] = set() + raw_failure_kinds = tuple(sorted(RAW_FAILURE_EVIDENCE_KINDS)) + terminal_raw_failure_kinds = tuple(sorted(_TERMINAL_RAW_FAILURE_EVIDENCE_KINDS)) + raw_failure_placeholders = ", ".join("?" for _ in raw_failure_kinds) + terminal_raw_failure_placeholders = ", ".join("?" for _ in terminal_raw_failure_kinds) + # The path batch binds once for observation receipts and once for raw rows. + path_batch_size = max(1, (500 - len(raw_failure_kinds) - len(terminal_raw_failure_kinds)) // 2) + pending = set(source_paths) + while pending: + batch = tuple(sorted(pending)[:path_batch_size]) + pending.difference_update(batch) + placeholders = ", ".join("?" for _ in batch) + rows = conn.execute( + f""" + WITH latest_raw_observation AS ( + SELECT raw_id, acquired_at_ms, observation_rowid + FROM ( + SELECT + ref_id AS raw_id, + acquired_at_ms, + rowid AS observation_rowid, + ROW_NUMBER() OVER ( + PARTITION BY ref_id + ORDER BY acquired_at_ms DESC, rowid DESC + ) AS observation_rank + FROM blob_refs + WHERE ref_type = 'raw_payload' + AND source_path IN ({placeholders}) + ) + WHERE observation_rank = 1 + ), + newest_per_coordinate AS ( + SELECT raw_id, source_path, origin, source_index, parse_error, + validation_status, validated_at_ms, parsed_at_ms + FROM ( + SELECT + raw.raw_id, + raw.source_path, + raw.origin, + raw.source_index, + raw.parse_error, + raw.validation_status, + raw.validated_at_ms, + raw.parsed_at_ms, + ROW_NUMBER() OVER ( + PARTITION BY raw.source_path, raw.origin, raw.source_index + ORDER BY + COALESCE(observation.acquired_at_ms, raw.acquired_at_ms) DESC, + COALESCE(observation.observation_rowid, raw.rowid) DESC + ) AS coordinate_rank + FROM raw_sessions AS raw + LEFT JOIN latest_raw_observation AS observation ON observation.raw_id = raw.raw_id + WHERE raw.source_path IN ({placeholders}) + ) + WHERE coordinate_rank = 1 + ), + terminal_artifacts AS ( + SELECT artifact.raw_id + FROM raw_artifacts AS artifact + JOIN newest_per_coordinate AS evidence_raw ON evidence_raw.raw_id = artifact.raw_id + WHERE artifact.parse_as_session = 0 + AND ( + ( + artifact.artifact_kind NOT IN ({raw_failure_placeholders}) + AND ( + evidence_raw.parsed_at_ms IS NULL + OR artifact.last_observed_at_ms >= evidence_raw.parsed_at_ms + ) + ) + OR ( + artifact.artifact_kind IN ({terminal_raw_failure_placeholders}) + AND ( + evidence_raw.parse_error IS NOT NULL + OR ( + evidence_raw.validation_status = 'failed' + AND ( + evidence_raw.parsed_at_ms IS NULL + OR evidence_raw.validated_at_ms IS NULL + -- A legacy tie has no proven winner; + -- retain it rather than deleting raw + -- authority based on an arbitrary side. + OR evidence_raw.validated_at_ms >= evidence_raw.parsed_at_ms + ) + ) + ) + ) + ) + ), + terminal_evidence AS ( + SELECT raw_id FROM terminal_artifacts + UNION + SELECT evidence_raw.raw_id + FROM newest_per_coordinate AS evidence_raw + JOIN raw_membership_census AS census ON census.raw_id = evidence_raw.raw_id + WHERE census.status = 'non_session' + ) + SELECT DISTINCT terminal_raw.source_path + FROM terminal_evidence AS artifact + JOIN newest_per_coordinate AS terminal_raw ON terminal_raw.raw_id = artifact.raw_id + WHERE NOT EXISTS ( + SELECT 1 + FROM newest_per_coordinate AS coordinate + WHERE coordinate.source_path = terminal_raw.source_path + AND NOT EXISTS ( + SELECT 1 + FROM terminal_evidence AS current_artifact + WHERE current_artifact.raw_id = coordinate.raw_id + ) + ) + """, + (*batch, *batch, *raw_failure_kinds, *terminal_raw_failure_kinds), + ).fetchall() + result.update(str(row[0]) for row in rows) + return result + + +def _terminal_artifact_raw_ids( + conn: sqlite3.Connection, + *, + source_paths: Iterable[Path] | None = None, +) -> frozenset[str]: + """Return all retained raw evidence for paths with terminal current observations.""" + + selected_paths = ( + {str(row[0]) for row in conn.execute("SELECT DISTINCT source_path FROM raw_sessions").fetchall()} + if source_paths is None + else {str(path) for path in source_paths} + ) + terminal_paths = _terminal_artifact_paths(conn, selected_paths) + if not terminal_paths: + return frozenset() + raw_ids: set[str] = set() + pending = set(terminal_paths) + while pending: + batch = tuple(sorted(pending)[:500]) + pending.difference_update(batch) + placeholders = ", ".join("?" for _ in batch) + rows = conn.execute( + f"SELECT raw_id FROM raw_sessions WHERE source_path IN ({placeholders})", + batch, + ).fetchall() + raw_ids.update(str(row[0]) for row in rows) + return frozenset(raw_ids) + + def _ops_cursor_byte_offsets(ops_db_path: Path) -> dict[str, _OpsCursorAuthority]: if not ops_db_path.is_file(): raise RawRetentionSafetyError(f"ops tier is unavailable: {ops_db_path}") diff --git a/polylogue/storage/repair.py b/polylogue/storage/repair.py index ff9102247e..e6673ba001 100644 --- a/polylogue/storage/repair.py +++ b/polylogue/storage/repair.py @@ -45,7 +45,7 @@ ) from polylogue.pipeline.ids import session_content_hash, session_revision_projection from polylogue.pipeline.ids import session_id as make_session_id -from polylogue.storage.archive_identity import archive_file_set_root +from polylogue.storage.archive_identity import archive_file_set_root, resolve_active_index_path from polylogue.storage.blob_repair import count_orphaned_blobs_sync, repair_orphaned_blobs_data from polylogue.storage.blob_store import BlobStore from polylogue.storage.insights.session.repair_assessment import ( @@ -61,6 +61,7 @@ count_unclassified_message_type_sync, ) from polylogue.storage.raw_authority import ( + RAW_AUTHORITY_PARSER_FINGERPRINT, RAW_REPLAY_NO_PROGRESS_REASON, SUPERSEDED_MEMBERSHIP_FINGERPRINTS, RawAuthorityCensusReceipt, @@ -83,6 +84,7 @@ validate_raw_replay_application_receipt, validate_raw_replay_plan, ) +from polylogue.storage.sqlite.queries.raw_state import raw_provider_origin_sql if TYPE_CHECKING: # ``revision_backfill`` imports ``ArchiveStore``, which (via @@ -1228,6 +1230,8 @@ def _cas_refine_quarantined_accepted_raw( def inspect_quarantined_accepted_raws( config: Config, raw_ids_with_keys: list[tuple[str, str]], + *, + index_db_path: Path | None = None, ) -> tuple[QuarantinedAcceptedRawRepairItem, ...]: """Return exact typed quarantine-refinement proofs without mutation. @@ -1246,7 +1250,7 @@ def inspect_quarantined_accepted_raws( raise ValueError("raw ids must be lowercase SHA-256 identifiers") archive_root = _raw_materialization_archive_root(config) source_db = archive_root / "source.db" - index_db = archive_root / "index.db" + index_db = index_db_path or config.current_db_path() if not source_db.exists() or not index_db.exists(): raise RuntimeError("source or index tier is missing") with closing(sqlite3.connect(f"file:{source_db}?mode=ro", uri=True)) as conn: @@ -2995,6 +2999,8 @@ def _inspect_browser_capture_origin_strategy( def inspect_browser_capture_origin_mismatches( config: Config, raw_ids: list[str], + *, + index_db_path: Path | None = None, ) -> tuple[BrowserCaptureOriginRepairItem, ...]: """Return the exact admitted browser-origin strategy for each raw. @@ -3010,7 +3016,7 @@ def inspect_browser_capture_origin_mismatches( raise ValueError("raw ids must be lowercase SHA-256 identifiers") archive_root = _raw_materialization_archive_root(config) source_db = archive_root / "source.db" - index_db = archive_root / "index.db" + index_db = index_db_path or config.current_db_path() if not source_db.exists() or not index_db.exists(): raise RuntimeError("source or index tier is missing") with closing(sqlite3.connect(f"file:{index_db}?mode=ro", uri=True)) as conn: @@ -3241,7 +3247,10 @@ def ineligible(reason: str) -> BrowserCanonicalAuthorityConflictWitness: def inspect_browser_canonical_authority_conflicts( - config: Config, raw_ids: list[str] + config: Config, + raw_ids: list[str], + *, + index_db_path: Path | None = None, ) -> BrowserCanonicalAuthorityConflictReport: """Build read-only evidence packets for browser-capture raws a safe rekey refuses. @@ -3268,7 +3277,7 @@ def inspect_browser_canonical_authority_conflicts( raise ValueError("raw ids must be lowercase SHA-256 identifiers") archive_root = _raw_materialization_archive_root(config) source_db = archive_root / "source.db" - index_db = archive_root / "index.db" + index_db = index_db_path or config.current_db_path() if not source_db.exists() or not index_db.exists(): raise RuntimeError("source or index tier is missing") @@ -3772,20 +3781,37 @@ def _raw_materialization_archive_root(config: Config) -> Path: def _raw_materialization_index_path(config: Config, archive_root: Path) -> Path: - """Return the active derived tier while keeping durable tiers at root.""" - return config.db_path if config.db_path.name == "index.db" else archive_root / "index.db" + """Return an explicit index override or the archive's active generation.""" + del archive_root + return config.current_db_path() def _raw_artifact_coordinate_predicate(*, artifact_alias: str, raw_alias: str) -> str: """Correlate evidence with the exact failed artifact observation.""" + provider_origin = raw_provider_origin_sql(table_alias=raw_alias) return f""" AND {artifact_alias}.raw_id IS {raw_alias}.raw_id - AND {artifact_alias}.origin IS {raw_alias}.origin + AND ( + {artifact_alias}.origin IS {raw_alias}.origin + OR {artifact_alias}.origin IS ({provider_origin}) + ) AND {artifact_alias}.source_path IS {raw_alias}.source_path AND {artifact_alias}.source_index IS {raw_alias}.source_index """ +def _failed_validation_overrides_parse_predicate(*, raw_alias: str) -> str: + """Return the durable state in which validation blocks raw replay.""" + return f""" + COALESCE({raw_alias}.validation_status, '') = 'failed' + AND ( + {raw_alias}.parsed_at_ms IS NULL + OR {raw_alias}.validated_at_ms IS NULL + OR {raw_alias}.validated_at_ms >= {raw_alias}.parsed_at_ms + ) + """ + + def _raw_materialization_candidate_ids( config: Config, *, @@ -3850,10 +3876,11 @@ def _raw_materialization_candidate_ids( if raw_artifact_id is not None: raw_filter = "AND r.raw_id = ?" params.append(raw_artifact_id) + effective_origin = raw_provider_origin_sql(table_alias="r") origin_filter = "" provider_origin = _raw_materialization_origin_from_provider(provider) if provider_origin is not None: - origin_filter += " AND r.origin = ?" + origin_filter += f" AND {effective_origin} = ?" params.append(provider_origin) if source_family is not None: origin_filter += " AND r.origin = ?" @@ -3866,8 +3893,9 @@ def _raw_materialization_candidate_ids( terminal_pair_placeholders = ", ".join("(?, ?)" for _ in RAW_FAILURE_TERMINAL_EVIDENCE_SUPPORT_STATUS_PAIRS) rows = conn.execute( f""" - SELECT r.raw_id, r.origin, r.native_id, r.source_path, r.blob_hash, r.blob_size, - r.acquired_at_ms, r.parsed_at_ms, + SELECT r.raw_id, r.origin, {effective_origin} AS provider_origin, + r.native_id, r.source_path, r.blob_hash, r.blob_size, + r.acquired_at_ms, r.parsed_at_ms, r.validated_at_ms, r.parse_error, ( SELECT a.artifact_kind @@ -3910,6 +3938,15 @@ def _raw_materialization_candidate_ids( AND (m.decision IS NULL OR m.decision IN ('ambiguous', 'deferred')) ) ) AS membership_authority_complete + , EXISTS ( + SELECT 1 + FROM raw_membership_census AS c + WHERE c.raw_id = r.raw_id + AND c.parser_fingerprint = ? + AND c.status = 'non_session' + AND r.parsed_at_ms IS NOT NULL + AND r.parse_error IS NULL + ) AS membership_non_session_terminal , EXISTS ( SELECT 1 FROM raw_membership_census AS c @@ -3940,7 +3977,7 @@ def _raw_materialization_candidate_ids( LEFT JOIN index_tier.sessions AS s_by_raw ON s_by_raw.raw_id = r.raw_id LEFT JOIN index_tier.sessions AS s_by_native ON r.native_id IS NOT NULL - AND s_by_native.origin = r.origin + AND s_by_native.origin = {effective_origin} AND s_by_native.native_id = r.native_id LEFT JOIN raw_sessions AS existing_native_raw ON existing_native_raw.raw_id = s_by_native.raw_id @@ -3949,10 +3986,14 @@ def _raw_materialization_candidate_ids( s_by_native.native_id IS NULL OR existing_native_raw.raw_id IS NULL ) - -- A failed worker validation is not replay authority. Keep - -- the raw bytes and their diagnostics, but require a fresh - -- validation outcome before materialization can select them. - AND COALESCE(r.validation_status, '') != 'failed' + -- A failed worker validation is replay authority only until a + -- successful parse records a durable parsed timestamp. Keep + -- that historical diagnostic, but do not let it block an + -- index reset from replaying successfully parsed raw bytes. + -- Equal legacy timestamps are indeterminate. Do not replay + -- and overwrite either authority until a monotonic transition + -- resolves the ambiguity. + AND NOT ({_failed_validation_overrides_parse_predicate(raw_alias="r")}) AND ( r.parse_error IS NULL OR r.parse_error = 'OperationalError: database is locked' @@ -3981,6 +4022,10 @@ def _raw_materialization_candidate_ids( AND (terminal_evidence.artifact_kind, terminal_evidence.support_status) IN ( {terminal_pair_placeholders} ) + AND ( + r.parse_error IS NOT NULL + OR ({_failed_validation_overrides_parse_predicate(raw_alias="r")}) + ) ) AND NOT ( COALESCE(r.validation_status, '') = 'skipped' @@ -3995,6 +4040,7 @@ def _raw_materialization_candidate_ids( [ *sorted(RAW_FAILURE_REPLAY_AUTHORITY_EVIDENCE_KINDS), RAW_FAILURE_DEFERRED_SUPPORT_STATUS, + RAW_AUTHORITY_PARSER_FINGERPRINT, BYTE_AUTHORITY_CENSUS_DETAIL, BYTE_AUTHORITY_CENSUS_DETAIL, *sorted(RAW_FAILURE_REPLAY_AUTHORITY_EVIDENCE_KINDS), @@ -4044,13 +4090,15 @@ def _raw_materialization_candidate_ids( continue if _raw_materialized_by_source_path_native(materialized_aliases, row): continue + if bool(row["membership_non_session_terminal"]): + continue if _raw_materialization_parsed_non_session_artifact(archive_root, row): continue blob_hash = row["blob_hash"].hex() if isinstance(row["blob_hash"], bytes) else str(row["blob_hash"]) if blob_store.exists(blob_hash): raw_id = str(row["raw_id"]) raw_ids.append(raw_id) - raw_origins[raw_id] = str(row["origin"] or "") + raw_origins[raw_id] = str(row["provider_origin"] or "") raw_source_paths[raw_id] = str(row["source_path"] or "") raw_acquired_at_ms[raw_id] = int(row["acquired_at_ms"] or 0) blob_size = row["blob_size"] @@ -4072,8 +4120,12 @@ def _raw_materialization_candidate_ids( for offset in range(0, len(expanded_raw_ids), 500): raw_id_chunk = expanded_raw_ids[offset : offset + 500] placeholders = ",".join("?" for _ in raw_id_chunk) + expanded_effective_origin = raw_provider_origin_sql(table_alias="raw_sessions") for row in conn.execute( - f"SELECT raw_id, blob_size, origin, source_path FROM raw_sessions WHERE raw_id IN ({placeholders})", + f""" + SELECT raw_id, blob_size, {expanded_effective_origin}, source_path + FROM raw_sessions WHERE raw_id IN ({placeholders}) + """, raw_id_chunk, ): rid = str(row[0]) @@ -4135,9 +4187,10 @@ def _raw_materialization_parser_census_candidates( if raw_artifact_id is not None: filters.append("r.raw_id = ?") params.append(raw_artifact_id) + effective_origin = raw_provider_origin_sql(table_alias="r") provider_origin = _raw_materialization_origin_from_provider(provider) if provider_origin is not None: - filters.append("r.origin = ?") + filters.append(f"{effective_origin} = ?") params.append(provider_origin) if source_family is not None: filters.append("r.origin = ?") @@ -4149,7 +4202,8 @@ def _raw_materialization_parser_census_candidates( where = f"WHERE {' AND '.join(filters)}" if filters else "" rows = conn.execute( f""" - SELECT r.raw_id, r.blob_size, r.origin, r.source_path, r.acquired_at_ms + SELECT r.raw_id, r.blob_size, {effective_origin} AS provider_origin, + r.source_path, r.acquired_at_ms FROM raw_sessions AS r {where} ORDER BY r.acquired_at_ms DESC, r.raw_id ASC @@ -4165,7 +4219,7 @@ def _raw_materialization_parser_census_candidates( raw_id = str(row["raw_id"]) raw_ids.append(raw_id) raw_blob_bytes[raw_id] = int(row["blob_size"] or 0) - raw_origins[raw_id] = str(row["origin"] or "") + raw_origins[raw_id] = str(row["provider_origin"] or "") raw_source_paths[raw_id] = str(row["source_path"] or "") raw_acquired_at_ms[raw_id] = int(row["acquired_at_ms"] or 0) from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore @@ -4178,8 +4232,12 @@ def _raw_materialization_parser_census_candidates( for offset in range(0, len(expanded_raw_ids), 500): raw_id_chunk = expanded_raw_ids[offset : offset + 500] placeholders = ",".join("?" for _ in raw_id_chunk) + expanded_effective_origin = raw_provider_origin_sql(table_alias="raw_sessions") for row in conn.execute( - f"SELECT raw_id, blob_size, origin, source_path FROM raw_sessions WHERE raw_id IN ({placeholders})", + f""" + SELECT raw_id, blob_size, {expanded_effective_origin}, source_path + FROM raw_sessions WHERE raw_id IN ({placeholders}) + """, raw_id_chunk, ): raw_id = str(row[0]) @@ -4275,18 +4333,23 @@ def raw_materialization_readonly_descriptors( placeholders = ",".join("?" for _ in raw_id_chunk) rows = conn.execute( f""" - SELECT raw_id, origin, capture_mode, lower(hex(blob_hash)), source_path, revision_kind, blob_size + SELECT raw_id, origin, detected_provider, capture_mode, + lower(hex(blob_hash)), source_path, revision_kind, blob_size FROM raw_sessions WHERE raw_id IN ({placeholders}) """, raw_id_chunk, ).fetchall() for row in rows: result[str(row[0])] = ( - provider_from_origin(Origin.from_string(str(row[1])), family_hint=row[2]), - str(row[3]), + ( + Provider.from_string(str(row[2])) + if row[2] is not None + else provider_from_origin(Origin.from_string(str(row[1])), family_hint=row[3]) + ), str(row[4]), - RawRevisionKind(str(row[5])), - int(row[6]), + str(row[5]), + RawRevisionKind(str(row[6])), + int(row[7]), ) return result @@ -4355,12 +4418,13 @@ def _raw_materialization_ordered_components( candidates: RawMaterializationCandidates, *, archive_root: Path, + index_db_path: Path | None = None, ) -> list[tuple[str, ...]]: """Order complete components fairly without splitting authority cohorts.""" candidate_ids = set(candidates.raw_ids) source_components = candidates.authority_components or tuple((raw_id,) for raw_id in candidates.raw_ids) components = [component for component in source_components if candidate_ids.intersection(component)] - plans = build_raw_replay_plans(archive_root, components) + plans = build_raw_replay_plans(archive_root, components, index_db_path=index_db_path) plan_ids = {plan.input_raw_ids: plan.plan_id for plan in plans} last_attempts = raw_replay_plan_last_attempts(archive_root) @@ -4456,7 +4520,11 @@ def raw_materialization_whale_pass_candidate( ): if not candidates.raw_ids: continue - ordered_components = _raw_materialization_ordered_components(candidates, archive_root=archive_root) + ordered_components = _raw_materialization_ordered_components( + candidates, + archive_root=archive_root, + index_db_path=_raw_materialization_index_path(config, archive_root), + ) for component in ordered_components: if census_only and not blocked_raw_ids.intersection(component): continue @@ -4527,10 +4595,15 @@ def _raw_authority_postflight_snapshot( candidates: RawMaterializationCandidates, *, max_payload_bytes: int, + index_db_path: Path | None = None, ) -> tuple[tuple[RawReplayPlan, ...], dict[str, object]]: """Build the complete post-pass plan inventory and typed residual debt.""" - components = _raw_materialization_ordered_components(candidates, archive_root=archive_root) - plans = build_raw_replay_plans(archive_root, components) + components = _raw_materialization_ordered_components( + candidates, + archive_root=archive_root, + index_db_path=index_db_path, + ) + plans = build_raw_replay_plans(archive_root, components, index_db_path=index_db_path) blocked_plan_ids = tuple( sorted( plan.plan_id @@ -4687,7 +4760,7 @@ def _raw_replay_plan_outcome( SELECT 1 FROM raw_sessions WHERE raw_id IN ({placeholders}) - AND validation_status = 'failed' + AND ({_failed_validation_overrides_parse_predicate(raw_alias="raw_sessions")}) UNION ALL SELECT 1 FROM raw_sessions @@ -4756,6 +4829,7 @@ def _raw_replay_plan_outcome( def _raw_replay_plan_outcomes( archive_root: Path, + index_db: Path, plans: Sequence[RawReplayPlan], *, remaining: RawMaterializationCandidates, @@ -4765,7 +4839,7 @@ def _raw_replay_plan_outcomes( return () with closing(sqlite3.connect(f"file:{archive_root / 'source.db'}?mode=ro", uri=True)) as conn: conn.row_factory = sqlite3.Row - conn.execute("ATTACH DATABASE ? AS index_tier", (str(archive_root / "index.db"),)) + conn.execute("ATTACH DATABASE ? AS index_tier", (str(index_db),)) return tuple( _raw_replay_plan_outcome(conn, plan, remaining=remaining, no_progress=no_progress) for plan in plans ) @@ -5065,7 +5139,7 @@ def raw_materialization_scale_profile(config: Config) -> dict[str, object]: def _raw_materialized_by_source_path_native(materialized_aliases: set[tuple[str, str]], row: sqlite3.Row) -> bool: - origin = str(row["origin"] or "") + origin = str(row["provider_origin"] or "") if not origin: return False for native_id in _source_path_native_id_candidates(str(row["source_path"] or "")): @@ -5084,7 +5158,7 @@ def _raw_materialization_parsed_non_session_artifact(archive_root: Path, row: sq return ( parsed_non_session_artifact_reason( archive_root=archive_root, - origin=str(row["origin"] or ""), + origin=str(row["provider_origin"] or ""), source_path=str(row["source_path"] or ""), blob_hash=blob_hash, ) @@ -5113,7 +5187,6 @@ def _source_path_native_id_candidates(source_path: str) -> tuple[str, ...]: def _open_archive_index_connection() -> sqlite3.Connection: from polylogue.paths import archive_root - from polylogue.storage.archive_identity import resolve_active_index_path conn = sqlite3.connect(resolve_active_index_path(archive_root())) conn.row_factory = sqlite3.Row @@ -5562,6 +5635,16 @@ def _internal_derived_repair_result( ) +def raw_materialization_lease_refusal_result(error: BaseException) -> RepairResult: + """Translate active-generation lease refusal into the repair contract.""" + return _internal_derived_repair_result( + "raw_materialization", + repaired_count=0, + success=False, + detail=f"Skipped raw materialization while offline index rebuild owns archive: {error}", + ) + + def _archive_debt_status( target_name: str, *, @@ -5880,6 +5963,30 @@ def count_superseded_raw_snapshots_sync(conn: sqlite3.Connection) -> int: def repair_superseded_raw_snapshots(config: Config, dry_run: bool = False) -> RepairResult: + """Delete redundant raw snapshots while promotion cannot change the protected set.""" + + if dry_run: + return _repair_superseded_raw_snapshots(config, dry_run=True) + + from polylogue.storage.index_generation import ActiveWriterLease, RebuildLeaseUnavailableError + + lease = ActiveWriterLease(_raw_materialization_archive_root(config)) + try: + lease.acquire() + except RebuildLeaseUnavailableError as exc: + return _repair_result( + "superseded_raw_snapshots", + repaired_count=0, + success=False, + detail=f"Skipped destructive raw cleanup: {exc}", + ) + try: + return _repair_superseded_raw_snapshots(config, dry_run=False) + finally: + lease.close() + + +def _repair_superseded_raw_snapshots(config: Config, dry_run: bool = False) -> RepairResult: from polylogue.storage.raw_retention import ( RawRetentionSafetyError, active_raw_retention_authority, @@ -5890,7 +5997,7 @@ def repair_superseded_raw_snapshots(config: Config, dry_run: bool = False) -> Re archive_root = _raw_materialization_archive_root(config) repair_db_path = archive_root / "source.db" if repair_db_path.exists(): - index_db_path = archive_root / "index.db" + index_db_path = _raw_materialization_index_path(config, archive_root) with closing(open_connection(repair_db_path)) as conn, conn: conn.row_factory = sqlite3.Row try: @@ -6051,7 +6158,6 @@ def repair_session_insights( clearing the active daemon's debt ledger. """ from polylogue.paths import archive_root as _resolve_archive_root - from polylogue.storage.archive_identity import resolve_active_index_path from polylogue.storage.insights.session.rebuild import ( rebuild_archive_session_insights, refresh_session_insight_aggregates_sync, @@ -6209,6 +6315,58 @@ def repair_raw_materialization( progress_callback: ProgressCallback | None = None, prefetch_cache: RawParsePrefetchCache | None = None, max_pass_seconds: float | None = None, +) -> RepairResult: + """Converge one raw-materialization pass under active-generation ownership.""" + + def run() -> RepairResult: + return _repair_raw_materialization( + config, + dry_run=dry_run, + raw_artifact_id=raw_artifact_id, + provider=provider, + source_family=source_family, + source_root=source_root, + raw_artifact_limit=raw_artifact_limit, + max_payload_bytes=max_payload_bytes, + ingest_workers=ingest_workers, + commit_batch_size=commit_batch_size, + progress_callback=progress_callback, + prefetch_cache=prefetch_cache, + max_pass_seconds=max_pass_seconds, + ) + + if dry_run: + return run() + + from polylogue.storage.index_generation import ActiveWriterLease, RebuildLeaseUnavailableError + + archive_root = _raw_materialization_archive_root(config) + lease = ActiveWriterLease(archive_root) + try: + lease.acquire() + except RebuildLeaseUnavailableError as exc: + return raw_materialization_lease_refusal_result(exc) + try: + return run() + finally: + lease.close() + + +def _repair_raw_materialization( + config: Config, + dry_run: bool = False, + *, + raw_artifact_id: str | None = None, + provider: str | None = None, + source_family: str | None = None, + source_root: Path | None = None, + raw_artifact_limit: int | None = None, + max_payload_bytes: int = RAW_MATERIALIZATION_EXECUTE_BLOB_LIMIT_BYTES, + ingest_workers: int | None = None, + commit_batch_size: int | None = None, + progress_callback: ProgressCallback | None = None, + prefetch_cache: RawParsePrefetchCache | None = None, + max_pass_seconds: float | None = None, ) -> RepairResult: """Converge retained raws through typed per-session revision authority. @@ -6278,7 +6436,8 @@ def _pass_deadline_exceeded() -> bool: return max_pass_seconds is not None and (time.monotonic() - pass_started_monotonic) >= max_pass_seconds archive_root = _raw_materialization_archive_root(config) - recovered_censuses = recover_interrupted_raw_authority_censuses(archive_root) + index_db = _raw_materialization_index_path(config, archive_root) + recovered_censuses = recover_interrupted_raw_authority_censuses(archive_root, index_db_path=index_db) for recovered_census_id, recovered_scope in recovered_censuses: recovered_envelope = recovered_scope.get("max_payload_bytes") recovered_max_payload_bytes = ( @@ -6289,6 +6448,7 @@ def _pass_deadline_exceeded() -> bool: archive_root, recovered_candidates, max_payload_bytes=recovered_max_payload_bytes, + index_db_path=index_db, ) finalize_raw_authority_census( archive_root, @@ -6344,7 +6504,11 @@ def _pass_deadline_exceeded() -> bool: raise ValueError("raw_artifact_limit must be positive") census_components_attempted = 0 if uncensused_raw_ids: - preliminary_components = _raw_materialization_ordered_components(census_candidates, archive_root=archive_root) + preliminary_components = _raw_materialization_ordered_components( + census_candidates, + archive_root=archive_root, + index_db_path=index_db, + ) for component in preliminary_components: if not uncensused_raw_ids.intersection(component): continue @@ -6363,6 +6527,7 @@ def _pass_deadline_exceeded() -> bool: try: census_historical_revision_evidence( archive_root, + active_index_path=index_db, selected_raw_ids=[seed], max_payload_bytes=max_payload_bytes, ingest_workers=ingest_workers, @@ -6491,8 +6656,12 @@ def _pass_deadline_exceeded() -> bool: census_receipt=census_receipt, ) candidate_raw_ids = candidates.raw_ids - ordered_components = _raw_materialization_ordered_components(candidates, archive_root=archive_root) - plans = build_raw_replay_plans(archive_root, ordered_components) + ordered_components = _raw_materialization_ordered_components( + candidates, + archive_root=archive_root, + index_db_path=index_db, + ) + plans = build_raw_replay_plans(archive_root, ordered_components, index_db_path=index_db) plan_by_component = {plan.input_raw_ids: plan for plan in plans} all_blocked_components = [ component @@ -6759,7 +6928,7 @@ def _pass_deadline_exceeded() -> bool: stale_outcomes: list[RawReplayPlanOutcome] = [] validated_plans: list[RawReplayPlan] = [] for plan in executable_plans: - valid, observed = validate_raw_replay_plan(archive_root, plan) + valid, observed = validate_raw_replay_plan(archive_root, plan, index_db_path=index_db) if valid: validated_plans.append(plan) else: @@ -6788,6 +6957,7 @@ def _pass_deadline_exceeded() -> bool: archive_root, stale_candidates, max_payload_bytes=max_payload_bytes, + index_db_path=index_db, ) census_receipt = finalize_raw_authority_census( archive_root, @@ -6828,7 +6998,7 @@ def _pass_deadline_exceeded() -> bool: # writer-hot table before this bounded live pass; this is the same # planner invariant seeded for a fresh index bootstrap, without turning # raw materialization into a full rebuild. - with closing(sqlite3.connect(archive_root / "index.db", timeout=60)) as planner_conn: + with closing(sqlite3.connect(index_db, timeout=60)) as planner_conn: planner_conn.execute("PRAGMA busy_timeout = 60000") # A freshly reset index uses representative bootstrap statistics. # ``ANALYZE blocks`` on an empty table deletes that seed and brings @@ -6880,6 +7050,7 @@ def _pass_deadline_exceeded() -> bool: try: part = backfill_historical_revision_evidence( archive_root, + active_index_path=index_db, selected_raw_ids=[raw_id], max_payload_bytes=max_payload_bytes, ingest_workers=ingest_workers, @@ -6927,7 +7098,7 @@ def _pass_deadline_exceeded() -> bool: continue except Exception as exc: logger.exception("raw replay plan %s failed", plan.plan_id) - application_receipt = raw_replay_application_receipt(archive_root, plan) + application_receipt = raw_replay_application_receipt(archive_root, plan, index_db_path=index_db) receipt_valid, receipt_problems = validate_raw_replay_application_receipt(plan, application_receipt) if receipt_valid: outcome = RawReplayPlanOutcome( @@ -6940,7 +7111,11 @@ def _pass_deadline_exceeded() -> bool: ) record_raw_replay_outcome(archive_root, census_receipt.census_id, outcome) else: - plan_still_valid, _ = validate_raw_replay_plan(archive_root, plan) + plan_still_valid, _ = validate_raw_replay_plan( + archive_root, + plan, + index_db_path=index_db, + ) if plan_still_valid: outcome = RawReplayPlanOutcome( plan.plan_id, @@ -6984,9 +7159,11 @@ def _pass_deadline_exceeded() -> bool: # ``_raw_replay_plan_outcome`` types this TERMINAL (not RETRYABLE) so # it stops being silently reselected forever. no_progress = part.replayed_logical_sources == 0 and part.quarantined == 0 and part.adoption_deferred == 0 - component_outcomes = _raw_replay_plan_outcomes(archive_root, [plan], remaining=current, no_progress=no_progress) + component_outcomes = _raw_replay_plan_outcomes( + archive_root, index_db, [plan], remaining=current, no_progress=no_progress + ) for outcome in component_outcomes: - application_receipt = raw_replay_application_receipt(archive_root, plan) + application_receipt = raw_replay_application_receipt(archive_root, plan, index_db_path=index_db) receipted = dataclasses.replace(outcome, application_receipt=application_receipt) if outcome.status is RawReplayPlanStatus.EXECUTED: receipt_valid, receipt_problems = validate_raw_replay_application_receipt(plan, application_receipt) @@ -7047,7 +7224,10 @@ def _pass_deadline_exceeded() -> bool: ) plan_outcomes = tuple(execution_outcomes) + blocked_plan_outcomes post_plans, post_residual = _raw_authority_postflight_snapshot( - archive_root, remaining, max_payload_bytes=max_payload_bytes + archive_root, + remaining, + max_payload_bytes=max_payload_bytes, + index_db_path=index_db, ) census_receipt = finalize_raw_authority_census( archive_root, @@ -7321,6 +7501,7 @@ def run_selected_maintenance( "preview_superseded_raw_snapshots", "preview_message_type_backfill", "preview_session_insights", + "raw_materialization_lease_refusal_result", "raw_materialization_replay_backlog", "raw_materialization_scale_profile", "repair_empty_sessions", diff --git a/polylogue/storage/sqlite/archive_tiers/archive.py b/polylogue/storage/sqlite/archive_tiers/archive.py index 3841333e58..7180bb4914 100644 --- a/polylogue/storage/sqlite/archive_tiers/archive.py +++ b/polylogue/storage/sqlite/archive_tiers/archive.py @@ -216,6 +216,8 @@ raw_revision_descriptor, raw_revision_head_raw_id, raw_revision_material, + raw_revision_observation_order, + raw_revision_observed_at_ms, raw_revision_rebuild_selection, raw_revision_replay_adoptable, raw_revision_replay_plan, @@ -240,6 +242,7 @@ deterministic_blob_hash, deterministic_raw_session_id, list_hook_events, + record_raw_container_coordinate, write_source_hook_event, ) from polylogue.storage.sqlite.archive_tiers.types import ( @@ -1779,15 +1782,16 @@ def __init__( self._active_writer_lease = ActiveWriterLease(archive_root) self._active_writer_lease.acquire() - try: - assert_writable_archive_identity( - configured_root=configured_archive_root(), - active_root=archive_root, - ) - except Exception: - self._active_writer_lease.close() - self._active_writer_lease = None - raise + if not source_tier_acquisition: + try: + assert_writable_archive_identity( + configured_root=configured_archive_root(), + active_root=archive_root, + ) + except Exception: + self._active_writer_lease.close() + self._active_writer_lease = None + raise else: from polylogue.storage.index_generation import IndexGeneration, IndexGenerationStore @@ -1867,7 +1871,6 @@ def _initialize_store( ) -> None: self.archive_root = archive_root self.source_db_path = archive_root / "source.db" - self.index_db_path = self._frozen_index_path or archive_root / "index.db" self.embeddings_db_path = archive_root / "embeddings.db" self.user_db_path = archive_root / "user.db" self.ops_db_path = archive_root / "ops.db" @@ -1904,6 +1907,16 @@ def _initialize_store( self._tags_relation = "session_tags" self._blob_publisher = ArchiveBlobPublisher(self.source_db_path, self.archive_root / "blob") return + if self._frozen_index_path is not None: + self.index_db_path = self._frozen_index_path + else: + # The configured root owns the durable tiers, while an active + # generation can keep index.db elsewhere. A writable open must + # follow the same pointer as readiness and live ingest instead of + # silently mutating a stale conventional root/index.db shadow. + from polylogue.storage.archive_identity import resolve_active_index_path + + self.index_db_path = resolve_active_index_path(archive_root) if self._frozen_source_validation: # Candidate admission derives every decision from source.db and # frozen blob bytes. Requiring an index handle here would make the @@ -2111,6 +2124,10 @@ def commit(self) -> None: publication receipts; bulk cadence applies to the derived index. """ self._require_writable("commit archive writes") + if self._source_tier_acquisition: + if self._source_conn is not None: + self._source_conn.commit() + return self._conn.commit() self._consume_index_blob_receipts() self._flush_pending_raw_parse_states() @@ -2123,6 +2140,10 @@ def rollback(self) -> None: Used by a bulk caller to discard an uncommitted, half-applied batch when a write raises, before propagating the error. """ + if self._source_tier_acquisition: + if self._source_conn is not None: + self._source_conn.rollback() + return self._conn.rollback() self._pending_index_blob_receipts.clear() self._pending_raw_parse_states.clear() @@ -2441,6 +2462,23 @@ def write_raw_blob_ref( post_parse=post_parse, ) + def record_raw_container_coordinate( + self, + raw_id: str, + *, + coordinate_format: Literal["zip-v2"], + entry_ordinal: int, + split_index: int, + ) -> None: + self._require_writable("record source.db container coordinate") + record_raw_container_coordinate( + self._ensure_source_conn(), + raw_id, + coordinate_format=coordinate_format, + entry_ordinal=entry_ordinal, + split_index=split_index, + ) + def admit_raw_artifact_payload( self, *, @@ -2681,7 +2719,9 @@ def raw_revision_rebuild_selection( ) -> tuple[tuple[tuple[str, int], ...], tuple[str, ...]]: return raw_revision_rebuild_selection(self, raw_ids) - def raw_membership_census_rows(self, raw_ids: Sequence[str] | None = None) -> tuple[tuple[str, int, bool], ...]: + def raw_membership_census_rows( + self, raw_ids: Sequence[str] | None = None + ) -> tuple[tuple[str, int, bool, int], ...]: return raw_membership_census_rows(self, raw_ids) def raw_payload_sizes(self, raw_ids: Sequence[str]) -> dict[str, int]: @@ -2744,6 +2784,12 @@ def raw_membership_raw_ids( def raw_revision_acquired_at_ms(self, raw_id: str) -> int: return raw_revision_acquired_at_ms(self, raw_id) + def raw_revision_observed_at_ms(self, raw_id: str) -> int: + return raw_revision_observed_at_ms(self, raw_id) + + def raw_revision_observation_order(self, raw_id: str) -> tuple[int, int]: + return raw_revision_observation_order(self, raw_id) + def raw_membership_rebuild_raw_ids(self, logical_source_key: str) -> tuple[str, ...]: return raw_membership_rebuild_raw_ids(self, logical_source_key) diff --git a/polylogue/storage/sqlite/archive_tiers/bootstrap.py b/polylogue/storage/sqlite/archive_tiers/bootstrap.py index e124c0ff10..d736cdb9d7 100644 --- a/polylogue/storage/sqlite/archive_tiers/bootstrap.py +++ b/polylogue/storage/sqlite/archive_tiers/bootstrap.py @@ -4,6 +4,7 @@ import os import sqlite3 +import threading from dataclasses import dataclass from pathlib import Path from typing import Literal @@ -76,6 +77,13 @@ def ddl(self) -> str: } +# ``OwnedArchiveLocation`` protects against other processes. Its deliberate +# reentrancy permits nested production opens in one process, so bootstrap also +# needs this process-local serialization around the fresh durable receipt +# protocol. +_ACTIVE_ARCHIVE_BOOTSTRAP_LOCK = threading.RLock() + + def archive_tier_spec(tier: ArchiveTier) -> ArchiveTierSpec: """Return the database-file spec for one durability tier.""" return ARCHIVE_TIER_SPECS[tier] @@ -310,7 +318,7 @@ def initialize_archive_database( conn.close() -def initialize_active_archive_root(root: Path) -> None: +def _initialize_active_archive_root(root: Path) -> None: """Create or initialize every tier database in an archive root.""" from polylogue.operations.durable_change_train import audit_adoption_receipt_path, recover_pending_audit_adoption from polylogue.storage.archive_identity import ( @@ -429,9 +437,11 @@ def classify_paths() -> tuple[bool, bool]: if not recovering_fresh_durable_bootstrap and not pre_marker_adoption: assert_owned_root() reconcile_durable_change_trains_on_startup(root) + location = ArchiveLocation.resolve(root) for spec in ARCHIVE_TIER_SPECS.values(): assert_owned_root() - initialize_archive_database(root / spec.filename, spec.tier) + tier_path = location.active_index_path if spec.tier is ArchiveTier.INDEX else root / spec.filename + initialize_archive_database(tier_path, spec.tier) # Mutation composition performs source/audit reconciliation immediately # before it consumes authority. Ordinary archive opens stay read-only # with respect to continuity, including their steady-state path. @@ -453,6 +463,13 @@ def classify_paths() -> tuple[bool, bool]: pending_bootstrap_path.unlink(missing_ok=True) +def initialize_active_archive_root(root: Path) -> None: + """Create or initialize every active archive tier under one local bootstrap owner.""" + + with _ACTIVE_ARCHIVE_BOOTSTRAP_LOCK: + _initialize_active_archive_root(root) + + def reconcile_durable_change_trains_on_startup(root: Path) -> tuple[Path, ...]: """Reconcile persisted durable trains without executing migration SQL.""" from polylogue.storage.sqlite.durable_change_train import reconcile_durable_change_train_startup diff --git a/polylogue/storage/sqlite/archive_tiers/revision_governance.py b/polylogue/storage/sqlite/archive_tiers/revision_governance.py index ba32d52a0d..40df0a0888 100644 --- a/polylogue/storage/sqlite/archive_tiers/revision_governance.py +++ b/polylogue/storage/sqlite/archive_tiers/revision_governance.py @@ -1516,7 +1516,7 @@ def raw_revision_descriptor( store._ensure_source_conn() .execute( """ - SELECT origin, capture_mode, lower(hex(blob_hash)), source_path, revision_kind, blob_size + SELECT origin, detected_provider, capture_mode, lower(hex(blob_hash)), source_path, revision_kind, blob_size FROM raw_sessions WHERE raw_id = ? """, (raw_id,), @@ -1526,11 +1526,15 @@ def raw_revision_descriptor( if row is None: raise KeyError(raw_id) return ( - provider_from_origin(Origin.from_string(str(row[0])), family_hint=row[1]), - str(row[2]), + ( + Provider.from_string(str(row[1])) + if row[1] is not None + else provider_from_origin(Origin.from_string(str(row[0])), family_hint=row[2]) + ), str(row[3]), - RawRevisionKind(str(row[4])), - int(row[5]), + str(row[4]), + RawRevisionKind(str(row[5])), + int(row[6]), ) @@ -1709,25 +1713,38 @@ def raw_revision_rebuild_selection( def raw_membership_census_rows( store: RawRevisionGovernanceHost, raw_ids: Sequence[str] | None = None -) -> tuple[tuple[str, int, bool], ...]: +) -> tuple[tuple[str, int, bool, int], ...]: """Return retained raws and whether durable evidence says they are non-sessions.""" conn = store._ensure_source_conn() columns = """ r.raw_id, r.source_index, - EXISTS(SELECT 1 FROM raw_artifacts AS a WHERE a.raw_id = r.raw_id AND a.parse_as_session = 0) + ( + EXISTS(SELECT 1 FROM raw_artifacts AS a WHERE a.raw_id = r.raw_id AND a.parse_as_session = 0) + OR EXISTS( + SELECT 1 FROM raw_membership_census AS c + WHERE c.raw_id = r.raw_id + AND c.parser_fingerprint = ? + AND c.status = 'non_session' + AND r.parsed_at_ms IS NOT NULL + AND r.parse_error IS NULL + ) + ), + r.rowid """ if raw_ids is None: - rows = conn.execute(f"SELECT {columns} FROM raw_sessions AS r ORDER BY r.raw_id").fetchall() + rows = conn.execute( + f"SELECT {columns} FROM raw_sessions AS r ORDER BY r.raw_id", (RAW_AUTHORITY_PARSER_FINGERPRINT,) + ).fetchall() elif raw_ids: placeholders = ",".join("?" for _ in raw_ids) rows = conn.execute( f"SELECT {columns} FROM raw_sessions AS r WHERE r.raw_id IN ({placeholders}) ORDER BY r.raw_id", - tuple(raw_ids), + (RAW_AUTHORITY_PARSER_FINGERPRINT, *raw_ids), ).fetchall() else: rows = [] - return tuple((str(row[0]), int(row[1]), bool(row[2])) for row in rows) + return tuple((str(row[0]), int(row[1]), bool(row[2]), int(row[3])) for row in rows) def raw_payload_sizes(store: RawRevisionGovernanceHost, raw_ids: Sequence[str]) -> dict[str, int]: @@ -1767,8 +1784,6 @@ def replace_raw_membership_census( ).fetchone() if revision is None: raise RuntimeError(f"membership census raw is missing: {raw_id}") - if revision[0] is not None and str(revision[1]) != RawRevisionKind.FULL.value: - raise RuntimeError("only self-contained full raws can move to membership governance") dependent = conn.execute( """ SELECT 1 FROM raw_sessions @@ -2192,6 +2207,38 @@ def raw_revision_acquired_at_ms(store: RawRevisionGovernanceHost, raw_id: str) - return int(row[0]) +def raw_revision_observed_at_ms(store: RawRevisionGovernanceHost, raw_id: str) -> int: + """Return the latest durable observation receipt for a retained raw. + + ``raw_sessions.acquired_at_ms`` is deliberately immutable because the raw + id is content-derived. Re-observing identical bytes refreshes the + ``blob_refs`` raw-payload receipt instead, which is the ordering authority + for replaying mutable state snapshots. + """ + return raw_revision_observation_order(store, raw_id)[0] + + +def raw_revision_observation_order(store: RawRevisionGovernanceHost, raw_id: str) -> tuple[int, int]: + """Return the latest observation timestamp and its durable receipt order.""" + conn = store._ensure_source_conn() + row = conn.execute( + """ + SELECT acquired_at_ms, rowid + FROM blob_refs + WHERE ref_id = ? AND ref_type = 'raw_payload' + ORDER BY acquired_at_ms DESC, rowid DESC + LIMIT 1 + """, + (raw_id,), + ).fetchone() + if row is not None: + return int(row[0]), int(row[1]) + row = conn.execute("SELECT acquired_at_ms, rowid FROM raw_sessions WHERE raw_id = ?", (raw_id,)).fetchone() + if row is None: + raise KeyError(f"unknown raw revision {raw_id}") + return int(row[0]), int(row[1]) + + def raw_membership_rebuild_raw_ids(store: RawRevisionGovernanceHost, logical_source_key: str) -> tuple[str, ...]: """Return census candidates excluding quarantined full rows with another authority key.""" rows = ( diff --git a/polylogue/storage/sqlite/archive_tiers/source.py b/polylogue/storage/sqlite/archive_tiers/source.py index 344f567149..dccde7e983 100644 --- a/polylogue/storage/sqlite/archive_tiers/source.py +++ b/polylogue/storage/sqlite/archive_tiers/source.py @@ -22,7 +22,7 @@ from polylogue.storage.sqlite.archive_tiers.types import ProvenRevisionAuthority from polylogue.storage.sqlite.audit_continuity import AUDIT_CONTINUITY_GENESIS_HEAD_SHA256 -SOURCE_SCHEMA_VERSION = 32 +SOURCE_SCHEMA_VERSION = 33 SOURCE_DDL = f""" CREATE TABLE IF NOT EXISTS raw_sessions ( @@ -58,6 +58,14 @@ CHECK ({check("revision_authority", RawRevisionAuthority)}) ,revision_authority_evidence TEXT CHECK(revision_authority_evidence IS NULL OR revision_authority_evidence IN ('live_source_verification_v1')) + ,detected_provider TEXT CHECK ({nullable_check("detected_provider", Provider)}) +) STRICT; + +CREATE TABLE IF NOT EXISTS raw_container_coordinates ( + raw_id TEXT PRIMARY KEY REFERENCES raw_sessions(raw_id) ON DELETE CASCADE, + coordinate_format TEXT NOT NULL CHECK(coordinate_format = 'zip-v2'), + entry_ordinal INTEGER NOT NULL CHECK(entry_ordinal >= 0), + split_index INTEGER NOT NULL CHECK(split_index >= 0) ) STRICT; CREATE INDEX IF NOT EXISTS idx_raw_sessions_origin diff --git a/polylogue/storage/sqlite/archive_tiers/source_write.py b/polylogue/storage/sqlite/archive_tiers/source_write.py index d5886488ae..6383596040 100644 --- a/polylogue/storage/sqlite/archive_tiers/source_write.py +++ b/polylogue/storage/sqlite/archive_tiers/source_write.py @@ -280,6 +280,40 @@ def record_capture_mode_observation( ) +def record_raw_container_coordinate( + conn: sqlite3.Connection, + raw_id: str, + *, + coordinate_format: Literal["zip-v2"], + entry_ordinal: int, + split_index: int, + manage_transaction: bool = True, +) -> None: + """Persist one content-independent container coordinate for a raw row.""" + if entry_ordinal < 0 or split_index < 0: + raise ValueError("container entry ordinal and split index must be non-negative") + with conn if manage_transaction else nullcontext(): + conn.execute( + """ + INSERT OR IGNORE INTO raw_container_coordinates ( + raw_id, coordinate_format, entry_ordinal, split_index + ) VALUES (?, ?, ?, ?) + """, + (raw_id, coordinate_format, entry_ordinal, split_index), + ) + stored = conn.execute( + """ + SELECT coordinate_format, entry_ordinal, split_index + FROM raw_container_coordinates + WHERE raw_id = ? + """, + (raw_id,), + ).fetchone() + expected = (coordinate_format, entry_ordinal, split_index) + if stored is None or tuple(stored) != expected: + raise ValueError(f"raw container coordinate changed for {raw_id}") + + def read_capture_mode_resolution(conn: sqlite3.Connection, raw_id: str) -> CaptureModeResolution: """Read every acquisition mode ever observed for ``raw_id``, explicitly ambiguous or not. @@ -1273,9 +1307,9 @@ def upsert_raw_artifact( """ failure_kind = _is_raw_failure_artifact_kind(artifact.artifact_kind) coordinate_predicate = ( - "raw_id = ? AND origin = ? AND source_path = ? AND source_index = ?" + "a.raw_id = ? AND a.origin = ? AND a.source_path = ? AND a.source_index = ?" if failure_kind - else "origin = ? AND source_path = ? AND source_index = ? AND artifact_kind NOT IN (" + else "a.origin = ? AND a.source_path = ? AND a.source_index = ? AND a.artifact_kind NOT IN (" + ", ".join("?" for _ in RAW_FAILURE_EVIDENCE_KINDS) + ")" ) @@ -1292,13 +1326,60 @@ def upsert_raw_artifact( with conn if manage_transaction else nullcontext(): existing = conn.execute( f""" - SELECT artifact_id - FROM raw_artifacts + SELECT a.artifact_id, a.raw_id + FROM raw_artifacts AS a WHERE {coordinate_predicate} """, coordinate_params, ).fetchone() if existing is not None: + # One coordinate has one authority carrier. A delayed census of + # stale retained bytes must not replace a carrier observed later. + if str(existing[1]) != raw_id: + incoming_receipt = conn.execute( + """ + SELECT acquired_at_ms, rowid FROM blob_refs + WHERE ref_id = ? AND ref_type = 'raw_payload' + ORDER BY acquired_at_ms DESC, rowid DESC LIMIT 1 + """, + (raw_id,), + ).fetchone() + existing_receipt = conn.execute( + """ + SELECT acquired_at_ms, rowid FROM blob_refs + WHERE ref_id = ? AND ref_type = 'raw_payload' + ORDER BY acquired_at_ms DESC, rowid DESC LIMIT 1 + """, + (str(existing[1]),), + ).fetchone() + if (incoming_receipt is None) != (existing_receipt is None): + raise RuntimeError( + "cannot compare artifact observation order across incompatible raw-payload receipt coverage" + ) + if incoming_receipt is None: + incoming_observation = conn.execute( + "SELECT acquired_at_ms, rowid FROM raw_sessions WHERE raw_id = ?", + (raw_id,), + ).fetchone() + existing_observation = conn.execute( + "SELECT acquired_at_ms, rowid FROM raw_sessions WHERE raw_id = ?", + (str(existing[1]),), + ).fetchone() + else: + incoming_observation = incoming_receipt + existing_observation = existing_receipt + if incoming_observation is None: + raise KeyError(raw_id) + if existing_observation is None: + raise KeyError(str(existing[1])) + existing_order = (int(existing_observation[0]), int(existing_observation[1])) + incoming_order = (int(incoming_observation[0]), int(incoming_observation[1])) + if existing_order >= incoming_order: + conn.execute( + "UPDATE raw_artifacts SET first_observed_at_ms = MIN(first_observed_at_ms, ?) WHERE artifact_id = ?", + (artifact.first_observed_at_ms, str(existing[0])), + ) + return artifact = replace(artifact, artifact_id=str(existing[0])) _insert_artifact(conn, raw_id, artifact) @@ -1416,6 +1497,7 @@ def _enum_value(value: object) -> str | None: "read_raw_artifact", "read_archive_raw_session_envelope", "record_capture_mode_observation", + "record_raw_container_coordinate", "record_excised_blob_hash", "pending_raw_logical_source_key", "upsert_raw_artifact", diff --git a/polylogue/storage/sqlite/migrations/source/033.train.json b/polylogue/storage/sqlite/migrations/source/033.train.json new file mode 100644 index 0000000000..49e77188f6 --- /dev/null +++ b/polylogue/storage/sqlite/migrations/source/033.train.json @@ -0,0 +1,97 @@ +{ + "manifest_format": "polylogue.durable-change-train.v1", + "train_id": "train:source:v33", + "tier": "source", + "current_version": 32, + "target_version": 33, + "slot": 33, + "owner_ref": "github:pull/3952#discussion_r3775839929", + "migration": { + "tier": "source", + "target_version": 33, + "slot": 33, + "path": "033_detected_raw_provider.sql", + "owner_ref": "polylogue/storage/sqlite/migrations/source/033_detected_raw_provider.sql", + "sql_sha256": "8823c3de62eed0e93c7254222e24577442e146d6a76f8f1c810316b521ae6a81", + "requires_backup": true + }, + "riders": [ + { + "rider_id": "rider:detected-raw-provider", + "owner_ref": "github:pull/3952#discussion_r3775839929", + "schema_objects": ["column:raw_sessions.detected_provider"], + "runtime_consumers": [ + { + "consumer_id": "raw-state-update", + "production_ref": "polylogue.storage.sqlite.raw_state_update:compile_raw_state_update", + "behavior_proof_ref": "proof:source-v33:preserve-acquisition-origin", + "roles": ["write"] + }, + { + "consumer_id": "revision-provider-resolution", + "production_ref": "polylogue.storage.sqlite.archive_tiers.revision_governance:raw_revision_descriptor", + "behavior_proof_ref": "proof:source-v33:reuse-detected-provider", + "roles": ["read"] + }, + { + "consumer_id": "raw-record-hydration", + "production_ref": "polylogue.storage.sqlite.queries.mappers_archive:_row_to_raw_session", + "behavior_proof_ref": "proof:source-v33:split-acquisition-and-parser-identity", + "roles": ["read"] + } + ], + "behavior_proof_refs": [ + "proof:source-v33:preserve-acquisition-origin", + "proof:source-v33:reuse-detected-provider", + "proof:source-v33:split-acquisition-and-parser-identity" + ], + "after_rider_ids": [], + "trust_floor_exception_ref": null + }, + { + "rider_id": "rider:raw-container-coordinate", + "owner_ref": "github:pull/3952#discussion_r3779604611", + "schema_objects": ["table:raw_container_coordinates"], + "runtime_consumers": [ + { + "consumer_id": "live-zip-coordinate-write", + "production_ref": "polylogue.sources.live.batch:_record_zip_container_coordinate", + "behavior_proof_ref": "proof:source-v33:persist-zip-coordinate", + "roles": ["write"] + }, + { + "consumer_id": "raw-blob-source-replacement", + "production_ref": "polylogue.storage.blob_integrity:replace_raw_backed_blob_reference_debt_from_source", + "behavior_proof_ref": "proof:source-v33:reuse-zip-coordinate-after-replacement", + "roles": ["read", "write"] + } + ], + "behavior_proof_refs": [ + "proof:source-v33:persist-zip-coordinate", + "proof:source-v33:reuse-zip-coordinate-after-replacement" + ], + "after_rider_ids": [], + "trust_floor_exception_ref": null + } + ], + "ordering_constraints": [], + "drop_constraints": [], + "row_change_allowances": [], + "backup_plan_ref": "backup-profile:source-tier", + "state": "declared", + "revision": 0, + "declared_at_ms": 0, + "admitted_at_ms": null, + "admission_evidence_ref": null, + "fresh_ddl_parity": null, + "reservation": null, + "backup_authorization": null, + "pre_apply_evidence": null, + "apply_evidence": null, + "proof": null, + "failure": null, + "released_at_ms": null, + "release_evidence_ref": null, + "proof_refs": [], + "manifest_sha256": "7d5d512f5932e6d0a49bdaea48d7a54052116c180d0f24c4dfcbc84244f3381a" +} diff --git a/polylogue/storage/sqlite/migrations/source/033_detected_raw_provider.sql b/polylogue/storage/sqlite/migrations/source/033_detected_raw_provider.sql new file mode 100644 index 0000000000..8c892914f8 --- /dev/null +++ b/polylogue/storage/sqlite/migrations/source/033_detected_raw_provider.sql @@ -0,0 +1,21 @@ +-- Parser classification is not acquisition identity. Retain the exact +-- provider-wire result separately so replay can reuse it without rewriting +-- raw_sessions.origin and breaking deterministic reacquisition. +ALTER TABLE raw_sessions ADD COLUMN detected_provider TEXT CHECK ( + (detected_provider IN ( + 'chatgpt', 'claude-ai', 'claude-design', 'claude-code', 'codex', + 'gemini', 'gemini-cli', 'hermes', 'antigravity', 'beads', 'grok', + 'drive', 'unknown' + ) OR detected_provider IS NULL) +); + +-- ZIP member raw ids bind the content hash while source_index stores a paired +-- central-directory ordinal and within-member split index. Source replacement +-- may legitimately change the row's blob hash without changing raw_id, so the +-- coordinate format must remain independently durable for later recovery. +CREATE TABLE raw_container_coordinates ( + raw_id TEXT PRIMARY KEY REFERENCES raw_sessions(raw_id) ON DELETE CASCADE, + coordinate_format TEXT NOT NULL CHECK(coordinate_format = 'zip-v2'), + entry_ordinal INTEGER NOT NULL CHECK(entry_ordinal >= 0), + split_index INTEGER NOT NULL CHECK(split_index >= 0) +) STRICT; diff --git a/polylogue/storage/sqlite/queries/mappers_archive.py b/polylogue/storage/sqlite/queries/mappers_archive.py index 67423a6851..355d732195 100644 --- a/polylogue/storage/sqlite/queries/mappers_archive.py +++ b/polylogue/storage/sqlite/queries/mappers_archive.py @@ -180,14 +180,15 @@ def _row_to_raw_session(row: sqlite3.Row) -> RawSessionRecord: validation_mode = _row_text(row, "validation_mode") blob_hash_value = _row_get(row, "blob_hash") blob_hash = bytes(blob_hash_value).hex() if isinstance(blob_hash_value, (bytes, bytearray)) else None - # raw_sessions carries a single ``origin`` column (#1743). The in-memory - # record still exposes provider-wire ``source_name``/``payload_provider``; - # both project from the stored origin. + # Acquisition origin is immutable raw identity. A later parser may retain + # its exact provider separately without rewriting that identity. capture_mode = _row_text(row, "capture_mode") - provider = provider_from_origin( + detected_provider = _row_text(row, "detected_provider") + acquisition_provider = provider_from_origin( Origin.from_string(row["origin"]), family_hint=capture_mode, ) + provider = Provider.from_string(detected_provider) if detected_provider is not None else acquisition_provider logical_source_key = _row_text(row, "logical_source_key") source_revision = _row_text(row, "source_revision") generation = _row_int(row, "acquisition_generation") @@ -210,7 +211,7 @@ def _row_to_raw_session(row: sqlite3.Row) -> RawSessionRecord: blob_hash=blob_hash, payload_provider=provider, capture_mode=Provider.from_string(capture_mode) if capture_mode is not None else None, - source_name=provider.value, + source_name=acquisition_provider.value, source_path=row["source_path"], source_index=row["source_index"], blob_size=row["blob_size"], diff --git a/polylogue/storage/sqlite/queries/raw_reads.py b/polylogue/storage/sqlite/queries/raw_reads.py index 02242b643d..596197780a 100644 --- a/polylogue/storage/sqlite/queries/raw_reads.py +++ b/polylogue/storage/sqlite/queries/raw_reads.py @@ -243,6 +243,7 @@ async def get_raw_session_states(conn: aiosqlite.Connection, raw_ids: list[str]) SELECT raw_id, origin, + detected_provider, capture_mode, source_path, parsed_at_ms, @@ -256,7 +257,12 @@ async def get_raw_session_states(conn: aiosqlite.Connection, raw_ids: list[str]) rows = await cursor.fetchall() def _state(row: aiosqlite.Row) -> RawSessionState: - provider = provider_from_origin(Origin.from_string(row["origin"]), family_hint=row["capture_mode"]) + detected_provider = row["detected_provider"] + provider = ( + Provider.from_string(str(detected_provider)) + if detected_provider is not None + else provider_from_origin(Origin.from_string(row["origin"]), family_hint=row["capture_mode"]) + ) return RawSessionState( raw_id=row["raw_id"], source_name=provider.value, diff --git a/polylogue/storage/sqlite/queries/raw_state.py b/polylogue/storage/sqlite/queries/raw_state.py index 93f707ee69..ffc1c2b2bc 100644 --- a/polylogue/storage/sqlite/queries/raw_state.py +++ b/polylogue/storage/sqlite/queries/raw_state.py @@ -12,9 +12,39 @@ from polylogue.storage.sqlite.connection import _build_source_scope_filter from polylogue.storage.sqlite.raw_state_update import compile_raw_state_update -# raw_sessions carries a single ``origin`` column (#1743). Provider-token -# filters translate the token to its canonical origin value before matching. -RAW_ORIGIN_FILTER_SQL = "origin" + +def raw_provider_origin_sql(*, table_alias: str | None = None) -> str: + """Project parser-classified provider evidence into Origin vocabulary. + + Acquisition ``origin`` remains immutable raw identity. Provider-scoped + readers use this expression so a later positive parser classification is + visible without rewriting that identity. ``table_alias`` keeps the same + contract usable in joined repair and sampling queries. + """ + prefix = f"{table_alias}." if table_alias else "" + detected = f"{prefix}detected_provider" + origin = f"{prefix}origin" + return f""" +CASE {detected} + WHEN 'chatgpt' THEN 'chatgpt-export' + WHEN 'claude-ai' THEN 'claude-ai-export' + WHEN 'claude-design' THEN 'claude-design-session' + WHEN 'claude-code' THEN 'claude-code-session' + WHEN 'codex' THEN 'codex-session' + WHEN 'gemini' THEN 'aistudio-drive' + WHEN 'drive' THEN 'aistudio-drive' + WHEN 'gemini-cli' THEN 'gemini-cli-session' + WHEN 'hermes' THEN 'hermes-session' + WHEN 'antigravity' THEN 'antigravity-session' + WHEN 'beads' THEN 'beads-issue' + WHEN 'grok' THEN 'grok-export' + WHEN 'unknown' THEN 'unknown-export' + ELSE {origin} +END +""".strip() + + +RAW_ORIGIN_FILTER_SQL = raw_provider_origin_sql() def origin_filter_value(token: str) -> str: @@ -217,6 +247,7 @@ async def reset_validation_status( "coerce_provider", "coerce_status", "origin_filter_value", + "raw_provider_origin_sql", "mark_raw_parsed", "mark_raw_validated", "reset_parse_status", diff --git a/polylogue/storage/sqlite/queries/raw_writes.py b/polylogue/storage/sqlite/queries/raw_writes.py index 2025a31f0d..b685eace3d 100644 --- a/polylogue/storage/sqlite/queries/raw_writes.py +++ b/polylogue/storage/sqlite/queries/raw_writes.py @@ -17,12 +17,9 @@ async def save_raw_session( record: RawSessionRecord, transaction_depth: int, ) -> bool: - # payload_provider wins when the payload has been classified; otherwise fall - # back to the source_name token (#1743 collapses both onto origin). - if record.payload_provider is not None: - origin = origin_from_provider(record.payload_provider) - else: - origin = origin_from_provider(Provider.from_string(record.source_name or "unknown")) + acquisition_provider = Provider.from_string(record.source_name or "unknown") + origin = origin_from_provider(acquisition_provider) + detected_provider = record.payload_provider # Only the acquisition path can assert a capture mode. A hydrated legacy # row has ``None`` here even though its compatibility projection supplies # a canonical payload provider; writing that projection back must not turn @@ -40,17 +37,18 @@ async def save_raw_session( cursor = await conn.execute( """ INSERT OR IGNORE INTO raw_sessions ( - raw_id, origin, capture_mode, native_id, source_path, source_index, blob_hash, + raw_id, origin, detected_provider, capture_mode, native_id, source_path, source_index, blob_hash, blob_size, acquired_at_ms, file_mtime_ms, parsed_at_ms, parse_error, validated_at_ms, validation_status, validation_error, validation_drift_count, validation_mode, detection_warnings_json, logical_source_key, revision_kind, source_revision, predecessor_source_revision, predecessor_raw_id, baseline_raw_id, append_start_offset, append_end_offset, acquisition_generation, revision_authority, revision_authority_evidence - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( record.raw_id, origin.value, + detected_provider.value if detected_provider is not None else None, capture_mode.value if capture_mode is not None else None, None, record.source_path, diff --git a/polylogue/storage/sqlite/raw_state_update.py b/polylogue/storage/sqlite/raw_state_update.py index c145503b4f..de858d7031 100644 --- a/polylogue/storage/sqlite/raw_state_update.py +++ b/polylogue/storage/sqlite/raw_state_update.py @@ -5,7 +5,6 @@ import json from polylogue.core.enums import Provider, ValidationMode, ValidationStatus -from polylogue.core.sources import origin_from_provider from polylogue.storage.raw.models import UNSET, RawSessionStateUpdate, _RawStateUnset from polylogue.storage.sqlite.archive_tiers.write import _timestamp_ms @@ -18,9 +17,27 @@ def compile_raw_state_update( """Compile one typed mutation for either SQLite connection adapter.""" set_clauses: list[str] = [] params: list[object] = [] + parsed_at_ms = _timestamp_ms(state.parsed_at) if isinstance(state.parsed_at, str) else None + validation_transition = state.validation_status is not UNSET or state.validation_error is not UNSET if state.parsed_at is not UNSET: - set_clauses.append("parsed_at_ms = ?") - params.append(_timestamp_ms(state.parsed_at) if isinstance(state.parsed_at, str) else None) + if parsed_at_ms is None: + if isinstance(state.parsed_at, str): + raise ValueError(f"parsed_at must be a valid timestamp, got {state.parsed_at!r}") + set_clauses.append("parsed_at_ms = ?") + params.append(None) + elif validation_transition: + # SQLite evaluates every SET expression from the old row. A + # combined update records validation first and parse second, so + # advance parse by two from either old transition (and one from + # this validation clock) to preserve that authority ordering even + # when wall time is equal or moves backward. + set_clauses.append( + "parsed_at_ms = MAX(?, ? + 1, COALESCE(parsed_at_ms + 2, ?), COALESCE(validated_at_ms + 2, ?))" + ) + params.extend((parsed_at_ms, now_ms, parsed_at_ms, parsed_at_ms)) + else: + set_clauses.append("parsed_at_ms = MAX(?, COALESCE(parsed_at_ms + 1, ?), COALESCE(validated_at_ms + 1, ?))") + params.extend((parsed_at_ms, parsed_at_ms, parsed_at_ms)) if state.parse_error is not UNSET: set_clauses.append("parse_error = ?") params.append(state.parse_error[:2000] if isinstance(state.parse_error, str) else state.parse_error) @@ -47,15 +64,15 @@ def compile_raw_state_update( provider = state.payload_provider elif isinstance(state.validation_provider, Provider): provider = state.validation_provider - set_clauses.append("origin = COALESCE(?, origin)") - params.append(origin_from_provider(provider).value if provider is not None else None) + set_clauses.append("detected_provider = COALESCE(?, detected_provider)") + params.append(provider.value if provider is not None else None) if state.detection_warnings is not UNSET: warnings = state.detection_warnings set_clauses.append("detection_warnings_json = ?") params.append(json.dumps([warnings[:2000]]) if isinstance(warnings, str) and warnings else "[]") - if state.validation_status is not UNSET or state.validation_error is not UNSET: - set_clauses.append("validated_at_ms = ?") - params.append(now_ms) + if validation_transition: + set_clauses.append("validated_at_ms = MAX(?, COALESCE(validated_at_ms + 1, ?), COALESCE(parsed_at_ms + 1, ?))") + params.extend((now_ms, now_ms, now_ms)) return tuple(set_clauses), tuple(params) diff --git a/tests/unit/browser_capture/test_receiver.py b/tests/unit/browser_capture/test_receiver.py index f2f6aaadf8..e228ee7ec4 100644 --- a/tests/unit/browser_capture/test_receiver.py +++ b/tests/unit/browser_capture/test_receiver.py @@ -100,6 +100,10 @@ def _seed_browser_capture_archive( raw_id: str = "raw-capture", message_count: int = 1, parse_error: str | None = None, + validation_status: str | None = None, + validation_error: str | None = None, + parsed_at_ms: int | None = None, + validated_at_ms: int | None = None, updated_at_ms: int | None = None, ) -> None: with sqlite3.connect(archive_root / "source.db") as conn: @@ -110,16 +114,31 @@ def _seed_browser_capture_archive( origin TEXT, native_id TEXT, source_path TEXT, - parse_error TEXT + parse_error TEXT, + validation_status TEXT, + validation_error TEXT, + parsed_at_ms INTEGER, + validated_at_ms INTEGER ) """ ) conn.execute( """ - INSERT INTO raw_sessions (raw_id, origin, native_id, source_path, parse_error) - VALUES (?, ?, ?, ?, ?) + INSERT INTO raw_sessions ( + raw_id, origin, native_id, source_path, parse_error, validation_status, validation_error, parsed_at_ms, validated_at_ms + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) """, - (raw_id, "chatgpt-export", native_id, f"browser-capture/chatgpt/{native_id}.json", parse_error), + ( + raw_id, + "chatgpt-export", + native_id, + f"browser-capture/chatgpt/{native_id}.json", + parse_error, + validation_status, + validation_error, + parsed_at_ms, + validated_at_ms, + ), ) with sqlite3.connect(archive_root / "index.db") as conn: conn.execute( @@ -684,6 +703,19 @@ def test_receiver_archive_state_reports_missing_without_spool_or_archive(tmp_pat assert Path(state.artifact_ref).is_absolute() is False +def test_receiver_archive_state_tolerates_invalid_active_index_pointer(tmp_path: Path) -> None: + envelope = BrowserCaptureEnvelope.model_validate(_payload()) + write_capture_envelope(envelope, spool_path=tmp_path) + (tmp_path / ".index-active-pointer").write_text("not-an-index.db\n", encoding="utf-8") + + state = BrowserCaptureArchiveStatePayload.model_validate( + existing_capture_state("chatgpt", "conv-123", spool_path=tmp_path, archive_root=tmp_path) + ) + + assert state.state == "spooled_only" + assert state.indexed_session_exists is False + + def test_receiver_archive_state_requires_indexed_messages(tmp_path: Path) -> None: envelope = BrowserCaptureEnvelope.model_validate(_payload()) write_capture_envelope(envelope, spool_path=tmp_path) @@ -763,6 +795,75 @@ def test_receiver_archive_state_surfaces_raw_failure(tmp_path: Path) -> None: assert state.failure_source == "raw_parse" +def test_receiver_uses_active_index_and_ignores_historical_validation_failure(tmp_path: Path) -> None: + """The public state reads the promoted generation, not its stale shadow.""" + envelope = BrowserCaptureEnvelope.model_validate(_payload()) + write_capture_envelope(envelope, spool_path=tmp_path) + _seed_browser_capture_archive( + tmp_path, + validation_status="failed", + parsed_at_ms=1, + validated_at_ms=0, + message_count=0, + ) + active_index = tmp_path / "generations" / "active" / "index.db" + active_index.parent.mkdir(parents=True) + with sqlite3.connect(active_index) as conn: + conn.execute( + "CREATE TABLE sessions (session_id TEXT, raw_id TEXT, native_id TEXT, message_count INTEGER, updated_at_ms INTEGER)" + ) + conn.execute("INSERT INTO sessions VALUES ('chatgpt-export:conv-123', 'raw-capture', 'conv-123', 1, NULL)") + (tmp_path / ".index-active-pointer").write_text(f"{active_index}\n", encoding="utf-8") + + state = BrowserCaptureArchiveStatePayload.model_validate( + existing_capture_state("chatgpt", "conv-123", spool_path=tmp_path, archive_root=tmp_path) + ) + + assert state.state == "archived" + assert state.latest_failure is None + assert state.indexed_message_count == 1 + + +def test_receiver_surfaces_validation_failure_newer_than_parse(tmp_path: Path) -> None: + envelope = BrowserCaptureEnvelope.model_validate(_payload()) + write_capture_envelope(envelope, spool_path=tmp_path) + _seed_browser_capture_archive( + tmp_path, + validation_status="failed", + parsed_at_ms=1, + validated_at_ms=2, + validation_error="strict validation rejected current bytes", + ) + + state = BrowserCaptureArchiveStatePayload.model_validate( + existing_capture_state("chatgpt", "conv-123", spool_path=tmp_path, archive_root=tmp_path) + ) + + assert state.state == "failed" + assert state.latest_failure == "strict validation rejected current bytes" + assert state.failure_source == "raw_validation" + + +def test_receiver_surfaces_indeterminate_raw_state_order_without_choosing_validation(tmp_path: Path) -> None: + envelope = BrowserCaptureEnvelope.model_validate(_payload()) + write_capture_envelope(envelope, spool_path=tmp_path) + _seed_browser_capture_archive( + tmp_path, + validation_status="failed", + parsed_at_ms=1, + validated_at_ms=1, + validation_error="equal-time failure", + ) + + state = BrowserCaptureArchiveStatePayload.model_validate( + existing_capture_state("chatgpt", "conv-123", spool_path=tmp_path, archive_root=tmp_path) + ) + + assert state.state == "failed" + assert state.latest_failure == "raw validation and parse timestamps are indeterminate" + assert state.failure_source == "raw_state_order" + + def test_receiver_echoes_safe_request_id_header(tmp_path: Path) -> None: with _running_receiver(tmp_path) as (host, port): conn = HTTPConnection(host, port) diff --git a/tests/unit/core/test_config.py b/tests/unit/core/test_config.py index 161ce26744..6fea7a8081 100644 --- a/tests/unit/core/test_config.py +++ b/tests/unit/core/test_config.py @@ -9,6 +9,7 @@ import sys from io import StringIO from pathlib import Path +from unittest.mock import MagicMock import pytest @@ -47,6 +48,10 @@ def test_config_with_sources(self, tmp_path: Path) -> None: assert config.sources[0].name == "inbox" assert config.sources[1].name == "claude-code" + def test_config_mock_spec_exposes_explicit_database_tracking(self) -> None: + """Consumers cloning Config can inspect its explicit-path contract.""" + assert hasattr(MagicMock(spec=Config), "_db_path_explicit") + def test_config_db_path_default(self, workspace_env: dict[str, Path]) -> None: """db_path defaults to the resolved index.db database path.""" config = Config( @@ -99,6 +104,38 @@ def test_config_db_path_follows_active_generation_pointer(self, tmp_path: Path) assert config.db_path == active_index + def test_with_sources_keeps_implicit_active_generation_tracking(self, tmp_path: Path) -> None: + archive_root = tmp_path / "archive" + archive_root.mkdir() + first = tmp_path / "first" / "index.db" + second = tmp_path / "second" / "index.db" + first.parent.mkdir() + second.parent.mkdir() + first.touch() + second.touch() + pointer = archive_root / ".index-active-pointer" + pointer.write_text(str(first), encoding="utf-8") + + clone = Config(archive_root=archive_root, render_root=tmp_path / "render", sources=[]).with_sources([]) + pointer.write_text(str(second), encoding="utf-8") + + assert clone.current_db_path() == second + + def test_current_db_path_honors_explicit_nonstandard_filename(self, tmp_path: Path) -> None: + archive_root = tmp_path / "archive" + archive_root.mkdir() + active = tmp_path / "generation" / "index.db" + explicit = tmp_path / "selected" / "archive.db" + active.parent.mkdir() + explicit.parent.mkdir() + active.touch() + explicit.touch() + (archive_root / ".index-active-pointer").write_text(str(active), encoding="utf-8") + + config = Config(archive_root=archive_root, render_root=tmp_path / "render", sources=[], db_path=explicit) + + assert config.current_db_path() == explicit + def test_config_db_path_warns_on_stale_conventional_index_shadowing_pointer( self, tmp_path: Path, caplog: pytest.LogCaptureFixture ) -> None: diff --git a/tests/unit/core/test_sampling.py b/tests/unit/core/test_sampling.py index bda1bb458e..2b61480f84 100644 --- a/tests/unit/core/test_sampling.py +++ b/tests/unit/core/test_sampling.py @@ -262,6 +262,112 @@ def test_claude_ai_reads_db_rows_stored_under_claude(self, tmp_path: Path) -> No assert len(result) == 1 assert result[0]["uuid"] == "conv-1" + def test_sampling_keeps_successfully_reparsed_historical_validation_failure(self, tmp_path: Path) -> None: + db = _archive_index_db(tmp_path) + _insert_raw_session( + db_path=db, + origin="claude-ai-export", + source_path="/tmp/sessions.json", + raw_content=json.dumps( + [ + { + "uuid": "reparsed", + "name": "Retained", + "summary": "successful reparse", + "created_at": "2026-01-01T00:00:00Z", + "updated_at": "2026-01-01T00:05:00Z", + "account": {"uuid": "acct-reparsed"}, + "chat_messages": [], + } + ] + ).encode(), + ) + with sqlite3.connect(db.with_name("source.db")) as conn: + cursor = conn.execute( + "UPDATE raw_sessions SET parsed_at_ms = 1, validated_at_ms = 0, validation_status = 'failed'" + ) + assert cursor.rowcount == 1 + conn.commit() + + result = load_samples_from_db("claude-ai", db_path=db) + + assert len(result) == 1 + assert result[0]["uuid"] == "reparsed" + + def test_sampling_quarantines_validation_failure_newer_than_parse(self, tmp_path: Path) -> None: + db = _archive_index_db(tmp_path) + raw_id = _insert_raw_session( + db_path=db, + origin="claude-ai-export", + source_path="/tmp/rejected.json", + raw_content=b'{"uuid":"rejected","chat_messages":[]}', + ) + with sqlite3.connect(db.with_name("source.db")) as conn: + cursor = conn.execute( + "UPDATE raw_sessions SET parsed_at_ms = 1, validated_at_ms = 2, validation_status = 'failed' WHERE raw_id = ?", + (raw_id,), + ) + assert cursor.rowcount == 1 + conn.commit() + outcomes: list[dict[str, object]] = [] + + result = list( + iter_schema_units( + "claude-ai", + db_path=db, + full_corpus=True, + terminal_recorder=lambda **outcome: outcomes.append(outcome), + ) + ) + + assert result == [] + assert outcomes == [ + { + "raw_id": raw_id, + "status": "quarantined", + "artifact_kind": None, + "source_path": "/tmp/rejected.json", + "reason": "source_validation_failed", + } + ] + + def test_sampling_records_equal_raw_transition_timestamps_as_indeterminate(self, tmp_path: Path) -> None: + db = _archive_index_db(tmp_path) + raw_id = _insert_raw_session( + db_path=db, + origin="claude-ai-export", + source_path="/tmp/equal-time.json", + raw_content=b'{"uuid":"equal-time","chat_messages":[]}', + ) + with sqlite3.connect(db.with_name("source.db")) as conn: + cursor = conn.execute( + "UPDATE raw_sessions SET parsed_at_ms = 1, validated_at_ms = 1, validation_status = 'failed' WHERE raw_id = ?", + (raw_id,), + ) + assert cursor.rowcount == 1 + conn.commit() + outcomes: list[dict[str, object]] = [] + + result = list( + iter_schema_units( + "claude-ai", + db_path=db, + full_corpus=True, + terminal_recorder=lambda **outcome: outcomes.append(outcome), + ) + ) + + assert result == [] + assert outcomes == [ + { + "raw_id": raw_id, + "status": "quarantined", + "artifact_kind": None, + "source_path": "/tmp/equal-time.json", + "reason": "source_validation_parse_order_ambiguous", + } + ] + def test_record_provider_sampling_streams_without_full_envelope( self, tmp_path: Path, diff --git a/tests/unit/core/test_schema_validation.py b/tests/unit/core/test_schema_validation.py index d678c6aa2f..f196b5690f 100644 --- a/tests/unit/core/test_schema_validation.py +++ b/tests/unit/core/test_schema_validation.py @@ -748,6 +748,53 @@ def validate(self, _sample: object) -> ValidationResult: assert stats.decode_errors == 0 +def test_verify_raw_corpus_filters_and_parses_by_detected_provider( + db_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """An UNKNOWN acquisition classified as Codex remains in the Codex corpus.""" + + class _AlwaysValidValidator: + provider = "codex" + + def validation_samples(self, payload: object, max_samples: int = 16) -> list[object]: + del max_samples + return [payload] + + def validate(self, _sample: object) -> ValidationResult: + return ValidationResult(is_valid=True) + + selected_providers: list[str] = [] + + def validator_for_payload(provider: str, *_args: object, **_kwargs: object) -> _AlwaysValidValidator: + selected_providers.append(provider) + return _AlwaysValidValidator() + + monkeypatch.setattr("polylogue.schemas.validation.corpus.SchemaValidator.for_payload", validator_for_payload) + raw_id = _insert_raw_record( + db_path=db_path, + raw_id="raw-unknown-codex", + source_name="unknown", + source_path="/tmp/learned-codex.jsonl", + raw_content=( + b'{"type":"session_meta","payload":{"id":"learned-codex"}}\n' + b'{"type":"response_item","payload":{"type":"message","role":"user",' + b'"content":[{"type":"input_text","text":"retained"}]}}\n' + ), + ) + with sqlite3.connect(db_path.parent / "source.db") as conn: + conn.execute("UPDATE raw_sessions SET detected_provider = 'codex' WHERE raw_id = ?", (raw_id,)) + + report = verify_raw_corpus( + db_path=db_path, + request=SchemaVerificationRequest(providers=["codex"], max_samples=16), + ) + + assert report.total_records == 1 + assert report.providers["codex"].valid_records == 1 + assert selected_providers == ["codex"] + + def test_verify_raw_corpus_counts_missing_schema_as_skipped(db_path: Path) -> None: _insert_raw_record( db_path=db_path, @@ -870,6 +917,37 @@ def test_verify_raw_corpus_quarantine_malformed_updates_validation_state(db_path assert isinstance(row["parse_error"], str) and "Malformed JSONL lines" in row["parse_error"] +def test_verify_raw_corpus_quarantine_advances_past_an_existing_parse_transition(db_path: Path) -> None: + """A wall clock behind the parse transition must not reverse raw-state authority.""" + raw_id = _insert_raw_record( + db_path=db_path, + raw_id="raw-codex-quarantine-order", + source_name="codex", + source_path="/tmp/quarantine-order.jsonl", + raw_content=( + b'{"type":"session_meta"}\nnot json at all\n{"type":"response_item","payload":{"type":"message"}}' + ), + ) + with sqlite3.connect(db_path.parent / "source.db") as conn: + conn.execute("UPDATE raw_sessions SET parsed_at_ms = 9999999999999 WHERE raw_id = ?", (raw_id,)) + conn.commit() + + from polylogue.schemas.validation.corpus import apply_quarantine_updates + + with sqlite3.connect(db_path.parent / "source.db") as conn: + apply_quarantine_updates( + conn, + updates=[(raw_id, "malformed retained JSONL", "codex", "codex")], + ) + + with sqlite3.connect(db_path.parent / "source.db") as conn: + parsed_at_ms, validated_at_ms, validation_status = conn.execute( + "SELECT parsed_at_ms, validated_at_ms, validation_status FROM raw_sessions WHERE raw_id = ?", (raw_id,) + ).fetchone() + assert validation_status == "failed" + assert validated_at_ms > parsed_at_ms + + def test_verify_raw_corpus_quarantine_empty_payload_updates_validation_state(db_path: Path) -> None: raw_id = _insert_raw_record( db_path=db_path, diff --git a/tests/unit/daemon/test_daemon_cli.py b/tests/unit/daemon/test_daemon_cli.py index c13cf41a1a..691a317843 100644 --- a/tests/unit/daemon/test_daemon_cli.py +++ b/tests/unit/daemon/test_daemon_cli.py @@ -3,12 +3,14 @@ import asyncio import contextlib import functools +import hashlib import inspect import os import sqlite3 import stat import threading import time +from collections.abc import Iterator from pathlib import Path from types import SimpleNamespace from typing import Any, cast @@ -664,7 +666,7 @@ def fake_repair(*_args: object, **_kwargs: object) -> object: return SimpleNamespace(success=True, repaired_count=1, detail="unexpected writer call") monkeypatch.setattr("polylogue.product.raw_authority.repair_materialization", fake_repair) - monkeypatch.setattr(daemon_cli, "_close_raw_materialization_fts", lambda _path: None) + monkeypatch.setattr(daemon_cli, "_close_raw_materialization_fts", lambda _path, *, ops_db_path: None) monkeypatch.setattr(daemon_cli, "_emit_raw_materialization_pass", lambda _result: None) with pytest.raises(RuntimeError, match="source-selection gate blocked"): @@ -1054,7 +1056,12 @@ def test_raw_materialization_closes_fts_on_cancellation( from polylogue.daemon import cli as daemon_cli archive = tmp_path / "archive" - closed: list[Path] = [] + active_index = tmp_path / "generations" / "active" / "index.db" + active_index.parent.mkdir(parents=True) + active_index.touch() + archive.mkdir() + (archive / ".index-active-pointer").write_text(str(active_index), encoding="utf-8") + closed: list[tuple[Path, Path]] = [] class FakeRestoreResult: restored_count = 0 @@ -1070,12 +1077,189 @@ def cancel_repair(*_args: object, **_kwargs: object) -> object: lambda *_args, **_kwargs: FakeRestoreResult(), ) monkeypatch.setattr("polylogue.storage.repair.repair_raw_materialization", cancel_repair) - monkeypatch.setattr(daemon_cli, "_close_raw_materialization_fts", closed.append) + monkeypatch.setattr( + daemon_cli, + "_close_raw_materialization_fts", + lambda index_db, *, ops_db_path: closed.append((index_db, ops_db_path)), + ) with pytest.raises(asyncio.CancelledError): daemon_cli._drain_raw_materialization_once() - assert closed == [archive / "index.db"] + assert closed == [(active_index, archive / "ops.db")] + + +@pytest.mark.parametrize("whale", [False, True]) +def test_raw_materialization_holds_pinned_generation_lease_through_fts_closure( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + whale: bool, +) -> None: + """FTS closure must finish under the same promotion-excluding lease as replay.""" + from polylogue.daemon import cli as daemon_cli + + archive = tmp_path / "archive" + active_index = tmp_path / "generations" / "active" / "index.db" + active_index.parent.mkdir(parents=True) + active_index.touch() + archive.mkdir() + (archive / ".index-active-pointer").write_text(str(active_index), encoding="utf-8") + lease_events: list[str] = [] + held = 0 + + @contextlib.contextmanager + def fake_generation_lease(_config: Config) -> Any: + nonlocal held + held += 1 + lease_events.append("acquire") + try: + yield active_index + finally: + lease_events.append("close") + held -= 1 + + class FakeRestoreResult: + restored_count = 0 + + def restore_debt(*_args: object, **_kwargs: object) -> FakeRestoreResult: + assert held == 1 + lease_events.append("restore") + return FakeRestoreResult() + + result = SimpleNamespace( + success=True, + repaired_count=1, + detail="repaired", + metrics={"raw_materialization_remaining_candidate_count": 0}, + ) + closed: list[tuple[Path, Path]] = [] + + monkeypatch.setattr("polylogue.paths.archive_root", lambda: archive) + monkeypatch.setattr("polylogue.paths.render_root", lambda: tmp_path / "render") + monkeypatch.setattr("polylogue.readiness.capability.raw_frontier_source_selection_block_reason", lambda _root: None) + monkeypatch.setattr( + "polylogue.storage.blob_integrity.restore_direct_blob_reference_debt", + restore_debt, + ) + + def recover_frontier(_config: Config) -> tuple[()]: + assert held == 1 + lease_events.append("recover") + return () + + def resolve_stale(_config: Config) -> int: + assert held == 1 + lease_events.append("stale") + return 0 + + monkeypatch.setattr("polylogue.product.raw_authority.recover_interrupted_frontier", recover_frontier) + monkeypatch.setattr("polylogue.product.raw_authority.auto_resolve_stale_plan_blockers", resolve_stale) + monkeypatch.setattr("polylogue.product.raw_authority.repair_materialization", lambda *_args, **_kwargs: result) + monkeypatch.setattr("polylogue.product.raw_authority.materialization_generation_lease", fake_generation_lease) + monkeypatch.setattr(daemon_cli, "_emit_raw_materialization_pass", lambda _result: None) + + def converge_frontier(_config: Config, **_kwargs: object) -> int: + assert held == 1 + lease_events.append("frontier") + return 0 + + monkeypatch.setattr(daemon_cli, "_converge_raw_authority_frontier", converge_frontier) + + def close_fts(index_db: Path, *, ops_db_path: Path) -> None: + assert held == 1 + closed.append((index_db, ops_db_path)) + lease_events.append("fts") + + monkeypatch.setattr(daemon_cli, "_close_raw_materialization_fts", close_fts) + + if whale: + assert ( + daemon_cli._run_raw_materialization_whale_pass_once(raw_artifact_id="raw-whale", max_payload_bytes=123) + is result + ) + else: + assert daemon_cli._drain_raw_materialization_once().repaired_sessions == 1 + + assert closed == [(active_index, archive / "ops.db")] + expected_events = ( + ["acquire", "fts", "close"] if whale else ["acquire", "restore", "recover", "stale", "fts", "frontier", "close"] + ) + assert lease_events == expected_events + assert held == 0 + + +@pytest.mark.parametrize("whale", [False, True]) +def test_raw_materialization_outer_lease_refusal_preserves_typed_result( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + whale: bool, +) -> None: + """Both daemon routes must emit the repair contract when pinning is refused.""" + from polylogue.daemon import cli as daemon_cli + from polylogue.storage.index_generation import ActiveWriterLease, RebuildLeaseUnavailableError + from polylogue.storage.repair import RepairResult + + archive = tmp_path / "archive" + archive.mkdir() + emitted: list[RepairResult] = [] + + def refuse_outer_lease(_lease: ActiveWriterLease) -> None: + raise RebuildLeaseUnavailableError("offline rebuild is active") + + def reject_restore(*_args: object, **_kwargs: object) -> None: + raise AssertionError("blob-reference restoration requires an acquired generation pin") + + def reject_repair(*_args: object, **_kwargs: object) -> None: + raise AssertionError("repair must not run when the outer generation pin is refused") + + monkeypatch.setattr("polylogue.paths.archive_root", lambda: archive) + monkeypatch.setattr("polylogue.paths.render_root", lambda: tmp_path / "render") + monkeypatch.setattr("polylogue.readiness.capability.raw_frontier_source_selection_block_reason", lambda _root: None) + monkeypatch.setattr( + "polylogue.storage.blob_integrity.restore_direct_blob_reference_debt", + reject_restore, + ) + monkeypatch.setattr( + "polylogue.product.raw_authority.recover_interrupted_frontier", + lambda _config: pytest.fail("frontier recovery requires an acquired generation pin"), + ) + monkeypatch.setattr( + "polylogue.product.raw_authority.auto_resolve_stale_plan_blockers", + lambda _config: pytest.fail("stale-plan recovery requires an acquired generation pin"), + ) + monkeypatch.setattr("polylogue.product.raw_authority.repair_materialization", reject_repair) + monkeypatch.setattr(ActiveWriterLease, "acquire", refuse_outer_lease) + monkeypatch.setattr(daemon_cli, "_emit_raw_materialization_pass", emitted.append) + monkeypatch.setattr( + daemon_cli, + "_converge_raw_authority_frontier", + lambda _config, **_kwargs: pytest.fail("frontier convergence requires an acquired generation pin"), + ) + monkeypatch.setattr( + daemon_cli, + "_close_raw_materialization_fts", + lambda *_args, **_kwargs: pytest.fail("FTS closure requires an acquired generation pin"), + ) + + if whale: + returned = daemon_cli._run_raw_materialization_whale_pass_once( + raw_artifact_id="raw-whale", + max_payload_bytes=123, + ) + assert returned is emitted[0] + else: + counts = daemon_cli._drain_raw_materialization_once() + assert counts.repaired_sessions == 0 + + assert len(emitted) == 1 + result = emitted[0] + assert isinstance(result, RepairResult) + assert result.name == "raw_materialization" + assert result.success is False + assert result.repaired_count == 0 + assert result.detail == ( + "Skipped raw materialization while offline index rebuild owns archive: offline rebuild is active" + ) def test_raw_materialization_fts_failure_records_durable_debt( @@ -1084,13 +1268,16 @@ def test_raw_materialization_fts_failure_records_durable_debt( ) -> None: from polylogue.daemon import cli as daemon_cli - index_db = tmp_path / "index.db" + index_db = tmp_path / "generations" / "active" / "index.db" + ops_db = tmp_path / "ops.db" + index_db.parent.mkdir(parents=True) index_db.touch() calls: list[tuple[str, str, str, str | None]] = [] class FakeCursor: - def __init__(self, db: Path) -> None: + def __init__(self, db: Path, *, ops_db_path: Path) -> None: assert db == index_db + assert ops_db_path == ops_db def clear_convergence_debt(self, **_kwargs: object) -> None: raise AssertionError("failed FTS repair must not clear debt") @@ -1109,7 +1296,7 @@ def record_convergence_debt( monkeypatch.setattr("polylogue.daemon.convergence_stages.repair_fts_surface", lambda *_args: False) monkeypatch.setattr("polylogue.sources.live.cursor.CursorStore", FakeCursor) - daemon_cli._close_raw_materialization_fts(index_db) + daemon_cli._close_raw_materialization_fts(index_db, ops_db_path=ops_db) assert calls == [ ( @@ -1127,13 +1314,16 @@ def test_raw_materialization_fts_success_clears_prior_debt( ) -> None: from polylogue.daemon import cli as daemon_cli - index_db = tmp_path / "index.db" + index_db = tmp_path / "generations" / "active" / "index.db" + ops_db = tmp_path / "ops.db" + index_db.parent.mkdir(parents=True) index_db.touch() cleared: list[dict[str, object]] = [] class FakeCursor: - def __init__(self, db: Path) -> None: + def __init__(self, db: Path, *, ops_db_path: Path) -> None: assert db == index_db + assert ops_db_path == ops_db def clear_convergence_debt(self, **kwargs: object) -> None: cleared.append(kwargs) @@ -1145,7 +1335,7 @@ def record_convergence_debt(self, **_kwargs: object) -> None: monkeypatch.setattr("polylogue.daemon.convergence_stages.repair_fts_surface", lambda *_args: True) monkeypatch.setattr("polylogue.sources.live.cursor.CursorStore", FakeCursor) - daemon_cli._close_raw_materialization_fts(index_db) + daemon_cli._close_raw_materialization_fts(index_db, ops_db_path=ops_db) assert cleared == [{"subject_type": "fts_surface", "subject_id": "messages_fts", "stage": "fts"}] @@ -1156,13 +1346,16 @@ def test_raw_materialization_fts_exception_becomes_explicit_debt( ) -> None: from polylogue.daemon import cli as daemon_cli - index_db = tmp_path / "index.db" + index_db = tmp_path / "generations" / "active" / "index.db" + ops_db = tmp_path / "ops.db" + index_db.parent.mkdir(parents=True) index_db.touch() errors: list[str | None] = [] class FakeCursor: - def __init__(self, db: Path) -> None: + def __init__(self, db: Path, *, ops_db_path: Path) -> None: assert db == index_db + assert ops_db_path == ops_db def record_convergence_debt(self, *, error: str | None = None, **_kwargs: object) -> None: errors.append(error) @@ -1174,7 +1367,7 @@ def record_convergence_debt(self, *, error: str | None = None, **_kwargs: object ) monkeypatch.setattr("polylogue.sources.live.cursor.CursorStore", FakeCursor) - daemon_cli._close_raw_materialization_fts(index_db) + daemon_cli._close_raw_materialization_fts(index_db, ops_db_path=ops_db) assert errors == ["FTS repair failed after raw materialization: RuntimeError: injected FTS failure"] @@ -2230,7 +2423,10 @@ def test_explicit_browser_capture_root_uses_spool_override_classifier(tmp_path: ) -def test_run_live_watcher_stops_on_keyboard_interrupt() -> None: +def test_run_live_watcher_stops_on_keyboard_interrupt( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: from polylogue.daemon import cli as daemon_cli class FakePolylogue: @@ -2241,6 +2437,12 @@ async def __aexit__(self, *exc: object) -> None: return None stopped: list[bool] = [] + shutdown_timeouts: list[float] = [] + + class Coordinator: + async def shutdown(self, *, timeout: float) -> bool: + shutdown_timeouts.append(timeout) + return True class FakeWatcher: stopped = False @@ -2256,14 +2458,165 @@ def stop(self) -> None: stopped.append(self.stopped) sources = (WatchSource(name="codex", root=Path("/tmp/codex")),) + monkeypatch.setattr("polylogue.paths.archive_root", lambda: tmp_path / "archive") with ( patch.object(daemon_cli, "Polylogue", FakePolylogue), patch.object(daemon_cli, "LiveWatcher", FakeWatcher), + patch.object(daemon_cli, "daemon_write_coordinator", return_value=Coordinator()), ): asyncio.run(daemon_cli.run_live_watcher(sources=sources, debounce_s=1.0)) assert stopped == [True] + assert shutdown_timeouts == [5.0] + + +def test_run_live_watcher_refuses_before_entry_while_rebuild_lease_is_held( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + from polylogue.daemon import cli as daemon_cli + from polylogue.storage.index_generation import RebuildLease, RebuildLeaseUnavailableError + + archive_root_path = tmp_path / "archive" + archive_root_path.mkdir() + monkeypatch.setattr("polylogue.paths.archive_root", lambda: archive_root_path) + + class ForbiddenPolylogue: + def __init__(self) -> None: + raise AssertionError("standalone watcher entered archive before rebuild refusal") + + monkeypatch.setattr(daemon_cli, "Polylogue", ForbiddenPolylogue) + with ( + RebuildLease(archive_root_path), + pytest.raises( + RebuildLeaseUnavailableError, + match="offline index rebuild owns archive", + ), + ): + asyncio.run(daemon_cli.run_live_watcher(sources=(), debounce_s=1.0)) + + +def test_live_watcher_cancellation_during_drain_retains_rebuild_exclusion( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + from polylogue.daemon import cli as daemon_cli + from polylogue.storage.index_generation import RebuildLease, RebuildLeaseUnavailableError + + archive_root_path = tmp_path / "archive" + archive_root_path.mkdir() + monkeypatch.setattr("polylogue.paths.archive_root", lambda: archive_root_path) + + class FakePolylogue: + async def __aenter__(self) -> object: + return object() + + async def __aexit__(self, *exc: object) -> None: + return None + + class FakeWatcher: + def __init__(self, *_args: object, **_kwargs: object) -> None: + pass + + async def run(self) -> None: + return None + + def stop(self) -> None: + return None + + class BlockingCoordinator: + def __init__(self) -> None: + self.shutdown_started = asyncio.Event() + + async def shutdown(self, *, timeout: float) -> bool: + assert timeout == 5.0 + self.shutdown_started.set() + await asyncio.Event().wait() + return False + + coordinator = BlockingCoordinator() + + async def exercise() -> None: + with ( + patch.object(daemon_cli, "Polylogue", FakePolylogue), + patch.object(daemon_cli, "LiveWatcher", FakeWatcher), + patch.object(daemon_cli, "daemon_write_coordinator", return_value=coordinator), + ): + task = asyncio.create_task(daemon_cli.run_live_watcher(sources=(), debounce_s=1.0)) + await coordinator.shutdown_started.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + asyncio.run(exercise()) + + with pytest.raises(RebuildLeaseUnavailableError, match="index rebuild lease is already held"): + with RebuildLease(archive_root_path): + pass + + +def test_live_watcher_stop_failure_still_retains_undrained_rebuild_exclusion( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """A watcher stop exception cannot bypass coordinator drain authority.""" + from polylogue.daemon import cli as daemon_cli + from polylogue.product import raw_authority + from polylogue.storage.index_generation import RebuildLease, RebuildLeaseUnavailableError + + archive_root_path = tmp_path / "archive" + archive_root_path.mkdir() + monkeypatch.setattr("polylogue.paths.archive_root", lambda: archive_root_path) + + class FakePolylogue: + async def __aenter__(self) -> object: + return object() + + async def __aexit__(self, *exc: object) -> None: + return None + + class FailingStopWatcher: + def __init__(self, *_args: object, **_kwargs: object) -> None: + pass + + async def run(self) -> None: + return None + + def stop(self) -> None: + raise RuntimeError("watcher stop failed") + + class UndrainedCoordinator: + async def shutdown(self, *, timeout: float) -> bool: + assert timeout == 5.0 + return False + + captured: list[raw_authority.ArchiveWriterRebuildExclusion] = [] + real_exclusion = raw_authority.archive_writer_rebuild_exclusion + + @contextlib.contextmanager + def capture_exclusion(root: Path) -> Iterator[raw_authority.ArchiveWriterRebuildExclusion]: + with real_exclusion(root) as exclusion: + captured.append(exclusion) + yield exclusion + + monkeypatch.setattr(raw_authority, "archive_writer_rebuild_exclusion", capture_exclusion) + with ( + patch.object(daemon_cli, "Polylogue", FakePolylogue), + patch.object(daemon_cli, "LiveWatcher", FailingStopWatcher), + patch.object(daemon_cli, "daemon_write_coordinator", return_value=UndrainedCoordinator()), + pytest.raises(RuntimeError, match="watcher stop failed"), + ): + asyncio.run(daemon_cli.run_live_watcher(sources=(), debounce_s=1.0)) + + assert len(captured) == 1 + with pytest.raises(RebuildLeaseUnavailableError, match="index rebuild lease is already held"): + with RebuildLease(archive_root_path): + pass + + captured[0].release() + with RebuildLease(archive_root_path): + pass def test_ensure_fts_startup_readiness_skips_old_non_blocks_shape( @@ -3365,6 +3718,65 @@ def test_reconcile_blob_publications_clears_terminal_receipts_at_startup( assert conn.execute("SELECT COUNT(*) FROM blob_publication_reservations").fetchone()[0] == 0 +def test_daemon_rebuild_lease_refusal_precedes_startup_blob_reconciliation( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """An offline rebuild must refuse the daemon before durable startup writes.""" + from polylogue.daemon import cli as daemon_cli + from polylogue.storage.blob_publication import ArchiveBlobPublisher + from polylogue.storage.blob_store import BlobStore + from polylogue.storage.index_generation import RebuildLease, RebuildLeaseUnavailableError + from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root + + archive_root_path = tmp_path / "archive" + initialize_active_archive_root(archive_root_path) + monkeypatch.setenv("POLYLOGUE_ARCHIVE_ROOT", str(archive_root_path)) + + source_db = archive_root_path / "source.db" + publisher = ArchiveBlobPublisher(source_db, BlobStore(archive_root_path / "blob").root) + publisher.write_from_bytes(b"startup-rebuild-refusal") + publisher.flush() + with sqlite3.connect(source_db) as conn: + assert conn.execute("SELECT COUNT(*) FROM blob_publication_reservations").fetchone()[0] == 1 + + def archive_digest() -> str: + digest = hashlib.sha256() + for path in sorted(archive_root_path.rglob("*")): + if not path.is_file(): + continue + relative = path.relative_to(archive_root_path).as_posix().encode() + payload = path.read_bytes() + digest.update(len(relative).to_bytes(8, "big")) + digest.update(relative) + digest.update(len(payload).to_bytes(8, "big")) + digest.update(payload) + return digest.hexdigest() + + with RebuildLease(archive_root_path): + before = archive_digest() + with pytest.raises( + RebuildLeaseUnavailableError, + match="offline index rebuild owns archive", + ): + asyncio.run( + daemon_cli.run_daemon_services( + sources=(), + debounce_s=1.0, + enable_watch=False, + enable_browser_capture=False, + browser_capture_host="127.0.0.1", + browser_capture_port=8765, + browser_capture_spool_path=None, + ) + ) + assert archive_digest() == before + + with sqlite3.connect(source_db) as conn: + assert conn.execute("SELECT COUNT(*) FROM blob_publication_reservations").fetchone()[0] == 1 + assert not (archive_root_path / "daemon.pid").exists() + + def test_run_daemon_services_stops_live_watcher_on_failure() -> None: from polylogue.daemon import cli as daemon_cli @@ -3411,6 +3823,83 @@ def stop(self) -> None: assert stopped == [True] +def test_daemon_cleanup_failure_retains_rebuild_exclusion_until_process_exit( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Cleanup errors before coordinator shutdown must never reopen rebuilds.""" + from polylogue.daemon import cli as daemon_cli + from polylogue.product import raw_authority + from polylogue.storage.index_generation import RebuildLease, RebuildLeaseUnavailableError + + async def noop() -> None: + return None + + class FakePolylogue: + async def __aenter__(self) -> object: + return object() + + async def __aexit__(self, *exc: object) -> None: + return None + + class FakeWatcher: + def __init__(self, *_args: object, **_kwargs: object) -> None: + pass + + async def run(self) -> None: + raise RuntimeError("watch stopped") + + def stop(self) -> None: + return None + + captured: list[raw_authority.ArchiveWriterRebuildExclusion] = [] + archive_roots: list[Path] = [] + real_exclusion = raw_authority.archive_writer_rebuild_exclusion + + @contextlib.contextmanager + def capture_exclusion(archive_root: Path) -> Iterator[raw_authority.ArchiveWriterRebuildExclusion]: + archive_roots.append(archive_root) + with real_exclusion(archive_root) as exclusion: + captured.append(exclusion) + yield exclusion + + def fail_shutdown_marker() -> None: + raise RuntimeError("shutdown marker failed") + + monkeypatch.setattr(raw_authority, "archive_writer_rebuild_exclusion", capture_exclusion) + with ( + patch.object(daemon_cli, "Polylogue", FakePolylogue), + patch.object(daemon_cli, "LiveWatcher", FakeWatcher), + patch.object(daemon_cli, "_reconcile_blob_publications", noop), + patch.object( + daemon_cli, + "_mark_interrupted_live_ingest_attempts_on_shutdown", + fail_shutdown_marker, + ), + pytest.raises(RuntimeError, match="shutdown marker failed"), + ): + asyncio.run( + daemon_cli.run_daemon_services( + sources=(WatchSource(name="codex", root=Path("/tmp/codex")),), + debounce_s=1.0, + enable_watch=True, + enable_browser_capture=False, + browser_capture_host="127.0.0.1", + browser_capture_port=8765, + browser_capture_spool_path=None, + ) + ) + + assert len(captured) == 1 + assert len(archive_roots) == 1 + with pytest.raises(RebuildLeaseUnavailableError, match="index rebuild lease is already held"): + with RebuildLease(archive_roots[0]): + pass + + captured[0].release() + with RebuildLease(archive_roots[0]): + pass + + def test_lifecycle_heartbeat_runs_without_index_stats(monkeypatch: pytest.MonkeyPatch) -> None: """The degraded daemon heartbeat must not depend on index.db existing.""" from polylogue.daemon import cli as daemon_cli @@ -3683,6 +4172,28 @@ def test_pidfile_remains_locked_until_admitted_writers_are_drained( os.close(successor_fd) +def test_rebuild_exclusion_survives_an_undrained_writer_timeout(tmp_path: Path) -> None: + from polylogue.daemon import cli as daemon_cli + from polylogue.product.raw_authority import archive_writer_rebuild_exclusion + from polylogue.storage.index_generation import RebuildLease, RebuildLeaseUnavailableError + + archive_root_path = tmp_path / "archive" + archive_root_path.mkdir() + with archive_writer_rebuild_exclusion(archive_root_path) as exclusion: + daemon_cli._retain_rebuild_exclusion_for_undrained_writer( + exclusion, + writer_drained=False, + ) + + with pytest.raises(RebuildLeaseUnavailableError, match="index rebuild lease is already held"): + with RebuildLease(archive_root_path): + pass + + exclusion.release() + with RebuildLease(archive_root_path): + pass + + def test_shutdown_lifecycle_event_is_bounded_when_writer_gate_is_stuck(tmp_path: Path) -> None: from polylogue.daemon import cli as daemon_cli diff --git a/tests/unit/daemon/test_provenance_endpoint.py b/tests/unit/daemon/test_provenance_endpoint.py index 49f59431ef..db59a92870 100644 --- a/tests/unit/daemon/test_provenance_endpoint.py +++ b/tests/unit/daemon/test_provenance_endpoint.py @@ -387,6 +387,7 @@ def test_quarantine_surfaces_when_validation_failed(self, workspace_env: dict[st raw_id=raw_id, source_path="/tmp/x.json", blob_size=len(payload_bytes), + parsed_at_ms=None, validation_status="failed", ) @@ -395,6 +396,43 @@ def test_quarantine_surfaces_when_validation_failed(self, workspace_env: dict[st assert result["quarantined"] is True assert result["quarantine_reason"] == "validation_failed" + def test_historical_validation_failure_is_not_current_quarantine(self, workspace_env: dict[str, Path]) -> None: + raw_id = _seed_raw_blob(b"{}") + session_id = _seed_archive_provenance( + session_id="c-historical-validation", + raw_id=raw_id, + source_path="/tmp/x.json", + blob_size=2, + validated_at_ms=1_767_225_601_000, + validation_status="failed", + ) + + result = build_provenance_payload(session_id) + + assert result is not None + assert result["validation_status"] == "failed" + assert result["parsed_at"] is not None + assert result["quarantined"] is False + assert result["quarantine_reason"] is None + + def test_equal_raw_transition_timestamps_surface_order_ambiguity(self, workspace_env: dict[str, Path]) -> None: + raw_id = _seed_raw_blob(b"{}") + session_id = _seed_archive_provenance( + session_id="c-ambiguous-validation", + raw_id=raw_id, + source_path="/tmp/x.json", + blob_size=2, + parsed_at_ms=1_767_225_602_000, + validated_at_ms=1_767_225_602_000, + validation_status="failed", + ) + + result = build_provenance_payload(session_id) + + assert result is not None + assert result["quarantined"] is True + assert result["quarantine_reason"] == "validation_parse_order_ambiguous" + def test_quarantine_surfaces_when_no_raw_artifact(self, workspace_env: dict[str, Path]) -> None: session_id = _seed_archive_provenance( session_id="c-orphan", diff --git a/tests/unit/daemon/test_raw_parse_recovery.py b/tests/unit/daemon/test_raw_parse_recovery.py index 513c3b0ac6..ca13453932 100644 --- a/tests/unit/daemon/test_raw_parse_recovery.py +++ b/tests/unit/daemon/test_raw_parse_recovery.py @@ -28,13 +28,14 @@ import pytest -from polylogue.core.enums import Provider +from polylogue.core.enums import Provider, ValidationStatus from polylogue.core.errors import RawCASFrontierError from polylogue.core.raw_failure_evidence import RawFailureEvidenceKind from polylogue.daemon.convergence import DaemonConverger, StageState from polylogue.daemon.convergence_stages import make_raw_parse_recovery_stage from polylogue.sources.live.cursor import CursorStore from polylogue.storage.archive_identity import archive_file_set_root +from polylogue.storage.raw.models import RawSessionStateUpdate from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root @@ -313,13 +314,14 @@ def test_raw_parse_recovery_skips_validation_failed_cas_frontier_failure(tmp_pat ) with sqlite3.connect(tmp_path / "source.db") as conn: conn.execute("UPDATE raw_sessions SET validation_status = 'failed' WHERE raw_id = ?", (raw_id,)) + assert conn.total_changes == 1 conn.commit() assert make_raw_parse_recovery_stage(tmp_path / "index.db").check(path) is False def test_raw_parse_recovery_drains_previously_parsed_cas_frontier_failure(tmp_path: Path) -> None: - """CAS authority replays an unmaterialized raw even when parsing had completed.""" + """A stale validation failure does not suppress newer parse authority.""" initialize_active_archive_root(tmp_path) path = tmp_path / "previously-parsed-cas-frontier.json" raw_id = _write_stuck_raw(tmp_path, source_path=str(path)) @@ -331,6 +333,45 @@ def test_raw_parse_recovery_drains_previously_parsed_cas_frontier_failure(tmp_pa provider=Provider.CHATGPT, error=RawCASFrontierError("frontier changed after parsing completed"), ) + with sqlite3.connect(tmp_path / "source.db") as conn: + parsed_at_ms = int( + conn.execute("SELECT parsed_at_ms FROM raw_sessions WHERE raw_id = ?", (raw_id,)).fetchone()[0] + ) + conn.execute( + "UPDATE raw_sessions SET validation_status = 'failed', validated_at_ms = ? WHERE raw_id = ?", + (parsed_at_ms - 1, raw_id), + ) + assert conn.total_changes == 1 + conn.commit() + + stage = make_raw_parse_recovery_stage(tmp_path / "index.db") + + assert stage.check(path) is True + assert stage.execute(path) is True + assert stage.check(path) is False + assert _sessions_for_raw(tmp_path, raw_id) == [("conv-stuck", raw_id)] + + +def test_raw_parse_recovery_uses_monotonic_parse_state_after_failed_validation(tmp_path: Path) -> None: + """The probe and repair route agree when a later parse supersedes validation.""" + initialize_active_archive_root(tmp_path) + path = tmp_path / "monotonic-validation-recovery.json" + raw_id = _write_stuck_raw(tmp_path, source_path=str(path)) + + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + archive.finalize_raw_parse_state( + raw_id, + state=RawSessionStateUpdate( + parsed_at="1970-01-01T00:00:00.001Z", + validation_status=ValidationStatus.FAILED, + validation_error="older validation failure", + ), + ) + archive.mark_raw_parse_failed( + raw_id, + provider=Provider.CHATGPT, + error=RawCASFrontierError("retry after the later parser state"), + ) stage = make_raw_parse_recovery_stage(tmp_path / "index.db") @@ -340,6 +381,32 @@ def test_raw_parse_recovery_drains_previously_parsed_cas_frontier_failure(tmp_pa assert _sessions_for_raw(tmp_path, raw_id) == [("conv-stuck", raw_id)] +def test_raw_parse_recovery_skips_current_validation_failure_after_prior_parse(tmp_path: Path) -> None: + """A current validation failure cannot leave CAS recovery permanently pending.""" + initialize_active_archive_root(tmp_path) + path = tmp_path / "current-validation-failed-cas-frontier.json" + raw_id = _write_stuck_raw(tmp_path, source_path=str(path)) + + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + archive.mark_raw_parse_succeeded(raw_id, provider=Provider.CHATGPT) + archive.mark_raw_parse_failed( + raw_id, + provider=Provider.CHATGPT, + error=RawCASFrontierError("frontier changed before current validation failure"), + ) + with sqlite3.connect(tmp_path / "source.db") as conn: + parsed_at_ms = int( + conn.execute("SELECT parsed_at_ms FROM raw_sessions WHERE raw_id = ?", (raw_id,)).fetchone()[0] + ) + conn.execute( + "UPDATE raw_sessions SET validation_status = 'failed', validated_at_ms = ? WHERE raw_id = ?", + (parsed_at_ms, raw_id), + ) + conn.commit() + + assert make_raw_parse_recovery_stage(tmp_path / "index.db").check(path) is False + + def test_raw_parse_recovery_source_open_failure_is_failed_and_retryable( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 829793d5d5..5558909a84 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -2077,6 +2077,35 @@ def portable_watch(*paths: Path, **kwargs: object) -> object: assert observation == CheckoutMutationObservation(changed=True, unavailable=False, observed_path="tracked.py") +def test_checkout_mutation_monitor_prepares_paths_before_backend_startup_deadline( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + monitor = CheckoutMutationMonitor(tmp_path) + original_watched_directories = monitor._watched_directories + discovery_threads: list[threading.Thread] = [] + + def observed_discovery() -> list[Path]: + discovery_threads.append(threading.current_thread()) + return original_watched_directories() + + def portable_watch(*_paths: Path, **kwargs: object) -> object: + yield set() + stop_event = kwargs["stop_event"] + assert isinstance(stop_event, threading.Event) + stop_event.wait() + + monkeypatch.setattr(monitor, "_watched_directories", observed_discovery) + monkeypatch.setattr(watchfiles, "watch", portable_watch) + + monitor.start() + observation = monitor.finish() + + assert discovery_threads[0] is threading.main_thread() + assert observation == CheckoutMutationObservation(changed=False, unavailable=False) + + def test_checkout_mutation_monitor_rejects_source_topology_changed_during_startup( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -2259,6 +2288,89 @@ def test_checkout_mutation_monitor_observes_transient_head_ref_change(tmp_path: ) +def test_checkout_mutation_monitor_ignores_shared_packed_refs_when_current_ref_is_loose( + tmp_path: Path, +) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.email", "tests@example.invalid"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.name", "Polylogue Tests"], cwd=tmp_path, check=True) + tracked = tmp_path / "tracked.py" + tracked.write_text("value = 1\n", encoding="utf-8") + subprocess.run(["git", "add", "tracked.py"], cwd=tmp_path, check=True) + subprocess.run(["git", "commit", "-qm", "seed"], cwd=tmp_path, check=True) + subprocess.run(["git", "pack-refs", "--all", "--no-prune"], cwd=tmp_path, check=True) + + monitor = CheckoutMutationMonitor(tmp_path) + monitor._watched_directories() + monitor._record_change(tmp_path / ".git" / "packed-refs") + + assert monitor.finish() == CheckoutMutationObservation(changed=False, unavailable=False) + + +def test_checkout_mutation_monitor_watches_packed_refs_when_current_ref_is_packed( + tmp_path: Path, +) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.email", "tests@example.invalid"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.name", "Polylogue Tests"], cwd=tmp_path, check=True) + tracked = tmp_path / "tracked.py" + tracked.write_text("value = 1\n", encoding="utf-8") + subprocess.run(["git", "add", "tracked.py"], cwd=tmp_path, check=True) + subprocess.run(["git", "commit", "-qm", "seed"], cwd=tmp_path, check=True) + subprocess.run(["git", "pack-refs", "--all", "--prune"], cwd=tmp_path, check=True) + branch = subprocess.run( + ["git", "symbolic-ref", "HEAD"], cwd=tmp_path, check=True, capture_output=True, text=True + ).stdout.strip() + loose_ref = Path( + subprocess.run( + ["git", "rev-parse", "--path-format=absolute", "--git-path", branch], + cwd=tmp_path, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + ) + assert not loose_ref.exists() + + monitor = CheckoutMutationMonitor(tmp_path) + monitor._watched_directories() + monitor._record_change(tmp_path / ".git" / "packed-refs") + + assert monitor.finish() == CheckoutMutationObservation( + changed=True, + unavailable=False, + observed_path=".git/packed-refs", + ) + + +def test_checkout_mutation_monitor_ignores_shared_packed_refs_when_head_is_detached( + tmp_path: Path, +) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.email", "tests@example.invalid"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.name", "Polylogue Tests"], cwd=tmp_path, check=True) + tracked = tmp_path / "tracked.py" + tracked.write_text("value = 1\n", encoding="utf-8") + subprocess.run(["git", "add", "tracked.py"], cwd=tmp_path, check=True) + subprocess.run(["git", "commit", "-qm", "seed"], cwd=tmp_path, check=True) + subprocess.run(["git", "switch", "--detach", "--quiet"], cwd=tmp_path, check=True) + packed_refs = Path( + subprocess.run( + ["git", "rev-parse", "--path-format=absolute", "--git-path", "packed-refs"], + cwd=tmp_path, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + ) + + monitor = CheckoutMutationMonitor(tmp_path) + monitor._watched_directories() + monitor._record_change(packed_refs) + + assert monitor.finish() == CheckoutMutationObservation(changed=False, unavailable=False) + + @pytest.mark.uses_real_clock("waits for the filesystem watcher to witness a loose ref created from packed authority") def test_checkout_mutation_monitor_observes_packed_nested_branch_ref_change(tmp_path: Path) -> None: subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) @@ -4939,6 +5051,36 @@ def test_verify_withholds_success_when_checkout_fingerprint_is_unavailable( assert checkout_step["final_worktree_fingerprint"] == fingerprints[1] +def test_verify_classifies_unavailable_mutation_monitor_separately( + capsys: pytest.CaptureFixture[str], +) -> None: + class _UnavailableMonitor: + def __init__(self, _root: Path) -> None: + pass + + def start(self) -> None: + pass + + def finish(self) -> CheckoutMutationObservation: + return CheckoutMutationObservation(changed=False, unavailable=True) + + with ( + patch("devtools.verify._run", return_value=(0, 0.01, {})), + patch("devtools.verify._git_head", return_value="head"), + patch("devtools.verify._save_history"), + patch("devtools.verify._stamp_head"), + patch("devtools.verify._notify"), + patch("devtools.verify.CheckoutMutationMonitor", _UnavailableMonitor), + patch("devtools.verify.worktree_fingerprint", return_value="stable"), + ): + rc = main(["--quick", "--json"]) + + assert rc == 125 + payload = json.loads(capsys.readouterr().out) + checkout_step = next(step for step in payload["steps"] if step["name"] == "checkout stability") + assert checkout_step["diagnosis"] == "checkout_mutation_monitor_unavailable" + + def test_verify_rejects_git_head_change_with_matching_worktree_fingerprints( capsys: pytest.CaptureFixture[str], ) -> None: diff --git a/tests/unit/product/test_raw_authority.py b/tests/unit/product/test_raw_authority.py index 4574213629..e6e47d18a9 100644 --- a/tests/unit/product/test_raw_authority.py +++ b/tests/unit/product/test_raw_authority.py @@ -7,6 +7,7 @@ from polylogue.config import Config from polylogue.product import raw_authority +from polylogue.storage.index_generation import RebuildLease, RebuildLeaseUnavailableError from polylogue.storage.raw_authority import raw_authority_detail_query_handle from polylogue.storage.raw_reconciler import RawAuthorityFrontierApplyReport @@ -89,6 +90,43 @@ def test_frontier_apply_report_rejects_incoherent_counts() -> None: ) +def test_materialization_generation_lease_pins_active_index_and_excludes_promotion(tmp_path: Path) -> None: + active_index = tmp_path / "generations" / "active" / "index.db" + active_index.parent.mkdir(parents=True) + active_index.touch() + (tmp_path / ".index-active-pointer").write_text(str(active_index), encoding="utf-8") + config = Config(archive_root=tmp_path, render_root=tmp_path / "render", sources=[]) + + with raw_authority.materialization_generation_lease(config) as index_db: + assert index_db == active_index + with pytest.raises(RebuildLeaseUnavailableError): + with RebuildLease(tmp_path): + pass + + +def test_materialization_generation_lease_uses_explicit_split_root(tmp_path: Path) -> None: + configured_root = tmp_path / "configured" + active_root = tmp_path / "active" + configured_root.mkdir() + active_root.mkdir() + active_index = active_root / "index.db" + active_index.touch() + config = Config( + archive_root=configured_root, + render_root=tmp_path / "render", + sources=[], + db_path=active_index, + ) + + with raw_authority.materialization_generation_lease(config) as index_db: + assert index_db == active_index + with pytest.raises(RebuildLeaseUnavailableError): + with RebuildLease(active_root): + pass + with RebuildLease(configured_root): + pass + + @pytest.mark.parametrize( ("selected_plan_ids", "preview_census_id", "outcome_plan_id", "message"), [ diff --git a/tests/unit/sources/test_artifact_taxonomy.py b/tests/unit/sources/test_artifact_taxonomy.py index c1f2273acb..fdcb98f86b 100644 --- a/tests/unit/sources/test_artifact_taxonomy.py +++ b/tests/unit/sources/test_artifact_taxonomy.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import sqlite3 from pathlib import Path from polylogue.archive.artifact_taxonomy import ArtifactKind, classify_artifact, classify_artifact_path @@ -285,6 +286,38 @@ def record_terminal( ] +def test_schema_sampling_uses_detected_provider_for_unknown_acquisition(workspace_env: dict[str, Path]) -> None: + """Provider-scoped schema reads include source-only raws learned during replay.""" + archive_root = workspace_env["archive_root"] + payload = ( + b'{"type":"user","uuid":"message-1","sessionId":"learned-session",' + b'"parentUuid":null,"message":{"role":"user","content":"hello"}}\n' + ) + with ArchiveStore(archive_root) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.UNKNOWN, + payload=payload, + source_path="/captures/learned/session.jsonl", + acquired_at_ms=1, + ) + with sqlite3.connect(archive_root / "source.db") as conn: + conn.execute( + "UPDATE raw_sessions SET detected_provider = 'claude-code' WHERE raw_id = ?", + (raw_id,), + ) + + units = list( + _iter_schema_units_from_db( + Provider.CLAUDE_CODE, + db_path=archive_root / "index.db", + config=resolve_provider_config(Provider.CLAUDE_CODE), + ) + ) + + assert units + assert {unit.raw_id for unit in units} == {raw_id} + + def test_tool_result_sidecar_never_classifies_as_session_even_when_content_looks_like_one() -> None: """Regression for polylogue-omsw: a ``tool-results/`` sidecar must never become a session regardless of its content, only its path. diff --git a/tests/unit/sources/test_convergence_debt_deferred_vocabulary.py b/tests/unit/sources/test_convergence_debt_deferred_vocabulary.py index e62422944a..3694b1e480 100644 --- a/tests/unit/sources/test_convergence_debt_deferred_vocabulary.py +++ b/tests/unit/sources/test_convergence_debt_deferred_vocabulary.py @@ -32,8 +32,63 @@ convergence_debt_from_states, is_deferred_stage_state, ) +from polylogue.sources.live.convergence_debt_retry import convergence_debt_source_path from polylogue.sources.live.convergence_outcome import record_convergence_outcome from polylogue.sources.live.cursor import CursorStore +from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_archive_database +from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier + + +def test_convergence_debt_lookups_follow_the_active_index_generation(tmp_path: Path) -> None: + """Outcome and retry lookups ignore a stale conventional index database.""" + + source_db = tmp_path / "source.db" + shadow_index = tmp_path / "index.db" + active_index = tmp_path / "generations" / "active" / "index.db" + initialize_archive_database(source_db, ArchiveTier.SOURCE) + initialize_archive_database(shadow_index, ArchiveTier.INDEX) + initialize_archive_database(active_index, ArchiveTier.INDEX) + source_path = tmp_path / "active.jsonl" + source_path.write_text("{}", encoding="utf-8") + with sqlite3.connect(source_db) as conn: + conn.execute( + """ + INSERT INTO raw_sessions ( + raw_id, origin, native_id, source_path, source_index, blob_hash, blob_size, acquired_at_ms + ) VALUES ('raw-active', 'codex-session', 'active', ?, 0, ?, 1, 1) + """, + (str(source_path), bytes(32)), + ) + conn.commit() + with sqlite3.connect(active_index) as conn: + conn.execute( + """ + INSERT INTO sessions (native_id, origin, raw_id, title, content_hash) + VALUES ('active', 'codex-session', 'raw-active', 'active', ?) + """, + (bytes(32),), + ) + conn.commit() + (tmp_path / ".index-active-pointer").write_text(f"{active_index}\n", encoding="utf-8") + cursor = CursorStore(tmp_path / "ops.db") + debt = ConvergenceDebt(path=source_path, stage="fts", error="deferred", deferred=True) + + record_convergence_outcome(cursor, source_path, (debt,), archive_root=tmp_path) + with sqlite3.connect(tmp_path / "ops.db") as conn: + session_debts = conn.execute( + "SELECT target_id FROM convergence_debt WHERE target_type = 'session_id'" + ).fetchall() + assert ( + convergence_debt_source_path( + conn, + subject_type="session_id", + subject_id="codex-session:active", + archive_root=tmp_path, + ) + == source_path + ) + + assert session_debts == [("codex-session:active",)] def test_is_deferred_stage_state_true_only_for_pending() -> None: diff --git a/tests/unit/sources/test_hook_spool.py b/tests/unit/sources/test_hook_spool.py index a93e2ad1f8..d44aa60301 100644 --- a/tests/unit/sources/test_hook_spool.py +++ b/tests/unit/sources/test_hook_spool.py @@ -2,13 +2,14 @@ from __future__ import annotations +import asyncio import json import os import re import sqlite3 import subprocess import sys -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Callable, Coroutine from io import StringIO from pathlib import Path from types import SimpleNamespace @@ -32,6 +33,7 @@ from polylogue.sources.live import LiveWatcher, WatchSource from polylogue.sources.live.cursor import CursorStore from polylogue.sources.parsers.hermes_lifecycle import DURABLE_FINALIZE, PER_TURN_END +from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root @pytest.mark.parametrize( @@ -267,6 +269,197 @@ async def emit_first_hook(*roots: Path, **_kwargs: object) -> AsyncIterator[set[ assert conn.execute("SELECT session_native_id FROM raw_hook_events").fetchone() == ("session-1",) +@pytest.mark.asyncio +async def test_live_watcher_drains_hook_spool_from_added_directory_notification( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """An added hook shard drains immediately without waiting for catch-up.""" + + spool_root = tmp_path / "hooks" + pending = pending_hook_spool_dir(spool_root) + archive_root = tmp_path / "archive" + archive_root.mkdir() + watcher = LiveWatcher( + cast(Any, SimpleNamespace(archive_root=archive_root, backend=None)), + (WatchSource(name="hooks", root=pending, suffixes=(".json",)),), + cursor=CursorStore(archive_root / "ops.db"), + ) + + async def emit_added_shard(*roots: Path, **_kwargs: object) -> AsyncIterator[set[tuple[Change, str]]]: + assert roots == (pending,) + event_path = enqueue_hook_event( + event_id="directory-notification", + provider="codex", + event_type="SessionStart", + session_id="session-1", + timestamp="2026-07-12T10:00:00Z", + payload={"cwd": "/workspace"}, + root=spool_root, + ) + yield {(Change.added, str(event_path.parent))} + + monkeypatch.setattr(watchfiles, "awatch", emit_added_shard) + + await watcher._watch_changes([pending]) + watcher.stop() + + assert list(acknowledged_hook_spool_dir(spool_root).rglob("directory-notification.json")) != [] + with sqlite3.connect(archive_root / "source.db") as conn: + assert conn.execute("SELECT session_native_id FROM raw_hook_events").fetchone() == ("session-1",) + + +@pytest.mark.uses_real_clock("exercises production retry polling against a delayed atomic hook publish") +@pytest.mark.asyncio +async def test_live_watcher_retries_added_hook_shard_until_atomic_publish( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A late first envelope drains without its own child-file notification.""" + + spool_root = tmp_path / "hooks" + pending = pending_hook_spool_dir(spool_root) + archive_root = tmp_path / "archive" + archive_root.mkdir() + watcher = LiveWatcher( + cast(Any, SimpleNamespace(archive_root=archive_root, backend=None)), + (WatchSource(name="hooks", root=pending, suffixes=(".json",)),), + cursor=CursorStore(archive_root / "ops.db"), + ) + monkeypatch.setattr("polylogue.sources.hooks._day_shard", lambda: "2026-08-12") + shard = pending / "2026-08-12" + publish_task: asyncio.Task[None] | None = None + + async def publish_after_fixed_grace() -> None: + # The old one-shot 50 ms re-drain has already completed by the time + # this producer publishes. The retry must keep watching this shard. + await asyncio.sleep(0.10) + enqueue_hook_event( + event_id="published-after-directory-event", + provider="codex", + event_type="SessionStart", + session_id="session-2", + timestamp="2026-07-12T10:00:00Z", + payload={"cwd": "/workspace"}, + root=spool_root, + ) + + async def emit_empty_shard(*roots: Path, **_kwargs: object) -> AsyncIterator[set[tuple[Change, str]]]: + nonlocal publish_task + assert roots == (pending,) + shard.mkdir(parents=True) + publish_task = asyncio.create_task(publish_after_fixed_grace()) + yield {(Change.added, str(shard))} + + monkeypatch.setattr(watchfiles, "awatch", emit_empty_shard) + + await watcher._watch_changes([pending]) + assert publish_task is not None + await publish_task + retry_task = watcher._hook_spool_directory_retry_tasks[shard.resolve()] + await asyncio.wait_for(retry_task, timeout=1.0) + + assert list(acknowledged_hook_spool_dir(spool_root).rglob("published-after-directory-event.json")) != [] + with sqlite3.connect(archive_root / "source.db") as conn: + assert conn.execute("SELECT session_native_id FROM raw_hook_events").fetchone() == ("session-2",) + + +@pytest.mark.asyncio +async def test_hook_shard_retry_replacement_remains_tracked_until_stop( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """An old done callback cannot untrack its replacement retry task.""" + + class ControlledTask: + def __init__(self) -> None: + self._done = False + self._cancelled = False + self.callbacks: list[Callable[[ControlledTask], None]] = [] + + def done(self) -> bool: + return self._done + + def cancel(self) -> None: + self._cancelled = True + + def cancelled(self) -> bool: + return self._cancelled + + def result(self) -> None: + return None + + def add_done_callback(self, callback: Callable[[ControlledTask], None]) -> None: + self.callbacks.append(callback) + + def finish(self) -> None: + self._done = True + for callback in self.callbacks: + callback(self) + + created: list[ControlledTask] = [] + + def create_task(coro: Coroutine[Any, Any, None]) -> ControlledTask: + coro.close() + task = ControlledTask() + created.append(task) + return task + + spool_root = tmp_path / "hooks" + watcher = LiveWatcher( + cast(Any, SimpleNamespace(archive_root=tmp_path / "archive", backend=None)), + (WatchSource(name="hooks", root=pending_hook_spool_dir(spool_root), suffixes=(".json",)),), + cursor=CursorStore(tmp_path / "archive" / "ops.db"), + ) + monkeypatch.setattr(asyncio, "create_task", create_task) + directory = pending_hook_spool_dir(spool_root) / "2026-08-13" + + watcher._schedule_hook_spool_directory_retry(directory) + first = created[0] + first._done = True + watcher._schedule_hook_spool_directory_retry(directory) + replacement = created[1] + first.finish() + + tracked_replacement = cast(object, watcher._hook_spool_directory_retry_tasks[directory.resolve()]) + assert tracked_replacement is replacement + watcher.stop() + assert replacement.cancelled() is True + + +@pytest.mark.uses_real_clock("exercises retry polling while a pending hook envelope remains unacknowledged") +@pytest.mark.asyncio +async def test_hook_shard_retry_waits_for_durable_acknowledgement( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A failed drain keeps the directory retry alive until the envelope moves.""" + + directory = tmp_path / "hooks" / "pending" / "2026-08-12" + directory.mkdir(parents=True) + envelope = directory / "retry.json" + envelope.write_text("{}", encoding="utf-8") + watcher = LiveWatcher( + cast(Any, SimpleNamespace(archive_root=tmp_path / "archive", backend=None)), + (WatchSource(name="hooks", root=directory.parent, suffixes=(".json",)),), + cursor=CursorStore(tmp_path / "ops.db"), + ) + drains = 0 + + async def drain_until_acknowledged() -> None: + nonlocal drains + drains += 1 + if drains == 2: + envelope.unlink() + + monkeypatch.setattr(watcher, "_drain_hook_spool", drain_until_acknowledged) + + await watcher._retry_hook_spool_directory_until_populated(directory) + + assert drains == 2 + assert not envelope.exists() + + def test_hook_spool_retains_sqlite_failures_for_retry( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -295,6 +488,44 @@ def fail_persistence(*_args: object, **_kwargs: object) -> None: assert event_path.exists() +def test_hook_spool_drain_remains_source_only_when_derived_generation_is_unavailable(tmp_path: Path) -> None: + from polylogue.core.degraded import DegradedReason, clear_degraded, set_degraded + + archive_root = tmp_path / "archive" + spool_root = tmp_path / "hooks" + initialize_active_archive_root(archive_root) + pointer = archive_root / ".index-active-pointer" + pointer.write_bytes(b"\xff") + enqueue_hook_event( + event_id="derived-only-hook", + provider="codex", + event_type="PostToolUse", + session_id="session-1", + timestamp="2026-08-13T05:00:00Z", + payload={"tool_name": "exec"}, + root=spool_root, + ) + set_degraded( + DegradedReason( + code="schema_version_mismatch", + message="derived generation unavailable", + derived_only=True, + ) + ) + try: + result = drain_hook_event_spool(archive_root, root=spool_root) + finally: + clear_degraded() + + assert result.acknowledged == 1 + assert pointer.read_bytes() == b"\xff" + with sqlite3.connect(archive_root / "source.db") as conn: + assert conn.execute( + "SELECT hook_event_id FROM raw_hook_events WHERE hook_event_id = ?", + ("hook:derived-only-hook",), + ).fetchone() == ("hook:derived-only-hook",) + + @pytest.mark.parametrize( ("provider", "session_id"), [("claude-code", "claude-session"), ("codex", "codex-session")], @@ -551,8 +782,6 @@ def test_drain_opens_archive_once_per_pass_and_honors_limit( ) -> None: """One archive open per drain pass (never per record), bounded by limit, with remaining telling the caller to drain again.""" - from types import SimpleNamespace - from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore spool_root = tmp_path / "hooks" @@ -568,17 +797,16 @@ def test_drain_opens_archive_once_per_pass_and_honors_limit( ) archive_root = tmp_path / "archive" open_calls = 0 - real_open = ArchiveStore.open_existing + from polylogue.sources.live import archive_open + + real_open = archive_open._open_archive_for_live_write - def counting_open(root: Path, *, read_only: bool = True, read_timeout: float = 5.0) -> ArchiveStore: + def counting_open(root: Path) -> ArchiveStore: nonlocal open_calls open_calls += 1 - return real_open(root, read_only=read_only, read_timeout=read_timeout) + return real_open(root) - monkeypatch.setattr( - "polylogue.sources.hooks.ArchiveStore", - SimpleNamespace(open_existing=counting_open), - ) + monkeypatch.setattr(archive_open, "_open_archive_for_live_write", counting_open) first = drain_hook_event_spool(archive_root, root=spool_root, limit=2) assert first.acknowledged == 2 diff --git a/tests/unit/sources/test_live_batch_support.py b/tests/unit/sources/test_live_batch_support.py index 513df715ce..13369a5936 100644 --- a/tests/unit/sources/test_live_batch_support.py +++ b/tests/unit/sources/test_live_batch_support.py @@ -5,6 +5,7 @@ import json import os import sqlite3 +import zipfile from dataclasses import replace from hashlib import sha256 from pathlib import Path @@ -14,6 +15,7 @@ import pytest import polylogue.sources.live.watcher as live_watcher +from polylogue.archive.artifact_taxonomy import classify_artifact_path from polylogue.archive.message.roles import Role from polylogue.archive.revision_authority import ( HISTORICAL_NON_PREFIX_GOVERNANCE_DETAIL, @@ -30,6 +32,7 @@ from polylogue.sources.live.append_ingest import ingest_append_plans from polylogue.sources.live.batch import ( _MAX_APPEND_PLAN_PAYLOAD_BYTES, + CursorAuthorityBlockedError, LiveBatchProcessor, _ArchiveFullWriteResult, append_capability_receipt, @@ -50,6 +53,12 @@ ) from polylogue.sources.live.cursor import CursorStore from polylogue.sources.parsers.base import ParsedMessage, ParsedSession +from polylogue.sources.revision_backfill import ( + backfill_historical_revision_evidence, + validate_frozen_source_authority, +) +from polylogue.sources.source_acquisition_components import stream_preserved_zip_entry_raw_data +from polylogue.sources.source_parsing import has_decoded_session_evidence from polylogue.storage.blob_store import BlobStore from polylogue.storage.raw_authority import RAW_AUTHORITY_PARSER_FINGERPRINT from polylogue.storage.raw_failure_lifecycle import read_raw_failure_lifecycle @@ -102,8 +111,6 @@ def test_append_capability_receipt_is_keyed_to_live_identity_contract( initialize_active_archive_root, initialize_archive_database, ) -from polylogue.storage.sqlite.archive_tiers.index import INDEX_SCHEMA_VERSION -from polylogue.storage.sqlite.archive_tiers.source import SOURCE_SCHEMA_VERSION from polylogue.storage.sqlite.archive_tiers.source_write import ( ArchiveSourceArtifact, read_archive_raw_session_envelope, @@ -114,6 +121,13 @@ def test_append_capability_receipt_is_keyed_to_live_identity_contract( _ARCHIVE_STORAGE_TIERS = ",".join(spec.tier.value for spec in ARCHIVE_TIER_SPECS.values()) +def _complete_archive_storage_probe_fields() -> dict[str, object]: + return _archive_storage_probe_fields( + present={spec.tier for spec in ARCHIVE_TIER_SPECS.values()}, + versions={spec.tier: spec.version for spec in ARCHIVE_TIER_SPECS.values()}, + ) + + def _archive_storage_probe_fields( *, present: set[ArchiveTier], @@ -212,7 +226,7 @@ def _seed_live_append_plan( archive_root: Path, *, native_id: str, -) -> tuple[Path, _AppendPlan, object]: +) -> tuple[Path, _AppendPlan, object, LiveBatchProcessor]: root = archive_root / "sessions" root.mkdir() path = root / f"{native_id}.jsonl" @@ -240,7 +254,37 @@ def _seed_live_append_plan( handle.write(append) plan = processor._append_plan(path) assert isinstance(plan, _AppendPlan) - return path, plan, _append_owner(archive_root) + return path, plan, _append_owner(archive_root), processor + + +def _seed_claude_live_append_plan( + archive_root: Path, + *, + native_id: str, + append: bytes, +) -> tuple[Path, _AppendPlan, object, LiveBatchProcessor]: + root = archive_root / "claude-projects" + root.mkdir() + path = root / f"{native_id}.jsonl" + baseline = ( + f'{{"parentUuid":null,"type":"user","message":{{"role":"user","content":"zero"}},' + f'"uuid":"message-0","timestamp":"2026-06-02T00:00:00Z","sessionId":"{native_id}"}}\n' + ).encode() + path.write_bytes(baseline) + index_db = archive_root / "index.db" + processor = LiveBatchProcessor( + cast(Any, SimpleNamespace(archive_root=archive_root, backend=SimpleNamespace(db_path=index_db))), + (WatchSource(name="claude-code", root=root),), + cursor=CursorStore(index_db), + parser_fingerprint="test-parser", + ) + seeded = asyncio.run(processor.ingest_files([path], emit_event=False)) + assert seeded.succeeded_file_count == 1 + with path.open("ab") as handle: + handle.write(append) + plan = processor._append_plan(path) + assert isinstance(plan, _AppendPlan) + return path, plan, _append_owner(archive_root), processor def test_live_append_replay_streams_retained_jsonl_raw( @@ -250,7 +294,7 @@ def test_live_append_replay_streams_retained_jsonl_raw( """Append replay must not resurrect eager blob materialization.""" from polylogue.storage.blob_publication import ArchiveBlobPublisher - _path, plan, owner = _seed_live_append_plan(tmp_path, native_id="streamed-append") + _path, plan, owner, _processor = _seed_live_append_plan(tmp_path, native_id="streamed-append") monkeypatch.setattr( ArchiveBlobPublisher, "read_all", @@ -263,6 +307,221 @@ def test_live_append_replay_streams_retained_jsonl_raw( assert result.failed == [] +def test_live_append_acquires_with_unreadable_active_pointer( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + from polylogue.core.degraded import DegradedReason, clear_degraded, set_degraded + + _path, plan, owner, _processor = _seed_live_append_plan(tmp_path, native_id="degraded-append") + (tmp_path / ".index-active-pointer").write_bytes(b"\xff") + set_degraded( + DegradedReason( + code="schema_version_mismatch", + message="derived generation unavailable", + derived_only=True, + ) + ) + monkeypatch.setattr( + "polylogue.sources.dispatch.parse_stream_payload", + lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("source-only append must not parse")), + ) + try: + result = ingest_append_plans(cast(Any, owner), [plan]) + finally: + clear_degraded() + + assert result.succeeded == [plan] + assert result.failed == [] + with sqlite3.connect(tmp_path / "source.db") as conn: + append_row = conn.execute( + """ + SELECT logical_source_key, revision_kind, predecessor_raw_id, + baseline_raw_id, append_start_offset, append_end_offset, + revision_authority + FROM raw_sessions + WHERE source_index = -1 + """ + ).fetchone() + assert append_row is not None + assert append_row[:2] == ("codex:degraded-append", "append") + assert append_row[2] is not None + assert append_row[3] is not None + assert append_row[4:] == ( + plan.start_offset, + plan.last_complete_newline, + "byte_proven", + ) + + +def test_source_only_file_history_append_binds_before_artifact_classification(tmp_path: Path) -> None: + from polylogue.core.degraded import DegradedReason, clear_degraded, set_degraded + + native_id = "source-only-history" + append = ( + f'{{"type":"file-history-snapshot","sessionId":"{native_id}",' + '"uuid":"history-1","snapshot":{},"timestamp":"2026-06-02T00:00:01Z"}\n' + ).encode() + _path, plan, owner, _processor = _seed_claude_live_append_plan( + tmp_path, + native_id=native_id, + append=append, + ) + assert plan.native_id_hint == native_id + assert plan.acquisition_native_id_hint is None + set_degraded( + DegradedReason( + code="schema_version_mismatch", + message="derived generation unavailable", + derived_only=True, + ) + ) + try: + result = ingest_append_plans(cast(Any, owner), [plan]) + finally: + clear_degraded() + + assert result.succeeded == [plan] + assert result.failed == [] + assert result.deferred == [] + with sqlite3.connect(tmp_path / "source.db") as conn: + append_row = conn.execute( + """ + SELECT logical_source_key, revision_kind, predecessor_raw_id, + baseline_raw_id, append_start_offset, append_end_offset, + revision_authority, native_id + FROM raw_sessions + WHERE source_index = -1 + """ + ).fetchone() + artifact_count = conn.execute("SELECT COUNT(*) FROM raw_artifacts").fetchone() + assert append_row is not None + assert append_row[:2] == (f"claude-code:{native_id}", "append") + assert append_row[2] is not None + assert append_row[3] is not None + assert append_row[4:] == ( + plan.start_offset, + plan.last_complete_newline, + "byte_proven", + None, + ) + assert artifact_count == (0,) + + +def test_source_only_quarantined_append_is_deferred(tmp_path: Path) -> None: + from polylogue.core.degraded import DegradedReason, clear_degraded, set_degraded + + path = tmp_path / "quarantined-source-only.jsonl" + payload = ( + b'{"type":"response_item","payload":{"type":"message","id":"message-1",' + b'"role":"assistant","content":[{"type":"output_text","text":"one"}]}}\n' + ) + path.write_bytes(payload) + plan = replace( + _append_plan(path, payload, payload_hash=sha256(payload).hexdigest()), + native_id_hint="quarantined-source-only", + acquisition_native_id_hint="quarantined-source-only", + ) + set_degraded( + DegradedReason( + code="schema_version_mismatch", + message="derived generation unavailable", + derived_only=True, + ) + ) + try: + result = ingest_append_plans(cast(Any, _append_owner(tmp_path)), [plan]) + finally: + clear_degraded() + + assert result.succeeded == [] + assert result.failed == [] + assert result.deferred == [plan] + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute( + "SELECT logical_source_key, revision_kind, revision_authority FROM raw_sessions" + ).fetchone() == ("codex:quarantined-source-only", "append", "quarantined") + + +def test_claude_append_retry_preserves_legacy_null_acquisition_identity(tmp_path: Path) -> None: + native_id = "claude-legacy-append" + append = ( + f'{{"parentUuid":"message-0","type":"assistant","message":{{"role":"assistant",' + f'"content":[{{"type":"text","text":"one"}}]}},"uuid":"message-1",' + f'"timestamp":"2026-06-02T00:00:01Z","sessionId":"{native_id}"}}\n' + ).encode() + _path, plan, owner, _processor = _seed_claude_live_append_plan( + tmp_path, + native_id=native_id, + append=append, + ) + assert plan.native_id_hint == native_id + assert plan.acquisition_native_id_hint is None + + legacy_plan = replace(plan, native_id_hint=None, acquisition_native_id_hint=None) + first = ingest_append_plans(cast(Any, owner), [legacy_plan]) + assert first.succeeded == [legacy_plan] + assert first.failed == [] + with sqlite3.connect(tmp_path / "source.db") as conn: + before_retry = conn.execute("SELECT raw_id, native_id FROM raw_sessions WHERE source_index = -1").fetchall() + assert len(before_retry) == 1 + assert before_retry[0][1] is None + + retry = ingest_append_plans(cast(Any, owner), [plan]) + + assert retry.succeeded == [plan] + assert retry.failed == [] + assert retry.deferred == [] + with sqlite3.connect(tmp_path / "source.db") as conn: + after_retry = conn.execute( + "SELECT raw_id, native_id, revision_authority FROM raw_sessions WHERE source_index = -1" + ).fetchall() + assert after_retry == [(before_retry[0][0], None, "byte_proven")] + + +def test_derived_only_live_append_candidate_uses_source_acquisition(tmp_path: Path) -> None: + """The managed batch route must not plan an index-backed append while derived-only.""" + + import hashlib + + from polylogue.core.degraded import DegradedReason, clear_degraded, set_degraded + + path, _plan, _owner, processor = _seed_live_append_plan(tmp_path, native_id="degraded-managed-append") + index_db = tmp_path / "index.db" + index_digest_before = hashlib.sha256(index_db.read_bytes()).hexdigest() + with sqlite3.connect(tmp_path / "source.db") as conn: + raw_count_before = int( + conn.execute("SELECT COUNT(*) FROM raw_sessions WHERE source_path = ?", (str(path),)).fetchone()[0] + ) + pointer = tmp_path / ".index-active-pointer" + pointer.write_bytes(b"\xff") + set_degraded( + DegradedReason( + code="schema_version_mismatch", + message="derived generation unavailable", + derived_only=True, + ) + ) + try: + metrics = asyncio.run(processor.ingest_files([path], emit_event=False)) + finally: + clear_degraded() + + assert metrics.succeeded_file_count == 1 + assert metrics.append_file_count == 0 + assert metrics.full_file_count == 1 + assert pointer.read_bytes() == b"\xff" + assert hashlib.sha256(index_db.read_bytes()).hexdigest() == index_digest_before + with sqlite3.connect(tmp_path / "source.db") as conn: + rows = conn.execute( + """SELECT parsed_at_ms, parse_error FROM raw_sessions + WHERE source_path = ? ORDER BY acquired_at_ms DESC, raw_id DESC""", + (str(path),), + ).fetchall() + assert len(rows) == raw_count_before + 1 + assert rows[0] == (None, None) + + def test_live_full_replay_streams_retained_jsonl_raw( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -305,6 +564,7 @@ def test_live_full_replay_streams_retained_jsonl_raw( def test_full_ingest_acquires_but_does_not_parse_when_derived_tier_degraded( tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, ) -> None: """polylogue-gbs02: a derived-only degraded reason must still acquire raw content. @@ -327,30 +587,637 @@ def test_full_ingest_acquires_but_does_not_parse_when_derived_tier_degraded( b'{"type":"response_item","payload":{"type":"message","id":"message-0","role":"user",' b'"content":[{"type":"input_text","text":"zero"}]}}\n' ) + json_path = root / "degraded-full.json" + json_path.write_bytes(b'{"mapping":{"root":{"message":{"author":{"role":"user"}}}}}') + classified_path = root / "subagents" / "worker" / "agent-degraded.meta.json" + classified_path.parent.mkdir(parents=True) + classified_path.write_bytes(b'{"mapping":{"root":{"message":{"author":{"role":"user"}}}}}') + index_db = tmp_path / "index.db" + processor = LiveBatchProcessor( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=index_db))), + (WatchSource(name="claude-code", root=root),), + cursor=CursorStore(index_db), + parser_fingerprint="test-parser", + ) + set_degraded( + DegradedReason( + code="schema_version_mismatch", + message="index.db:46!=57", + derived_only=True, + ) + ) + monkeypatch.setattr( + "polylogue.sources.live.batch._parse_payload_as_session_artifact", + lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("must not decode source-only evidence")), + ) + monkeypatch.setattr( + "polylogue.sources.live.batch._jsonl_provider_and_session_artifact", + lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("must not classify source-only JSONL")), + ) + monkeypatch.setattr( + "polylogue.sources.live.batch.has_decoded_session_evidence", + lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("must not inspect source-only JSON evidence")), + ) + monkeypatch.setattr( + "polylogue.sources.live.batch._detect_provider_from_raw_bytes", + lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("must not detect source-only provider")), + ) + try: + result = processor._ingest_full_paths_sync([path, json_path, classified_path], source_name="claude-code") + finally: + clear_degraded() + + assert result.succeeded == [path, json_path, classified_path] + assert result.failed == [] + with sqlite3.connect(tmp_path / "source.db") as conn: + raw_states = conn.execute("SELECT parsed_at_ms, parse_error FROM raw_sessions ORDER BY source_path").fetchall() + artifact_rows = conn.execute( + "SELECT COUNT(*) FROM raw_artifacts WHERE source_path = ?", (str(classified_path),) + ).fetchone() + assert raw_states == [(None, None), (None, None), (None, None)] + assert artifact_rows == (0,) + + +def test_source_only_full_ingest_refuses_missing_durable_source_tier(tmp_path: Path) -> None: + """An established archive cannot silently bootstrap over source.db loss.""" + from polylogue.core.degraded import DegradedReason, clear_degraded, set_degraded + + initialize_active_archive_root(tmp_path) + (tmp_path / "source.db").unlink() + root = tmp_path / "sessions" + root.mkdir() + path = root / "pending.jsonl" + path.write_text('{"opaque":"must remain pending"}\n', encoding="utf-8") + index_db = tmp_path / "index.db" + processor = LiveBatchProcessor( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=index_db))), + (WatchSource(name="claude-code", root=root),), + cursor=CursorStore(index_db), + parser_fingerprint="test-parser", + ) + + set_degraded(DegradedReason(code="schema_version_mismatch", message="index unavailable", derived_only=True)) + try: + result = processor._ingest_full_paths_sync([path], source_name="claude-code") + finally: + clear_degraded() + + assert result.succeeded == [] + assert result.failed == [path] + assert result.source_payload_read_bytes == 0 + assert not (tmp_path / "source.db").exists() + + +def test_source_only_antigravity_metadata_stays_pending_with_mutable_companion(tmp_path: Path) -> None: + """Cursor authority cannot cover metadata while omitting its sibling bytes.""" + from polylogue.core.degraded import DegradedReason, clear_degraded, set_degraded + + initialize_active_archive_root(tmp_path) + root = tmp_path / "antigravity" + metadata = root / "brain" / "work-session" / "plan.md.metadata.json" + metadata.parent.mkdir(parents=True) + metadata.write_text('{"summary":"plan"}', encoding="utf-8") + companion = metadata.with_name("plan.md") + companion.write_text("contemporaneous body", encoding="utf-8") + index_db = tmp_path / "index.db" + cursor = CursorStore(index_db) + processor = LiveBatchProcessor( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=index_db))), + (WatchSource(name="antigravity", root=root),), + cursor=cursor, + parser_fingerprint="test-parser", + ) + + set_degraded(DegradedReason(code="schema_version_mismatch", message="index unavailable", derived_only=True)) + try: + metrics = asyncio.run(processor.ingest_files([metadata], emit_event=False)) + finally: + clear_degraded() + + assert metrics.succeeded_file_count == 0 + assert metrics.failed_paths == [str(metadata)] + cursor_record = cursor.get_record(metadata) + assert cursor_record is not None + assert cursor_record.excluded is False + assert cursor_record.failure_count == 1 + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT COUNT(*) FROM raw_sessions").fetchone() == (0,) + + +def test_source_only_full_ingest_streams_admitted_zip_members_without_decoding( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The production full-ingest ZIP route must retain bytes before decode.""" + from polylogue.core.degraded import DegradedReason, clear_degraded, set_degraded + + initialize_active_archive_root(tmp_path) + root = tmp_path / "sessions" + root.mkdir() + bundle = root / "degraded.zip" + member_names = ("sessions/one.jsonl", "sessions/two.json") + with zipfile.ZipFile(bundle, "w") as zf: + zf.writestr(member_names[0], b'{"opaque":"first"}\n') + zf.writestr(member_names[1], b'{"opaque":"second"}') + index_db = tmp_path / "index.db" + processor = LiveBatchProcessor( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=index_db))), + (WatchSource(name="claude-code", root=root),), + cursor=CursorStore(index_db), + parser_fingerprint="test-parser", + ) + set_degraded(DegradedReason(code="schema_version_mismatch", message="index unavailable", derived_only=True)) + for target in ( + "polylogue.sources.live.batch.iter_zip_entry_raw_data", + "polylogue.sources.live.batch.LiveBatchProcessor._sniff_zip_provider", + "polylogue.sources.live.batch._detect_provider_from_raw_bytes", + "polylogue.sources.source_acquisition_components.iter_entry_payloads", + "polylogue.sources.source_acquisition_components.classify_artifact", + ): + monkeypatch.setattr( + target, lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("must not decode ZIP")) + ) + try: + result = processor._ingest_full_paths_sync([bundle], source_name="claude-code") + finally: + clear_degraded() + + assert result.succeeded == [bundle] + assert result.failed == [] + with sqlite3.connect(tmp_path / "source.db") as conn: + rows = conn.execute( + "SELECT source_path, source_index, parsed_at_ms, parse_error FROM raw_sessions ORDER BY source_index" + ).fetchall() + assert rows == [ + (f"{bundle}:{member_names[0]}", 0, None, None), + (f"{bundle}:{member_names[1]}", 1, None, None), + ] + + +def test_source_only_full_ingest_bounds_oversized_ndjson_sampling( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The production NDJSON route reaches streaming retention before eager decode.""" + from polylogue.core.degraded import DegradedReason, clear_degraded, set_degraded + + initialize_active_archive_root(tmp_path) + root = tmp_path / "inbox" + root.mkdir() + source = root / "oversized.ndjson" + payload = ( + json.dumps( + { + "type": "session_meta", + "payload": {"id": "oversized-record", "padding": "x" * 128_000}, + } + ).encode() + + b"\n" + ) + source.write_bytes(payload) + processor = LiveBatchProcessor( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=tmp_path / "index.db"))), + (WatchSource(name="inbox", root=root, suffixes=(".ndjson",)),), + cursor=CursorStore(tmp_path / "index.db"), + parser_fingerprint="test-parser", + ) + monkeypatch.setattr("polylogue.sources.live.batch._STREAMING_FULL_INGEST_BYTES", 1) + monkeypatch.setattr( + "polylogue.sources.live.batch_support.json_loads", + lambda _raw: (_ for _ in ()).throw(AssertionError("sampling must not decode an oversized physical record")), + ) + + set_degraded(DegradedReason(code="schema_version_mismatch", message="index unavailable", derived_only=True)) + try: + result = processor._ingest_full_paths_sync([source], source_name="inbox") + finally: + clear_degraded() + + assert result.succeeded == [source] + assert result.failed == [] + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT origin, blob_size, parsed_at_ms, parse_error FROM raw_sessions").fetchall() == [ + ("unknown-export", len(payload), None, None) + ] + + +def test_source_only_zip_read_failure_remains_retryable_after_partial_copy( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The real source-only route must not exclude a transiently unreadable ZIP.""" + from polylogue.core.degraded import DegradedReason, clear_degraded, set_degraded + + initialize_active_archive_root(tmp_path) + root = tmp_path / "sessions" + root.mkdir() + bundle = root / "retry.zip" + member_names = ("sessions/one.jsonl", "sessions/two.jsonl") + with zipfile.ZipFile(bundle, "w") as zf: + zf.writestr(member_names[0], b'{"opaque":"first"}\n') + zf.writestr(member_names[1], b'{"opaque":"second"}\n') + index_db = tmp_path / "index.db" + cursor = CursorStore(index_db) + processor = LiveBatchProcessor( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=index_db))), + (WatchSource(name="claude-code", root=root),), + cursor=cursor, + parser_fingerprint="test-parser", + ) + original_stream = stream_preserved_zip_entry_raw_data + calls = 0 + + def fail_after_first_copy(*args: Any, **kwargs: Any) -> Any: + nonlocal calls + calls += 1 + if calls == 2: + raise OSError("transient ZIP read failure") + return original_stream(*args, **kwargs) + + monkeypatch.setattr( + "polylogue.sources.live.batch.stream_preserved_zip_entry_raw_data", + fail_after_first_copy, + ) + set_degraded(DegradedReason(code="schema_version_mismatch", message="index unavailable", derived_only=True)) + try: + failed = asyncio.run(processor.ingest_files([bundle], emit_event=False)) + + assert failed.succeeded_file_count == 0 + assert failed.failed_file_count == 1 + failed_cursor = cursor.get_record(bundle) + assert failed_cursor is not None + assert failed_cursor.failure_count == 1 + assert failed_cursor.excluded is False + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT COUNT(*) FROM raw_sessions").fetchone() == (0,) + + retried = asyncio.run(processor.ingest_files([bundle], emit_event=False)) + finally: + clear_degraded() + + assert retried.succeeded_file_count == 1 + assert retried.failed_file_count == 0 + recovered_cursor = cursor.get_record(bundle) + assert recovered_cursor is not None + assert recovered_cursor.failure_count == 0 + assert recovered_cursor.excluded is False + with sqlite3.connect(tmp_path / "source.db") as conn: + retained = conn.execute("SELECT source_path, source_index FROM raw_sessions ORDER BY source_index").fetchall() + assert retained == [ + (f"{bundle}:{member_names[0]}", 0), + (f"{bundle}:{member_names[1]}", 1), + ] + + +def test_source_only_zip_replay_resolves_unknown_chatgpt_member_and_keeps_duplicate_coordinates( + tmp_path: Path, +) -> None: + """Recovery, not acquisition, resolves UNKNOWN ZIP bytes and replays each coordinate.""" + from polylogue.core.degraded import DegradedReason, clear_degraded, set_degraded + + initialize_active_archive_root(tmp_path) + root = tmp_path / "inbox" + root.mkdir() + bundle = root / "export.zip" + payload = json.dumps( + [ + { + "id": "zip-chatgpt", + "conversation_id": "zip-chatgpt", + "title": "ZIP recovery", + "create_time": 1_700_000_000, + "update_time": 1_700_000_001, + "current_node": "assistant-node", + "mapping": { + "user-node": { + "id": "user-node", + "parent": None, + "children": ["assistant-node"], + "message": { + "id": "user-message", + "author": {"role": "user"}, + "content": {"content_type": "text", "parts": ["recover ZIP"]}, + "create_time": 1_700_000_000, + }, + }, + "assistant-node": { + "id": "assistant-node", + "parent": "user-node", + "children": [], + "message": { + "id": "assistant-message", + "author": {"role": "assistant"}, + "content": {"content_type": "text", "parts": ["replayed"]}, + "create_time": 1_700_000_001, + }, + }, + }, + } + ], + sort_keys=True, + ).encode() + with zipfile.ZipFile(bundle, "w") as zf: + zf.writestr("first/conversations.json", payload) + zf.writestr("second/conversations.json", payload) + index_db = tmp_path / "index.db" + processor = LiveBatchProcessor( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=index_db))), + (WatchSource(name="unknown", root=root),), + cursor=CursorStore(index_db), + parser_fingerprint="test-parser", + ) + + set_degraded(DegradedReason(code="schema_version_mismatch", message="index unavailable", derived_only=True)) + try: + result = processor._ingest_full_paths_sync([bundle], source_name="unknown") + finally: + clear_degraded() + + assert result.succeeded == [bundle] + with sqlite3.connect(tmp_path / "source.db") as conn: + before_replay = conn.execute( + "SELECT raw_id, hex(blob_hash), source_path, source_index, origin FROM raw_sessions ORDER BY source_index" + ).fetchall() + assert len(before_replay) == 2 + assert len({row[0] for row in before_replay}) == 2 + assert len({row[1] for row in before_replay}) == 1 + assert [row[2:] for row in before_replay] == [ + (f"{bundle}:first/conversations.json", 0, "unknown-export"), + (f"{bundle}:second/conversations.json", 1, "unknown-export"), + ] + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute( + "SELECT raw_id, coordinate_format, entry_ordinal, split_index " + "FROM raw_container_coordinates ORDER BY entry_ordinal" + ).fetchall() == [ + (before_replay[0][0], "zip-v2", 0, 0), + (before_replay[1][0], "zip-v2", 1, 0), + ] + + replay = backfill_historical_revision_evidence(tmp_path) + + assert replay.replayed_logical_sources == 2 + with sqlite3.connect(tmp_path / "index.db") as conn: + assert conn.execute("SELECT native_id, message_count FROM sessions").fetchall() == [("zip-chatgpt", 2)] + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT origin, detected_provider FROM raw_sessions ORDER BY source_index").fetchall() == [ + ("unknown-export", "chatgpt"), + ("unknown-export", "chatgpt"), + ] + assert conn.execute( + "SELECT raw_id, coordinate_format, entry_ordinal, split_index " + "FROM raw_container_coordinates ORDER BY entry_ordinal" + ).fetchall() == [ + (before_replay[0][0], "zip-v2", 0, 0), + (before_replay[1][0], "zip-v2", 1, 0), + ] + + reobserved = processor._ingest_full_paths_sync([bundle], source_name="unknown") + + assert reobserved.succeeded == [bundle] + assert reobserved.failed == [] + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT origin, detected_provider FROM raw_sessions ORDER BY source_index").fetchall() == [ + ("unknown-export", "chatgpt"), + ("unknown-export", "chatgpt"), + ] + assert conn.execute( + "SELECT raw_id, coordinate_format, entry_ordinal, split_index " + "FROM raw_container_coordinates ORDER BY entry_ordinal" + ).fetchall() == [ + (before_replay[0][0], "zip-v2", 0, 0), + (before_replay[1][0], "zip-v2", 1, 0), + ] + + +def test_zip_duplicate_member_coordinates_match_normal_and_source_only_routes(tmp_path: Path) -> None: + """Central-directory ordinal and within-member split remain independent.""" + root = tmp_path / "inbox" + root.mkdir() + bundle = root / "duplicates.zip" + member_name = "sessions/duplicate.jsonl" + payload = ( + b'{"type":"session_meta","payload":{"id":"duplicate-coordinate"}}\n' + b'{"type":"response_item","payload":{"type":"message","role":"user",' + b'"content":[{"type":"input_text","text":"retained twice"}]}}\n' + ) + with zipfile.ZipFile(bundle, "w") as zf: + zf.writestr("ignored/readme.txt", b"not admitted") + zf.writestr(member_name, payload) + with pytest.warns(UserWarning, match="Duplicate name"): + zf.writestr(member_name, payload) + + index_db = tmp_path / "index.db" + processor = LiveBatchProcessor( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=index_db))), + (WatchSource(name="codex", root=root),), + cursor=CursorStore(index_db), + parser_fingerprint="test-parser", + ) + blob_store = BlobStore(tmp_path / "blob") + + normal_records, _normal_bytes = processor._extract_zip_member_records( + bundle, + blob_store=blob_store, + fallback_provider=Provider.CODEX, + file_mtime="2026-08-13T00:00:00+00:00", + ) + source_only_result = processor._extract_source_only_zip_member_records( + bundle, + blob_store=blob_store, + fallback_provider=Provider.CODEX, + file_mtime="2026-08-13T00:00:00+00:00", + ) + + assert source_only_result is not None + source_only_records, _source_only_bytes = source_only_result + normal_ids = [raw_id for raw_id, _record in normal_records] + source_only_ids = [raw_id for raw_id, _record in source_only_records] + assert len(normal_ids) == 2 + assert len(set(normal_ids)) == 2 + assert source_only_ids == normal_ids + assert [record.source_index for _raw_id, record in normal_records] == [1, 3] + assert [record.source_index for _raw_id, record in source_only_records] == [1, 3] + + +def test_source_only_full_ingest_snapshots_unrecognized_codex_state_without_shape_probe( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A degraded source tier retains a valid but future-shaped Codex state DB.""" + from polylogue.core.degraded import DegradedReason, clear_degraded, set_degraded + + initialize_active_archive_root(tmp_path) + root = tmp_path / "codex" + root.mkdir() + state_db = root / "state_5.sqlite" + _write_plain_sqlite_db(state_db) + index_db = tmp_path / "index.db" + processor = LiveBatchProcessor( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=index_db))), + (WatchSource(name="codex", root=root),), + cursor=CursorStore(index_db), + parser_fingerprint="test-parser", + ) + set_degraded(DegradedReason(code="schema_version_mismatch", message="index unavailable", derived_only=True)) + monkeypatch.setattr( + "polylogue.sources.parsers.codex_state.is_in_scope_codex_sqlite_path", + lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("must not inspect source-only state schema")), + ) + try: + result = processor._ingest_full_paths_sync([state_db], source_name="codex") + finally: + clear_degraded() + + assert result.succeeded == [state_db] + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT source_path, parsed_at_ms FROM raw_sessions").fetchall() == [(str(state_db), None)] + + +def test_source_only_foreign_sqlite_name_cannot_claim_codex_authority(tmp_path: Path) -> None: + """A foreign watch source cannot turn a filename into Codex authority.""" + from polylogue.core.degraded import DegradedReason, clear_degraded, set_degraded + + initialize_active_archive_root(tmp_path) + root = tmp_path / "inbox" + state_db = root / "state_5.sqlite" + _write_plain_sqlite_db(state_db) + index_db = tmp_path / "index.db" + processor = LiveBatchProcessor( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=index_db))), + (WatchSource(name="inbox", root=root, suffixes=(".sqlite",)),), + cursor=CursorStore(index_db), + parser_fingerprint="test-parser", + ) + + set_degraded(DegradedReason(code="schema_version_mismatch", message="index unavailable", derived_only=True)) + try: + result = processor._ingest_full_paths_sync([state_db], source_name="inbox") + finally: + clear_degraded() + + assert result.succeeded == [] + assert result.failed == [] + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT origin FROM raw_sessions").fetchall() == [] + + +def _write_codex_thread_state_db(path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with sqlite3.connect(path) as conn: + conn.executescript( + """ + CREATE TABLE threads ( + id TEXT PRIMARY KEY, + title TEXT NOT NULL, + cwd TEXT NOT NULL, + created_at_ms INTEGER NOT NULL, + updated_at_ms INTEGER NOT NULL, + source TEXT NOT NULL, + model TEXT, + agent_nickname TEXT, + agent_role TEXT, + archived INTEGER NOT NULL DEFAULT 0 + ); + CREATE TABLE thread_spawn_edges ( + parent_thread_id TEXT NOT NULL, + child_thread_id TEXT NOT NULL PRIMARY KEY, + status TEXT NOT NULL + ); + """ + ) + conn.execute( + "INSERT INTO threads VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ("codex-thread", "Recover retained state", "/work", 1, 2, "cli", "gpt-5", None, None, 0), + ) + conn.execute( + "INSERT INTO thread_spawn_edges VALUES (?, ?, ?)", + ("codex-thread", "codex-child", "closed"), + ) + + +def test_source_only_codex_state_recovery_replays_retained_thread_evidence(tmp_path: Path) -> None: + """Frozen validation admits state as non-session before mutable replay.""" + from polylogue.core.degraded import DegradedReason, clear_degraded, set_degraded + + initialize_active_archive_root(tmp_path) + root = tmp_path / "codex" + state_db = root / "state_5.sqlite" + _write_codex_thread_state_db(state_db) + index_db = tmp_path / "index.db" + processor = LiveBatchProcessor( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=index_db))), + (WatchSource(name="codex", root=root),), + cursor=CursorStore(index_db), + parser_fingerprint="test-parser", + ) + + set_degraded(DegradedReason(code="schema_version_mismatch", message="index unavailable", derived_only=True)) + try: + assert processor._ingest_full_paths_sync([state_db], source_name="codex").succeeded == [state_db] + finally: + clear_degraded() + + source_before = sha256((tmp_path / "source.db").read_bytes()).hexdigest() + validate_frozen_source_authority(tmp_path) + assert sha256((tmp_path / "source.db").read_bytes()).hexdigest() == source_before + + replay = backfill_historical_revision_evidence(tmp_path) + + assert replay.scanned == 1 + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT parsed_at_ms IS NOT NULL FROM raw_sessions").fetchone() == (1,) + assert conn.execute( + "SELECT hook_event_id, event_type FROM raw_hook_events ORDER BY hook_event_id" + ).fetchall() == [ + ("codex-thread-spawn-edge:codex-thread:codex-child", "codex_thread_spawn_edge"), + ("codex-thread-title:codex-thread", "codex_thread_title"), + ] + + +@pytest.mark.parametrize("state_name", ["state.db", "verification_evidence.db"]) +def test_source_only_hermes_named_sqlite_uses_consistent_backup_before_generic_capture( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + state_name: str, +) -> None: + """A direct file copy loses an uncheckpointed WAL row; the snapshot retains it.""" + from polylogue.core.degraded import DegradedReason, clear_degraded, set_degraded + + initialize_active_archive_root(tmp_path) + root = tmp_path / "hermes" + state_db = root / state_name + state_db.parent.mkdir(parents=True) + writer = sqlite3.connect(state_db) + writer.execute("PRAGMA journal_mode=WAL") + writer.execute("CREATE TABLE retained_wal_row (value TEXT NOT NULL)") + writer.commit() + writer.execute("INSERT INTO retained_wal_row VALUES ('must survive')") + writer.commit() index_db = tmp_path / "index.db" processor = LiveBatchProcessor( cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=index_db))), - (WatchSource(name="codex", root=root),), + (WatchSource(name="hermes", root=root),), cursor=CursorStore(index_db), parser_fingerprint="test-parser", ) - set_degraded( - DegradedReason( - code="schema_version_mismatch", - message="index.db:46!=57", - derived_only=True, - ) + monkeypatch.setattr("polylogue.sources.parsers.hermes_state.looks_like_state_db_path", lambda *_a, **_k: False) + monkeypatch.setattr( + "polylogue.sources.parsers.hermes_verification.looks_like_verification_evidence_db_path", + lambda *_a, **_k: False, ) + + set_degraded(DegradedReason(code="schema_version_mismatch", message="index unavailable", derived_only=True)) try: - result = processor._ingest_full_paths_sync([path], source_name="codex") + assert processor._ingest_full_paths_sync([state_db], source_name="hermes").succeeded == [state_db] finally: clear_degraded() + writer.close() - assert result.succeeded == [path] - assert result.failed == [] - parsed_at_ms, parse_error = _raw_parse_state(tmp_path) - assert parsed_at_ms is None - assert parse_error is None + with sqlite3.connect(tmp_path / "source.db") as conn: + blob_hash = str(conn.execute("SELECT hex(blob_hash) FROM raw_sessions").fetchone()[0]).lower() + with sqlite3.connect(BlobStore(tmp_path / "blob").blob_path(blob_hash)) as snapshot: + assert snapshot.execute("SELECT value FROM retained_wal_row").fetchall() == [("must survive",)] def test_full_ingest_acquires_when_index_is_genuinely_semantic_distance_stale( @@ -394,6 +1261,8 @@ def test_full_ingest_acquires_when_index_is_genuinely_semantic_distance_stale( finally: conn.close() index_digest_before = hashlib.sha256(index_db.read_bytes()).hexdigest() + pointer = tmp_path / ".index-active-pointer" + pointer.write_bytes(b"\xff") processor = LiveBatchProcessor( cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=index_db))), @@ -409,17 +1278,66 @@ def test_full_ingest_acquires_when_index_is_genuinely_semantic_distance_stale( ) ) try: - result = processor._ingest_full_paths_sync([path], source_name="codex") + metrics = asyncio.run(processor.ingest_files([path], emit_event=False)) finally: clear_degraded() - assert result.succeeded == [path], f"failed={result.failed}" + assert metrics.succeeded_file_count == 1 + assert metrics.failed_file_count == 0 parsed_at_ms, parse_error = _raw_parse_state(tmp_path) assert parsed_at_ms is None assert parse_error is None assert hashlib.sha256(index_db.read_bytes()).hexdigest() == index_digest_before, ( "the stale index tier must never be opened for write during acquire-only ingest" ) + assert pointer.read_bytes() == b"\xff" + + +def test_live_raw_compaction_holds_generation_lease_through_delete( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The protected set and destructive cleanup observe one unpromotable generation.""" + + from polylogue.storage import raw_retention + from polylogue.storage.index_generation import RebuildLease, RebuildLeaseUnavailableError + from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root + + initialize_active_archive_root(tmp_path) + root = tmp_path / "sessions" + root.mkdir() + path = root / "session.jsonl" + path.write_text("{}\n", encoding="utf-8") + processor = LiveBatchProcessor( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=tmp_path / "index.db"))), + (WatchSource(name="codex", root=root),), + cursor=CursorStore(tmp_path / "ops.db"), + parser_fingerprint="test-parser", + ) + phases: list[str] = [] + + def assert_promotion_excluded(*_args: object, **_kwargs: object) -> SimpleNamespace: + with pytest.raises(RebuildLeaseUnavailableError): + with RebuildLease(tmp_path): + pass + phases.append("authority") + return SimpleNamespace(protected_raw_ids=frozenset(), eligible_raw_ids=frozenset()) + + def assert_delete_excluded(*_args: object, **_kwargs: object) -> SimpleNamespace: + with pytest.raises(RebuildLeaseUnavailableError): + with RebuildLease(tmp_path): + pass + phases.append("delete") + return SimpleNamespace(errors=()) + + monkeypatch.setattr(raw_retention, "active_raw_retention_authority", assert_promotion_excluded) + monkeypatch.setattr(raw_retention, "compact_paths_superseded_raw_snapshots", assert_delete_excluded) + + processor._compact_superseded_raw_snapshots([path]) + + assert phases == ["authority", "delete"] + with RebuildLease(tmp_path): + pass def test_full_ingest_empty_jsonl_is_not_misclassified_as_truncated( @@ -457,7 +1375,7 @@ def test_full_ingest_empty_jsonl_is_not_misclassified_as_truncated( def test_full_ingest_unknown_export_without_sessions_records_terminal_evidence(tmp_path: Path) -> None: - root = tmp_path / "unknown" + root = tmp_path / "chatgpt" root.mkdir() path = root / "export.jsonl" path.write_bytes(b"") @@ -477,6 +1395,107 @@ def test_full_ingest_unknown_export_without_sessions_records_terminal_evidence(t assert artifact == ("terminal_unknown_export_no_session", "unsupported_parseable", 0) +def test_full_ingest_unknown_weak_path_ndjson_records_terminal_evidence(tmp_path: Path) -> None: + """NDJSON takes the same strict terminal classification route as JSONL.""" + + root = tmp_path / "chatgpt" + path = root / "analysis" / "export.ndjson" + path.parent.mkdir(parents=True) + path.write_bytes(b"") + db_path = tmp_path / "archive.sqlite" + processor = LiveBatchProcessor( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=db_path))), + (WatchSource(name="unknown", root=root, suffixes=(".jsonl", ".ndjson")),), + cursor=CursorStore(db_path), + parser_fingerprint="test-parser", + ) + + result = processor._ingest_full_paths_sync([path], source_name="unknown") + + assert result.succeeded == [path] + with sqlite3.connect(tmp_path / "source.db") as conn: + artifact = conn.execute("SELECT artifact_kind, parse_as_session FROM raw_artifacts").fetchone() + assert artifact == ("terminal_unknown_export_no_session", 0) + + +@pytest.mark.parametrize( + ("payload", "expected_artifact"), + [ + (b"{", ("terminal_unknown_json_decode", "decode_failed")), + (b"", ("terminal_unknown_json_decode", "decode_failed")), + ], +) +def test_full_ingest_unknown_weak_path_json_retains_terminal_evidence( + tmp_path: Path, + payload: bytes, + expected_artifact: tuple[str, str], +) -> None: + """Unknown weak-path JSON reaches durable generic terminal handling.""" + + root = tmp_path / "unknown" + path = root / "analysis" / "export.json" + path.parent.mkdir(parents=True) + path.write_bytes(payload) + path_artifact = classify_artifact_path(path, provider=Provider.UNKNOWN) + assert path_artifact is not None and not path_artifact.parse_as_session + assert not has_decoded_session_evidence(path, provider=Provider.UNKNOWN) + + db_path = tmp_path / "archive.sqlite" + processor = LiveBatchProcessor( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=db_path))), + (WatchSource(name="unknown", root=root),), + cursor=CursorStore(db_path), + parser_fingerprint="test-parser", + ) + + result = processor._ingest_full_paths_sync([path], source_name="unknown") + + assert result.succeeded == [path] + assert result.failed == [] + with sqlite3.connect(tmp_path / "source.db") as conn: + raw = conn.execute( + "SELECT raw_id, blob_size, parse_error FROM raw_sessions WHERE source_path = ?", (str(path),) + ).fetchone() + artifact = conn.execute( + """ + SELECT artifact_kind, support_status + FROM raw_artifacts + WHERE raw_id = ? + """, + (raw[0],) if raw is not None else (None,), + ).fetchone() + # The preconditions above would take the weak path-exclusion branch if + # the production unknown-JSON exemption were removed. + assert raw is not None + assert raw[1] == len(payload) + assert isinstance(raw[2], str) + assert artifact == expected_artifact + + +def test_full_ingest_unknown_weak_directory_still_excludes_strong_sidecar(tmp_path: Path) -> None: + """A weak directory cannot override a definitive non-session filename.""" + + root = tmp_path / "unknown" + path = root / "analysis" / "sessions-index.json" + path.parent.mkdir(parents=True) + path.write_text('{"mapping":{"looks":"conversational"}}', encoding="utf-8") + path_artifact = classify_artifact_path(path, provider=Provider.UNKNOWN) + assert path_artifact is not None and path_artifact.kind.value == "metadata_document" + processor = LiveBatchProcessor( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=tmp_path / "ops.db"))), + (WatchSource(name="unknown", root=root, suffixes=(".json",)),), + cursor=CursorStore(tmp_path / "ops.db"), + parser_fingerprint="test-parser", + ) + + result = processor._ingest_full_paths_sync([path], source_name="unknown") + + assert result.succeeded == [] + assert result.failed == [] + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT COUNT(*) FROM raw_sessions").fetchone() == (0,) + + def test_full_ingest_unknown_malformed_jsonl_records_terminal_decode_and_stops_retrying(tmp_path: Path) -> None: """Complete malformed JSONL lines are terminal decode evidence, not no-session evidence.""" root = tmp_path / "unknown" @@ -653,6 +1672,43 @@ def grow_source_after_capture(**kwargs: object) -> bool: assert artifact == ("deferred_hot_jsonl_capture", "partial_decode", 1) +def test_full_ingest_applies_incomplete_record_guard_to_jsonl_txt( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The supported ``.jsonl.txt`` wire suffix has JSONL tail authority too.""" + from polylogue.sources.live import batch as live_batch + + root = tmp_path / "sessions" + root.mkdir() + path = root / "active.jsonl.txt" + captured = b'{"type":"session_meta"' + path.write_bytes(captured) + processor = LiveBatchProcessor( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=tmp_path / "archive.sqlite"))), + (WatchSource(name="codex", root=root),), + cursor=CursorStore(tmp_path / "archive.sqlite"), + parser_fingerprint="test-parser", + ) + monkeypatch.setattr( + "polylogue.sources.live.batch._jsonl_provider_and_session_artifact", + lambda _path, fallback_provider: (fallback_provider, True), + ) + boundary_check = live_batch._captured_jsonl_ends_at_record_boundary + + def grow_source_after_capture(**kwargs: object) -> bool: + path.write_bytes(captured + b"\n") + return boundary_check(**kwargs) # type: ignore[arg-type] + + monkeypatch.setattr(live_batch, "_captured_jsonl_ends_at_record_boundary", grow_source_after_capture) + + result = processor._ingest_full_paths_sync([path], source_name="codex") + + assert result.succeeded == [path] + _parsed_at_ms, parse_error = _raw_parse_state(tmp_path) + assert isinstance(parse_error, str) and parse_error.endswith("complete record boundary") + + def test_full_ingest_claude_partial_jsonl_has_provider_specific_evidence( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -691,7 +1747,7 @@ def grow_source_after_capture(**kwargs: object) -> bool: assert artifact == ("deferred_claude_code_partial_jsonl", "partial_decode", 1) -def test_streamed_incomplete_jsonl_capture_defers_then_replays_completed_source( +def test_streamed_incomplete_jsonl_capture_defers_completed_source_until_authority_recovers( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -741,16 +1797,16 @@ def complete_source_after_capture(**kwargs: object) -> bool: artifact = conn.execute("SELECT artifact_kind FROM raw_artifacts ORDER BY last_observed_at_ms DESC").fetchone() assert artifact == ("deferred_hot_jsonl_capture",) - replayed = asyncio.run(processor.ingest_files([path])) + with pytest.raises(CursorAuthorityBlockedError, match="source-selection gate blocked"): + asyncio.run(processor.ingest_files([path])) - assert replayed.full_file_count == 1 - assert replayed.append_file_count == 0 - assert replayed.succeeded_file_count == 1 final_cursor = cursor.get_record(path) assert final_cursor is not None - assert final_cursor.failure_count == 0 + assert final_cursor.byte_offset == 0 + assert final_cursor.byte_size == len(completed) + assert final_cursor.deferred_end_offset is None with sqlite3.connect(index_db) as conn: - assert conn.execute("SELECT native_id FROM messages").fetchall() == [("message-0",)] + assert conn.execute("SELECT native_id FROM messages").fetchall() == [] def test_full_ingest_rejects_incomplete_jsonl_without_hot_prefix_proof( @@ -923,13 +1979,121 @@ def test_streaming_sized_full_ingest_uses_archive( assert conn.execute("SELECT COUNT(*) FROM raw_sessions").fetchone()[0] == 1 -def test_full_ingest_writes_archive_with_route_observability( +def test_large_weak_path_uses_streaming_route_before_decoded_evidence( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_archive_database - from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier + """A weak path cannot force an eager whole-file evidence decode.""" + + root = tmp_path / "unknown" + path = root / "analysis" / "export.json" + path.parent.mkdir(parents=True) + path.write_bytes( + json.dumps( + { + "id": "weak-large", + "title": "weak large export", + "create_time": 1781442866.0, + "update_time": 1781442966.0, + "current_node": "assistant-node", + "mapping": { + "root": {"id": "root", "message": None, "parent": None, "children": ["user-node"]}, + "user-node": { + "id": "user-node", + "parent": "root", + "children": ["assistant-node"], + "message": { + "id": "weak-u1", + "author": {"role": "user"}, + "content": {"content_type": "text", "parts": ["question"]}, + "metadata": {}, + }, + }, + "assistant-node": { + "id": "assistant-node", + "parent": "user-node", + "children": [], + "message": { + "id": "weak-a1", + "author": {"role": "assistant"}, + "content": {"content_type": "text", "parts": ["answer"]}, + "metadata": {}, + }, + }, + }, + } + ).encode() + + (b" " * (9 * 1024 * 1024)) + ) + db_path = tmp_path / "archive.sqlite" + processor = LiveBatchProcessor( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=db_path))), + (WatchSource(name="chatgpt", root=root, suffixes=(".json",)),), + cursor=CursorStore(db_path), + parser_fingerprint="test-parser", + ) + monkeypatch.setattr("polylogue.sources.live.batch._STREAMING_FULL_INGEST_BYTES", 1) + monkeypatch.setattr("polylogue.sources.live.batch_support._STREAMING_FULL_INGEST_BYTES", 1) + monkeypatch.setattr( + "polylogue.sources.live.batch.has_decoded_session_evidence", + lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("large input decoded before streaming route")), + ) + phases: list[str] = [] + + def heartbeat(phase: str, **_kwargs: object) -> None: + phases.append(phase) + + result = processor._ingest_full_paths_sync([path], source_name="chatgpt", heartbeat=heartbeat) + + assert result.failed == [] + assert "full_blob_copy" in phases + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT COUNT(*) FROM raw_sessions").fetchone() == (1,) + + +def test_threshold_crossing_strong_sidecar_is_excluded_before_streaming( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A definitive sidecar path must never reach large-JSON admission.""" + + root = tmp_path / "chatgpt" + root.mkdir() + path = root / "sessions-index.json" + path.write_bytes(b"{}") + db_path = tmp_path / "archive.sqlite" + processor = LiveBatchProcessor( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=db_path))), + (WatchSource(name="chatgpt", root=root, suffixes=(".json",)),), + cursor=CursorStore(db_path), + parser_fingerprint="test-parser", + ) + monkeypatch.setattr("polylogue.sources.live.batch._STREAMING_FULL_INGEST_BYTES", 1) + monkeypatch.setattr("polylogue.sources.live.batch_support._STREAMING_FULL_INGEST_BYTES", 1) + monkeypatch.setattr( + "polylogue.sources.live.batch_support._large_non_jsonl_path_can_stream", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("strong sidecar reached large-file streaming admission") + ), + ) + + result = processor._ingest_full_paths_sync([path], source_name="chatgpt") + + assert result.succeeded == [] + assert result.failed == [] + assert not _parse_payload_as_session_artifact( + path, + provider=Provider.CHATGPT, + payload=b'{"mapping":{"session":"would otherwise look like an export"}}', + ) + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT COUNT(*) FROM raw_sessions").fetchone() == (0,) + +def test_full_ingest_writes_archive_with_route_observability( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: root = tmp_path / "sessions" root.mkdir() source = root / "full-v1.jsonl" @@ -940,8 +2104,7 @@ def test_full_ingest_writes_archive_with_route_observability( source.write_bytes(payload) index_db = tmp_path / "index.db" source_db = tmp_path / "source.db" - initialize_archive_database(index_db, ArchiveTier.INDEX) - initialize_archive_database(source_db, ArchiveTier.SOURCE) + initialize_active_archive_root(tmp_path) cursor = CursorStore(index_db) processor = LiveBatchProcessor( cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=index_db))), @@ -997,14 +2160,7 @@ def heartbeat( "storage_write_tiers": "source,index", "archive_active": True, "archive_bootstrapped": False, - **_archive_storage_probe_fields( - present={ArchiveTier.SOURCE, ArchiveTier.INDEX, ArchiveTier.OPS}, - versions={ - ArchiveTier.SOURCE: SOURCE_SCHEMA_VERSION, - ArchiveTier.INDEX: INDEX_SCHEMA_VERSION, - ArchiveTier.OPS: 1, - }, - ), + **_complete_archive_storage_probe_fields(), } write_event = next(payload for phase, payload in stage_events if phase == "full_archive_write") assert write_event == { @@ -1033,9 +2189,6 @@ def test_streaming_full_ingest_writes_archive_from_blob( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_archive_database - from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier - root = tmp_path / "sessions" root.mkdir() source = root / "stream-v1.jsonl" @@ -1046,8 +2199,7 @@ def test_streaming_full_ingest_writes_archive_from_blob( source.write_bytes(payload) index_db = tmp_path / "index.db" source_db = tmp_path / "source.db" - initialize_archive_database(index_db, ArchiveTier.INDEX) - initialize_archive_database(source_db, ArchiveTier.SOURCE) + initialize_active_archive_root(tmp_path) cursor = CursorStore(index_db) processor = LiveBatchProcessor( cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=index_db))), @@ -1090,14 +2242,7 @@ def heartbeat( "storage_write_tiers": "source,index", "archive_active": True, "archive_bootstrapped": False, - **_archive_storage_probe_fields( - present={ArchiveTier.SOURCE, ArchiveTier.INDEX, ArchiveTier.OPS}, - versions={ - ArchiveTier.SOURCE: SOURCE_SCHEMA_VERSION, - ArchiveTier.INDEX: INDEX_SCHEMA_VERSION, - ArchiveTier.OPS: 1, - }, - ), + **_complete_archive_storage_probe_fields(), } write_event = next(payload for phase, payload in stage_events if phase == "full_archive_write") assert write_event == { @@ -1127,9 +2272,6 @@ def test_streaming_sized_browser_capture_json_uses_native_payload_detection( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_archive_database - from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier - root = tmp_path / "browser-capture" / "chatgpt" root.mkdir(parents=True) source = root / "native-capture.json" @@ -1195,8 +2337,7 @@ def test_streaming_sized_browser_capture_json_uses_native_payload_detection( source.write_text(json.dumps(capture_payload), encoding="utf-8") index_db = tmp_path / "index.db" source_db = tmp_path / "source.db" - initialize_archive_database(index_db, ArchiveTier.INDEX) - initialize_archive_database(source_db, ArchiveTier.SOURCE) + initialize_active_archive_root(tmp_path) cursor = CursorStore(index_db) processor = LiveBatchProcessor( cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=index_db))), @@ -1249,9 +2390,6 @@ def test_generic_large_browser_capture_json_uses_prefix_detection_without_unknow tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_archive_database - from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier - root = tmp_path / "inbox" root.mkdir() source = root / "large-browser-capture.json" @@ -1281,8 +2419,7 @@ def test_generic_large_browser_capture_json_uses_prefix_detection_without_unknow source.write_text(json.dumps(capture_payload), encoding="utf-8") index_db = tmp_path / "index.db" source_db = tmp_path / "source.db" - initialize_archive_database(index_db, ArchiveTier.INDEX) - initialize_archive_database(source_db, ArchiveTier.SOURCE) + initialize_active_archive_root(tmp_path) cursor = CursorStore(index_db) processor = LiveBatchProcessor( cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=index_db))), @@ -2689,8 +3826,6 @@ def test_live_append_chain_survives_post_ingest_compaction( protect_chain: bool, ) -> None: from polylogue.storage.blob_publication import ArchiveBlobPublisher - from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_archive_database - from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier root = tmp_path / "sessions" root.mkdir() @@ -2703,8 +3838,7 @@ def test_live_append_chain_survives_post_ingest_compaction( path.write_bytes(payload) index_db = tmp_path / "index.db" source_db = tmp_path / "source.db" - initialize_archive_database(index_db, ArchiveTier.INDEX) - initialize_archive_database(source_db, ArchiveTier.SOURCE) + initialize_active_archive_root(tmp_path) cursor = CursorStore(index_db) processor = LiveBatchProcessor( cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=index_db))), @@ -2874,11 +4008,7 @@ def test_append_ingest_proves_byte_authority_at_capture_without_reconciler(tmp_p path.write_bytes(baseline) index_db = tmp_path / "index.db" source_db = tmp_path / "source.db" - from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_archive_database - from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier - - initialize_archive_database(index_db, ArchiveTier.INDEX) - initialize_archive_database(source_db, ArchiveTier.SOURCE) + initialize_active_archive_root(tmp_path) cursor = CursorStore(index_db) processor = LiveBatchProcessor( cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=index_db))), @@ -4524,7 +5654,7 @@ def test_append_admission_bind_failure_persists_exact_pending_envelope_and_retri tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - _path, plan, owner = _seed_live_append_plan(tmp_path, native_id="append-admission-retry") + _path, plan, owner, _processor = _seed_live_append_plan(tmp_path, native_id="append-admission-retry") original_bind = ArchiveStore.bind_raw_revision fail_once = True @@ -4588,7 +5718,7 @@ def fail_bind(self: ArchiveStore, raw_id: str, revision: RawRevisionEnvelope, ** assert conn.execute("SELECT COUNT(*) FROM raw_sessions WHERE source_index = -1").fetchone() == (1,) -def test_public_full_blob_batch_bind_failure_persists_bytes_and_retries( +def test_public_full_blob_batch_bind_failure_persists_bytes_and_blocks_unsafe_retry( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -4651,13 +5781,12 @@ def fail_bind(self: ArchiveStore, raw_id: str, revision: RawRevisionEnvelope, ** ) assert isinstance(row[13], str) and "injected blob bind failure" in row[13] - retry = asyncio.run(processor.ingest_files([source], emit_event=False)) + with pytest.raises(CursorAuthorityBlockedError, match="source-selection gate blocked"): + asyncio.run(processor.ingest_files([source], emit_event=False)) - assert retry.full_file_count == 1 - assert retry.failed_file_count == 0 with sqlite3.connect(tmp_path / "source.db") as conn: assert conn.execute("SELECT COUNT(*) FROM raw_sessions").fetchone() == (1,) - bound = conn.execute( + retained = conn.execute( """ SELECT logical_source_key, revision_kind, source_revision, predecessor_source_revision, predecessor_raw_id, baseline_raw_id, @@ -4666,18 +5795,18 @@ def fail_bind(self: ArchiveStore, raw_id: str, revision: RawRevisionEnvelope, ** FROM raw_sessions """ ).fetchone() - assert bound == ( - "codex:blob-retry", + assert retained == ( + f"pending-raw:codex-session:0:{source}:{raw_id}", "full", sha256(payload).hexdigest(), None, None, - raw_id, None, None, - 0, - "byte_proven", None, + 0, + "quarantined", + row[13], ) @@ -4769,7 +5898,7 @@ def test_append_index_failure_never_marks_raw_success( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - _path, plan, owner = _seed_live_append_plan(tmp_path, native_id="index-fail") + _path, plan, owner, _processor = _seed_live_append_plan(tmp_path, native_id="index-fail") def fail_index(*_args: object, **_kwargs: object) -> object: raise sqlite3.IntegrityError("injected index commit failure") @@ -4790,14 +5919,7 @@ def test_append_multi_session_payload_is_rejected_before_index_write( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - path = tmp_path / "append-multi.jsonl" - # Real, parseable content -- not a bare `{}` -- because polylogue-xwkh's - # declared-non-session-artifact gate now refuses that shape before this - # test's mocked parse_payload (returning two sessions) is ever reached. - payload = b'{"type":"event_msg","payload":{"type":"user_message","message":"hello"}}\n' - path.write_bytes(payload) - plan = _append_plan(path, payload, payload_hash="multi") - owner = _append_owner(tmp_path) + path, plan, owner, _processor = _seed_live_append_plan(tmp_path, native_id="append-multi") # polylogue-9ykn: a message-less ParsedSession carries no positive # conversational evidence and is refused before this test's own # "more than one session" check ever runs -- give each session one real @@ -4814,15 +5936,15 @@ def test_append_multi_session_payload_is_rejected_before_index_write( messages=[ParsedMessage(provider_message_id="multi-2-0", role=Role.USER, text="hello")], ), ] - monkeypatch.setattr("polylogue.sources.dispatch.parse_payload", lambda *_args, **_kwargs: sessions) + monkeypatch.setattr("polylogue.sources.dispatch.parse_stream_payload", lambda *_args, **_kwargs: sessions) result = ingest_append_plans(cast(Any, owner), [plan]) assert result.failed == [plan] - parsed_at_ms, parse_error = _raw_parse_state(tmp_path) + parsed_at_ms, parse_error = _append_raw_parse_state(tmp_path) assert parsed_at_ms is None assert isinstance(parse_error, str) and "did not prove one session and cursor identity" in parse_error with sqlite3.connect(tmp_path / "index.db") as conn: - assert conn.execute("SELECT COUNT(*) FROM sessions").fetchone()[0] == 0 + assert conn.execute("SELECT native_id FROM sessions").fetchall() == [("append-multi",)] def test_full_multi_session_failure_retries_without_success_mapping( @@ -4935,12 +6057,10 @@ def test_full_ingest_skips_durably_excised_content_without_aborting_batch( ``write_raw_payload`` -> ``write_source_raw_session`` gate, which is a different call site (polylogue-re4a). """ - from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_archive_database from polylogue.storage.sqlite.archive_tiers.source_write import ( deterministic_blob_hash, record_excised_blob_hash, ) - from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier root = tmp_path / "sessions" root.mkdir() @@ -4958,7 +6078,7 @@ def test_full_ingest_skips_durably_excised_content_without_aborting_batch( # Pre-mark the excised file's exact content hash as durably excised, # mirroring a prior real `polylogue ops excise` apply. - initialize_archive_database(tmp_path / "source.db", ArchiveTier.SOURCE) + initialize_active_archive_root(tmp_path) source_conn = sqlite3.connect(tmp_path / "source.db") try: record_excised_blob_hash( @@ -6480,7 +7600,7 @@ def test_append_crash_after_index_commit_repairs_idempotently( class SimulatedProcessCrash(BaseException): pass - _path, plan, owner = _seed_live_append_plan(tmp_path, native_id="crash-retry") + _path, plan, owner, _processor = _seed_live_append_plan(tmp_path, native_id="crash-retry") # polylogue-1r9c: mark_raw_parse_succeeded is called internally by # revision_governance.py (a direct module-internal function reference), # not through ArchiveStore's `self.` dispatch -- patch it there. diff --git a/tests/unit/sources/test_live_watcher.py b/tests/unit/sources/test_live_watcher.py index 7f508c3175..df38579db7 100644 --- a/tests/unit/sources/test_live_watcher.py +++ b/tests/unit/sources/test_live_watcher.py @@ -357,6 +357,7 @@ async def test_active_index_pointer_keeps_shadow_index_unmodified(tmp_path: Path projection = reconcile._projection_for(tmp_path) sample = projection.cursor_ahead_samples[0] shadow_before = shadow_index.read_bytes() + active_before = active_index.read_bytes() with scoped_cursor_authority_authorization( source_path_digest=cursor_authority_path_digest(source_path), cursor_byte_offset=sample.cursor_byte_offset, @@ -368,6 +369,11 @@ async def test_active_index_pointer_keeps_shadow_index_unmodified(tmp_path: Path assert metrics.full_file_count == 1 assert shadow_index.read_bytes() == shadow_before + assert active_index.read_bytes() != active_before + with sqlite3.connect(shadow_index) as conn: + conn.execute("DELETE FROM sessions") + conn.commit() + assert watcher._reconcile_archived_cursor(source_path, stat=source_path.stat()) is True watcher.stop() @@ -1568,6 +1574,88 @@ def test_hermes_wal_revision_triggers_resnapshot_and_maps_sidecar_event(tmp_path writer.close() +def test_watch_filter_accepts_directories_but_not_unmatched_files_under_broad_roots(tmp_path: Path) -> None: + """The watch backend wakes only for source suffixes or real directories.""" + + root = tmp_path / "codex-state" + root.mkdir() + unmatched = root / "history.log" + unmatched.write_text("noise", encoding="utf-8") + child_directory = root / "new-session" + child_directory.mkdir() + watcher, _full_ingest = _make_watcher( + tmp_path, + root, + sources=(WatchSource(name="codex-state", root=root, suffixes=(".jsonl",)),), + ) + + assert watcher._watch_filter(object(), str(unmatched)) is False + assert watcher._watch_filter(object(), str(child_directory)) is True + + +def test_added_directory_scan_rejects_file_symlinks_escaping_source_root( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Recursive add recovery applies the same resolved containment as live events.""" + + root = tmp_path / "watched" + added = root / "new-directory" + added.mkdir(parents=True) + internal = added / "inside.jsonl" + internal.write_text("{}\n", encoding="utf-8") + external = tmp_path / "outside.jsonl" + external.write_text("secret\n", encoding="utf-8") + escaping = added / "escaping.jsonl" + escaping.symlink_to(external) + watcher, _full_ingest = _make_watcher( + tmp_path, + root, + sources=(WatchSource(name="codex", root=root, suffixes=(".jsonl",)),), + ) + enqueued: list[Path] = [] + monkeypatch.setattr(watcher, "_enqueue", enqueued.append) + + assert watcher._canonical_watch_path(escaping) is None + watcher._enqueue_added_directory(added) + + assert enqueued == [internal] + + +def test_added_directory_scan_retains_a_deeper_root_under_outer_ignore( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """An outer ignore rule cannot hide a configured nested source root.""" + + outer = tmp_path / "codex-state" + ignored = outer / "runtime" + inner = ignored / "sessions" + inner.mkdir(parents=True) + session = inner / "session.jsonl" + session.write_text("{}\n", encoding="utf-8") + watcher, _full_ingest = _make_watcher( + tmp_path, + outer, + sources=( + WatchSource( + name="codex-state", + root=outer, + suffixes=(".sqlite",), + ignored_dir_names=frozenset({"runtime"}), + ), + WatchSource(name="codex", root=inner, suffixes=(".jsonl",)), + ), + ) + enqueued: list[Path] = [] + monkeypatch.setattr(watcher, "_enqueue", enqueued.append) + + assert watcher._watch_filter(object(), str(ignored)) is True + watcher._enqueue_added_directory(ignored) + + assert enqueued == [session] + + def test_hermes_cursor_records_acquisition_revision_not_live_tail(tmp_path: Path) -> None: root = tmp_path / "hermes" root.mkdir() @@ -2176,6 +2264,7 @@ async def test_live_full_ingest_preserves_complete_workflow_journal_revisions( } assert summary.call_count == 1 assert summary.journal_result_count == 1 + assert processor.require_cursor_authority() is None finally: await archive.close() @@ -2294,6 +2383,7 @@ async def test_live_append_atof_shared_file_multi_session_boundary_retains_all_e await processor.ingest_files([source_path], emit_event=False) replayed = _atof_event_uuids_by_session(workspace_env["archive_root"]) assert replayed == event_uuids_by_session + assert processor.require_cursor_authority() is None finally: await archive.close() @@ -2433,6 +2523,7 @@ async def test_live_full_ingest_over_ambiguous_membership_preserves_durable_debt second = await processor.ingest_files([source_path], emit_event=False) assert second.succeeded_file_count == 1, "ambiguous membership debt is not retried as a file failure (#3282)" assert second.failed_file_count == 0 + assert processor.require_cursor_authority() is None record = cursor.get_record(source_path) assert record is not None @@ -2862,7 +2953,7 @@ async def test_live_full_ingest_excludes_non_session_sidecars_before_raw_storage @pytest.mark.asyncio -async def test_live_full_ingest_excludes_invalid_jsonl_sidecars_before_raw_storage( +async def test_live_full_ingest_excludes_known_provider_invalid_jsonl_sidecars_before_raw_storage( workspace_env: dict[str, Path], ) -> None: root = workspace_env["data_root"] / "projects" @@ -2875,7 +2966,7 @@ async def test_live_full_ingest_excludes_invalid_jsonl_sidecars_before_raw_stora cursor = CursorStore(db_path) processor = LiveBatchProcessor( archive, - (WatchSource(name="projects", root=root),), + (WatchSource(name="claude-code", root=root),), cursor=cursor, parser_fingerprint=live_watcher._PARSER_FINGERPRINT, ) @@ -3233,6 +3324,56 @@ def test_catch_up_processes_pre_existing_files(tmp_path: Path) -> None: assert parse_sources.await_count == 1 +def test_catch_up_acquires_source_without_reading_unavailable_index( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The real catch-up planner and batch route remain source-only while derived-only.""" + + from polylogue.core.degraded import DegradedReason, clear_degraded, set_degraded + + with ArchiveStore.open_existing(tmp_path, read_only=False): + pass + root = tmp_path / "sessions" + root.mkdir() + path = root / "degraded-catch-up.jsonl" + path.write_bytes( + b'{"type":"session_meta","payload":{"id":"degraded-catch-up"}}\n' + b'{"type":"response_item","payload":{"type":"message","id":"message-0","role":"user",' + b'"content":[{"type":"input_text","text":"zero"}]}}\n' + ) + pointer = tmp_path / ".index-active-pointer" + pointer.write_bytes(b"\xff") + watcher = LiveWatcher( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=tmp_path / "index.db"))), + (WatchSource(name="codex", root=root),), + cursor=CursorStore(tmp_path / "cursor.sqlite", ops_db_path=tmp_path / "ops.db"), + ) + parse_stage = watcher._parse_stage + assert parse_stage is not None + monkeypatch.setattr(parse_stage, "warm", lambda *_args: (_ for _ in ()).throw(AssertionError("must not prewarm"))) + set_degraded( + DegradedReason( + code="schema_version_mismatch", + message="derived generation unavailable", + derived_only=True, + ) + ) + try: + asyncio.run(watcher._catch_up([root])) + finally: + clear_degraded() + parse_stage.shutdown() + + assert pointer.read_bytes() == b"\xff" + with sqlite3.connect(tmp_path / "source.db") as conn: + row = conn.execute( + """SELECT parsed_at_ms, parse_error FROM raw_sessions + WHERE source_path = ? ORDER BY acquired_at_ms DESC, raw_id DESC LIMIT 1""", + (str(path),), + ).fetchone() + assert row == (None, None) + + def test_catch_up_skips_already_processed(tmp_path: Path) -> None: root = tmp_path / "src" root.mkdir() @@ -3478,6 +3619,135 @@ def test_watch_source_accepts_configured_suffixes(tmp_path: Path) -> None: assert src.accepts(tmp_path / "README.md") is False +def test_source_accepts_prefers_most_specific_nested_root(tmp_path: Path) -> None: + """A nested explicit root owns its files regardless of source order.""" + root = tmp_path / "codex" + sessions = root / "sessions" + sessions.mkdir(parents=True) + path = sessions / "session.jsonl" + path.write_text("{}\n", encoding="utf-8") + watcher = LiveWatcher( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=tmp_path / "index.db"))), + ( + WatchSource(name="codex-state", root=root, suffixes=(".sqlite",)), + WatchSource(name="codex", root=sessions, suffixes=(".jsonl",)), + ), + cursor=CursorStore(tmp_path / "cursor.db"), + ) + parse_stage = watcher._parse_stage + assert parse_stage is not None + + try: + assert watcher._source_accepts(path) is True + assert watcher._source_name_for(path) == "codex" + assert watcher._batch_processor._source_name_for(path) == "codex" + directory_source = watcher._source_for_directory(sessions) + assert directory_source is not None + assert directory_source.name == "codex" + candidates = watcher._scan_catch_up_candidates([root, sessions]) + assert [(candidate.path, candidate.source_name) for candidate in candidates] == [(path, "codex")] + finally: + parse_stage.shutdown() + + +@pytest.mark.asyncio +async def test_hook_spool_directory_retry_retries_sqlite_operational_error(tmp_path: Path) -> None: + """A transient spool-drain lock follows the normal delayed retry path.""" + shard = tmp_path / "pending" / "2026-08-13" + shard.mkdir(parents=True) + (shard / "event.json").write_text("{}", encoding="utf-8") + watcher = LiveWatcher( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=tmp_path / "index.db"))), + (), + cursor=CursorStore(tmp_path / "cursor.db"), + ) + parse_stage = watcher._parse_stage + assert parse_stage is not None + calls = 0 + + async def drain() -> None: + nonlocal calls + calls += 1 + if calls == 1: + raise sqlite3.OperationalError("database is locked") + (shard / "event.json").unlink() + + watcher._drain_hook_spool = drain # type: ignore[method-assign] + try: + await watcher._retry_hook_spool_directory_until_populated(shard) + finally: + parse_stage.shutdown() + + assert calls == 2 + + +@pytest.mark.asyncio +async def test_hook_spool_directory_retry_rejects_non_lock_sqlite_error(tmp_path: Path) -> None: + """A corrupt or incompatible spool database is not misclassified as contention.""" + + shard = tmp_path / "pending" / "2026-08-13" + shard.mkdir(parents=True) + (shard / "event.json").write_text("{}", encoding="utf-8") + watcher = LiveWatcher( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=tmp_path / "index.db"))), + (), + cursor=CursorStore(tmp_path / "cursor.db"), + ) + parse_stage = watcher._parse_stage + assert parse_stage is not None + + async def drain() -> None: + raise sqlite3.OperationalError("no such table: hook_events") + + watcher._drain_hook_spool = drain # type: ignore[method-assign] + try: + with pytest.raises(sqlite3.OperationalError, match="no such table"): + await watcher._retry_hook_spool_directory_until_populated(shard) + finally: + parse_stage.shutdown() + + +@pytest.mark.asyncio +async def test_scheduled_hook_spool_retry_observes_and_logs_failure( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A failed detached retry is retrieved and reported instead of becoming an unhandled task.""" + + shard = tmp_path / "pending" / "2026-08-13" + shard.mkdir(parents=True) + (shard / "event.json").write_text("{}", encoding="utf-8") + watcher = LiveWatcher( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=tmp_path / "index.db"))), + (), + cursor=CursorStore(tmp_path / "cursor.db"), + ) + parse_stage = watcher._parse_stage + assert parse_stage is not None + recorded_logger = MagicMock() + monkeypatch.setattr(live_watcher, "logger", recorded_logger) + + async def drain() -> None: + raise sqlite3.OperationalError("database disk image is malformed") + + watcher._drain_hook_spool = drain # type: ignore[method-assign] + try: + watcher._schedule_hook_spool_directory_retry(shard) + task = watcher._hook_spool_directory_retry_tasks[shard.resolve()] + with contextlib.suppress(sqlite3.OperationalError): + await task + if watcher._hook_spool_directory_retry_tasks: + await asyncio.sleep(0) + finally: + parse_stage.shutdown() + + assert watcher._hook_spool_directory_retry_tasks == {} + recorded_logger.exception.assert_called_once() + assert recorded_logger.exception.call_args.args == ( + "live.watcher: hook spool directory retry failed for %s", + shard.resolve(), + ) + + def test_inbox_source_accepts_zip_and_archive_formats() -> None: """#1683: inbox must accept .zip (GDPR exports), .json, .jsonl, .ndjson.""" from polylogue.sources.live.watcher import default_sources @@ -3650,6 +3920,81 @@ async def _drive() -> None: asyncio.run(_drive()) +def test_periodic_catch_up_adds_configured_nested_root_created_after_start( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """A late nested root remains recoverable after its add event is missed.""" + outer = tmp_path / "sources" + nested = outer / "late-codex" + outer.mkdir() + watcher, parse_sources = _make_watcher( + tmp_path, + outer, + sources=( + WatchSource(name="outer", root=outer, suffixes=(".jsonl",)), + WatchSource(name="nested", root=nested, suffixes=(".jsonl",)), + ), + ) + monkeypatch.setattr(live_watcher, "_PERIODIC_CATCH_UP_INTERVAL_S", 0.02) + + async def _drive() -> None: + task = asyncio.create_task(watcher._periodic_catch_up([outer])) + await asyncio.sleep(0.03) + nested.mkdir() + (nested / "missed.jsonl").write_text('{"type":"session_meta","payload":{"id":"late"}}\n') + for _ in range(60): + if parse_sources.await_count >= 1: + break + await asyncio.sleep(0.05) + watcher.stop() + task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await task + assert parse_sources.await_count >= 1 + + asyncio.run(_drive()) + + +def test_watcher_run_periodically_rediscovers_nested_root_created_after_start( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """The daemon route refreshes configured roots after its initial watch snapshot.""" + outer = tmp_path / "sources" + nested = outer / "late-codex" + outer.mkdir() + watcher, parse_sources = _make_watcher( + tmp_path, + outer, + sources=( + WatchSource(name="outer", root=outer, suffixes=(".jsonl",)), + WatchSource(name="nested", root=nested, suffixes=(".jsonl",)), + ), + ) + monkeypatch.setattr(live_watcher, "_PERIODIC_CATCH_UP_INTERVAL_S", 0.02) + + async def wait_for_stop(_roots: list[Path]) -> None: + await watcher._stop.wait() + + monkeypatch.setattr(watcher, "_watch_changes", wait_for_stop) + + async def _drive() -> None: + task = asyncio.create_task(watcher.run()) + await asyncio.wait_for(watcher.catch_up_complete.wait(), timeout=1.0) + nested.mkdir() + (nested / "missed.jsonl").write_text('{"type":"session_meta","payload":{"id":"late"}}\n') + for _ in range(60): + if parse_sources.await_count >= 1: + break + await asyncio.sleep(0.05) + watcher.stop() + await asyncio.wait_for(task, timeout=1.0) + assert parse_sources.await_count >= 1 + + asyncio.run(_drive()) + + def test_periodic_catch_up_backs_off_after_each_reconciliation_pass( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, diff --git a/tests/unit/sources/test_live_watcher_locking.py b/tests/unit/sources/test_live_watcher_locking.py index edc3b09e99..15fc4b1c94 100644 --- a/tests/unit/sources/test_live_watcher_locking.py +++ b/tests/unit/sources/test_live_watcher_locking.py @@ -3,6 +3,8 @@ from __future__ import annotations import asyncio +import contextlib +import selectors import sqlite3 import subprocess import sys @@ -31,6 +33,7 @@ def _make_watcher(tmp_path: Path, root: Path, *, debounce_s: float = 0.01) -> Li @pytest.mark.parametrize("route", ["append", "full"]) +@pytest.mark.uses_real_clock("requires a bounded subprocess exit deadline while the injected writer remains blocked") def test_real_watcher_writer_routes_cannot_pin_process_exit(route: str) -> None: script = textwrap.dedent( f""" @@ -61,8 +64,13 @@ async def main() -> None: cursor=cursor, write_coordinator=coordinator, ) + # This proof targets the writer bridge's process-exit semantics. + # Disable the independent prefetch lane so an executor worker + # cannot determine the subprocess lifetime instead. + watcher._parse_stage.shutdown() + watcher._parse_stage = None + watcher._batch_processor._parse_stage = None started = threading.Event() - def stuck(*args, **kwargs): started.set() threading.Event().wait() @@ -95,21 +103,39 @@ def stuck(*args, **kwargs): with contextlib.suppress(asyncio.CancelledError): await caller assert await coordinator.shutdown(timeout=0.01) is False + # The injected writer remains blocked through interpreter + # termination. A non-daemon bridge thread would pin this + # subprocess after the loop closes. + watcher.stop() + print("ready-for-interpreter-exit", flush=True) asyncio.run(main()) """ ) - completed = subprocess.run( + process = subprocess.Popen( [sys.executable, "-c", script], cwd=Path(__file__).parents[3], - capture_output=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True, - timeout=6.0, - check=False, ) - - assert completed.returncode == 0, completed.stderr + assert process.stdout is not None + selector = selectors.DefaultSelector() + selector.register(process.stdout, selectors.EVENT_READ) + try: + assert selector.select(timeout=30.0), "writer subprocess did not reach its exit boundary" + assert process.stdout.readline().strip() == "ready-for-interpreter-exit" + stdout, stderr = process.communicate(timeout=2.0) + except BaseException: + process.kill() + with contextlib.suppress(subprocess.TimeoutExpired): + process.communicate(timeout=2.0) + raise + finally: + selector.close() + + assert process.returncode == 0, stdout + stderr @pytest.mark.asyncio diff --git a/tests/unit/sources/test_revision_backfill.py b/tests/unit/sources/test_revision_backfill.py index 7f90f41870..76ba01cfc3 100644 --- a/tests/unit/sources/test_revision_backfill.py +++ b/tests/unit/sources/test_revision_backfill.py @@ -3,10 +3,13 @@ import json import sqlite3 import time +from collections.abc import Iterator +from contextlib import contextmanager from io import BytesIO from pathlib import Path -from typing import Any +from typing import Any, BinaryIO +import ijson import pytest from polylogue.archive.ingest_flags import ( @@ -20,6 +23,7 @@ from polylogue.sources import revision_backfill from polylogue.sources.decoders import _iter_json_stream from polylogue.sources.dispatch import parse_payload +from polylogue.sources.parsers import codex_state from polylogue.sources.parsers.base import ParsedSession from polylogue.sources.revision_backfill import ( RawParsePrefetchCache, @@ -28,13 +32,18 @@ _parse_one, backfill_historical_revision_evidence, census_historical_revision_evidence, + validate_frozen_source_authority, ) +from polylogue.storage.artifacts.inspection import inspect_raw_artifact from polylogue.storage.blob_publication import ArchiveBlobPublisher +from polylogue.storage.blob_store import BlobStore from polylogue.storage.raw_authority import RAW_AUTHORITY_PARSER_FINGERPRINT +from polylogue.storage.raw_retention import RawRetentionAuthority, active_raw_retention_authority from polylogue.storage.sqlite.archive_tiers import revision_governance as archive_revision_governance from polylogue.storage.sqlite.archive_tiers import write as archive_tier_write from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root +from polylogue.storage.sqlite.async_sqlite import SQLiteBackend from tests.infra.revision_backfill_benchmark import ( REVISION_CHAIN_SHAPE, WHALE_BEARING_SHAPE, @@ -156,6 +165,423 @@ def test_parse_one_replays_single_session_state_db_bytes_via_temp_spill(tmp_path assert sessions[0].messages[0].text == "hi" +def test_unknown_retained_stream_replay_scans_past_oversized_first_record_without_eager_payload( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """UNKNOWN JSONL scans past an oversized first record before streaming replay.""" + initialize_active_archive_root(tmp_path) + payload = ( + json.dumps({"opaque": "x" * 9_000}, sort_keys=True).encode() + b"\n" + b'{"type":"session_meta","payload":{"id":"unknown-stream","timestamp":"2026-06-01T00:00:00Z"}}\n' + b'{"type":"response_item","payload":{"type":"message","id":"m1","role":"user",' + b'"content":[{"type":"input_text","text":"prefix detected replay"}]}}\n' + ) + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.UNKNOWN, + payload=payload, + source_path="unknown-member.jsonl", + acquired_at_ms=1, + ) + + def reject_eager_material(_raw_id: str) -> tuple[Provider, bytes, str, RawRevisionKind]: + raise AssertionError("eager payload read") + + monkeypatch.setattr(archive, "raw_revision_material", reject_eager_material) + sessions = revision_backfill.parse_retained_raw_sessions(archive, raw_id) + + assert [session.provider_session_id for session in sessions] == ["unknown-stream"] + + +def test_unknown_retained_codex_record_scans_provider_key_past_8k_padding( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """A late Codex discriminator in one oversized record remains visible.""" + initialize_active_archive_root(tmp_path) + late_session_meta = json.dumps( + { + "padding": "x" * (revision_backfill._REPLAY_PROVIDER_DETECTION_PREFIX_BYTES + 512), + "type": "session_meta", + "payload": {"id": "late-codex", "timestamp": "2026-06-01T00:00:00Z"}, + }, + separators=(",", ":"), + ).encode() + payload = ( + late_session_meta + + b"\n" + + b'{"type":"response_item","payload":{"type":"message","id":"m1","role":"user",' + + b'"content":[{"type":"input_text","text":"late discriminator"}]}}\n' + ) + assert ( + b'"type":"session_meta"' not in late_session_meta[: revision_backfill._REPLAY_PROVIDER_DETECTION_PREFIX_BYTES] + ) + + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.UNKNOWN, + payload=payload, + source_path="late-codex.jsonl", + acquired_at_ms=1, + ) + + monkeypatch.setattr( + archive, + "raw_revision_material", + lambda *_args, **_kwargs: pytest.fail("late Codex evidence must select the streaming route"), + ) + sessions = revision_backfill.parse_retained_raw_sessions(archive, raw_id) + + assert [session.provider_session_id for session in sessions] == ["late-codex"] + + +def test_unknown_retained_malformed_huge_record_stays_unknown_inside_total_budget() -> None: + """Malformed data cannot make a discriminator beyond the scan envelope authoritative.""" + + class CountingReader: + def __init__(self, payload: bytes) -> None: + self._payload = BytesIO(payload) + self.bytes_read = 0 + + def readline(self, size: int = -1) -> bytes: + chunk = self._payload.readline(size) + self.bytes_read += len(chunk) + return chunk + + def read(self, size: int = -1) -> bytes: + chunk = self._payload.read(size) + self.bytes_read += len(chunk) + return chunk + + def seek(self, offset: int, whence: int = 0) -> int: + return self._payload.seek(offset, whence) + + payload = ( + b'{"padding":"' + + b"x" * (revision_backfill._REPLAY_PROVIDER_DETECTION_MAX_SCAN_BYTES + 16_384) + + b'","type":"session_meta","payload":{"id":"outside-budget"}' + ) + reader = CountingReader(payload) + + provider, _evidence = revision_backfill._detect_unknown_retained_provider(reader, "huge.jsonl") + + assert provider is Provider.UNKNOWN + assert reader.bytes_read <= revision_backfill._REPLAY_PROVIDER_DETECTION_MAX_SCAN_BYTES + + +def test_unknown_retained_oversized_provider_record_never_uses_eager_payload( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """Positive prefix evidence survives a record larger than the total scan cap.""" + initialize_active_archive_root(tmp_path) + payload = ( + json.dumps( + { + "sessionId": "oversized-only-provider-record", + "uuid": "message-1", + "type": "user", + "message": {"role": "user", "content": [{"type": "text", "text": "x" * 80_000}]}, + } + ).encode() + + b"\n" + ) + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.UNKNOWN, + payload=payload, + source_path="oversized-only.jsonl", + acquired_at_ms=1, + ) + + def reject_eager_material(_raw_id: str) -> tuple[Provider, bytes, str, RawRevisionKind]: + raise AssertionError("eager payload read") + + monkeypatch.setattr(archive, "raw_revision_material", reject_eager_material) + sessions = revision_backfill.parse_retained_raw_sessions(archive, raw_id) + + assert [session.provider_session_id for session in sessions] == ["oversized-only-provider-record"] + + +def test_unknown_retained_jsonl_detection_caps_total_scan_before_typed_failure( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """An unidentifiable retained JSONL blob stops at the detection envelope.""" + initialize_active_archive_root(tmp_path) + payload = (b'{"opaque":"' + b"x" * 9_000 + b'"}\n') * 32 + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.UNKNOWN, + payload=payload, + source_path="opaque.jsonl", + acquired_at_ms=1, + ) + read_bytes = 0 + original_open = archive.open_raw_revision_material + + class CountingReader: + def __init__(self, wrapped: BinaryIO) -> None: + self._wrapped = wrapped + + def read(self, size: int = -1) -> bytes: + nonlocal read_bytes + chunk = self._wrapped.read(size) + read_bytes += len(chunk) + return chunk + + def readline(self, size: int = -1) -> bytes: + nonlocal read_bytes + chunk = self._wrapped.readline(size) + read_bytes += len(chunk) + return chunk + + @contextmanager + def tracked_open(requested_raw_id: str) -> Iterator[tuple[Provider, CountingReader, str, RawRevisionKind]]: + with original_open(requested_raw_id) as (provider, stream, source_path, kind): + yield provider, CountingReader(stream), source_path, kind + + monkeypatch.setattr(archive, "open_raw_revision_material", tracked_open) + monkeypatch.setattr( + archive, + "raw_revision_material", + lambda *_args, **_kwargs: pytest.fail("unidentified JSONL must not fall through to eager blob loading"), + ) + + with pytest.raises(ValueError, match="retained UNKNOWN provider remained unresolved"): + revision_backfill.parse_retained_raw_sessions(archive, raw_id) + + assert read_bytes <= revision_backfill._REPLAY_PROVIDER_DETECTION_MAX_SCAN_BYTES + + +def test_frozen_source_validation_treats_codex_state_as_non_session_evidence( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """Frozen validation must route Codex state SQLite past the JSON parser.""" + initialize_active_archive_root(tmp_path) + payload = _codex_thread_state_snapshot_bytes(tmp_path, "frozen state") + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + archive.write_raw_payload( + provider=Provider.CODEX, + payload=payload, + source_path=str(tmp_path / "codex" / "state_5.sqlite"), + acquired_at_ms=1, + ) + + census_historical_revision_evidence(tmp_path) + parsed_raw_ids: list[str] = [] + + def record_parse_dispatch(_archive: ArchiveStore, raw_ids: list[str], **_kwargs: object) -> dict[object, object]: + parsed_raw_ids.extend(raw_ids) + return {} + + monkeypatch.setattr(revision_backfill, "_parse_retained_raws", record_parse_dispatch) + + validate_frozen_source_authority(tmp_path) + assert parsed_raw_ids == [] + + +def test_frozen_codex_state_budget_blocks_before_sqlite_classification( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """Frozen-source validation rejects an oversized state snapshot before opening it.""" + initialize_active_archive_root(tmp_path) + payload = _codex_thread_state_snapshot_bytes(tmp_path, "frozen oversized state") + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.CODEX, + payload=payload, + source_path=str(tmp_path / "codex" / "state_5.sqlite"), + acquired_at_ms=1, + ) + + monkeypatch.setattr( + codex_state, + "classify_codex_sqlite_path", + lambda *_args, **_kwargs: pytest.fail("payload budget must block before Codex SQLite classification"), + ) + + with pytest.raises(revision_backfill.RawRevisionReplayResourceBlockedError) as blocked: + validate_frozen_source_authority(tmp_path, max_payload_bytes=1) + + assert blocked.value.raw_ids == (raw_id,) + + +def test_unknown_retained_stream_census_worker_scans_past_oversized_first_record( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """The production census worker must discover a later bounded JSONL record.""" + initialize_active_archive_root(tmp_path) + payload = ( + json.dumps({"opaque": "x" * 9_000}, sort_keys=True).encode() + + b"\n" + + b'{"type":"session_meta","payload":{"id":"unknown-worker","timestamp":"2026-06-01T00:00:00Z"}}\n' + + b'{"type":"response_item","payload":{"type":"message","id":"m1","role":"user",' + + b'"content":[{"type":"input_text","text":"worker replay"}]}}\n' + ) + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.UNKNOWN, + payload=payload, + source_path="unknown-worker.jsonl", + acquired_at_ms=1, + ) + provider, blob_hash, source_path, kind, _payload_size = archive.raw_revision_descriptor(raw_id) + + monkeypatch.setattr( + ArchiveBlobPublisher, + "read_all", + lambda *_args, **_kwargs: pytest.fail("UNKNOWN stream census must not eagerly read the blob"), + ) + same_raw_id, sessions, error = revision_backfill.census_parse_worker( + raw_id, + provider.value, + blob_hash, + source_path, + False, + str(tmp_path / "blob"), + str(tmp_path / "source.db"), + kind.value, + None, + ) + + assert same_raw_id == raw_id + assert error is None + assert sessions is not None + assert [session.provider_session_id for session in sessions] == ["unknown-worker"] + + +def test_unknown_retained_nonstream_jsonl_keeps_complete_payload_fallback(tmp_path: Path) -> None: + """Positive bounded document evidence may select eager non-stream replay.""" + initialize_active_archive_root(tmp_path) + document = _chatgpt_session("large-jsonl-document", "bounded evidence") + document["padding"] = "x" * 9_000 + payload = json.dumps(document, sort_keys=True).encode() + b"\n" + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.UNKNOWN, + payload=payload, + source_path="large-chatgpt.jsonl", + acquired_at_ms=1, + ) + sessions = revision_backfill.parse_retained_raw_sessions(archive, raw_id) + + assert [session.provider_session_id for session in sessions] == ["large-jsonl-document"] + + +def test_unknown_retained_document_scans_past_oversized_leading_value(tmp_path: Path) -> None: + """A complete ChatGPT document must scan beyond its bounded prefix. + + The raw is intentionally a source-only UNKNOWN ``conversations.json`` + whose provider-defining fields follow an oversized leading value. This + drives the historical replay chokepoint against a real archive, rather + than testing the structural scanner in isolation. + """ + initialize_active_archive_root(tmp_path) + document = {"padding": "x" * 9_000, **_chatgpt_session("large-document", "bounded evidence")} + payload = json.dumps([document]).encode() + assert b'"mapping"' not in payload[: revision_backfill._REPLAY_PROVIDER_DETECTION_PREFIX_BYTES] + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + archive.write_raw_payload( + provider=Provider.UNKNOWN, + payload=payload, + source_path="export/conversations.json", + acquired_at_ms=1, + ) + + backfill_historical_revision_evidence(tmp_path) + + with sqlite3.connect(tmp_path / "index.db") as conn: + assert conn.execute("SELECT session_id FROM sessions").fetchall() == [("chatgpt-export:large-document",)] + + +def test_unknown_retained_document_caps_oversized_scalar_before_structural_scan( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """The retained-document route never hands a whole giant scalar to ijson.""" + initialize_active_archive_root(tmp_path) + payload = json.dumps({"padding": "x" * 128_000, "metadata": {"shape": "unknown"}}).encode() + observed_string_bytes: list[int] = [] + original_parse = ijson.parse + + def guarded_parse(*args: object, **kwargs: object) -> Any: + for prefix, event, value in original_parse(*args, **kwargs): + if event == "string": + observed_string_bytes.append(len(str(value).encode())) + assert observed_string_bytes[-1] <= revision_backfill._REPLAY_PROVIDER_DETECTION_PREFIX_BYTES + yield prefix, event, value + + monkeypatch.setattr(ijson, "parse", guarded_parse) + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.UNKNOWN, + payload=payload, + source_path="export/unknown-document.json", + acquired_at_ms=1, + ) + + def reject_eager_material(_raw_id: str) -> tuple[Provider, bytes, str, RawRevisionKind]: + raise AssertionError("unclassified document must not use eager payload materialization") + + monkeypatch.setattr(archive, "raw_revision_material", reject_eager_material) + with pytest.raises(ValueError, match="remained unresolved after bounded scan"): + revision_backfill.parse_retained_raw_sessions(archive, raw_id) + + assert max(observed_string_bytes) == revision_backfill._REPLAY_PROVIDER_DETECTION_PREFIX_BYTES + + +def test_unknown_retained_array_ignores_fragment_only_mapping_before_real_provider(tmp_path: Path) -> None: + """An unrelated mapping fragment cannot claim a whole document sequence.""" + initialize_active_archive_root(tmp_path) + payload = json.dumps( + [ + {"mapping": {"foreign-node": {"message": None}}, "metadata": "not a conversation"}, + { + "uuid": "later-claude-provider", + "name": "Later Claude provider", + "chat_messages": [ + { + "uuid": "claude-message", + "sender": "human", + "text": "real provider evidence", + "created_at": "2026-08-13T00:00:00Z", + } + ], + }, + ] + ).encode() + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + archive.write_raw_payload( + provider=Provider.UNKNOWN, + payload=payload, + source_path="export/unknown-array.json", + acquired_at_ms=1, + ) + + backfill_historical_revision_evidence(tmp_path) + + with sqlite3.connect(tmp_path / "index.db") as conn: + assert conn.execute("SELECT session_id FROM sessions").fetchall() == [ + ("claude-ai-export:later-claude-provider",) + ] + + +def test_parsed_session_spill_uses_the_pinned_active_index_directory(tmp_path: Path) -> None: + """Repair spill churn follows the generation being repaired, not a shadow index.""" + archive_root = tmp_path / "archive" + archive_root.mkdir() + active_index = tmp_path / "external-generation" / "index.db" + active_index.parent.mkdir() + active_index.touch() + (archive_root / "index.db").touch() + + with revision_backfill._ParsedSessionSpill( + archive_root, + index_path=active_index, + max_cached_payload_bytes=None, + ) as spill: + assert spill.path.parent == active_index.parent + + @pytest.mark.parametrize( "source_path_suffix", [ @@ -190,6 +616,71 @@ def test_parse_one_refuses_declared_fact_artifacts(tmp_path: Path, source_path_s assert sessions == [] +def test_parse_one_recovery_accepts_session_evidence_at_a_declared_fact_path(tmp_path: Path) -> None: + """Source-only raw recovery decodes evidence before assigning fact taxonomy.""" + source_path = tmp_path / ".claude" / "projects" / "proj" / "subagents" / "workflows" / "wf" / "journal.jsonl" + payload = ( + b'{"parentUuid":null,"type":"user","sessionId":"wf","message":{"role":"user","content":"recover me"},' + b'"uuid":"user-1","timestamp":"2025-01-01T00:00:00Z"}\n' + b'{"parentUuid":"user-1","type":"assistant","sessionId":"wf","message":{"role":"assistant",' + b'"content":[{"type":"text","text":"recovered"}]},"uuid":"assistant-1",' + b'"timestamp":"2025-01-01T00:00:01Z"}\n' + ) + + sessions = _parse_one(Provider.CLAUDE_CODE, payload, str(source_path)) + + assert len(sessions) == 1 + assert [message.text for message in sessions[0].messages] == ["recover me", "recovered"] + + +def test_parse_stream_recovery_accepts_session_evidence_at_a_declared_fact_path(tmp_path: Path) -> None: + """The streamed replay route must inspect fact-path records before refusing them.""" + source_path = tmp_path / ".claude" / "projects" / "proj" / "subagents" / "workflows" / "wf" / "journal.jsonl" + payload = BytesIO( + b'{"parentUuid":null,"type":"user","sessionId":"wf","message":{"role":"user","content":"recover me"},' + b'"uuid":"user-1","timestamp":"2025-01-01T00:00:00Z"}\n' + b'{"parentUuid":"user-1","type":"assistant","sessionId":"wf","message":{"role":"assistant",' + b'"content":[{"type":"text","text":"recovered"}]},"uuid":"assistant-1",' + b'"timestamp":"2025-01-01T00:00:01Z"}\n' + ) + + sessions = revision_backfill._parse_stream(Provider.CLAUDE_CODE, payload, str(source_path)) + + assert len(sessions) == 1 + assert [message.text for message in sessions[0].messages] == ["recover me", "recovered"] + + +def test_backfill_scans_declared_stream_past_non_session_prefix(tmp_path: Path) -> None: + """Later Claude records outrank an arbitrarily long fact-artifact prefix. + + The production backfill route must not turn the first 64 non-session + records into permanent artifact authority when later records prove a + session. The archive assertion fails if replay rejects that bounded + prefix before parsing the rest of the retained JSONL. + """ + initialize_active_archive_root(tmp_path) + source_path = str(tmp_path / ".claude" / "projects" / "proj" / "subagents" / "workflows" / "wf" / "journal.jsonl") + payload = _relationship_index_jsonl_bytes(64) + ( + b'{"parentUuid":null,"type":"user","sessionId":"late-session","message":{"role":"user","content":"late evidence"},' + b'"uuid":"late-user","timestamp":"2025-01-01T00:00:00Z"}\n' + b'{"parentUuid":"late-user","type":"assistant","sessionId":"late-session","message":{"role":"assistant",' + b'"content":[{"type":"text","text":"late reply"}]},"uuid":"late-assistant",' + b'"timestamp":"2025-01-01T00:00:01Z"}\n' + ) + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + archive.write_raw_payload( + provider=Provider.CLAUDE_CODE, + payload=payload, + source_path=source_path, + acquired_at_ms=1, + ) + + backfill_historical_revision_evidence(tmp_path) + + with sqlite3.connect(tmp_path / "index.db") as conn: + assert conn.execute("SELECT session_id FROM sessions").fetchall() == [("claude-code-session:late-session",)] + + def _relationship_index_jsonl_bytes(count: int = 8) -> bytes: """Bytes shaped like the real sinex analysis artifact from polylogue-9ykn (gvgi): a graph-edge index sitting under a watched Claude Code directory, @@ -350,6 +841,587 @@ def test_historical_backfill_streams_codex_raw_without_eager_blob_read( assert result.replayed_logical_sources == 1 +def _codex_thread_state_snapshot_bytes(tmp_path: Path, title: str) -> bytes: + state_path = tmp_path / f"{title}.sqlite" + with sqlite3.connect(state_path) as conn: + conn.executescript( + """ + CREATE TABLE threads ( + id TEXT PRIMARY KEY, title TEXT, cwd TEXT, created_at_ms INTEGER, + updated_at_ms INTEGER, source TEXT, model TEXT, agent_nickname TEXT, + agent_role TEXT, archived INTEGER + ); + CREATE TABLE thread_spawn_edges ( + parent_thread_id TEXT, child_thread_id TEXT, status TEXT + ); + """ + ) + conn.execute( + "INSERT INTO threads VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ("codex-state-thread", title, "/work", 1, 1, "cli", None, None, None, 0), + ) + conn.commit() + return state_path.read_bytes() + + +def test_codex_state_replay_applies_payload_budget_before_sqlite_parse(tmp_path: Path) -> None: + """A bounded census defers a state snapshot before it can write evidence.""" + initialize_active_archive_root(tmp_path) + payload = _codex_thread_state_snapshot_bytes(tmp_path, "oversized state") + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.CODEX, + payload=payload, + source_path=str(tmp_path / "codex" / "state_5.sqlite"), + acquired_at_ms=1, + ) + + with pytest.raises(revision_backfill.RawRevisionReplayResourceBlockedError) as blocked: + census_historical_revision_evidence(tmp_path, max_payload_bytes=1) + + assert blocked.value.raw_ids == (raw_id,) + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute( + "SELECT parsed_at_ms, parse_error FROM raw_sessions WHERE raw_id = ?", (raw_id,) + ).fetchone() == (None, None) + assert conn.execute("SELECT COUNT(*) FROM raw_hook_events").fetchone() == (0,) + + +def test_backfill_replays_codex_state_by_latest_raw_observation(tmp_path: Path) -> None: + """A retained A -> B -> A state sequence leaves A's title current. + + Reacquiring A reuses its content-derived raw id, so this proves replay + orders its snapshot application by the latest durable raw-payload receipt, + not ``raw_sessions.acquired_at_ms`` from A's first observation. + """ + initialize_active_archive_root(tmp_path) + source_path = str(tmp_path / "codex" / "state_5.sqlite") + snapshot_a = _codex_thread_state_snapshot_bytes(tmp_path, "title A") + snapshot_b = _codex_thread_state_snapshot_bytes(tmp_path, "title B") + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + archive.write_raw_payload( + provider=Provider.CODEX, payload=snapshot_a, source_path=source_path, acquired_at_ms=1 + ) + archive.write_raw_payload( + provider=Provider.CODEX, payload=snapshot_b, source_path=source_path, acquired_at_ms=1 + ) + archive.write_raw_payload( + provider=Provider.CODEX, payload=snapshot_a, source_path=source_path, acquired_at_ms=1 + ) + + census_historical_revision_evidence(tmp_path) + + with sqlite3.connect(tmp_path / "source.db") as conn: + payload_json = conn.execute( + "SELECT payload_json FROM raw_hook_events WHERE hook_event_id = 'codex-thread-title:codex-state-thread'" + ).fetchone() + assert payload_json is not None + assert json.loads(str(payload_json[0]))["title"] == "title A" + + +def test_backfill_replays_equal_time_codex_state_by_raw_acquisition_order(tmp_path: Path) -> None: + """Equal-time Codex snapshots retain the later raw insertion as authority.""" + initialize_active_archive_root(tmp_path) + source_path = str(tmp_path / "codex" / "state_5.sqlite") + older_snapshot = _codex_thread_state_snapshot_bytes(tmp_path, "older title") + newer_snapshot = _codex_thread_state_snapshot_bytes(tmp_path, "newer title") + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + archive.write_raw_payload( + provider=Provider.CODEX, + payload=older_snapshot, + source_path=source_path, + acquired_at_ms=1, + raw_id="z-older-state", + ) + archive.write_raw_payload( + provider=Provider.CODEX, + payload=newer_snapshot, + source_path=source_path, + acquired_at_ms=1, + raw_id="a-newer-state", + ) + + census_historical_revision_evidence(tmp_path) + + with sqlite3.connect(tmp_path / "source.db") as conn: + payload_json = conn.execute( + "SELECT payload_json FROM raw_hook_events WHERE hook_event_id = 'codex-thread-title:codex-state-thread'" + ).fetchone() + assert payload_json is not None + assert json.loads(str(payload_json[0]))["title"] == "newer title" + + +def test_backfill_terminalizes_source_only_declared_artifact(tmp_path: Path) -> None: + """Replay turns a decoded fact-sidecar raw into terminal source authority. + + This exercises the same retained-raw replay path as recovery: the + source-only raw starts pending, the parser confirms it is a workflow + artifact, and the source tier must retain both typed artifact evidence and + a successful parse receipt so it is not selected forever. + """ + initialize_active_archive_root(tmp_path) + source_path = str(tmp_path / ".claude" / "projects" / "proj" / "subagents" / "workflows" / "wf" / "journal.jsonl") + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.CLAUDE_CODE, + payload=b'{"contentKey":"workflow-artifact","agentId":"agent"}\n', + source_path=source_path, + acquired_at_ms=1, + ) + + backfill_historical_revision_evidence(tmp_path) + + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute( + "SELECT parsed_at_ms IS NOT NULL FROM raw_sessions WHERE raw_id = ?", (raw_id,) + ).fetchone() == (1,) + assert conn.execute("SELECT parse_as_session FROM raw_artifacts WHERE raw_id = ?", (raw_id,)).fetchone() == (0,) + assert conn.execute("SELECT status FROM raw_membership_census WHERE raw_id = ?", (raw_id,)).fetchone() == ( + "non_session", + ) + assert conn.execute( + "SELECT status, logical_keys_json FROM raw_authority_parser_census WHERE raw_id = ?", (raw_id,) + ).fetchone() == ("complete", "[]") + + +@pytest.mark.asyncio +async def test_backfill_terminalizes_detected_unknown_empty_artifact(tmp_path: Path) -> None: + """Detected provider evidence must survive an empty retained replay.""" + initialize_active_archive_root(tmp_path) + source_path = str(tmp_path / ".claude" / "projects" / "proj" / "subagents" / "workflows" / "wf" / "journal.jsonl") + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.UNKNOWN, + payload=( + b'{"type":"file-history-snapshot","messageId":"history-message",' + b'"sessionId":"history-only-session","snapshot":{"trackedFileBackups":{}}}\n' + ), + source_path=source_path, + acquired_at_ms=1, + ) + + backfill_historical_revision_evidence(tmp_path) + + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute( + "SELECT origin, detected_provider, parsed_at_ms IS NOT NULL FROM raw_sessions WHERE raw_id = ?", (raw_id,) + ).fetchone() == ( + "unknown-export", + "claude-code", + 1, + ) + assert conn.execute("SELECT parse_as_session FROM raw_artifacts WHERE raw_id = ?", (raw_id,)).fetchone() == (0,) + assert conn.execute( + "SELECT status, logical_keys_json FROM raw_authority_parser_census WHERE raw_id = ?", (raw_id,) + ).fetchone() == ("complete", "[]") + + terminal_artifact_id = str( + conn.execute("SELECT artifact_id FROM raw_artifacts WHERE raw_id = ?", (raw_id,)).fetchone()[0] + ) + + backend = SQLiteBackend(db_path=tmp_path / "index.db") + try: + record = await backend.get_raw_session(raw_id) + assert record is not None + refreshed = inspect_raw_artifact(record, blob_store=BlobStore(tmp_path / "blob")) + assert refreshed.observation_id == terminal_artifact_id + assert await backend.save_artifact_observation(refreshed) is False + finally: + await backend.close() + + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute( + """ + SELECT COUNT(*) FROM raw_artifacts + WHERE origin = 'claude-code-session' AND source_path = ? AND source_index = 0 + """, + (source_path,), + ).fetchone() == (1,) + + +def test_backfill_persists_detected_provider_for_empty_ordinary_session_path(tmp_path: Path) -> None: + """Empty replay retains parser identity without mutating acquisition identity.""" + initialize_active_archive_root(tmp_path) + source_path = str(tmp_path / ".claude" / "projects" / "proj" / "history-only-session.jsonl") + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.UNKNOWN, + payload=( + b'{"type":"file-history-snapshot","messageId":"history-message",' + b'"sessionId":"history-only-session","snapshot":{"trackedFileBackups":{}}}\n' + ), + source_path=source_path, + acquired_at_ms=1, + ) + + census_historical_revision_evidence(tmp_path) + + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute( + "SELECT origin, detected_provider, parsed_at_ms IS NOT NULL FROM raw_sessions WHERE raw_id = ?", (raw_id,) + ).fetchone() == ("unknown-export", "claude-code", 1) + assert conn.execute("SELECT COUNT(*) FROM raw_artifacts WHERE raw_id = ?", (raw_id,)).fetchone() == (0,) + assert conn.execute("SELECT status FROM raw_membership_census WHERE raw_id = ?", (raw_id,)).fetchone() == ( + "non_session", + ) + + with ArchiveStore.open_existing(tmp_path, read_only=True) as archive: + assert archive.raw_membership_census_rows([raw_id])[0][2] + + with sqlite3.connect(tmp_path / "source.db") as conn: + assert active_raw_retention_authority( + conn, + index_db_path=tmp_path / "index.db", + ) == RawRetentionAuthority( + protected_raw_ids=frozenset({raw_id}), + eligible_raw_ids=frozenset(), + ) + + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + assert ( + archive.write_raw_payload( + provider=Provider.UNKNOWN, + payload=( + b'{"type":"file-history-snapshot","messageId":"history-message",' + b'"sessionId":"history-only-session","snapshot":{"trackedFileBackups":{}}}\n' + ), + source_path=source_path, + acquired_at_ms=2, + ) + == raw_id + ) + + +def test_backfill_leaves_undetected_empty_raw_replayable(tmp_path: Path) -> None: + """An unknown shape is not terminal merely because it produced no sessions.""" + initialize_active_archive_root(tmp_path) + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.UNKNOWN, + payload=b'{"future_provider_shape":true}\n', + source_path=str(tmp_path / "future.jsonl"), + acquired_at_ms=1, + ) + + census_historical_revision_evidence(tmp_path) + + with ArchiveStore.open_existing(tmp_path, read_only=True) as archive: + assert not archive.raw_membership_census_rows([raw_id])[0][2] + + +def test_backfill_retires_stale_revision_governance_for_empty_replay(tmp_path: Path) -> None: + """A current zero-session parse cannot remain in a stale full-revision plan.""" + initialize_active_archive_root(tmp_path) + source_path = str(tmp_path / ".claude" / "projects" / "proj" / "history-only-session.jsonl") + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.CLAUDE_CODE, + payload=b'{"type":"file-history-snapshot","sessionId":"history-only","snapshot":{}}\n', + source_path=source_path, + acquired_at_ms=1, + ) + archive.bind_raw_revision( + raw_id, + RawRevisionEnvelope( + logical_source_key="claude-code-session:stale-session", + kind=RawRevisionKind.FULL, + source_revision=raw_id, + acquisition_generation=0, + authority=RawRevisionAuthority.QUARANTINED, + ), + ) + + census_historical_revision_evidence(tmp_path) + + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute( + "SELECT logical_source_key, revision_kind, revision_authority FROM raw_sessions WHERE raw_id = ?", (raw_id,) + ).fetchone() == (None, "unknown", "quarantined") + + +def test_backfill_preserves_empty_append_revision_governance(tmp_path: Path) -> None: + """A terminal empty APPEND remains reconstructible through its byte envelope.""" + initialize_active_archive_root(tmp_path) + source_path = str(tmp_path / ".claude" / "projects" / "proj" / "append.jsonl") + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.CLAUDE_CODE, + payload=b'{"type":"file-history-snapshot","sessionId":"append-only","snapshot":{}}\n', + source_path=source_path, + source_index=0, + acquired_at_ms=1, + ) + archive.bind_raw_revision( + raw_id, + RawRevisionEnvelope( + logical_source_key="claude-code-session:append-only", + kind=RawRevisionKind.APPEND, + source_revision=raw_id, + predecessor_source_revision="predecessor-revision", + predecessor_raw_id="predecessor-raw", + baseline_raw_id="baseline-raw", + append_start_offset=10, + append_end_offset=20, + acquisition_generation=2, + authority=RawRevisionAuthority.BYTE_PROVEN, + ), + ) + + census_historical_revision_evidence(tmp_path) + + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute( + """ + SELECT logical_source_key, revision_kind, predecessor_source_revision, + predecessor_raw_id, baseline_raw_id, append_start_offset, + append_end_offset, acquisition_generation, revision_authority + FROM raw_sessions WHERE raw_id = ? + """, + (raw_id,), + ).fetchone() == ( + "claude-code-session:append-only", + "append", + "predecessor-revision", + "predecessor-raw", + "baseline-raw", + 10, + 20, + 2, + "byte_proven", + ) + assert conn.execute("SELECT status FROM raw_membership_census WHERE raw_id = ?", (raw_id,)).fetchone() == ( + "non_session", + ) + + +def test_backfill_fallback_terminalization_preserves_each_source_index(tmp_path: Path) -> None: + """A deferred byte-growth member keeps its own artifact coordinate.""" + initialize_active_archive_root(tmp_path) + source_path = str(tmp_path / ".claude" / "projects" / "proj" / "subagents" / "workflows" / "wf" / "journal.jsonl") + older_payload = b'{"contentKey":"older","agentId":"agent"}\n' + head_payload = older_payload + b'{"contentKey":"head","agentId":"agent"}\n' + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + archive.write_raw_payload( + provider=Provider.CLAUDE_CODE, + payload=older_payload, + source_path=source_path, + source_index=4, + acquired_at_ms=1, + ) + archive.write_raw_payload( + provider=Provider.CLAUDE_CODE, + payload=head_payload, + source_path=source_path, + source_index=9, + acquired_at_ms=2, + ) + + census_historical_revision_evidence(tmp_path) + + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT source_index FROM raw_artifacts ORDER BY source_index").fetchall() == [(4,), (9,)] + + +def test_terminal_artifact_receipts_roll_back_together(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """A failed terminal census cannot expose only its artifact carrier.""" + initialize_active_archive_root(tmp_path) + source_path = str(tmp_path / ".claude" / "projects" / "proj" / "subagents" / "workflows" / "wf" / "journal.jsonl") + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.CLAUDE_CODE, + payload=b'{"contentKey":"workflow-artifact","agentId":"agent"}\n', + source_path=source_path, + acquired_at_ms=1, + ) + + def fail_census(*_args: object, **_kwargs: object) -> None: + raise RuntimeError("injected terminal census failure") + + monkeypatch.setattr(ArchiveStore, "replace_raw_membership_census", fail_census) + with pytest.raises(RuntimeError, match="injected terminal census failure"): + backfill_historical_revision_evidence(tmp_path) + + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT COUNT(*) FROM raw_artifacts WHERE raw_id = ?", (raw_id,)).fetchone() == (0,) + assert conn.execute("SELECT parsed_at_ms FROM raw_sessions WHERE raw_id = ?", (raw_id,)).fetchone() == (None,) + assert conn.execute( + "SELECT COUNT(*) FROM raw_authority_parser_census WHERE raw_id = ?", (raw_id,) + ).fetchone() == (0,) + + +def test_batched_terminal_artifact_receipts_roll_back_together(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Batched empty outcomes retain one transaction through their batch boundary.""" + initialize_active_archive_root(tmp_path) + raw_ids: list[str] = [] + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + for index in range(2): + raw_ids.append( + archive.write_raw_payload( + provider=Provider.CLAUDE_CODE, + payload=f'{{"contentKey":"workflow-{index}","agentId":"agent"}}\n'.encode(), + source_path=str( + tmp_path + / ".claude" + / "projects" + / "proj" + / "subagents" + / "workflows" + / f"wf-{index}" + / "journal.jsonl" + ), + acquired_at_ms=index + 1, + ) + ) + + original_replace = ArchiveStore.replace_raw_membership_census + calls = 0 + + def fail_second_census( + self: ArchiveStore, + raw_id: str, + sessions: list[ParsedSession] | None, + *, + parser_fingerprint: str, + censused_at_ms: int, + detail: str = "", + retire_full_revision_governance: bool = False, + manage_transaction: bool = True, + ) -> None: + nonlocal calls + calls += 1 + if calls == 2: + raise RuntimeError("injected second terminal census failure") + original_replace( + self, + raw_id, + sessions, + parser_fingerprint=parser_fingerprint, + censused_at_ms=censused_at_ms, + detail=detail, + retire_full_revision_governance=retire_full_revision_governance, + manage_transaction=manage_transaction, + ) + + monkeypatch.setattr(ArchiveStore, "replace_raw_membership_census", fail_second_census) + with pytest.raises(RuntimeError, match="injected second terminal census failure"): + census_historical_revision_evidence(tmp_path, commit_batch_size=2) + + with sqlite3.connect(tmp_path / "source.db") as conn: + placeholders = ", ".join("?" for _ in raw_ids) + assert conn.execute( + f"SELECT COUNT(*) FROM raw_artifacts WHERE raw_id IN ({placeholders})", raw_ids + ).fetchone() == (0,) + assert conn.execute( + f"SELECT COUNT(*) FROM raw_authority_parser_census WHERE raw_id IN ({placeholders})", raw_ids + ).fetchone() == (0,) + + +def test_backfill_preserves_latest_terminal_artifact_observation(tmp_path: Path) -> None: + """A delayed older replay cannot replace a newer coordinate carrier.""" + initialize_active_archive_root(tmp_path) + source_path = str(tmp_path / ".claude" / "projects" / "proj" / "subagents" / "workflows" / "wf" / "journal.jsonl") + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + older_raw_id = archive.write_raw_payload( + provider=Provider.CLAUDE_CODE, + payload=b'{"contentKey":"workflow-artifact","agentId":"old"}\n', + source_path=source_path, + acquired_at_ms=1, + raw_id="z-older-artifact", + ) + newer_raw_id = archive.write_raw_payload( + provider=Provider.CLAUDE_CODE, + payload=b'{"contentKey":"workflow-artifact","agentId":"new"}\n', + source_path=source_path, + acquired_at_ms=2, + raw_id="a-newer-artifact", + ) + + backfill_historical_revision_evidence(tmp_path) + + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT COUNT(*) FROM raw_artifacts").fetchone() == (1,) + assert conn.execute("SELECT raw_id, last_observed_at_ms FROM raw_artifacts").fetchone() == (newer_raw_id, 2) + assert older_raw_id > newer_raw_id + assert conn.execute( + "SELECT COUNT(*) FROM raw_sessions WHERE raw_id IN (?, ?) AND parsed_at_ms IS NOT NULL", + (older_raw_id, newer_raw_id), + ).fetchone() == (2,) + + with ArchiveStore.open_existing(tmp_path, read_only=True) as archive: + assert all(row[2] for row in archive.raw_membership_census_rows([older_raw_id, newer_raw_id])) + + +def test_backfill_uses_raw_observation_order_for_equal_time_artifacts(tmp_path: Path) -> None: + """Legacy receipt-free observations use raw insertion order, not raw-id order.""" + initialize_active_archive_root(tmp_path) + source_path = str(tmp_path / ".claude" / "projects" / "proj" / "subagents" / "workflows" / "wf" / "journal.jsonl") + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + older_raw_id = archive.write_raw_payload( + provider=Provider.CLAUDE_CODE, + payload=b'{"contentKey":"workflow-artifact","agentId":"old"}\n', + source_path=source_path, + acquired_at_ms=1, + raw_id="a-older-artifact", + ) + newer_raw_id = archive.write_raw_payload( + provider=Provider.CLAUDE_CODE, + payload=b'{"contentKey":"workflow-artifact","agentId":"new"}\n', + source_path=source_path, + acquired_at_ms=1, + raw_id="z-newer-artifact", + ) + + with sqlite3.connect(tmp_path / "source.db") as conn: + # Legacy rows can lack receipts entirely. The fallback must compare + # both observations through raw_sessions, never one rowid per table. + conn.execute("DELETE FROM blob_refs WHERE ref_id IN (?, ?)", (older_raw_id, newer_raw_id)) + conn.commit() + + backfill_historical_revision_evidence(tmp_path) + + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT COUNT(*) FROM raw_artifacts").fetchone() == (1,) + assert conn.execute("SELECT raw_id, last_observed_at_ms FROM raw_artifacts").fetchone() == (newer_raw_id, 1) + assert older_raw_id < newer_raw_id + + +def test_backfill_preserves_latest_repeated_artifact_observation(tmp_path: Path) -> None: + """A -> B -> A reacquisition restores A as the coordinate authority.""" + initialize_active_archive_root(tmp_path) + source_path = str(tmp_path / ".claude" / "projects" / "proj" / "subagents" / "workflows" / "wf" / "journal.jsonl") + payload_a = b'{"contentKey":"workflow-artifact","agentId":"a"}\n' + payload_b = b'{"contentKey":"workflow-artifact","agentId":"b"}\n' + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_a = archive.write_raw_payload( + provider=Provider.CLAUDE_CODE, + payload=payload_a, + source_path=source_path, + acquired_at_ms=1, + ) + raw_b = archive.write_raw_payload( + provider=Provider.CLAUDE_CODE, + payload=payload_b, + source_path=source_path, + acquired_at_ms=2, + ) + assert ( + archive.write_raw_payload( + provider=Provider.CLAUDE_CODE, + payload=payload_a, + source_path=source_path, + acquired_at_ms=3, + ) + == raw_a + ) + + backfill_historical_revision_evidence(tmp_path) + + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT COUNT(*) FROM raw_artifacts").fetchone() == (1,) + assert conn.execute("SELECT raw_id, last_observed_at_ms FROM raw_artifacts").fetchone() == (raw_a, 3) + assert raw_a != raw_b + + def test_historical_backfill_selects_prefix_newest_independent_of_acquisition_order(tmp_path: Path) -> None: initialize_active_archive_root(tmp_path) baseline = ( diff --git a/tests/unit/storage/test_archive_tiers_archive.py b/tests/unit/storage/test_archive_tiers_archive.py index 41fb39da51..99c548d462 100644 --- a/tests/unit/storage/test_archive_tiers_archive.py +++ b/tests/unit/storage/test_archive_tiers_archive.py @@ -105,6 +105,15 @@ def acquire_then_replace( assert not (root / ".maintenance-state").exists() +def test_source_tier_acquisition_does_not_resolve_active_index(tmp_path: Path) -> None: + """Acquire-only writes remain available while the derived pointer is unreadable.""" + initialize_active_archive_root(tmp_path) + (tmp_path / ".index-active-pointer").write_bytes(b"\xff") + + with ArchiveStore.open_source_tier_acquisition(tmp_path) as archive: + assert archive.source_db_path == tmp_path / "source.db" + + def test_active_archive_root_facade_writes_reads_and_searches_archive_db(tmp_path: Path) -> None: session = ParsedSession( source_name=Provider.CODEX, diff --git a/tests/unit/storage/test_archive_tiers_source_write.py b/tests/unit/storage/test_archive_tiers_source_write.py index 0cafa8418e..7223e8902f 100644 --- a/tests/unit/storage/test_archive_tiers_source_write.py +++ b/tests/unit/storage/test_archive_tiers_source_write.py @@ -276,6 +276,56 @@ def test_source_artifact_upsert_keeps_coordinate_deduplication_and_raw_failure_f assert tuple(ordinary) == ("ordinary-coordinate", raw_ids[0], "session_export") +def test_source_artifact_upsert_refreshes_current_equal_time_carrier(tmp_path: Path) -> None: + """The current raw may refine its own coordinate even at the same timestamp.""" + conn = _connect(tmp_path / "source.db") + raw_id = write_source_raw_session( + conn, + origin=Origin.CODEX_SESSION, + source_path="/tmp/current.jsonl", + source_index=0, + payload=b"current", + acquired_at_ms=1, + ) + upsert_raw_artifact( + conn, + raw_id, + ArchiveSourceArtifact( + artifact_id="deferred-current", + origin=Origin.CODEX_SESSION, + source_path="/tmp/current.jsonl", + source_index=0, + artifact_kind="deferred_cas_frontier", + classification_reason="deferred", + support_status=ArtifactSupportStatus.PARTIAL_DECODE, + ), + ) + upsert_raw_artifact( + conn, + raw_id, + ArchiveSourceArtifact( + artifact_id="terminal-current", + origin=Origin.CODEX_SESSION, + source_path="/tmp/current.jsonl", + source_index=0, + artifact_kind="terminal_corrupt_input", + classification_reason="corrupt", + support_status=ArtifactSupportStatus.DECODE_FAILED, + ), + ) + + row = conn.execute( + "SELECT artifact_id, artifact_kind, support_status, classification_reason FROM raw_artifacts" + ).fetchone() + assert row is not None + assert tuple(row) == ( + "deferred-current", + "terminal_corrupt_input", + ArtifactSupportStatus.DECODE_FAILED.value, + "corrupt", + ) + + def test_archive_tiers_source_writer_replays_hook_events_idempotently(tmp_path: Path) -> None: conn = _connect(tmp_path / "source.db") payload = b'{"kind":"session","messages":["hello"]}' diff --git a/tests/unit/storage/test_artifact_loss_surfacing.py b/tests/unit/storage/test_artifact_loss_surfacing.py index 5487c01c48..3912ad8419 100644 --- a/tests/unit/storage/test_artifact_loss_surfacing.py +++ b/tests/unit/storage/test_artifact_loss_surfacing.py @@ -1,19 +1,22 @@ -"""Regression tests: artifact DECODE_FAILED/PARTIAL_DECODE covers the whole file (#1745). +"""Regression tests: artifact inspection covers the whole retained stream. -The artifact support status is derived from raw inspection. Inspection reads -only a 64 KB prefix to bound memory, so malformed JSONL content *past* the -prefix used to be invisible and the artifact was never flagged. These tests -assert that loss past the prefix is surfaced via a full-scan fallback. +Inspection starts from a 64 KB prefix to bound memory, then uses rolling stream +passes for whole-file loss accounting and positive session evidence. These +tests preserve both duties without weakening definitive sidecar exclusions. """ from __future__ import annotations from collections.abc import Iterator +from io import BytesIO from pathlib import Path import pytest +from polylogue.archive.raw_payload.decode import scan_jsonl_session_artifact from polylogue.core.enums import ArtifactSupportStatus, Provider +from polylogue.core.json import JSONValue +from polylogue.schemas.observation import schema_cluster_id from polylogue.storage.artifacts.inspection import ( _INSPECTION_PREFIX_BYTES, inspect_raw_artifact, @@ -41,13 +44,14 @@ def _write_record( content: bytes, source_path: str, source_name: str = "claude-code", + provider: Provider = Provider.CLAUDE_CODE, ) -> RawSessionRecord: raw_id, blob_size = store.write_from_bytes(content) return RawSessionRecord( raw_id=raw_id, source_name=source_name, source_path=source_path, - payload_provider=Provider.CLAUDE_CODE, + payload_provider=provider, source_index=None, blob_size=blob_size, acquired_at="2026-01-01T00:00:00+00:00", @@ -103,3 +107,197 @@ def test_clean_large_jsonl_is_not_flagged(blob_store: BlobStore) -> None: ArtifactSupportStatus.PARTIAL_DECODE, ArtifactSupportStatus.DECODE_FAILED, } + + +def test_large_codex_stream_is_not_terminalized_from_session_meta_prefix(blob_store: BlobStore) -> None: + session_meta = b'{"type":"session_meta","payload":{"id":"large-codex"}}\n' + message = ( + b'{"type":"response_item","payload":{"type":"message","id":"message-1",' + b'"role":"user","content":[{"type":"input_text","text":"hello"}]}}\n' + ) + padding = ( + b'{"type":"response_item","payload":{"type":"token_count","padding":"' + + (b"x" * (_INSPECTION_PREFIX_BYTES * 2)) + + b'"}}\n' + ) + content = session_meta + message + padding + assert len(content) > _INSPECTION_PREFIX_BYTES + + record = _write_record( + blob_store, + content=content, + source_path="codex/large-session.jsonl", + source_name="codex", + provider=Provider.CODEX, + ) + observation = inspect_raw_artifact(record) + + assert observation.parse_as_session is True + assert observation.artifact_kind == "session_record_stream" + assert observation.classification_reason == "parser-supported Codex session record stream" + + +def test_codex_stream_recovers_when_first_record_exceeds_inspection_prefix(blob_store: BlobStore) -> None: + session_meta = ( + b'{"type":"session_meta","payload":{"id":"large-first-record","base_instructions":{"text":"' + + (b"x" * (_INSPECTION_PREFIX_BYTES * 2)) + + b'"}}}\n' + ) + message = ( + b'{"type":"response_item","payload":{"type":"message","id":"message-1",' + b'"role":"user","content":[{"type":"input_text","text":"hello"}]}}\n' + ) + assert session_meta.find(b"\n") > _INSPECTION_PREFIX_BYTES + + record = _write_record( + blob_store, + content=session_meta + message, + source_path="codex/large-first-record.jsonl", + source_name="codex", + provider=Provider.CODEX, + ) + observation = inspect_raw_artifact(record) + + assert observation.parse_as_session is True + assert observation.artifact_kind == "session_record_stream" + assert observation.wire_format == "jsonl" + assert observation.decode_error is None + assert observation.malformed_jsonl_lines == 0 + assert observation.support_status is ArtifactSupportStatus.SUPPORTED_PARSEABLE + assert observation.resolved_package_version == "v1" + assert observation.resolved_element_kind == "session_record_stream" + expected_message: JSONValue = { + "type": "response_item", + "payload": { + "type": "message", + "id": "message-1", + "role": "user", + "content": [{"type": "input_text", "text": "hello"}], + }, + } + assert observation.cohort_id == schema_cluster_id([expected_message], "session_record_stream") + + +def test_recovery_reader_discards_oversized_record_in_bounded_chunks() -> None: + class BoundedReadlineStream(BytesIO): + def readline(self, size: int | None = -1, /) -> bytes: + assert isinstance(size, int) + assert 0 < size <= _INSPECTION_PREFIX_BYTES + 1 + return super().readline(size) + + oversized = b'{"ignored":"' + (b"x" * (_INSPECTION_PREFIX_BYTES * 3)) + b'"}\n' + message = ( + b'{"type":"response_item","payload":{"type":"message","id":"message-1",' + b'"role":"user","content":[{"type":"input_text","text":"hello"}]}}\n' + ) + + scan = scan_jsonl_session_artifact( + BoundedReadlineStream(oversized + message), + provider=Provider.CODEX, + source_path="codex/bounded.jsonl", + max_record_bytes=_INSPECTION_PREFIX_BYTES, + ) + + assert scan.artifact is not None + assert scan.artifact.parse_as_session is True + assert scan.oversized_records == 1 + assert len(scan.sample) == 1 + + +@pytest.mark.parametrize( + ("provider", "source_name"), + [ + pytest.param(Provider.CLAUDE_CODE, "claude-code", id="claude-code"), + pytest.param(Provider.CODEX, "codex", id="codex"), + ], +) +def test_single_oversized_provider_record_under_weak_path_remains_parse_candidate( + blob_store: BlobStore, + provider: Provider, + source_name: str, +) -> None: + if provider is Provider.CLAUDE_CODE: + content = ( + b'{"type":"user","uuid":"message-1","sessionId":"oversized-session",' + b'"parentUuid":null,"message":{"role":"user","content":"' + + (b"x" * (_INSPECTION_PREFIX_BYTES * 2)) + + b'"}}\n' + ) + else: + content = ( + b'{"type":"response_item","payload":{"type":"message","id":"message-1",' + b'"role":"user","content":[{"type":"input_text","text":"' + + (b"x" * (_INSPECTION_PREFIX_BYTES * 2)) + + b'"}]}}\n' + ) + assert content.find(b"\n") > _INSPECTION_PREFIX_BYTES + record = _write_record( + blob_store, + content=content, + source_path=f"{source_name}/analysis/re-homed-session.jsonl", + source_name=source_name, + provider=provider, + ) + + observation = inspect_raw_artifact(record) + + assert observation.parse_as_session is True + assert observation.schema_eligible is False + assert observation.artifact_kind == "session_record_stream" + assert observation.support_status is ArtifactSupportStatus.RECOGNIZED_UNPARSED + assert observation.malformed_jsonl_lines == 0 + assert observation.decode_error is None + + +def test_recovered_stream_retains_subagent_artifact_kind(blob_store: BlobStore) -> None: + oversized = b'{"ignored":"' + (b"x" * (_INSPECTION_PREFIX_BYTES * 2)) + b'"}\n' + message = ( + b'{"parentUuid":null,"type":"user","sessionId":"agent-session",' + b'"message":{"role":"user","content":"hello"},' + b'"uuid":"user-1","timestamp":"2026-01-01T00:00:00Z"}\n' + ) + record = _write_record( + blob_store, + content=oversized + message, + source_path="projects/project/subagents/agent-abcd.jsonl", + ) + + observation = inspect_raw_artifact(record) + + assert observation.parse_as_session is True + assert observation.artifact_kind == "agent_transcript" + assert observation.cohort_id == schema_cluster_id( + [ + { + "parentUuid": None, + "type": "user", + "sessionId": "agent-session", + "message": {"role": "user", "content": "hello"}, + "uuid": "user-1", + "timestamp": "2026-01-01T00:00:00Z", + } + ], + "agent_transcript", + ) + + +def test_rolling_scan_preserves_tool_result_sidecar_exclusion(blob_store: BlobStore) -> None: + content = ( + b'{"parentUuid":null,"type":"user","sessionId":"embedded",' + b'"message":{"role":"user","content":"copied transcript"},' + b'"uuid":"user-1","timestamp":"2026-01-01T00:00:00Z"}\n' + b'{"parentUuid":"user-1","type":"assistant","sessionId":"embedded",' + b'"message":{"role":"assistant","content":[{"type":"text","text":"copied reply"}]},' + b'"uuid":"assistant-1","timestamp":"2026-01-01T00:00:01Z"}\n' + ) + record = _write_record( + blob_store, + content=content, + source_path="projects/project/session/tool-results/copied-transcript.jsonl", + ) + + observation = inspect_raw_artifact(record) + + assert observation.parse_as_session is False + assert observation.schema_eligible is False + assert observation.artifact_kind == "tool_result_sidecar" diff --git a/tests/unit/storage/test_blob_integrity.py b/tests/unit/storage/test_blob_integrity.py index a59be45beb..5083039bc2 100644 --- a/tests/unit/storage/test_blob_integrity.py +++ b/tests/unit/storage/test_blob_integrity.py @@ -14,6 +14,7 @@ from polylogue.archive import zip_admission from polylogue.archive.message.roles import Role from polylogue.core.enums import BlockType, Provider +from polylogue.core.raw_coordinates import zip_member_raw_id, zip_member_source_index from polylogue.sources.parsers.base import ParsedAttachment, ParsedContentBlock, ParsedMessage, ParsedSession from polylogue.storage import blob_integrity from polylogue.storage.blob_gc import run_blob_gc_report @@ -1021,6 +1022,111 @@ def fail_open(*args: object, **kwargs: object) -> object: assert reason == "ambiguous_container_member" +def test_blob_recovery_uses_v2_entry_ordinal_without_consuming_split_index(tmp_path: Path) -> None: + """ZIP coordinates survive one replacement and authorize the next recovery.""" + initialize_active_archive_root(tmp_path) + source_db = tmp_path / "source.db" + store = BlobStore(tmp_path / "blob") + zip_source = tmp_path / "duplicate-v2.zip" + member = "sessions/duplicate.json" + old_selected_payloads = (b'{"member":"old-first-one"}', b'{"member":"old-second-one"}') + current_member_payloads = ( + b'[{"member":"current-first-zero"},{"member":"current-first-one"}]', + b'[{"member":"current-second-zero"},{"member":"current-second-one"}]', + ) + current_selected_payloads = ( + b'{"member":"current-first-one"}', + b'{"member":"current-second-one"}', + ) + split_index = 1 + with zipfile.ZipFile(zip_source, "w") as archive: + archive.writestr(member, current_member_payloads[0]) + with pytest.warns(UserWarning, match="Duplicate name"): + archive.writestr(member, current_member_payloads[1]) + + source_path = f"{zip_source}:{member}" + old_hashes = tuple(hashlib.sha256(payload).hexdigest() for payload in old_selected_payloads) + current_hashes = tuple(hashlib.sha256(payload).hexdigest() for payload in current_selected_payloads) + coordinates = tuple( + zip_member_source_index(entry_ordinal=ordinal, split_index=split_index) + for ordinal in range(len(current_member_payloads)) + ) + raw_ids = tuple( + zip_member_raw_id( + source_path=source_path, + entry_ordinal=ordinal, + split_index=split_index, + blob_hash=old_hashes[ordinal], + ) + for ordinal in range(len(current_member_payloads)) + ) + with sqlite3.connect(source_db) as conn: + conn.executemany( + """ + INSERT INTO raw_sessions ( + raw_id, origin, native_id, source_path, source_index, blob_hash, + blob_size, acquired_at_ms, file_mtime_ms + ) VALUES (?, 'codex-session', NULL, ?, ?, ?, ?, 1, 1) + """, + [ + (raw_id, source_path, source_index, bytes.fromhex(blob_hash), len(payload)) + for raw_id, source_index, blob_hash, payload in zip( + raw_ids, coordinates, old_hashes, old_selected_payloads, strict=True + ) + ], + ) + conn.executemany( + """ + INSERT INTO blob_refs (blob_hash, ref_id, ref_type, source_path, size_bytes, acquired_at_ms) + VALUES (?, ?, 'raw_payload', ?, ?, 1) + """, + [ + (bytes.fromhex(blob_hash), raw_id, source_path, len(payload)) + for raw_id, blob_hash, payload in zip(raw_ids, old_hashes, old_selected_payloads, strict=True) + ], + ) + + first = replace_raw_backed_blob_reference_debt_from_source( + source_db, + store=store, + dry_run=False, + manifest_path=tmp_path / "duplicate-v2-first-replacement.jsonl", + ) + + assert first.replaced_rows == 2 + assert first.written_blobs == 2 + assert all(store.exists(blob_hash) for blob_hash in current_hashes) + with sqlite3.connect(source_db) as conn: + assert conn.execute( + "SELECT raw_id, lower(hex(blob_hash)), source_index FROM raw_sessions ORDER BY source_index" + ).fetchall() == list(zip(raw_ids, current_hashes, coordinates, strict=True)) + assert conn.execute( + "SELECT raw_id, coordinate_format, entry_ordinal, split_index " + "FROM raw_container_coordinates ORDER BY entry_ordinal" + ).fetchall() == [ + (raw_ids[0], "zip-v2", 0, split_index), + (raw_ids[1], "zip-v2", 1, split_index), + ] + + for blob_hash in current_hashes: + store.blob_path(blob_hash).unlink() + + second = replace_raw_backed_blob_reference_debt_from_source( + source_db, + store=store, + dry_run=False, + manifest_path=tmp_path / "duplicate-v2-second-replacement.jsonl", + ) + + assert second.replaced_rows == 2 + assert second.written_blobs == 2 + assert tuple(store.read_all(blob_hash) for blob_hash in current_hashes) == current_selected_payloads + with sqlite3.connect(source_db) as conn: + assert conn.execute( + "SELECT raw_id, lower(hex(blob_hash)), source_index FROM raw_sessions ORDER BY source_index" + ).fetchall() == list(zip(raw_ids, current_hashes, coordinates, strict=True)) + + def test_blob_recovery_rejects_oversized_container_member_before_open( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/storage/test_browser_capture_origin_repair.py b/tests/unit/storage/test_browser_capture_origin_repair.py index b39a8ee732..591d45d781 100644 --- a/tests/unit/storage/test_browser_capture_origin_repair.py +++ b/tests/unit/storage/test_browser_capture_origin_repair.py @@ -606,6 +606,27 @@ def test_unified_frontier_applies_browser_origin_without_incident_receipt(tmp_pa assert not (tmp_path / "recovery").exists() +def test_unified_frontier_strategy_uses_the_selected_active_generation(tmp_path: Path) -> None: + raw_id = _seed_mismatched_browser_head(tmp_path) + active_dir = tmp_path / "generations" / "active" + active_dir.mkdir(parents=True) + active_index = active_dir / "index.db" + (tmp_path / "index.db").rename(active_index) + (tmp_path / "index.db").write_bytes(b"shadow index is not authoritative") + (tmp_path / ".index-active-pointer").write_text(str(active_index), encoding="utf-8") + config = Config( + archive_root=tmp_path, + render_root=tmp_path / "render", + sources=[], + ) + + census = inspect_raw_authority_frontier(config) + selected = next(item for item in census.items if item.raw_id == raw_id) + + assert selected.state is RawAuthorityFrontierState.SAFELY_REKEYABLE + assert selected.actuator is RawAuthorityActuator.COPY_FORWARD_ORIGIN + + def test_unified_frontier_restores_equivalent_canonical_browser_head(tmp_path: Path) -> None: mismatched_raw_id = _seed_mismatched_browser_head(tmp_path) canonical_raw_id = _seed_equivalent_canonical_head(tmp_path, mismatched_raw_id) diff --git a/tests/unit/storage/test_duplicate_raw_identity_repair.py b/tests/unit/storage/test_duplicate_raw_identity_repair.py index 4d8f6a6134..1f8105d9d3 100644 --- a/tests/unit/storage/test_duplicate_raw_identity_repair.py +++ b/tests/unit/storage/test_duplicate_raw_identity_repair.py @@ -184,6 +184,26 @@ def test_unified_frontier_census_plans_duplicate_alias_with_stable_evidence(tmp_ assert json.loads(persisted[3])["accepted_content_hash"] +def test_duplicate_alias_census_uses_active_generation_not_shadow_index(tmp_path: Path) -> None: + stale_raw_id, _canonical_raw_id, _session_id, _logical_key = _seed_duplicate_raw_pair(tmp_path) + active_index = tmp_path / "generations" / "active" / "index.db" + active_index.parent.mkdir(parents=True) + (tmp_path / "index.db").replace(active_index) + (tmp_path / ".index-active-pointer").write_text(str(active_index), encoding="utf-8") + (tmp_path / "index.db").write_bytes(b"not a sqlite database") + + config = Config( + archive_root=tmp_path, + render_root=tmp_path / "render", + sources=[], + ) + census = inspect_raw_authority_frontier(config) + + duplicate = next(item for item in census.items if item.raw_id == stale_raw_id) + assert duplicate.state is RawAuthorityFrontierState.DUPLICATE_ALIAS + assert duplicate.actuator is RawAuthorityActuator.FOLD_DUPLICATE_ALIAS + + def test_unified_frontier_census_prioritizes_missing_bytes_over_safe_actuation(tmp_path: Path) -> None: stale_raw_id, _canonical_raw_id, _session_id, _logical_key = _seed_duplicate_raw_pair(tmp_path) with sqlite3.connect(tmp_path / "source.db") as conn: diff --git a/tests/unit/storage/test_parse_tracking.py b/tests/unit/storage/test_parse_tracking.py index 35a5251de4..6aea9b4e89 100644 --- a/tests/unit/storage/test_parse_tracking.py +++ b/tests/unit/storage/test_parse_tracking.py @@ -194,6 +194,81 @@ async def test_update_raw_state_truncates_error_fields(self, backend: SQLiteBack assert rec.validation_error is not None assert len(rec.validation_error) == 2000 + @pytest.mark.parametrize("wall_clock_ms", [1000, 999]) + async def test_failed_validation_after_parse_advances_past_identical_or_backward_clock( + self, backend: SQLiteBackend, monkeypatch: pytest.MonkeyPatch, wall_clock_ms: int + ) -> None: + """A failed revalidation cannot tie or precede the parse it supersedes.""" + from polylogue.storage.sqlite.queries import raw_state as raw_state_queries + + await self._save_raw(backend, raw_id="parse-then-failed-validation") + await backend.update_raw_state( + "parse-then-failed-validation", + state=RawSessionStateUpdate(parsed_at="1970-01-01T00:00:01Z"), + ) + monkeypatch.setattr(raw_state_queries, "_now_ms", lambda: wall_clock_ms) + await backend.mark_raw_validated("parse-then-failed-validation", status="failed", error="rejected") + + with sqlite3.connect(backend._source_db_path) as conn: + row = conn.execute( + "SELECT parsed_at_ms, validated_at_ms, validation_status FROM raw_sessions WHERE raw_id = ?", + ("parse-then-failed-validation",), + ).fetchone() + assert row == (1000, 1001, "failed") + + @pytest.mark.parametrize( + ("parsed_at", "expected"), + [ + ("1970-01-01T00:00:01Z", (1001, 1000, "failed")), + ("1970-01-01T00:00:00.999Z", (1001, 1000, "failed")), + ], + ) + async def test_successful_parse_after_validation_advances_past_identical_or_backward_clock( + self, + backend: SQLiteBackend, + monkeypatch: pytest.MonkeyPatch, + parsed_at: str, + expected: tuple[int, int, str], + ) -> None: + """A later parse wins even if its injected wall clock is older.""" + from polylogue.storage.sqlite.queries import raw_state as raw_state_queries + + await self._save_raw(backend, raw_id="validation-then-parse") + monkeypatch.setattr(raw_state_queries, "_now_ms", lambda: 1000) + await backend.mark_raw_validated("validation-then-parse", status="failed", error="rejected") + await backend.update_raw_state( + "validation-then-parse", + state=RawSessionStateUpdate(parsed_at=parsed_at, parse_error=None), + ) + + with sqlite3.connect(backend._source_db_path) as conn: + row = conn.execute( + "SELECT parsed_at_ms, validated_at_ms, validation_status FROM raw_sessions WHERE raw_id = ?", + ("validation-then-parse",), + ).fetchone() + assert row == expected + + async def test_malformed_parse_timestamp_cannot_clear_existing_parse_authority( + self, backend: SQLiteBackend + ) -> None: + await self._save_raw(backend, raw_id="malformed-parse-timestamp") + await backend.update_raw_state( + "malformed-parse-timestamp", + state=RawSessionStateUpdate(parsed_at="1970-01-01T00:00:01Z"), + ) + + with pytest.raises(ValueError, match="parsed_at must be a valid timestamp"): + await backend.update_raw_state( + "malformed-parse-timestamp", + state=RawSessionStateUpdate(parsed_at="not-a-timestamp"), + ) + + with sqlite3.connect(backend._source_db_path) as conn: + row = conn.execute( + "SELECT parsed_at_ms FROM raw_sessions WHERE raw_id = ?", ("malformed-parse-timestamp",) + ).fetchone() + assert row == (1000,) + class TestMarkRawValidated: """Tests for mark_raw_validated backend method.""" diff --git a/tests/unit/storage/test_raw_authority_ledger.py b/tests/unit/storage/test_raw_authority_ledger.py index 945c57ab9d..4c4064927b 100644 --- a/tests/unit/storage/test_raw_authority_ledger.py +++ b/tests/unit/storage/test_raw_authority_ledger.py @@ -20,6 +20,7 @@ from polylogue.storage import raw_authority as raw_authority_mod from polylogue.storage import raw_reconciler as raw_reconciler_mod from polylogue.storage import repair as repair_mod +from polylogue.storage.archive_identity import resolve_active_index_path from polylogue.storage.archive_readiness import raw_materialization_readiness_snapshot, raw_materialization_ready from polylogue.storage.blob_store import BlobStore from polylogue.storage.raw_authority import ( @@ -42,11 +43,12 @@ from polylogue.storage.raw_reconciler import RawAuthorityFrontierState, inspect_raw_authority_frontier from polylogue.storage.repair import RepairResult, repair_raw_materialization from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore -from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root +from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root, initialize_archive_database +from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier def _config(root: Path) -> Config: - return Config(archive_root=root, render_root=root / "render", sources=[], db_path=root / "archive.db") + return Config(archive_root=root, render_root=root / "render", sources=[]) def _read_detail_document(root: Path, query_handle: str, *, chunk_chars: int = 256) -> dict[str, object]: @@ -900,13 +902,41 @@ def test_interrupted_apply_recovers_exact_durable_postconditions(tmp_path: Path) assert fts_hits_after_resume == fts_hits_before_resume +def test_interrupted_recovery_receives_repair_pinned_index_path(tmp_path: Path) -> None: + initialize_active_archive_root(tmp_path) + _write_codex_raw(tmp_path, native_id="pinned-recovery", source_path="pinned-recovery.jsonl", acquired_at_ms=1) + + with patch.object(repair_mod, "raw_replay_application_receipt", side_effect=RuntimeError("synthetic crash")): + with pytest.raises(RuntimeError, match="synthetic crash"): + repair_raw_materialization(_config(tmp_path)) + + expected_index = resolve_active_index_path(tmp_path) + recover = raw_authority_mod.recover_interrupted_raw_authority_censuses + received: list[Path | None] = [] + + def capture_pinned_index(root: Path, *, index_db_path: Path | None = None) -> tuple[tuple[str, JSONDocument], ...]: + received.append(index_db_path) + return recover(root, index_db_path=index_db_path) + + with patch.object(repair_mod, "recover_interrupted_raw_authority_censuses", side_effect=capture_pinned_index): + result = repair_raw_materialization(_config(tmp_path)) + + assert result.metrics["raw_materialization_recovered_census_count"] == 1.0 + assert received == [expected_index] + + def test_parsed_timestamp_without_exact_application_receipt_fails_closed(tmp_path: Path) -> None: initialize_active_archive_root(tmp_path) _write_codex_raw(tmp_path, native_id="receipt", source_path="receipt.jsonl", acquired_at_ms=1) real_receipt = raw_authority_mod.raw_replay_application_receipt - def incomplete_receipt(root: Path, plan: RawReplayPlan) -> JSONDocument: - payload = dict(real_receipt(root, plan)) + def incomplete_receipt( + root: Path, + plan: RawReplayPlan, + *, + index_db_path: Path | None = None, + ) -> JSONDocument: + payload = dict(real_receipt(root, plan, index_db_path=index_db_path)) payload["head_rows"] = [] return json_document(payload) @@ -920,6 +950,53 @@ def incomplete_receipt(root: Path, plan: RawReplayPlan) -> JSONDocument: ) +def test_application_receipt_reads_the_active_generation_not_shadow_index(tmp_path: Path) -> None: + initialize_active_archive_root(tmp_path) + raw_id = _write_codex_raw(tmp_path, native_id="active-receipt", source_path="active.jsonl", acquired_at_ms=1) + assert repair_raw_materialization(_config(tmp_path)).success is True + plan = build_raw_replay_plans(tmp_path, ((raw_id,),))[0] + active_index = tmp_path / "generations" / "active" / "index.db" + initialize_archive_database(active_index, ArchiveTier.INDEX) + (tmp_path / ".index-active-pointer").write_text(str(active_index), encoding="utf-8") + + receipt = raw_authority_mod.raw_replay_application_receipt(tmp_path, plan) + + assert receipt["index_db_path"] == str(active_index) + assert receipt["application_rows"] == [] + + +def test_replay_plan_build_and_validation_read_the_active_generation(tmp_path: Path) -> None: + initialize_active_archive_root(tmp_path) + raw_id = _write_codex_raw(tmp_path, native_id="active-plan", source_path="active-plan.jsonl", acquired_at_ms=1) + assert repair_raw_materialization(_config(tmp_path)).success is True + shadow_plan = build_raw_replay_plans(tmp_path, ((raw_id,),))[0] + assert shadow_plan.index_preconditions["sessions"] + + active_index = tmp_path / "generations" / "active" / "index.db" + initialize_archive_database(active_index, ArchiveTier.INDEX) + (tmp_path / ".index-active-pointer").write_text(str(active_index), encoding="utf-8") + + active_plan = build_raw_replay_plans(tmp_path, ((raw_id,),))[0] + valid, observed = validate_raw_replay_plan(tmp_path, shadow_plan) + + assert active_plan.index_preconditions["sessions"] == [] + assert valid is False + assert observed == active_plan.to_dict() + + +def test_frontier_census_reads_the_active_generation_not_shadow_index(tmp_path: Path) -> None: + initialize_active_archive_root(tmp_path) + active_index = tmp_path / "generations" / "active" / "index.db" + initialize_archive_database(active_index, ArchiveTier.INDEX) + (tmp_path / ".index-active-pointer").write_text(str(active_index), encoding="utf-8") + (tmp_path / "index.db").write_bytes(b"not a sqlite database") + + census = inspect_raw_authority_frontier(_config(tmp_path)) + + assert census.accepted_head_count == 0 + assert census.plan_count == 0 + + @pytest.mark.parametrize("field", ["session_id", "accepted_raw_id", "accepted_content_hash"]) def test_application_receipt_requires_exact_application_authority(tmp_path: Path, field: str) -> None: initialize_active_archive_root(tmp_path) @@ -1305,7 +1382,12 @@ def _seed_ambiguous_membership_component( conn.commit() (plan,) = build_raw_replay_plans(tmp_path, [(raw_id,)]) empty_remaining = repair_mod.RawMaterializationCandidates([], 0, 0) - (outcome,) = repair_mod._raw_replay_plan_outcomes(tmp_path, [plan], remaining=empty_remaining) + (outcome,) = repair_mod._raw_replay_plan_outcomes( + tmp_path, + resolve_active_index_path(tmp_path), + [plan], + remaining=empty_remaining, + ) return raw_id, outcome diff --git a/tests/unit/storage/test_raw_retention.py b/tests/unit/storage/test_raw_retention.py index 60011c82ca..f514ae99c3 100644 --- a/tests/unit/storage/test_raw_retention.py +++ b/tests/unit/storage/test_raw_retention.py @@ -12,6 +12,7 @@ from polylogue.archive.revision_authority import RawRevisionAuthority, RawRevisionEnvelope, RawRevisionKind from polylogue.core.enums import Provider from polylogue.sources.parsers.base import ParsedMessage, ParsedSession +from polylogue.storage import raw_retention as raw_retention_mod from polylogue.storage.archive_readiness import raw_materialization_readiness_snapshot from polylogue.storage.blob_store import BlobStore from polylogue.storage.raw_retention import ( @@ -30,6 +31,11 @@ from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root, initialize_archive_database from polylogue.storage.sqlite.archive_tiers.ops_write import upsert_ingest_cursor +from polylogue.storage.sqlite.archive_tiers.source_write import ( + ArchiveSourceArtifact, + upsert_raw_artifact, + write_source_raw_session, +) from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier @@ -37,6 +43,17 @@ def _write_blob(store: BlobStore, payload: bytes) -> tuple[str, int]: return store.write_from_bytes(payload) +def test_unavailable_frontier_preserves_empty_healthy_source_reason() -> None: + """A healthy source check must not inherit an unrelated pointer failure.""" + projection = raw_retention_mod.unknown_raw_frontier_integrity_projection( + "active index pointer unavailable", + missing_source_raw_status="healthy", + missing_source_raw_reason="", + ) + + assert projection.missing_source_raw_reason == "" + + def _ensure_archive_source_schema(conn: sqlite3.Connection) -> None: conn.execute( """CREATE TABLE raw_sessions ( @@ -417,6 +434,85 @@ def test_real_revision_receipt_authorizes_only_current_byte_head_supersession(tm ) +def test_scoped_terminal_retention_avoids_archive_wide_raw_inventory(tmp_path: Path) -> None: + """Terminal authority scans only the caller's source-path scope.""" + + source_db = tmp_path / "source.db" + source_path = tmp_path / "terminal.json" + initialize_archive_database(source_db, ArchiveTier.SOURCE) + with sqlite3.connect(source_db) as conn: + conn.execute( + """ + INSERT INTO raw_sessions ( + raw_id, origin, native_id, source_path, source_index, blob_hash, blob_size, acquired_at_ms + ) VALUES ('raw-terminal', 'unknown-export', 'terminal', ?, 0, ?, 1, 3) + """, + (str(source_path), bytes.fromhex("03" * 32)), + ) + conn.execute( + """ + INSERT INTO raw_artifacts ( + artifact_id, raw_id, origin, source_path, source_index, artifact_kind, + support_status, classification_reason, parse_as_session, schema_eligible, + malformed_jsonl_lines, first_observed_at_ms, last_observed_at_ms + ) VALUES ('artifact-terminal', 'raw-terminal', 'unknown-export', ?, 0, + 'workflow_journal', 'unknown', 'terminal', 0, 0, 0, 3, 3) + """, + (str(source_path),), + ) + conn.commit() + statements: list[str] = [] + conn.set_trace_callback(statements.append) + terminal_paths = raw_retention_mod._terminal_artifact_paths(conn, {str(source_path)}) + + raw_reads = [" ".join(statement.split()).upper() for statement in statements if "RAW_SESSIONS" in statement.upper()] + assert raw_reads + # Every raw_sessions scan in this cursor-scoped route must carry the + # source-path scope. Assert the SQL shape, not one historical rendering. + assert all("SOURCE_PATH IN (" in statement for statement in raw_reads) + assert terminal_paths == {str(source_path)} + + +def test_terminal_retention_batches_make_progress_when_failure_kinds_fill_budget( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source_db = tmp_path / "source.db" + source_paths = {tmp_path / "first.json", tmp_path / "second.json"} + initialize_archive_database(source_db, ArchiveTier.SOURCE) + with sqlite3.connect(source_db) as conn: + for index, source_path in enumerate(sorted(source_paths)): + raw_id = f"raw-{index}" + conn.execute( + """ + INSERT INTO raw_sessions ( + raw_id, origin, native_id, source_path, source_index, blob_hash, blob_size, acquired_at_ms + ) VALUES (?, 'unknown-export', ?, ?, 0, ?, 1, ?) + """, + (raw_id, raw_id, str(source_path), bytes([index + 1]) * 32, index + 1), + ) + conn.execute( + """ + INSERT INTO raw_artifacts ( + artifact_id, raw_id, origin, source_path, source_index, artifact_kind, + support_status, classification_reason, parse_as_session, schema_eligible, + malformed_jsonl_lines, first_observed_at_ms, last_observed_at_ms + ) VALUES (?, ?, 'unknown-export', ?, 0, + 'workflow_journal', 'unknown', 'terminal', 0, 0, 0, ?, ?) + """, + (f"artifact-{index}", raw_id, str(source_path), index + 1, index + 1), + ) + monkeypatch.setattr(raw_retention_mod, "RAW_FAILURE_EVIDENCE_KINDS", frozenset(f"raw-{i}" for i in range(250))) + monkeypatch.setattr( + raw_retention_mod, + "_TERMINAL_RAW_FAILURE_EVIDENCE_KINDS", + frozenset(f"terminal-{i}" for i in range(250)), + ) + + assert raw_retention_mod._terminal_artifact_paths(conn, {str(path) for path in source_paths}) == { + str(path) for path in source_paths + } + + def test_semantic_head_receipt_authorizes_no_raw_deletion(tmp_path: Path) -> None: old_raw_id, new_raw_id = _seed_real_full_supersession(tmp_path) with sqlite3.connect(tmp_path / "index.db") as conn: @@ -556,6 +652,430 @@ def test_active_raw_protection_rejects_empty_index_over_retained_source(tmp_path assert conn.execute("SELECT COUNT(*) FROM raw_sessions").fetchone() == (2,) +def test_current_terminal_artifact_authorizes_historical_raws_but_not_later_session_raw(tmp_path: Path) -> None: + """Terminal artifact authority follows the current coordinate receipt, not every old raw. + + ``raw_artifacts`` intentionally keeps one current carrier per ordinary + source coordinate while ``raw_sessions`` keeps every acquisition. A + current workflow/fact artifact may therefore retain historical raw rows + without a duplicate artifact receipt. Conversely, a later unclassified + raw must remove that exemption rather than allowing the old terminal + receipt to mask a cursor-authority gap. + """ + source_db = tmp_path / "source.db" + index_db = tmp_path / "index.db" + ops_db = tmp_path / "ops.db" + source_path = tmp_path / "journal.jsonl" + initialize_archive_database(source_db, ArchiveTier.SOURCE) + initialize_archive_database(index_db, ArchiveTier.INDEX) + with sqlite3.connect(source_db) as conn: + _insert_revision_raw( + conn, + raw_id="raw-journal-old", + source_path=source_path, + acquired_at_ms=1, + kind="unknown", + source_revision="old", + generation=0, + blob_size=10, + authority="quarantined", + ) + _insert_revision_raw( + conn, + raw_id="raw-journal-current", + source_path=source_path, + acquired_at_ms=2, + kind="unknown", + source_revision="current", + generation=0, + blob_size=20, + authority="quarantined", + ) + conn.execute( + """ + INSERT INTO raw_artifacts ( + artifact_id, raw_id, origin, source_path, source_index, + artifact_kind, support_status, classification_reason, + parse_as_session, schema_eligible, malformed_jsonl_lines, + first_observed_at_ms, last_observed_at_ms + ) VALUES (?, ?, 'claude-code-session', ?, 0, 'workflow_journal', + 'unknown', 'typed terminal artifact', 0, 0, 0, 1, 2) + """, + ("artifact-journal", "raw-journal-current", str(source_path)), + ) + conn.commit() + _seed_ops_cursor(ops_db, source_path=source_path, byte_offset=20) + + with sqlite3.connect(source_db) as conn: + authority = active_raw_retention_authority(conn, index_db_path=index_db) + snapshot = raw_frontier_integrity_snapshot(conn, index_db_path=index_db, ops_db_path=ops_db) + + assert authority == RawRetentionAuthority( + protected_raw_ids=frozenset({"raw-journal-old", "raw-journal-current"}), + eligible_raw_ids=frozenset(), + ) + assert snapshot.cursor_ahead_status == "healthy" + assert snapshot.cursor_authority_gap_count == 0 + + with sqlite3.connect(source_db) as conn: + _insert_revision_raw( + conn, + raw_id="raw-conversational-later", + source_path=source_path, + acquired_at_ms=3, + kind="full", + source_revision="later", + generation=0, + blob_size=30, + authority="asserted", + ) + conn.commit() + _seed_ops_cursor(ops_db, source_path=source_path, byte_offset=30) + + with sqlite3.connect(source_db) as conn: + with pytest.raises(RawRetentionSafetyError, match="index has no raw authority"): + active_raw_retention_authority(conn, index_db_path=index_db) + snapshot = raw_frontier_integrity_snapshot(conn, index_db_path=index_db, ops_db_path=ops_db) + + assert snapshot.cursor_ahead_status == "unknown" + assert snapshot.cursor_authority_gap_count == 1 + assert snapshot.cursor_authority_gap_samples[0].state == "source_raws_without_accepted_head" + + +def test_terminal_coordinate_uses_latest_repeated_raw_observation(tmp_path: Path) -> None: + """A→B→A ranks A's reacquisition receipt, not its first raw-row time.""" + source_db = tmp_path / "source.db" + index_db = tmp_path / "index.db" + source_path = tmp_path / "repeated.jsonl" + initialize_archive_database(source_db, ArchiveTier.SOURCE) + initialize_archive_database(index_db, ArchiveTier.INDEX) + with sqlite3.connect(source_db) as conn: + raw_a = write_source_raw_session( + conn, + origin="claude-code-session", + source_path=str(source_path), + source_index=0, + payload=b"session A", + acquired_at_ms=1, + ) + raw_b = write_source_raw_session( + conn, + origin="claude-code-session", + source_path=str(source_path), + source_index=0, + payload=b"terminal B", + acquired_at_ms=2, + ) + upsert_raw_artifact( + conn, + raw_b, + ArchiveSourceArtifact( + artifact_id="artifact-repeated-coordinate", + origin="claude-code-session", + source_path=str(source_path), + source_index=0, + artifact_kind="workflow_journal", + classification_reason="terminal B", + parse_as_session=False, + first_observed_at_ms=2, + last_observed_at_ms=2, + ), + ) + assert ( + write_source_raw_session( + conn, + origin="claude-code-session", + source_path=str(source_path), + source_index=0, + payload=b"session A", + acquired_at_ms=3, + ) + == raw_a + ) + assert conn.execute("SELECT acquired_at_ms FROM raw_sessions WHERE raw_id = ?", (raw_a,)).fetchone() == (1,) + assert conn.execute( + "SELECT acquired_at_ms FROM blob_refs WHERE ref_type = 'raw_payload' AND ref_id = ?", (raw_a,) + ).fetchone() == (3,) + + assert raw_retention_mod._terminal_artifact_paths(conn, {str(source_path)}) == set() + with pytest.raises(RawRetentionSafetyError, match="index has no raw authority"): + active_raw_retention_authority(conn, index_db_path=index_db) + + +def test_terminal_cursor_exemption_requires_every_source_coordinate(tmp_path: Path) -> None: + """A terminal sibling cannot hide an unheaded conversational coordinate.""" + + source_db = tmp_path / "source.db" + index_db = tmp_path / "index.db" + ops_db = tmp_path / "ops.db" + source_path = tmp_path / "bundle.jsonl" + initialize_archive_database(source_db, ArchiveTier.SOURCE) + initialize_archive_database(index_db, ArchiveTier.INDEX) + with sqlite3.connect(source_db) as conn: + conn.executemany( + """ + INSERT INTO raw_sessions ( + raw_id, origin, native_id, source_path, source_index, blob_hash, blob_size, acquired_at_ms + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + ("raw-terminal", "claude-code-session", "terminal", str(source_path), 0, bytes(32), 1, 1), + ("raw-session", "claude-code-session", "session", str(source_path), 1, bytes([1]) * 32, 1, 2), + ), + ) + conn.execute( + """ + INSERT INTO raw_artifacts ( + artifact_id, raw_id, origin, source_path, source_index, artifact_kind, + support_status, classification_reason, parse_as_session, schema_eligible, + malformed_jsonl_lines, first_observed_at_ms, last_observed_at_ms + ) VALUES (?, ?, ?, ?, ?, 'workflow_journal', 'unknown', 'terminal coordinate', 0, 0, 0, 1, 1) + """, + ("artifact-terminal", "raw-terminal", "claude-code-session", str(source_path), 0), + ) + conn.commit() + _seed_ops_cursor(ops_db, source_path=source_path, byte_offset=2) + + with sqlite3.connect(source_db) as conn: + snapshot = raw_frontier_integrity_snapshot(conn, index_db_path=index_db, ops_db_path=ops_db) + + assert snapshot.cursor_ahead_status == "unknown" + assert snapshot.cursor_authority_gap_count == 1 + assert snapshot.cursor_authority_gap_samples[0].state == "source_raws_without_accepted_head" + + +def test_resolution_carrier_cannot_authorize_cursor_without_accepted_head(tmp_path: Path) -> None: + """A superseded deferred-CAS receipt is resolution evidence, not terminal authority.""" + + source_db = tmp_path / "source.db" + index_db = tmp_path / "index.db" + ops_db = tmp_path / "ops.db" + source_path = tmp_path / "replaced-attempt.jsonl" + initialize_archive_database(source_db, ArchiveTier.SOURCE) + initialize_archive_database(index_db, ArchiveTier.INDEX) + with sqlite3.connect(source_db) as conn: + conn.execute( + """ + INSERT INTO raw_sessions ( + raw_id, origin, native_id, source_path, source_index, blob_hash, blob_size, acquired_at_ms + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, + ("raw-resolution", "claude-code-session", "resolution", str(source_path), 0, bytes(32), 1, 1), + ) + conn.execute( + """ + INSERT INTO raw_artifacts ( + artifact_id, raw_id, origin, source_path, source_index, artifact_kind, + support_status, classification_reason, parse_as_session, schema_eligible, + malformed_jsonl_lines, first_observed_at_ms, last_observed_at_ms + ) VALUES (?, ?, ?, ?, ?, ?, 'unknown', 'deferred attempt replaced', 0, 0, 0, 1, 1) + """, + ( + "artifact-resolution", + "raw-resolution", + "claude-code-session", + str(source_path), + 0, + "terminal_superseded_deferred_cas_frontier", + ), + ) + conn.commit() + _seed_ops_cursor(ops_db, source_path=source_path, byte_offset=1) + + with sqlite3.connect(source_db) as conn: + snapshot = raw_frontier_integrity_snapshot(conn, index_db_path=index_db, ops_db_path=ops_db) + + assert snapshot.cursor_ahead_status == "unknown" + assert snapshot.cursor_authority_gap_count == 1 + assert snapshot.cursor_authority_gap_samples[0].state == "source_raws_without_accepted_head" + + +def test_successful_reparse_revokes_stale_terminal_failure_cursor_authority(tmp_path: Path) -> None: + """A successful production parse state makes an old terminal carrier historical.""" + + initialize_active_archive_root(tmp_path) + source_path = tmp_path / "reparsed-export.json" + with sqlite3.connect(tmp_path / "source.db") as conn: + conn.execute( + """ + INSERT INTO raw_sessions ( + raw_id, origin, native_id, source_path, source_index, + blob_hash, blob_size, acquired_at_ms, parse_error, + validated_at_ms, validation_status, validation_error, + validation_drift_count, validation_mode + ) VALUES (?, ?, ?, ?, 0, ?, 1, 1, ?, 1, 'failed', ?, 3, 'strict') + """, + ( + "raw-terminal-reparsed", + "codex-session", + "reparsed", + str(source_path), + bytes(32), + "ValueError: unsupported export shape", + "schema validation failed", + ), + ) + conn.execute( + """ + INSERT INTO raw_artifacts ( + artifact_id, raw_id, origin, source_path, source_index, + artifact_kind, support_status, classification_reason, + parse_as_session, schema_eligible, malformed_jsonl_lines, + first_observed_at_ms, last_observed_at_ms + ) VALUES (?, ?, ?, ?, 0, ?, ?, ?, 0, 0, 0, 1, 1) + """, + ( + "artifact-terminal-reparsed", + "raw-terminal-reparsed", + "codex-session", + str(source_path), + "terminal_unsupported_shape", + "unsupported_parseable", + "terminal parse failure", + ), + ) + conn.commit() + _seed_ops_cursor(tmp_path / "ops.db", source_path=source_path, byte_offset=1) + + with sqlite3.connect(tmp_path / "source.db") as conn: + before = raw_frontier_integrity_snapshot( + conn, + index_db_path=tmp_path / "index.db", + ops_db_path=tmp_path / "ops.db", + ) + assert before.cursor_ahead_status == "healthy" + + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + archive.mark_raw_parse_succeeded("raw-terminal-reparsed", provider=Provider.CODEX) + + with sqlite3.connect(tmp_path / "source.db") as conn: + state = conn.execute( + """ + SELECT parsed_at_ms, parse_error, validated_at_ms, validation_status, + validation_error, validation_drift_count, validation_mode + FROM raw_sessions + WHERE raw_id = ? + """, + ("raw-terminal-reparsed",), + ).fetchone() + stale_artifact = conn.execute( + "SELECT artifact_kind FROM raw_artifacts WHERE raw_id = ?", + ("raw-terminal-reparsed",), + ).fetchone() + after = raw_frontier_integrity_snapshot( + conn, + index_db_path=tmp_path / "index.db", + ops_db_path=tmp_path / "ops.db", + ) + + assert state is not None + parsed_at_ms, parse_error, validated_at_ms, validation_status, validation_error, drift_count, mode = state + assert parsed_at_ms is not None + assert (parse_error, validated_at_ms, validation_status, validation_error, drift_count, mode) == ( + None, + 1, + "failed", + "schema validation failed", + 3, + "strict", + ) + assert stale_artifact == ("terminal_unsupported_shape",) + assert after.cursor_ahead_status == "unknown" + assert after.cursor_authority_gap_count == 1 + assert after.cursor_authority_gap_samples[0].state == "source_raws_without_accepted_head" + + +def test_successful_reparse_revokes_stale_ordinary_artifact_cursor_authority(tmp_path: Path) -> None: + """An ordinary sidecar classification cannot stay terminal after a later parse.""" + + initialize_active_archive_root(tmp_path) + source_path = tmp_path / "reparsed-sidecar.json" + with sqlite3.connect(tmp_path / "source.db") as conn: + conn.execute( + """ + INSERT INTO raw_sessions ( + raw_id, origin, native_id, source_path, source_index, + blob_hash, blob_size, acquired_at_ms, parsed_at_ms + ) VALUES (?, ?, ?, ?, 0, ?, 1, 1, 20) + """, + ( + "raw-ordinary-reparsed", + "codex-session", + "ordinary-reparsed", + str(source_path), + bytes(32), + ), + ) + conn.execute( + """ + INSERT INTO raw_artifacts ( + artifact_id, raw_id, origin, source_path, source_index, + artifact_kind, support_status, classification_reason, + parse_as_session, schema_eligible, malformed_jsonl_lines, + first_observed_at_ms, last_observed_at_ms + ) VALUES (?, ?, ?, ?, 0, 'session_metadata', 'supported_parseable', ?, 0, 0, 0, 10, 10) + """, + ( + "artifact-ordinary-reparsed", + "raw-ordinary-reparsed", + "codex-session", + str(source_path), + "ordinary sidecar classification before parser support", + ), + ) + conn.commit() + _seed_ops_cursor(tmp_path / "ops.db", source_path=source_path, byte_offset=1) + + with sqlite3.connect(tmp_path / "source.db") as conn: + snapshot = raw_frontier_integrity_snapshot( + conn, + index_db_path=tmp_path / "index.db", + ops_db_path=tmp_path / "ops.db", + ) + + assert snapshot.cursor_ahead_status == "unknown" + assert snapshot.cursor_authority_gap_count == 1 + assert snapshot.cursor_authority_gap_samples[0].state == "source_raws_without_accepted_head" + + +def test_terminal_artifact_retention_batches_source_paths_below_sqlite_limit(tmp_path: Path) -> None: + """Terminal evidence remains protectable when more than one SQL batch is needed.""" + + source_db = tmp_path / "source.db" + index_db = tmp_path / "index.db" + initialize_archive_database(source_db, ArchiveTier.SOURCE) + initialize_archive_database(index_db, ArchiveTier.INDEX) + with sqlite3.connect(source_db) as conn: + for number in range(501): + source_path = tmp_path / f"terminal-{number}.jsonl" + raw_id = f"raw-terminal-{number}" + conn.execute( + """ + INSERT INTO raw_sessions ( + raw_id, origin, native_id, source_path, source_index, blob_hash, blob_size, acquired_at_ms + ) VALUES (?, 'claude-code-session', ?, ?, 0, ?, 1, ?) + """, + (raw_id, raw_id, str(source_path), number.to_bytes(32, "big"), number), + ) + conn.execute( + """ + INSERT INTO raw_artifacts ( + artifact_id, raw_id, origin, source_path, source_index, artifact_kind, + support_status, classification_reason, parse_as_session, schema_eligible, + malformed_jsonl_lines, first_observed_at_ms, last_observed_at_ms + ) VALUES (?, ?, 'claude-code-session', ?, 0, 'workflow_journal', + 'unknown', 'terminal', 0, 0, 0, ?, ?) + """, + (f"artifact-{number}", raw_id, str(source_path), number, number), + ) + conn.commit() + conn.setlimit(sqlite3.SQLITE_LIMIT_VARIABLE_NUMBER, 500) + authority = active_raw_retention_authority(conn, index_db_path=index_db) + + assert authority.protected_raw_ids == frozenset(f"raw-terminal-{number}" for number in range(501)) + assert authority.eligible_raw_ids == frozenset() + + def test_active_raw_protection_rejects_incomplete_predecessor_chain(tmp_path: Path) -> None: source_db = tmp_path / "source.db" index_db = tmp_path / "index.db" @@ -1768,6 +2288,38 @@ def test_raw_frontier_integrity_semantic_membership_cursor_is_intentionally_not_ assert snapshot.overall_status == "healthy" +def test_raw_frontier_integrity_reports_missing_semantic_head_source_raw(tmp_path: Path) -> None: + """Semantic membership skips byte validation only after finding its source raw.""" + + source_db = tmp_path / "source.db" + index_db = tmp_path / "index.db" + ops_db = tmp_path / "ops.db" + initialize_archive_database(source_db, ArchiveTier.SOURCE) + initialize_archive_database(index_db, ArchiveTier.INDEX) + _seed_index_authority( + index_db, + session_raw_id="raw-missing-semantic", + accepted_raw_id="raw-missing-semantic", + accepted_revision="semantic-revision", + generation=0, + frontier=1, + append_end_offset=None, + ) + with sqlite3.connect(index_db) as conn: + conn.execute("UPDATE raw_revision_heads SET accepted_frontier_kind = 'semantic'") + conn.commit() + initialize_archive_database(ops_db, ArchiveTier.OPS) + + with sqlite3.connect(source_db) as conn: + snapshot = raw_frontier_integrity_snapshot(conn, index_db_path=index_db, ops_db_path=ops_db) + + assert snapshot.broken_head_status == "violated" + assert snapshot.broken_head_count == 1 + assert snapshot.broken_head_checked_count == 1 + assert snapshot.broken_head_samples[0].accepted_raw_id == "raw-missing-semantic" + assert "missing from source tier" in snapshot.broken_head_samples[0].reason + + def test_raw_frontier_integrity_snapshot_cursor_at_exact_accepted_frontier_is_healthy(tmp_path: Path) -> None: """A cursor sitting exactly at the accepted frontier (not past it) is healthy. @@ -1952,6 +2504,92 @@ def test_raw_frontier_integrity_projection_preserves_violation_when_sibling_is_u assert projection.available is False +def test_raw_frontier_integrity_projection_follows_active_index_pointer(tmp_path: Path) -> None: + """A promoted index, rather than a stale conventional shadow, governs frontier health.""" + + source_db = tmp_path / "source.db" + shadow_index = tmp_path / "index.db" + active_index = tmp_path / "generations" / "active" / "index.db" + ops_db = tmp_path / "ops.db" + source_path = tmp_path / "session.jsonl" + source_path.write_text("{}\n", encoding="utf-8") + initialize_archive_database(source_db, ArchiveTier.SOURCE) + initialize_archive_database(shadow_index, ArchiveTier.INDEX) + initialize_archive_database(active_index, ArchiveTier.INDEX) + initialize_archive_database(ops_db, ArchiveTier.OPS) + with sqlite3.connect(source_db) as conn: + _insert_revision_raw( + conn, + raw_id="raw-active", + source_path=source_path, + acquired_at_ms=1, + kind="full", + source_revision="revision-1", + generation=1, + blob_size=10, + ) + conn.commit() + _seed_index_authority( + active_index, + session_raw_id="raw-active", + accepted_raw_id="raw-active", + accepted_revision="revision-1", + generation=1, + frontier=10, + append_end_offset=None, + ) + _seed_ops_cursor(ops_db, source_path=source_path, byte_offset=10) + (tmp_path / ".index-active-pointer").write_text(f"{active_index}\n", encoding="utf-8") + + projection = raw_frontier_integrity_projection( + tmp_path, + {"available": True, "lost_source_evidence_count": 0}, + ) + + assert projection.broken_head_status == "healthy" + assert projection.cursor_ahead_status == "healthy" + assert projection.overall_status == "healthy" + assert projection.available is True + + +def test_raw_frontier_integrity_projection_reports_malformed_active_pointer(tmp_path: Path) -> None: + """Status reads degrade to an unavailable projection when a pointer is invalid.""" + initialize_archive_database(tmp_path / "source.db", ArchiveTier.SOURCE) + initialize_archive_database(tmp_path / "ops.db", ArchiveTier.OPS) + (tmp_path / ".index-active-pointer").write_text("relative/index.db\n", encoding="utf-8") + + projection = raw_frontier_integrity_projection( + tmp_path, + {"available": True, "lost_source_evidence_count": 0}, + ) + + assert projection.available is False + assert projection.overall_status == "unknown" + assert "active index pointer" in projection.broken_head_reason + + +def test_raw_frontier_projection_retains_known_missing_source_violation_when_pointer_is_invalid(tmp_path: Path) -> None: + """An unavailable active pointer cannot erase known source-tier loss.""" + initialize_archive_database(tmp_path / "source.db", ArchiveTier.SOURCE) + initialize_archive_database(tmp_path / "ops.db", ArchiveTier.OPS) + (tmp_path / ".index-active-pointer").write_text("relative/index.db\n", encoding="utf-8") + + projection = raw_frontier_integrity_projection( + tmp_path, + { + "available": True, + "lost_source_evidence_count": 1, + "lost_source_evidence_samples": [{"session_id": "missing-session"}], + }, + ) + + assert projection.available is False + assert projection.overall_status == "violated" + assert projection.missing_source_raw_status == "violated" + assert projection.missing_source_raw_count == 1 + assert projection.missing_source_raw_samples == ({"session_id": "missing-session"},) + + @pytest.mark.parametrize("index_kind", ["missing", "malformed"]) def test_raw_frontier_integrity_snapshot_unavailable_index_tier_is_unknown_never_healthy( tmp_path: Path, @@ -2007,6 +2645,22 @@ def test_raw_frontier_integrity_snapshot_unavailable_source_tier_is_unknown_neve assert "unreadable" in snapshot.broken_head_reason +def test_active_retention_translates_missing_source_authority_table(tmp_path: Path) -> None: + """Cleanup callers receive the typed fail-closed exception contract.""" + source_db = tmp_path / "source.db" + index_db = tmp_path / "index.db" + initialize_archive_database(index_db, ArchiveTier.INDEX) + with sqlite3.connect(source_db) as conn: + conn.execute("CREATE TABLE placeholder (id INTEGER PRIMARY KEY)") + conn.commit() + + with ( + sqlite3.connect(source_db) as conn, + pytest.raises(RawRetentionSafetyError, match="raw retention authority is unreadable"), + ): + active_raw_retention_authority(conn, index_db_path=index_db) + + def test_raw_frontier_integrity_snapshot_partial_source_schema_is_unknown_not_violated(tmp_path: Path) -> None: source_db = tmp_path / "source.db" index_db = tmp_path / "index.db" diff --git a/tests/unit/storage/test_repair.py b/tests/unit/storage/test_repair.py index 7cb78bcbe7..c82b2b9b8c 100644 --- a/tests/unit/storage/test_repair.py +++ b/tests/unit/storage/test_repair.py @@ -12,8 +12,9 @@ import pytest from polylogue.config import Config -from polylogue.core.enums import ArtifactSupportStatus +from polylogue.core.enums import ArtifactSupportStatus, Provider from polylogue.core.errors import RawCASFrontierError +from polylogue.core.json import json_document from polylogue.core.raw_failure_evidence import RawFailureEvidenceKind from polylogue.daemon.status import raw_failure_info_for_root from polylogue.maintenance.models import DerivedModelStatus @@ -23,14 +24,96 @@ from polylogue.storage.blob_store import BlobStore from polylogue.storage.insights.session.repair_assessment import assess_session_insight_repairs from polylogue.storage.insights.session.runtime import SessionInsightCounts, SessionInsightStatusSnapshot -from polylogue.storage.raw_authority import RawReplayPlan, RawReplayPlanOutcome -from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_archive_database +from polylogue.storage.raw.models import RawSessionStateUpdate +from polylogue.storage.raw_authority import RawReplayPlan, RawReplayPlanOutcome, RawReplayPlanStatus +from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root, initialize_archive_database from polylogue.storage.sqlite.archive_tiers.source_write import ArchiveSourceArtifact, upsert_raw_artifact from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier def _config(tmp_path: Path) -> Config: - return Config(archive_root=tmp_path, render_root=tmp_path, sources=[], db_path=tmp_path / "archive.db") + return Config(archive_root=tmp_path, render_root=tmp_path, sources=[]) + + +def test_raw_materialization_binds_current_generation_under_writer_lease( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Promotion cannot race generation resolution, replay, and postconditions.""" + from polylogue.storage.index_generation import RebuildLease, RebuildLeaseUnavailableError + + initialize_active_archive_root(tmp_path) + config = Config(archive_root=tmp_path, render_root=tmp_path, sources=[]) + active_index = tmp_path / "generations" / "active" / "index.db" + initialize_archive_database(active_index, ArchiveTier.INDEX) + (tmp_path / ".index-active-pointer").write_text(str(active_index), encoding="utf-8") + + class InnerReachedError(RuntimeError): + pass + + def inspect_inner(*_args: object, **_kwargs: object) -> Any: + assert config.current_db_path() == active_index + with pytest.raises(RebuildLeaseUnavailableError): + with RebuildLease(tmp_path): + pass + raise InnerReachedError + + monkeypatch.setattr(repair_mod, "_repair_raw_materialization", inspect_inner) + + with pytest.raises(InnerReachedError): + repair_mod.repair_raw_materialization(config) + with RebuildLease(tmp_path): + pass + + +def test_raw_snapshot_cleanup_binds_authority_and_delete_under_writer_lease( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Promotion cannot change the protected generation during destructive raw cleanup.""" + + from polylogue.storage.index_generation import RebuildLease, RebuildLeaseUnavailableError + + initialize_active_archive_root(tmp_path) + config = Config(archive_root=tmp_path, render_root=tmp_path, sources=[]) + + class InnerReachedError(RuntimeError): + pass + + def inspect_inner(*_args: object, **_kwargs: object) -> Any: + with pytest.raises(RebuildLeaseUnavailableError): + with RebuildLease(tmp_path): + pass + raise InnerReachedError + + monkeypatch.setattr(repair_mod, "_repair_superseded_raw_snapshots", inspect_inner) + + with pytest.raises(InnerReachedError): + repair_mod.repair_superseded_raw_snapshots(config) + with RebuildLease(tmp_path): + pass + + +def test_raw_materialization_returns_a_typed_failure_while_rebuild_owns_archive(tmp_path: Path) -> None: + """A lease conflict cannot abort a caller aggregating repair results.""" + from polylogue.storage.index_generation import RebuildLease + + initialize_active_archive_root(tmp_path) + with RebuildLease(tmp_path): + result = repair_mod.repair_raw_materialization(_config(tmp_path)) + + assert result.success is False + assert "offline index rebuild owns archive" in result.detail + + +def test_raw_snapshot_cleanup_returns_a_typed_failure_while_rebuild_owns_archive(tmp_path: Path) -> None: + """Destructive raw cleanup reports a lease conflict through RepairResult.""" + from polylogue.storage.index_generation import RebuildLease + + initialize_active_archive_root(tmp_path) + with RebuildLease(tmp_path): + result = repair_mod.repair_superseded_raw_snapshots(_config(tmp_path)) + + assert result.success is False + assert "offline index rebuild owns archive" in result.detail def test_raw_materialization_reparses_legacy_indexed_raw_before_receipting(tmp_path: Path) -> None: @@ -140,6 +223,17 @@ def _complete_bounded_raw_census(config: Config, *, limit: int) -> tuple[repair_ raise AssertionError("bounded raw census did not quiesce") +def _repair_after_persisted_census( + config: Config, + *, + dry_run: bool = False, + raw_artifact_id: str | None = None, +) -> repair_mod.RepairResult: + """Exercise replay only after the durable parser census reaches quiescence.""" + _complete_bounded_raw_census(config, limit=1_000) + return repair_mod.repair_raw_materialization(config, dry_run=dry_run, raw_artifact_id=raw_artifact_id) + + def _status( *, source_documents: int = 0, @@ -376,8 +470,7 @@ def fail_unrelated(*_args: object, **_kwargs: object) -> int: def test_raw_materialization_preview_counts_replayable_rows_without_erasing_missing_blobs(tmp_path: Path) -> None: config = _config(tmp_path) - initialize_archive_database(tmp_path / "source.db", ArchiveTier.SOURCE) - initialize_archive_database(tmp_path / "index.db", ArchiveTier.INDEX) + initialize_active_archive_root(tmp_path) blob_store = BlobStore(tmp_path / "blob") replayable_raw_id, replayable_size = blob_store.write_from_bytes(b'{"mapping":{}}') materialized_raw_id, materialized_size = blob_store.write_from_bytes(b'{"mapping":{"done":{}}}') @@ -449,39 +542,17 @@ def test_raw_materialization_preview_counts_replayable_rows_without_erasing_miss result = repair_mod.repair_raw_materialization(config, dry_run=True) assert result.repaired_count == 0 - assert result.success is True - assert result.metrics == { - "raw_materialization_candidate_count": 1.0, - "raw_materialization_selected_count": 1.0, - "raw_materialization_missing_blob_count": 1.0, - "raw_materialization_missing_blob_source_available_count": 0.0, - "raw_materialization_missing_blob_source_missing_count": 1.0, - "raw_materialization_already_parsed_count": 0.0, - "raw_materialization_total_blob_bytes": float(replayable_size), - "raw_materialization_max_blob_bytes": float(replayable_size), - "raw_materialization_selected_total_blob_bytes": float(replayable_size), - "raw_materialization_selected_max_blob_bytes": float(replayable_size), - "raw_materialization_adoption_deferred_count": 0.0, - "raw_materialization_authority_quarantined_count": 0.0, - "raw_materialization_byte_authority_fragment_count": 0.0, - "raw_materialization_byte_authority_pending_count": 0.0, - "raw_materialization_byte_authority_quarantined_count": 0.0, - "raw_materialization_before_component_count": 1.0, - "raw_materialization_selected_executable_component_count": 1.0, - "raw_materialization_selected_blocked_component_count": 0.0, - "raw_materialization_census_sequence": 1.0, - "raw_materialization_census_fixed_point": 0.0, - } - assert "per-session revision authority" in result.detail - assert "selected raw payload bytes total=" in result.detail - assert "largest=" in result.detail - assert "1 raw rows remain blocked by missing blobs (1 with source paths missing)" in result.detail + assert result.success is False + assert result.census_receipt is not None + assert result.census_receipt.quiescent is False + assert result.metrics["raw_materialization_census_incomplete_raw_count"] == 1.0 + assert result.metrics["raw_materialization_missing_blob_count"] == 1.0 + assert "persisted parser census" in result.detail def test_raw_materialization_replays_same_native_when_index_raw_link_is_dangling(tmp_path: Path) -> None: config = _config(tmp_path) - initialize_archive_database(tmp_path / "source.db", ArchiveTier.SOURCE) - initialize_archive_database(tmp_path / "index.db", ArchiveTier.INDEX) + initialize_active_archive_root(tmp_path) blob_store = BlobStore(tmp_path / "blob") replacement_raw_id, replacement_size = blob_store.write_from_bytes(b'{"mapping":{"replacement":{}}}') @@ -515,7 +586,7 @@ def test_raw_materialization_replays_same_native_when_index_raw_link_is_dangling ) index_conn.commit() - result = repair_mod.repair_raw_materialization(config, dry_run=True) + result = _repair_after_persisted_census(config, dry_run=True) assert result.success is True assert result.repaired_count == 0 @@ -526,9 +597,7 @@ def test_raw_materialization_split_root_routes_authority_replay(tmp_path: Path) configured_root = tmp_path / "configured" routed_root = tmp_path / "routed" configured_root.mkdir() - routed_root.mkdir() - initialize_archive_database(routed_root / "source.db", ArchiveTier.SOURCE) - initialize_archive_database(routed_root / "index.db", ArchiveTier.INDEX) + initialize_active_archive_root(routed_root) raw_id, raw_size = BlobStore(routed_root / "blob").write_from_bytes( b'{"mapping":{"routed":{"id":"routed","message":{"id":"m1","author":{"role":"user"},' b'"content":{"content_type":"text","parts":["hi"]}},"parent":null,"children":[]}},' @@ -561,7 +630,7 @@ def test_raw_materialization_split_root_routes_authority_replay(tmp_path: Path) ) backlog = repair_mod.raw_materialization_replay_backlog(config) - result = repair_mod.repair_raw_materialization(config) + result = _repair_after_persisted_census(config) assert backlog["execution_blocked"] is False assert backlog["execution_block_reason"] is None @@ -842,6 +911,188 @@ def test_raw_materialization_validation_failure_cannot_reuse_deferred_authority( assert backlog["candidate_count"] == 0 +def test_raw_materialization_replays_successful_raw_with_historical_validation_failure(tmp_path: Path) -> None: + """Index reset replays a successful raw while retaining its failed-validation history.""" + from polylogue.core.enums import Provider + from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore + from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root + + initialize_active_archive_root(tmp_path) + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.CODEX, + payload=( + b'{"type":"session_meta","payload":{"id":"historical-validation"}}\n' + b'{"type":"response_item","payload":{"type":"message","id":"m1","role":"user",' + b'"content":[{"type":"input_text","text":"repair retained validation"}]}}\n' + ), + source_path="historical-validation.jsonl", + acquired_at_ms=1, + ) + + assert repair_mod.repair_raw_materialization(_config(tmp_path)).success is True + + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + archive.finalize_raw_parse_state( + raw_id, + state=RawSessionStateUpdate( + parsed_at=None, + parse_error=None, + validation_status="failed", + validation_error="validator rejected an earlier observation", + ), + ) + archive.record_raw_failure_evidence( + raw_id, + provider=Provider.CODEX, + source_path="historical-validation.jsonl", + source_index=0, + acquired_at_ms=1, + kind=RawFailureEvidenceKind.TERMINAL_CORRUPT_INPUT, + ) + archive.mark_raw_parse_succeeded(raw_id, provider=Provider.CODEX) + + with sqlite3.connect(tmp_path / "source.db") as conn: + raw_state = conn.execute( + "SELECT parsed_at_ms, parse_error, validation_status, validation_error FROM raw_sessions WHERE raw_id = ?", + (raw_id,), + ).fetchone() + assert raw_state is not None + assert raw_state[0] is not None + assert raw_state[1:] == (None, "failed", "validator rejected an earlier observation") + assert conn.execute( + "SELECT artifact_kind, support_status FROM raw_artifacts WHERE raw_id = ?", + (raw_id,), + ).fetchone() == ("terminal_corrupt_input", "decode_failed") + + # A reset removes only the derived projection; durable raw evidence and + # its historical validation diagnosis remain available to replay. Leave + # the populated conventional index as a stale shadow: the production + # planner and replay postcondition must use this promoted empty generation. + active_index = tmp_path / "generations" / "active" / "index.db" + initialize_archive_database(active_index, ArchiveTier.INDEX) + (tmp_path / ".index-active-pointer").write_text(f"{active_index}\n", encoding="utf-8") + with sqlite3.connect(tmp_path / "index.db") as conn: + shadow_applications_before = conn.execute( + "SELECT COUNT(*) FROM raw_revision_applications WHERE raw_id = ?", (raw_id,) + ).fetchone() + + replay = repair_mod.repair_raw_materialization(_config(tmp_path)) + + assert replay.success is True + assert replay.repaired_count == 1 + with sqlite3.connect(active_index) as conn: + assert conn.execute("SELECT COUNT(*) FROM sessions WHERE raw_id = ?", (raw_id,)).fetchone() == (1,) + with sqlite3.connect(tmp_path / "index.db") as conn: + assert conn.execute("SELECT COUNT(*) FROM sessions WHERE raw_id = ?", (raw_id,)).fetchone() == (1,) + assert ( + conn.execute("SELECT COUNT(*) FROM raw_revision_applications WHERE raw_id = ?", (raw_id,)).fetchone() + == shadow_applications_before + ) + + +@pytest.mark.parametrize("validation_offset", [0, 1]) +def test_raw_materialization_refuses_non_parse_authoritative_validation_failure( + tmp_path: Path, validation_offset: int +) -> None: + """A newer failure or legacy tie must not replay and overwrite raw authority.""" + from polylogue.core.enums import Provider + from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore + from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root + + initialize_active_archive_root(tmp_path) + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.CODEX, + payload=( + b'{"type":"session_meta","payload":{"id":"later-validation-failure"}}\n' + b'{"type":"response_item","payload":{"type":"message","id":"m1","role":"user",' + b'"content":[{"type":"input_text","text":"current failure"}]}}\n' + ), + source_path="later-validation-failure.jsonl", + acquired_at_ms=1, + ) + + config = _config(tmp_path) + assert repair_mod.repair_raw_materialization(config).success is True + with sqlite3.connect(tmp_path / "source.db") as conn: + parsed_at_ms = int( + conn.execute("SELECT parsed_at_ms FROM raw_sessions WHERE raw_id = ?", (raw_id,)).fetchone()[0] + ) + conn.execute( + """ + UPDATE raw_sessions + SET validation_status = 'failed', validation_error = ?, validated_at_ms = ? + WHERE raw_id = ? + """, + ("strict validation rejected the later observation", parsed_at_ms + validation_offset, raw_id), + ) + conn.commit() + + active_index = tmp_path / "generations" / "after-validation" / "index.db" + initialize_archive_database(active_index, ArchiveTier.INDEX) + (tmp_path / ".index-active-pointer").write_text(f"{active_index}\n", encoding="utf-8") + + assert raw_id not in repair_mod._raw_materialization_candidate_ids(config).raw_ids + assert repair_mod.raw_materialization_replay_backlog(config)["candidate_count"] == 0 + + +def test_raw_replay_plan_marks_tied_validation_component_terminal(tmp_path: Path) -> None: + """A tied failed member cannot make an otherwise parsed component look executed.""" + from polylogue.core.enums import Provider + from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore + + initialize_active_archive_root(tmp_path) + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + parsed_raw_id = archive.write_raw_payload( + provider=Provider.CODEX, + payload=b'{"type":"session_meta","payload":{"id":"parsed-member"}}\n', + source_path="parsed-member.jsonl", + acquired_at_ms=1, + ) + tied_raw_id = archive.write_raw_payload( + provider=Provider.CODEX, + payload=b'{"type":"session_meta","payload":{"id":"tied-member"}}\n', + source_path="tied-member.jsonl", + acquired_at_ms=2, + ) + archive.finalize_raw_parse_state( + parsed_raw_id, + state=RawSessionStateUpdate(parsed_at="1970-01-01T00:00:00.001Z"), + ) + archive.finalize_raw_parse_state( + tied_raw_id, + state=RawSessionStateUpdate(parsed_at="1970-01-01T00:00:00.001Z"), + ) + + with sqlite3.connect(tmp_path / "source.db") as conn: + cursor = conn.execute( + """ + UPDATE raw_sessions + SET validation_status = 'failed', validation_error = ?, validated_at_ms = parsed_at_ms + WHERE raw_id = ? + """, + ("rejected at the same legacy millisecond", tied_raw_id), + ) + assert cursor.rowcount == 1 + conn.commit() + + plan = RawReplayPlan( + "raw-replay:tied-validation-component", + "0" * 64, + (parsed_raw_id, tied_raw_id), + ("codex:tied-validation-component",), + json_document({}), + json_document({}), + json_document({}), + ) + remaining = repair_mod.RawMaterializationCandidates(raw_ids=[], missing_blobs=0, already_parsed=0) + + outcome = repair_mod._raw_replay_plan_outcomes(tmp_path, tmp_path / "index.db", [plan], remaining=remaining)[0] + + assert outcome.status is RawReplayPlanStatus.TERMINAL + + @pytest.mark.parametrize("artifact_kind", ["deferred_hot_jsonl_capture", "deferred_claude_code_partial_jsonl"]) def test_raw_materialization_does_not_replay_hot_partial_capture(tmp_path: Path, artifact_kind: str) -> None: """Hot partial evidence stays deferred until a complete source observation arrives.""" @@ -1359,9 +1610,7 @@ def test_raw_materialization_split_root_classifies_parsed_sidecar_from_routed_bl configured_root = tmp_path / "configured" routed_root = tmp_path / "routed" configured_root.mkdir() - routed_root.mkdir() - initialize_archive_database(routed_root / "source.db", ArchiveTier.SOURCE) - initialize_archive_database(routed_root / "index.db", ArchiveTier.INDEX) + initialize_active_archive_root(routed_root) raw_id, raw_size = BlobStore(routed_root / "blob").write_from_bytes(b'{"type":"session_meta"}\n') with sqlite3.connect(routed_root / "source.db") as source_conn: source_conn.execute( @@ -1391,13 +1640,33 @@ def test_raw_materialization_split_root_classifies_parsed_sidecar_from_routed_bl db_path=routed_root / "index.db", ) - result = repair_mod.repair_raw_materialization(config, dry_run=True) + result = _repair_after_persisted_census(config, dry_run=True) assert result.success is True assert result.repaired_count == 0 assert result.metrics["raw_materialization_candidate_count"] == 0.0 +def test_raw_materialization_skips_current_non_session_census(tmp_path: Path) -> None: + """A successful zero-session census settles an otherwise unknown sidecar shape.""" + from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore + + initialize_active_archive_root(tmp_path) + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.CLAUDE_CODE, + payload=b'{"type":"queue-operation","operation":"compact"}\n', + source_path=str(tmp_path / "ordinary.jsonl"), + acquired_at_ms=1, + ) + + assert raw_id in repair_mod._raw_materialization_candidate_ids(_config(tmp_path)).raw_ids + + census_historical_revision_evidence(tmp_path) + + assert raw_id not in repair_mod._raw_materialization_candidate_ids(_config(tmp_path)).raw_ids + + def test_superseded_raw_cleanup_protects_split_index_referenced_raw_ids(tmp_path: Path) -> None: config = _config(tmp_path) initialize_archive_database(tmp_path / "source.db", ArchiveTier.SOURCE) @@ -1453,6 +1722,107 @@ def test_superseded_raw_cleanup_protects_split_index_referenced_raw_ids(tmp_path assert "skipped 1 active revision raw rows" in result.detail +def test_superseded_raw_cleanup_follows_active_index_pointer(tmp_path: Path) -> None: + config = _config(tmp_path) + initialize_archive_database(tmp_path / "source.db", ArchiveTier.SOURCE) + shadow_index = tmp_path / "index.db" + active_index = tmp_path / "generations" / "active" / "index.db" + initialize_archive_database(shadow_index, ArchiveTier.INDEX) + initialize_archive_database(active_index, ArchiveTier.INDEX) + source_file = tmp_path / "source.jsonl" + source_file.write_text("{}", encoding="utf-8") + + with sqlite3.connect(tmp_path / "source.db") as source_conn: + source_conn.executemany( + """ + INSERT INTO raw_sessions ( + raw_id, origin, native_id, source_path, source_index, blob_hash, blob_size, acquired_at_ms + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + ( + "raw-referenced-old", + "chatgpt-export", + "native-old", + str(source_file), + 0, + bytes.fromhex("11" * 32), + 10, + 1, + ), + ( + "raw-newer", + "chatgpt-export", + "native-newer", + str(source_file), + 0, + bytes.fromhex("22" * 32), + 11, + 2, + ), + ), + ) + source_conn.commit() + with sqlite3.connect(active_index) as index_conn: + index_conn.execute( + """ + INSERT INTO sessions (native_id, origin, raw_id, title, content_hash) + VALUES (?, ?, ?, ?, ?) + """, + ("native-old", "chatgpt-export", "raw-referenced-old", "old", bytes(32)), + ) + index_conn.commit() + (tmp_path / ".index-active-pointer").write_text(f"{active_index}\n", encoding="utf-8") + + result = repair_mod.repair_superseded_raw_snapshots(config, dry_run=True) + + assert result.success is True + assert result.repaired_count == 0 + assert "skipped 1 active revision raw rows" in result.detail + + +def test_superseded_raw_cleanup_preserves_explicit_index_override(tmp_path: Path) -> None: + """An explicit generation remains cleanup authority even when a pointer differs.""" + + initialize_archive_database(tmp_path / "source.db", ArchiveTier.SOURCE) + pointer_index = tmp_path / "generations" / "pointer" / "index.db" + explicit_index = tmp_path / "generations" / "explicit" / "index.db" + initialize_archive_database(pointer_index, ArchiveTier.INDEX) + initialize_archive_database(explicit_index, ArchiveTier.INDEX) + source_file = tmp_path / "source.jsonl" + source_file.write_text("{}", encoding="utf-8") + with sqlite3.connect(tmp_path / "source.db") as source_conn: + source_conn.executemany( + """ + INSERT INTO raw_sessions ( + raw_id, origin, native_id, source_path, source_index, blob_hash, blob_size, acquired_at_ms + ) VALUES (?, 'chatgpt-export', ?, ?, 0, ?, ?, ?) + """, + ( + ("raw-explicit", "native-explicit", str(source_file), bytes.fromhex("11" * 32), 10, 1), + ("raw-newer", "native-newer", str(source_file), bytes.fromhex("22" * 32), 11, 2), + ), + ) + source_conn.commit() + with sqlite3.connect(explicit_index) as index_conn: + index_conn.execute( + """ + INSERT INTO sessions (native_id, origin, raw_id, title, content_hash) + VALUES ('native-explicit', 'chatgpt-export', 'raw-explicit', 'explicit', ?) + """, + (bytes(32),), + ) + index_conn.commit() + (tmp_path / ".index-active-pointer").write_text(f"{pointer_index}\n", encoding="utf-8") + config = Config(archive_root=tmp_path, render_root=tmp_path, sources=[], db_path=explicit_index) + + result = repair_mod.repair_superseded_raw_snapshots(config, dry_run=True) + + assert result.success is True + assert result.repaired_count == 0 + assert "skipped 1 active revision raw rows" in result.detail + + def test_superseded_raw_cleanup_allows_history_before_active_full(tmp_path: Path) -> None: config = _config(tmp_path) initialize_archive_database(tmp_path / "source.db", ArchiveTier.SOURCE) @@ -1519,7 +1889,7 @@ def test_superseded_raw_cleanup_fails_closed_without_index(tmp_path: Path) -> No initialize_archive_database(tmp_path / "source.db", ArchiveTier.SOURCE) # This valid but unrelated legacy anchor must never authorize deletion # from the split archive_root/source.db file set. - initialize_archive_database(config.db_path, ArchiveTier.INDEX) + initialize_archive_database(tmp_path / "archive.db", ArchiveTier.INDEX) source_file = tmp_path / "source.jsonl" source_file.write_text("{}", encoding="utf-8") with sqlite3.connect(tmp_path / "source.db") as conn: @@ -1600,8 +1970,7 @@ def test_raw_materialization_retries_restored_missing_blob_parse_errors(tmp_path def test_raw_materialization_replays_parsed_rows_when_index_is_empty(tmp_path: Path) -> None: config = _config(tmp_path) - initialize_archive_database(tmp_path / "source.db", ArchiveTier.SOURCE) - initialize_archive_database(tmp_path / "index.db", ArchiveTier.INDEX) + initialize_active_archive_root(tmp_path) blob_store = BlobStore(tmp_path / "blob") raw_id, blob_size = blob_store.write_from_bytes(b'{"mapping":{"already-parsed":{}}}') @@ -1627,7 +1996,7 @@ def test_raw_materialization_replays_parsed_rows_when_index_is_empty(tmp_path: P ) source_conn.commit() - result = repair_mod.repair_raw_materialization(config, dry_run=True) + result = _repair_after_persisted_census(config, dry_run=True) assert result.repaired_count == 0 assert result.success is True @@ -1638,8 +2007,7 @@ def test_raw_materialization_replays_parsed_rows_when_index_is_empty(tmp_path: P def test_raw_materialization_replays_parsed_rows_after_interrupted_index_rebuild(tmp_path: Path) -> None: config = _config(tmp_path) - initialize_archive_database(tmp_path / "source.db", ArchiveTier.SOURCE) - initialize_archive_database(tmp_path / "index.db", ArchiveTier.INDEX) + initialize_active_archive_root(tmp_path) blob_store = BlobStore(tmp_path / "blob") remaining_raw_id, remaining_size = blob_store.write_from_bytes(b'{"mapping":{"remaining":{}}}') done_raw_id, done_size = blob_store.write_from_bytes(b'{"mapping":{"done":{}}}') @@ -1689,7 +2057,7 @@ def test_raw_materialization_replays_parsed_rows_after_interrupted_index_rebuild ) index_conn.commit() - result = repair_mod.repair_raw_materialization(config, dry_run=True) + result = _repair_after_persisted_census(config, dry_run=True) assert result.repaired_count == 0 assert result.success is True @@ -1903,8 +2271,7 @@ def conversation(session_id: str) -> dict[str, object]: def test_raw_materialization_reports_uncensused_append_fragments_as_pending_debt(tmp_path: Path) -> None: config = _config(tmp_path) - initialize_archive_database(tmp_path / "source.db", ArchiveTier.SOURCE) - initialize_archive_database(tmp_path / "index.db", ArchiveTier.INDEX) + initialize_active_archive_root(tmp_path) blob_store = BlobStore(tmp_path / "blob") raw_id, blob_size = blob_store.write_from_bytes(b'{"fragment":true}') with sqlite3.connect(tmp_path / "source.db") as source_conn: @@ -1929,18 +2296,21 @@ def test_raw_materialization_reports_uncensused_append_fragments_as_pending_debt assert backlog["durable_authority_debt_count"] == 1 assert backlog["byte_authority_pending_count"] == 1 assert targeted.success is False - assert "pending byte-authority adjudication" in targeted.detail + assert targeted.census_receipt is not None + assert targeted.census_receipt.quiescent is False + assert "persisted parser census" in targeted.detail with sqlite3.connect(tmp_path / "source.db") as source_conn: - source_conn.execute( + cursor = source_conn.execute( """ - INSERT INTO raw_membership_census ( - raw_id, parser_fingerprint, status, member_count, censused_at_ms, detail - ) VALUES (?, 'test', 'failed', 0, 2, - 'append fragments are governed by byte revision authority') + UPDATE raw_membership_census + SET parser_fingerprint = 'test', status = 'failed', member_count = 0, + censused_at_ms = 2, detail = 'append fragments are governed by byte revision authority' + WHERE raw_id = ? """, (raw_id,), ) + assert cursor.rowcount == 1 source_conn.commit() governed = repair_mod._raw_materialization_candidate_ids(config) @@ -1964,8 +2334,7 @@ def test_raw_materialization_reports_uncensused_append_fragments_as_pending_debt def test_raw_materialization_ordinary_replay_reaches_two_call_fixed_point(tmp_path: Path) -> None: config = _config(tmp_path) - initialize_archive_database(tmp_path / "source.db", ArchiveTier.SOURCE) - initialize_archive_database(tmp_path / "index.db", ArchiveTier.INDEX) + initialize_active_archive_root(tmp_path) payload = b"""{ "id": "fixed-point", "title": "fixed point", @@ -2008,7 +2377,7 @@ def test_raw_materialization_ordinary_replay_reaches_two_call_fixed_point(tmp_pa ) source_conn.commit() - first = repair_mod.repair_raw_materialization(config) + first = _repair_after_persisted_census(config) with sqlite3.connect(tmp_path / "index.db") as index_conn: receipts_after_first = index_conn.execute( "SELECT decision_id, raw_id, decision FROM raw_revision_applications ORDER BY decision_id" @@ -2048,8 +2417,7 @@ def test_raw_materialization_no_progress_component_terminalizes_instead_of_loopi automatically reselected on the next pass. """ config = _config(tmp_path) - initialize_archive_database(tmp_path / "source.db", ArchiveTier.SOURCE) - initialize_archive_database(tmp_path / "index.db", ArchiveTier.INDEX) + initialize_active_archive_root(tmp_path) payload = b"""{ "id": "orphan-append", "title": "orphan append", @@ -2107,7 +2475,7 @@ def test_raw_materialization_no_progress_component_terminalizes_instead_of_loopi ) source_conn.commit() - first = repair_mod.repair_raw_materialization(config) + first = _repair_after_persisted_census(config) assert first.success is False assert first.repaired_count == 0 assert first.metrics.get("raw_materialization_no_progress_count") == 1.0 @@ -2148,8 +2516,7 @@ def test_raw_materialization_uses_authority_replay_not_legacy_batch_parser( monkeypatch: pytest.MonkeyPatch, ) -> None: config = _config(tmp_path) - initialize_archive_database(tmp_path / "source.db", ArchiveTier.SOURCE) - initialize_archive_database(tmp_path / "index.db", ArchiveTier.INDEX) + initialize_active_archive_root(tmp_path) blob_store = BlobStore(tmp_path / "blob") first_raw_id, first_size = blob_store.write_from_bytes( b'{"mapping":{"first":{"id":"first","message":{"id":"m1","author":{"role":"user"},' @@ -2208,7 +2575,7 @@ async def parse_from_raw(self, *, raw_ids: list[str], **kwargs: object) -> objec monkeypatch.setattr(parsing_module, "ParsingService", FakeParsingService) - result = repair_mod.repair_raw_materialization(config) + result = _repair_after_persisted_census(config) assert result.success is True assert result.repaired_count == 2 @@ -2221,8 +2588,7 @@ def test_raw_materialization_ordinary_repair_preserves_newer_index_state( monkeypatch: pytest.MonkeyPatch, ) -> None: config = _config(tmp_path) - initialize_archive_database(tmp_path / "source.db", ArchiveTier.SOURCE) - initialize_archive_database(tmp_path / "index.db", ArchiveTier.INDEX) + initialize_active_archive_root(tmp_path) older_payload = b"""{ "id": "logical-session", "title": "older raw snapshot", @@ -2291,6 +2657,13 @@ def test_raw_materialization_ordinary_repair_preserves_newer_index_state( fts_hits_before = index_conn.execute( "SELECT rowid FROM messages_fts WHERE messages_fts MATCH 'newer' ORDER BY rowid" ).fetchall() + message_ids_before = [ + str(message_id) + for (message_id,) in index_conn.execute( + "SELECT message_id FROM messages WHERE session_id = ? ORDER BY position", + (session_id,), + ).fetchall() + ] assert len(fts_hits_before) == 1 class UnexpectedParsingService: @@ -2299,7 +2672,7 @@ def __init__(self, *_args: object, **_kwargs: object) -> None: monkeypatch.setattr("polylogue.pipeline.services.parsing.ParsingService", UnexpectedParsingService) - result = repair_mod.repair_raw_materialization(config, dry_run=False) + result = _repair_after_persisted_census(config) assert result.success is False assert result.repaired_count == 0 @@ -2321,7 +2694,7 @@ def __init__(self, *_args: object, **_kwargs: object) -> None: "SELECT rowid FROM messages_fts WHERE messages_fts MATCH 'newer' ORDER BY rowid" ).fetchall() assert row == ("newer-index-raw", "newer indexed state", newer_hash, 1) - assert message_ids == ["chatgpt-export:logical-session:newer-message"] + assert message_ids == message_ids_before assert fts_hits_after == fts_hits_before with sqlite3.connect(tmp_path / "source.db") as source_conn: raw_state = source_conn.execute( @@ -2485,8 +2858,7 @@ def test_raw_materialization_raw_artifact_filter_counts_only_target(tmp_path: Pa def test_raw_materialization_excludes_already_parsed_non_materialized_rows(tmp_path: Path) -> None: config = _config(tmp_path) - initialize_archive_database(tmp_path / "source.db", ArchiveTier.SOURCE) - initialize_archive_database(tmp_path / "index.db", ArchiveTier.INDEX) + initialize_active_archive_root(tmp_path) blob_store = BlobStore(tmp_path / "blob") replayable_raw_id, replayable_size = blob_store.write_from_bytes(b'{"mapping":{"pending":{}}}') parsed_raw_id, parsed_size = blob_store.write_from_bytes(b'{"mapping":{"parsed":{}}}') @@ -2525,13 +2897,13 @@ def test_raw_materialization_excludes_already_parsed_non_materialized_rows(tmp_p ) source_conn.commit() - result = repair_mod.repair_raw_materialization(config, dry_run=True) + result = _repair_after_persisted_census(config, dry_run=True) assert result.repaired_count == 0 assert result.metrics["raw_materialization_candidate_count"] == 2.0 assert "1 already parsed but not materialized" in result.detail - scoped = repair_mod.repair_raw_materialization(config, dry_run=True, raw_artifact_id=parsed_raw_id) + scoped = _repair_after_persisted_census(config, dry_run=True, raw_artifact_id=parsed_raw_id) assert scoped.repaired_count == 0 assert scoped.metrics["raw_materialization_candidate_count"] == 1.0 @@ -2581,8 +2953,7 @@ def test_raw_materialization_excludes_parsed_non_session_artifacts(tmp_path: Pat def test_raw_materialization_explicit_scope_includes_already_parsed_rows(tmp_path: Path) -> None: config = _config(tmp_path) - initialize_archive_database(tmp_path / "source.db", ArchiveTier.SOURCE) - initialize_archive_database(tmp_path / "index.db", ArchiveTier.INDEX) + initialize_active_archive_root(tmp_path) blob_store = BlobStore(tmp_path / "blob") parsed_raw_id, parsed_size = blob_store.write_from_bytes(b'{"items":[]}') @@ -2608,6 +2979,7 @@ def test_raw_materialization_explicit_scope_includes_already_parsed_rows(tmp_pat ) source_conn.commit() + _complete_bounded_raw_census(config, limit=1_000) broad = repair_mod.repair_raw_materialization(config, dry_run=True) by_family = repair_mod.repair_raw_materialization(config, dry_run=True, source_family="gemini-cli-session") by_root = repair_mod.repair_raw_materialization(config, dry_run=True, source_root=Path("/captures/gemini")) @@ -2628,6 +3000,7 @@ def test_raw_materialization_scope_filters_count_only_matching_raw_rows(tmp_path claude_raw_id, claude_size = blob_store.write_from_bytes(b'{"parentUuid":null,"sessionId":"claude-a"}') codex_raw_id, codex_size = blob_store.write_from_bytes(b'{"items":[]}') other_root_raw_id, other_root_size = blob_store.write_from_bytes(b'{"parentUuid":null,"sessionId":"claude-b"}') + learned_raw_id, learned_size = blob_store.write_from_bytes(b'{"parentUuid":null,"sessionId":"claude-learned"}') with sqlite3.connect(tmp_path / "source.db") as source_conn: source_conn.executemany( @@ -2669,6 +3042,22 @@ def test_raw_materialization_scope_filters_count_only_matching_raw_rows(tmp_path ), ), ) + source_conn.execute( + """ + INSERT INTO raw_sessions ( + raw_id, origin, detected_provider, native_id, source_path, source_index, + blob_hash, blob_size, acquired_at_ms + ) VALUES (?, 'unknown-export', 'claude-code', ?, ?, 0, ?, ?, ?) + """, + ( + learned_raw_id, + "claude-learned", + "/captures/claude/learned.jsonl", + bytes.fromhex(learned_raw_id), + learned_size, + 4, + ), + ) source_conn.commit() by_provider = repair_mod.repair_raw_materialization(config, dry_run=True, provider="claude-code") @@ -2678,9 +3067,18 @@ def test_raw_materialization_scope_filters_count_only_matching_raw_rows(tmp_path assert by_provider.repaired_count == 0 assert by_family.repaired_count == 0 assert by_root.repaired_count == 0 - assert by_provider.metrics["raw_materialization_candidate_count"] == 2.0 - assert by_provider.metrics["raw_materialization_total_blob_bytes"] == float(claude_size + other_root_size) - assert by_provider.metrics["raw_materialization_max_blob_bytes"] == float(max(claude_size, other_root_size)) + assert by_provider.metrics["raw_materialization_candidate_count"] == 3.0 + assert by_provider.metrics["raw_materialization_total_blob_bytes"] == float( + claude_size + other_root_size + learned_size + ) + assert by_provider.metrics["raw_materialization_max_blob_bytes"] == float( + max(claude_size, other_root_size, learned_size) + ) + census_candidates = repair_mod._raw_materialization_parser_census_candidates( + config, + provider="claude-code", + ) + assert set(census_candidates.raw_ids) == {claude_raw_id, other_root_raw_id, learned_raw_id} def test_raw_materialization_uses_authority_substrate_not_legacy_ingest_stage( @@ -2806,8 +3204,10 @@ def __init__(self, **_kwargs: object) -> None: assert result.metrics["raw_materialization_executed_count"] == 0.0 assert result.metrics["raw_materialization_execute_blob_limit_bytes"] == float(1024 * 1024 * 1024) assert parser_fingerprint.endswith(":resource-blocked:1073741824") - assert repeated.success is True - assert repeated_census_count == first_census_count + assert repeated.success is False + assert len(repeated.plan_outcomes) == 1 + assert repeated.plan_outcomes[0].status.value == "terminal" + assert repeated_census_count == first_census_count + 1 def test_raw_materialization_classifies_oversized_stream_record_replay( @@ -2984,10 +3384,11 @@ def test_raw_materialization_blocks_aggregate_sub_limit_cohort_before_blob_open( assert result.success is False assert result.metrics["raw_materialization_resource_blocked_count"] == 2.0 assert len(result.plan_outcomes) == 1 - assert result.plan_outcomes[0].status.value == "deferred" - assert repeated.success is True - assert repeated.plan_outcomes == () - assert "unchanged plan(s) remain deferred" in repeated.detail + assert result.plan_outcomes[0].status.value == "terminal" + assert repeated.success is False + assert len(repeated.plan_outcomes) == 1 + assert repeated.plan_outcomes[0].status.value == "terminal" + assert "aggregate payload exceeds 1.0 GiB" in repeated.detail assert "aggregate payload exceeds 1.0 GiB" in result.detail @@ -3218,7 +3619,7 @@ def test_raw_materialization_durable_ledger_survives_ops_reset_for_fairness( ) first = repair_mod.repair_raw_materialization(_config(tmp_path), raw_artifact_limit=1) - assert first.plan_outcomes[0].status.value == "deferred" + assert first.plan_outcomes[0].status.value == "terminal" (tmp_path / "ops.db").unlink() second = repair_mod.repair_raw_materialization(_config(tmp_path), raw_artifact_limit=1) @@ -3261,7 +3662,10 @@ def retry_oldest(*args: Any, selected_raw_ids: list[str] | None = None, **kwargs mutation.setattr(revision_backfill, "backfill_historical_revision_evidence", retry_oldest) if remove_fair_rotation: - def acquisition_only_order(candidates: Any, *, archive_root: Path) -> list[tuple[str, ...]]: + def acquisition_only_order( + candidates: Any, *, archive_root: Path, index_db_path: Path + ) -> list[tuple[str, ...]]: + del index_db_path return sorted( candidates.authority_components, key=lambda component: min(candidates.raw_acquired_at_ms[raw_id] for raw_id in component), @@ -3330,7 +3734,10 @@ def run(*, prefer_cheap: bool) -> tuple[tuple[str, ...], str]: with monkeypatch.context() as mutation: if prefer_cheap: - def cheap_first_order(candidates: Any, *, archive_root: Path) -> list[tuple[str, ...]]: + def cheap_first_order( + candidates: Any, *, archive_root: Path, index_db_path: Path + ) -> list[tuple[str, ...]]: + del index_db_path candidate_ids = set(candidates.raw_ids) source_components = candidates.authority_components or tuple( (raw_id,) for raw_id in candidates.raw_ids