diff --git a/docs/plans/layering-surface-baseline.json b/docs/plans/layering-surface-baseline.json index d1e67f7e52..5e883f7d89 100644 --- a/docs/plans/layering-surface-baseline.json +++ b/docs/plans/layering-surface-baseline.json @@ -229,11 +229,6 @@ "file": "polylogue/cli/archive_query.py", "import": "polylogue.storage.sqlite.archive_tiers.write" }, - { - "target": "polylogue/cli", - "file": "polylogue/cli/click_app.py", - "import": "polylogue.storage.archive_identity" - }, { "target": "polylogue/cli", "file": "polylogue/cli/click_app.py", @@ -1444,11 +1439,6 @@ "file": "polylogue/daemon/similarity.py", "import": "polylogue.storage.archive_identity" }, - { - "target": "polylogue/daemon", - "file": "polylogue/daemon/similarity.py", - "import": "polylogue.storage.sqlite.sqlite_vec_extension" - }, { "target": "polylogue/daemon", "file": "polylogue/daemon/status.py", @@ -1554,11 +1544,6 @@ "file": "polylogue/mcp/server_prompts.py", "import": "polylogue.storage.sqlite.archive_tiers.archive" }, - { - "target": "polylogue/mcp", - "file": "polylogue/mcp/server_prompts.py", - "import": "polylogue.storage.sqlite.archive_tiers.write" - }, { "target": "polylogue/mcp", "file": "polylogue/mcp/server_resources.py", diff --git a/docs/plans/layering.yaml b/docs/plans/layering.yaml index 91aa3c33e8..18c78b4037 100644 --- a/docs/plans/layering.yaml +++ b/docs/plans/layering.yaml @@ -39,6 +39,8 @@ writer_modules: [admit_raw_and_parsed_result, apply_raw_membership_classification, apply_raw_revision_replay, classify_raw_revision_cohort_for_frozen_candidate, classify_raw_revision_cohort_for_live_watch, classify_raw_revision_cohort_for_rebuild_repair, + finalize_raw_parse_state, + mark_raw_parse_failed, release_provisional_full_revisions, replace_raw_membership_census, write_parsed_for_retained_raw, write_parsed_for_retained_raw_result, write_raw_and_parsed, write_raw_and_parsed_result, write_raw_blob_and_parsed, write_raw_blob_and_parsed_result] diff --git a/polylogue/core/errors.py b/polylogue/core/errors.py index 63dac90bb5..129342361a 100644 --- a/polylogue/core/errors.py +++ b/polylogue/core/errors.py @@ -32,6 +32,12 @@ class PolylogueError(Exception): http_status_code: int = HTTPStatus.INTERNAL_SERVER_ERROR +class RawCASFrontierError(PolylogueError): + """Retryable compare-and-swap conflict while advancing raw authority.""" + + is_transient = True + + class DatabaseError(PolylogueError): """Base class for database errors.""" @@ -93,5 +99,6 @@ def __init__(self, message: str, *, readiness_status: str) -> None: "DatabaseError", "EmbeddingRetrievalNotReadyError", "PolylogueError", + "RawCASFrontierError", "SchemaVersionMismatchError", ] diff --git a/polylogue/core/raw_failure_evidence.py b/polylogue/core/raw_failure_evidence.py index 3ad12d0784..367bde3e95 100644 --- a/polylogue/core/raw_failure_evidence.py +++ b/polylogue/core/raw_failure_evidence.py @@ -7,6 +7,7 @@ from __future__ import annotations +import json from enum import StrEnum from polylogue.core.enums import ArtifactSupportStatus @@ -16,35 +17,197 @@ class RawFailureEvidenceKind(StrEnum): """Durable lifecycle evidence attached to a retained raw artifact.""" DEFERRED_HOT_JSONL_CAPTURE = "deferred_hot_jsonl_capture" + DEFERRED_CLAUDE_CODE_PARTIAL_JSONL = "deferred_claude_code_partial_jsonl" + DEFERRED_CAS_FRONTIER = "deferred_cas_frontier" + # Historical rows written before CAS evidence was made provider-neutral. + # Keep this token readable until a backup-gated migration or re-observation + # receipt has converted every retained row. + DEFERRED_CODEX_CAS_FRONTIER = "deferred_codex_cas_frontier" TERMINAL_CORRUPT_INPUT = "terminal_corrupt_input" + TERMINAL_SUPERSEDED_DEFERRED_CAS_FRONTIER = "terminal_superseded_deferred_cas_frontier" + TERMINAL_UNKNOWN_JSON_DECODE = "terminal_unknown_json_decode" + TERMINAL_UNKNOWN_EXPORT_NO_SESSION = "terminal_unknown_export_no_session" TERMINAL_UNSUPPORTED_SHAPE = "terminal_unsupported_shape" @property def support_status(self) -> ArtifactSupportStatus: - if self is RawFailureEvidenceKind.DEFERRED_HOT_JSONL_CAPTURE: + if self is RawFailureEvidenceKind.TERMINAL_SUPERSEDED_DEFERRED_CAS_FRONTIER: + return ArtifactSupportStatus.UNKNOWN + if self in { + RawFailureEvidenceKind.DEFERRED_HOT_JSONL_CAPTURE, + RawFailureEvidenceKind.DEFERRED_CLAUDE_CODE_PARTIAL_JSONL, + RawFailureEvidenceKind.DEFERRED_CAS_FRONTIER, + RawFailureEvidenceKind.DEFERRED_CODEX_CAS_FRONTIER, + }: return ArtifactSupportStatus.PARTIAL_DECODE - if self is RawFailureEvidenceKind.TERMINAL_CORRUPT_INPUT: + if self in { + RawFailureEvidenceKind.TERMINAL_CORRUPT_INPUT, + RawFailureEvidenceKind.TERMINAL_UNKNOWN_JSON_DECODE, + }: return ArtifactSupportStatus.DECODE_FAILED return ArtifactSupportStatus.UNSUPPORTED_PARSEABLE @property def lifecycle(self) -> str: - return "deferred" if self is RawFailureEvidenceKind.DEFERRED_HOT_JSONL_CAPTURE else "terminal" + if self is RawFailureEvidenceKind.TERMINAL_SUPERSEDED_DEFERRED_CAS_FRONTIER: + return "resolution" + return "deferred" if self.value in RAW_FAILURE_DEFERRED_EVIDENCE_KINDS else "terminal" + + +RAW_FAILURE_TRUSTED_PROVENANCE = "worker-disposition-v1" +RAW_FAILURE_VALIDATION_FAILURE_KINDS = frozenset( + { + RawFailureEvidenceKind.TERMINAL_CORRUPT_INPUT.value, + RawFailureEvidenceKind.TERMINAL_UNKNOWN_JSON_DECODE.value, + } +) + + +def raw_failure_classification_reason( + *, + diagnostic: str | None, + evidence_ref: str | None, + outcome_code: str, + remediation: str | None, + retryable: bool | None, + trusted_validation_failure: bool, +) -> str: + """Encode the typed carrier, including proof for a validation failure.""" + payload: dict[str, object] = { + "diagnostic": diagnostic, + "evidence_ref": evidence_ref, + "outcome_code": outcome_code, + "remediation": remediation, + "retryable": retryable, + } + if trusted_validation_failure: + payload["provenance"] = RAW_FAILURE_TRUSTED_PROVENANCE + return json.dumps(payload, sort_keys=True, separators=(",", ":")) + + +def has_trusted_raw_failure_provenance( + classification_reason: object, + *, + artifact_kind: RawFailureEvidenceKind, + outcome_code: object, +) -> bool: + """Check the structural worker receipt required for corrupt evidence.""" + if artifact_kind.value not in RAW_FAILURE_VALIDATION_FAILURE_KINDS: + return False + if str(outcome_code) != "corrupt_input": + return False + if not isinstance(classification_reason, str): + return False + try: + payload = json.loads(classification_reason) + except (TypeError, ValueError): + return False + return isinstance(payload, dict) and payload.get("provenance") == RAW_FAILURE_TRUSTED_PROVENANCE + + +def raw_failure_outcome_code(classification_reason: object) -> object: + """Read the typed outcome code from a structured failure carrier.""" + if not isinstance(classification_reason, str): + return None + try: + payload = json.loads(classification_reason) + except (TypeError, ValueError): + return None + return payload.get("outcome_code") if isinstance(payload, dict) else None + + +def validated_raw_failure_evidence_kind( + artifact_kind: object, + support_status: object, + *, + validation_failed: bool, + classification_reason: object = None, + outcome_code: object = None, +) -> RawFailureEvidenceKind | None: + """Return a typed kind only for a complete, self-consistent carrier. + + Decode failures are reported by the worker as validation failures because + the payload cannot satisfy the input contract. A matching terminal + corrupt-input/decode carrier explains that state; deferred evidence still + requires validation to have passed or been skipped. + """ + if artifact_kind is None or support_status is None: + return None + try: + evidence_kind = RawFailureEvidenceKind(str(artifact_kind)) + except ValueError: + return None + if evidence_kind.support_status.value != str(support_status): + return None + if validation_failed and not has_trusted_raw_failure_provenance( + classification_reason, + artifact_kind=evidence_kind, + outcome_code=outcome_code, + ): + return None + return evidence_kind RAW_FAILURE_EVIDENCE_KINDS = frozenset(kind.value for kind in RawFailureEvidenceKind) -RAW_FAILURE_DEFERRED_EVIDENCE_KINDS = frozenset({RawFailureEvidenceKind.DEFERRED_HOT_JSONL_CAPTURE.value}) +RAW_FAILURE_DEFERRED_EVIDENCE_KINDS = frozenset( + { + RawFailureEvidenceKind.DEFERRED_HOT_JSONL_CAPTURE.value, + RawFailureEvidenceKind.DEFERRED_CLAUDE_CODE_PARTIAL_JSONL.value, + RawFailureEvidenceKind.DEFERRED_CAS_FRONTIER.value, + RawFailureEvidenceKind.DEFERRED_CODEX_CAS_FRONTIER.value, + } +) +# Only frontier conflicts authorize retained-raw replay. Hot captures remain +# deferred until a complete source observation arrives; replaying their +# truncated blob would advance the cursor past the record that later bytes +# complete. +RAW_FAILURE_REPLAY_AUTHORITY_EVIDENCE_KINDS = frozenset( + { + RawFailureEvidenceKind.DEFERRED_CAS_FRONTIER.value, + RawFailureEvidenceKind.DEFERRED_CODEX_CAS_FRONTIER.value, + } +) +# Every deferred raw-failure carrier represents a partial decode. Consumers +# selecting retry authority must validate this companion field as well as the +# closed kind, or contradictory rows can authorize replay. +RAW_FAILURE_DEFERRED_SUPPORT_STATUS = ArtifactSupportStatus.PARTIAL_DECODE.value +RAW_FAILURE_EVIDENCE_SUPPORT_STATUS_PAIRS = tuple( + sorted((kind.value, kind.support_status.value) for kind in RawFailureEvidenceKind) +) +RAW_FAILURE_LIFECYCLE_EVIDENCE_SUPPORT_STATUS_PAIRS = tuple( + sorted( + (kind.value, kind.support_status.value) + for kind in RawFailureEvidenceKind + if kind.lifecycle in {"deferred", "terminal"} + ) +) RAW_FAILURE_TERMINAL_EVIDENCE_KINDS = frozenset( { RawFailureEvidenceKind.TERMINAL_CORRUPT_INPUT.value, + RawFailureEvidenceKind.TERMINAL_UNKNOWN_JSON_DECODE.value, + RawFailureEvidenceKind.TERMINAL_UNKNOWN_EXPORT_NO_SESSION.value, RawFailureEvidenceKind.TERMINAL_UNSUPPORTED_SHAPE.value, } ) +RAW_FAILURE_TERMINAL_EVIDENCE_SUPPORT_STATUS_PAIRS = tuple( + sorted((kind.value, kind.support_status.value) for kind in RawFailureEvidenceKind if kind.lifecycle == "terminal") +) __all__ = [ "RAW_FAILURE_DEFERRED_EVIDENCE_KINDS", + "RAW_FAILURE_DEFERRED_SUPPORT_STATUS", "RAW_FAILURE_EVIDENCE_KINDS", + "RAW_FAILURE_EVIDENCE_SUPPORT_STATUS_PAIRS", + "RAW_FAILURE_LIFECYCLE_EVIDENCE_SUPPORT_STATUS_PAIRS", + "RAW_FAILURE_REPLAY_AUTHORITY_EVIDENCE_KINDS", "RAW_FAILURE_TERMINAL_EVIDENCE_KINDS", + "RAW_FAILURE_TERMINAL_EVIDENCE_SUPPORT_STATUS_PAIRS", + "RAW_FAILURE_TRUSTED_PROVENANCE", + "RAW_FAILURE_VALIDATION_FAILURE_KINDS", "RawFailureEvidenceKind", + "has_trusted_raw_failure_provenance", + "raw_failure_classification_reason", + "raw_failure_outcome_code", + "validated_raw_failure_evidence_kind", ] diff --git a/polylogue/daemon/status.py b/polylogue/daemon/status.py index 00c9204808..985ebcbed7 100644 --- a/polylogue/daemon/status.py +++ b/polylogue/daemon/status.py @@ -8,7 +8,7 @@ import sqlite3 import sys import threading -from collections.abc import Callable +from collections.abc import Callable, Mapping from datetime import UTC, datetime from pathlib import Path from typing import Any, Literal, cast @@ -21,6 +21,7 @@ from polylogue.core.payload_coercion import required_str as _required_str from polylogue.core.payload_coercion import row_float as _row_float from polylogue.core.payload_coercion import row_int as _row_int +from polylogue.core.raw_failure_evidence import raw_failure_outcome_code, validated_raw_failure_evidence_kind from polylogue.core.stats import percentile from polylogue.daemon.catchup_status import ( CatchupStatus as CatchupStatus, @@ -81,6 +82,21 @@ logger = get_logger(__name__) + +def _authoritative_lifecycle_artifact_kind(sample: Mapping[str, object]) -> str | None: + """Project a typed kind only after validating the complete lifecycle row.""" + evidence_kind = validated_raw_failure_evidence_kind( + sample.get("artifact_kind"), + sample.get("support_status"), + validation_failed=str(sample.get("validation_status") or "") == "failed", + classification_reason=sample.get("classification_reason"), + outcome_code=raw_failure_outcome_code(sample.get("classification_reason")), + ) + if evidence_kind is None or sample.get("lifecycle") != evidence_kind.lifecycle: + return None + return evidence_kind.value + + # Backwards-compatible alias for the stuck threshold (#1246). The "stale" # rollup field stays in the typed status payload to avoid breaking # downstream consumers, but the threshold itself is owned by @@ -399,7 +415,21 @@ class RawFailureSample(BaseModel): :attr:`operation_id` and the typed planner :attr:`locator`. """ - failure_kind: Literal["decode_error", "parse_error", "schema_violation", "maintenance", "unknown"] + failure_kind: Literal[ + "decode_error", + "parse_error", + "schema_violation", + "maintenance", + "unknown", + "deferred_hot_jsonl_capture", + "deferred_claude_code_partial_jsonl", + "deferred_cas_frontier", + "deferred_codex_cas_frontier", + "terminal_corrupt_input", + "terminal_unknown_json_decode", + "terminal_unknown_export_no_session", + "terminal_unsupported_shape", + ] provider_hint: str | None = None redacted_error: str = "" source: Literal["ingest", "maintenance"] = "ingest" @@ -933,6 +963,12 @@ def _archive_raw_failure_info( for sample in lifecycle_snapshot.samples if sample.get("raw_id") is not None and sample.get("lifecycle") is not None } + validated_artifact_kind_by_raw_id = { + str(sample["raw_id"]): artifact_kind + for sample in lifecycle_snapshot.samples + if sample.get("raw_id") is not None + and (artifact_kind := _authoritative_lifecycle_artifact_kind(sample)) is not None + } samples: list[RawFailureSample] = [] rows_by_raw_id: dict[str, sqlite3.Row | tuple[object, ...]] = {} if sample_ids: @@ -955,8 +991,11 @@ def _archive_raw_failure_info( val_status = str(row[3] or "") if row[3] else "" val_err = str(row[4] or "") if row[4] else "" origin = str(row[1]) if row[1] else None - if "JSONDecodeError" in parse_err or "decode error" in parse_err.lower(): - kind: Literal["decode_error", "parse_error", "schema_violation", "unknown"] = "decode_error" + artifact_kind = validated_artifact_kind_by_raw_id.get(raw_id) + if artifact_kind is not None: + kind = cast(Any, artifact_kind) + elif "JSONDecodeError" in parse_err or "decode error" in parse_err.lower(): + kind = "decode_error" elif val_status == "failed": kind = "schema_violation" elif parse_err: diff --git a/polylogue/maintenance/raw_failure_disposition_apply.py b/polylogue/maintenance/raw_failure_disposition_apply.py index fa6fd59482..04744943b1 100644 --- a/polylogue/maintenance/raw_failure_disposition_apply.py +++ b/polylogue/maintenance/raw_failure_disposition_apply.py @@ -19,7 +19,7 @@ from typing import cast from polylogue.config import Config -from polylogue.core.raw_failure_evidence import RawFailureEvidenceKind +from polylogue.core.raw_failure_evidence import RawFailureEvidenceKind, raw_failure_classification_reason from polylogue.maintenance.offline_guard import offline_maintenance_block_reason from polylogue.paths import render_root from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier @@ -166,7 +166,20 @@ def _apply_candidate( ( candidate.disposition_kind.value, candidate.disposition_kind.support_status.value, - candidate.disposition_kind.value, + raw_failure_classification_reason( + diagnostic=candidate.detail, + evidence_ref=f"raw-failure-disposition:{manifest_sha256}", + outcome_code=( + "corrupt_input" + if candidate.disposition_kind is RawFailureEvidenceKind.TERMINAL_CORRUPT_INPUT + else "unsupported_shape" + ), + remediation="retain the reviewed terminal disposition until a forced reparse is authorized", + retryable=False, + trusted_validation_failure=( + candidate.disposition_kind is RawFailureEvidenceKind.TERMINAL_CORRUPT_INPUT + ), + ), disposed_at_ms, row["artifact_id"], ), diff --git a/polylogue/pipeline/services/ingest_batch/_core.py b/polylogue/pipeline/services/ingest_batch/_core.py index 6b317da74b..b63d63ee5a 100644 --- a/polylogue/pipeline/services/ingest_batch/_core.py +++ b/polylogue/pipeline/services/ingest_batch/_core.py @@ -28,13 +28,14 @@ from polylogue.archive.revision_authority import RawRevisionAuthority, RawRevisionEnvelope, RawRevisionKind from polylogue.archive.revision_replay import RevisionReplayPlan from polylogue.archive.write_gateway import ArchiveWriteGateway, WriteOperation -from polylogue.core.enums import BlockType, Provider +from polylogue.core.enums import BlockType, IngestOutcome, Provider from polylogue.core.memory import release_process_memory from polylogue.core.metrics import ( read_current_rss_mb, read_peak_rss_children_mb, read_peak_rss_self_mb, ) +from polylogue.core.raw_failure_evidence import RawFailureEvidenceKind from polylogue.logging import get_logger from polylogue.pipeline.ids import session_id as make_session_id from polylogue.pipeline.payload_types import MaterializeStageObservation, ParseBatchObservation @@ -1181,6 +1182,11 @@ def _record_outcome(summary: _IngestBatchSummary, ir: IngestRecordResult) -> Non parse_error=ir.parse_error, error=ir.error, had_sessions=bool(ir.sessions), + outcome_code=ir.outcome_code, + retryable=ir.retryable, + evidence_ref=ir.evidence_ref, + remediation=ir.remediation, + diagnostic=ir.diagnostic, ) if ir.serialized_size_bytes is not None: summary.total_result_bytes += ir.serialized_size_bytes @@ -2266,9 +2272,10 @@ def _failed_raw_state_update( parse_error=error, detection_warnings=error[:500] if error else None, ) + diagnostic = outcome.diagnostic or outcome.parse_error return RawSessionStateUpdate( parse_error=outcome.parse_error, - detection_warnings=outcome.parse_error[:500] if outcome.parse_error else None, + detection_warnings=diagnostic[:500] if diagnostic else None, payload_provider=outcome.payload_provider, validation_status=outcome.validation_status, validation_error=outcome.validation_error or error, @@ -2276,6 +2283,20 @@ def _failed_raw_state_update( ) +def _raw_failure_evidence_kind(outcome: _RawIngestOutcome | None) -> RawFailureEvidenceKind | None: + """Map terminal worker input outcomes to closed source-tier carriers.""" + if outcome is None: + return None + try: + outcome_code = IngestOutcome.from_string(outcome.outcome_code) + except ValueError: + return None + return { + IngestOutcome.CORRUPT_INPUT: RawFailureEvidenceKind.TERMINAL_CORRUPT_INPUT, + IngestOutcome.UNSUPPORTED_SHAPE: RawFailureEvidenceKind.TERMINAL_UNSUPPORTED_SHAPE, + }.get(outcome_code) + + async def _persist_batch_raw_state_updates( service: _ParsingServiceRawStateLike, backend: _BulkConnectionBackendLike, @@ -2334,6 +2355,8 @@ async def stage_accepted_payloads( for rid in succeeded_raw_ids: if rid in skipped_raw_ids: continue + if source_backend is not None: + await source_backend.supersede_deferred_cas_evidence(rid) await service.repository.update_raw_state( rid, state=_successful_raw_state_update( @@ -2350,6 +2373,8 @@ async def stage_accepted_payloads( for rid in skipped_raw_ids: if rid in failed_raw_ids: continue + if source_backend is not None: + await source_backend.supersede_deferred_cas_evidence(rid) await service.repository.update_raw_state( rid, state=_skipped_raw_state_update( @@ -2371,6 +2396,22 @@ async def stage_accepted_payloads( validation_mode=validation_mode, ), ) + outcome = outcomes.get(rid) + evidence_kind = _raw_failure_evidence_kind(outcome) + if source_backend is not None: + if outcome is not None and evidence_kind is not None: + await source_backend.save_raw_failure_evidence( + rid, + artifact_kind=evidence_kind.value, + support_status=evidence_kind.support_status.value, + outcome_code=outcome.outcome_code, + retryable=outcome.retryable, + evidence_ref=outcome.evidence_ref, + remediation=outcome.remediation, + diagnostic=outcome.diagnostic, + ) + elif outcome is not None: + await source_backend.retire_raw_failure_evidence(rid) return time.perf_counter() - raw_state_update_started diff --git a/polylogue/pipeline/services/ingest_batch/_models.py b/polylogue/pipeline/services/ingest_batch/_models.py index 6d85d41ee3..7ae768e4c1 100644 --- a/polylogue/pipeline/services/ingest_batch/_models.py +++ b/polylogue/pipeline/services/ingest_batch/_models.py @@ -40,6 +40,23 @@ def bulk_connection(self) -> AbstractAsyncContextManager[None]: ... class _SourceTierBackendLike(_BulkConnectionBackendLike, Protocol): def connection(self) -> AbstractAsyncContextManager[aiosqlite.Connection]: ... + async def supersede_deferred_cas_evidence(self, raw_id: str) -> None: ... + + async def retire_raw_failure_evidence(self, raw_id: str) -> None: ... + + async def save_raw_failure_evidence( + self, + raw_id: str, + *, + artifact_kind: str, + support_status: str, + outcome_code: str, + retryable: bool | None, + evidence_ref: str | None, + remediation: str | None, + diagnostic: str | None, + ) -> None: ... + class _ConnectionBackendLike(Protocol): def connection(self) -> AbstractAsyncContextManager[aiosqlite.Connection]: ... @@ -54,6 +71,14 @@ class _RawIngestOutcome: parse_error: str | None error: str | None had_sessions: bool + # Keep the worker's typed disposition intact through the batch summary so + # the raw-state persistence boundary can retain the same evidence instead + # of reconstructing it from free-form error text. + outcome_code: str = "success" + retryable: bool | None = False + evidence_ref: str | None = None + remediation: str | None = None + diagnostic: str | None = None @dataclass(slots=True) diff --git a/polylogue/pipeline/services/ingest_worker.py b/polylogue/pipeline/services/ingest_worker.py index 51c18f5b2e..8d50113696 100644 --- a/polylogue/pipeline/services/ingest_worker.py +++ b/polylogue/pipeline/services/ingest_worker.py @@ -111,6 +111,7 @@ class IngestRecordResult: retryable: bool | None = False evidence_ref: str | None = None remediation: str | None = None + diagnostic: str | None = None # polylogue-azf7: set when on-demand sidecar-assembly enrichment raised # and this record's sessions were materialized unenriched (native-id # title etc.) as a result. Distinct from ``error`` -- the record still @@ -250,6 +251,7 @@ def _record_result( retryable=(disposition.retryable if disposition is not None else None), evidence_ref=(disposition.evidence_ref if disposition is not None else None), remediation=(disposition.remediation if disposition is not None else None), + diagnostic=(disposition.diagnostic if disposition is not None else None), sessions_unenriched=sessions_unenriched, ), measure_serialized_size=context.measure_serialized_size, diff --git a/polylogue/pipeline/services/planning_backlog.py b/polylogue/pipeline/services/planning_backlog.py index 91051e71ce..3f25ce20ec 100644 --- a/polylogue/pipeline/services/planning_backlog.py +++ b/polylogue/pipeline/services/planning_backlog.py @@ -56,6 +56,7 @@ async def collect_parse_backlog( validation_statuses=list(query_spec.validation_statuses) if query_spec.validation_statuses is not None else None, + exclude_terminal_failure_evidence=query_spec.exclude_terminal_failure_evidence, ): if raw_id not in exclude: backlog_parse_ids.append(raw_id) diff --git a/polylogue/sources/decoder_json.py b/polylogue/sources/decoder_json.py index f3cfb6b0f2..91c97ee0b1 100644 --- a/polylogue/sources/decoder_json.py +++ b/polylogue/sources/decoder_json.py @@ -75,6 +75,21 @@ def __init__( ) +class JsonlDecodeError(ValueError): + """A complete JSONL record could not be decoded. + + Tolerant JSONL consumers may continue after malformed records, but callers + that need fail-closed evidence can opt into raising this typed signal after + the stream has been consumed. + """ + + def __init__(self, path_name: str, *, line_number: int, cause: BaseException) -> None: + self.path_name = path_name + self.line_number = line_number + self.cause = cause + super().__init__(f"JSONL decode failed for {path_name} at line {line_number}: {cause}") + + def _is_json_value(value: object) -> TypeGuard[JsonValue]: if value is None or isinstance(value, (str, int, float, bool)): return True @@ -113,25 +128,26 @@ def _yield_jsonl_pending( *, is_last: bool, path_name: str, -) -> tuple[list[JsonValue], int]: + line_number: int, +) -> tuple[list[JsonValue], int, int | None]: try: parsed = json_loads(raw_pending) except JSONDecodeError: parsed = None else: if _is_json_value(parsed): - return ([parsed], 0) + return ([parsed], 0, None) logger_obj.debug("Skipping non-JSON-compatible decoded line from %s", path_name) - return ([], 0) + return ([], 0, None) if isinstance(raw_pending, bytes): decoded = decode_json_bytes_with(logger_obj, raw_pending) if not decoded: if is_last: logger_obj.debug("Skipping undecodable trailing line from %s", path_name) - return ([], 0) + return ([], 1, line_number) else: - return ([], 1) + return ([], 1, line_number) else: decoded = raw_pending @@ -140,33 +156,42 @@ def _yield_jsonl_pending( except json.JSONDecodeError as exc: if is_last: logger_obj.debug("Skipping truncated trailing line in %s: %s", path_name, exc) - return ([], 0) - return ([], 1) + return ([], 1, line_number) + return ([], 1, line_number) if _is_json_value(parsed): - return ([parsed], 0) + return ([parsed], 0, None) logger_obj.debug("Skipping non-JSON-compatible decoded line from %s", path_name) - return ([], 0) + return ([], 0, None) def _iter_jsonl_stream( logger_obj: LoggerLike, handle: JsonReadable, path_name: str, + *, + fail_on_decode_error: bool = False, ) -> Iterable[JsonValue]: error_count = 0 pending: bytes | str | None = None + physical_line_number = 0 + pending_line_number: int | None = None + first_decode_error_line: int | None = None for line in handle: + physical_line_number += 1 raw = line.strip() if not raw: continue if pending is not None: - records, new_errors = _yield_jsonl_pending( + records, new_errors, error_line = _yield_jsonl_pending( logger_obj, pending, is_last=False, path_name=path_name, + line_number=pending_line_number or physical_line_number, ) + if first_decode_error_line is None and error_line is not None: + first_decode_error_line = error_line error_count += new_errors if new_errors: if error_count <= 3: @@ -175,16 +200,28 @@ def _iter_jsonl_stream( logger_obj.warning("Skipping further invalid JSON lines in %s...", path_name) yield from records pending = raw + pending_line_number = physical_line_number if pending is not None: - records, _new_errors = _yield_jsonl_pending( + records, new_errors, error_line = _yield_jsonl_pending( logger_obj, pending, is_last=True, path_name=path_name, + line_number=pending_line_number or physical_line_number, ) + if first_decode_error_line is None and error_line is not None: + first_decode_error_line = error_line + error_count += new_errors yield from records + if fail_on_decode_error and error_count: + raise JsonlDecodeError( + path_name, + line_number=first_decode_error_line or physical_line_number, + cause=ValueError("malformed JSONL record"), + ) + if error_count > 3: logger_obj.warning("Skipped %d invalid JSON lines in %s", error_count, path_name) @@ -251,12 +288,18 @@ def iter_json_stream_with( handle: JsonReadable, path_name: str, unpack_lists: bool = True, + fail_on_decode_error: bool = False, ) -> Iterable[JsonValue]: normalized_path = path_name.lower() if normalized_path.endswith((".jsonl", ".jsonl.txt", ".ndjson")) or any( marker in normalized_path for marker in (".jsonl.", ".ndjson.") ): - yield from _iter_jsonl_stream(logger_obj, handle, path_name) + yield from _iter_jsonl_stream( + logger_obj, + handle, + path_name, + fail_on_decode_error=fail_on_decode_error, + ) return # The ijson multi-strategy parse below rewinds via ``handle.seek(0)``. A @@ -306,13 +349,27 @@ def iter_json_stream_with( yield data -def iter_json_stream(handle: JsonReadable, path_name: str, unpack_lists: bool = True) -> Iterable[JsonValue]: - yield from iter_json_stream_with(logger, ijson, handle, path_name, unpack_lists) +def iter_json_stream( + handle: JsonReadable, + path_name: str, + unpack_lists: bool = True, + *, + fail_on_decode_error: bool = False, +) -> Iterable[JsonValue]: + yield from iter_json_stream_with( + logger, + ijson, + handle, + path_name, + unpack_lists, + fail_on_decode_error=fail_on_decode_error, + ) __all__ = [ "ENCODING_GUESSES", "IjsonModuleLike", + "JsonlDecodeError", "JsonReadable", "JsonValue", "LoggerLike", diff --git a/polylogue/sources/decoders.py b/polylogue/sources/decoders.py index 2d3842a744..615edc93fd 100644 --- a/polylogue/sources/decoders.py +++ b/polylogue/sources/decoders.py @@ -9,6 +9,7 @@ from polylogue.logging import get_logger from polylogue.sources.decoder_json import ( + JsonlDecodeError, JsonValue, decode_json_bytes_with, iter_json_stream_with, @@ -34,13 +35,23 @@ def _iter_json_stream( handle: BinaryIO | IO[bytes], path_name: str, unpack_lists: bool = True, + *, + fail_on_decode_error: bool = False, ) -> Iterable[JsonValue]: - yield from iter_json_stream_with(logger, ijson, handle, path_name, unpack_lists) + yield from iter_json_stream_with( + logger, + ijson, + handle, + path_name, + unpack_lists, + fail_on_decode_error=fail_on_decode_error, + ) __all__ = [ "_decode_json_bytes", "_iter_json_stream", + "JsonlDecodeError", "_ZipEntryValidator", "_zip_entry_provider_hint", "_process_zip", diff --git a/polylogue/sources/live/batch.py b/polylogue/sources/live/batch.py index cfd79ecb05..94e1f5d392 100644 --- a/polylogue/sources/live/batch.py +++ b/polylogue/sources/live/batch.py @@ -14,6 +14,7 @@ from datetime import UTC, datetime from hashlib import sha256 from io import BytesIO +from json import JSONDecodeError as StdlibJSONDecodeError from json import dumps as json_dumps from json import loads as json_loads from pathlib import Path @@ -49,7 +50,11 @@ read_peak_rss_self_mb, ) from polylogue.core.provider_identity import canonical_acquisition_provider -from polylogue.core.raw_failure_evidence import RAW_FAILURE_EVIDENCE_KINDS, RawFailureEvidenceKind +from polylogue.core.raw_failure_evidence import ( + RAW_FAILURE_EVIDENCE_KINDS, + RAW_FAILURE_LIFECYCLE_EVIDENCE_SUPPORT_STATUS_PAIRS, + RawFailureEvidenceKind, +) from polylogue.logging import get_logger from polylogue.pipeline.ids import session_revision_projection from polylogue.pipeline.ingest_outcomes import ( @@ -59,8 +64,9 @@ success_disposition, ) from polylogue.pipeline.services.ingest_batch._models import _IngestBatchSummary +from polylogue.sources.decoder_json import PartialJsonStreamError from polylogue.sources.decoder_zip import ZipBombError, open_bounded_zip_entry -from polylogue.sources.decoders import _iter_json_stream, _ZipEntryValidator +from polylogue.sources.decoders import JsonlDecodeError, _iter_json_stream, _ZipEntryValidator from polylogue.sources.dispatch import ( _detect_provider_from_raw_bytes, is_stream_record_provider, @@ -338,6 +344,10 @@ def _write_codex_thread_state_evidence( ) +def _is_json_stream_decode_error(error: BaseException) -> bool: + return isinstance(error, (StdlibJSONDecodeError, UnicodeDecodeError, PartialJsonStreamError, JsonlDecodeError)) + + LiveBatchEventEmitter = Callable[[str, dict[str, object]], None] LiveBatchSyncRunner = Callable[..., Awaitable[Any]] P = ParamSpec("P") @@ -486,6 +496,10 @@ def _captured_jsonl_ends_at_record_boundary( @dataclass(slots=True) class _ArchiveFullWriteResult: raw_ids: dict[str, str] = field(default_factory=dict) + # Terminal refusals are durably retained and therefore handled by this + # observation. Keep them separate from accepted raw ids so deferred + # authority failures remain retryable. + terminal_raw_ids: dict[str, str] = field(default_factory=dict) # A raw whose membership census does not produce an accepted session is # still a durably acquired, successfully parsed source observation. The # decision can be pending for the materialization conveyor or already @@ -1961,7 +1975,7 @@ def _ingest_full_paths_sync( elif path.suffix.lower() == ".jsonl": provider, parse_as_session = _jsonl_provider_and_session_artifact(path, fallback_provider) source_name = provider.value - if not parse_as_session: + if not parse_as_session and provider is not Provider.UNKNOWN: self._mark_excluded_cursor(path, stat, source_name=source_name) continue if stat.st_size >= _STREAMING_FULL_INGEST_BYTES: @@ -2026,7 +2040,10 @@ def _ingest_full_paths_sync( continue provider = _detect_provider_from_raw_bytes(payload, path.name, fallback_provider) source_name = provider.value - if not _parse_payload_as_session_artifact(path, provider=provider, payload=payload): + if ( + not _parse_payload_as_session_artifact(path, provider=provider, payload=payload) + and provider is not Provider.UNKNOWN + ): self._mark_excluded_cursor(path, stat, source_name=source_name) continue raw_id, blob_size = blob_store.write_from_bytes(payload) @@ -2079,7 +2096,10 @@ def _ingest_full_paths_sync( continue provider = _detect_provider_from_raw_bytes(payload, path.name, fallback_provider) source_name = provider.value - if not _parse_payload_as_session_artifact(path, provider=provider, payload=payload): + if ( + not _parse_payload_as_session_artifact(path, provider=provider, payload=payload) + and provider is not Provider.UNKNOWN + ): self._mark_excluded_cursor(path, stat, source_name=source_name) continue raw_id, blob_size = blob_store.write_from_bytes(payload) @@ -2176,10 +2196,16 @@ def _ingest_full_paths_sync( for raw_id in raw_by_id if raw_id not in archive_write.raw_ids and raw_id not in archive_write.deferred_raw_ids + and raw_id not in archive_write.terminal_raw_ids and raw_id not in archive_write.skipped_raw_ids ) raw_by_id = { - (archive_write.raw_ids.get(raw_id) or archive_write.deferred_raw_ids.get(raw_id) or raw_id): path + ( + archive_write.raw_ids.get(raw_id) + or archive_write.deferred_raw_ids.get(raw_id) + or archive_write.terminal_raw_ids.get(raw_id) + or raw_id + ): path for raw_id, path in raw_by_id.items() } if heartbeat is not None: @@ -2309,6 +2335,7 @@ def _ingest_full_records_archive( break provider: Provider | None = None source_raw_id: str | None = None + acquired_at_ms = 0 try: record_timings: dict[str, float] = {} t0 = time.perf_counter() @@ -2425,18 +2452,24 @@ def _ingest_full_records_archive( blob_hash=blob_hash, blob_size=record.blob_size, ): + evidence_kind = ( + RawFailureEvidenceKind.DEFERRED_CLAUDE_CODE_PARTIAL_JSONL + if provider is Provider.CLAUDE_CODE + else RawFailureEvidenceKind.DEFERRED_HOT_JSONL_CAPTURE + ) archive.record_raw_failure_evidence( source_raw_id, provider=provider, source_path=record.source_path, source_index=record.source_index or 0, acquired_at_ms=acquired_at_ms, - kind=RawFailureEvidenceKind.DEFERRED_HOT_JSONL_CAPTURE, + kind=evidence_kind, ) archive.mark_raw_parse_failed( source_raw_id, provider=provider, error=ValueError("captured JSONL payload ends before a complete record boundary"), + preserve_existing_failure_evidence=True, ) result.raw_ids[record.raw_id] = source_raw_id _accumulate_stage_timings(result.stage_timings_s, record_timings) @@ -2453,6 +2486,7 @@ def _ingest_full_records_archive( source_raw_id, provider=provider, error=ValueError("captured JSONL payload ends before a complete record boundary"), + preserve_existing_failure_evidence=True, ) result.raw_ids[record.raw_id] = source_raw_id _accumulate_stage_timings(result.stage_timings_s, record_timings) @@ -2518,23 +2552,43 @@ def _ingest_full_records_archive( with blob_store.open(blob_hash) as payload_handle: sessions = parse_stream_payload( provider, - _iter_json_stream(payload_handle, source_name), + _iter_json_stream( + payload_handle, + source_name, + fail_on_decode_error=provider is Provider.UNKNOWN, + ), fallback_id, source_path=record.source_path, ) else: sessions = parse_stream_payload( provider, - _iter_json_stream(BytesIO(payload), source_name), + _iter_json_stream( + BytesIO(payload), + source_name, + fail_on_decode_error=provider is Provider.UNKNOWN, + ), fallback_id, source_path=record.source_path, ) else: if payload is None: with blob_store.open(blob_hash) as payload_handle: - payloads = list(_iter_json_stream(payload_handle, source_name)) + payloads = list( + _iter_json_stream( + payload_handle, + source_name, + fail_on_decode_error=provider is Provider.UNKNOWN, + ) + ) else: - payloads = list(_iter_json_stream(BytesIO(payload), source_name)) + payloads = list( + _iter_json_stream( + BytesIO(payload), + source_name, + fail_on_decode_error=provider is Provider.UNKNOWN, + ) + ) sessions = parse_payload( provider, payloads, @@ -2560,7 +2614,11 @@ def _ingest_full_records_archive( source_path=record.source_path, source_index=record.source_index or 0, acquired_at_ms=acquired_at_ms, - kind=RawFailureEvidenceKind.TERMINAL_UNSUPPORTED_SHAPE, + kind=( + RawFailureEvidenceKind.TERMINAL_UNKNOWN_EXPORT_NO_SESSION + if provider is Provider.UNKNOWN + else RawFailureEvidenceKind.TERMINAL_UNSUPPORTED_SHAPE + ), ) archive.mark_raw_parse_failed( source_raw_id, @@ -2568,6 +2626,7 @@ def _ingest_full_records_archive( error=ValueError( "parsed raw payload produced no sessions with positive conversational evidence" ), + preserve_existing_failure_evidence=True, ) result.raw_ids[record.raw_id] = source_raw_id _accumulate_stage_timings(result.stage_timings_s, record_timings) @@ -2806,7 +2865,24 @@ def _ingest_full_records_archive( ) raise if provider is not None and source_raw_id is not None: - archive.mark_raw_parse_failed(source_raw_id, provider=provider, error=exc) + preserve_existing_failure_evidence = False + if provider is Provider.UNKNOWN and _is_json_stream_decode_error(exc): + archive.record_raw_failure_evidence( + source_raw_id, + provider=provider, + source_path=record.source_path, + source_index=record.source_index or 0, + acquired_at_ms=acquired_at_ms, + kind=RawFailureEvidenceKind.TERMINAL_UNKNOWN_JSON_DECODE, + ) + result.terminal_raw_ids[record.raw_id] = source_raw_id + preserve_existing_failure_evidence = True + archive.mark_raw_parse_failed( + source_raw_id, + provider=provider, + error=exc, + preserve_existing_failure_evidence=preserve_existing_failure_evidence, + ) logger.warning( "live.watcher: archive full ingest failed for %s: %s: %s", record.source_path, @@ -3298,6 +3374,10 @@ def _raw_failure_requires_full_replay(self, path: Path, raw_id: str) -> bool: if not source_db.exists(): return False placeholders = ", ".join("?" for _ in RAW_FAILURE_EVIDENCE_KINDS) + support_pairs = " OR ".join( + "(a.artifact_kind = ? AND a.support_status = ?)" + for _ in RAW_FAILURE_LIFECYCLE_EVIDENCE_SUPPORT_STATUS_PAIRS + ) try: conn = sqlite3.connect(f"file:{source_db}?mode=ro", uri=True) try: @@ -3308,12 +3388,21 @@ def _raw_failure_requires_full_replay(self, path: Path, raw_id: str) -> bool: FROM raw_sessions AS r JOIN raw_artifacts AS a ON a.raw_id = r.raw_id WHERE r.raw_id = ? + AND r.origin IS a.origin + AND r.source_path IS a.source_path AND r.source_path = ? + AND r.source_index IS a.source_index AND r.parse_error IS NOT NULL AND a.artifact_kind IN ({placeholders}) + AND ({support_pairs}) LIMIT 1 """, - (raw_id, str(path), *sorted(RAW_FAILURE_EVIDENCE_KINDS)), + ( + raw_id, + str(path), + *sorted(RAW_FAILURE_EVIDENCE_KINDS), + *[value for pair in RAW_FAILURE_LIFECYCLE_EVIDENCE_SUPPORT_STATUS_PAIRS for value in pair], + ), ).fetchone() is not None ) diff --git a/polylogue/sources/live/parse_prefetch.py b/polylogue/sources/live/parse_prefetch.py index a1c70dddf2..90605fb22e 100644 --- a/polylogue/sources/live/parse_prefetch.py +++ b/polylogue/sources/live/parse_prefetch.py @@ -141,12 +141,22 @@ def live_parse_worker( if is_stream: sessions = parse_stream_payload( provider, - _iter_json_stream(BytesIO(payload), source_name), + _iter_json_stream( + BytesIO(payload), + source_name, + fail_on_decode_error=provider is Provider.UNKNOWN, + ), fallback_id, source_path=source_path, ) else: - payloads = list(_iter_json_stream(BytesIO(payload), source_name)) + payloads = list( + _iter_json_stream( + BytesIO(payload), + source_name, + fail_on_decode_error=provider is Provider.UNKNOWN, + ) + ) sessions = parse_payload(provider, payloads, fallback_id, source_path=source_path) return cache_key, sessions, None except Exception as exc: diff --git a/polylogue/storage/raw/artifacts.py b/polylogue/storage/raw/artifacts.py index 54948874a0..974b7bd17e 100644 --- a/polylogue/storage/raw/artifacts.py +++ b/polylogue/storage/raw/artifacts.py @@ -20,6 +20,7 @@ class RawBacklogQuerySpec: require_unparsed: bool require_unvalidated: bool = False validation_statuses: tuple[str, ...] | None = None + exclude_terminal_failure_evidence: bool = False @dataclass(frozen=True, slots=True) @@ -72,6 +73,7 @@ def parse_backlog_query_spec(*, force_reparse: bool = False) -> RawBacklogQueryS return RawBacklogQuerySpec( require_unparsed=not force_reparse, validation_statuses=None if force_reparse else tuple(status.value for status in _PARSEABLE_VALIDATION_STATUSES), + exclude_terminal_failure_evidence=not force_reparse, ) diff --git a/polylogue/storage/raw_failure_lifecycle.py b/polylogue/storage/raw_failure_lifecycle.py index fa90cc8371..0c94c8820f 100644 --- a/polylogue/storage/raw_failure_lifecycle.py +++ b/polylogue/storage/raw_failure_lifecycle.py @@ -16,7 +16,9 @@ from typing import Literal from polylogue.core.raw_failure_evidence import ( - RawFailureEvidenceKind, + RAW_FAILURE_LIFECYCLE_EVIDENCE_SUPPORT_STATUS_PAIRS, + raw_failure_outcome_code, + validated_raw_failure_evidence_kind, ) from polylogue.storage.sqlite.connection_profile import open_readonly_connection @@ -82,23 +84,23 @@ def _lifecycle( support_status: object, *, validation_failed: bool, + classification_reason: object = None, ) -> RawFailureLifecycle: """Classify an artifact only when its closed evidence is self-consistent.""" - if validation_failed: - return "unexplained" - if artifact_kind is None or support_status is None: - return "unexplained" - kind = str(artifact_kind) - support = str(support_status) - try: - evidence_kind = RawFailureEvidenceKind(kind) - except ValueError: - return "unexplained" - if evidence_kind.lifecycle not in {"deferred", "terminal"}: - return "unexplained" - if evidence_kind.support_status.value != support: + evidence_kind = validated_raw_failure_evidence_kind( + artifact_kind, + support_status, + validation_failed=validation_failed, + classification_reason=classification_reason, + outcome_code=raw_failure_outcome_code(classification_reason), + ) + if evidence_kind is None: return "unexplained" - return "deferred" if evidence_kind.lifecycle == "deferred" else "terminal" + if evidence_kind.lifecycle == "deferred": + return "deferred" + if evidence_kind.lifecycle == "terminal": + return "terminal" + return "unexplained" def read_raw_failure_lifecycle(source_db: Path, *, sample_limit: int = 10) -> RawFailureLifecycleSnapshot: @@ -140,19 +142,31 @@ def read_raw_failure_lifecycle(source_db: Path, *, sample_limit: int = 10) -> Ra OR r.validation_status = 'failed' ) """ + typed_failure_placeholders = ", ".join("(?, ?)" for _ in RAW_FAILURE_LIFECYCLE_EVIDENCE_SUPPORT_STATUS_PAIRS) + valid_failure_artifacts_cte = ( + """ + , valid_failure_artifacts AS ( + SELECT a.* + FROM raw_artifacts AS a + WHERE (a.artifact_kind, a.support_status) IN (""" + + typed_failure_placeholders + + """) + ) + """ + ) latest_artifact_join = """ - LEFT JOIN raw_artifacts AS a - ON a.raw_id = f.raw_id - AND a.origin = f.origin - AND a.source_path = f.source_path - AND a.source_index = f.source_index + LEFT JOIN valid_failure_artifacts AS a + ON a.raw_id IS f.raw_id + AND a.origin IS f.origin + AND a.source_path IS f.source_path + AND a.source_index IS f.source_index AND NOT EXISTS ( SELECT 1 - FROM raw_artifacts AS newer - WHERE newer.raw_id = a.raw_id - AND newer.origin = a.origin - AND newer.source_path = a.source_path - AND newer.source_index = a.source_index + FROM valid_failure_artifacts AS newer + WHERE newer.raw_id IS a.raw_id + AND newer.origin IS a.origin + AND newer.source_path IS a.source_path + AND newer.source_index IS a.source_index AND (newer.last_observed_at_ms > a.last_observed_at_ms OR (newer.last_observed_at_ms = a.last_observed_at_ms AND newer.artifact_id > a.artifact_id)) @@ -161,40 +175,41 @@ def read_raw_failure_lifecycle(source_db: Path, *, sample_limit: int = 10) -> Ra if has_artifacts: summary_sql = ( failed_cte + + valid_failure_artifacts_cte + """ SELECT f.origin, f.validation_status, a.artifact_kind, a.support_status, + a.classification_reason, COUNT(*) AS failure_count FROM failed AS f """ + latest_artifact_join + """ - GROUP BY f.origin, f.validation_status, a.artifact_kind, a.support_status - ORDER BY f.origin, f.validation_status, a.artifact_kind, a.support_status + GROUP BY f.origin, f.validation_status, a.artifact_kind, a.support_status, + a.classification_reason + ORDER BY f.origin, f.validation_status, a.artifact_kind, a.support_status, + a.classification_reason """ ) sample_sql = ( failed_cte + + valid_failure_artifacts_cte + """ , sampled AS ( SELECT f.raw_id, f.origin, f.validation_status, f.acquired_at_ms, - a.artifact_kind, a.support_status + a.artifact_kind, a.support_status, a.classification_reason FROM failed AS f """ + latest_artifact_join + """ ORDER BY CASE - WHEN f.validation_status = 'failed' THEN 0 - WHEN (a.artifact_kind, a.support_status) IN ( - ('deferred_hot_jsonl_capture', 'partial_decode'), - ('terminal_corrupt_input', 'decode_failed'), - ('terminal_unsupported_shape', 'unsupported_parseable') - ) THEN 1 + WHEN a.artifact_kind IS NOT NULL THEN 0 ELSE 2 END, f.acquired_at_ms DESC, f.raw_id DESC LIMIT ? ) - SELECT raw_id, origin, validation_status, artifact_kind, support_status + SELECT raw_id, origin, validation_status, artifact_kind, support_status, + classification_reason FROM sampled """ ) @@ -202,7 +217,7 @@ def read_raw_failure_lifecycle(source_db: Path, *, sample_limit: int = 10) -> Ra summary_sql = ( failed_cte + """ - SELECT f.origin, f.validation_status, NULL, NULL, COUNT(*) AS failure_count + SELECT f.origin, f.validation_status, NULL, NULL, NULL, COUNT(*) AS failure_count FROM failed AS f GROUP BY f.origin, f.validation_status ORDER BY f.origin, f.validation_status @@ -211,14 +226,18 @@ def read_raw_failure_lifecycle(source_db: Path, *, sample_limit: int = 10) -> Ra sample_sql = ( failed_cte + """ - SELECT raw_id, origin, validation_status, NULL, NULL + SELECT raw_id, origin, validation_status, NULL, NULL, NULL FROM failed ORDER BY acquired_at_ms DESC, raw_id DESC LIMIT ? """ ) - summary_rows = conn.execute(summary_sql).fetchall() - sample_rows = conn.execute(sample_sql, (sample_limit,)).fetchall() + typed_failure_params = tuple( + value for pair in RAW_FAILURE_LIFECYCLE_EVIDENCE_SUPPORT_STATUS_PAIRS for value in pair + ) + summary_rows = conn.execute(summary_sql, typed_failure_params if has_artifacts else ()).fetchall() + sample_params: tuple[object, ...] = typed_failure_params + (sample_limit,) if has_artifacts else (sample_limit,) + sample_rows = conn.execute(sample_sql, sample_params).fetchall() except sqlite3.Error as exc: logger.warning("could not read raw failure lifecycle", exc_info=exc) return RawFailureLifecycleSnapshot(False, reason=f"could not read raw failure lifecycle: {exc}") @@ -233,9 +252,15 @@ def read_raw_failure_lifecycle(source_db: Path, *, sample_limit: int = 10) -> Ra origin = str(row[0] or "unknown") artifact_kind = str(row[2]) if row[2] is not None else None support_status = str(row[3]) if row[3] is not None else None + classification_reason = row[4] validation_failed = str(row[1] or "") == "failed" - lifecycle = _lifecycle(artifact_kind, support_status, validation_failed=validation_failed) - count = int(row[4]) + lifecycle = _lifecycle( + artifact_kind, + support_status, + validation_failed=validation_failed, + classification_reason=classification_reason, + ) + count = int(row[5]) counts[lifecycle] += count by_origin[origin] += count by_artifact_kind[artifact_kind or ""] += count @@ -243,6 +268,7 @@ def read_raw_failure_lifecycle(source_db: Path, *, sample_limit: int = 10) -> Ra origin = str(row[1] or "unknown") artifact_kind = str(row[3]) if row[3] is not None else None support_status = str(row[4]) if row[4] is not None else None + classification_reason = row[5] samples.append( { "raw_id": str(row[0]), @@ -253,7 +279,9 @@ def read_raw_failure_lifecycle(source_db: Path, *, sample_limit: int = 10) -> Ra artifact_kind, support_status, validation_failed=str(row[2] or "") == "failed", + classification_reason=classification_reason, ), + "classification_reason": str(classification_reason) if classification_reason is not None else None, } ) return RawFailureLifecycleSnapshot( diff --git a/polylogue/storage/repair.py b/polylogue/storage/repair.py index 5f82b8171a..f61ef0b076 100644 --- a/polylogue/storage/repair.py +++ b/polylogue/storage/repair.py @@ -24,8 +24,15 @@ from polylogue.archive.revision_replay import ApplicationDecision from polylogue.config import Config from polylogue.core.enums import Origin, Provider +from polylogue.core.errors import RawCASFrontierError from polylogue.core.json import JSONDocument, json_document from polylogue.core.protocols import ProgressCallback +from polylogue.core.raw_failure_evidence import ( + RAW_FAILURE_DEFERRED_EVIDENCE_KINDS, + RAW_FAILURE_DEFERRED_SUPPORT_STATUS, + RAW_FAILURE_REPLAY_AUTHORITY_EVIDENCE_KINDS, + RAW_FAILURE_TERMINAL_EVIDENCE_SUPPORT_STATUS_PAIRS, +) from polylogue.core.sources import origin_from_provider, origin_provider_fiber, provider_from_origin from polylogue.logging import get_logger from polylogue.maintenance.models import DerivedModelStatus, MaintenanceCategory @@ -103,22 +110,15 @@ RAW_MATERIALIZATION_COMMIT_BATCH_SIZE = 20 RAW_MATERIALIZATION_OUTCOME_SAMPLE_LIMIT = 8 _TRANSIENT_LOCK_PARSE_ERROR = "OperationalError: database is locked" -#: polylogue-5iz4: ``MembershipReplayConflictError``'s ``parse_error`` text -#: (``f"{type(exc).__name__}: {exc}"``, set by ``mark_raw_parse_failed`` in -#: ``storage/sqlite/archive_tiers/revision_governance.py``) always starts -#: with this exact type name regardless of the exception's own -#: human-readable message wording, which is not itself stable (it has -#: already drifted once since PR #2718 introduced this guard under -#: different phrasing). Matching on the type name here is what keeps a raw -#: that hit this guard retry-eligible even after the message wording -#: changes again. +# Membership replay conflicts are typed on new writes, so their diagnostic +# wording is not an authority signal. Keep the prefix as a bounded bridge for +# rows written before typed evidence was persisted. Removing this bridge needs +# a backup-gated migration or re-observation receipt for every retained row. _MEMBERSHIP_REPLAY_CONFLICT_ERROR_PREFIX = "MembershipReplayConflictError:" -# These are durable parse-error values written before the corresponding -# revision guards acquired typed exception classes. They name an authority -# refusal, not malformed source evidence: replay must reconsider them against -# the current accepted frontier. Keep the complete legacy values exact so a -# broad RuntimeError prefix cannot turn unrelated parser failures into an -# unbounded repair loop. +# These complete values are the two legacy CAS refusals observed before the +# typed exception/evidence path existed. They authorize reconsideration only +# as exact authority markers, never arbitrary RuntimeError prose. Removal is +# gated by the same backup-bound migration or re-observation receipt. _LEGACY_OLDER_ACCEPTED_FRONTIER_ERROR = "RuntimeError: raw revision CAS rejected an older accepted frontier" _LEGACY_UNCONVERTIBLE_BYTE_HEAD_ERROR = "RuntimeError: membership replay cannot replace an unconvertible byte head" _QUARANTINED_ACCEPTED_RAW_REPAIR_DETAIL = "repair:accepted_quarantined_raw_exact_byte_and_semantic_proof" @@ -1221,7 +1221,7 @@ def _cas_refine_quarantined_accepted_raw( ), ) if cursor.rowcount != 1: - raise RuntimeError(f"source authority CAS failed for {item.raw_id}") + raise RawCASFrontierError(f"source authority CAS failed for {item.raw_id}") def inspect_quarantined_accepted_raws( @@ -2705,13 +2705,13 @@ def _finalize_browser_origin_copy_forward_index(conn: sqlite3.Connection, item: ), ) if cursor.rowcount != 1: - raise RuntimeError(f"semantic canonical-head CAS failed for {item.raw_id}") + raise RawCASFrontierError(f"semantic canonical-head CAS failed for {item.raw_id}") cursor = conn.execute( "UPDATE sessions SET raw_id = ? WHERE session_id = ? AND raw_id = ?", (item.copy_forward_raw_id, item.session_id, item.raw_id), ) if cursor.rowcount != 1: - raise RuntimeError(f"session raw pointer CAS failed for {item.raw_id}") + raise RawCASFrontierError(f"session raw pointer CAS failed for {item.raw_id}") record_revision_application_sync( conn, RevisionApplicationReceipt( @@ -2801,7 +2801,7 @@ def _retire_browser_origin_legacy_head( ), ).rowcount if deleted != 1: - raise RuntimeError(f"obsolete browser-origin head CAS failed for {item.raw_id}") + raise RawCASFrontierError(f"obsolete browser-origin head CAS failed for {item.raw_id}") def _restore_browser_origin_canonical_head(conn: sqlite3.Connection, item: BrowserCaptureOriginRepairItem) -> None: @@ -2819,7 +2819,7 @@ def _restore_browser_origin_canonical_head(conn: sqlite3.Connection, item: Brows (item.replacement_raw_id, item.session_id, item.raw_id), ) if cursor.rowcount != 1: - raise RuntimeError(f"session raw pointer CAS failed for {item.raw_id}") + raise RawCASFrontierError(f"session raw pointer CAS failed for {item.raw_id}") _retire_browser_origin_legacy_head( conn, item, @@ -3662,7 +3662,7 @@ def _apply_duplicate_raw_identity_repair(conn: sqlite3.Connection, item: Duplica (item.canonical_raw_id, item.session_id, item.stale_raw_id), ) if cursor.rowcount != 1: - raise RuntimeError(f"session raw pointer CAS failed for {item.stale_raw_id}") + raise RawCASFrontierError(f"session raw pointer CAS failed for {item.stale_raw_id}") record_revision_application_sync( conn, RevisionApplicationReceipt( @@ -3770,6 +3770,16 @@ def _raw_materialization_archive_root(config: Config) -> Path: return archive_file_set_root(archive_root=config.archive_root, db_path=config.db_path) +def _raw_artifact_coordinate_predicate(*, artifact_alias: str, raw_alias: str) -> str: + """Correlate evidence with the exact failed artifact observation.""" + return f""" + AND {artifact_alias}.raw_id IS {raw_alias}.raw_id + AND {artifact_alias}.origin IS {raw_alias}.origin + AND {artifact_alias}.source_path IS {raw_alias}.source_path + AND {artifact_alias}.source_index IS {raw_alias}.source_index + """ + + def _raw_materialization_candidate_ids( config: Config, *, @@ -3847,11 +3857,22 @@ def _raw_materialization_candidate_ids( normalized_root = str(source_root).rstrip("/") source_root_filter = " AND (r.source_path = ? OR r.source_path LIKE ?)" params.extend((normalized_root, f"{normalized_root}/%")) + terminal_pair_placeholders = ", ".join("(?, ?)" for _ in RAW_FAILURE_TERMINAL_EVIDENCE_SUPPORT_STATUS_PAIRS) rows = conn.execute( f""" SELECT r.raw_id, r.origin, r.native_id, r.source_path, r.blob_hash, r.blob_size, r.acquired_at_ms, r.parsed_at_ms, r.parse_error, + ( + SELECT a.artifact_kind + FROM raw_artifacts AS a + WHERE 1 = 1 + {_raw_artifact_coordinate_predicate(artifact_alias="a", raw_alias="r")} + AND a.artifact_kind IN ({", ".join("?" for _ in RAW_FAILURE_REPLAY_AUTHORITY_EVIDENCE_KINDS)}) + AND a.support_status = ? + ORDER BY a.last_observed_at_ms DESC, a.artifact_id DESC + LIMIT 1 + ) AS failure_artifact_kind, EXISTS ( SELECT 1 FROM index_tier.raw_revision_applications AS a @@ -3922,30 +3943,39 @@ def _raw_materialization_candidate_ids( s_by_native.native_id IS NULL OR existing_native_raw.raw_id IS NULL ) + -- A failed worker validation is not replay authority. Keep + -- the raw bytes and their diagnostics, but require a fresh + -- validation outcome before materialization can select them. + AND COALESCE(r.validation_status, '') != 'failed' AND ( r.parse_error IS NULL OR r.parse_error = 'OperationalError: database is locked' OR ( r.parse_error LIKE 'decode:%No such file or directory:%' ) - OR ( - -- polylogue-5iz4: MembershipReplayConflictError - -- (storage/sqlite/archive_tiers/revision_governance.py) is a - -- transient, retry-eligible refusal by construction -- a - -- later pass over the SAME durable raw bytes can succeed - -- once sibling evidence resolves or the accepted head - -- itself changes. Matching by exception TYPE name (stable) - -- rather than the human-readable message text (which has - -- already drifted twice since #2718 introduced this guard) - -- is what keeps this retry-eligible even after the guard's - -- own wording changes again. - r.parse_error LIKE '{_MEMBERSHIP_REPLAY_CONFLICT_ERROR_PREFIX}%' + OR EXISTS ( + SELECT 1 + FROM raw_artifacts AS retry_evidence + WHERE 1 = 1 + {_raw_artifact_coordinate_predicate(artifact_alias="retry_evidence", raw_alias="r")} + AND retry_evidence.artifact_kind IN ({", ".join("?" for _ in RAW_FAILURE_REPLAY_AUTHORITY_EVIDENCE_KINDS)}) + AND retry_evidence.support_status = ? ) + OR r.parse_error LIKE '{_MEMBERSHIP_REPLAY_CONFLICT_ERROR_PREFIX}%' OR r.parse_error IN ( '{_LEGACY_OLDER_ACCEPTED_FRONTIER_ERROR}', '{_LEGACY_UNCONVERTIBLE_BYTE_HEAD_ERROR}' ) ) + AND NOT EXISTS ( + SELECT 1 + FROM raw_artifacts AS terminal_evidence + WHERE 1 = 1 + {_raw_artifact_coordinate_predicate(artifact_alias="terminal_evidence", raw_alias="r")} + AND (terminal_evidence.artifact_kind, terminal_evidence.support_status) IN ( + {terminal_pair_placeholders} + ) + ) AND NOT ( COALESCE(r.validation_status, '') = 'skipped' AND r.parsed_at_ms IS NOT NULL @@ -3956,7 +3986,16 @@ def _raw_materialization_candidate_ids( {source_root_filter} ORDER BY r.acquired_at_ms DESC, r.raw_id ASC """, - [BYTE_AUTHORITY_CENSUS_DETAIL, BYTE_AUTHORITY_CENSUS_DETAIL, *params], + [ + *sorted(RAW_FAILURE_REPLAY_AUTHORITY_EVIDENCE_KINDS), + RAW_FAILURE_DEFERRED_SUPPORT_STATUS, + BYTE_AUTHORITY_CENSUS_DETAIL, + BYTE_AUTHORITY_CENSUS_DETAIL, + *sorted(RAW_FAILURE_REPLAY_AUTHORITY_EVIDENCE_KINDS), + RAW_FAILURE_DEFERRED_SUPPORT_STATUS, + *[value for pair in RAW_FAILURE_TERMINAL_EVIDENCE_SUPPORT_STATUS_PAIRS for value in pair], + *params, + ], ).fetchall() adoption_deferred = 0 authority_quarantined = 0 @@ -3992,7 +4031,10 @@ def _raw_materialization_candidate_ids( byte_authority_pending += 1 byte_authority_pending_raw_ids.append(row_raw_id) continue - if row["parse_error"] and not _raw_materialization_retryable_missing_blob_error(row["parse_error"]): + if row["parse_error"] and not _raw_materialization_retryable_missing_blob_error( + row["parse_error"], + row["failure_artifact_kind"] in RAW_FAILURE_REPLAY_AUTHORITY_EVIDENCE_KINDS, + ): continue if _raw_materialized_by_source_path_native(materialized_aliases, row): continue @@ -4175,21 +4217,14 @@ def _raw_materialization_component_stream_safe( return all(_raw_materialization_stream_safe(candidates, raw_id) for raw_id in component) -def _raw_materialization_retryable_missing_blob_error(parse_error: object) -> bool: +def _raw_materialization_retryable_missing_blob_error(parse_error: object, durable_retryable: bool = False) -> bool: + if durable_retryable: + return True if not isinstance(parse_error, str): return False return ( parse_error == _TRANSIENT_LOCK_PARSE_ERROR or (parse_error.startswith("decode:") and "No such file or directory" in parse_error) - # polylogue-5iz4: MembershipReplayConflictError - # (storage/sqlite/archive_tiers/revision_governance.py) is a - # transient, retry-eligible refusal by construction -- see the SQL - # candidate query's matching clause above for the full rationale. - # This Python-side check re-validates every row the SQL WHERE - # clause already passed and is the true single source of truth for - # retry eligibility (the SQL clause is a pre-filter, not merely an - # optimization the SQL and this function must independently agree, - # or a row that clears the SQL gate is silently re-excluded here). or parse_error.startswith(_MEMBERSHIP_REPLAY_CONFLICT_ERROR_PREFIX) or parse_error in { @@ -4511,9 +4546,22 @@ def _raw_replay_plan_outcome( UNION ALL SELECT 1 FROM raw_sessions + WHERE raw_id IN ({placeholders}) + AND validation_status = 'failed' + UNION ALL + SELECT 1 + FROM raw_sessions WHERE raw_id IN ({placeholders}) AND parse_error IS NOT NULL AND parse_error != ? + AND NOT EXISTS ( + SELECT 1 + FROM raw_artifacts AS retry_evidence + WHERE 1 = 1 + {_raw_artifact_coordinate_predicate(artifact_alias="retry_evidence", raw_alias="raw_sessions")} + AND retry_evidence.artifact_kind IN ({", ".join("?" for _ in RAW_FAILURE_DEFERRED_EVIDENCE_KINDS)}) + AND retry_evidence.support_status = ? + ) UNION ALL SELECT 1 FROM raw_membership_census @@ -4526,7 +4574,10 @@ def _raw_replay_plan_outcome( *component, superseded_json, *component, + *component, _TRANSIENT_LOCK_PARSE_ERROR, + *sorted(RAW_FAILURE_DEFERRED_EVIDENCE_KINDS), + RAW_FAILURE_DEFERRED_SUPPORT_STATUS, *component, ), ).fetchone() diff --git a/polylogue/storage/sqlite/archive_tiers/archive.py b/polylogue/storage/sqlite/archive_tiers/archive.py index 21c54a1953..e86b804d46 100644 --- a/polylogue/storage/sqlite/archive_tiers/archive.py +++ b/polylogue/storage/sqlite/archive_tiers/archive.py @@ -2790,8 +2790,21 @@ def apply_raw_membership_classification( def finalize_raw_parse_state(self, raw_id: str, *, state: RawSessionStateUpdate) -> None: return finalize_raw_parse_state(self, raw_id, state=state) - def mark_raw_parse_failed(self, raw_id: str, *, provider: Provider, error: BaseException) -> None: - return mark_raw_parse_failed(self, raw_id, provider=provider, error=error) + def mark_raw_parse_failed( + self, + raw_id: str, + *, + provider: Provider, + error: BaseException, + preserve_existing_failure_evidence: bool = False, + ) -> None: + return mark_raw_parse_failed( + self, + raw_id, + provider=provider, + error=error, + preserve_existing_failure_evidence=preserve_existing_failure_evidence, + ) def record_raw_failure_evidence( self, diff --git a/polylogue/storage/sqlite/archive_tiers/revision_governance.py b/polylogue/storage/sqlite/archive_tiers/revision_governance.py index 886c3a0a38..059bdab70f 100644 --- a/polylogue/storage/sqlite/archive_tiers/revision_governance.py +++ b/polylogue/storage/sqlite/archive_tiers/revision_governance.py @@ -129,7 +129,12 @@ ) from polylogue.archive.session_revision_membership import MembershipClassification, MembershipDecision from polylogue.core.enums import Origin, Provider -from polylogue.core.raw_failure_evidence import RawFailureEvidenceKind +from polylogue.core.errors import RawCASFrontierError +from polylogue.core.raw_failure_evidence import ( + RAW_FAILURE_DEFERRED_SUPPORT_STATUS, + RawFailureEvidenceKind, + raw_failure_classification_reason, +) from polylogue.core.sources import origin_from_provider, provider_from_origin from polylogue.pipeline.ids import SessionRevisionProjection, session_content_hash, session_revision_projection from polylogue.pipeline.ids import session_id as make_session_id @@ -181,7 +186,7 @@ class ActiveByteRevisionChainError(RuntimeError): """A byte-identical revision chain cannot admit a conflicting sibling.""" -class MembershipReplayConflictError(RuntimeError): +class MembershipReplayConflictError(RawCASFrontierError): """Membership replay refused to move an accepted head this pass. Raised by ``apply_raw_membership_classification`` when it cannot safely @@ -192,17 +197,9 @@ class MembershipReplayConflictError(RuntimeError): over the same durable raw bytes can succeed once sibling evidence resolves or the accepted head itself changes (polylogue-5iz4). - A dedicated subclass exists so ``mark_raw_parse_failed``'s ``parse_error`` - text (``f"{type(exc).__name__}: {exc}"``) carries a stable, matchable - marker for retry-eligibility checks (``storage/repair.py``'s raw - materialization candidate query) independent of this class's own - human-readable message wording, which has already drifted twice (#2718's - original "unconvertible byte head" phrasing no longer appears anywhere in - this module) and will keep drifting as the guard is refined. A plain - ``RuntimeError`` gives the retry-candidate query nothing durable to match - on beyond the exact message text, which is how a raw that hit this guard - under old wording got silently excluded from every future rebuild even - after the guard's conditions no longer applied to it. + A dedicated subclass exists so the raw-failure boundary can persist a + structured retryable evidence kind. The free-form ``parse_error`` remains + a diagnostic only and is not an authorization signal for replay. """ @@ -3023,6 +3020,13 @@ def apply_raw_membership_classification( def finalize_raw_parse_state(store: RawRevisionGovernanceHost, raw_id: str, *, state: RawSessionStateUpdate) -> None: """Commit one typed source parse state after its index outcome.""" + # A generic state update is also used by the retained-raw index route when + # it has no typed worker disposition to persist. Retire any prior + # failure authority before recording that new untyped failure; otherwise + # an old terminal/deferred carrier can continue to authorize replay after + # the current attempt has failed for a different reason. + if isinstance(state.parse_error, str) and state.parse_error: + _retire_raw_failure_evidence(store, raw_id, manage_transaction=False) apply_source_raw_state_update( store._ensure_source_conn(), raw_id, @@ -3032,10 +3036,43 @@ def finalize_raw_parse_state(store: RawRevisionGovernanceHost, raw_id: str, *, s def mark_raw_parse_failed( - store: RawRevisionGovernanceHost, raw_id: str, *, provider: Provider, error: BaseException + store: RawRevisionGovernanceHost, + raw_id: str, + *, + provider: Provider, + error: BaseException, + preserve_existing_failure_evidence: bool = False, ) -> None: """Persist a bounded parse/index failure for retained raw evidence.""" - finalize_raw_parse_state(store, raw_id, state=_raw_parse_failure_state(provider, error)) + conn = store._ensure_source_conn() + with conn: + if isinstance(error, RawCASFrontierError): + row = conn.execute( + "SELECT source_path, source_index, acquired_at_ms FROM raw_sessions WHERE raw_id = ?", + (raw_id,), + ).fetchone() + if row is not None: + _retire_raw_failure_evidence(store, raw_id, manage_transaction=False) + record_raw_failure_evidence( + store, + raw_id, + provider=provider, + source_path=str(row[0] or raw_id), + source_index=int(row[1] or 0), + acquired_at_ms=int(row[2] or int(time.time() * 1000)), + kind=RawFailureEvidenceKind.DEFERRED_CAS_FRONTIER, + manage_transaction=False, + ) + else: + if not preserve_existing_failure_evidence: + _supersede_deferred_cas_evidence(store, raw_id, provider=provider, manage_transaction=False) + _retire_raw_failure_evidence(store, raw_id, manage_transaction=False) + apply_source_raw_state_update( + conn, + raw_id, + state=_raw_parse_failure_state(provider, error), + manage_transaction=False, + ) def record_raw_failure_evidence( @@ -3047,6 +3084,7 @@ def record_raw_failure_evidence( source_index: int, acquired_at_ms: int, kind: RawFailureEvidenceKind, + manage_transaction: bool = True, ) -> None: """Persist a closed parse-outcome classification beside retained bytes.""" from hashlib import sha256 @@ -3055,6 +3093,24 @@ def record_raw_failure_evidence( from polylogue.storage.sqlite.archive_tiers.source_write import ArchiveSourceArtifact, upsert_raw_artifact artifact_id = "raw-failure:" + sha256(f"{raw_id}:{kind.value}".encode()).hexdigest() + raw_row = ( + store._ensure_source_conn() + .execute( + "SELECT validation_status FROM raw_sessions WHERE raw_id = ?", + (raw_id,), + ) + .fetchone() + ) + validation_failed = raw_row is not None and str(raw_row[0] or "") == "failed" + outcome_code = ( + "corrupt_input" + if kind + in { + RawFailureEvidenceKind.TERMINAL_CORRUPT_INPUT, + RawFailureEvidenceKind.TERMINAL_UNKNOWN_JSON_DECODE, + } + else kind.value + ) upsert_raw_artifact( store._ensure_source_conn(), raw_id, @@ -3064,19 +3120,160 @@ def record_raw_failure_evidence( source_path=source_path, source_index=source_index, artifact_kind=kind.value, - classification_reason=kind.value, + classification_reason=raw_failure_classification_reason( + diagnostic=None, + evidence_ref=None, + outcome_code=outcome_code, + remediation=None, + retryable=False, + trusted_validation_failure=( + validation_failed + and kind + in { + RawFailureEvidenceKind.TERMINAL_CORRUPT_INPUT, + RawFailureEvidenceKind.TERMINAL_UNKNOWN_JSON_DECODE, + } + and outcome_code == "corrupt_input" + ), + ), support_status=kind.support_status, - parse_as_session=kind is RawFailureEvidenceKind.DEFERRED_HOT_JSONL_CAPTURE, - schema_eligible=kind is RawFailureEvidenceKind.DEFERRED_HOT_JSONL_CAPTURE, + parse_as_session=kind.lifecycle == "deferred", + schema_eligible=kind.lifecycle == "deferred", first_observed_at_ms=acquired_at_ms, last_observed_at_ms=acquired_at_ms, ), + manage_transaction=manage_transaction, + ) + + +def _supersede_deferred_cas_evidence( + store: RawRevisionGovernanceHost, + raw_id: str, + *, + provider: Provider, + manage_transaction: bool = True, +) -> None: + """Terminalize deferred CAS evidence once its attempt has resolved. + + ``raw_artifacts`` stores the latest observation for a source coordinate, + not an attempt history. Replace only an exact-coordinate deferred CAS + observation, so a neighboring artifact cannot be consumed or cleared by + this raw's outcome. + """ + conn = store._ensure_source_conn() + row = conn.execute( + """ + SELECT origin, source_path, source_index + FROM raw_sessions + WHERE raw_id = ? + """, + (raw_id,), + ).fetchone() + if row is None: + return + origin, source_path, source_index = row + deferred = conn.execute( + """ + SELECT 1 + FROM raw_artifacts + WHERE raw_id = ? + AND origin IS ? + AND source_path IS ? + AND source_index IS ? + AND artifact_kind IN (?, ?) + AND support_status = ? + LIMIT 1 + """, + ( + raw_id, + origin, + source_path, + source_index, + RawFailureEvidenceKind.DEFERRED_CAS_FRONTIER.value, + RawFailureEvidenceKind.DEFERRED_CODEX_CAS_FRONTIER.value, + RAW_FAILURE_DEFERRED_SUPPORT_STATUS, + ), + ).fetchone() + if deferred is None: + return + record_raw_failure_evidence( + store, + raw_id, + provider=provider, + source_path=str(source_path or raw_id), + source_index=int(source_index or 0), + acquired_at_ms=int(time.time() * 1000), + kind=RawFailureEvidenceKind.TERMINAL_SUPERSEDED_DEFERRED_CAS_FRONTIER, + manage_transaction=manage_transaction, ) +def _retire_raw_failure_evidence( + store: RawRevisionGovernanceHost, + raw_id: str, + *, + manage_transaction: bool = True, +) -> None: + """Retire stale failure evidence before an untyped current failure.""" + conn = store._ensure_source_conn() + row = conn.execute( + "SELECT origin, source_path, source_index FROM raw_sessions WHERE raw_id = ?", + (raw_id,), + ).fetchone() + if row is None: + return + retired_kinds = { + kind.value + for kind in RawFailureEvidenceKind + if kind is not RawFailureEvidenceKind.TERMINAL_SUPERSEDED_DEFERRED_CAS_FRONTIER + } + placeholders = ", ".join("?" for _ in retired_kinds) + with conn if manage_transaction else nullcontext(): + conn.execute( + f""" + UPDATE raw_artifacts + SET artifact_kind = ?, + support_status = ?, + classification_reason = ?, + parse_as_session = 0, + schema_eligible = 0 + WHERE raw_id = ? + AND origin IS ? + AND source_path IS ? + AND source_index IS ? + AND artifact_kind IN ({placeholders}) + """, + ( + RawFailureEvidenceKind.TERMINAL_SUPERSEDED_DEFERRED_CAS_FRONTIER.value, + RawFailureEvidenceKind.TERMINAL_SUPERSEDED_DEFERRED_CAS_FRONTIER.support_status.value, + raw_failure_classification_reason( + diagnostic=None, + evidence_ref=None, + outcome_code="failure_attempt_replaced", + remediation="inspect the current parser failure before retrying", + retryable=False, + trusted_validation_failure=False, + ), + raw_id, + row[0], + row[1], + row[2], + *sorted(retired_kinds), + ), + ) + + def mark_raw_parse_succeeded(store: RawRevisionGovernanceHost, raw_id: str, *, provider: Provider) -> None: """Finalize one retained raw payload after every derived session commits.""" - finalize_raw_parse_state(store, raw_id, state=_raw_parse_success_state(provider)) + conn = store._ensure_source_conn() + with conn: + _supersede_deferred_cas_evidence(store, raw_id, provider=provider, manage_transaction=False) + apply_source_raw_state_update( + conn, + raw_id, + state=_raw_parse_success_state(provider), + manage_transaction=False, + ) def _flush_pending_raw_parse_states(store: RawRevisionGovernanceHost) -> None: @@ -3085,6 +3282,14 @@ def _flush_pending_raw_parse_states(store: RawRevisionGovernanceHost) -> None: source_conn = store._ensure_source_conn() with source_conn: for raw_id, state in store._pending_raw_parse_states: + provider = state.payload_provider + if isinstance(provider, Provider) and isinstance(state.parsed_at, str) and state.parse_error is None: + _supersede_deferred_cas_evidence( + store, + raw_id, + provider=provider, + manage_transaction=False, + ) apply_source_raw_state_update( source_conn, raw_id, @@ -3130,7 +3335,14 @@ def _index_parsed_for_retained_raw( ) except Exception as exc: if not _is_frozen_candidate(store): - finalize_raw_parse_state(store, raw_id, state=_raw_parse_failure_state(provider, exc)) + if isinstance(exc, RawCASFrontierError): + # A retained-raw CAS refusal is retryable authority evidence. + # Persist that carrier with the first source-tier failure + # mutation so a crash cannot leave only the generic parse + # diagnostic behind. + mark_raw_parse_failed(store, raw_id, provider=provider, error=exc) + else: + finalize_raw_parse_state(store, raw_id, state=_raw_parse_failure_state(provider, exc)) raise if finalize_raw_parse and not _is_frozen_candidate(store): success_state = _raw_parse_success_state(provider) diff --git a/polylogue/storage/sqlite/archive_tiers/source.py b/polylogue/storage/sqlite/archive_tiers/source.py index f85ba4a5bf..ce23563903 100644 --- a/polylogue/storage/sqlite/archive_tiers/source.py +++ b/polylogue/storage/sqlite/archive_tiers/source.py @@ -21,7 +21,7 @@ from polylogue.storage.sqlite.archive_tiers.common import check, literal_check, nullable_check from polylogue.storage.sqlite.archive_tiers.types import ProvenRevisionAuthority -SOURCE_SCHEMA_VERSION = 29 +SOURCE_SCHEMA_VERSION = 30 SOURCE_DDL = f""" CREATE TABLE IF NOT EXISTS raw_sessions ( @@ -576,7 +576,32 @@ ) STRICT; CREATE UNIQUE INDEX IF NOT EXISTS idx_raw_artifacts_source_identity -ON raw_artifacts(origin, source_path, source_index); +ON raw_artifacts(origin, source_path, source_index) +WHERE artifact_kind NOT IN ( + 'deferred_hot_jsonl_capture', + 'deferred_claude_code_partial_jsonl', + 'deferred_cas_frontier', + 'deferred_codex_cas_frontier', + 'terminal_corrupt_input', + 'terminal_superseded_deferred_cas_frontier', + 'terminal_unknown_json_decode', + 'terminal_unknown_export_no_session', + 'terminal_unsupported_shape' +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_raw_artifacts_failure_identity +ON raw_artifacts(raw_id, origin, source_path, source_index) +WHERE artifact_kind IN ( + 'deferred_hot_jsonl_capture', + 'deferred_claude_code_partial_jsonl', + 'deferred_cas_frontier', + 'deferred_codex_cas_frontier', + 'terminal_corrupt_input', + 'terminal_superseded_deferred_cas_frontier', + 'terminal_unknown_json_decode', + 'terminal_unknown_export_no_session', + 'terminal_unsupported_shape' +); CREATE INDEX IF NOT EXISTS idx_raw_artifacts_raw_id ON raw_artifacts(raw_id); diff --git a/polylogue/storage/sqlite/archive_tiers/source_write.py b/polylogue/storage/sqlite/archive_tiers/source_write.py index fccd7a310c..d5886488ae 100644 --- a/polylogue/storage/sqlite/archive_tiers/source_write.py +++ b/polylogue/storage/sqlite/archive_tiers/source_write.py @@ -15,6 +15,7 @@ from polylogue.archive.revision_authority import RawRevisionAuthority, RawRevisionEnvelope from polylogue.core.enums import ArtifactSupportStatus, Origin, Provider, ValidationMode, ValidationStatus +from polylogue.core.raw_failure_evidence import RAW_FAILURE_EVIDENCE_KINDS from polylogue.storage.introspection import table_exists as _table_exists from polylogue.storage.raw.models import RawSessionStateUpdate from polylogue.storage.sqlite.raw_state_update import compile_raw_state_update @@ -52,6 +53,11 @@ def __init__(self, *, blob_hash: bytes, source_path: str) -> None: PENDING_RAW_LOGICAL_SOURCE_PREFIX = "pending-raw:" +def _is_raw_failure_artifact_kind(artifact_kind: object) -> bool: + value = getattr(artifact_kind, "value", artifact_kind) + return str(value) in RAW_FAILURE_EVIDENCE_KINDS + + def pending_raw_logical_source_key(*, origin: Origin | str, source_path: str, source_index: int, raw_id: str) -> str: """Return the typed identity used until a parser proves the session key.""" origin_value = _enum_value(origin) @@ -1251,16 +1257,46 @@ def _insert_artifact(conn: sqlite3.Connection, raw_id: str, artifact: ArchiveSou ) -def upsert_raw_artifact(conn: sqlite3.Connection, raw_id: str, artifact: ArchiveSourceArtifact) -> None: - """Attach or refresh typed artifact evidence for an existing raw row.""" - with conn: +def upsert_raw_artifact( + conn: sqlite3.Connection, + raw_id: str, + artifact: ArchiveSourceArtifact, + *, + manage_transaction: bool = True, +) -> None: + """Attach or refresh typed artifact evidence for an existing raw row. + + Ordinary artifact observations retain one carrier per source coordinate. + Failure evidence is attempt-scoped, so each retained raw gets its own + carrier while repeated evidence for that raw and coordinate remains an + idempotent replacement. + """ + failure_kind = _is_raw_failure_artifact_kind(artifact.artifact_kind) + coordinate_predicate = ( + "raw_id = ? AND origin = ? AND source_path = ? AND source_index = ?" + if failure_kind + else "origin = ? AND source_path = ? AND source_index = ? AND artifact_kind NOT IN (" + + ", ".join("?" for _ in RAW_FAILURE_EVIDENCE_KINDS) + + ")" + ) + coordinate_params: tuple[object, ...] + if failure_kind: + coordinate_params = (raw_id, _enum_value(artifact.origin), artifact.source_path, artifact.source_index) + else: + coordinate_params = ( + _enum_value(artifact.origin), + artifact.source_path, + artifact.source_index, + *sorted(RAW_FAILURE_EVIDENCE_KINDS), + ) + with conn if manage_transaction else nullcontext(): existing = conn.execute( - """ + f""" SELECT artifact_id FROM raw_artifacts - WHERE origin = ? AND source_path = ? AND source_index = ? + WHERE {coordinate_predicate} """, - (_enum_value(artifact.origin), artifact.source_path, artifact.source_index), + coordinate_params, ).fetchone() if existing is not None: artifact = replace(artifact, artifact_id=str(existing[0])) diff --git a/polylogue/storage/sqlite/async_sqlite_raw.py b/polylogue/storage/sqlite/async_sqlite_raw.py index b4548aa333..c28d1ff91f 100644 --- a/polylogue/storage/sqlite/async_sqlite_raw.py +++ b/polylogue/storage/sqlite/async_sqlite_raw.py @@ -8,7 +8,7 @@ import aiosqlite -from polylogue.core.enums import Provider, ValidationMode, ValidationStatus +from polylogue.core.enums import ArtifactSupportStatus, Provider, ValidationMode, ValidationStatus from polylogue.storage.raw.models import RawSessionState, RawSessionStateUpdate from polylogue.storage.runtime import ArtifactObservationRecord, RawSessionRecord from polylogue.storage.sqlite.queries import artifacts as artifacts_q @@ -35,6 +35,7 @@ def _raw_id_query( require_unparsed: bool = False, require_unvalidated: bool = False, validation_statuses: list[str] | None = None, + exclude_terminal_failure_evidence: bool = False, ) -> tuple[str, tuple[str, ...]]: """Build the canonical scoped raw-ID query.""" return self.queries.raw_id_query( @@ -43,6 +44,7 @@ def _raw_id_query( require_unparsed=require_unparsed, require_unvalidated=require_unvalidated, validation_statuses=validation_statuses, + exclude_terminal_failure_evidence=exclude_terminal_failure_evidence, ) async def iter_raw_ids( @@ -53,6 +55,7 @@ async def iter_raw_ids( require_unparsed: bool = False, require_unvalidated: bool = False, validation_statuses: list[str] | None = None, + exclude_terminal_failure_evidence: bool = False, page_size: int = 1000, ) -> AsyncIterator[str]: """Iterate raw session IDs for a pipeline state slice.""" @@ -62,6 +65,7 @@ async def iter_raw_ids( require_unparsed=require_unparsed, require_unvalidated=require_unvalidated, validation_statuses=validation_statuses, + exclude_terminal_failure_evidence=exclude_terminal_failure_evidence, page_size=page_size, ): yield rid @@ -74,6 +78,7 @@ async def iter_raw_headers( require_unparsed: bool = False, require_unvalidated: bool = False, validation_statuses: list[str] | None = None, + exclude_terminal_failure_evidence: bool = False, page_size: int = 1000, ) -> AsyncIterator[tuple[str, int]]: """Iterate raw session IDs with blob sizes for lightweight batching.""" @@ -83,6 +88,7 @@ async def iter_raw_headers( require_unparsed=require_unparsed, require_unvalidated=require_unvalidated, validation_statuses=validation_statuses, + exclude_terminal_failure_evidence=exclude_terminal_failure_evidence, page_size=page_size, ): yield raw_header @@ -104,6 +110,51 @@ async def save_artifact_observation(self, record: ArtifactObservationRecord) -> async with self._get_connection() as conn: return await artifacts_q.save_artifact_observation(conn, record, self._transaction_depth) + async def save_raw_failure_evidence( + self, + raw_id: str, + *, + artifact_kind: str, + support_status: ArtifactSupportStatus | str, + outcome_code: str, + retryable: bool | None, + evidence_ref: str | None, + remediation: str | None, + diagnostic: str | None, + ) -> None: + """Persist typed worker failure evidence in the active source transaction.""" + async with self._get_connection() as conn: + await artifacts_q.save_raw_failure_evidence( + conn, + raw_id, + artifact_kind=artifact_kind, + support_status=support_status, + outcome_code=outcome_code, + retryable=retryable, + evidence_ref=evidence_ref, + remediation=remediation, + diagnostic=diagnostic, + transaction_depth=self._transaction_depth, + ) + + async def supersede_deferred_cas_evidence(self, raw_id: str) -> None: + """Expire exact-coordinate CAS retry authority in the active transaction.""" + async with self._get_connection() as conn: + await artifacts_q.supersede_deferred_cas_evidence( + conn, + raw_id, + transaction_depth=self._transaction_depth, + ) + + async def retire_raw_failure_evidence(self, raw_id: str) -> None: + """Retire stale failure evidence for an untyped current attempt.""" + async with self._get_connection() as conn: + await artifacts_q.retire_raw_failure_evidence( + conn, + raw_id, + transaction_depth=self._transaction_depth, + ) + async def get_raw_session(self, raw_id: str) -> RawSessionRecord | None: """Retrieve a raw session by ID.""" async with self._get_connection() as conn: diff --git a/polylogue/storage/sqlite/migrations/source/030.train.json b/polylogue/storage/sqlite/migrations/source/030.train.json new file mode 100644 index 0000000000..82bc72d0ca --- /dev/null +++ b/polylogue/storage/sqlite/migrations/source/030.train.json @@ -0,0 +1,82 @@ +{ + "manifest_format": "polylogue.durable-change-train.v1", + "train_id": "train:source:v30", + "tier": "source", + "current_version": 29, + "target_version": 30, + "slot": 30, + "owner_ref": "polylogue-dyica", + "migration": { + "tier": "source", + "target_version": 30, + "slot": 30, + "path": "030_raw_failure_coordinate_carriers.sql", + "owner_ref": "polylogue/storage/sqlite/migrations/source/030_raw_failure_coordinate_carriers.sql", + "sql_sha256": "f8951c5fbdbc3095ff007d86240dcfcca307867bd21dcb0a97e6aac7f5ccec0c", + "requires_backup": true + }, + "riders": [ + { + "rider_id": "rider:raw-failure-coordinate-carriers", + "owner_ref": "polylogue-dyica", + "schema_objects": [ + "index:idx_raw_artifacts_source_identity", + "index:idx_raw_artifacts_failure_identity" + ], + "runtime_consumers": [ + { + "consumer_id": "raw-artifact-upsert", + "production_ref": "polylogue.storage.sqlite.archive_tiers.source_write:upsert_raw_artifact", + "behavior_proof_ref": "proof:source-v30:raw-artifact-upsert", + "roles": ["read", "write"] + }, + { + "consumer_id": "raw-failure-lifecycle", + "production_ref": "polylogue.storage.raw_failure_lifecycle:read_raw_failure_lifecycle", + "behavior_proof_ref": "proof:source-v30:raw-failure-lifecycle", + "roles": ["read"] + }, + { + "consumer_id": "raw-materialization-replay", + "production_ref": "polylogue.storage.repair:_raw_materialization_candidate_ids", + "behavior_proof_ref": "proof:source-v30:raw-materialization-replay", + "roles": ["read"] + } + ], + "behavior_proof_refs": [ + "proof:source-v30:raw-artifact-upsert", + "proof:source-v30:raw-failure-lifecycle", + "proof:source-v30:raw-materialization-replay" + ], + "after_rider_ids": [], + "trust_floor_exception_ref": null + } + ], + "ordering_constraints": [], + "drop_constraints": [ + { + "object_ref": "index:idx_raw_artifacts_source_identity", + "after_rider_ids": [], + "copy_forward_proof_ref": "proof:source-v30:raw-failure-coordinate-carriers-index-replacement", + "consent_ref": "polylogue-dyica" + } + ], + "row_change_allowances": [], + "backup_plan_ref": "backup-profile:source-tier", + "state": "declared", + "revision": 0, + "declared_at_ms": 0, + "admitted_at_ms": null, + "admission_evidence_ref": null, + "fresh_ddl_parity": null, + "reservation": null, + "backup_authorization": null, + "pre_apply_evidence": null, + "apply_evidence": null, + "proof": null, + "failure": null, + "released_at_ms": null, + "release_evidence_ref": null, + "proof_refs": [], + "manifest_sha256": "6353701b80a8152ab8eca23bd6f073fc006d6b8cacf4ca8ba872b75329f91be8" +} diff --git a/polylogue/storage/sqlite/migrations/source/030_raw_failure_coordinate_carriers.sql b/polylogue/storage/sqlite/migrations/source/030_raw_failure_coordinate_carriers.sql new file mode 100644 index 0000000000..cfc647aa01 --- /dev/null +++ b/polylogue/storage/sqlite/migrations/source/030_raw_failure_coordinate_carriers.sql @@ -0,0 +1,34 @@ +-- Replace the single raw-artifact coordinate carrier with two explicit +-- uniqueness domains. Ordinary artifact observations remain unique by source +-- coordinate. Typed raw-failure evidence is unique per retained raw and +-- exact coordinate, so a later acquisition cannot move authority away from +-- an older failed raw. +DROP INDEX IF EXISTS idx_raw_artifacts_source_identity; + +CREATE UNIQUE INDEX idx_raw_artifacts_source_identity +ON raw_artifacts(origin, source_path, source_index) +WHERE artifact_kind NOT IN ( + 'deferred_hot_jsonl_capture', + 'deferred_claude_code_partial_jsonl', + 'deferred_cas_frontier', + 'deferred_codex_cas_frontier', + 'terminal_corrupt_input', + 'terminal_superseded_deferred_cas_frontier', + 'terminal_unknown_json_decode', + 'terminal_unknown_export_no_session', + 'terminal_unsupported_shape' +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_raw_artifacts_failure_identity +ON raw_artifacts(raw_id, origin, source_path, source_index) +WHERE artifact_kind IN ( + 'deferred_hot_jsonl_capture', + 'deferred_claude_code_partial_jsonl', + 'deferred_cas_frontier', + 'deferred_codex_cas_frontier', + 'terminal_corrupt_input', + 'terminal_superseded_deferred_cas_frontier', + 'terminal_unknown_json_decode', + 'terminal_unknown_export_no_session', + 'terminal_unsupported_shape' +); diff --git a/polylogue/storage/sqlite/queries/artifacts.py b/polylogue/storage/sqlite/queries/artifacts.py index 9bffae820e..9e9ef7e221 100644 --- a/polylogue/storage/sqlite/queries/artifacts.py +++ b/polylogue/storage/sqlite/queries/artifacts.py @@ -15,7 +15,15 @@ import aiosqlite -from polylogue.core.enums import Provider +from polylogue.core.enums import ArtifactSupportStatus, Provider +from polylogue.core.raw_failure_evidence import ( + RAW_FAILURE_DEFERRED_SUPPORT_STATUS, + RAW_FAILURE_EVIDENCE_KINDS, + RAW_FAILURE_REPLAY_AUTHORITY_EVIDENCE_KINDS, + RawFailureEvidenceKind, + raw_failure_classification_reason, + validated_raw_failure_evidence_kind, +) from polylogue.core.sources import origin_from_provider from polylogue.storage.blob_store import get_blob_store from polylogue.storage.runtime import ArtifactObservationRecord @@ -24,6 +32,9 @@ "RAW_ARTIFACT_UPSERT_SQL", "artifact_observation_params", "save_artifact_observation", + "save_raw_failure_evidence", + "retire_raw_failure_evidence", + "supersede_deferred_cas_evidence", ] @@ -118,6 +129,236 @@ async def save_artifact_observation( return not existed +async def save_raw_failure_evidence( + conn: aiosqlite.Connection, + raw_id: str, + *, + artifact_kind: str, + support_status: ArtifactSupportStatus | str, + outcome_code: str, + retryable: bool | None, + evidence_ref: str | None, + remediation: str | None, + diagnostic: str | None, + artifact_id: str | None = None, + transaction_depth: int, +) -> None: + """Persist one typed worker disposition beside its retained raw bytes. + + The raw row supplies the origin and exact source coordinate inside the + caller's source-tier transaction. Failure disposition details live in + the existing structured ``classification_reason`` carrier, while the + diagnostic remains available through ``decode_error``. + """ + status = ArtifactSupportStatus.from_string(str(support_status)) + raw_cursor = await conn.execute( + """ + SELECT origin, source_path, source_index, acquired_at_ms, validation_status + FROM raw_sessions + WHERE raw_id = ? + """, + (raw_id,), + ) + raw_row = await raw_cursor.fetchone() + if raw_row is None: + raise KeyError(raw_id) + origin, source_path, source_index, acquired_at_ms, validation_status = raw_row + validation_failed = str(validation_status or "") == "failed" + evidence_kind = validated_raw_failure_evidence_kind( + artifact_kind, + status, + validation_failed=False, + ) + if evidence_kind is None: + raise ValueError(f"invalid closed raw-failure evidence pair: {artifact_kind!r}/{status.value!r}") + + if artifact_id is None: + failure_kind_placeholders = ", ".join("?" for _ in RAW_FAILURE_EVIDENCE_KINDS) + existing_cursor = await conn.execute( + f""" + SELECT artifact_id + FROM raw_artifacts + WHERE raw_id = ? + AND origin = ? + AND source_path = ? + AND source_index = ? + AND artifact_kind IN ({failure_kind_placeholders}) + LIMIT 1 + """, + (raw_id, origin, source_path, source_index, *sorted(RAW_FAILURE_EVIDENCE_KINDS)), + ) + existing_row = await existing_cursor.fetchone() + artifact_id = ( + str(existing_row[0]) + if existing_row is not None + else "raw-failure:" + hashlib.sha256(f"{raw_id}:{origin}:{source_path}:{source_index}".encode()).hexdigest() + ) + classification_reason = raw_failure_classification_reason( + diagnostic=diagnostic, + evidence_ref=evidence_ref, + outcome_code=outcome_code, + remediation=remediation, + retryable=retryable, + trusted_validation_failure=( + validation_failed + and evidence_kind.value in {"terminal_corrupt_input", "terminal_unknown_json_decode"} + and outcome_code == "corrupt_input" + ), + ) + await conn.execute( + RAW_ARTIFACT_UPSERT_SQL, + ( + artifact_id, + raw_id, + origin, + source_path, + source_index, + evidence_kind.value, + evidence_kind.support_status.value, + classification_reason, + int(evidence_kind.lifecycle == "deferred"), + int(evidence_kind.lifecycle == "deferred"), + 0, + diagnostic, + None, + None, + None, + acquired_at_ms, + acquired_at_ms, + ), + ) + if transaction_depth == 0: + await conn.commit() + + +async def supersede_deferred_cas_evidence( + conn: aiosqlite.Connection, + raw_id: str, + *, + transaction_depth: int, +) -> None: + """Terminalize the exact deferred CAS carrier for one retained raw. + + The artifact ID is selected from the raw's own coordinate before the + replacement, so an ordinary carrier or neighboring deferred observation + cannot be consumed by a successful batch update. + """ + raw_cursor = await conn.execute( + """ + SELECT origin, source_path, source_index + FROM raw_sessions + WHERE raw_id = ? + """, + (raw_id,), + ) + raw_row = await raw_cursor.fetchone() + if raw_row is None: + return + origin, source_path, source_index = raw_row + placeholders = ", ".join("?" for _ in RAW_FAILURE_REPLAY_AUTHORITY_EVIDENCE_KINDS) + deferred_cursor = await conn.execute( + f""" + SELECT artifact_id + FROM raw_artifacts + WHERE raw_id = ? + AND origin IS ? + AND source_path IS ? + AND source_index IS ? + AND artifact_kind IN ({placeholders}) + AND support_status = ? + ORDER BY last_observed_at_ms DESC, artifact_id DESC + LIMIT 1 + """, + ( + raw_id, + origin, + source_path, + source_index, + *sorted(RAW_FAILURE_REPLAY_AUTHORITY_EVIDENCE_KINDS), + RAW_FAILURE_DEFERRED_SUPPORT_STATUS, + ), + ) + deferred_row = await deferred_cursor.fetchone() + if deferred_row is None: + return + await save_raw_failure_evidence( + conn, + raw_id, + artifact_kind="terminal_superseded_deferred_cas_frontier", + support_status="unknown", + outcome_code="cas_frontier_resolved", + retryable=False, + evidence_ref=None, + remediation=None, + diagnostic=None, + artifact_id=str(deferred_row[0]), + transaction_depth=transaction_depth, + ) + + +async def retire_raw_failure_evidence( + conn: aiosqlite.Connection, + raw_id: str, + *, + transaction_depth: int, +) -> None: + """Retire an older failure carrier before recording an untyped attempt. + + A new parser failure without a closed worker disposition must remain + unexplained. Reusing an earlier terminal or deferred carrier would make + that unrelated failure appear resolved or retry-authorized. Keep the + artifact row for durable receipt references, but make it non-lifecycle + resolution evidence; the retained raw and current parse diagnostic remain + intact. + """ + raw_cursor = await conn.execute( + "SELECT origin, source_path, source_index FROM raw_sessions WHERE raw_id = ?", + (raw_id,), + ) + raw_row = await raw_cursor.fetchone() + if raw_row is None: + return + origin, source_path, source_index = raw_row + retired_kinds = RAW_FAILURE_EVIDENCE_KINDS - { + RawFailureEvidenceKind.TERMINAL_SUPERSEDED_DEFERRED_CAS_FRONTIER.value + } + placeholders = ", ".join("?" for _ in retired_kinds) + await conn.execute( + f""" + UPDATE raw_artifacts + SET artifact_kind = ?, + support_status = ?, + classification_reason = ?, + parse_as_session = 0, + schema_eligible = 0 + WHERE raw_id = ? + AND origin IS ? + AND source_path IS ? + AND source_index IS ? + AND artifact_kind IN ({placeholders}) + """, + ( + RawFailureEvidenceKind.TERMINAL_SUPERSEDED_DEFERRED_CAS_FRONTIER.value, + RawFailureEvidenceKind.TERMINAL_SUPERSEDED_DEFERRED_CAS_FRONTIER.support_status.value, + raw_failure_classification_reason( + diagnostic=None, + evidence_ref=None, + outcome_code="failure_attempt_replaced", + remediation="inspect the current parser failure before retrying", + retryable=False, + trusted_validation_failure=False, + ), + raw_id, + origin, + source_path, + source_index, + *sorted(retired_kinds), + ), + ) + if transaction_depth == 0: + await conn.commit() + + def _hook_observed_at_ms(value: object, fallback: str) -> int: candidate = value if isinstance(value, str) else fallback try: diff --git a/polylogue/storage/sqlite/queries/raw_reads.py b/polylogue/storage/sqlite/queries/raw_reads.py index ffa683d0de..02242b643d 100644 --- a/polylogue/storage/sqlite/queries/raw_reads.py +++ b/polylogue/storage/sqlite/queries/raw_reads.py @@ -7,6 +7,7 @@ import aiosqlite from polylogue.core.enums import Origin, Provider +from polylogue.core.raw_failure_evidence import RAW_FAILURE_TERMINAL_EVIDENCE_SUPPORT_STATUS_PAIRS from polylogue.core.sources import provider_from_origin from polylogue.storage.raw.models import RawSessionState from polylogue.storage.runtime import RawSessionRecord @@ -36,6 +37,7 @@ def _raw_select_query( require_unparsed: bool = False, require_unvalidated: bool = False, validation_statuses: list[str] | None = None, + exclude_terminal_failure_evidence: bool = False, ) -> tuple[str, tuple[str, ...]]: where_clauses: list[str] = [] params: list[str] = [] @@ -51,6 +53,20 @@ def _raw_select_query( placeholders = ",".join("?" for _ in validation_statuses) where_clauses.append(f"validation_status IN ({placeholders})") params.extend(validation_statuses) + if exclude_terminal_failure_evidence: + placeholders = ",".join("(?, ?)" for _ in RAW_FAILURE_TERMINAL_EVIDENCE_SUPPORT_STATUS_PAIRS) + where_clauses.append( + "NOT EXISTS (" + "SELECT 1 FROM raw_artifacts AS terminal_failure " + "WHERE terminal_failure.raw_id = raw_sessions.raw_id " + "AND terminal_failure.origin IS raw_sessions.origin " + "AND terminal_failure.source_path IS raw_sessions.source_path " + "AND terminal_failure.source_index IS raw_sessions.source_index " + f"AND (terminal_failure.artifact_kind, terminal_failure.support_status) IN ({placeholders})" + ")" + ) + for pair in RAW_FAILURE_TERMINAL_EVIDENCE_SUPPORT_STATUS_PAIRS: + params.extend(pair) predicate, scope_params = _build_source_path_scope_filter(source_paths) if predicate: @@ -71,6 +87,7 @@ def raw_id_query( require_unparsed: bool = False, require_unvalidated: bool = False, validation_statuses: list[str] | None = None, + exclude_terminal_failure_evidence: bool = False, ) -> tuple[str, tuple[str, ...]]: return _raw_select_query( "raw_id", @@ -79,6 +96,7 @@ def raw_id_query( require_unparsed=require_unparsed, require_unvalidated=require_unvalidated, validation_statuses=validation_statuses, + exclude_terminal_failure_evidence=exclude_terminal_failure_evidence, ) @@ -89,6 +107,7 @@ def raw_header_query( require_unparsed: bool = False, require_unvalidated: bool = False, validation_statuses: list[str] | None = None, + exclude_terminal_failure_evidence: bool = False, ) -> tuple[str, tuple[str, ...]]: return _raw_select_query( "raw_id, blob_size", @@ -97,6 +116,7 @@ def raw_header_query( require_unparsed=require_unparsed, require_unvalidated=require_unvalidated, validation_statuses=validation_statuses, + exclude_terminal_failure_evidence=exclude_terminal_failure_evidence, ) @@ -108,6 +128,7 @@ async def iter_raw_ids( require_unparsed: bool = False, require_unvalidated: bool = False, validation_statuses: list[str] | None = None, + exclude_terminal_failure_evidence: bool = False, page_size: int = 1000, ) -> AsyncIterator[str]: sql, params = raw_id_query( @@ -116,6 +137,7 @@ async def iter_raw_ids( require_unparsed=require_unparsed, require_unvalidated=require_unvalidated, validation_statuses=validation_statuses, + exclude_terminal_failure_evidence=exclude_terminal_failure_evidence, ) cursor = await conn.execute(sql, params) while True: @@ -134,6 +156,7 @@ async def iter_raw_headers( require_unparsed: bool = False, require_unvalidated: bool = False, validation_statuses: list[str] | None = None, + exclude_terminal_failure_evidence: bool = False, page_size: int = 1000, ) -> AsyncIterator[tuple[str, int]]: sql, params = raw_header_query( @@ -142,6 +165,7 @@ async def iter_raw_headers( require_unparsed=require_unparsed, require_unvalidated=require_unvalidated, validation_statuses=validation_statuses, + exclude_terminal_failure_evidence=exclude_terminal_failure_evidence, ) cursor = await conn.execute(sql, params) while True: diff --git a/polylogue/storage/sqlite/query_store_maintenance.py b/polylogue/storage/sqlite/query_store_maintenance.py index b1bccbef41..b449e48818 100644 --- a/polylogue/storage/sqlite/query_store_maintenance.py +++ b/polylogue/storage/sqlite/query_store_maintenance.py @@ -24,6 +24,7 @@ def raw_id_query( require_unparsed: bool = False, require_unvalidated: bool = False, validation_statuses: list[str] | None = None, + exclude_terminal_failure_evidence: bool = False, ) -> tuple[str, tuple[str, ...]]: return raw_queries.raw_id_query( source_paths=source_paths, @@ -31,6 +32,7 @@ def raw_id_query( require_unparsed=require_unparsed, require_unvalidated=require_unvalidated, validation_statuses=validation_statuses, + exclude_terminal_failure_evidence=exclude_terminal_failure_evidence, ) async def iter_raw_ids( @@ -41,6 +43,7 @@ async def iter_raw_ids( require_unparsed: bool = False, require_unvalidated: bool = False, validation_statuses: list[str] | None = None, + exclude_terminal_failure_evidence: bool = False, page_size: int = 1000, ) -> AsyncIterator[str]: async with self._connection_factory() as conn: @@ -51,6 +54,7 @@ async def iter_raw_ids( require_unparsed=require_unparsed, require_unvalidated=require_unvalidated, validation_statuses=validation_statuses, + exclude_terminal_failure_evidence=exclude_terminal_failure_evidence, page_size=page_size, ): yield raw_id @@ -63,6 +67,7 @@ async def iter_raw_headers( require_unparsed: bool = False, require_unvalidated: bool = False, validation_statuses: list[str] | None = None, + exclude_terminal_failure_evidence: bool = False, page_size: int = 1000, ) -> AsyncIterator[tuple[str, int]]: async with self._connection_factory() as conn: @@ -73,6 +78,7 @@ async def iter_raw_headers( require_unparsed=require_unparsed, require_unvalidated=require_unvalidated, validation_statuses=validation_statuses, + exclude_terminal_failure_evidence=exclude_terminal_failure_evidence, page_size=page_size, ): yield raw_header diff --git a/tests/unit/daemon/test_raw_failure_sample.py b/tests/unit/daemon/test_raw_failure_sample.py index 152058f84b..917e43d4db 100644 --- a/tests/unit/daemon/test_raw_failure_sample.py +++ b/tests/unit/daemon/test_raw_failure_sample.py @@ -4,7 +4,7 @@ import sqlite3 from pathlib import Path -from typing import cast +from typing import Any, cast from unittest.mock import patch import pytest @@ -12,6 +12,12 @@ from polylogue.core.enums import ArtifactSupportStatus from polylogue.core.json import JSONDocument +from polylogue.core.raw_failure_evidence import ( + RAW_FAILURE_DEFERRED_EVIDENCE_KINDS, + RAW_FAILURE_LIFECYCLE_EVIDENCE_SUPPORT_STATUS_PAIRS, + RAW_FAILURE_TERMINAL_EVIDENCE_KINDS, + RawFailureEvidenceKind, +) from polylogue.daemon.status import ( DaemonStatus, RawFailureSample, @@ -49,10 +55,44 @@ def test_rejects_invalid_failure_kind(self) -> None: RawFailureSample(failure_kind="invalid_kind") # type: ignore[arg-type] def test_all_valid_failure_kinds(self) -> None: - for kind in ("decode_error", "parse_error", "schema_violation", "maintenance", "unknown"): - sample = RawFailureSample(failure_kind=kind) + for kind in ( + "decode_error", + "parse_error", + "schema_violation", + "maintenance", + "unknown", + *RAW_FAILURE_DEFERRED_EVIDENCE_KINDS, + *RAW_FAILURE_TERMINAL_EVIDENCE_KINDS, + ): + sample = RawFailureSample(failure_kind=cast(Any, kind)) assert sample.failure_kind == kind + def test_raw_evidence_kinds_have_closed_lifecycle_partition(self) -> None: + assert ( + frozenset( + { + "deferred_hot_jsonl_capture", + "deferred_claude_code_partial_jsonl", + "deferred_cas_frontier", + "deferred_codex_cas_frontier", + } + ) + == RAW_FAILURE_DEFERRED_EVIDENCE_KINDS + ) + assert ( + frozenset( + { + "terminal_corrupt_input", + "terminal_unknown_json_decode", + "terminal_unknown_export_no_session", + "terminal_unsupported_shape", + } + ) + == RAW_FAILURE_TERMINAL_EVIDENCE_KINDS + ) + for value in (*RAW_FAILURE_DEFERRED_EVIDENCE_KINDS, *RAW_FAILURE_TERMINAL_EVIDENCE_KINDS): + assert RawFailureEvidenceKind(value).lifecycle in {"deferred", "terminal"} + def test_redacts_absolute_file_paths(self) -> None: sample = RawFailureSample( failure_kind="decode_error", @@ -336,6 +376,60 @@ def test_raw_failure_info_generic_parse_error_kind(self, tmp_path: Path) -> None # Non-JSON parse error → "parse_error" (the error IS a parse error, just not JSON-specific) assert sample.failure_kind == "parse_error" + def test_raw_failure_info_prefers_failure_evidence_over_newer_artifact(self, tmp_path: Path) -> None: + index_db = _seed_archive_raw_session( + tmp_path, + raw_id="raw-multi-artifact", + origin="claude-code-session", + native_id="native-multi-artifact", + source_path="/data/failure.jsonl", + parse_error="captured JSONL payload ends before a complete record boundary", + ) + with sqlite3.connect(tmp_path / "source.db") as conn: + upsert_raw_artifact( + conn, + "raw-multi-artifact", + ArchiveSourceArtifact( + artifact_id="failure-evidence", + origin="claude-code-session", + source_path="/data/failure.jsonl", + source_index=0, + artifact_kind="deferred_claude_code_partial_jsonl", + classification_reason="deferred_claude_code_partial_jsonl", + support_status=ArtifactSupportStatus.PARTIAL_DECODE, + parse_as_session=True, + schema_eligible=True, + first_observed_at_ms=100, + last_observed_at_ms=100, + ), + ) + upsert_raw_artifact( + conn, + "raw-multi-artifact", + ArchiveSourceArtifact( + artifact_id="newer-unrelated-artifact", + origin="claude-code-session", + source_path="/data/newer.sqlite", + source_index=0, + artifact_kind="sqlite_state_database", + classification_reason="sqlite_state_database", + support_status=ArtifactSupportStatus.UNKNOWN, + first_observed_at_ms=200, + last_observed_at_ms=200, + ), + ) + + with ( + patch("polylogue.daemon.status.archive_root", return_value=tmp_path), + patch("polylogue.daemon.status._active_status_db_path", return_value=index_db), + ): + info = _raw_failure_info() + + samples = cast(list[RawFailureSample], info["samples"]) + assert len(samples) == 1 + assert samples[0].failure_kind == "deferred_claude_code_partial_jsonl" + assert samples[0].lifecycle == "deferred" + def test_raw_failure_info_separates_closed_lifecycle_evidence(self, tmp_path: Path) -> None: index_db = _seed_archive_raw_session( tmp_path, @@ -405,6 +499,56 @@ def test_raw_failure_info_separates_closed_lifecycle_evidence(self, tmp_path: Pa assert info["unexplained_failures"] == 1 samples = cast(list[RawFailureSample], info["samples"]) assert {sample.lifecycle for sample in samples} == {"deferred", "terminal", "unexplained"} + by_kind = {sample.provider_hint: sample.failure_kind for sample in samples} + assert by_kind["claude-code-session"] == "deferred_hot_jsonl_capture" + assert by_kind["unknown-export"] == "terminal_unsupported_shape" + assert by_kind["codex-session"] == "parse_error" + + def test_lifecycle_sampling_prioritizes_every_valid_typed_evidence_pair(self, tmp_path: Path) -> None: + """Every closed typed kind remains inspectable before unexplained rows.""" + for index, (kind, support_status) in enumerate(RAW_FAILURE_LIFECYCLE_EVIDENCE_SUPPORT_STATUS_PAIRS): + _seed_archive_raw_session( + tmp_path, + raw_id=f"raw-typed-{index}", + origin="codex-session", + native_id=f"typed-{index}", + source_path=f"/data/typed-{index}.jsonl", + parse_error=f"typed failure {index}", + acquired_at_ms=1_770_000_000_000 + index, + ) + with sqlite3.connect(tmp_path / "source.db") as conn: + upsert_raw_artifact( + conn, + f"raw-typed-{index}", + ArchiveSourceArtifact( + artifact_id=f"typed-evidence-{index}", + origin="codex-session", + source_path=f"/data/typed-{index}.jsonl", + source_index=0, + artifact_kind=kind, + classification_reason=kind, + support_status=ArtifactSupportStatus(support_status), + ), + ) + _seed_archive_raw_session( + tmp_path, + raw_id="raw-unexplained-newest", + origin="codex-session", + native_id="unexplained-newest", + source_path="/data/unexplained-newest.jsonl", + parse_error="unexplained failure", + acquired_at_ms=1_770_000_000_999, + ) + + snapshot = read_raw_failure_lifecycle( + tmp_path / "source.db", + sample_limit=len(RAW_FAILURE_LIFECYCLE_EVIDENCE_SUPPORT_STATUS_PAIRS), + ) + + assert {sample["artifact_kind"] for sample in snapshot.samples} == { + kind for kind, _support_status in RAW_FAILURE_LIFECYCLE_EVIDENCE_SUPPORT_STATUS_PAIRS + } + assert all(sample["lifecycle"] in {"deferred", "terminal"} for sample in snapshot.samples) def test_raw_failure_info_uses_root_source_tier_for_pointer_index(self, tmp_path: Path) -> None: generation = tmp_path / "generation" @@ -511,6 +655,97 @@ def test_raw_failure_lifecycle_rejects_mismatched_or_malformed_artifacts(self, t assert info["terminal_rejections"] == snapshot.terminal assert info["unexplained_failures"] == snapshot.unexplained + def test_raw_failure_status_does_not_project_unvalidated_typed_kinds(self, tmp_path: Path) -> None: + """Status uses the lifecycle reader's validation, support, and kind checks.""" + index_db = _seed_archive_raw_session( + tmp_path, + raw_id="raw-validation-failed", + origin="codex-session", + native_id="validation-failed", + source_path="/data/validation-failed.jsonl", + validation_status="failed", + validation_error="schema drift", + ) + _seed_archive_raw_session( + tmp_path, + raw_id="raw-kind-contradiction", + origin="codex-session", + native_id="kind-contradiction", + source_path="/data/kind-contradiction.jsonl", + parse_error="parser failed after artifact observation", + ) + _seed_archive_raw_session( + tmp_path, + raw_id="raw-malformed-carrier", + origin="codex-session", + native_id="malformed-carrier", + source_path="/data/malformed-carrier.jsonl", + parse_error="parser failed with malformed evidence", + ) + with sqlite3.connect(tmp_path / "source.db") as conn: + upsert_raw_artifact( + conn, + "raw-validation-failed", + ArchiveSourceArtifact( + artifact_id="validation-failed-evidence", + origin="codex-session", + source_path="/data/validation-failed.jsonl", + source_index=0, + artifact_kind="terminal_corrupt_input", + classification_reason="terminal_corrupt_input", + support_status=ArtifactSupportStatus.DECODE_FAILED, + ), + ) + upsert_raw_artifact( + conn, + "raw-kind-contradiction", + ArchiveSourceArtifact( + artifact_id="kind-contradiction-evidence", + origin="codex-session", + source_path="/data/kind-contradiction.jsonl", + source_index=0, + artifact_kind="terminal_corrupt_input", + classification_reason="terminal_corrupt_input", + support_status=ArtifactSupportStatus.UNSUPPORTED_PARSEABLE, + ), + ) + upsert_raw_artifact( + conn, + "raw-malformed-carrier", + ArchiveSourceArtifact( + artifact_id="malformed-carrier-evidence", + origin="codex-session", + source_path="/data/malformed-carrier.jsonl", + source_index=0, + artifact_kind="terminal_corrupt_input", + classification_reason="terminal_corrupt_input", + support_status=ArtifactSupportStatus.DECODE_FAILED, + ), + ) + # Directly mutate the persisted carrier to simulate a malformed + # producer row without weakening the typed write boundary. + conn.execute( + "UPDATE raw_artifacts SET artifact_kind = ? WHERE artifact_id = ?", + ("malformed_evidence_kind", "malformed-carrier-evidence"), + ) + conn.commit() + + snapshot = read_raw_failure_lifecycle(tmp_path / "source.db") + with ( + patch("polylogue.daemon.status.archive_root", return_value=tmp_path), + patch("polylogue.daemon.status._active_status_db_path", return_value=index_db), + ): + info = _raw_failure_info() + + samples = cast(list[RawFailureSample], info["samples"]) + by_origin_error = {sample.redacted_error: sample for sample in samples} + assert snapshot.unexplained == 3 + assert info["unexplained_failures"] == 3 + assert by_origin_error["schema drift"].failure_kind == "schema_violation" + assert by_origin_error["parser failed after artifact observation"].failure_kind == "parse_error" + assert by_origin_error["parser failed with malformed evidence"].failure_kind == "parse_error" + assert all(sample.failure_kind != "terminal_corrupt_input" for sample in samples) + def test_daemon_status_lifecycle_counts_match_the_shared_projection(self, tmp_path: Path) -> None: """The health/status source is the same lifecycle projection as preflight.""" index_db = _seed_archive_raw_session( diff --git a/tests/unit/maintenance/test_raw_failure_disposition_apply.py b/tests/unit/maintenance/test_raw_failure_disposition_apply.py index 700f6bba9f..341c56b03c 100644 --- a/tests/unit/maintenance/test_raw_failure_disposition_apply.py +++ b/tests/unit/maintenance/test_raw_failure_disposition_apply.py @@ -2,6 +2,7 @@ from __future__ import annotations +import hashlib import json import sqlite3 from pathlib import Path @@ -120,6 +121,9 @@ def _validate(path: Path, tier: object, *, connection: sqlite3.Connection) -> Pa "previous_classification_reason, disposition_kind, tool_version, detail " "FROM raw_failure_disposition_receipts" ).fetchone() + classification_reason = conn.execute( + "SELECT classification_reason FROM raw_artifacts WHERE raw_id = ?", (raw_id,) + ).fetchone()[0] assert receipt == ( raw_id, "coordinator_session_stream", @@ -129,6 +133,14 @@ def _validate(path: Path, tier: object, *, connection: sqlite3.Connection) -> Pa TOOL_VERSION, "empty retained byte stream", ) + assert json.loads(classification_reason) == { + "diagnostic": "empty retained byte stream", + "evidence_ref": f"raw-failure-disposition:{hashlib.sha256(manifest.read_bytes()).hexdigest()}", + "outcome_code": "corrupt_input", + "provenance": "worker-disposition-v1", + "remediation": "retain the reviewed terminal disposition until a forced reparse is authorized", + "retryable": False, + } def test_apply_refuses_duplicate_or_nonterminal_manifest_entries(tmp_path: Path) -> None: diff --git a/tests/unit/pipeline/test_ingest_batch.py b/tests/unit/pipeline/test_ingest_batch.py index c82c58d811..c5d8184b4b 100644 --- a/tests/unit/pipeline/test_ingest_batch.py +++ b/tests/unit/pipeline/test_ingest_batch.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import os import sqlite3 from collections.abc import AsyncIterator, Callable @@ -19,8 +20,11 @@ import polylogue.pipeline.services.ingest_batch._core as ingest_batch_core from polylogue.archive.ingest_flags import DOM_FALLBACK_INGEST_FLAG, NATIVE_BROWSER_CAPTURE_INGEST_FLAG from polylogue.archive.message.roles import Role -from polylogue.core.enums import BlockType, Provider +from polylogue.config import Config +from polylogue.core.enums import ArtifactSupportStatus, BlockType, Origin, Provider +from polylogue.core.raw_failure_evidence import RawFailureEvidenceKind from polylogue.core.types import SessionId +from polylogue.daemon.status import RawFailureSample, raw_failure_info_for_root from polylogue.pipeline.ids import session_id as make_session_id from polylogue.pipeline.services import ingest_worker as ingest_worker_mod from polylogue.pipeline.services.ingest_batch import ( @@ -64,11 +68,18 @@ from polylogue.storage.blob_store import BlobStore from polylogue.storage.insights.session.refresh import SessionInsightRefreshChunkObservation from polylogue.storage.raw.models import RawSessionStateUpdate +from polylogue.storage.raw_failure_lifecycle import read_raw_failure_lifecycle +from polylogue.storage.repository import SessionRepository from polylogue.storage.runtime import RawSessionRecord from polylogue.storage.search.cache import get_cache_stats from polylogue.storage.search.runtime import search_messages from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root +from polylogue.storage.sqlite.archive_tiers.source_write import ( + ArchiveSourceArtifact, + upsert_raw_artifact, + write_source_raw_session, +) from polylogue.storage.sqlite.archive_tiers.write import _attachment_id from polylogue.storage.sqlite.async_sqlite import SQLiteBackend from polylogue.storage.sqlite.connection import open_connection @@ -198,6 +209,29 @@ def test_primary_mode_keeps_unconfirmed_revision_out_of_index_and_fts( assert summary.publication_payload_bytes == 0 +def test_batch_projection_preserves_worker_disposition_fields() -> None: + summary = _IngestBatchSummary() + ingest_batch_core._record_outcome( + summary, + IngestRecordResult( + raw_id="raw-worker-failure", + error="schema rejected", + outcome_code="validation_rejected", + retryable=False, + evidence_ref="schema_validation_strict", + remediation="repair source schema", + diagnostic="missing required field: messages", + ), + ) + + outcome = summary.outcomes["raw-worker-failure"] + assert outcome.outcome_code == "validation_rejected" + assert outcome.retryable is False + assert outcome.evidence_ref == "schema_validation_strict" + assert outcome.remediation == "repair source schema" + assert outcome.diagnostic == "missing required field: messages" + + def test_primary_transport_resolution_precedes_index_connection( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -3496,6 +3530,31 @@ def test_failed_raw_state_update_keeps_validation_only_failure_out_of_parse_erro ) +def test_failed_raw_state_update_persists_worker_diagnostic_at_boundary() -> None: + outcome = _RawIngestOutcome( + raw_id="raw-1", + payload_provider="chatgpt", + validation_status="failed", + validation_error="schema mismatch", + parse_error="parse failed", + error="parse failed", + had_sessions=False, + outcome_code="validation_rejected", + retryable=False, + evidence_ref="schema_validation_strict", + remediation="repair source schema", + diagnostic="missing required field: messages", + ) + + state = _failed_raw_state_update( + outcome=outcome, + error="parse failed", + validation_mode="strict", + ) + + assert state.detection_warnings == "missing required field: messages" + + def test_unattributed_batch_elapsed_subtracts_setup_and_teardown() -> None: summary = _IngestBatchSummary( setup_elapsed_s=0.12, @@ -3672,3 +3731,507 @@ async def _bulk_connection() -> AsyncIterator[None]: assert state.parse_error is None assert state.validation_error == "bad schema" assert state.validation_status == "failed" + + +@pytest.mark.asyncio +async def test_persist_batch_raw_state_updates_persists_terminal_worker_disposition_to_source( + tmp_path: Path, +) -> None: + """The ordinary batch boundary retains typed terminal evidence at the raw coordinate.""" + initialize_active_archive_root(tmp_path) + with sqlite3.connect(tmp_path / "source.db") as conn: + raw_id = write_source_raw_session( + conn, + origin=Origin.CODEX_SESSION, + source_path="batch-unsupported.jsonl", + source_index=7, + payload=b"unsupported-shape", + acquired_at_ms=1, + ) + + repository = SessionRepository(backend=SQLiteBackend(db_path=tmp_path / "index.db"), archive_root=tmp_path) + service = SimpleNamespace(repository=repository) + outcome = _RawIngestOutcome( + raw_id=raw_id, + payload_provider="codex", + validation_status="passed", + validation_error=None, + parse_error="parse: session artifact produced no materializable sessions", + error="parse: session artifact produced no materializable sessions", + had_sessions=False, + outcome_code="unsupported_shape", + retryable=False, + evidence_ref="empty_parsed_sessions", + remediation="open a source-support issue", + diagnostic="worker rejected unsupported shape", + ) + + try: + await _persist_batch_raw_state_updates( + service, + repository.backend, + outcomes={raw_id: outcome}, + succeeded_raw_ids=set(), + skipped_raw_ids=set(), + failed_raw_ids={raw_id: outcome.error or "worker failure"}, + validation_mode="strict", + ) + finally: + await repository.close() + + with sqlite3.connect(tmp_path / "source.db") as conn: + state = conn.execute( + "SELECT parse_error, source_path, source_index FROM raw_sessions WHERE raw_id = ?", + (raw_id,), + ).fetchone() + artifact = conn.execute( + """ + SELECT raw_id, origin, source_path, source_index, artifact_kind, support_status, + classification_reason, decode_error + FROM raw_artifacts + WHERE raw_id = ? + """, + (raw_id,), + ).fetchone() + + assert state == (outcome.parse_error, "batch-unsupported.jsonl", 7) + assert artifact is not None + assert tuple(artifact[:6]) == ( + raw_id, + Origin.CODEX_SESSION.value, + "batch-unsupported.jsonl", + 7, + RawFailureEvidenceKind.TERMINAL_UNSUPPORTED_SHAPE.value, + "unsupported_parseable", + ) + carrier = json.loads(artifact[6]) + assert carrier == { + "diagnostic": outcome.diagnostic, + "evidence_ref": outcome.evidence_ref, + "outcome_code": outcome.outcome_code, + "remediation": outcome.remediation, + "retryable": False, + } + assert artifact[7] == outcome.diagnostic + + lifecycle = read_raw_failure_lifecycle(tmp_path / "source.db") + assert lifecycle.terminal == 1 + assert lifecycle.unexplained == 0 + assert lifecycle.blocking is False + status = raw_failure_info_for_root(tmp_path) + assert status["terminal_rejections"] == 1 + assert status["unexplained_failures"] == 0 + samples = cast(list[RawFailureSample], status["samples"]) + assert samples[0].failure_kind == RawFailureEvidenceKind.TERMINAL_UNSUPPORTED_SHAPE.value + assert lifecycle.state == "degraded" + + +@pytest.mark.asyncio +async def test_persist_batch_success_supersedes_deferred_cas_evidence_in_source_transaction( + tmp_path: Path, +) -> None: + """The async batch success route revokes stale CAS replay authority.""" + initialize_active_archive_root(tmp_path) + with sqlite3.connect(tmp_path / "source.db") as conn: + raw_id = write_source_raw_session( + conn, + origin=Origin.CODEX_SESSION, + source_path="batch-success.jsonl", + source_index=2, + payload=b"batch-success", + acquired_at_ms=1, + ) + upsert_raw_artifact( + conn, + raw_id, + ArchiveSourceArtifact( + artifact_id="deferred-cas", + origin=Origin.CODEX_SESSION, + source_path="batch-success.jsonl", + source_index=2, + artifact_kind=RawFailureEvidenceKind.DEFERRED_CAS_FRONTIER.value, + classification_reason="deferred CAS", + support_status=ArtifactSupportStatus.PARTIAL_DECODE, + parse_as_session=True, + schema_eligible=True, + first_observed_at_ms=1, + last_observed_at_ms=1, + ), + ) + + repository = SessionRepository(backend=SQLiteBackend(db_path=tmp_path / "index.db"), archive_root=tmp_path) + service = SimpleNamespace(repository=repository) + outcome = _RawIngestOutcome( + raw_id=raw_id, + payload_provider="codex", + validation_status="passed", + validation_error=None, + parse_error=None, + error=None, + had_sessions=True, + ) + try: + await _persist_batch_raw_state_updates( + service, + repository.backend, + outcomes={raw_id: outcome}, + succeeded_raw_ids={raw_id}, + skipped_raw_ids=set(), + failed_raw_ids={}, + validation_mode="strict", + ) + finally: + await repository.close() + + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute( + "SELECT artifact_id, artifact_kind, support_status FROM raw_artifacts WHERE raw_id = ?", + (raw_id,), + ).fetchone() == ( + "deferred-cas", + RawFailureEvidenceKind.TERMINAL_SUPERSEDED_DEFERRED_CAS_FRONTIER.value, + "unknown", + ) + assert ( + conn.execute("SELECT parse_error, parsed_at_ms FROM raw_sessions WHERE raw_id = ?", (raw_id,)).fetchone()[0] + is None + ) + + lifecycle = read_raw_failure_lifecycle(tmp_path / "source.db") + assert lifecycle.deferred == 0 + assert lifecycle.terminal == 0 + assert lifecycle.unexplained == 0 + + +@pytest.mark.asyncio +async def test_persist_batch_untyped_failure_retires_stale_terminal_evidence(tmp_path: Path) -> None: + """A later untyped parser failure cannot inherit an older terminal cause.""" + initialize_active_archive_root(tmp_path) + with sqlite3.connect(tmp_path / "source.db") as conn: + raw_id = write_source_raw_session( + conn, + origin=Origin.CODEX_SESSION, + source_path="untyped-successor.jsonl", + source_index=0, + payload=b"untyped-successor", + acquired_at_ms=1, + ) + + repository = SessionRepository(backend=SQLiteBackend(db_path=tmp_path / "index.db"), archive_root=tmp_path) + service = SimpleNamespace(repository=repository) + terminal_outcome = _RawIngestOutcome( + raw_id=raw_id, + payload_provider="codex", + validation_status="passed", + validation_error=None, + parse_error="unsupported shape", + error="unsupported shape", + had_sessions=False, + outcome_code="unsupported_shape", + evidence_ref="shape", + remediation="support it", + diagnostic="unsupported shape", + ) + untyped_outcome = _RawIngestOutcome( + raw_id=raw_id, + payload_provider="codex", + validation_status="passed", + validation_error=None, + parse_error="parser defect", + error="parser defect", + had_sessions=False, + outcome_code="parser_defect", + diagnostic="parser defect", + ) + try: + await _persist_batch_raw_state_updates( + service, + repository.backend, + outcomes={raw_id: terminal_outcome}, + succeeded_raw_ids=set(), + skipped_raw_ids=set(), + failed_raw_ids={raw_id: terminal_outcome.error or "failure"}, + validation_mode="strict", + ) + await _persist_batch_raw_state_updates( + service, + repository.backend, + outcomes={raw_id: untyped_outcome}, + succeeded_raw_ids=set(), + skipped_raw_ids=set(), + failed_raw_ids={raw_id: untyped_outcome.error or "failure"}, + validation_mode="strict", + ) + finally: + await repository.close() + + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT artifact_kind FROM raw_artifacts WHERE raw_id = ?", (raw_id,)).fetchone() == ( + RawFailureEvidenceKind.TERMINAL_SUPERSEDED_DEFERRED_CAS_FRONTIER.value, + ) + lifecycle = read_raw_failure_lifecycle(tmp_path / "source.db") + assert lifecycle.terminal == 0 + assert lifecycle.unexplained == 1 + assert lifecycle.blocking is True + + +@pytest.mark.asyncio +async def test_process_ingest_batch_public_route_retires_deferred_cas_resolution( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The public async batch route applies CAS resolution after index commit.""" + initialize_active_archive_root(tmp_path) + payload = (Path(__file__).parents[2] / "fixtures" / "chatgpt" / "native-conversation-v1.json").read_bytes() + BlobStore(tmp_path / "blob").write_from_bytes(payload) + with sqlite3.connect(tmp_path / "source.db") as conn: + raw_id = write_source_raw_session( + conn, + origin=Origin.CHATGPT_EXPORT, + source_path="public-batch.json", + source_index=0, + payload=payload, + acquired_at_ms=1, + ) + upsert_raw_artifact( + conn, + raw_id, + ArchiveSourceArtifact( + artifact_id="public-deferred-cas", + origin=Origin.CHATGPT_EXPORT, + source_path="public-batch.json", + source_index=0, + artifact_kind=RawFailureEvidenceKind.DEFERRED_CAS_FRONTIER.value, + classification_reason="deferred CAS", + support_status=ArtifactSupportStatus.PARTIAL_DECODE, + parse_as_session=True, + schema_eligible=True, + first_observed_at_ms=1, + last_observed_at_ms=1, + ), + ) + conn.commit() + + config = Config(archive_root=tmp_path, render_root=tmp_path / "render", sources=[]) + monkeypatch.setattr( + "polylogue.config.load_polylogue_config", + lambda: SimpleNamespace(schema_validation="advisory", sinex_mode="off"), + ) + repository = SessionRepository(backend=SQLiteBackend(db_path=tmp_path / "index.db"), archive_root=tmp_path) + service = ParsingService(repository=repository, archive_root=tmp_path, config=config, ingest_workers=1) + parse_result = ParseResult() + try: + await ingest_batch_core.process_ingest_batch( + service, + repository.backend, + [raw_id], + parse_result, + None, + repair_message_fts=False, + ) + finally: + await repository.close() + + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute( + "SELECT parsed_at_ms IS NOT NULL, parse_error FROM raw_sessions WHERE raw_id = ?", (raw_id,) + ).fetchone() == (1, None) + assert conn.execute( + "SELECT artifact_kind, support_status FROM raw_artifacts WHERE raw_id = ?", (raw_id,) + ).fetchone() == ( + RawFailureEvidenceKind.TERMINAL_SUPERSEDED_DEFERRED_CAS_FRONTIER.value, + "unknown", + ) + assert parse_result.processed_ids + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("payload", "diagnostic"), + [ + (b"", "decode: Input is a zero-length, empty document"), + (b"{", "decode: Input data was truncated"), + ], + ids=["zero-length", "decode-failure"], +) +async def test_persist_batch_corrupt_input_remains_terminal_in_lifecycle( + tmp_path: Path, + payload: bytes, + diagnostic: str, +) -> None: + """Worker validation failure plus typed corrupt evidence is explainable.""" + initialize_active_archive_root(tmp_path) + with sqlite3.connect(tmp_path / "source.db") as conn: + raw_id = write_source_raw_session( + conn, + origin=Origin.CODEX_SESSION, + source_path="batch-corrupt.jsonl", + source_index=0, + payload=payload, + acquired_at_ms=1, + ) + + repository = SessionRepository(backend=SQLiteBackend(db_path=tmp_path / "index.db"), archive_root=tmp_path) + service = SimpleNamespace(repository=repository) + outcome = _RawIngestOutcome( + raw_id=raw_id, + payload_provider="codex", + validation_status="failed", + validation_error="payload failed validation", + parse_error=diagnostic, + error=diagnostic, + had_sessions=False, + outcome_code="corrupt_input", + retryable=False, + evidence_ref="decode", + remediation="retain and inspect raw bytes", + diagnostic=diagnostic, + ) + try: + await _persist_batch_raw_state_updates( + service, + repository.backend, + outcomes={raw_id: outcome}, + succeeded_raw_ids=set(), + skipped_raw_ids=set(), + failed_raw_ids={raw_id: diagnostic}, + validation_mode="strict", + ) + finally: + await repository.close() + + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute( + "SELECT validation_status, parse_error FROM raw_sessions WHERE raw_id = ?", (raw_id,) + ).fetchone() == ("failed", diagnostic) + assert conn.execute( + "SELECT artifact_kind, support_status FROM raw_artifacts WHERE raw_id = ?", (raw_id,) + ).fetchone() == (RawFailureEvidenceKind.TERMINAL_CORRUPT_INPUT.value, "decode_failed") + + lifecycle = read_raw_failure_lifecycle(tmp_path / "source.db") + assert lifecycle.validation_failures == 1 + assert lifecycle.terminal == 1 + assert lifecycle.unexplained == 0 + assert lifecycle.blocking is False + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("payload", "diagnostic"), + [ + (b"", "decode: Input is a zero-length, empty document"), + (b"{", "decode: Input data was truncated"), + ], + ids=["zero-length-public-route", "decode-failure-public-route"], +) +async def test_process_ingest_batch_public_route_persists_corrupt_input_readiness( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + payload: bytes, + diagnostic: str, +) -> None: + """The real worker route makes corrupt input terminal and status-readable.""" + initialize_active_archive_root(tmp_path) + BlobStore(tmp_path / "blob").write_from_bytes(payload) + with sqlite3.connect(tmp_path / "source.db") as conn: + raw_id = write_source_raw_session( + conn, + origin=Origin.CODEX_SESSION, + source_path="public-corrupt.jsonl", + source_index=0, + payload=payload, + acquired_at_ms=1, + ) + + config = Config(archive_root=tmp_path, render_root=tmp_path / "render", sources=[]) + monkeypatch.setattr( + "polylogue.config.load_polylogue_config", + lambda: SimpleNamespace(schema_validation="advisory", sinex_mode="off"), + ) + repository = SessionRepository(backend=SQLiteBackend(db_path=tmp_path / "index.db"), archive_root=tmp_path) + service = ParsingService(repository=repository, archive_root=tmp_path, config=config, ingest_workers=1) + parse_result = ParseResult() + try: + await ingest_batch_core.process_ingest_batch( + service, + repository.backend, + [raw_id], + parse_result, + None, + repair_message_fts=False, + ) + finally: + await repository.close() + + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute( + "SELECT validation_status, parse_error FROM raw_sessions WHERE raw_id = ?", (raw_id,) + ).fetchone() == ("failed", diagnostic) + assert conn.execute( + "SELECT artifact_kind, support_status FROM raw_artifacts WHERE raw_id = ?", (raw_id,) + ).fetchone() == (RawFailureEvidenceKind.TERMINAL_CORRUPT_INPUT.value, "decode_failed") + + lifecycle = read_raw_failure_lifecycle(tmp_path / "source.db") + assert lifecycle.terminal == 1 + assert lifecycle.unexplained == 0 + assert lifecycle.blocking is False + status = raw_failure_info_for_root(tmp_path) + assert status["terminal_rejections"] == 1 + assert status["unexplained_failures"] == 0 + + +@pytest.mark.asyncio +async def test_persist_batch_raw_state_updates_rolls_back_typed_evidence_with_raw_state( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A source-tier carrier failure rolls back the paired raw-state mutation.""" + initialize_active_archive_root(tmp_path) + with sqlite3.connect(tmp_path / "source.db") as conn: + raw_id = write_source_raw_session( + conn, + origin=Origin.CODEX_SESSION, + source_path="batch-atomic.jsonl", + source_index=3, + payload=b"unsupported-shape", + acquired_at_ms=1, + ) + + repository = SessionRepository(backend=SQLiteBackend(db_path=tmp_path / "index.db"), archive_root=tmp_path) + service = SimpleNamespace(repository=repository) + outcome = _RawIngestOutcome( + raw_id=raw_id, + payload_provider="codex", + validation_status="passed", + validation_error=None, + parse_error="unsupported shape", + error="unsupported shape", + had_sessions=False, + outcome_code="unsupported_shape", + retryable=False, + evidence_ref="shape", + remediation="support it", + diagnostic="unsupported", + ) + + async def fail_evidence(*_args: object, **_kwargs: object) -> None: + raise RuntimeError("carrier write failed") + + monkeypatch.setattr(repository.source_backend, "save_raw_failure_evidence", fail_evidence) + with pytest.raises(RuntimeError, match="carrier write failed"): + await _persist_batch_raw_state_updates( + service, + repository.backend, + outcomes={raw_id: outcome}, + succeeded_raw_ids=set(), + skipped_raw_ids=set(), + failed_raw_ids={raw_id: "unsupported shape"}, + validation_mode="strict", + ) + await repository.close() + + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute( + "SELECT parse_error, validation_status FROM raw_sessions WHERE raw_id = ?", (raw_id,) + ).fetchone() == (None, None) + assert conn.execute("SELECT COUNT(*) FROM raw_artifacts WHERE raw_id = ?", (raw_id,)).fetchone() == (0,) diff --git a/tests/unit/pipeline/test_parsing_service.py b/tests/unit/pipeline/test_parsing_service.py index 98a8d12f63..2d6c1afb78 100644 --- a/tests/unit/pipeline/test_parsing_service.py +++ b/tests/unit/pipeline/test_parsing_service.py @@ -5,6 +5,7 @@ import asyncio import hashlib import json +import sqlite3 import time from collections.abc import AsyncGenerator, Awaitable, Callable from datetime import datetime, timezone @@ -17,17 +18,22 @@ from polylogue.config import Config, Source from polylogue.core.enums import Provider from polylogue.core.errors import DatabaseError +from polylogue.core.raw_failure_evidence import RawFailureEvidenceKind from polylogue.pipeline.payload_types import ParseBatchObservation from polylogue.pipeline.services.acquisition import AcquireResult, AcquisitionService from polylogue.pipeline.services.acquisition_records import ScanResult from polylogue.pipeline.services.ingest_worker import _fallback_id from polylogue.pipeline.services.parsing import ParseResult, ParsingService from polylogue.pipeline.services.planning import PlanningService +from polylogue.pipeline.services.planning_backlog import collect_parse_backlog from polylogue.pipeline.services.validation import ValidationService # used by TestPlanningService from polylogue.sources.parsers.base import RawSessionData +from polylogue.storage.raw_failure_lifecycle import read_raw_failure_lifecycle from polylogue.storage.repository import SessionRepository from polylogue.storage.runtime import RawSessionRecord +from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root from polylogue.storage.sqlite.async_sqlite import SQLiteBackend +from tests.infra.storage_records import make_raw_session pytestmark = pytest.mark.uses_real_clock("Same as test_async_index: acquired_at is opaque metadata.") @@ -69,6 +75,134 @@ def _parse_batch_observation( } +async def test_parse_backlog_excludes_terminal_failure_authority_until_forced_reparse(tmp_path: Path) -> None: + """Scheduled parse selection stops retrying a typed terminal refusal.""" + initialize_active_archive_root(tmp_path) + source_db = tmp_path / "source.db" + backend = SQLiteBackend(db_path=source_db) + try: + await backend.save_raw_session( + make_raw_session( + raw_id="terminal-unsupported", + source_name="codex-session", + source_path="unsupported.jsonl", + validation_status="skipped", + parse_error="unsupported shape", + blob_size=1, + acquired_at="2026-08-09T00:00:00+00:00", + ) + ) + await backend.save_raw_failure_evidence( + "terminal-unsupported", + artifact_kind=RawFailureEvidenceKind.TERMINAL_UNSUPPORTED_SHAPE.value, + support_status=RawFailureEvidenceKind.TERMINAL_UNSUPPORTED_SHAPE.support_status.value, + outcome_code="unsupported_shape", + retryable=False, + evidence_ref="shape", + remediation="support it", + diagnostic="unsupported shape", + ) + + assert await collect_parse_backlog(backend, source_paths=None) == [] + assert await collect_parse_backlog(backend, source_paths=None, force_reparse=True) == ["terminal-unsupported"] + finally: + await backend.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "mutation", + ["wrong-support-status", "wrong-coordinate"], +) +async def test_parse_backlog_keeps_malformed_terminal_evidence_retryable( + tmp_path: Path, + mutation: str, +) -> None: + """Only an exact, typed terminal carrier suppresses scheduled retry.""" + initialize_active_archive_root(tmp_path) + source_db = tmp_path / "source.db" + backend = SQLiteBackend(db_path=source_db) + raw_id = "malformed-terminal" + try: + await backend.save_raw_session( + make_raw_session( + raw_id=raw_id, + source_name="codex-session", + source_path="unsupported.jsonl", + validation_status="skipped", + parse_error="unsupported shape", + blob_size=1, + acquired_at="2026-08-09T00:00:00+00:00", + ) + ) + await backend.save_raw_failure_evidence( + raw_id, + artifact_kind=RawFailureEvidenceKind.TERMINAL_UNSUPPORTED_SHAPE.value, + support_status=RawFailureEvidenceKind.TERMINAL_UNSUPPORTED_SHAPE.support_status.value, + outcome_code="unsupported_shape", + retryable=False, + evidence_ref="shape", + remediation="support it", + diagnostic="unsupported shape", + ) + finally: + await backend.close() + + with sqlite3.connect(source_db) as conn: + if mutation == "wrong-support-status": + conn.execute( + "UPDATE raw_artifacts SET support_status = 'partial_decode' WHERE raw_id = ?", + (raw_id,), + ) + else: + conn.execute( + "UPDATE raw_artifacts SET source_path = 'other.jsonl' WHERE raw_id = ?", + (raw_id,), + ) + + backend = SQLiteBackend(db_path=source_db) + try: + assert await collect_parse_backlog(backend, source_paths=None) == [raw_id] + finally: + await backend.close() + + +@pytest.mark.asyncio +async def test_validation_failed_unsupported_terminal_evidence_remains_unexplained(tmp_path: Path) -> None: + """Validation failure authority is limited to corrupt/decode outcomes.""" + initialize_active_archive_root(tmp_path) + source_db = tmp_path / "source.db" + backend = SQLiteBackend(db_path=source_db) + try: + await backend.save_raw_session( + make_raw_session( + raw_id="validation-failed-unsupported", + source_name="codex-session", + source_path="unsupported.jsonl", + validation_status="failed", + parse_error="unsupported shape", + blob_size=1, + ) + ) + await backend.save_raw_failure_evidence( + "validation-failed-unsupported", + artifact_kind=RawFailureEvidenceKind.TERMINAL_UNSUPPORTED_SHAPE.value, + support_status=RawFailureEvidenceKind.TERMINAL_UNSUPPORTED_SHAPE.support_status.value, + outcome_code="unsupported_shape", + retryable=False, + evidence_ref="shape", + remediation="support it", + diagnostic="unsupported shape", + ) + finally: + await backend.close() + + lifecycle = read_raw_failure_lifecycle(source_db) + assert lifecycle.terminal == 0 + assert lifecycle.unexplained == 1 + assert lifecycle.blocking is True + + class TestParseResultMerge: async def test_merge_thread_safe(self) -> None: result = ParseResult() diff --git a/tests/unit/sinex/test_ingest_atomicity.py b/tests/unit/sinex/test_ingest_atomicity.py index 2c6b604fbc..7ec07afac3 100644 --- a/tests/unit/sinex/test_ingest_atomicity.py +++ b/tests/unit/sinex/test_ingest_atomicity.py @@ -66,6 +66,10 @@ async def connection(self) -> AsyncIterator[_AsyncConnection]: assert self.active is not None yield _AsyncConnection(self.active) + async def supersede_deferred_cas_evidence(self, raw_id: str) -> None: + del raw_id + assert self.active is not None + class _Repository: def __init__(self, backend: _SourceBackend) -> None: diff --git a/tests/unit/sources/test_decoders.py b/tests/unit/sources/test_decoders.py index 97a20ae5fc..1e2089a75f 100644 --- a/tests/unit/sources/test_decoders.py +++ b/tests/unit/sources/test_decoders.py @@ -11,9 +11,11 @@ import zipfile from pathlib import Path +import pytest from hypothesis import given, settings from hypothesis import strategies as st +from polylogue.sources.decoder_json import JsonlDecodeError from polylogue.sources.decoders import ( MAX_AGGREGATE_UNCOMPRESSED_SIZE, MAX_UNCOMPRESSED_SIZE, @@ -187,6 +189,12 @@ def test_jsonl_txt_extension(self) -> None: items = list(_iter_json_stream(handle, "data.jsonl.txt")) assert len(items) == 2 + def test_strict_jsonl_decode_reports_physical_offending_line(self) -> None: + content = b'{"valid": 1}\n\nnot json at all\n{"later": 2}\n' + with pytest.raises(JsonlDecodeError) as exc_info: + list(_iter_json_stream(io.BytesIO(content), "data.jsonl", fail_on_decode_error=True)) + assert exc_info.value.line_number == 3 + # ============================================================================= # _ZipEntryValidator diff --git a/tests/unit/sources/test_live_batch_support.py b/tests/unit/sources/test_live_batch_support.py index 4d74f40642..6b35a1c84e 100644 --- a/tests/unit/sources/test_live_batch_support.py +++ b/tests/unit/sources/test_live_batch_support.py @@ -23,6 +23,7 @@ ) from polylogue.archive.session_revision_membership import MembershipClassification from polylogue.core.enums import ArtifactSupportStatus, Provider +from polylogue.core.raw_failure_evidence import RAW_FAILURE_EVIDENCE_KINDS from polylogue.pipeline.ids import session_content_hash, session_revision_projection from polylogue.sources.dispatch import parse_payload from polylogue.sources.live import LiveWatcher, WatchSource @@ -46,6 +47,7 @@ from polylogue.sources.parsers.base import ParsedMessage, ParsedSession from polylogue.storage.blob_store import BlobStore from polylogue.storage.raw_authority import RAW_AUTHORITY_PARSER_FINGERPRINT +from polylogue.storage.raw_failure_lifecycle import read_raw_failure_lifecycle from polylogue.storage.sqlite.archive_tiers import archive as archive_tier_module from polylogue.storage.sqlite.archive_tiers import revision_governance as archive_revision_governance from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore @@ -408,6 +410,154 @@ def test_full_ingest_empty_jsonl_is_not_misclassified_as_truncated( assert artifact == ("terminal_unsupported_shape", "unsupported_parseable", 0) +def test_full_ingest_unknown_export_without_sessions_records_terminal_evidence(tmp_path: Path) -> None: + root = tmp_path / "unknown" + root.mkdir() + path = root / "export.jsonl" + path.write_bytes(b"") + db_path = tmp_path / "archive.sqlite" + processor = LiveBatchProcessor( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=db_path))), + (WatchSource(name="unknown", root=root),), + cursor=CursorStore(db_path), + parser_fingerprint="test-parser", + ) + + result = processor._ingest_full_paths_sync([path], source_name="unknown") + + assert result.succeeded == [path] + with sqlite3.connect(tmp_path / "source.db") as conn: + artifact = conn.execute("SELECT artifact_kind, support_status, parse_as_session FROM raw_artifacts").fetchone() + assert artifact == ("terminal_unknown_export_no_session", "unsupported_parseable", 0) + + +def test_full_ingest_unknown_malformed_jsonl_records_terminal_decode_and_stops_retrying(tmp_path: Path) -> None: + """Complete malformed JSONL lines are terminal decode evidence, not no-session evidence.""" + root = tmp_path / "unknown" + root.mkdir() + path = root / "malformed.jsonl" + path.write_bytes(b'{"broken":}\n{"also_broken":}\n') + db_path = tmp_path / "archive.sqlite" + processor = LiveBatchProcessor( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=db_path))), + (WatchSource(name="unknown", root=root),), + cursor=CursorStore(db_path), + parser_fingerprint="test-parser", + ) + + first = processor._ingest_full_paths_sync([path], source_name="unknown") + second = processor._ingest_full_paths_sync([path], source_name="unknown") + + assert first.succeeded == [path] + assert first.failed == [] + assert second.succeeded == [path] + assert second.failed == [] + with sqlite3.connect(tmp_path / "source.db") as conn: + artifact = conn.execute("SELECT artifact_kind, support_status FROM raw_artifacts").fetchone() + assert artifact == ("terminal_unknown_json_decode", "decode_failed") + lifecycle = read_raw_failure_lifecycle(tmp_path / "source.db") + assert lifecycle.terminal == 1 + assert lifecycle.unexplained == 0 + + +def test_full_ingest_unknown_malformed_final_jsonl_record_records_terminal_decode(tmp_path: Path) -> None: + """A malformed final JSONL record contributes to strict decode evidence.""" + root = tmp_path / "unknown" + root.mkdir() + path = root / "malformed-final.jsonl" + path.write_bytes(b'{"only_broken":}\n') + db_path = tmp_path / "archive.sqlite" + processor = LiveBatchProcessor( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=db_path))), + (WatchSource(name="unknown", root=root),), + cursor=CursorStore(db_path), + parser_fingerprint="test-parser", + ) + + result = processor._ingest_full_paths_sync([path], source_name="unknown") + + assert result.succeeded == [path] + assert result.failed == [] + with sqlite3.connect(tmp_path / "source.db") as conn: + artifact = conn.execute("SELECT artifact_kind, support_status FROM raw_artifacts").fetchone() + assert artifact == ("terminal_unknown_json_decode", "decode_failed") + + +def test_full_ingest_unknown_json_decode_records_terminal_decode_evidence(tmp_path: Path) -> None: + root = tmp_path / "unknown" + root.mkdir() + path = root / "export.json" + path.write_bytes(b"{") + db_path = tmp_path / "archive.sqlite" + processor = LiveBatchProcessor( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=db_path))), + (WatchSource(name="unknown", root=root),), + cursor=CursorStore(db_path), + parser_fingerprint="test-parser", + ) + + result = processor._ingest_full_paths_sync([path], source_name="unknown") + + assert result.succeeded == [path] + assert result.failed == [] + with sqlite3.connect(tmp_path / "source.db") as conn: + artifact = conn.execute("SELECT artifact_kind, support_status, parse_as_session FROM raw_artifacts").fetchone() + assert artifact == ("terminal_unknown_json_decode", "decode_failed", 0) + + +def test_full_ingest_unknown_invalid_utf8_records_terminal_decode_evidence(tmp_path: Path) -> None: + root = tmp_path / "unknown" + root.mkdir() + path = root / "export.json" + path.write_bytes(b"\xff") + db_path = tmp_path / "archive.sqlite" + processor = LiveBatchProcessor( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=db_path))), + (WatchSource(name="unknown", root=root),), + cursor=CursorStore(db_path), + parser_fingerprint="test-parser", + ) + + result = processor._ingest_full_paths_sync([path], source_name="unknown") + + assert result.succeeded == [path] + assert result.failed == [] + with sqlite3.connect(tmp_path / "source.db") as conn: + artifact = conn.execute("SELECT artifact_kind, support_status, parse_as_session FROM raw_artifacts").fetchone() + assert artifact == ("terminal_unknown_json_decode", "decode_failed", 0) + + +def test_full_ingest_unknown_semantic_value_error_remains_unexplained( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = tmp_path / "unknown" + root.mkdir() + path = root / "export.json" + path.write_bytes(b'{"unrelated": "payload"}') + db_path = tmp_path / "archive.sqlite" + processor = LiveBatchProcessor( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=db_path))), + (WatchSource(name="unknown", root=root),), + cursor=CursorStore(db_path), + parser_fingerprint="test-parser", + ) + + def raise_semantic_value_error(*_args: object, **_kwargs: object) -> list[ParsedSession]: + raise ValueError("semantic parser rejection") + + monkeypatch.setattr("polylogue.sources.live.batch.parse_payload", raise_semantic_value_error) + + result = processor._ingest_full_paths_sync([path], source_name="unknown") + + assert result.succeeded == [] + with sqlite3.connect(tmp_path / "source.db") as conn: + artifact_kinds = {row[0] for row in conn.execute("SELECT artifact_kind FROM raw_artifacts")} + assert not artifact_kinds & RAW_FAILURE_EVIDENCE_KINDS + lifecycle = read_raw_failure_lifecycle(tmp_path / "source.db") + assert lifecycle.unexplained == 1 + + def test_full_ingest_defers_incomplete_jsonl_only_after_hot_prefix_proof( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -457,6 +607,44 @@ def grow_source_after_capture(**kwargs: object) -> bool: assert artifact == ("deferred_hot_jsonl_capture", "partial_decode", 1) +def test_full_ingest_claude_partial_jsonl_has_provider_specific_evidence( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + from polylogue.sources.live import batch as live_batch + + root = tmp_path / "claude" + root.mkdir() + path = root / "active.jsonl" + captured = b'{"type":"assistant"' + path.write_bytes(captured) + db_path = tmp_path / "archive.sqlite" + processor = LiveBatchProcessor( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=db_path))), + (WatchSource(name="claude-code", root=root),), + cursor=CursorStore(db_path), + parser_fingerprint="test-parser", + ) + monkeypatch.setattr( + "polylogue.sources.live.batch._jsonl_provider_and_session_artifact", + lambda _path, fallback_provider: (fallback_provider, True), + ) + boundary_check = live_batch._captured_jsonl_ends_at_record_boundary + + def grow_source_after_capture(**kwargs: object) -> bool: + path.write_bytes(captured + b"\n") + return boundary_check(**kwargs) # type: ignore[arg-type] + + monkeypatch.setattr(live_batch, "_captured_jsonl_ends_at_record_boundary", grow_source_after_capture) + + result = processor._ingest_full_paths_sync([path], source_name="claude-code") + + assert result.succeeded == [path] + with sqlite3.connect(tmp_path / "source.db") as conn: + artifact = conn.execute("SELECT artifact_kind, support_status, parse_as_session FROM raw_artifacts").fetchone() + assert artifact == ("deferred_claude_code_partial_jsonl", "partial_decode", 1) + + def test_streamed_incomplete_jsonl_capture_defers_then_replays_completed_source( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -3606,6 +3794,72 @@ def test_raw_failure_cursor_guard_uses_root_source_tier_for_pointer_index(tmp_pa assert processor._cursor_references_raw_failure_requiring_full_replay(path, record) +def test_raw_failure_cursor_guard_rejects_contradictory_or_mismatched_evidence(tmp_path: Path) -> None: + """Append fallback requires the same source coordinate and valid support status.""" + root = tmp_path / "sessions" + root.mkdir() + path = root / "guard.jsonl" + path.write_bytes(b'{"type":"session_meta","payload":{"id":"guard"}}\n') + index_db = tmp_path / "index.db" + initialize_active_archive_root(tmp_path) + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + mismatched_raw_id = archive.write_raw_payload( + provider=Provider.CODEX, + payload=path.read_bytes(), + source_path=str(path), + source_index=1, + acquired_at_ms=1, + ) + contradictory_raw_id = archive.write_raw_payload( + provider=Provider.CODEX, + payload=path.read_bytes() + b"2", + source_path=str(path), + source_index=0, + acquired_at_ms=2, + ) + with sqlite3.connect(tmp_path / "source.db") as source_conn: + source_conn.executemany( + "UPDATE raw_sessions SET parse_error = ? WHERE raw_id = ?", + [("mismatched coordinate", mismatched_raw_id), ("contradictory support", contradictory_raw_id)], + ) + upsert_raw_artifact( + source_conn, + mismatched_raw_id, + ArchiveSourceArtifact( + artifact_id="mismatched-coordinate-evidence", + origin="chatgpt-export", + source_path=str(path), + source_index=0, + artifact_kind="deferred_cas_frontier", + classification_reason="deferred_cas_frontier", + support_status=ArtifactSupportStatus.PARTIAL_DECODE, + ), + ) + upsert_raw_artifact( + source_conn, + contradictory_raw_id, + ArchiveSourceArtifact( + artifact_id="contradictory-support-evidence", + origin="codex-session", + source_path=str(path), + source_index=0, + artifact_kind="deferred_cas_frontier", + classification_reason="deferred_cas_frontier", + support_status=ArtifactSupportStatus.DECODE_FAILED, + ), + ) + source_conn.commit() + processor = LiveBatchProcessor( + cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=index_db))), + (WatchSource(name="codex", root=root),), + cursor=CursorStore(index_db), + parser_fingerprint="test-parser", + ) + + assert processor._raw_failure_requires_full_replay(path, mismatched_raw_id) is False + assert processor._raw_failure_requires_full_replay(path, contradictory_raw_id) is False + + def test_captured_incomplete_jsonl_is_rejected_after_source_disappears( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -3876,7 +4130,7 @@ def test_append_parse_failure_retains_typed_raw_failure( plan = _append_plan(path, payload, payload_hash="bad") owner = _append_owner(tmp_path) monkeypatch.setattr( - "polylogue.sources.dispatch.parse_payload", + "polylogue.sources.dispatch.parse_stream_payload", lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("injected append parse failure")), ) @@ -5501,9 +5755,18 @@ def session(native_id: str, *texts: str) -> ParsedSession: # retry-candidate query (storage/repair.py) nothing stable to match # once the message text drifts -- exactly what happened to a real # production session that hit this guard under #2718's original - # wording and was never retried again. MembershipReplayConflictError - # gives that query a message-text-independent marker to key on. - assert parse_error.startswith("MembershipReplayConflictError:") + # The structured evidence row, not the diagnostic wording, is the + # retry authorization. + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute( + """ + SELECT a.artifact_kind + FROM raw_artifacts AS a + JOIN raw_sessions AS r ON r.raw_id = a.raw_id + WHERE r.source_path = ? + """, + (str(older_bundle),), + ).fetchone() == ("deferred_cas_frontier",) with sqlite3.connect(index_db) as conn: assert conn.execute("SELECT message_count FROM sessions WHERE native_id = 'shared'").fetchone() == (2,) head_after = conn.execute( @@ -5706,12 +5969,24 @@ def _parse_stream_payload_stub( (str(incident_recovery),), ).fetchone() assert parse_error is not None - # polylogue-5iz4 / #3646: the retry-eligible marker, not a bare - # RuntimeError -- this is what lets a later pass ever try again. + # The typed evidence is authoritative for new rows. The recognized prefix + # remains a bounded compatibility bridge for this historical diagnostic. assert parse_error.startswith("MembershipReplayConflictError:") + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute( + """ + SELECT a.artifact_kind + FROM raw_artifacts AS a + JOIN raw_sessions AS r ON r.raw_id = a.raw_id + WHERE r.source_path = ? + """, + (str(incident_recovery),), + ).fetchone() == ("deferred_cas_frontier",) from polylogue.storage.repair import _raw_materialization_retryable_missing_blob_error assert _raw_materialization_retryable_missing_blob_error(parse_error) is True + assert _raw_materialization_retryable_missing_blob_error("RuntimeError: unrelated parser failure") is False + assert _raw_materialization_retryable_missing_blob_error(parse_error, True) is True with sqlite3.connect(index_db) as conn: assert ( diff --git a/tests/unit/sources/test_live_watcher_parse_stage_equivalence.py b/tests/unit/sources/test_live_watcher_parse_stage_equivalence.py index 8c48f55af8..167445305d 100644 --- a/tests/unit/sources/test_live_watcher_parse_stage_equivalence.py +++ b/tests/unit/sources/test_live_watcher_parse_stage_equivalence.py @@ -33,10 +33,12 @@ import pytest from polylogue import Polylogue -from polylogue.sources.live.batch import LiveBatchProcessor +from polylogue.core.enums import Provider +from polylogue.sources.live.batch import LiveBatchProcessor, _live_parse_stage_candidates from polylogue.sources.live.cursor import CursorStore from polylogue.sources.live.parse_prefetch import LiveParseStage from polylogue.sources.live.watcher import _PARSER_FINGERPRINT, WatchSource +from polylogue.storage.raw_failure_lifecycle import read_raw_failure_lifecycle _VOLATILE_COLUMNS: dict[str, frozenset[str]] = { "raw_sessions": frozenset({"acquired_at_ms", "parsed_at_ms"}), @@ -208,3 +210,41 @@ def delayed_worker(*args: object, **kwargs: object) -> object: assert _canonical_snapshot(baseline_root) == _canonical_snapshot(prefetch_root) assert _raw_sessions_source_path_order(prefetch_root) == _raw_sessions_source_path_order(baseline_root) assert _raw_sessions_source_path_order(prefetch_root) == tuple(str(path) for path in paths) + + +@pytest.mark.asyncio +async def test_unknown_mixed_jsonl_prefetch_falls_back_to_strict_decode(tmp_path: Path) -> None: + """Strict prefetch refuses a partial unknown-provider conversation.""" + root = tmp_path / "unknown" + root.mkdir() + path = root / "mixed.jsonl" + path.write_bytes(b'{"id":"unknown-1","messages":[{"id":"m1","role":"user","content":"hello"}]}\n{"broken":}\n') + candidates = _live_parse_stage_candidates([path], fallback_provider=Provider.UNKNOWN) + assert len(candidates) == 1 + assert candidates[0].provider is Provider.UNKNOWN + assert candidates[0].is_stream is False + polylogue = Polylogue(archive_root=tmp_path, db_path=tmp_path / "index.db") + stage = LiveParseStage(max_workers=1, max_inflight_bytes=10_000_000) + processor = LiveBatchProcessor( + polylogue, + (WatchSource(name="unknown", root=root),), + cursor=CursorStore(tmp_path / "index.db"), + parser_fingerprint=_PARSER_FINGERPRINT, + parse_stage=stage, + ) + try: + result = await processor.ingest_files([path], emit_event=False) + finally: + stage.shutdown() + + assert result.failed_file_count == 0 + assert result.succeeded_file_count == 1 + assert len(stage.cache) == 0 + with _connect(tmp_path / "index.db") as conn: + assert conn.execute("SELECT COUNT(*) FROM sessions").fetchone()[0] == 0 + with _connect(tmp_path / "source.db") as conn: + artifact = conn.execute("SELECT artifact_kind, support_status FROM raw_artifacts").fetchone() + assert (artifact[0], artifact[1]) == ("terminal_unknown_json_decode", "decode_failed") + lifecycle = read_raw_failure_lifecycle(tmp_path / "source.db") + assert lifecycle.terminal == 1 + assert lifecycle.unexplained == 0 diff --git a/tests/unit/storage/test_archive_tiers_source_write.py b/tests/unit/storage/test_archive_tiers_source_write.py index d71ad8565c..0cafa8418e 100644 --- a/tests/unit/storage/test_archive_tiers_source_write.py +++ b/tests/unit/storage/test_archive_tiers_source_write.py @@ -23,6 +23,7 @@ read_history_sidecar, read_hook_event, read_raw_artifact, + upsert_raw_artifact, write_history_sidecar, write_source_raw_session, write_source_raw_session_blob_ref, @@ -165,6 +166,116 @@ def test_archive_tiers_source_writer_materializes_raw_session_with_blob_ref(tmp_ assert list_hook_events(conn, origin=Origin.CLAUDE_CODE_SESSION, session_native_id="session-1") == (hook_event,) +def test_source_artifact_upsert_keeps_coordinate_deduplication_and_raw_failure_fanout( + tmp_path: Path, +) -> None: + conn = _connect(tmp_path / "source.db") + raw_ids = [ + write_source_raw_session( + conn, + origin=Origin.CODEX_SESSION, + source_path="/tmp/shared.jsonl", + source_index=0, + payload=f"payload-{suffix}".encode(), + acquired_at_ms=index + 1, + ) + for index, suffix in enumerate(("old", "new")) + ] + + upsert_raw_artifact( + conn, + raw_ids[0], + ArchiveSourceArtifact( + artifact_id="failure-old", + origin=Origin.CODEX_SESSION, + source_path="/tmp/shared.jsonl", + source_index=0, + artifact_kind="deferred_cas_frontier", + classification_reason="deferred_cas_frontier", + support_status=ArtifactSupportStatus.PARTIAL_DECODE, + ), + ) + upsert_raw_artifact( + conn, + raw_ids[1], + ArchiveSourceArtifact( + artifact_id="failure-new", + origin=Origin.CODEX_SESSION, + source_path="/tmp/shared.jsonl", + source_index=0, + artifact_kind="deferred_cas_frontier", + classification_reason="deferred_cas_frontier", + support_status=ArtifactSupportStatus.PARTIAL_DECODE, + ), + ) + upsert_raw_artifact( + conn, + raw_ids[0], + ArchiveSourceArtifact( + artifact_id="failure-old-refresh", + origin=Origin.CODEX_SESSION, + source_path="/tmp/shared.jsonl", + source_index=0, + artifact_kind="terminal_corrupt_input", + classification_reason="terminal_corrupt_input", + support_status=ArtifactSupportStatus.DECODE_FAILED, + ), + ) + + rows = conn.execute( + """ + SELECT raw_id, origin, source_path, source_index, artifact_kind, support_status + FROM raw_artifacts + WHERE origin = ? AND source_path = ? AND source_index = 0 + ORDER BY raw_id + """, + (Origin.CODEX_SESSION.value, "/tmp/shared.jsonl"), + ).fetchall() + assert {tuple(row) for row in rows} == { + ( + raw_ids[0], + Origin.CODEX_SESSION.value, + "/tmp/shared.jsonl", + 0, + "terminal_corrupt_input", + ArtifactSupportStatus.DECODE_FAILED.value, + ), + ( + raw_ids[1], + Origin.CODEX_SESSION.value, + "/tmp/shared.jsonl", + 0, + "deferred_cas_frontier", + ArtifactSupportStatus.PARTIAL_DECODE.value, + ), + } + + upsert_raw_artifact( + conn, + raw_ids[0], + ArchiveSourceArtifact( + artifact_id="ordinary-coordinate", + origin=Origin.CODEX_SESSION, + source_path="/tmp/shared.jsonl", + source_index=0, + artifact_kind="session_export", + classification_reason="session_export", + support_status=ArtifactSupportStatus.SUPPORTED_PARSEABLE, + ), + ) + ordinary = conn.execute( + """ + SELECT artifact_id, raw_id, artifact_kind + FROM raw_artifacts + WHERE origin = ? AND source_path = ? AND source_index = 0 + AND artifact_kind = 'session_export' + """, + (Origin.CODEX_SESSION.value, "/tmp/shared.jsonl"), + ).fetchone() + assert ordinary is not None + assert tuple(ordinary) == ("ordinary-coordinate", raw_ids[0], "session_export") + + def test_archive_tiers_source_writer_replays_hook_events_idempotently(tmp_path: Path) -> None: conn = _connect(tmp_path / "source.db") payload = b'{"kind":"session","messages":["hello"]}' diff --git a/tests/unit/storage/test_durable_migrations.py b/tests/unit/storage/test_durable_migrations.py index 0d755056a7..83666526e2 100644 --- a/tests/unit/storage/test_durable_migrations.py +++ b/tests/unit/storage/test_durable_migrations.py @@ -467,6 +467,182 @@ def test_symlinked_user_tier_uses_resolved_attestation_authority( conn.close() +def _restore_source_pre_v30_raw_artifact_indexes(conn: sqlite3.Connection) -> None: + """Restore the raw-artifact index shape that existed before migration 030.""" + conn.executescript( + """ + DROP INDEX IF EXISTS idx_raw_artifacts_failure_identity; + DROP INDEX IF EXISTS idx_raw_artifacts_source_identity; + CREATE UNIQUE INDEX idx_raw_artifacts_source_identity + ON raw_artifacts(origin, source_path, source_index); + """ + ) + + +def _create_source_v29_raw_failure_fixture(path: Path, *, archive_root: Path) -> tuple[str, str, str]: + """Build the exact pre-v30 source shape used by migration 030.""" + current_indexes = """ +CREATE UNIQUE INDEX IF NOT EXISTS idx_raw_artifacts_source_identity +ON raw_artifacts(origin, source_path, source_index) +WHERE artifact_kind NOT IN ( + 'deferred_hot_jsonl_capture', + 'deferred_claude_code_partial_jsonl', + 'deferred_cas_frontier', + 'deferred_codex_cas_frontier', + 'terminal_corrupt_input', + 'terminal_superseded_deferred_cas_frontier', + 'terminal_unknown_json_decode', + 'terminal_unknown_export_no_session', + 'terminal_unsupported_shape' +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_raw_artifacts_failure_identity +ON raw_artifacts(raw_id, origin, source_path, source_index) +WHERE artifact_kind IN ( + 'deferred_hot_jsonl_capture', + 'deferred_claude_code_partial_jsonl', + 'deferred_cas_frontier', + 'deferred_codex_cas_frontier', + 'terminal_corrupt_input', + 'terminal_superseded_deferred_cas_frontier', + 'terminal_unknown_json_decode', + 'terminal_unknown_export_no_session', + 'terminal_unsupported_shape' +); +""" + legacy_index = """ +CREATE UNIQUE INDEX IF NOT EXISTS idx_raw_artifacts_source_identity +ON raw_artifacts(origin, source_path, source_index); +""" + v29_ddl = SOURCE_DDL.replace(current_indexes, legacy_index) + assert v29_ddl != SOURCE_DDL + path.unlink(missing_ok=True) + blob_store = BlobStore(archive_root / "blob") + ordinary_blob, ordinary_size = blob_store.write_from_bytes(b"ordinary-v29") + failure_a_blob, failure_a_size = blob_store.write_from_bytes(b"failure-a-v29") + failure_b_blob, failure_b_size = blob_store.write_from_bytes(b"failure-b-v29") + conn = sqlite3.connect(path) + try: + conn.executescript(v29_ddl) + conn.execute("PRAGMA user_version = 29") + conn.executemany( + """ + INSERT INTO raw_sessions ( + raw_id, origin, source_path, source_index, blob_hash, blob_size, acquired_at_ms + ) VALUES (?, 'codex-session', ?, ?, ?, ?, ?) + """, + [ + ("raw-v29-ordinary", "/v29/ordinary.json", 0, bytes.fromhex(ordinary_blob), ordinary_size, 1), + ("raw-v29-failure-a", "/v29/failure.jsonl", 0, bytes.fromhex(failure_a_blob), failure_a_size, 2), + ("raw-v29-failure-b", "/v29/failure.jsonl", 1, bytes.fromhex(failure_b_blob), failure_b_size, 3), + ], + ) + conn.execute( + """ + INSERT INTO raw_artifacts ( + artifact_id, raw_id, origin, source_path, source_index, artifact_kind, + support_status, classification_reason, first_observed_at_ms, last_observed_at_ms + ) VALUES ('artifact-v29-ordinary', 'raw-v29-ordinary', 'codex-session', + '/v29/ordinary.json', 0, 'session_export', 'supported_parseable', + 'ordinary-v29', 1, 1) + """ + ) + conn.executemany( + """ + INSERT INTO raw_artifacts ( + artifact_id, raw_id, origin, source_path, source_index, artifact_kind, + support_status, classification_reason, first_observed_at_ms, last_observed_at_ms + ) VALUES (?, ?, 'codex-session', '/v29/failure.jsonl', ?, + 'deferred_cas_frontier', 'partial_decode', 'deferred-v29', ?, ?) + """, + [ + ("artifact-v29-failure-a", "raw-v29-failure-a", 0, 2, 2), + ("artifact-v29-failure-b", "raw-v29-failure-b", 1, 3, 3), + ], + ) + conn.commit() + finally: + conn.close() + return ordinary_blob, failure_a_blob, failure_b_blob + + +def test_source_tier_v29_applies_only_migration_030_and_preserves_failure_coordinates( + workspace_env: dict[str, Path], + tmp_path: Path, +) -> None: + """The source-v30 index split preserves ordinary and retained-raw evidence.""" + db_path = workspace_env["archive_root"] / "source.db" + _create_source_v29_raw_failure_fixture(db_path, archive_root=workspace_env["archive_root"]) + manifest = _verified_backup_manifest(tmp_path / "backup-source-v29") + + with sqlite3.connect(db_path) as conn: + result = migrate_archive_tier(conn, ArchiveTier.SOURCE, backup_manifest=manifest) + assert result.from_version == 29 + assert result.to_version == 30 + assert result.applied_versions == (30,) + assert int(conn.execute("PRAGMA user_version").fetchone()[0]) == 30 + + index_names = {str(row[1]) for row in conn.execute("PRAGMA index_list('raw_artifacts')")} + assert {"idx_raw_artifacts_source_identity", "idx_raw_artifacts_failure_identity"} <= index_names + index_columns = { + name: tuple(str(column[2]) for column in conn.execute(f"PRAGMA index_info('{name}')")) + for name in ("idx_raw_artifacts_source_identity", "idx_raw_artifacts_failure_identity") + } + assert index_columns["idx_raw_artifacts_source_identity"] == ("origin", "source_path", "source_index") + assert index_columns["idx_raw_artifacts_failure_identity"] == ( + "raw_id", + "origin", + "source_path", + "source_index", + ) + assert { + tuple(row) + for row in conn.execute( + """ + SELECT artifact_id, raw_id, source_path, source_index, artifact_kind + FROM raw_artifacts + ORDER BY artifact_id + """ + ) + } == { + ( + "artifact-v29-failure-a", + "raw-v29-failure-a", + "/v29/failure.jsonl", + 0, + "deferred_cas_frontier", + ), + ( + "artifact-v29-failure-b", + "raw-v29-failure-b", + "/v29/failure.jsonl", + 1, + "deferred_cas_frontier", + ), + ( + "artifact-v29-ordinary", + "raw-v29-ordinary", + "/v29/ordinary.json", + 0, + "session_export", + ), + } + + conn.execute( + """ + INSERT INTO raw_artifacts ( + artifact_id, raw_id, origin, source_path, source_index, artifact_kind, + support_status, classification_reason, first_observed_at_ms, last_observed_at_ms + ) VALUES ('artifact-v30-failure-same-coordinate', 'raw-v29-ordinary', + 'codex-session', '/v29/failure.jsonl', 0, 'deferred_cas_frontier', + 'partial_decode', 'new-v30', 4, 4) + """ + ) + assert conn.execute( + "SELECT COUNT(*) FROM raw_artifacts WHERE source_path = '/v29/failure.jsonl' AND source_index = 0" + ).fetchone() == (2,) + + def test_source_tier_v1_migrates_to_current_without_native_uniqueness( workspace_env: dict[str, Path], tmp_path: Path, @@ -681,6 +857,13 @@ def test_source_tier_v7_expands_origin_checks_with_verified_backup( " ,blob_hash BLOB CHECK(blob_hash IS NULL OR length(blob_hash) = 32)\n", "", ) + # The v7 fixture also predates the v22 source-hash index. Removing the + # column without removing its index makes SQLite reject the fixture before + # migration code gets a chance to exercise the copy-forward. + old_ddl = old_ddl.replace( + "CREATE INDEX IF NOT EXISTS idx_raw_hook_events_source_hash\nON raw_hook_events(source_path, blob_hash);\n", + "", + ) # Migration 010 adds `excised_content` (polylogue-27m) -- a v7 snapshot # predates it, same as it predates the beads-origin/capture_mode diffs # stripped above. Without this, the fixture (built from the CURRENT @@ -738,6 +921,7 @@ def test_source_tier_v7_expands_origin_checks_with_verified_backup( "raw_quarantine_group_dedup_receipts", "raw_unknown_export_reclassification_receipts", "raw_non_session_duplicate_exclusion_receipts", + "raw_failure_disposition_receipts", ): conn.execute(f"DROP TABLE IF EXISTS {table_name}") conn.execute("PRAGMA user_version = 7") @@ -868,6 +1052,12 @@ def _create_source_v20_with_stale_origin_check(path: Path, *, archive_root: Path conn = sqlite3.connect(path) try: conn.executescript(_source_v20_ddl_with_stale_origin_check()) + _restore_source_pre_v30_raw_artifact_indexes(conn) + # v29 is not present in a v20 archive. The broad origin-check rewrite + # above intentionally affects every current table, so remove this + # later table explicitly instead of handing migration 029 a malformed + # pre-existing copy that CREATE IF NOT EXISTS would preserve. + conn.execute("DROP TABLE raw_failure_disposition_receipts") conn.execute("PRAGMA user_version = 20") conn.execute( """ @@ -1144,10 +1334,12 @@ def test_source_tier_v24_repairs_raw_hook_event_origin_check( ) with sqlite3.connect(db_path) as conn: conn.executescript(SOURCE_DDL) + _restore_source_pre_v30_raw_artifact_indexes(conn) for table_name in ( "raw_quarantine_group_dedup_receipts", "raw_unknown_export_reclassification_receipts", "raw_non_session_duplicate_exclusion_receipts", + "raw_failure_disposition_receipts", ): conn.execute(f"DROP TABLE IF EXISTS {table_name}") conn.executescript( @@ -1486,6 +1678,7 @@ def test_source_tier_v13_adds_raw_sessions_blob_hash_index( conn = sqlite3.connect(db_path) try: conn.executescript(SOURCE_DDL) + _restore_source_pre_v30_raw_artifact_indexes(conn) conn.execute("DROP TABLE raw_sessions") conn.executescript( """ @@ -1533,6 +1726,7 @@ def test_source_tier_v13_adds_raw_sessions_blob_hash_index( "raw_quarantine_group_dedup_receipts", "raw_unknown_export_reclassification_receipts", "raw_non_session_duplicate_exclusion_receipts", + "raw_failure_disposition_receipts", ): conn.execute(f"DROP TABLE IF EXISTS {table_name}") conn.commit() @@ -1596,6 +1790,7 @@ def test_source_tier_v14_adds_raw_sessions_blob_hash_raw_id_index( conn = sqlite3.connect(db_path) try: conn.executescript(SOURCE_DDL) + _restore_source_pre_v30_raw_artifact_indexes(conn) conn.execute("DROP TABLE raw_sessions") conn.executescript( """ @@ -1644,6 +1839,7 @@ def test_source_tier_v14_adds_raw_sessions_blob_hash_raw_id_index( "raw_quarantine_group_dedup_receipts", "raw_unknown_export_reclassification_receipts", "raw_non_session_duplicate_exclusion_receipts", + "raw_failure_disposition_receipts", ): conn.execute(f"DROP TABLE IF EXISTS {table_name}") conn.commit() diff --git a/tests/unit/storage/test_repair.py b/tests/unit/storage/test_repair.py index 3691982624..3926d49985 100644 --- a/tests/unit/storage/test_repair.py +++ b/tests/unit/storage/test_repair.py @@ -12,6 +12,10 @@ import pytest from polylogue.config import Config +from polylogue.core.enums import ArtifactSupportStatus +from polylogue.core.errors import RawCASFrontierError +from polylogue.core.raw_failure_evidence import RawFailureEvidenceKind +from polylogue.daemon.status import raw_failure_info_for_root from polylogue.maintenance.models import DerivedModelStatus from polylogue.sources.revision_backfill import census_historical_revision_evidence from polylogue.storage import repair as repair_mod @@ -21,6 +25,7 @@ from polylogue.storage.insights.session.runtime import SessionInsightCounts, SessionInsightStatusSnapshot from polylogue.storage.raw_authority import RawReplayPlan, RawReplayPlanOutcome from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_archive_database +from polylogue.storage.sqlite.archive_tiers.source_write import ArchiveSourceArtifact, upsert_raw_artifact from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier @@ -512,16 +517,8 @@ def test_raw_materialization_retries_typed_transient_lock_failure(tmp_path: Path ).fetchone() == (1, None) -def test_raw_materialization_retries_legacy_frontier_authority_failures(tmp_path: Path) -> None: - """polylogue-5iz4: a MembershipReplayConflictError parse_error is retryable. - - The durable raw rows use the exact legacy strings observed in the live - frontier: a stale accepted-frontier CAS and the old membership-replay - guard. They are authority refusals, so the real repair selector must - reconsider them against the current head. An unrelated RuntimeError - remains excluded, which prevents a broad RuntimeError retry from making - the check vacuous. - """ +def test_raw_materialization_retries_only_with_deferred_frontier_evidence(tmp_path: Path) -> None: + """CAS retryability comes from durable evidence, never parse-error prose.""" from polylogue.core.enums import Provider from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root @@ -533,6 +530,7 @@ def test_raw_materialization_retries_legacy_frontier_authority_failures(tmp_path "cas": b'{"mapping":{"cas":{}}}', "membership": b'{"mapping":{"membership":{}}}', "stale": b'{"mapping":{"stale":{}}}', + "sibling": b'{"mapping":{"sibling":{}}}', } with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: raw_ids = { @@ -552,19 +550,69 @@ def test_raw_materialization_retries_legacy_frontier_authority_failures(tmp_path UPDATE raw_sessions SET parsed_at_ms = ?, parse_error = ? WHERE raw_id = ? """, [ - ( - 3, - "MembershipReplayConflictError: membership replay cannot replace a head " - "with unresolved byte-append evidence: logical_source_key='codex:whale'", - raw_ids["retryable"], - ), - (4, "RuntimeError: raw revision CAS rejected an older accepted frontier", raw_ids["cas"]), - (4, "RuntimeError: membership replay cannot replace an unconvertible byte head", raw_ids["membership"]), + (3, "changed wording for retryable frontier", raw_ids["retryable"]), + (4, "another changed wording", raw_ids["cas"]), + (4, "third changed wording", raw_ids["membership"]), (4, "RuntimeError: unrelated parser failure", raw_ids["stale"]), + (4, "RuntimeError: unrelated parser failure", raw_ids["sibling"]), ], ) + for name in ("retryable", "cas", "membership"): + upsert_raw_artifact( + source_conn, + raw_ids[name], + ArchiveSourceArtifact( + artifact_id=f"deferred-{name}", + origin="codex-session", + source_path=f"{name}.jsonl", + source_index=0, + artifact_kind="deferred_cas_frontier", + classification_reason="deferred_cas_frontier", + support_status=ArtifactSupportStatus.PARTIAL_DECODE, + parse_as_session=True, + schema_eligible=True, + first_observed_at_ms=100, + last_observed_at_ms=100, + ), + ) + upsert_raw_artifact( + source_conn, + raw_ids["cas"], + ArchiveSourceArtifact( + artifact_id="newer-unrelated-cas-artifact", + origin="codex-session", + source_path="cas.sqlite", + source_index=0, + artifact_kind="sqlite_state_database", + classification_reason="sqlite_state_database", + support_status=ArtifactSupportStatus.UNKNOWN, + first_observed_at_ms=200, + last_observed_at_ms=200, + ), + ) + upsert_raw_artifact( + source_conn, + raw_ids["sibling"], + ArchiveSourceArtifact( + artifact_id="deferred-on-sibling-coordinate", + origin="codex-session", + source_path="sibling-other.jsonl", + source_index=0, + artifact_kind="deferred_cas_frontier", + classification_reason="deferred_cas_frontier", + support_status=ArtifactSupportStatus.PARTIAL_DECODE, + parse_as_session=True, + schema_eligible=True, + first_observed_at_ms=100, + last_observed_at_ms=100, + ), + ) source_conn.commit() + candidates = repair_mod._raw_materialization_candidate_ids(config) + assert raw_ids["cas"] in candidates.raw_ids + assert raw_ids["sibling"] not in candidates.raw_ids + result = repair_mod.repair_raw_materialization(config, dry_run=True) assert result.metrics["raw_materialization_candidate_count"] == 3.0 @@ -573,8 +621,179 @@ def test_raw_materialization_retries_legacy_frontier_authority_failures(tmp_path ) -def test_raw_materialization_repairs_legacy_stale_frontier_failure(tmp_path: Path) -> None: - """The legacy CAS failure reaches the real replay actuator and clears only on success.""" +def test_raw_materialization_rejects_contradictory_deferred_evidence(tmp_path: Path) -> None: + """A deferred kind with terminal support cannot authorize replay.""" + from polylogue.core.enums import Provider + from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore + from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root + + initialize_active_archive_root(tmp_path) + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.CODEX, + payload=b'{"mapping":{"contradictory":true}}', + source_path="contradictory.jsonl", + acquired_at_ms=1, + ) + with sqlite3.connect(tmp_path / "source.db") as source_conn: + source_conn.execute( + "UPDATE raw_sessions SET parsed_at_ms = 2, parse_error = ? WHERE raw_id = ?", + ("changed wording", raw_id), + ) + upsert_raw_artifact( + source_conn, + raw_id, + ArchiveSourceArtifact( + artifact_id="contradictory-deferred-evidence", + origin="codex-session", + source_path="contradictory.jsonl", + source_index=0, + artifact_kind="deferred_cas_frontier", + classification_reason="deferred_cas_frontier", + support_status=ArtifactSupportStatus.DECODE_FAILED, + parse_as_session=True, + schema_eligible=True, + ), + ) + source_conn.commit() + + candidates = repair_mod._raw_materialization_candidate_ids(_config(tmp_path)) + + assert raw_id not in candidates.raw_ids + + +def test_raw_materialization_requires_exact_failed_artifact_coordinate(tmp_path: Path) -> None: + """A deferred neighbor cannot authorize replay for another raw coordinate.""" + from polylogue.core.enums import Provider + from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore + from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root + + initialize_active_archive_root(tmp_path) + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.CODEX, + payload=b'{"neighbor":true}', + source_path="target.jsonl", + acquired_at_ms=1, + ) + with sqlite3.connect(tmp_path / "source.db") as source_conn: + source_conn.execute( + "UPDATE raw_sessions SET parsed_at_ms = 2, parse_error = ? WHERE raw_id = ?", + ("deferred failure", raw_id), + ) + for suffix, origin, source_path, source_index in ( + ("origin", "claude-code-session", "target.jsonl", 0), + ("path", "codex-session", "neighbor.jsonl", 0), + ("index", "codex-session", "target.jsonl", 1), + ): + upsert_raw_artifact( + source_conn, + raw_id, + ArchiveSourceArtifact( + artifact_id=f"neighbor-{suffix}", + origin=origin, + source_path=source_path, + source_index=source_index, + artifact_kind="deferred_cas_frontier", + classification_reason="deferred_cas_frontier", + support_status=ArtifactSupportStatus.PARTIAL_DECODE, + parse_as_session=True, + schema_eligible=True, + ), + ) + source_conn.commit() + + candidates = repair_mod._raw_materialization_candidate_ids(_config(tmp_path)) + + assert raw_id not in candidates.raw_ids + + +def test_raw_materialization_validation_failure_cannot_reuse_deferred_authority(tmp_path: Path) -> None: + """Repair and its public backlog report share the worker validation gate.""" + from polylogue.core.enums import Provider + from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore + from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root + + initialize_active_archive_root(tmp_path) + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.CODEX, + payload=b'{"deferred":true}', + source_path="validation-failed.jsonl", + acquired_at_ms=1, + ) + with sqlite3.connect(tmp_path / "source.db") as source_conn: + source_conn.execute( + "UPDATE raw_sessions SET parsed_at_ms = NULL, parse_error = ?, validation_status = 'failed' WHERE raw_id = ?", + ("decode: malformed input", raw_id), + ) + upsert_raw_artifact( + source_conn, + raw_id, + ArchiveSourceArtifact( + artifact_id="validation-failed-deferred", + origin="codex-session", + source_path="validation-failed.jsonl", + source_index=0, + artifact_kind="deferred_cas_frontier", + classification_reason="deferred_cas_frontier", + support_status=ArtifactSupportStatus.PARTIAL_DECODE, + parse_as_session=True, + schema_eligible=True, + ), + ) + source_conn.commit() + + config = _config(tmp_path) + assert raw_id not in repair_mod._raw_materialization_candidate_ids(config).raw_ids + backlog = repair_mod.raw_materialization_replay_backlog(config) + assert backlog["candidate_count"] == 0 + + +@pytest.mark.parametrize("artifact_kind", ["deferred_hot_jsonl_capture", "deferred_claude_code_partial_jsonl"]) +def test_raw_materialization_does_not_replay_hot_partial_capture(tmp_path: Path, artifact_kind: str) -> None: + """Hot partial evidence stays deferred until a complete source observation arrives.""" + from polylogue.core.enums import Provider + from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore + from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root + + initialize_active_archive_root(tmp_path) + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.CLAUDE_CODE, + payload=b'{"type":"message_start"}\n', + source_path="rollout.jsonl", + acquired_at_ms=1, + ) + with sqlite3.connect(tmp_path / "source.db") as source_conn: + source_conn.execute( + "UPDATE raw_sessions SET parsed_at_ms = 2, parse_error = ? WHERE raw_id = ?", + ("partial JSONL capture", raw_id), + ) + upsert_raw_artifact( + source_conn, + raw_id, + ArchiveSourceArtifact( + artifact_id=f"{artifact_kind}-{raw_id}", + origin="claude-code-session", + source_path="rollout.jsonl", + source_index=0, + artifact_kind=artifact_kind, + classification_reason=artifact_kind, + support_status=ArtifactSupportStatus.PARTIAL_DECODE, + parse_as_session=True, + schema_eligible=True, + ), + ) + source_conn.commit() + + candidates = repair_mod._raw_materialization_candidate_ids(_config(tmp_path)) + + assert raw_id not in candidates.raw_ids + + +def test_raw_materialization_repairs_deferred_stale_frontier_failure(tmp_path: Path) -> None: + """Durable frontier evidence reaches the real replay actuator.""" from polylogue.core.enums import Provider from polylogue.storage.raw_retention import raw_frontier_integrity_projection from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore @@ -618,6 +837,432 @@ def test_raw_materialization_repairs_legacy_stale_frontier_failure(tmp_path: Pat assert frontier.broken_head_count == 0 +def test_raw_materialization_preserves_bounded_historical_cas_retry_authority(tmp_path: Path) -> None: + """Historical CAS rows remain selectable, while arbitrary prose stays terminal.""" + from polylogue.core.enums import Provider + from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore + from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root + + initialize_active_archive_root(tmp_path) + errors = { + "prefix": "MembershipReplayConflictError: old guard wording", + "frontier": "RuntimeError: raw revision CAS rejected an older accepted frontier", + "byte": "RuntimeError: membership replay cannot replace an unconvertible byte head", + "unrelated": "RuntimeError: parser failed while decoding a session", + } + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_ids = { + name: archive.write_raw_payload( + provider=Provider.CODEX, + payload=f'{{"name":"{name}"}}'.encode(), + source_path=f"{name}.jsonl", + acquired_at_ms=index + 1, + ) + for index, name in enumerate(errors) + } + with sqlite3.connect(tmp_path / "source.db") as source_conn: + source_conn.executemany( + "UPDATE raw_sessions SET parsed_at_ms = 2, parse_error = ? WHERE raw_id = ?", + [(error, raw_ids[name]) for name, error in errors.items()], + ) + source_conn.commit() + + candidates = repair_mod._raw_materialization_candidate_ids(_config(tmp_path)) + + assert set(candidates.raw_ids) == {raw_ids["prefix"], raw_ids["frontier"], raw_ids["byte"]} + + +def test_raw_materialization_terminal_carrier_overrides_legacy_cas_marker(tmp_path: Path) -> None: + """A reviewed terminal carrier blocks legacy-marker replay authority.""" + from polylogue.core.enums import Origin, Provider + from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore + from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root + + initialize_active_archive_root(tmp_path) + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.CODEX, + payload=b'{"name":"reviewed-terminal"}', + source_path="reviewed-terminal.jsonl", + acquired_at_ms=1, + ) + with sqlite3.connect(tmp_path / "source.db") as source_conn: + source_conn.execute( + "UPDATE raw_sessions SET parse_error = ? WHERE raw_id = ?", + ("MembershipReplayConflictError: historical marker", raw_id), + ) + upsert_raw_artifact( + source_conn, + raw_id, + ArchiveSourceArtifact( + artifact_id="reviewed-terminal-carrier", + origin=Origin.CODEX_SESSION, + source_path="reviewed-terminal.jsonl", + source_index=0, + artifact_kind=RawFailureEvidenceKind.TERMINAL_UNSUPPORTED_SHAPE.value, + classification_reason="reviewed terminal disposition", + support_status=ArtifactSupportStatus.UNSUPPORTED_PARSEABLE, + parse_as_session=False, + schema_eligible=False, + first_observed_at_ms=2, + last_observed_at_ms=2, + ), + ) + source_conn.commit() + + assert repair_mod._raw_materialization_candidate_ids(_config(tmp_path)).raw_ids == [] + + +def test_raw_cas_frontier_error_is_typed_transient() -> None: + error = RawCASFrontierError("frontier changed") + + assert error.is_transient is True + + +def test_non_codex_cas_frontier_failure_persists_provider_neutral_evidence(tmp_path: Path) -> None: + from polylogue.core.enums import Provider + from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore + from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root + + initialize_active_archive_root(tmp_path) + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.CLAUDE_CODE, + payload=b'{"type":"session_meta","payload":{"id":"cas-frontier"}}\n', + source_path="rollout.jsonl", + acquired_at_ms=1, + ) + archive.mark_raw_parse_failed( + raw_id, + provider=Provider.CLAUDE_CODE, + error=RawCASFrontierError("frontier changed"), + ) + + with sqlite3.connect(tmp_path / "source.db") as source_conn: + assert source_conn.execute( + "SELECT artifact_kind, support_status, parse_as_session FROM raw_artifacts WHERE raw_id = ?", + (raw_id,), + ).fetchone() == ("deferred_cas_frontier", "partial_decode", 1) + + +def test_generic_parse_state_failure_retires_prior_failure_authority(tmp_path: Path) -> None: + """An untyped retained-raw failure cannot reuse an earlier replay carrier.""" + from polylogue.core.enums import Provider + from polylogue.storage.raw.models import RawSessionStateUpdate + from polylogue.storage.raw_failure_lifecycle import read_raw_failure_lifecycle + from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore + from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root + + initialize_active_archive_root(tmp_path) + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.CODEX, + payload=b'{"type":"session_meta","payload":{"id":"stale-authority"}}\n', + source_path="stale-authority.jsonl", + acquired_at_ms=1, + ) + archive.mark_raw_parse_failed( + raw_id, + provider=Provider.CODEX, + error=RawCASFrontierError("first frontier"), + ) + archive.finalize_raw_parse_state( + raw_id, + state=RawSessionStateUpdate( + parse_error="ValueError: later parser failure", + payload_provider=Provider.CODEX, + ), + ) + + with sqlite3.connect(tmp_path / "source.db") as source_conn: + assert source_conn.execute( + "SELECT artifact_kind, support_status, parse_as_session FROM raw_artifacts WHERE raw_id = ?", + (raw_id,), + ).fetchone() == ( + RawFailureEvidenceKind.TERMINAL_SUPERSEDED_DEFERRED_CAS_FRONTIER.value, + "unknown", + 0, + ) + + lifecycle = read_raw_failure_lifecycle(tmp_path / "source.db") + assert lifecycle.terminal == 0 + assert lifecycle.deferred == 0 + assert lifecycle.unexplained == 1 + assert repair_mod._raw_materialization_candidate_ids(_config(tmp_path)).raw_ids == [] + + +def test_failed_raw_lifecycle_preserves_exact_evidence_for_same_coordinate( + tmp_path: Path, +) -> None: + """Two retained failures at one coordinate keep independent replay authority.""" + from polylogue.core.enums import Provider + from polylogue.storage.raw_failure_lifecycle import read_raw_failure_lifecycle + from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore + from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root + + initialize_active_archive_root(tmp_path) + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + old_raw_id = archive.write_raw_payload( + provider=Provider.CODEX, + payload=b'{"revision":"old"}', + source_path="same-coordinate.jsonl", + source_index=0, + acquired_at_ms=1, + ) + new_raw_id = archive.write_raw_payload( + provider=Provider.CODEX, + payload=b'{"revision":"new"}', + source_path="same-coordinate.jsonl", + source_index=0, + acquired_at_ms=2, + ) + archive.mark_raw_parse_failed( + old_raw_id, + provider=Provider.CODEX, + error=RawCASFrontierError("old frontier"), + ) + archive.mark_raw_parse_failed( + new_raw_id, + provider=Provider.CODEX, + error=RawCASFrontierError("new frontier"), + ) + + with sqlite3.connect(tmp_path / "source.db") as source_conn: + rows = source_conn.execute( + """ + SELECT raw_id, origin, source_path, source_index, artifact_kind, support_status + FROM raw_artifacts + WHERE source_path = 'same-coordinate.jsonl' + ORDER BY raw_id + """ + ).fetchall() + assert {tuple(row) for row in rows} == { + ( + old_raw_id, + "codex-session", + "same-coordinate.jsonl", + 0, + RawFailureEvidenceKind.DEFERRED_CAS_FRONTIER.value, + "partial_decode", + ), + ( + new_raw_id, + "codex-session", + "same-coordinate.jsonl", + 0, + RawFailureEvidenceKind.DEFERRED_CAS_FRONTIER.value, + "partial_decode", + ), + } + + lifecycle = read_raw_failure_lifecycle(tmp_path / "source.db") + assert lifecycle.deferred == 2 + assert lifecycle.unexplained == 0 + assert {sample["raw_id"] for sample in lifecycle.samples} == {old_raw_id, new_raw_id} + candidates = repair_mod._raw_materialization_candidate_ids(_config(tmp_path)) + assert set(candidates.raw_ids) == {old_raw_id, new_raw_id} + + +def test_failed_raw_lifecycle_ignores_newer_ordinary_artifact_at_same_coordinate( + tmp_path: Path, +) -> None: + """A newer ordinary observation cannot hide a valid closed failure carrier.""" + from polylogue.core.enums import Origin + from polylogue.storage.raw_failure_lifecycle import read_raw_failure_lifecycle + from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root + from polylogue.storage.sqlite.archive_tiers.source_write import write_source_raw_session + + initialize_active_archive_root(tmp_path) + with sqlite3.connect(tmp_path / "source.db") as conn: + raw_id = write_source_raw_session( + conn, + origin=Origin.CODEX_SESSION, + source_path="coexisting.jsonl", + source_index=4, + payload=b"unsupported", + acquired_at_ms=1, + parse_error="worker rejected shape", + ) + upsert_raw_artifact( + conn, + raw_id, + ArchiveSourceArtifact( + artifact_id="failure-carrier", + origin=Origin.CODEX_SESSION, + source_path="coexisting.jsonl", + source_index=4, + artifact_kind=RawFailureEvidenceKind.TERMINAL_UNSUPPORTED_SHAPE.value, + classification_reason=RawFailureEvidenceKind.TERMINAL_UNSUPPORTED_SHAPE.value, + support_status=ArtifactSupportStatus.UNSUPPORTED_PARSEABLE, + first_observed_at_ms=10, + last_observed_at_ms=10, + ), + ) + upsert_raw_artifact( + conn, + raw_id, + ArchiveSourceArtifact( + artifact_id="ordinary-carrier", + origin=Origin.CODEX_SESSION, + source_path="coexisting.jsonl", + source_index=4, + artifact_kind="session_export", + classification_reason="ordinary re-observation", + support_status=ArtifactSupportStatus.SUPPORTED_PARSEABLE, + first_observed_at_ms=20, + last_observed_at_ms=20, + ), + ) + + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT COUNT(*) FROM raw_artifacts WHERE raw_id = ?", (raw_id,)).fetchone() == (2,) + + lifecycle = read_raw_failure_lifecycle(tmp_path / "source.db") + assert lifecycle.terminal == 1 + assert lifecycle.unexplained == 0 + assert lifecycle.blocking is False + assert lifecycle.state == "degraded" + assert lifecycle.samples[0]["artifact_kind"] == RawFailureEvidenceKind.TERMINAL_UNSUPPORTED_SHAPE.value + + +def test_cas_failure_evidence_rolls_back_with_parse_state( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A failed parse-state write cannot leave a committed CAS evidence receipt.""" + from polylogue.core.enums import Provider + from polylogue.storage.sqlite.archive_tiers import revision_governance + from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore + from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root + + initialize_active_archive_root(tmp_path) + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.CODEX, + payload=b'{"revision":"atomic"}', + source_path="atomic.jsonl", + acquired_at_ms=1, + ) + + def fail_state_update(*_args: object, **_kwargs: object) -> None: + raise RuntimeError("state update failed") + + monkeypatch.setattr(revision_governance, "apply_source_raw_state_update", fail_state_update) + with pytest.raises(RuntimeError, match="state update failed"): + archive.mark_raw_parse_failed( + raw_id, + provider=Provider.CODEX, + error=RawCASFrontierError("frontier"), + ) + + with sqlite3.connect(tmp_path / "source.db") as source_conn: + assert source_conn.execute("SELECT parse_error FROM raw_sessions WHERE raw_id = ?", (raw_id,)).fetchone() == ( + None, + ) + assert source_conn.execute("SELECT COUNT(*) FROM raw_artifacts WHERE raw_id = ?", (raw_id,)).fetchone() == (0,) + + +def test_deferred_cas_evidence_is_superseded_after_resolution_and_non_cas_failure(tmp_path: Path) -> None: + """Resolved deferred evidence cannot authorize a later replay attempt.""" + from polylogue.core.enums import Provider + from polylogue.storage.raw_failure_lifecycle import read_raw_failure_lifecycle + from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore + from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root + + initialize_active_archive_root(tmp_path) + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_success = archive.write_raw_payload( + provider=Provider.CODEX, + payload=b'{"name":"success"}', + source_path="success.jsonl", + acquired_at_ms=1, + ) + raw_failure = archive.write_raw_payload( + provider=Provider.CODEX, + payload=b'{"name":"failure"}', + source_path="failure.jsonl", + acquired_at_ms=2, + ) + with sqlite3.connect(tmp_path / "source.db") as source_conn: + for raw_id, source_path, neighbor_path in ( + (raw_success, "success.jsonl", "success-neighbor.jsonl"), + (raw_failure, "failure.jsonl", "failure-neighbor.jsonl"), + ): + for artifact_id, path in ((f"deferred-{raw_id}", source_path), (f"neighbor-{raw_id}", neighbor_path)): + upsert_raw_artifact( + source_conn, + raw_id, + ArchiveSourceArtifact( + artifact_id=artifact_id, + origin="codex-session", + source_path=path, + source_index=0, + artifact_kind="deferred_cas_frontier", + classification_reason="deferred_cas_frontier", + support_status=ArtifactSupportStatus.PARTIAL_DECODE, + parse_as_session=True, + schema_eligible=True, + ), + ) + source_conn.commit() + + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + archive.mark_raw_parse_succeeded(raw_success, provider=Provider.CODEX) + archive.mark_raw_parse_failed( + raw_failure, + provider=Provider.CODEX, + error=ValueError("unrelated parser failure"), + ) + + with sqlite3.connect(tmp_path / "source.db") as source_conn: + source_conn.execute( + "UPDATE raw_sessions SET parsed_at_ms = 3, parse_error = ? WHERE raw_id = ?", + ("later unrelated parser failure", raw_success), + ) + source_conn.commit() + observations = source_conn.execute( + """ + SELECT raw_id, source_path, artifact_kind, support_status + FROM raw_artifacts + WHERE raw_id IN (?, ?) + ORDER BY raw_id, source_path + """, + (raw_success, raw_failure), + ).fetchall() + + assert {tuple(row) for row in observations if row[1].endswith("neighbor.jsonl")} == { + (raw_failure, "failure-neighbor.jsonl", "deferred_cas_frontier", "partial_decode"), + (raw_success, "success-neighbor.jsonl", "deferred_cas_frontier", "partial_decode"), + } + assert {(row[0], row[1], row[2], row[3]) for row in observations if not row[1].endswith("neighbor.jsonl")} == { + ( + raw_failure, + "failure.jsonl", + RawFailureEvidenceKind.TERMINAL_SUPERSEDED_DEFERRED_CAS_FRONTIER.value, + "unknown", + ), + ( + raw_success, + "success.jsonl", + RawFailureEvidenceKind.TERMINAL_SUPERSEDED_DEFERRED_CAS_FRONTIER.value, + "unknown", + ), + } + + lifecycle = read_raw_failure_lifecycle(tmp_path / "source.db") + # The two CAS replacement receipts are bound, non-failure resolutions. + # The later unrelated parser failures therefore remain unexplained rather + # than being hidden behind a stale success carrier. + assert lifecycle.terminal == 0 + assert lifecycle.deferred == 0 + assert lifecycle.unexplained == 2 + status = raw_failure_info_for_root(tmp_path) + assert status["terminal_rejections"] == 0 + assert status["unexplained_failures"] == 2 + assert repair_mod._raw_materialization_candidate_ids(_config(tmp_path)).raw_ids == [] + assert repair_mod.raw_materialization_replay_backlog(_config(tmp_path))["candidate_count"] == 0 + + def test_raw_materialization_split_root_classifies_parsed_sidecar_from_routed_blob(tmp_path: Path) -> None: configured_root = tmp_path / "configured" routed_root = tmp_path / "routed" diff --git a/tests/unit/storage/test_revision_replay.py b/tests/unit/storage/test_revision_replay.py index 5c04acb05f..c2feb5d57f 100644 --- a/tests/unit/storage/test_revision_replay.py +++ b/tests/unit/storage/test_revision_replay.py @@ -27,6 +27,7 @@ classify_membership_revisions, ) from polylogue.core.enums import Provider +from polylogue.core.raw_failure_evidence import RawFailureEvidenceKind from polylogue.pipeline.ids import session_content_hash, session_revision_projection from polylogue.sources.dispatch import merge_parsed_session_chunks, parse_stream_payload from polylogue.sources.parsers.base import ParsedAttachment, ParsedMessage, ParsedSession @@ -1476,6 +1477,79 @@ def _apply_membership_head(archive: ArchiveStore, raw_id: str, session: ParsedSe ) +def test_batched_membership_success_supersedes_deferred_cas_evidence(tmp_path: Path) -> None: + """The positive commit-batch route must expire CAS retry authority too.""" + initialize_active_archive_root(tmp_path) + session = _parsed_session(("m0", "batched success")) + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = _write_quarantined_member(archive, "batched-cas", session) + archive.record_raw_failure_evidence( + raw_id, + provider=Provider.CODEX, + source_path="batched-cas.json", + source_index=0, + acquired_at_ms=2, + kind=RawFailureEvidenceKind.DEFERRED_CAS_FRONTIER, + ) + archive.apply_raw_membership_classification( + "codex:session", + MembershipClassification((raw_id,), (), ()), + {raw_id: session}, + {raw_id: session_revision_projection(session)}, + acquired_at_ms=3, + manage_transaction=False, + ) + archive.commit() + + artifact = ( + archive._ensure_source_conn() + .execute( + "SELECT artifact_kind FROM raw_artifacts WHERE raw_id = ? AND source_path = ?", + (raw_id, "batched-cas.json"), + ) + .fetchone() + ) + + assert artifact == (RawFailureEvidenceKind.TERMINAL_SUPERSEDED_DEFERRED_CAS_FRONTIER.value,) + + +def test_retained_index_cas_failure_persists_evidence_with_first_failure_state( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A retained-raw CAS failure cannot commit an untyped state first.""" + initialize_active_archive_root(tmp_path) + session = _parsed_session(("m0", "retained CAS failure")) + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = _write_quarantined_member(archive, "retained-cas-failure", session) + + def raise_conflict(*_args: object, **_kwargs: object) -> None: + raise archive_revision_governance.MembershipReplayConflictError("retained membership conflict") + + monkeypatch.setattr(archive_revision_governance, "_write_parsed_precedence_result", raise_conflict) + with pytest.raises(archive_revision_governance.MembershipReplayConflictError): + archive._index_parsed_for_retained_raw( + session, + raw_id=raw_id, + source_index=0, + stage_timings_s=None, + stage_timing_prefix="test", + manage_transaction=False, + preacquired_attachment_blobs={}, + finalize_raw_parse=False, + revision_authoritative=True, + ) + + with sqlite3.connect(tmp_path / "source.db") as source_conn: + assert source_conn.execute("SELECT parse_error FROM raw_sessions WHERE raw_id = ?", (raw_id,)).fetchone() == ( + "MembershipReplayConflictError: retained membership conflict", + ) + assert source_conn.execute( + "SELECT artifact_kind, support_status, parse_as_session FROM raw_artifacts " + "WHERE raw_id = ? ORDER BY artifact_id DESC LIMIT 1", + (raw_id,), + ).fetchone() == ("deferred_cas_frontier", "partial_decode", 1) + + def _head_row(archive: ArchiveStore) -> tuple[object, ...] | None: row = archive._conn.execute( """SELECT accepted_raw_id, accepted_frontier_kind, accepted_frontier