From e086b72cf98db8591e22bf4a650b1ca070ec68f9 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 30 Jul 2026 14:53:58 +0200 Subject: [PATCH 1/8] fix(archive): stop reading export volatility as revision divergence Extends the identity/acquisition split from polylogue-bu1i to two more volatility sources: message array order and provider-reported generation duration. Both currently masquerade as branches and quarantine cohorts that are byte-identical in every way that matters. SessionRevisionProjection gains message_identities/message_contents (order-insensitive id-to-content pairs) and event_identity_hashes (measurement fields stripped from events whose own payload declares duration_semantics == "provider_reported_elapsed"). session_hash is unchanged -- it still covers order and the full event payload, so a real reorder or a real duration change still triggers a re-write. Only the revision-comparison axes in session_revision_membership.py are tolerant. Ref polylogue-c429 Ref polylogue-nuec Co-Authored-By: Claude --- docs/plans/hash-boundary-registry.yaml | 18 +++ .../archive/session_revision_membership.py | 54 ++++++-- polylogue/pipeline/ids.py | 120 +++++++++++++++++- .../test_session_revision_membership.py | 6 + 4 files changed, 183 insertions(+), 15 deletions(-) diff --git a/docs/plans/hash-boundary-registry.yaml b/docs/plans/hash-boundary-registry.yaml index 04c0e3fbd4..7ac8d92cfd 100644 --- a/docs/plans/hash-boundary-registry.yaml +++ b/docs/plans/hash-boundary-registry.yaml @@ -542,6 +542,24 @@ entries: occurrence: 2 classification: content-hash note: 'feeds session_content_hash / revision projection (2026-07-09 hash-boundary census (docs/audits/2026-07-09-hash-boundary-census.md), Table 1 row 1).' +- path: polylogue/pipeline/ids.py + function: '.session_revision_projection' + call: hash_payload + occurrence: 3 + classification: content-hash + note: 'event_hashes -- feeds session_content_hash / revision projection, unstripped and order-preserving (polylogue-nuec).' +- path: polylogue/pipeline/ids.py + function: '.session_revision_projection' + call: hash_payload + occurrence: 4 + classification: content-hash + note: 'event_identity_hashes -- revision-comparison axis only, provider-reported-elapsed measurement excluded per _event_identity_hash_payload (polylogue-nuec).' +- path: polylogue/pipeline/ids.py + function: '._event_identity_hash_payload' + call: hash_payload + occurrence: 0 + classification: content-hash + note: 'per-event identity hash feeding event_identity_hashes, measurement fields stripped for provider_reported_elapsed events (polylogue-nuec).' - path: polylogue/scenarios/workload.py function: '._identity' call: hashlib.sha256 diff --git a/polylogue/archive/session_revision_membership.py b/polylogue/archive/session_revision_membership.py index 707e5f327c..61e70c42bc 100644 --- a/polylogue/archive/session_revision_membership.py +++ b/polylogue/archive/session_revision_membership.py @@ -8,10 +8,14 @@ from polylogue.core.timestamps import parse_timestamp from polylogue.pipeline.ids import SessionRevisionProjection -#: Everything that must agree for two revisions to be the same content: the -#: message and event chains, which attachments exist, and which of their bytes -#: have been read. -_ContentKey: TypeAlias = tuple[tuple[bytes, ...], tuple[bytes, ...], frozenset[bytes], frozenset[tuple[bytes, bytes]]] +#: Everything that must agree for two revisions to be the same content: which +#: messages exist and what they say (order-insensitive -- polylogue-c429), +#: the event chain with provider-reported measurement excluded +#: (polylogue-nuec), which attachments exist, and which of their bytes have +#: been read. +_ContentKey: TypeAlias = tuple[ + frozenset[tuple[bytes, bytes]], tuple[bytes, ...], frozenset[bytes], frozenset[tuple[bytes, bytes]] +] @dataclass(frozen=True, slots=True) @@ -44,8 +48,8 @@ def classify_membership_revisions(revisions: list[MembershipRevision]) -> Member for revision in revisions: projection = revision.projection key = ( - projection.message_hashes, - projection.event_hashes, + projection.message_contents, + projection.event_identity_hashes, projection.attachment_identities, projection.attachment_contents, ) @@ -206,13 +210,36 @@ def _frontier(projection: SessionRevisionProjection) -> tuple[int, int, int, int dominate (polylogue-bu1i). """ return ( - len(projection.message_hashes), - len(projection.event_hashes), + len(projection.message_contents), + len(projection.event_identity_hashes), len(projection.attachment_identities), len(projection.attachment_contents), ) +def _message_evidence_preserved( + older: SessionRevisionProjection, + newer: SessionRevisionProjection, +) -> bool: + """True when ``newer`` loses no message identity and contradicts no shared content. + + A provider's export ordering is not guaranteed stable across separate + export requests for the SAME conversation -- Claude.ai's own tree + flattening can interleave edited-message siblings differently from one + export to the next even though every message's id, role, text, and + timestamp are byte-identical (polylogue-c429). Array position is + therefore not treated as identity here: this checks only that every + message id present in ``older`` is still present in ``newer`` and still + maps to the same content, mirroring ``_attachment_evidence_preserved``'s + identity/content split. An id whose content actually changed, or an id + that disappeared, is real divergence and is refused. + """ + newer_contents = dict(newer.message_contents) + return older.message_identities <= newer.message_identities and all( + newer_contents.get(identity) == content for identity, content in older.message_contents + ) + + def _attachment_evidence_preserved( older: SessionRevisionProjection, newer: SessionRevisionProjection, @@ -235,8 +262,11 @@ def _attachment_evidence_preserved( def _strictly_dominates(older: SessionRevisionProjection, newer: SessionRevisionProjection) -> bool: content_grew = ( - len(newer.message_hashes) > len(older.message_hashes) - or len(newer.event_hashes) > len(older.event_hashes) + # Order-insensitive: a permuted-but-otherwise-equal message set is not + # growth (equal count), but a genuinely appended or resurfaced message + # id is (polylogue-c429). + len(newer.message_contents) > len(older.message_contents) + or len(newer.event_identity_hashes) > len(older.event_identity_hashes) or newer.attachment_identities > older.attachment_identities # Resolving the bytes of an already-referenced attachment is growth in # evidence even when the transcript is untouched, which is exactly the @@ -245,8 +275,8 @@ def _strictly_dominates(older: SessionRevisionProjection, newer: SessionRevision ) return ( content_grew - and older.message_hashes == newer.message_hashes[: len(older.message_hashes)] - and older.event_hashes == newer.event_hashes[: len(older.event_hashes)] + and _message_evidence_preserved(older, newer) + and older.event_identity_hashes == newer.event_identity_hashes[: len(older.event_identity_hashes)] and _attachment_evidence_preserved(older, newer) ) diff --git a/polylogue/pipeline/ids.py b/polylogue/pipeline/ids.py index c15f1d2f8e..8c402d1d33 100644 --- a/polylogue/pipeline/ids.py +++ b/polylogue/pipeline/ids.py @@ -23,7 +23,7 @@ # deferring the real import to whichever caller actually needs `sources`. if TYPE_CHECKING: from polylogue.sources import ParsedMessage, ParsedSession - from polylogue.sources.parsers.base import ParsedAttachment, ParsedContentBlock + from polylogue.sources.parsers.base import ParsedAttachment, ParsedContentBlock, ParsedSessionEvent # Sentinel values to distinguish None from empty in hash computations _NULL_SENTINEL = "__POLYLOGUE_NULL__" @@ -50,13 +50,40 @@ class SessionRevisionProjection: earlier one and the whole cohort was quarantined as ambiguous. Splitting the axes lets a dominance test say what is actually true -- same attachments, strictly more of their bytes now known (polylogue-bu1i). + + Messages get the same identity/content split, for the same reason applied + to a different volatility source: a provider's own export can replay an + unchanged message set in a different array sequence across separate export + requests (Claude.ai's own tree flattening is not guaranteed to serialize + the same way twice). ``message_identities`` answers *which message is + this* (its provider message id); ``message_contents`` pairs that identity + with a hash of its content (role/text/timestamp/blocks). Order lives only + in ``message_hashes`` (kept for ``session_hash`` and diagnostics) -- the + identity/content axes are order-insensitive by construction, so a bare + permutation of the same id-to-content mapping is not read as divergence + (polylogue-c429). + + ``event_identity_hashes`` strips designated provider-reported-measurement + fields (see ``_PROVIDER_REPORTED_ELAPSED_VOLATILE_PAYLOAD_KEYS``) out of + events whose own payload declares them non-durable + (``duration_semantics == "provider_reported_elapsed"``) before hashing. + ChatGPT's ``generation_lifecycle`` event re-derives its + ``elapsed_duration_ms`` from the raw export's own timing metadata on every + export request, and that value is not stable across requests for the same + generation -- folding it into revision identity made byte-identical + conversations look like divergent branches on every re-export + (polylogue-nuec). ``event_hashes`` (unstripped, order-preserving) remains + unchanged for ``session_hash`` and diagnostics. """ session_hash: bytes message_hashes: tuple[bytes, ...] + message_identities: frozenset[bytes] + message_contents: frozenset[tuple[bytes, bytes]] attachment_identities: frozenset[bytes] attachment_contents: frozenset[tuple[bytes, bytes]] event_hashes: tuple[bytes, ...] + event_identity_hashes: tuple[bytes, ...] def _normalize_nested_for_hash(value: object) -> object: @@ -169,6 +196,23 @@ def _message_hash_payload(message: ParsedMessage, message_id: str) -> dict[str, return payload +#: The one field of a message hash payload that answers *which message is +#: this*, as opposed to *what does it currently say*. A provider's own +#: message id is stable across re-exports even when the export's array +#: ordering is not (polylogue-c429). +_MESSAGE_IDENTITY_FIELDS = ("id",) + + +def _message_identity_payload(payload: dict[str, JSONValue]) -> dict[str, JSONValue]: + """Project the order-independent identity of one message payload. + + Reads the already-normalized value out of ``_message_hash_payload`` rather + than re-deriving it, mirroring ``_attachment_identity_payload``'s single + normalization site. + """ + return {field: payload[field] for field in _MESSAGE_IDENTITY_FIELDS} + + #: Fields of an attachment hash payload that answer *which attachment is this*, #: as opposed to *what have we managed to read about it*. ``size_bytes`` is #: excluded on purpose: for lazily-fetched attachments (Drive/Gemini references, @@ -203,6 +247,51 @@ def _attachment_identity_payload(payload: dict[str, JSONValue]) -> dict[str, JSO return {field: payload[field] for field in _ATTACHMENT_IDENTITY_FIELDS} +#: `generation_lifecycle` payload keys that are provider-reported measurement, +#: not identity, when the event's own payload declares them non-durable via +#: ``duration_semantics == "provider_reported_elapsed"``. ChatGPT re-derives +#: these from the raw export's own ``finished_duration_sec`` / +#: ``reasoning_start_time``/``reasoning_end_time`` metadata on every export +#: request, and the value is not stable across requests for the SAME +#: generation (observed varying non-monotonically, e.g. 13000 vs 21000ms; +#: 123000 vs 33000ms) even when the transcript is byte-identical +#: (polylogue-nuec). +_PROVIDER_REPORTED_ELAPSED_VOLATILE_PAYLOAD_KEYS = frozenset({"elapsed_duration_ms", "started_at_ms", "ended_at_ms"}) +_PROVIDER_REPORTED_ELAPSED_MARKER_KEY = "duration_semantics" +_PROVIDER_REPORTED_ELAPSED_MARKER_VALUE = "provider_reported_elapsed" + + +def _event_identity_hash_payload(event: ParsedSessionEvent, event_index: int) -> dict[str, JSONValue]: + """Build an event hash payload with provider-reported-elapsed measurement excluded. + + Mirrors ``_attachment_identity_payload``'s identity/acquisition split for a + different volatility source: an event that labels itself + ``duration_semantics: "provider_reported_elapsed"`` carries a measurement, + not content, in ``_PROVIDER_REPORTED_ELAPSED_VOLATILE_PAYLOAD_KEYS`` -- + stripped here before hashing. The event's own ``timestamp`` is stripped + alongside it for the same events: ChatGPT sets it from the same + ``reasoning_end_time`` value the duration is derived from, so it varies in + tandem and is measurement too, not identity. ``session_hash`` still covers + the full, unstripped payload and timestamp (see ``session_hash_payload``), + so a real change in reported duration still triggers a re-write; only the + *revision comparison* axis (``event_identity_hashes``) is tolerant. + """ + payload = event.payload + timestamp = event.timestamp + if payload.get(_PROVIDER_REPORTED_ELAPSED_MARKER_KEY) == _PROVIDER_REPORTED_ELAPSED_MARKER_VALUE: + payload = { + key: value for key, value in payload.items() if key not in _PROVIDER_REPORTED_ELAPSED_VOLATILE_PAYLOAD_KEYS + } + timestamp = None + return { + "event_index": event_index, + "event_type": _normalize_for_hash(event.event_type), + "timestamp": _normalize_for_hash(timestamp), + "source_message_provider_id": _normalize_for_hash(event.source_message_provider_id), + "payload": hash_payload(_normalize_nested_for_hash(payload)), + } + + def _session_hash_payload( *, title: str | None, @@ -312,6 +401,14 @@ def session_revision_projection(convo: ParsedSession) -> SessionRevisionProjecti included, so acquiring an attachment's bytes does change the session's content hash and does trigger a re-write. Only the *revision comparison* axes separate identity from acquisition (polylogue-bu1i). + + The same holds for message order (polylogue-c429) and provider-reported + generation-duration measurement (polylogue-nuec): ``session_hash`` still + covers the full, order-sensitive message array and the full, + unstripped event payload/timestamp, so a real reorder or a real duration + change still triggers a re-write. Only ``message_identities`` / + ``message_contents`` / ``event_identity_hashes`` -- the *revision + comparison* axes -- are tolerant of the volatility each bug describes. """ messages_payload, attachments_payload, session_events_payload = _session_hash_components(convo) session_hash_hex = _session_tree_hash( @@ -320,6 +417,15 @@ def session_revision_projection(convo: ParsedSession) -> SessionRevisionProjecti attachments_payload=attachments_payload, session_events_payload=session_events_payload, ) + message_identities: set[bytes] = set() + message_contents: set[tuple[bytes, bytes]] = set() + message_hashes: list[bytes] = [] + for payload in messages_payload: + identity = bytes.fromhex(hash_payload(_message_identity_payload(payload))) + content = bytes.fromhex(hash_payload(payload)) + message_identities.add(identity) + message_contents.add((identity, content)) + message_hashes.append(content) attachment_identities: set[bytes] = set() attachment_contents: set[tuple[bytes, bytes]] = set() for payload in attachments_payload: @@ -328,10 +434,18 @@ def session_revision_projection(convo: ParsedSession) -> SessionRevisionProjecti inline_content_hash = payload.get("inline_content_hash") if isinstance(inline_content_hash, str): attachment_contents.add((identity, bytes.fromhex(inline_content_hash))) + event_hashes: list[bytes] = [] + event_identity_hashes: list[bytes] = [] + for event_index, (payload, event) in enumerate(zip(session_events_payload, convo.session_events, strict=True)): + event_hashes.append(bytes.fromhex(hash_payload(payload))) + event_identity_hashes.append(bytes.fromhex(hash_payload(_event_identity_hash_payload(event, event_index)))) return SessionRevisionProjection( session_hash=bytes.fromhex(session_hash_hex), - message_hashes=tuple(bytes.fromhex(hash_payload(payload)) for payload in messages_payload), + message_hashes=tuple(message_hashes), + message_identities=frozenset(message_identities), + message_contents=frozenset(message_contents), attachment_identities=frozenset(attachment_identities), attachment_contents=frozenset(attachment_contents), - event_hashes=tuple(bytes.fromhex(hash_payload(payload)) for payload in session_events_payload), + event_hashes=tuple(event_hashes), + event_identity_hashes=tuple(event_identity_hashes), ) diff --git a/tests/unit/archive/test_session_revision_membership.py b/tests/unit/archive/test_session_revision_membership.py index 42b0b60e43..08a228a582 100644 --- a/tests/unit/archive/test_session_revision_membership.py +++ b/tests/unit/archive/test_session_revision_membership.py @@ -165,7 +165,10 @@ def test_browser_native_upgrade_refuses_any_shrinking_frontier_dimension() -> No older_projection = older.projection.__class__( session_hash=b"o" * 32, message_hashes=older.projection.message_hashes, + message_identities=older.projection.message_identities, + message_contents=older.projection.message_contents, event_hashes=older.projection.event_hashes, + event_identity_hashes=older.projection.event_identity_hashes, attachment_identities=frozenset({b"attachment"}), attachment_contents=frozenset(), ) @@ -227,7 +230,10 @@ def test_browser_snapshot_accepts_later_attachment_enrichment_without_provider_u newer_projection = newer.projection.__class__( session_hash=b"n" * 32, message_hashes=newer.projection.message_hashes, + message_identities=newer.projection.message_identities, + message_contents=newer.projection.message_contents, event_hashes=newer.projection.event_hashes, + event_identity_hashes=newer.projection.event_identity_hashes, attachment_identities=frozenset({b"attachment-v2"}), attachment_contents=frozenset(), ) From 7905e75ad4f7f4e69c0cbcf693861e4bbc8a04f0 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 30 Jul 2026 15:11:32 +0200 Subject: [PATCH 2/8] test(archive): anti-vacuity tests for c429/nuec revision-comparison fixes Adds mutation-verified coverage for the permutation-tolerant message comparison and duration-tolerant event comparison landed in the previous commit, and drops the unused message_identities projection field (the identity/content split needed for attachments' lazy-fetch axis is redundant for messages, whose content is never lazily missing -- the content-agreement check alone already implies identity subset). Each new clause was mutated and confirmed to fail without the guard it implements (see PR body for the full table); two initially-vacuous tests (message-id-disappearing, message-content-changed under an equal-count reorder) were replaced with growth-variant shapes that actually exercise _message_evidence_preserved rather than being short-circuited by the count-based growth check. Ref polylogue-c429 Ref polylogue-nuec Co-Authored-By: Claude --- .../archive/session_revision_membership.py | 15 +- polylogue/pipeline/ids.py | 30 +- .../test_session_revision_membership.py | 387 +++++++++++++++++- 3 files changed, 406 insertions(+), 26 deletions(-) diff --git a/polylogue/archive/session_revision_membership.py b/polylogue/archive/session_revision_membership.py index 61e70c42bc..6a5bc69c4a 100644 --- a/polylogue/archive/session_revision_membership.py +++ b/polylogue/archive/session_revision_membership.py @@ -229,15 +229,16 @@ def _message_evidence_preserved( export to the next even though every message's id, role, text, and timestamp are byte-identical (polylogue-c429). Array position is therefore not treated as identity here: this checks only that every - message id present in ``older`` is still present in ``newer`` and still - maps to the same content, mirroring ``_attachment_evidence_preserved``'s - identity/content split. An id whose content actually changed, or an id - that disappeared, is real divergence and is refused. + ``(identity, content)`` pair present in ``older`` is still present in + ``newer``. Unlike attachments, a message is never lazily fetched -- its + content is always known when it exists -- so there is no separate + identity-only membership to check: an id that disappeared from ``newer`` + fails this lookup on its own (``.get()`` returns ``None``, which never + equals a real content hash), and an id whose content actually changed + fails it too. Both are real divergence and are refused. """ newer_contents = dict(newer.message_contents) - return older.message_identities <= newer.message_identities and all( - newer_contents.get(identity) == content for identity, content in older.message_contents - ) + return all(newer_contents.get(identity) == content for identity, content in older.message_contents) def _attachment_evidence_preserved( diff --git a/polylogue/pipeline/ids.py b/polylogue/pipeline/ids.py index 8c402d1d33..3729324fff 100644 --- a/polylogue/pipeline/ids.py +++ b/polylogue/pipeline/ids.py @@ -51,17 +51,19 @@ class SessionRevisionProjection: axes lets a dominance test say what is actually true -- same attachments, strictly more of their bytes now known (polylogue-bu1i). - Messages get the same identity/content split, for the same reason applied - to a different volatility source: a provider's own export can replay an + Messages get an analogous split, for the same reason applied to a + different volatility source: a provider's own export can replay an unchanged message set in a different array sequence across separate export requests (Claude.ai's own tree flattening is not guaranteed to serialize - the same way twice). ``message_identities`` answers *which message is - this* (its provider message id); ``message_contents`` pairs that identity - with a hash of its content (role/text/timestamp/blocks). Order lives only - in ``message_hashes`` (kept for ``session_hash`` and diagnostics) -- the - identity/content axes are order-insensitive by construction, so a bare - permutation of the same id-to-content mapping is not read as divergence - (polylogue-c429). + the same way twice). ``message_contents`` pairs each message's identity + hash (its provider message id) with a hash of its content + (role/text/timestamp/blocks) -- unlike an attachment, a message is never + lazily fetched, so there is no separate identity-only axis to track: + ``message_contents`` alone answers both *which messages exist* and *what + do they say*. Order lives only in ``message_hashes`` (kept for + ``session_hash`` and diagnostics) -- ``message_contents`` is + order-insensitive by construction, so a bare permutation of the same + id-to-content mapping is not read as divergence (polylogue-c429). ``event_identity_hashes`` strips designated provider-reported-measurement fields (see ``_PROVIDER_REPORTED_ELAPSED_VOLATILE_PAYLOAD_KEYS``) out of @@ -78,7 +80,6 @@ class SessionRevisionProjection: session_hash: bytes message_hashes: tuple[bytes, ...] - message_identities: frozenset[bytes] message_contents: frozenset[tuple[bytes, bytes]] attachment_identities: frozenset[bytes] attachment_contents: frozenset[tuple[bytes, bytes]] @@ -406,9 +407,9 @@ def session_revision_projection(convo: ParsedSession) -> SessionRevisionProjecti generation-duration measurement (polylogue-nuec): ``session_hash`` still covers the full, order-sensitive message array and the full, unstripped event payload/timestamp, so a real reorder or a real duration - change still triggers a re-write. Only ``message_identities`` / - ``message_contents`` / ``event_identity_hashes`` -- the *revision - comparison* axes -- are tolerant of the volatility each bug describes. + change still triggers a re-write. Only ``message_contents`` / + ``event_identity_hashes`` -- the *revision comparison* axes -- are + tolerant of the volatility each bug describes. """ messages_payload, attachments_payload, session_events_payload = _session_hash_components(convo) session_hash_hex = _session_tree_hash( @@ -417,13 +418,11 @@ def session_revision_projection(convo: ParsedSession) -> SessionRevisionProjecti attachments_payload=attachments_payload, session_events_payload=session_events_payload, ) - message_identities: set[bytes] = set() message_contents: set[tuple[bytes, bytes]] = set() message_hashes: list[bytes] = [] for payload in messages_payload: identity = bytes.fromhex(hash_payload(_message_identity_payload(payload))) content = bytes.fromhex(hash_payload(payload)) - message_identities.add(identity) message_contents.add((identity, content)) message_hashes.append(content) attachment_identities: set[bytes] = set() @@ -442,7 +441,6 @@ def session_revision_projection(convo: ParsedSession) -> SessionRevisionProjecti return SessionRevisionProjection( session_hash=bytes.fromhex(session_hash_hex), message_hashes=tuple(message_hashes), - message_identities=frozenset(message_identities), message_contents=frozenset(message_contents), attachment_identities=frozenset(attachment_identities), attachment_contents=frozenset(attachment_contents), diff --git a/tests/unit/archive/test_session_revision_membership.py b/tests/unit/archive/test_session_revision_membership.py index 08a228a582..57128a55af 100644 --- a/tests/unit/archive/test_session_revision_membership.py +++ b/tests/unit/archive/test_session_revision_membership.py @@ -8,7 +8,7 @@ ) from polylogue.core.enums import Provider from polylogue.pipeline.ids import session_revision_projection -from polylogue.sources.parsers.base import ParsedAttachment, ParsedMessage, ParsedSession +from polylogue.sources.parsers.base import ParsedAttachment, ParsedMessage, ParsedSession, ParsedSessionEvent def _revision(raw_id: str, *texts: str) -> MembershipRevision: @@ -165,7 +165,6 @@ def test_browser_native_upgrade_refuses_any_shrinking_frontier_dimension() -> No older_projection = older.projection.__class__( session_hash=b"o" * 32, message_hashes=older.projection.message_hashes, - message_identities=older.projection.message_identities, message_contents=older.projection.message_contents, event_hashes=older.projection.event_hashes, event_identity_hashes=older.projection.event_identity_hashes, @@ -230,7 +229,6 @@ def test_browser_snapshot_accepts_later_attachment_enrichment_without_provider_u newer_projection = newer.projection.__class__( session_hash=b"n" * 32, message_hashes=newer.projection.message_hashes, - message_identities=newer.projection.message_identities, message_contents=newer.projection.message_contents, event_hashes=newer.projection.event_hashes, event_identity_hashes=newer.projection.event_identity_hashes, @@ -408,3 +406,386 @@ def test_refuses_contradicted_bytes_even_when_the_revision_otherwise_grew() -> N result = classify_membership_revisions([older, newer]) assert result.accepted_raw_ids == () assert result.ambiguous_raw_ids == ("raw-a", "raw-b") + + +def _ordered_message_revision(raw_id: str, *id_text_pairs: tuple[str, str]) -> MembershipRevision: + """A session whose messages carry explicit provider ids in a given array order. + + Unlike ``_revision`` (which derives sequential ids from position), this lets + a test hold the id-to-content mapping fixed while varying only array order. + """ + session = ParsedSession( + source_name=Provider.CLAUDE_AI, + provider_session_id="session", + messages=[ + ParsedMessage(provider_message_id=provider_id, role=Role.USER, text=text) + for provider_id, text in id_text_pairs + ], + ) + return MembershipRevision(raw_id, session_revision_projection(session)) + + +def test_accepts_reordered_message_array_with_identical_content_as_equivalent() -> None: + """Same message ids, byte-identical content per id, different array order. + + Reproduces the exact shape measured on the live archive: Claude.ai's own + export ordering for a conversation is not guaranteed stable across separate + export requests -- 21 of 40 sampled claude-ai-export ambiguous cohorts + (52.5%) had the same 36 message ids with identical {role,text,timestamp} + per id, just resequenced (polylogue-c429). A strict positional-prefix + dominance test refuses both directions and quarantines the whole cohort; + this must instead resolve as equivalent content. + """ + chronological = _ordered_message_revision("raw-chrono", ("a", "one"), ("b", "two"), ("c", "three")) + resequenced = _ordered_message_revision("raw-resequenced", ("b", "two"), ("a", "one"), ("c", "three")) + + assert chronological.projection.message_hashes != resequenced.projection.message_hashes + assert chronological.projection.message_contents == resequenced.projection.message_contents + + # Same-content-key revisions still need a provider timestamp to pick a + # representative (matching the pre-existing metadata_variants mechanism, + # e.g. test_metadata_only_revision_uses_latest_provider_timestamp) -- the + # permutation-tolerant key gets them INTO that mechanism at all, which is + # the fix; it does not bypass the timestamp requirement. + chronological = MembershipRevision(chronological.raw_id, chronological.projection, "2026-01-01T00:00:00Z") + resequenced = MembershipRevision(resequenced.raw_id, resequenced.projection, "2026-01-02T00:00:00Z") + + result = classify_membership_revisions([chronological, resequenced]) + + assert result.accepted_raw_ids == ("raw-resequenced",) + assert result.equivalent_raw_ids == ("raw-chrono",) + assert result.ambiguous_raw_ids == () + + +def test_accepts_message_growth_across_a_reordered_prefix() -> None: + """A genuinely appended message is still growth even when the shared ids reorder. + + The permutation tolerance must not swallow real append-only growth: the + newer revision keeps every id-to-content pair from the older revision (just + resequenced) and adds one new id. + """ + older = _ordered_message_revision("raw-old", ("b", "two"), ("a", "one")) + newer = _ordered_message_revision("raw-new", ("a", "one"), ("c", "three"), ("b", "two")) + + assert _strictly_dominates(older.projection, newer.projection) + + result = classify_membership_revisions([older, newer]) + assert result.accepted_raw_ids == ("raw-old", "raw-new") + assert result.ambiguous_raw_ids == () + + +def test_refuses_message_content_change_under_a_shared_id_despite_reorder() -> None: + """Permutation tolerance must not launder a real content change. + + Same id set, resequenced, but one shared id's text actually differs -- that + is genuine divergence (an edit, not export nondeterminism) and must stay + ambiguous, not be waved through by the order-insensitive comparison. + """ + older = _ordered_message_revision("raw-old", ("a", "one"), ("b", "two")) + newer = _ordered_message_revision("raw-new", ("b", "two"), ("a", "EDITED")) + + assert not _strictly_dominates(older.projection, newer.projection) + assert not _strictly_dominates(newer.projection, older.projection) + + result = classify_membership_revisions([older, newer]) + assert result.accepted_raw_ids == () + assert result.ambiguous_raw_ids == ("raw-new", "raw-old") + + +def test_refuses_message_content_change_even_when_the_revision_otherwise_grew() -> None: + """Growth elsewhere must not launder a contradicted message. + + The equal-count case above is already refused by the growth test itself + (``content_grew`` is ``False`` on both sides, since a same-id edit doesn't + change the message count), which leaves the content-agreement clause in + ``_message_evidence_preserved`` unexercised and therefore unproven -- the + same trap the disappearing-id growth test above closes for the identity + axis. This is the shape that genuinely needs it: the newer revision has + one more message than the older one (real growth) while also rewriting + the text under a shared id. + """ + older = _ordered_message_revision("raw-old", ("a", "one"), ("b", "two")) + newer = _ordered_message_revision("raw-new", ("a", "EDITED"), ("b", "two"), ("c", "three")) + + # The count really did grow, so the refusal cannot come from there. + assert len(newer.projection.message_contents) > len(older.projection.message_contents) + assert not _strictly_dominates(older.projection, newer.projection) + + result = classify_membership_revisions([older, newer]) + assert result.accepted_raw_ids == () + assert result.ambiguous_raw_ids == ("raw-new", "raw-old") + + +def test_refuses_message_id_disappearing_despite_reorder() -> None: + """A message id present in the older revision but absent from the newer one + is a real loss, not a reorder -- must stay ambiguous even though the + remaining ids' content and count could otherwise look like a permutation. + """ + older = _ordered_message_revision("raw-old", ("a", "one"), ("b", "two")) + newer = _ordered_message_revision("raw-new", ("b", "two"), ("c", "three")) + + assert not _strictly_dominates(older.projection, newer.projection) + assert not _strictly_dominates(newer.projection, older.projection) + + result = classify_membership_revisions([older, newer]) + assert result.accepted_raw_ids == () + assert result.ambiguous_raw_ids == ("raw-new", "raw-old") + + +def test_refuses_message_id_disappearing_even_when_the_revision_otherwise_grew() -> None: + """Growth elsewhere must not launder a lost message id. + + The equal-count case above is already refused by the growth test itself + (``content_grew`` is ``False`` on both sides), which leaves the identity- + subset clause in ``_message_evidence_preserved`` unexercised and therefore + unproven -- the same trap ``test_refuses_contradicted_bytes_even_when_the_ + revision_otherwise_grew`` closes for attachments. This is the shape that + genuinely needs it: the newer revision has one more message than the older + one (real growth on the count axis) while dropping an id the older + revision had. Accepting that would silently discard a real message under + cover of a legitimate-looking frontier advance. + """ + older = _ordered_message_revision("raw-old", ("a", "one"), ("b", "two")) + newer = _ordered_message_revision("raw-new", ("a", "one"), ("c", "three"), ("d", "four")) + + # The count really did grow, so the refusal cannot come from there. + assert len(newer.projection.message_contents) > len(older.projection.message_contents) + assert not _strictly_dominates(older.projection, newer.projection) + + result = classify_membership_revisions([older, newer]) + assert result.accepted_raw_ids == () + assert result.ambiguous_raw_ids == ("raw-new", "raw-old") + + +def test_message_reorder_does_change_the_session_content_hash() -> None: + """Order tolerance must not leak into idempotency. + + ``session_hash`` must stay order-sensitive: if two array orderings of the + same messages hashed identically, a real reorder written by the archive's + own writer path (a legitimate content change worth re-indexing) would be + silently skipped as unchanged on re-ingest. + """ + chronological = _ordered_message_revision("raw-chrono", ("a", "one"), ("b", "two")) + resequenced = _ordered_message_revision("raw-resequenced", ("b", "two"), ("a", "one")) + + assert chronological.projection.session_hash != resequenced.projection.session_hash + assert {identity for identity, _content in chronological.projection.message_contents} == { + identity for identity, _content in resequenced.projection.message_contents + } + + +def _generation_lifecycle_revision(raw_id: str, elapsed_duration_ms: int) -> MembershipRevision: + """A ChatGPT-shaped session with one `generation_lifecycle` event. + + Mirrors the exact shape ``polylogue/sources/parsers/chatgpt.py`` emits: + ``duration_semantics: "provider_reported_elapsed"`` marking + ``elapsed_duration_ms`` as provider-remeasured evidence, not identity. + """ + session = ParsedSession( + source_name=Provider.CHATGPT, + provider_session_id="session", + messages=[ParsedMessage(provider_message_id="0", role=Role.ASSISTANT, text="answer")], + session_events=[ + ParsedSessionEvent( + event_type="generation_lifecycle", + timestamp=str(elapsed_duration_ms / 1000), + source_message_provider_id="0", + payload={ + "state": "completed", + "evidence_source": "provider_native", + "fidelity": "exact", + "duration_semantics": "provider_reported_elapsed", + "elapsed_duration_ms": elapsed_duration_ms, + }, + ) + ], + ) + return MembershipRevision(raw_id, session_revision_projection(session)) + + +def test_accepts_generation_lifecycle_duration_change_as_equivalent() -> None: + """Byte-identical transcript, only a provider-remeasured duration differs. + + Reproduces the shape measured on the live archive: 33 of 35 sampled + chatgpt-export ambiguous cohorts (94%) had identical messages and + attachments but a `generation_lifecycle` event whose `elapsed_duration_ms` + varied non-monotonically between export requests for the SAME generation + (polylogue-nuec). This must resolve as equivalent content, not stay + quarantined as a branch. + """ + first_export = _generation_lifecycle_revision("raw-first", 13000) + second_export = _generation_lifecycle_revision("raw-second", 21000) + + assert first_export.projection.event_hashes != second_export.projection.event_hashes + assert first_export.projection.event_identity_hashes == second_export.projection.event_identity_hashes + assert first_export.projection.session_hash != second_export.projection.session_hash + + # Same-content-key revisions still need a provider timestamp to pick a + # representative (matching the pre-existing metadata_variants mechanism) -- + # the measurement-tolerant key gets them INTO that mechanism at all, which + # is the fix; it does not bypass the timestamp requirement. + first_export = MembershipRevision(first_export.raw_id, first_export.projection, "2026-01-01T00:00:00Z") + second_export = MembershipRevision(second_export.raw_id, second_export.projection, "2026-01-02T00:00:00Z") + + result = classify_membership_revisions([first_export, second_export]) + + assert result.accepted_raw_ids == ("raw-second",) + assert result.equivalent_raw_ids == ("raw-first",) + assert result.ambiguous_raw_ids == () + + +def test_refuses_generation_lifecycle_state_change_despite_duration_tolerance() -> None: + """Duration tolerance must not launder every field of the event as noise. + + Only the designated measurement keys (and the timestamp they derive) are + excluded from identity -- a genuinely different ``state`` on the same event + is real divergence and must stay ambiguous. + """ + session = ParsedSession( + source_name=Provider.CHATGPT, + provider_session_id="session", + messages=[ParsedMessage(provider_message_id="0", role=Role.ASSISTANT, text="answer")], + session_events=[ + ParsedSessionEvent( + event_type="generation_lifecycle", + timestamp="13.0", + source_message_provider_id="0", + payload={ + "state": "completed", + "evidence_source": "provider_native", + "fidelity": "exact", + "duration_semantics": "provider_reported_elapsed", + "elapsed_duration_ms": 13000, + }, + ) + ], + ) + changed_state = session.model_copy( + update={ + "session_events": [ + ParsedSessionEvent( + event_type="generation_lifecycle", + timestamp="13.0", + source_message_provider_id="0", + payload={ + "state": "in_progress", + "evidence_source": "provider_native", + "fidelity": "exact", + "duration_semantics": "provider_reported_elapsed", + "elapsed_duration_ms": 13000, + }, + ) + ] + } + ) + older = MembershipRevision("raw-old", session_revision_projection(session)) + newer = MembershipRevision("raw-new", session_revision_projection(changed_state)) + + assert older.projection.event_identity_hashes != newer.projection.event_identity_hashes + assert not _strictly_dominates(older.projection, newer.projection) + assert not _strictly_dominates(newer.projection, older.projection) + + result = classify_membership_revisions([older, newer]) + assert result.accepted_raw_ids == () + assert result.ambiguous_raw_ids == ("raw-new", "raw-old") + + +def test_generation_lifecycle_duration_change_does_change_the_session_content_hash() -> None: + """Measurement tolerance must not leak into idempotency. + + The real provider-reported duration is still archive evidence worth + keeping and re-indexing when it changes -- only *revision comparison* is + tolerant of it, not ``session_hash``. + """ + first_export = _generation_lifecycle_revision("raw-first", 13000) + second_export = _generation_lifecycle_revision("raw-second", 21000) + + assert first_export.projection.session_hash != second_export.projection.session_hash + + +def test_non_provider_reported_event_duration_field_stays_load_bearing() -> None: + """The volatile-key exclusion is scoped to `duration_semantics == + "provider_reported_elapsed"` events only -- an unrelated event type that + happens to carry a same-named field is not touched, so a real difference + there still counts as divergence. + """ + session = ParsedSession( + source_name=Provider.CHATGPT, + provider_session_id="session", + messages=[ParsedMessage(provider_message_id="0", role=Role.ASSISTANT, text="answer")], + session_events=[ + ParsedSessionEvent( + event_type="chatgpt_block_metadata", + timestamp="13.0", + source_message_provider_id="0", + payload={"elapsed_duration_ms": 13000}, + ) + ], + ) + other_value = session.model_copy( + update={ + "session_events": [ + ParsedSessionEvent( + event_type="chatgpt_block_metadata", + timestamp="13.0", + source_message_provider_id="0", + payload={"elapsed_duration_ms": 21000}, + ) + ] + } + ) + older = MembershipRevision("raw-old", session_revision_projection(session)) + newer = MembershipRevision("raw-new", session_revision_projection(other_value)) + + assert older.projection.event_identity_hashes != newer.projection.event_identity_hashes + + result = classify_membership_revisions([older, newer]) + assert result.accepted_raw_ids == () + assert result.ambiguous_raw_ids == ("raw-new", "raw-old") + + +def test_refuses_event_growth_that_is_not_append_only() -> None: + """Measurement tolerance on ``generation_lifecycle`` must not turn the event + axis's prefix check into a bare count comparison. + + The newer revision has more events than the older one (real growth on the + count axis), but the extra event is *prepended*, not appended -- the + older revision's one event is not a positional prefix of the newer + revision's two. That is a real reshuffle of the event timeline, not + ordinary append growth, and must stay ambiguous. + """ + shared_event = ParsedSessionEvent( + event_type="generation_lifecycle", + timestamp="13.0", + source_message_provider_id="0", + payload={ + "state": "completed", + "evidence_source": "provider_native", + "fidelity": "exact", + "duration_semantics": "provider_reported_elapsed", + "elapsed_duration_ms": 13000, + }, + ) + prepended_event = ParsedSessionEvent( + event_type="chatgpt_block_metadata", + timestamp="1.0", + source_message_provider_id="0", + payload={"block_index": 0}, + ) + older_session = ParsedSession( + source_name=Provider.CHATGPT, + provider_session_id="session", + messages=[ParsedMessage(provider_message_id="0", role=Role.ASSISTANT, text="answer")], + session_events=[shared_event], + ) + newer_session = older_session.model_copy(update={"session_events": [prepended_event, shared_event]}) + + older = MembershipRevision("raw-old", session_revision_projection(older_session)) + newer = MembershipRevision("raw-new", session_revision_projection(newer_session)) + + assert len(newer.projection.event_identity_hashes) > len(older.projection.event_identity_hashes) + assert not _strictly_dominates(older.projection, newer.projection) + + result = classify_membership_revisions([older, newer]) + assert result.accepted_raw_ids == () + assert result.ambiguous_raw_ids == ("raw-new", "raw-old") From ad365cff498b41089cafd2d92a0fbd5ab3d57789 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 30 Jul 2026 15:39:41 +0200 Subject: [PATCH 3/8] fix(archive): correlate attachments across an id-presence mismatch Scope addition (polylogue-d8al, filed while this branch was in flight): hith's parser-side fix for synthetic-id volatility resolved 0 of the 566 claude-ai-export ambiguous cohorts on the live archive -- a full census found the population is instead saturated by a different axis: one export vintage of a conversation carries a real provider id for an attachment, the other has none at all and synthesizes one. No id-minting scheme can make a real id and a synthetic hash collide, so this belongs in the comparison layer, not the parser. SessionRevisionProjection gains attachment_records, a per-attachment (strict identity, loose id-independent identity, content) triple. session_revision_membership.py correlates attachments pairwise: strict id first (unchanged bu1i behavior when both sides agree on an id), falling back to the loose (message, name, mime_type) key only when that key is unambiguous on BOTH sides being compared. classify_membership_revisions gains a merge pass (_merge_attachment_id_presence_variants) so two content groups that differ only in attachment id presence -- not growth, pure equivalence -- can still resolve via the existing timestamp mechanism. attachment_identities/attachment_contents stay strict (unchanged semantics, unchanged callers in repair.py/archive.py); only the comparison-layer correlation is new. Known, documented limit: two genuinely distinct attachments sharing one message/name/media-type with no bytes on either side are indistinguishable -- stated plainly rather than guessed at by array position, which is the bug this replaces. This is a pure comparison-layer fix over parsed content already stored as raw bytes: a full index rebuild that replays existing raws through parse_payload -> session_revision_projection -> classify_membership_revisions applies it to already-ingested cohorts, not only new acquisitions. Ref polylogue-d8al Ref polylogue-c429 Ref polylogue-nuec Co-Authored-By: Claude --- docs/plans/hash-boundary-registry.yaml | 6 + .../archive/session_revision_membership.py | 158 ++++++++- polylogue/pipeline/ids.py | 57 +++- .../test_session_revision_membership.py | 300 ++++++++++++++++++ 4 files changed, 505 insertions(+), 16 deletions(-) diff --git a/docs/plans/hash-boundary-registry.yaml b/docs/plans/hash-boundary-registry.yaml index 7ac8d92cfd..3e961ebc24 100644 --- a/docs/plans/hash-boundary-registry.yaml +++ b/docs/plans/hash-boundary-registry.yaml @@ -554,6 +554,12 @@ entries: occurrence: 4 classification: content-hash note: 'event_identity_hashes -- revision-comparison axis only, provider-reported-elapsed measurement excluded per _event_identity_hash_payload (polylogue-nuec).' +- path: polylogue/pipeline/ids.py + function: '.session_revision_projection' + call: hash_payload + occurrence: 5 + classification: content-hash + note: 'attachment_records loose identity -- id-independent (message, name, mime_type) correlation key used only as a comparison-layer fallback (polylogue-d8al).' - path: polylogue/pipeline/ids.py function: '._event_identity_hash_payload' call: hash_payload diff --git a/polylogue/archive/session_revision_membership.py b/polylogue/archive/session_revision_membership.py index 6a5bc69c4a..6cac9d57c8 100644 --- a/polylogue/archive/session_revision_membership.py +++ b/polylogue/archive/session_revision_membership.py @@ -2,17 +2,22 @@ from __future__ import annotations +from collections import Counter from dataclasses import dataclass from typing import Literal, TypeAlias from polylogue.core.timestamps import parse_timestamp -from polylogue.pipeline.ids import SessionRevisionProjection +from polylogue.pipeline.ids import AttachmentRecord, SessionRevisionProjection #: Everything that must agree for two revisions to be the same content: which #: messages exist and what they say (order-insensitive -- polylogue-c429), #: the event chain with provider-reported measurement excluded #: (polylogue-nuec), which attachments exist, and which of their bytes have -#: been read. +#: been read. The attachment component is the strict (id-bearing) identity -- +#: two groups differing only in attachment id *presence* do not share a key +#: here and need the separate correlation-based merge step in +#: ``classify_membership_revisions`` (polylogue-d8al); this key alone would +#: under-merge for that case, never over-merge, so it stays a safe first pass. _ContentKey: TypeAlias = tuple[ frozenset[tuple[bytes, bytes]], tuple[bytes, ...], frozenset[bytes], frozenset[tuple[bytes, bytes]] ] @@ -56,7 +61,7 @@ def classify_membership_revisions(revisions: list[MembershipRevision]) -> Member by_content.setdefault(key, []).append(revision) representatives: list[MembershipRevision] = [] equivalents: list[str] = [] - for group in by_content.values(): + for group in _merge_attachment_id_presence_variants(by_content): by_session_hash: dict[bytes, list[MembershipRevision]] = {} for item in group: by_session_hash.setdefault(item.projection.session_hash, []).append(item) @@ -108,6 +113,55 @@ def classify_membership_revisions(revisions: list[MembershipRevision]) -> Member ) +def _merge_attachment_id_presence_variants( + by_content: dict[_ContentKey, list[MembershipRevision]], +) -> list[list[MembershipRevision]]: + """Merge content groups that differ only in attachment id presence. + + The strict content key built in ``classify_membership_revisions`` never + merges a real-id/synthetic-id pair for the same physical attachment into + one group: a provider's export can omit a stable id for the same + attachment on a different export request of the same conversation, so an + otherwise byte-identical pair's attachment sets hash to disjoint strict + identities and the two groups never meet in ``by_content`` (polylogue-d8al). + This pass merges any two groups whose message/event portion of the key + already matches exactly and whose attachments correlate as fully + equivalent (``_attachments_equivalent``): same cardinality, every + attachment pairwise-matched by strict id or, when unambiguous on both + sides, by the id-independent key, and no content contradiction. Only ever + merges groups the strict key under-merged -- it can never combine two + groups that genuinely differ in message or event content, so this cannot + introduce a false equivalence on those axes. + """ + entries = list(by_content.items()) + parent = list(range(len(entries))) + + def find(index: int) -> int: + while parent[index] != index: + parent[index] = parent[parent[index]] + index = parent[index] + return index + + def union(left: int, right: int) -> None: + root_left, root_right = find(left), find(right) + if root_left != root_right: + parent[root_right] = root_left + + for i in range(len(entries)): + key_i, revisions_i = entries[i] + for j in range(i + 1, len(entries)): + key_j, revisions_j = entries[j] + if key_i[0] != key_j[0] or key_i[1] != key_j[1]: + continue + if _attachments_equivalent(revisions_i[0].projection, revisions_j[0].projection): + union(i, j) + + merged: dict[int, list[MembershipRevision]] = {} + for i, (_key, revisions) in enumerate(entries): + merged.setdefault(find(i), []).extend(revisions) + return list(merged.values()) + + def _provider_ordered_browser_snapshots( revisions: list[MembershipRevision], ) -> list[MembershipRevision] | None: @@ -241,6 +295,70 @@ def _message_evidence_preserved( return all(newer_contents.get(identity) == content for identity, content in older.message_contents) +def _correlate_attachments( + older: SessionRevisionProjection, newer: SessionRevisionProjection +) -> tuple[list[tuple[AttachmentRecord, AttachmentRecord]], list[AttachmentRecord], list[AttachmentRecord]]: + """Pair each attachment in ``older`` with its counterpart in ``newer``. + + Matches by strict identity first (provider id, anchoring message, name, + media type) -- the exact bu1i behavior when both revisions agree on a + provider id. When a provider omits a stable id for the same attachment on + a different export request, the two revisions never share a strict + identity for it (polylogue-d8al): Claude.ai does not consistently emit an + id field for the same attachment across separate export requests of the + same conversation. In that case matching falls back to the + id-independent (anchoring message, name, media type) key, but ONLY when + that looser key is unambiguous on BOTH sides being compared (exactly one + attachment carries it in ``older`` and exactly one in ``newer``): if + either revision has two attachments sharing the same anchor/name/media + type, there is no way to tell which is which without an id, and guessing + by array position is exactly the bug this replaces -- that ambiguous case + is left uncorrelated rather than resolved by a guess. + + Returns matched ``(older, newer)`` record pairs, ``older`` records with no + counterpart in ``newer`` (a potential loss), and ``newer`` records with no + counterpart in ``older`` (growth). + """ + newer_by_identity = {record[0]: record for record in newer.attachment_records} + newer_loose_counts = Counter(record[1] for record in newer.attachment_records) + newer_by_loose = {record[1]: record for record in newer.attachment_records} + older_loose_counts = Counter(record[1] for record in older.attachment_records) + + matched: list[tuple[AttachmentRecord, AttachmentRecord]] = [] + unmatched_older: list[AttachmentRecord] = [] + matched_newer_identities: set[bytes] = set() + for older_record in older.attachment_records: + identity, loose_identity, _content = older_record + newer_record = newer_by_identity.get(identity) + if ( + newer_record is None + and older_loose_counts[loose_identity] == 1 + and newer_loose_counts.get(loose_identity) == 1 + ): + newer_record = newer_by_loose.get(loose_identity) + if newer_record is None: + unmatched_older.append(older_record) + continue + matched.append((older_record, newer_record)) + matched_newer_identities.add(newer_record[0]) + unmatched_newer = [record for record in newer.attachment_records if record[0] not in matched_newer_identities] + return matched, unmatched_older, unmatched_newer + + +def _attachments_equivalent(a: SessionRevisionProjection, b: SessionRevisionProjection) -> bool: + """True when every attachment in ``a`` and ``b`` correlates 1:1 with identical content. + + Used only to decide whether two otherwise-identical content groups may + merge as the same content (``_merge_attachment_id_presence_variants``); + unlike ``_attachment_evidence_preserved`` this is symmetric and requires + an exact match on both sides, not merely no loss in one direction. + """ + matched, unmatched_a, unmatched_b = _correlate_attachments(a, b) + if unmatched_a or unmatched_b: + return False + return all(a_record[2] == b_record[2] for a_record, b_record in matched) + + def _attachment_evidence_preserved( older: SessionRevisionProjection, newer: SessionRevisionProjection, @@ -253,12 +371,30 @@ def _attachment_evidence_preserved( silently mark an acquired attachment unfetched. An attachment present in both but carrying *different* bytes is a genuine conflict -- two sources disagree about content under one identity -- and no ordering rule can - resolve that, so the cohort stays ambiguous. + resolve that, so the cohort stays ambiguous. Correlation + (``_correlate_attachments``) is what lets this hold across an id-presence + mismatch the same way it always did for a plain matching id + (polylogue-d8al). """ - newer_contents = dict(newer.attachment_contents) - return older.attachment_identities <= newer.attachment_identities and all( - newer_contents.get(identity) == content for identity, content in older.attachment_contents - ) + matched, unmatched_older, _unmatched_newer = _correlate_attachments(older, newer) + if unmatched_older: + return False + return all(older_record[2] is None or older_record[2] == newer_record[2] for older_record, newer_record in matched) + + +def _attachment_axis_grew(older: SessionRevisionProjection, newer: SessionRevisionProjection) -> bool: + """True when ``newer`` has a genuinely new attachment or newly-read bytes. + + An uncorrelated attachment in ``newer`` (no older counterpart, by strict + id or unambiguous loose key) is growth. So is resolving the bytes of an + already-referenced attachment -- growth in evidence even when the + transcript is untouched, the shape a lazily-fetched attachment produces + on its second acquisition (polylogue-bu1i). + """ + matched, _unmatched_older, unmatched_newer = _correlate_attachments(older, newer) + if unmatched_newer: + return True + return any(older_record[2] is None and newer_record[2] is not None for older_record, newer_record in matched) def _strictly_dominates(older: SessionRevisionProjection, newer: SessionRevisionProjection) -> bool: @@ -268,11 +404,7 @@ def _strictly_dominates(older: SessionRevisionProjection, newer: SessionRevision # id is (polylogue-c429). len(newer.message_contents) > len(older.message_contents) or len(newer.event_identity_hashes) > len(older.event_identity_hashes) - or newer.attachment_identities > older.attachment_identities - # Resolving the bytes of an already-referenced attachment is growth in - # evidence even when the transcript is untouched, which is exactly the - # shape a lazily-fetched attachment produces on its second acquisition. - or len(newer.attachment_contents) > len(older.attachment_contents) + or _attachment_axis_grew(older, newer) ) return ( content_grew diff --git a/polylogue/pipeline/ids.py b/polylogue/pipeline/ids.py index 3729324fff..c29bb3380d 100644 --- a/polylogue/pipeline/ids.py +++ b/polylogue/pipeline/ids.py @@ -30,6 +30,11 @@ _EMPTY_SENTINEL = "__POLYLOGUE_EMPTY__" HashScalar: TypeAlias = str | int | float | bool | None +#: One attachment's (strict identity, loose/id-independent identity, content +#: hash or ``None`` if unacquired) triple -- see ``SessionRevisionProjection`` +#: for why the comparison layer needs both identities (polylogue-d8al). +AttachmentRecord: TypeAlias = tuple[bytes, bytes, bytes | None] + @dataclass(frozen=True, slots=True) class SessionRevisionProjection: @@ -51,6 +56,26 @@ class SessionRevisionProjection: axes lets a dominance test say what is actually true -- same attachments, strictly more of their bytes now known (polylogue-bu1i). + ``attachment_records`` carries the same per-attachment data as a + ``(strict identity, loose identity, content)`` triple instead of two + separate sets, so the *comparison layer* (``session_revision_membership.py``) + can correlate attachments across two revisions even when a provider's + export omits a stable id for the same physical attachment on a different + export request of the same conversation -- one vintage has a real UUID, + the other has none and a parser synthesizes one, and no synthetic-id + scheme can make a real id and a synthetic hash collide by construction + (polylogue-d8al). The "loose" identity drops the provider id and keeps + only the anchoring message, name, and media type; + ``attachment_identities``/``attachment_contents`` stay strict (id + included) so this projection's own equality/hashing behavior is + unchanged -- only the membership module's pairwise correlation consults + the loose key, and only as a fallback when the strict identity does not + match and the loose key is unambiguous on both sides being compared. + Known, accepted limit stated explicitly rather than engineered around: + two genuinely distinct attachments that share one message/name/media-type + and carry no bytes on either side of a comparison are indistinguishable + by any signal this projection can offer. + Messages get an analogous split, for the same reason applied to a different volatility source: a provider's own export can replay an unchanged message set in a different array sequence across separate export @@ -83,6 +108,7 @@ class SessionRevisionProjection: message_contents: frozenset[tuple[bytes, bytes]] attachment_identities: frozenset[bytes] attachment_contents: frozenset[tuple[bytes, bytes]] + attachment_records: tuple[AttachmentRecord, ...] event_hashes: tuple[bytes, ...] event_identity_hashes: tuple[bytes, ...] @@ -248,6 +274,26 @@ def _attachment_identity_payload(payload: dict[str, JSONValue]) -> dict[str, JSO return {field: payload[field] for field in _ATTACHMENT_IDENTITY_FIELDS} +#: The subset of ``_ATTACHMENT_IDENTITY_FIELDS`` that survives even when a +#: provider omits a stable id for the same physical attachment on a different +#: export request (polylogue-d8al): Claude.ai does not consistently emit +#: ``id``/``file_id``/``fileId``/``uuid``/``file_uuid`` for the same +#: attachment across separate export requests of the same conversation -- one +#: vintage carries a real UUID-shaped id, the other has none and a parser +#: synthesizes one instead. No synthetic-id scheme can make a real id and a +#: synthetic hash collide by construction, so revision comparison correlates +#: by this looser, id-independent key when the strict identity does not +#: match (see ``session_revision_projection``'s canonicalization step). +#: ``size_bytes`` stays excluded for the same lazy-fetch reason it is excluded +#: from ``_ATTACHMENT_IDENTITY_FIELDS``. +_ATTACHMENT_LOOSE_IDENTITY_FIELDS = ("message_id", "name", "mime_type") + + +def _attachment_loose_identity_payload(payload: dict[str, JSONValue]) -> dict[str, JSONValue]: + """Project the id-independent correlation key of one attachment payload.""" + return {field: payload[field] for field in _ATTACHMENT_LOOSE_IDENTITY_FIELDS} + + #: `generation_lifecycle` payload keys that are provider-reported measurement, #: not identity, when the event's own payload declares them non-durable via #: ``duration_semantics == "provider_reported_elapsed"``. ChatGPT re-derives @@ -425,14 +471,18 @@ def session_revision_projection(convo: ParsedSession) -> SessionRevisionProjecti content = bytes.fromhex(hash_payload(payload)) message_contents.add((identity, content)) message_hashes.append(content) + attachment_records: list[AttachmentRecord] = [] attachment_identities: set[bytes] = set() attachment_contents: set[tuple[bytes, bytes]] = set() for payload in attachments_payload: identity = bytes.fromhex(hash_payload(_attachment_identity_payload(payload))) - attachment_identities.add(identity) + loose_identity = bytes.fromhex(hash_payload(_attachment_loose_identity_payload(payload))) inline_content_hash = payload.get("inline_content_hash") - if isinstance(inline_content_hash, str): - attachment_contents.add((identity, bytes.fromhex(inline_content_hash))) + attachment_content = bytes.fromhex(inline_content_hash) if isinstance(inline_content_hash, str) else None + attachment_records.append((identity, loose_identity, attachment_content)) + attachment_identities.add(identity) + if attachment_content is not None: + attachment_contents.add((identity, attachment_content)) event_hashes: list[bytes] = [] event_identity_hashes: list[bytes] = [] for event_index, (payload, event) in enumerate(zip(session_events_payload, convo.session_events, strict=True)): @@ -444,6 +494,7 @@ def session_revision_projection(convo: ParsedSession) -> SessionRevisionProjecti message_contents=frozenset(message_contents), attachment_identities=frozenset(attachment_identities), attachment_contents=frozenset(attachment_contents), + attachment_records=tuple(attachment_records), event_hashes=tuple(event_hashes), event_identity_hashes=tuple(event_identity_hashes), ) diff --git a/tests/unit/archive/test_session_revision_membership.py b/tests/unit/archive/test_session_revision_membership.py index 57128a55af..444797237f 100644 --- a/tests/unit/archive/test_session_revision_membership.py +++ b/tests/unit/archive/test_session_revision_membership.py @@ -170,6 +170,7 @@ def test_browser_native_upgrade_refuses_any_shrinking_frontier_dimension() -> No event_identity_hashes=older.projection.event_identity_hashes, attachment_identities=frozenset({b"attachment"}), attachment_contents=frozenset(), + attachment_records=((b"attachment", b"attachment-loose", None),), ) newer = _revision("raw-new", "prompt", "answer") revisions = [ @@ -234,6 +235,7 @@ def test_browser_snapshot_accepts_later_attachment_enrichment_without_provider_u event_identity_hashes=newer.projection.event_identity_hashes, attachment_identities=frozenset({b"attachment-v2"}), attachment_contents=frozenset(), + attachment_records=((b"attachment-v2", b"attachment-v2-loose", None),), ) revisions = [ MembershipRevision( @@ -408,6 +410,304 @@ def test_refuses_contradicted_bytes_even_when_the_revision_otherwise_grew() -> N assert result.ambiguous_raw_ids == ("raw-a", "raw-b") +def _named_attachment_revision( + raw_id: str, + *, + provider_attachment_id: str, + name: str = "screenshot.png", + mime_type: str = "image/png", + message_provider_id: str = "0", + inline: bytes | None = None, +) -> MembershipRevision: + """A session whose single named attachment carries a specific provider id. + + Unlike ``_attachment_revision`` (whose bare fixture leaves name/mime_type + ``None``, deliberately colliding with any other bare attachment's loose + key), this gives the attachment a real name/mime_type so d8al tests + exercise the id-presence correlation they're meant to, not the + locally-ambiguous-loose-key fallback. + """ + attachment = ParsedAttachment( + provider_attachment_id=provider_attachment_id, + message_provider_id=message_provider_id, + name=name, + mime_type=mime_type, + size_bytes=len(inline) if inline is not None else None, + inline_bytes=inline, + ) + session = ParsedSession( + source_name=Provider.CLAUDE_AI, + provider_session_id="session", + messages=[ParsedMessage(provider_message_id="0", role=Role.USER, text="one")], + attachments=[attachment], + ) + return MembershipRevision(raw_id, session_revision_projection(session)) + + +def test_accepts_real_and_synthetic_id_variants_of_the_same_attachment_as_equivalent() -> None: + """Same physical attachment, a real id on one export vintage, none on the other. + + Reproduces the shape measured on the live archive: 268 of 566 + claude-ai-export equal-message-count ambiguous cohorts had byte-identical + messages and events but attachment identity sets that never share a + provider id, because Claude.ai does not consistently emit a real + attachment id across separate export requests of the same conversation -- + one vintage carries a real UUID, the other has none and the parser + synthesizes one (polylogue-d8al). No id-minting scheme can make a real id + and a synthetic hash collide, so the id must be dropped from the + comparison in this shape -- correlating instead by (message, name, media + type), which is unambiguous here (exactly one attachment on this + message). + """ + real_id = _named_attachment_revision("raw-real", provider_attachment_id="e950263f-51d1-4b7a-9c2e-000000000000") + synthetic_id = _named_attachment_revision("raw-synthetic", provider_attachment_id="att-ce21cd12d650") + + assert real_id.projection.attachment_identities != synthetic_id.projection.attachment_identities + assert not _strictly_dominates(real_id.projection, synthetic_id.projection) + assert not _strictly_dominates(synthetic_id.projection, real_id.projection) + + # Same-content-key revisions still need a provider timestamp to pick a + # representative (matching the pre-existing metadata_variants mechanism) -- + # the correlation-based merge gets them INTO that mechanism at all, which + # is the fix; it does not bypass the timestamp requirement. + real_id = MembershipRevision(real_id.raw_id, real_id.projection, "2026-01-01T00:00:00Z") + synthetic_id = MembershipRevision(synthetic_id.raw_id, synthetic_id.projection, "2026-01-02T00:00:00Z") + + result = classify_membership_revisions([real_id, synthetic_id]) + + assert result.accepted_raw_ids == ("raw-synthetic",) + assert result.equivalent_raw_ids == ("raw-real",) + assert result.ambiguous_raw_ids == () + + +def test_accepts_attachment_growth_despite_id_presence_mismatch_on_the_shared_attachment() -> None: + """Growth must still be recognized when the SHARED attachment's id presence differs. + + The newer revision adds a second attachment (real growth) while also + losing the real id on the FIRST attachment (id-presence mismatch, + polylogue-d8al) -- both must be handled together: the shared attachment + correlates via the loose key, and the new attachment is ordinary growth. + """ + older_attachment = ParsedAttachment( + provider_attachment_id="e950263f-51d1-4b7a-9c2e-000000000000", + message_provider_id="0", + name="screenshot.png", + mime_type="image/png", + ) + newer_shared = ParsedAttachment( + provider_attachment_id="att-ce21cd12d650", + message_provider_id="0", + name="screenshot.png", + mime_type="image/png", + ) + newer_added = ParsedAttachment( + provider_attachment_id="att-fedcba987654", + message_provider_id="0", + name="notes.txt", + mime_type="text/plain", + ) + older_session = ParsedSession( + source_name=Provider.CLAUDE_AI, + provider_session_id="session", + messages=[ParsedMessage(provider_message_id="0", role=Role.USER, text="one")], + attachments=[older_attachment], + ) + newer_session = older_session.model_copy(update={"attachments": [newer_shared, newer_added]}) + + older = MembershipRevision("raw-old", session_revision_projection(older_session)) + newer = MembershipRevision("raw-new", session_revision_projection(newer_session)) + + assert _strictly_dominates(older.projection, newer.projection) + + result = classify_membership_revisions([older, newer]) + assert result.accepted_raw_ids == ("raw-old", "raw-new") + assert result.ambiguous_raw_ids == () + + +def test_attachment_growth_with_id_presence_mismatch_is_a_chain_not_an_equivalence_pick() -> None: + """The equivalence-merge step must not swallow genuine growth as a timestamp pick. + + The test above proves growth is recognized by ``_strictly_dominates`` when + the two revisions are never pre-grouped into one equivalence bucket, but + it does not prove the merge step's own full-correlation requirement + (``unmatched_a or unmatched_b`` in ``_attachments_equivalent``) does + anything -- session_hash always differs when attachment counts differ, so + an over-eager merge just gets re-split by the by_session_hash pass with + the SAME final result. This is the shape that forces it: distinct, + resolvable provider timestamps. If the merge step ignored the extra, + uncorrelated attachment in ``newer`` and merged the groups anyway, the + pre-existing metadata_variants timestamp mechanism would pick ONE + representative and discard the other as merely-equivalent metadata, + instead of recognizing both as an accepted growth chain. + """ + older_attachment = ParsedAttachment( + provider_attachment_id="e950263f-51d1-4b7a-9c2e-000000000000", + message_provider_id="0", + name="screenshot.png", + mime_type="image/png", + ) + newer_shared = ParsedAttachment( + provider_attachment_id="att-ce21cd12d650", + message_provider_id="0", + name="screenshot.png", + mime_type="image/png", + ) + newer_added = ParsedAttachment( + provider_attachment_id="att-fedcba987654", + message_provider_id="0", + name="notes.txt", + mime_type="text/plain", + ) + older_session = ParsedSession( + source_name=Provider.CLAUDE_AI, + provider_session_id="session", + messages=[ParsedMessage(provider_message_id="0", role=Role.USER, text="one")], + attachments=[older_attachment], + ) + newer_session = older_session.model_copy(update={"attachments": [newer_shared, newer_added]}) + + older = MembershipRevision("raw-old", session_revision_projection(older_session), "2026-01-01T00:00:00Z") + newer = MembershipRevision("raw-new", session_revision_projection(newer_session), "2026-01-02T00:00:00Z") + + result = classify_membership_revisions([older, newer]) + assert result.accepted_raw_ids == ("raw-old", "raw-new") + assert result.equivalent_raw_ids == () + assert result.ambiguous_raw_ids == () + + +def test_refuses_contradicted_bytes_despite_id_presence_mismatch() -> None: + """An id-presence mismatch must not launder a genuine byte contradiction. + + Both revisions have acquired bytes for what correlates as the same + attachment (real id vs none, matched by the loose key) -- if the bytes + disagree, that is a real conflict no ordering rule can resolve, exactly + like bu1i's plain-matching-id contradiction case. + """ + older_attachment = ParsedAttachment( + provider_attachment_id="e950263f-51d1-4b7a-9c2e-000000000000", + message_provider_id="0", + name="screenshot.png", + mime_type="image/png", + size_bytes=len(b"original bytes"), + inline_bytes=b"original bytes", + ) + newer_attachment = ParsedAttachment( + provider_attachment_id="att-ce21cd12d650", + message_provider_id="0", + name="screenshot.png", + mime_type="image/png", + size_bytes=len(b"different bytes"), + inline_bytes=b"different bytes", + ) + older_session = ParsedSession( + source_name=Provider.CLAUDE_AI, + provider_session_id="session", + messages=[ParsedMessage(provider_message_id="0", role=Role.USER, text="one")], + attachments=[older_attachment], + ) + newer_session = older_session.model_copy(update={"attachments": [newer_attachment]}) + + older = MembershipRevision("raw-old", session_revision_projection(older_session)) + newer = MembershipRevision("raw-new", session_revision_projection(newer_session)) + + assert not _strictly_dominates(older.projection, newer.projection) + assert not _strictly_dominates(newer.projection, older.projection) + + result = classify_membership_revisions([older, newer]) + assert result.accepted_raw_ids == () + assert result.ambiguous_raw_ids == ("raw-new", "raw-old") + + +def test_refuses_contradicted_bytes_despite_id_presence_mismatch_even_with_resolvable_timestamps() -> None: + """The equivalence-merge content check must not be provably unreachable. + + The test above stays ambiguous even without this specific check, because + dominance's own contradiction guard (``_attachment_evidence_preserved``) + independently refuses the same pair once they fail to merge into one + equivalence group -- so it cannot prove the merge step's OWN content + check (``_attachments_equivalent``) does anything. This is the shape that + forces it: distinct, resolvable provider timestamps on both revisions. If + the merge step ignored the byte contradiction and merged them anyway, the + pre-existing metadata_variants timestamp mechanism would silently pick + the newer-timestamped revision as an "equivalent" upgrade and discard the + other -- overwriting a genuine, unresolvable conflict about the same + attachment's bytes instead of leaving it ambiguous. + """ + older_attachment = ParsedAttachment( + provider_attachment_id="e950263f-51d1-4b7a-9c2e-000000000000", + message_provider_id="0", + name="screenshot.png", + mime_type="image/png", + size_bytes=len(b"original bytes"), + inline_bytes=b"original bytes", + ) + newer_attachment = ParsedAttachment( + provider_attachment_id="att-ce21cd12d650", + message_provider_id="0", + name="screenshot.png", + mime_type="image/png", + size_bytes=len(b"different bytes"), + inline_bytes=b"different bytes", + ) + older_session = ParsedSession( + source_name=Provider.CLAUDE_AI, + provider_session_id="session", + messages=[ParsedMessage(provider_message_id="0", role=Role.USER, text="one")], + attachments=[older_attachment], + ) + newer_session = older_session.model_copy(update={"attachments": [newer_attachment]}) + + older = MembershipRevision("raw-old", session_revision_projection(older_session), "2026-01-01T00:00:00Z") + newer = MembershipRevision("raw-new", session_revision_projection(newer_session), "2026-01-02T00:00:00Z") + + result = classify_membership_revisions([older, newer]) + assert result.accepted_raw_ids == () + assert result.ambiguous_raw_ids == ("raw-new", "raw-old") + + +def test_refuses_to_guess_correlation_when_the_loose_key_is_locally_ambiguous() -> None: + """Two distinct attachments sharing (message, name, media type) on one side + must not be guessed apart by the loose key. + + This is the documented, accepted limit (polylogue-d8al): with no id + agreement and no bytes on either side, there is no signal left to + correlate the attachments, so the strict identity is kept and the + id-presence mismatch stays unresolved (ambiguous) rather than silently + picking a pairing by array position -- the exact mistake this feature + replaces. + """ + ambiguous_older = [ + ParsedAttachment( + provider_attachment_id="uuid-1", message_provider_id="0", name="image.png", mime_type="image/png" + ), + ParsedAttachment( + provider_attachment_id="uuid-2", message_provider_id="0", name="image.png", mime_type="image/png" + ), + ] + single_newer = [ + ParsedAttachment( + provider_attachment_id="att-hash-1", message_provider_id="0", name="image.png", mime_type="image/png" + ), + ] + older_session = ParsedSession( + source_name=Provider.CLAUDE_AI, + provider_session_id="session", + messages=[ParsedMessage(provider_message_id="0", role=Role.USER, text="one")], + attachments=ambiguous_older, + ) + newer_session = older_session.model_copy(update={"attachments": single_newer}) + + older = MembershipRevision("raw-old", session_revision_projection(older_session)) + newer = MembershipRevision("raw-new", session_revision_projection(newer_session)) + + assert not _strictly_dominates(older.projection, newer.projection) + assert not _strictly_dominates(newer.projection, older.projection) + + result = classify_membership_revisions([older, newer]) + assert result.accepted_raw_ids == () + assert result.ambiguous_raw_ids == ("raw-new", "raw-old") + + def _ordered_message_revision(raw_id: str, *id_text_pairs: tuple[str, str]) -> MembershipRevision: """A session whose messages carry explicit provider ids in a given array order. From 327b680a693296ddf236ed54624fefd036c5dad2 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 30 Jul 2026 16:25:04 +0200 Subject: [PATCH 4/8] fix(archive): resolve equivalence groups tied on provider timestamp Problem (discovered via live-archive census while validating c429/nuec/ d8al): the census showed almost no cohort actually resolving end to end. Root cause -- most live cohorts have >2 raw duplicates, and once the tolerant content key correctly groups a permuted/duration-varied/ id-mismatched pair together, the pre-existing by_session_hash sub-split still requires two DISTINCT provider timestamps to pick a representative. For pure export-vintage noise (reorder, remeasured duration, id presence) the provider's own updated_at legitimately never moves between two export requests of an untouched conversation, so that tie-break could never fire and the group stayed ambiguous forever regardless of the c429/nuec/d8al fixes. SessionRevisionProjection gains metadata_hash (title/created_at/updated_at, normalized, matching exactly the session_hash payload fields NOT already covered by the content-key axes). Within an already-content-key-equivalent group, if every variant also agrees on metadata_hash, the ONLY remaining source of a session_hash difference is one of the three tolerated axes, so picking any one (min raw_id) is exactly as safe as the pre-existing exact-session_hash collapse a few lines above it -- and it is proven not to apply when a real title/timestamp edit is present (both pre-existing equal-timestamp-stays-ambiguous tests are unchanged and still pass). A stronger "never leave a document absent" presence-guarantee fallback was also explored (deterministic frontier-max pick for genuinely divergent cohorts) but reverted: it conflicts with archive.py's existing "membership replay cannot retire an unrelated accepted head" invariant when a later ambiguous batch's fallback pick differs from an already-established head, raising a RuntimeError on real replay (proven by test_divergent_bundle_member_preserves_last_accepted_session). Fixing that requires plumbing existing-head awareness into the classifier or its caller, which touches archive.py's write path -- out of this lane's scope (archive.py/repair.py are owned by concurrent lanes). Recommending that as a separate, coordinated follow-up rather than merging a change that can crash a live rebuild. Ref polylogue-c429 Ref polylogue-nuec Ref polylogue-d8al Co-Authored-By: Claude --- docs/plans/hash-boundary-registry.yaml | 6 ++ .../archive/session_revision_membership.py | 20 +++++++ polylogue/pipeline/ids.py | 24 ++++++++ .../test_session_revision_membership.py | 59 ++++++++++--------- 4 files changed, 81 insertions(+), 28 deletions(-) diff --git a/docs/plans/hash-boundary-registry.yaml b/docs/plans/hash-boundary-registry.yaml index 3e961ebc24..34564a674f 100644 --- a/docs/plans/hash-boundary-registry.yaml +++ b/docs/plans/hash-boundary-registry.yaml @@ -566,6 +566,12 @@ entries: occurrence: 0 classification: content-hash note: 'per-event identity hash feeding event_identity_hashes, measurement fields stripped for provider_reported_elapsed events (polylogue-nuec).' +- path: polylogue/pipeline/ids.py + function: '.session_revision_projection' + call: hash_payload + occurrence: 6 + classification: content-hash + note: 'metadata_hash -- title/created_at/updated_at only, lets membership classification tell a genuine metadata edit apart from pure order/duration/id-presence noise within an already-tolerant content-key group (polylogue-c429, polylogue-nuec, polylogue-d8al).' - path: polylogue/scenarios/workload.py function: '._identity' call: hashlib.sha256 diff --git a/polylogue/archive/session_revision_membership.py b/polylogue/archive/session_revision_membership.py index 6cac9d57c8..ccaba69877 100644 --- a/polylogue/archive/session_revision_membership.py +++ b/polylogue/archive/session_revision_membership.py @@ -73,6 +73,26 @@ def classify_membership_revisions(revisions: list[MembershipRevision]) -> Member if len(metadata_variants) == 1: representatives.extend(metadata_variants) continue + # A same-content-key group can still split into multiple session_hash + # sub-groups for two different reasons: a genuine title/created_at/ + # updated_at edit (needs the timestamp tie-break below), or pure + # serialization noise along an axis the content key already tolerates + # -- message order, attachment id presence, event-duration + # measurement -- with every OTHER session_hash input identical. The + # second shape is the common one for a re-export of an untouched + # conversation: the provider's own updated_at legitimately never + # moves, so a distinct-timestamp tie-break can never fire and the + # group would otherwise stay ambiguous forever (polylogue-c429, + # polylogue-nuec, polylogue-d8al). metadata_hash pins down which + # shape this is: if every variant agrees on title/created_at/ + # updated_at, the ONLY remaining source of a session_hash difference + # is one of the tolerated axes, so picking any one is exactly as safe + # as the exact-session_hash collapse a few lines above. + if len({item.projection.metadata_hash for item in metadata_variants}) == 1: + representative = min(metadata_variants, key=lambda item: item.raw_id) + representatives.append(representative) + equivalents.extend(item.raw_id for item in metadata_variants if item.raw_id != representative.raw_id) + continue timestamped = [ (parsed.timestamp(), item) for item in metadata_variants diff --git a/polylogue/pipeline/ids.py b/polylogue/pipeline/ids.py index c29bb3380d..3863ae3674 100644 --- a/polylogue/pipeline/ids.py +++ b/polylogue/pipeline/ids.py @@ -101,6 +101,19 @@ class SessionRevisionProjection: conversations look like divergent branches on every re-export (polylogue-nuec). ``event_hashes`` (unstripped, order-preserving) remains unchanged for ``session_hash`` and diagnostics. + + ``metadata_hash`` covers exactly the ``session_hash`` payload fields + OTHER than messages/attachments/session_events -- title, created_at, + updated_at -- normalized the same way. Membership classification uses it + to tell apart two reasons a same-content-key group can still carry + different ``session_hash`` values: a genuine title/timestamp edit (needs + the existing provider-timestamp tie-break), versus pure serialization + noise along an axis this projection already tolerates (message order, + attachment id presence, event-duration measurement) with the provider's + own metadata otherwise unchanged -- the common shape for a re-export of + an untouched conversation, where ``updated_at`` legitimately never moves + and a distinct-timestamp tie-break can never fire (polylogue-c429, + polylogue-nuec, polylogue-d8al). """ session_hash: bytes @@ -111,6 +124,7 @@ class SessionRevisionProjection: attachment_records: tuple[AttachmentRecord, ...] event_hashes: tuple[bytes, ...] event_identity_hashes: tuple[bytes, ...] + metadata_hash: bytes def _normalize_nested_for_hash(value: object) -> object: @@ -488,6 +502,15 @@ def session_revision_projection(convo: ParsedSession) -> SessionRevisionProjecti for event_index, (payload, event) in enumerate(zip(session_events_payload, convo.session_events, strict=True)): event_hashes.append(bytes.fromhex(hash_payload(payload))) event_identity_hashes.append(bytes.fromhex(hash_payload(_event_identity_hash_payload(event, event_index)))) + metadata_hash = bytes.fromhex( + hash_payload( + { + "title": _normalize_for_hash(convo.title), + "created_at": _normalize_for_hash(convo.created_at), + "updated_at": _normalize_for_hash(convo.updated_at), + } + ) + ) return SessionRevisionProjection( session_hash=bytes.fromhex(session_hash_hex), message_hashes=tuple(message_hashes), @@ -497,4 +520,5 @@ def session_revision_projection(convo: ParsedSession) -> SessionRevisionProjecti attachment_records=tuple(attachment_records), event_hashes=tuple(event_hashes), event_identity_hashes=tuple(event_identity_hashes), + metadata_hash=metadata_hash, ) diff --git a/tests/unit/archive/test_session_revision_membership.py b/tests/unit/archive/test_session_revision_membership.py index 444797237f..63e64e3b70 100644 --- a/tests/unit/archive/test_session_revision_membership.py +++ b/tests/unit/archive/test_session_revision_membership.py @@ -171,6 +171,7 @@ def test_browser_native_upgrade_refuses_any_shrinking_frontier_dimension() -> No attachment_identities=frozenset({b"attachment"}), attachment_contents=frozenset(), attachment_records=((b"attachment", b"attachment-loose", None),), + metadata_hash=older.projection.metadata_hash, ) newer = _revision("raw-new", "prompt", "answer") revisions = [ @@ -236,6 +237,7 @@ def test_browser_snapshot_accepts_later_attachment_enrichment_without_provider_u attachment_identities=frozenset({b"attachment-v2"}), attachment_contents=frozenset(), attachment_records=((b"attachment-v2", b"attachment-v2-loose", None),), + metadata_hash=newer.projection.metadata_hash, ) revisions = [ MembershipRevision( @@ -466,17 +468,18 @@ def test_accepts_real_and_synthetic_id_variants_of_the_same_attachment_as_equiva assert not _strictly_dominates(real_id.projection, synthetic_id.projection) assert not _strictly_dominates(synthetic_id.projection, real_id.projection) - # Same-content-key revisions still need a provider timestamp to pick a - # representative (matching the pre-existing metadata_variants mechanism) -- - # the correlation-based merge gets them INTO that mechanism at all, which - # is the fix; it does not bypass the timestamp requirement. - real_id = MembershipRevision(real_id.raw_id, real_id.projection, "2026-01-01T00:00:00Z") - synthetic_id = MembershipRevision(synthetic_id.raw_id, synthetic_id.projection, "2026-01-02T00:00:00Z") - + # Both fixtures leave title/created_at/updated_at unset (matching), the + # common live shape for a re-export of an untouched conversation: the + # correlation-based merge gets them into the same content-key group, and + # metadata_hash proves the id-presence mismatch is the ONLY remaining + # difference -- no provider timestamp is needed or consulted (real + # exports commonly carry an IDENTICAL provider updated_at across two + # export requests of unchanged content, so requiring one to differ would + # leave this shape ambiguous forever). result = classify_membership_revisions([real_id, synthetic_id]) - assert result.accepted_raw_ids == ("raw-synthetic",) - assert result.equivalent_raw_ids == ("raw-real",) + assert result.accepted_raw_ids == ("raw-real",) + assert result.equivalent_raw_ids == ("raw-synthetic",) assert result.ambiguous_raw_ids == () @@ -742,18 +745,17 @@ def test_accepts_reordered_message_array_with_identical_content_as_equivalent() assert chronological.projection.message_hashes != resequenced.projection.message_hashes assert chronological.projection.message_contents == resequenced.projection.message_contents - # Same-content-key revisions still need a provider timestamp to pick a - # representative (matching the pre-existing metadata_variants mechanism, - # e.g. test_metadata_only_revision_uses_latest_provider_timestamp) -- the - # permutation-tolerant key gets them INTO that mechanism at all, which is - # the fix; it does not bypass the timestamp requirement. - chronological = MembershipRevision(chronological.raw_id, chronological.projection, "2026-01-01T00:00:00Z") - resequenced = MembershipRevision(resequenced.raw_id, resequenced.projection, "2026-01-02T00:00:00Z") - + # Both fixtures leave title/created_at/updated_at unset (matching), the + # common live shape for a re-export of an untouched conversation: the + # permutation-tolerant content key gets them into the same group, and + # metadata_hash proves the reorder is the ONLY remaining difference -- no + # provider timestamp is needed (Claude.ai's own updated_at commonly + # carries the SAME value across two export requests of unchanged content, + # since re-sequencing an array is not a provider-visible edit). result = classify_membership_revisions([chronological, resequenced]) - assert result.accepted_raw_ids == ("raw-resequenced",) - assert result.equivalent_raw_ids == ("raw-chrono",) + assert result.accepted_raw_ids == ("raw-chrono",) + assert result.equivalent_raw_ids == ("raw-resequenced",) assert result.ambiguous_raw_ids == () @@ -920,17 +922,18 @@ def test_accepts_generation_lifecycle_duration_change_as_equivalent() -> None: assert first_export.projection.event_identity_hashes == second_export.projection.event_identity_hashes assert first_export.projection.session_hash != second_export.projection.session_hash - # Same-content-key revisions still need a provider timestamp to pick a - # representative (matching the pre-existing metadata_variants mechanism) -- - # the measurement-tolerant key gets them INTO that mechanism at all, which - # is the fix; it does not bypass the timestamp requirement. - first_export = MembershipRevision(first_export.raw_id, first_export.projection, "2026-01-01T00:00:00Z") - second_export = MembershipRevision(second_export.raw_id, second_export.projection, "2026-01-02T00:00:00Z") - + # Both fixtures leave title/created_at/updated_at unset (matching), the + # common live shape for a re-export of an untouched conversation: the + # measurement-tolerant content key gets them into the same group, and + # metadata_hash proves the duration change is the ONLY remaining + # difference -- no provider timestamp is needed (a remeasured duration is + # not a provider-visible edit to the conversation itself, so ChatGPT's + # own updated_at commonly carries the SAME value across two export + # requests of the same generation). result = classify_membership_revisions([first_export, second_export]) - assert result.accepted_raw_ids == ("raw-second",) - assert result.equivalent_raw_ids == ("raw-first",) + assert result.accepted_raw_ids == ("raw-first",) + assert result.equivalent_raw_ids == ("raw-second",) assert result.ambiguous_raw_ids == () From 39591e17df3a9a37771e52dbbe301677d2f06c8b Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 30 Jul 2026 17:12:06 +0200 Subject: [PATCH 5/8] refactor(archive): collapse revision comparison into one content-only relation Scope change (polylogue-aggz, filed while c429/nuec/d8al were in review): one day of investigation produced four separately-named volatility fixes (bu1i, c429, nuec, d8al/hith) and correspondingly four special-case code paths in the comparison layer. The operator's directive: stop naming failure cases and make them unrepresentable by one invariant instead. A conversation is a SET of items (messages, attachments, events) keyed by content-derived identity, each carrying only content-bearing fields -- nothing else may enter the value used to compare two acquisitions of it. Concretely: - Identity is never array position. message_contents/attachment_contents/ event_contents are frozensets, not ordered tuples. - Identity is derived from content, never a provider id whose presence is itself unstable. Attachment identity drops the provider id entirely (message_id, name, mime_type only) instead of using it when present and falling back when absent -- the strict/loose duality and its pairwise correlation machinery (_correlate_attachments, _attachments_equivalent, AttachmentRecord, the by_content/by_session_hash/metadata_variants/ timestamp-tiebreak layering) are deleted, not bypassed. - Acquisition state and provider-reported measurement are not content. Event content is built from an explicit per-event-type ALLOWLIST (_EVENT_CONTENT_PAYLOAD_ALLOWLIST) instead of a denylist of fields discovered volatile after shipping -- a new field a parser adds later is excluded from comparison by construction, not by someone remembering to strip it. Two revisions are now `equal` (same id set, equal content per id), `a_contains_b`/`b_contains_a` (one side's id set contains the other's with equal content on the overlap -- set containment, not sequence prefix, so a reorder never breaks it), or `conflict` (content disagrees on the overlap, or each side holds something the other lacks). One `_relation` function replaces the message dominance test, the event-hash prefix test, the attachment correlation pass, and the metadata-timestamp tiebreak. `_provider_ordered_browser_snapshots` was evaluated for deletion (per the same directive) and kept: browser-captured DOM/native snapshots synthesize their own local ids from DOM structure, not stable provider identity, so the content-only relation genuinely cannot correlate a DOM-to-native fidelity upgrade -- deleting it would silently break real browser-capture archival behavior this lane has no fixture coverage to verify against. A maximal-evidence presence-guarantee fallback for irreducible conflicts (deterministic by _frontier + raw_id, proven order-independent) is implemented and unit-tested as _maximal_evidence_fallback but NOT wired into classify_membership_revisions: doing so trips a real, documented archive.py write-back invariant (never retire an unrelated accepted head, with production incident history behind it -- polylogue-miwv, #3397/#3398) whenever the fallback pick differs from an already-established head, proven by two failing integration tests (test_divergent_bundle_member_preserves_last_accepted_session and its sibling). Landing it safely needs either the classifier or its caller to carry existing-head context, or the write-back guard to accept a re-affirmed-quarantined outcome -- both belong to archive.py's write path, out of this lane's scope. Reported as a concrete follow-up rather than forced through. Ref polylogue-aggz Ref polylogue-bu1i Ref polylogue-c429 Ref polylogue-nuec Ref polylogue-d8al Ref polylogue-hith Co-Authored-By: Claude --- docs/plans/hash-boundary-registry.yaml | 16 +- .../archive/session_revision_membership.py | 562 ++++---- polylogue/pipeline/ids.py | 339 ++--- .../test_session_revision_membership.py | 1252 +++++++---------- tests/unit/pipeline/test_pipeline_ids.py | 12 +- 5 files changed, 963 insertions(+), 1218 deletions(-) diff --git a/docs/plans/hash-boundary-registry.yaml b/docs/plans/hash-boundary-registry.yaml index 34564a674f..21fd693f3b 100644 --- a/docs/plans/hash-boundary-registry.yaml +++ b/docs/plans/hash-boundary-registry.yaml @@ -553,25 +553,25 @@ entries: call: hash_payload occurrence: 4 classification: content-hash - note: 'event_identity_hashes -- revision-comparison axis only, provider-reported-elapsed measurement excluded per _event_identity_hash_payload (polylogue-nuec).' + note: 'event base identity (event type + anchoring message), content-derived and position-independent, feeding event_contents (polylogue-aggz, polylogue-nuec).' - path: polylogue/pipeline/ids.py function: '.session_revision_projection' call: hash_payload occurrence: 5 classification: content-hash - note: 'attachment_records loose identity -- id-independent (message, name, mime_type) correlation key used only as a comparison-layer fallback (polylogue-d8al).' + note: 'per-event content hash (measurement excluded via _event_content_payload''s allowlist), feeding event_contents (polylogue-aggz, polylogue-nuec).' - path: polylogue/pipeline/ids.py - function: '._event_identity_hash_payload' + function: '.session_revision_projection' call: hash_payload - occurrence: 0 + occurrence: 6 classification: content-hash - note: 'per-event identity hash feeding event_identity_hashes, measurement fields stripped for provider_reported_elapsed events (polylogue-nuec).' + note: 'canonical event identity fold (base identity + content) used only when the base identity is locally ambiguous within one revision -- still content-derived, never the array index (polylogue-aggz).' - path: polylogue/pipeline/ids.py - function: '.session_revision_projection' + function: '._event_content_payload' call: hash_payload - occurrence: 6 + occurrence: 0 classification: content-hash - note: 'metadata_hash -- title/created_at/updated_at only, lets membership classification tell a genuine metadata edit apart from pure order/duration/id-presence noise within an already-tolerant content-key group (polylogue-c429, polylogue-nuec, polylogue-d8al).' + note: 'per-event content payload feeding event_contents, an explicit per-event-type ALLOWLIST of content-bearing fields rather than a denylist of fields discovered volatile after the fact (polylogue-aggz, polylogue-nuec).' - path: polylogue/scenarios/workload.py function: '._identity' call: hashlib.sha256 diff --git a/polylogue/archive/session_revision_membership.py b/polylogue/archive/session_revision_membership.py index ccaba69877..608d6b78c2 100644 --- a/polylogue/archive/session_revision_membership.py +++ b/polylogue/archive/session_revision_membership.py @@ -2,25 +2,21 @@ from __future__ import annotations -from collections import Counter from dataclasses import dataclass from typing import Literal, TypeAlias from polylogue.core.timestamps import parse_timestamp -from polylogue.pipeline.ids import AttachmentRecord, SessionRevisionProjection - -#: Everything that must agree for two revisions to be the same content: which -#: messages exist and what they say (order-insensitive -- polylogue-c429), -#: the event chain with provider-reported measurement excluded -#: (polylogue-nuec), which attachments exist, and which of their bytes have -#: been read. The attachment component is the strict (id-bearing) identity -- -#: two groups differing only in attachment id *presence* do not share a key -#: here and need the separate correlation-based merge step in -#: ``classify_membership_revisions`` (polylogue-d8al); this key alone would -#: under-merge for that case, never over-merge, so it stays a safe first pass. -_ContentKey: TypeAlias = tuple[ - frozenset[tuple[bytes, bytes]], tuple[bytes, ...], frozenset[bytes], frozenset[tuple[bytes, bytes]] -] +from polylogue.pipeline.ids import SessionRevisionProjection + +#: One content axis's relation between two revisions: total and decidable, +#: no residual category (polylogue-aggz). ``equal`` is the same identity set +#: with equal content per identity. ``a_contains_b``/``b_contains_a`` is one +#: side's identity set containing the other's with equal content on the +#: intersection -- set containment, not sequence prefix, so this subsumes +#: append-only growth without caring about array order. ``conflict`` is +#: either side disagreeing about content under a shared identity, or each +#: side holding an identity the other lacks (a genuine fork). +_Relation: TypeAlias = Literal["equal", "a_contains_b", "b_contains_a", "conflict"] @dataclass(frozen=True, slots=True) @@ -36,150 +32,251 @@ class MembershipRevision: @dataclass(frozen=True, slots=True) class MembershipClassification: + #: Materialized head(s): an append-only growth chain (oldest to newest, + #: by set containment). Empty only when the cohort is a genuine, + #: irreducible conflict (see ``ambiguous_raw_ids``) -- a presence- + #: guarantee fallback that would deterministically pick a maximal- + #: evidence head even then is designed and unit-tested + #: (``_maximal_evidence_fallback``) but not yet wired into this return + #: path; see that function's docstring for why. accepted_raw_ids: tuple[str, ...] + #: Raws proven to carry the SAME content as an accepted head (``equal`` + #: per ``_relation``) -- legitimately superseded, not debt. equivalent_raw_ids: tuple[str, ...] + #: Raws NOT reflected in ``accepted_raw_ids`` because no containment + #: relation could order them against it: unresolved conflict debt + #: attached to the session, retained as raw provenance only (never + #: deleted, never silently discarded). ambiguous_raw_ids: tuple[str, ...] +def _identities(contents: frozenset[tuple[bytes, bytes]]) -> frozenset[bytes]: + """Project the identity half of a (identity, content) pair set. + + Messages and events always carry content for every identity they have -- + unlike attachments, which project identity and content as two separate + fields because an attachment can be referenced before its bytes are + acquired (polylogue-bu1i) -- so their identity set is always exactly + this projection of their content set. + """ + return frozenset(identity for identity, _content in contents) + + +def _axis_relation( + identities_a: frozenset[bytes], + contents_a: frozenset[tuple[bytes, bytes]], + identities_b: frozenset[bytes], + contents_b: frozenset[tuple[bytes, bytes]], +) -> _Relation: + """Compare one content axis (messages, attachments, or events) between two revisions. + + An identity present without content on either side (an attachment + referenced but not yet acquired) is compatible with, never in conflict + with, the SAME identity carrying content on the other side -- unknown is + less evidence, not a disagreement (polylogue-bu1i). For messages and + events every identity always carries content, so this degrades to plain + set equality/containment for those two axes. + """ + content_a = dict(contents_a) + content_b = dict(contents_b) + shared = identities_a & identities_b + for identity in shared: + value_a, value_b = content_a.get(identity), content_b.get(identity) + if value_a is not None and value_b is not None and value_a != value_b: + return "conflict" + a_richer = bool(identities_a - identities_b) or any( + content_a.get(identity) is not None and content_b.get(identity) is None for identity in shared + ) + b_richer = bool(identities_b - identities_a) or any( + content_b.get(identity) is not None and content_a.get(identity) is None for identity in shared + ) + if a_richer and b_richer: + return "conflict" + if a_richer: + return "a_contains_b" + if b_richer: + return "b_contains_a" + return "equal" + + +def _relation(a: SessionRevisionProjection, b: SessionRevisionProjection) -> _Relation: + """Combine the message/attachment/event axes into one overall relation. + + Each axis is independently equal/contains/conflict; the whole document is + ``equal`` only if every axis is, ``a_contains_b``/``b_contains_a`` only if + every non-equal axis agrees on the SAME direction (a mix -- one axis grew + forward, another backward -- is exactly a fork: each side holds + something the other lacks, so it is a conflict too), and ``conflict`` if + any single axis already is. + """ + axes = ( + _axis_relation( + _identities(a.message_contents), a.message_contents, _identities(b.message_contents), b.message_contents + ), + _axis_relation(a.attachment_identities, a.attachment_contents, b.attachment_identities, b.attachment_contents), + _axis_relation( + _identities(a.event_contents), a.event_contents, _identities(b.event_contents), b.event_contents + ), + ) + if "conflict" in axes: + return "conflict" + directions = {axis for axis in axes if axis != "equal"} + if not directions: + return "equal" + if directions == {"a_contains_b"}: + return "a_contains_b" + if directions == {"b_contains_a"}: + return "b_contains_a" + return "conflict" + + +def _frontier(projection: SessionRevisionProjection) -> tuple[int, int, int, int]: + """Rank a revision along every axis on which it can only grow. + + Used only to order candidates for the pairwise ``_relation`` chain check + and to break ties deterministically in the presence-guarantee fallback + (stable regardless of processing order -- a rebuild must always resolve + the same cohort to the same head). + """ + return ( + len(projection.message_contents), + len(projection.event_contents), + len(projection.attachment_identities), + len(projection.attachment_contents), + ) + + def classify_membership_revisions(revisions: list[MembershipRevision]) -> MembershipClassification: - """Accept one total strict-growth chain; never choose between branches.""" + """Accept one total growth chain by set containment; never choose a branch silently. + + Two revisions are the same content, contain each other, or conflict -- + total and decidable, with no residual category (polylogue-aggz). This + replaces what used to be four separate mechanisms (a positional-prefix + message dominance test, a denylist-stripped ordered event-hash prefix + test, a strict-vs-loose attachment id correlation pass, and a + session-hash/metadata-timestamp tiebreak layered on top of all three) + with one relation, applied uniformly to every axis. + + A genuine, irreducible conflict -- no containment chain exists at all, + not export-vintage noise -- still quarantines every representative + (``ambiguous_raw_ids``) with nothing accepted, after browser-fidelity and + direct-export precedence have also failed to order the cohort. A + presence-guarantee alternative to that quarantine -- deterministically + materializing the maximal-evidence representative instead of withholding + every revision, with the conflict recorded as debt rather than absence + -- is designed and unit-tested (``_maximal_evidence_fallback``) but not + wired in here; see that function's docstring for the concrete, + proven reason (a real archive.py write-back invariant it would violate). + """ if not revisions: return MembershipClassification((), (), ()) - # Keyed on acquisition state as well as identity: two revisions are the same - # content only when they also agree on which attachment bytes are in hand, - # otherwise collapsing them as equivalents could discard the one that - # actually carries the bytes. - by_content: dict[_ContentKey, list[MembershipRevision]] = {} - for revision in revisions: - projection = revision.projection - key = ( - projection.message_contents, - projection.event_identity_hashes, - projection.attachment_identities, - projection.attachment_contents, - ) - by_content.setdefault(key, []).append(revision) + representatives: list[MembershipRevision] = [] equivalents: list[str] = [] - for group in _merge_attachment_id_presence_variants(by_content): - by_session_hash: dict[bytes, list[MembershipRevision]] = {} - for item in group: - by_session_hash.setdefault(item.projection.session_hash, []).append(item) - metadata_variants: list[MembershipRevision] = [] - for hash_group in by_session_hash.values(): - representative = min(hash_group, key=lambda item: item.raw_id) - metadata_variants.append(representative) - equivalents.extend(item.raw_id for item in hash_group if item.raw_id != representative.raw_id) - if len(metadata_variants) == 1: - representatives.extend(metadata_variants) - continue - # A same-content-key group can still split into multiple session_hash - # sub-groups for two different reasons: a genuine title/created_at/ - # updated_at edit (needs the timestamp tie-break below), or pure - # serialization noise along an axis the content key already tolerates - # -- message order, attachment id presence, event-duration - # measurement -- with every OTHER session_hash input identical. The - # second shape is the common one for a re-export of an untouched - # conversation: the provider's own updated_at legitimately never - # moves, so a distinct-timestamp tie-break can never fire and the - # group would otherwise stay ambiguous forever (polylogue-c429, - # polylogue-nuec, polylogue-d8al). metadata_hash pins down which - # shape this is: if every variant agrees on title/created_at/ - # updated_at, the ONLY remaining source of a session_hash difference - # is one of the tolerated axes, so picking any one is exactly as safe - # as the exact-session_hash collapse a few lines above. - if len({item.projection.metadata_hash for item in metadata_variants}) == 1: - representative = min(metadata_variants, key=lambda item: item.raw_id) - representatives.append(representative) - equivalents.extend(item.raw_id for item in metadata_variants if item.raw_id != representative.raw_id) + for revision in revisions: + match_index = next( + (i for i, rep in enumerate(representatives) if _relation(rep.projection, revision.projection) == "equal"), + None, + ) + if match_index is None: + representatives.append(revision) continue - timestamped = [ - (parsed.timestamp(), item) - for item in metadata_variants - if (parsed := parse_timestamp(item.provider_updated_at)) is not None - ] - timestamps = [timestamp for timestamp, _item in timestamped] - if len(timestamped) == len(metadata_variants) and len(set(timestamps)) == len(timestamps): - representative = max(timestamped, key=lambda pair: pair[0])[1] - representatives.append(representative) - equivalents.extend(item.raw_id for item in metadata_variants if item.raw_id != representative.raw_id) + incumbent = representatives[match_index] + incumbent_time = parse_timestamp(incumbent.provider_updated_at) + candidate_time = parse_timestamp(revision.provider_updated_at) + if ( + incumbent_time is not None + and candidate_time is not None + and candidate_time.timestamp() != incumbent_time.timestamp() + ): + winner, loser = ( + (revision, incumbent) + if candidate_time.timestamp() > incumbent_time.timestamp() + else (incumbent, revision) + ) else: - representatives.extend(metadata_variants) + # No distinguishing provider timestamp -- these two are already + # proven identical content, so which raw_id represents them does + # not matter for correctness; pick deterministically rather than + # requiring a timestamp that a re-export of an untouched + # conversation may never carry (its own provider updated_at + # legitimately does not move when nothing provider-visible + # changed). + winner, loser = sorted((incumbent, revision), key=lambda item: item.raw_id) + representatives[match_index] = winner + equivalents.append(loser.raw_id) + representatives.sort(key=lambda item: (_frontier(item.projection), item.raw_id)) - if any( - not _strictly_dominates(older.projection, newer.projection) + conflict = any( + _relation(older.projection, newer.projection) != "b_contains_a" for older, newer in zip(representatives, representatives[1:], strict=False) - ): - browser_order = _provider_ordered_browser_snapshots(representatives) - if browser_order is None: - direct_export = _direct_export_precedence(representatives) - if direct_export is None: - return MembershipClassification( - (), - tuple(sorted(equivalents)), - tuple(sorted(item.raw_id for item in representatives)), - ) - accepted, browser_capture_raw_ids = direct_export - return MembershipClassification( - (accepted.raw_id,), - tuple(sorted((*equivalents, *browser_capture_raw_ids))), - (), - ) - representatives = browser_order + ) + if not conflict: + return MembershipClassification( + tuple(item.raw_id for item in representatives), + tuple(sorted(equivalents)), + (), + ) + + browser_order = _provider_ordered_browser_snapshots(representatives) + if browser_order is not None: + return MembershipClassification( + tuple(item.raw_id for item in browser_order), + tuple(sorted(equivalents)), + (), + ) + direct_export = _direct_export_precedence(representatives) + if direct_export is not None: + accepted, browser_capture_raw_ids = direct_export + return MembershipClassification( + (accepted.raw_id,), + tuple(sorted((*equivalents, *browser_capture_raw_ids))), + (), + ) + # _maximal_evidence_fallback (below) is the intended long-term behavior + # for a genuine, irreducible conflict -- see its docstring for why it is + # not called here yet. Until it is wired in, an irreducible conflict is + # quarantined exactly as it always was: every representative in + # `ambiguous_raw_ids`, nothing accepted. return MembershipClassification( - tuple(item.raw_id for item in representatives), - tuple(sorted(equivalents)), (), + tuple(sorted(equivalents)), + tuple(sorted(item.raw_id for item in representatives)), ) -def _merge_attachment_id_presence_variants( - by_content: dict[_ContentKey, list[MembershipRevision]], -) -> list[list[MembershipRevision]]: - """Merge content groups that differ only in attachment id presence. - - The strict content key built in ``classify_membership_revisions`` never - merges a real-id/synthetic-id pair for the same physical attachment into - one group: a provider's export can omit a stable id for the same - attachment on a different export request of the same conversation, so an - otherwise byte-identical pair's attachment sets hash to disjoint strict - identities and the two groups never meet in ``by_content`` (polylogue-d8al). - This pass merges any two groups whose message/event portion of the key - already matches exactly and whose attachments correlate as fully - equivalent (``_attachments_equivalent``): same cardinality, every - attachment pairwise-matched by strict id or, when unambiguous on both - sides, by the id-independent key, and no content contradiction. Only ever - merges groups the strict key under-merged -- it can never combine two - groups that genuinely differ in message or event content, so this cannot - introduce a false equivalence on those axes. +def _maximal_evidence_fallback(representatives: list[MembershipRevision]) -> MembershipRevision: + """Deterministically select the maximal-evidence revision for an irreducible conflict. + + Designed and unit-tested as the presence-guarantee alternative to + quarantining every representative with nothing accepted: a document + should never simply vanish from the archive just because arbitration + could not order its revisions. Frontier order (most messages, most + events, most attachments, most acquired bytes) with a stable raw_id + tiebreak means the same cohort always resolves to the same head + regardless of processing order -- verified directly by + ``test_presence_guarantee_fallback_is_order_independent``. + + NOT called by ``classify_membership_revisions`` yet. Wiring it in would + make ``accepted_raw_ids`` non-empty for every non-empty input, moving the + other representatives into ``ambiguous_raw_ids`` as recorded conflict + debt instead of leaving the whole cohort headless. That trips a real, + carefully-designed invariant in + ``archive.py``'s ``apply_raw_membership_classification`` write-back, with + documented production incident history behind it (polylogue-miwv, PR + #3211): once a logical source has an accepted head, a later membership + pass may not silently retire it in favor of an unrelated raw unless that + raw is part of the new accepted/equivalent set. A fallback pick can + legitimately differ from an already-established head -- proven by two + real integration-test failures during verification + (``test_divergent_bundle_member_preserves_last_accepted_session`` and + its sibling), not a theoretical concern. Landing this safely needs + either the classifier or its caller to carry the existing head's raw_id + into this decision, or the write-back guard to accept a + re-affirmed-quarantined outcome explicitly -- both changes belong to + archive.py's write path, out of this lane's scope. """ - entries = list(by_content.items()) - parent = list(range(len(entries))) - - def find(index: int) -> int: - while parent[index] != index: - parent[index] = parent[parent[index]] - index = parent[index] - return index - - def union(left: int, right: int) -> None: - root_left, root_right = find(left), find(right) - if root_left != root_right: - parent[root_right] = root_left - - for i in range(len(entries)): - key_i, revisions_i = entries[i] - for j in range(i + 1, len(entries)): - key_j, revisions_j = entries[j] - if key_i[0] != key_j[0] or key_i[1] != key_j[1]: - continue - if _attachments_equivalent(revisions_i[0].projection, revisions_j[0].projection): - union(i, j) - - merged: dict[int, list[MembershipRevision]] = {} - for i, (_key, revisions) in enumerate(entries): - merged.setdefault(find(i), []).extend(revisions) - return list(merged.values()) + return max(representatives, key=lambda item: (_frontier(item.projection), item.raw_id)) def _provider_ordered_browser_snapshots( @@ -187,14 +284,22 @@ def _provider_ordered_browser_snapshots( ) -> list[MembershipRevision] | None: """Order compatible mutable browser snapshots by provider authority. - Browser-native payloads are complete snapshots, not append logs. ChatGPT - can move an already-present context/tool node when later work appears, and - can complete text in place under the same provider message id. A strict - serialized-content prefix therefore rejects ordinary provider progress. - Provider timestamps may resolve that progress only when stable message and - attachment identities are preserved. DOM-to-native is the sole fidelity - upgrade and may use different synthetic ids; a native-to-DOM downgrade is - never selected. + Kept as a genuine exception to the content-only relation above, not a + dead alternate path: browser-captured DOM and native snapshots synthesize + their OWN local message/attachment ids from DOM structure, not a stable + provider identity, so the same underlying message can carry a different + synthetic id in a DOM capture than in a native one -- the content-only + ``_relation`` genuinely cannot correlate them (it would read a + DOM-to-native fidelity upgrade as two disjoint, conflicting message id + sets). ``provider_message_ids``/``provider_attachment_ids`` on + ``MembershipRevision`` carry the browser-capture layer's own identity + evidence for exactly this reason. ChatGPT can also move an already-present + context/tool node when later work appears, and can complete text in place + under the same provider message id, so a strict serialized-content + comparison would reject ordinary provider progress; provider timestamps + resolve that progress only when stable message and attachment identities + are preserved. DOM-to-native is the sole fidelity upgrade and may use + different synthetic ids; a native-to-DOM downgrade is never selected. """ if not revisions or any(item.browser_snapshot_fidelity is None for item in revisions): @@ -227,7 +332,7 @@ def _direct_export_precedence( membership group carries no browser-capture provenance at all (``browser_snapshot_fidelity is None``) and at least one sibling is browser-capture-sourced, the non-browser candidate is authoritative for - session content regardless of byte-growth or provider-timestamp + session content regardless of set-containment or provider-timestamp comparisons; the browser-capture siblings are retained as raw provenance only, not indexed content (polylogue-z1c6). Returns ``None`` when this specific shape does not apply (zero or multiple non-browser @@ -273,165 +378,4 @@ def _browser_snapshot_dominates(older: MembershipRevision, newer: MembershipRevi ) -def _frontier(projection: SessionRevisionProjection) -> tuple[int, int, int, int]: - """Rank a revision along every axis on which it can only grow. - - Attachment acquisition is its own axis. Without it, a revision that added - nothing but the bytes of attachments it already referenced tied with its - predecessor on every dimension, the sort fell through to ``raw_id``, and the - dominance test was then run in whichever direction the hex happened to - order -- half the time backwards, against a revision that genuinely does - dominate (polylogue-bu1i). - """ - return ( - len(projection.message_contents), - len(projection.event_identity_hashes), - len(projection.attachment_identities), - len(projection.attachment_contents), - ) - - -def _message_evidence_preserved( - older: SessionRevisionProjection, - newer: SessionRevisionProjection, -) -> bool: - """True when ``newer`` loses no message identity and contradicts no shared content. - - A provider's export ordering is not guaranteed stable across separate - export requests for the SAME conversation -- Claude.ai's own tree - flattening can interleave edited-message siblings differently from one - export to the next even though every message's id, role, text, and - timestamp are byte-identical (polylogue-c429). Array position is - therefore not treated as identity here: this checks only that every - ``(identity, content)`` pair present in ``older`` is still present in - ``newer``. Unlike attachments, a message is never lazily fetched -- its - content is always known when it exists -- so there is no separate - identity-only membership to check: an id that disappeared from ``newer`` - fails this lookup on its own (``.get()`` returns ``None``, which never - equals a real content hash), and an id whose content actually changed - fails it too. Both are real divergence and are refused. - """ - newer_contents = dict(newer.message_contents) - return all(newer_contents.get(identity) == content for identity, content in older.message_contents) - - -def _correlate_attachments( - older: SessionRevisionProjection, newer: SessionRevisionProjection -) -> tuple[list[tuple[AttachmentRecord, AttachmentRecord]], list[AttachmentRecord], list[AttachmentRecord]]: - """Pair each attachment in ``older`` with its counterpart in ``newer``. - - Matches by strict identity first (provider id, anchoring message, name, - media type) -- the exact bu1i behavior when both revisions agree on a - provider id. When a provider omits a stable id for the same attachment on - a different export request, the two revisions never share a strict - identity for it (polylogue-d8al): Claude.ai does not consistently emit an - id field for the same attachment across separate export requests of the - same conversation. In that case matching falls back to the - id-independent (anchoring message, name, media type) key, but ONLY when - that looser key is unambiguous on BOTH sides being compared (exactly one - attachment carries it in ``older`` and exactly one in ``newer``): if - either revision has two attachments sharing the same anchor/name/media - type, there is no way to tell which is which without an id, and guessing - by array position is exactly the bug this replaces -- that ambiguous case - is left uncorrelated rather than resolved by a guess. - - Returns matched ``(older, newer)`` record pairs, ``older`` records with no - counterpart in ``newer`` (a potential loss), and ``newer`` records with no - counterpart in ``older`` (growth). - """ - newer_by_identity = {record[0]: record for record in newer.attachment_records} - newer_loose_counts = Counter(record[1] for record in newer.attachment_records) - newer_by_loose = {record[1]: record for record in newer.attachment_records} - older_loose_counts = Counter(record[1] for record in older.attachment_records) - - matched: list[tuple[AttachmentRecord, AttachmentRecord]] = [] - unmatched_older: list[AttachmentRecord] = [] - matched_newer_identities: set[bytes] = set() - for older_record in older.attachment_records: - identity, loose_identity, _content = older_record - newer_record = newer_by_identity.get(identity) - if ( - newer_record is None - and older_loose_counts[loose_identity] == 1 - and newer_loose_counts.get(loose_identity) == 1 - ): - newer_record = newer_by_loose.get(loose_identity) - if newer_record is None: - unmatched_older.append(older_record) - continue - matched.append((older_record, newer_record)) - matched_newer_identities.add(newer_record[0]) - unmatched_newer = [record for record in newer.attachment_records if record[0] not in matched_newer_identities] - return matched, unmatched_older, unmatched_newer - - -def _attachments_equivalent(a: SessionRevisionProjection, b: SessionRevisionProjection) -> bool: - """True when every attachment in ``a`` and ``b`` correlates 1:1 with identical content. - - Used only to decide whether two otherwise-identical content groups may - merge as the same content (``_merge_attachment_id_presence_variants``); - unlike ``_attachment_evidence_preserved`` this is symmetric and requires - an exact match on both sides, not merely no loss in one direction. - """ - matched, unmatched_a, unmatched_b = _correlate_attachments(a, b) - if unmatched_a or unmatched_b: - return False - return all(a_record[2] == b_record[2] for a_record, b_record in matched) - - -def _attachment_evidence_preserved( - older: SessionRevisionProjection, - newer: SessionRevisionProjection, -) -> bool: - """True when ``newer`` loses no attachment and contradicts no fetched bytes. - - Two distinct regressions are refused here. An attachment whose bytes were - read and are no longer present in the newer revision is a fidelity - *downgrade*: the newer revision knows strictly less, so accepting it would - silently mark an acquired attachment unfetched. An attachment present in - both but carrying *different* bytes is a genuine conflict -- two sources - disagree about content under one identity -- and no ordering rule can - resolve that, so the cohort stays ambiguous. Correlation - (``_correlate_attachments``) is what lets this hold across an id-presence - mismatch the same way it always did for a plain matching id - (polylogue-d8al). - """ - matched, unmatched_older, _unmatched_newer = _correlate_attachments(older, newer) - if unmatched_older: - return False - return all(older_record[2] is None or older_record[2] == newer_record[2] for older_record, newer_record in matched) - - -def _attachment_axis_grew(older: SessionRevisionProjection, newer: SessionRevisionProjection) -> bool: - """True when ``newer`` has a genuinely new attachment or newly-read bytes. - - An uncorrelated attachment in ``newer`` (no older counterpart, by strict - id or unambiguous loose key) is growth. So is resolving the bytes of an - already-referenced attachment -- growth in evidence even when the - transcript is untouched, the shape a lazily-fetched attachment produces - on its second acquisition (polylogue-bu1i). - """ - matched, _unmatched_older, unmatched_newer = _correlate_attachments(older, newer) - if unmatched_newer: - return True - return any(older_record[2] is None and newer_record[2] is not None for older_record, newer_record in matched) - - -def _strictly_dominates(older: SessionRevisionProjection, newer: SessionRevisionProjection) -> bool: - content_grew = ( - # Order-insensitive: a permuted-but-otherwise-equal message set is not - # growth (equal count), but a genuinely appended or resurfaced message - # id is (polylogue-c429). - len(newer.message_contents) > len(older.message_contents) - or len(newer.event_identity_hashes) > len(older.event_identity_hashes) - or _attachment_axis_grew(older, newer) - ) - return ( - content_grew - and _message_evidence_preserved(older, newer) - and older.event_identity_hashes == newer.event_identity_hashes[: len(older.event_identity_hashes)] - and _attachment_evidence_preserved(older, newer) - ) - - __all__ = ["MembershipClassification", "MembershipRevision", "classify_membership_revisions"] diff --git a/polylogue/pipeline/ids.py b/polylogue/pipeline/ids.py index 3863ae3674..208b08af3c 100644 --- a/polylogue/pipeline/ids.py +++ b/polylogue/pipeline/ids.py @@ -3,6 +3,7 @@ from __future__ import annotations import unicodedata +from collections import Counter from collections.abc import Mapping from dataclasses import dataclass from typing import TYPE_CHECKING, TypeAlias @@ -30,90 +31,64 @@ _EMPTY_SENTINEL = "__POLYLOGUE_EMPTY__" HashScalar: TypeAlias = str | int | float | bool | None -#: One attachment's (strict identity, loose/id-independent identity, content -#: hash or ``None`` if unacquired) triple -- see ``SessionRevisionProjection`` -#: for why the comparison layer needs both identities (polylogue-d8al). -AttachmentRecord: TypeAlias = tuple[bytes, bytes, bytes | None] - @dataclass(frozen=True, slots=True) class SessionRevisionProjection: - """Canonical content hashes used to prove append-only session growth. - - Attachments are projected on two axes deliberately, because folding them - into one hash makes acquisition look like divergence. ``attachment_identities`` - answers *which attachment is this* (provider id, anchoring message, name, - media type) and never changes once a provider has emitted the reference. - ``attachment_contents`` answers *have we read its bytes yet, and which bytes* - -- it carries an ``(identity, content)`` pair only for attachments whose - bytes are actually in hand. - - A single hash over both axes made an ordinary lazy fetch indistinguishable - from a branch: a Drive/Gemini document acquired before and after its - attachment bytes were resolved produced equal-cardinality *disjoint* hash - sets, so the later revision was neither a superset nor a prefix of the - earlier one and the whole cohort was quarantined as ambiguous. Splitting the - axes lets a dominance test say what is actually true -- same attachments, - strictly more of their bytes now known (polylogue-bu1i). - - ``attachment_records`` carries the same per-attachment data as a - ``(strict identity, loose identity, content)`` triple instead of two - separate sets, so the *comparison layer* (``session_revision_membership.py``) - can correlate attachments across two revisions even when a provider's - export omits a stable id for the same physical attachment on a different - export request of the same conversation -- one vintage has a real UUID, - the other has none and a parser synthesizes one, and no synthetic-id - scheme can make a real id and a synthetic hash collide by construction - (polylogue-d8al). The "loose" identity drops the provider id and keeps - only the anchoring message, name, and media type; - ``attachment_identities``/``attachment_contents`` stay strict (id - included) so this projection's own equality/hashing behavior is - unchanged -- only the membership module's pairwise correlation consults - the loose key, and only as a fallback when the strict identity does not - match and the loose key is unambiguous on both sides being compared. - Known, accepted limit stated explicitly rather than engineered around: - two genuinely distinct attachments that share one message/name/media-type - and carry no bytes on either side of a comparison are indistinguishable - by any signal this projection can offer. - - Messages get an analogous split, for the same reason applied to a - different volatility source: a provider's own export can replay an - unchanged message set in a different array sequence across separate export - requests (Claude.ai's own tree flattening is not guaranteed to serialize - the same way twice). ``message_contents`` pairs each message's identity - hash (its provider message id) with a hash of its content - (role/text/timestamp/blocks) -- unlike an attachment, a message is never - lazily fetched, so there is no separate identity-only axis to track: - ``message_contents`` alone answers both *which messages exist* and *what - do they say*. Order lives only in ``message_hashes`` (kept for - ``session_hash`` and diagnostics) -- ``message_contents`` is - order-insensitive by construction, so a bare permutation of the same - id-to-content mapping is not read as divergence (polylogue-c429). - - ``event_identity_hashes`` strips designated provider-reported-measurement - fields (see ``_PROVIDER_REPORTED_ELAPSED_VOLATILE_PAYLOAD_KEYS``) out of - events whose own payload declares them non-durable - (``duration_semantics == "provider_reported_elapsed"``) before hashing. - ChatGPT's ``generation_lifecycle`` event re-derives its - ``elapsed_duration_ms`` from the raw export's own timing metadata on every - export request, and that value is not stable across requests for the same - generation -- folding it into revision identity made byte-identical - conversations look like divergent branches on every re-export - (polylogue-nuec). ``event_hashes`` (unstripped, order-preserving) remains - unchanged for ``session_hash`` and diagnostics. - - ``metadata_hash`` covers exactly the ``session_hash`` payload fields - OTHER than messages/attachments/session_events -- title, created_at, - updated_at -- normalized the same way. Membership classification uses it - to tell apart two reasons a same-content-key group can still carry - different ``session_hash`` values: a genuine title/timestamp edit (needs - the existing provider-timestamp tie-break), versus pure serialization - noise along an axis this projection already tolerates (message order, - attachment id presence, event-duration measurement) with the provider's - own metadata otherwise unchanged -- the common shape for a re-export of - an untouched conversation, where ``updated_at`` legitimately never moves - and a distinct-timestamp tie-break can never fire (polylogue-c429, - polylogue-nuec, polylogue-d8al). + """Canonical content-only comparison value for a session revision. + + One invariant governs every field below: *a conversation is a SET of + items keyed by content-derived identity, each carrying only + content-bearing fields -- nothing else may enter the value used to + compare two acquisitions of it* (polylogue-aggz). Concretely: + + - Identity is never array position. ``message_contents``, + ``attachment_contents``, and ``event_contents`` are ``frozenset``s, not + ordered tuples -- a provider's export can replay an unchanged item set + in a different array sequence across separate export requests (proven + for Claude.ai messages and ChatGPT ``generation_lifecycle`` events: + same items, different sequence, every re-export), and a set has no + order to violate (polylogue-c429, polylogue-nuec). + - Identity is derived from content, never a provider id whose PRESENCE + is itself unstable. ``attachment_contents``' key is + ``(anchoring message, name, media type)`` -- never the provider's own + attachment id, which Claude.ai does not consistently emit for the same + attachment across export vintages (one vintage carries a real UUID, + the other has none) -- no id-minting scheme can make a real id and a + synthetic one collide, so the id is simply not part of identity + (polylogue-d8al, polylogue-hith). The same reasoning selects + ``(event type, anchoring message)`` for events, folding in the event's + own content hash only when that pair is ambiguous within one revision + (e.g. more than one ``chatgpt_block_metadata`` event on one message, + one per block) -- still content-derived, never the array index. + - Acquisition state and provider-reported measurement are not content. + An attachment's bytes may be known or not (``attachment_contents`` + omits an identity until its bytes are read, while + ``attachment_identities`` already knows the reference exists) -- + resolving them is evidence *growing*, not the attachment becoming a + different one (polylogue-bu1i). ChatGPT's ``generation_lifecycle`` + event re-derives ``elapsed_duration_ms`` from the raw export's own + timing metadata on every export request, and the value is not stable + across requests for the SAME generation even when the transcript is + byte-identical -- excluded from ``event_contents`` by + ``_EVENT_CONTENT_PAYLOAD_ALLOWLIST``, an explicit per-event-type + ALLOWLIST of content-bearing payload fields rather than a denylist of + fields discovered volatile after the fact (three volatility axes were + each found only after shipping: acquisition state, array order, + provider-reported duration -- a denylist means the next provider + quirk is silently invisible to comparison until someone notices and + files a bead; an allowlist means a NEW field a parser adds later + cannot silently enter identity without an explicit decision to add it). + + ``session_hash``, ``message_hashes`` (ordered), and ``event_hashes`` + (ordered, unstripped) are UNCHANGED by any of this: they still cover the + full, order-sensitive, unstripped payload, so a real reorder, a real + duration change, or newly-acquired bytes still change the session's + content hash and still trigger a re-write (idempotency is a different + question from revision *comparison*, and only the latter is + content-only). ``attachment_identities`` is kept as a plain + ``frozenset[bytes]`` of the same content-derived keys, read by + ``storage/repair.py``/``archive.py`` only via ``len()`` for a frontier + count. """ session_hash: bytes @@ -121,10 +96,8 @@ class SessionRevisionProjection: message_contents: frozenset[tuple[bytes, bytes]] attachment_identities: frozenset[bytes] attachment_contents: frozenset[tuple[bytes, bytes]] - attachment_records: tuple[AttachmentRecord, ...] event_hashes: tuple[bytes, ...] - event_identity_hashes: tuple[bytes, ...] - metadata_hash: bytes + event_contents: frozenset[tuple[bytes, bytes]] def _normalize_nested_for_hash(value: object) -> object: @@ -254,14 +227,29 @@ def _message_identity_payload(payload: dict[str, JSONValue]) -> dict[str, JSONVa return {field: payload[field] for field in _MESSAGE_IDENTITY_FIELDS} -#: Fields of an attachment hash payload that answer *which attachment is this*, -#: as opposed to *what have we managed to read about it*. ``size_bytes`` is -#: excluded on purpose: for lazily-fetched attachments (Drive/Gemini references, -#: browser capture) the provider states no size until the bytes are actually -#: read, so treating it as identity makes acquisition look like a different -#: attachment. ``inline_content_hash`` is excluded for the same reason and is -#: recovered separately as acquisition evidence. -_ATTACHMENT_IDENTITY_FIELDS = ("id", "message_id", "name", "mime_type") +#: Fields of an attachment hash payload that answer *which attachment is +#: this*, as opposed to *what have we managed to read about it* -- the +#: anchoring message plus content descriptors, content-derived and never the +#: provider's own attachment id. Claude.ai does not consistently emit +#: ``id``/``file_id``/``fileId``/``uuid``/``file_uuid`` for the same +#: attachment across separate export requests of the same conversation: one +#: vintage carries a real UUID-shaped id, the other has none (a positionally +#: -seeded synthetic id is not identity either -- polylogue-hith). No +#: id-minting scheme can make a real id and any synthetic value collide by +#: construction, so the id is excluded from identity altogether rather than +#: used when present (polylogue-d8al). ``size_bytes`` is excluded because for +#: lazily-fetched attachments (Drive/Gemini references, browser capture) the +#: provider states no size until the bytes are actually read, so treating it +#: as identity would make acquisition look like a different attachment +#: (polylogue-bu1i). ``inline_content_hash`` is excluded for the same reason +#: and is recovered separately as acquisition evidence in +#: ``attachment_contents``. +#: +#: Known, accepted limit stated explicitly rather than engineered around: two +#: genuinely distinct attachments that share one message/name/media-type and +#: carry no bytes on either side of a comparison are indistinguishable by any +#: signal this projection can offer. +_ATTACHMENT_IDENTITY_FIELDS = ("message_id", "name", "mime_type") def _attachment_hash_payload(attachment: ParsedAttachment) -> dict[str, JSONValue]: @@ -288,26 +276,6 @@ def _attachment_identity_payload(payload: dict[str, JSONValue]) -> dict[str, JSO return {field: payload[field] for field in _ATTACHMENT_IDENTITY_FIELDS} -#: The subset of ``_ATTACHMENT_IDENTITY_FIELDS`` that survives even when a -#: provider omits a stable id for the same physical attachment on a different -#: export request (polylogue-d8al): Claude.ai does not consistently emit -#: ``id``/``file_id``/``fileId``/``uuid``/``file_uuid`` for the same -#: attachment across separate export requests of the same conversation -- one -#: vintage carries a real UUID-shaped id, the other has none and a parser -#: synthesizes one instead. No synthetic-id scheme can make a real id and a -#: synthetic hash collide by construction, so revision comparison correlates -#: by this looser, id-independent key when the strict identity does not -#: match (see ``session_revision_projection``'s canonicalization step). -#: ``size_bytes`` stays excluded for the same lazy-fetch reason it is excluded -#: from ``_ATTACHMENT_IDENTITY_FIELDS``. -_ATTACHMENT_LOOSE_IDENTITY_FIELDS = ("message_id", "name", "mime_type") - - -def _attachment_loose_identity_payload(payload: dict[str, JSONValue]) -> dict[str, JSONValue]: - """Project the id-independent correlation key of one attachment payload.""" - return {field: payload[field] for field in _ATTACHMENT_LOOSE_IDENTITY_FIELDS} - - #: `generation_lifecycle` payload keys that are provider-reported measurement, #: not identity, when the event's own payload declares them non-durable via #: ``duration_semantics == "provider_reported_elapsed"``. ChatGPT re-derives @@ -321,31 +289,56 @@ def _attachment_loose_identity_payload(payload: dict[str, JSONValue]) -> dict[st _PROVIDER_REPORTED_ELAPSED_MARKER_KEY = "duration_semantics" _PROVIDER_REPORTED_ELAPSED_MARKER_VALUE = "provider_reported_elapsed" - -def _event_identity_hash_payload(event: ParsedSessionEvent, event_index: int) -> dict[str, JSONValue]: - """Build an event hash payload with provider-reported-elapsed measurement excluded. - - Mirrors ``_attachment_identity_payload``'s identity/acquisition split for a - different volatility source: an event that labels itself - ``duration_semantics: "provider_reported_elapsed"`` carries a measurement, - not content, in ``_PROVIDER_REPORTED_ELAPSED_VOLATILE_PAYLOAD_KEYS`` -- - stripped here before hashing. The event's own ``timestamp`` is stripped - alongside it for the same events: ChatGPT sets it from the same - ``reasoning_end_time`` value the duration is derived from, so it varies in - tandem and is measurement too, not identity. ``session_hash`` still covers - the full, unstripped payload and timestamp (see ``session_hash_payload``), - so a real change in reported duration still triggers a re-write; only the - *revision comparison* axis (``event_identity_hashes``) is tolerant. +#: Event payload ALLOWLIST by event type: only these fields, per type, ever +#: enter ``event_contents``. This is deliberately an allowlist, not a +#: denylist of fields discovered volatile after the fact -- three separate +#: volatility axes (attachment acquisition state, message array order, +#: provider-reported generation duration) were each found only after +#: shipping, one bead and one branch at a time, because a denylist design +#: means a NEW field silently enters identity the moment a parser starts +#: emitting it, until someone notices and adds it to the strip-list. Under +#: an allowlist, a field a parser adds later is excluded from comparison by +#: construction -- it takes an explicit decision to add it here before it +#: can affect identity, not an explicit decision to exclude it. +#: +#: Event types with no entry here compare their FULL payload (today's +#: behavior for every type except ``generation_lifecycle``, which is the +#: only one with proven volatility -- polylogue-nuec; the raw export's own +#: ``finished_duration_sec``/``reasoning_start_time``/``reasoning_end_time`` +#: metadata is not stable across separate export requests for the SAME +#: generation, so ``elapsed_duration_ms``/``started_at_ms``/``ended_at_ms`` +#: and the derived ``timestamp`` are excluded; ``duration_semantics`` merely +#: documents that fact and carries no content of its own). +_EVENT_CONTENT_PAYLOAD_ALLOWLIST: dict[str, frozenset[str]] = { + "generation_lifecycle": frozenset({"state", "evidence_source", "fidelity"}), +} + + +def _event_content_payload(event: ParsedSessionEvent) -> dict[str, JSONValue]: + """Build the position- and measurement-independent CONTENT payload for one event. + + Array position is not identity for events any more than for messages: + ChatGPT's ``generation_lifecycle`` events were independently observed to + reorder alongside their duration values across separate export requests + of the SAME conversation (same three durations, different array + positions each time) -- the same volatility polylogue-c429 found for + messages, on a different axis (polylogue-nuec). This never includes + ``event_index``; ``session_revision_projection`` builds identity purely + from this content plus the event's own type and anchoring message. """ - payload = event.payload - timestamp = event.timestamp - if payload.get(_PROVIDER_REPORTED_ELAPSED_MARKER_KEY) == _PROVIDER_REPORTED_ELAPSED_MARKER_VALUE: - payload = { - key: value for key, value in payload.items() if key not in _PROVIDER_REPORTED_ELAPSED_VOLATILE_PAYLOAD_KEYS - } + allowlist = _EVENT_CONTENT_PAYLOAD_ALLOWLIST.get(event.event_type) + if allowlist is None: + payload = event.payload + timestamp = event.timestamp + else: + payload = {key: value for key, value in event.payload.items() if key in allowlist} + # An event type with a registered allowlist also has its own + # provider-remeasured timestamp excluded: for generation_lifecycle, + # ChatGPT sets it from the same reasoning_end_time value the + # duration is derived from, so it varies in tandem and is + # measurement too, not content. timestamp = None return { - "event_index": event_index, "event_type": _normalize_for_hash(event.event_type), "timestamp": _normalize_for_hash(timestamp), "source_message_provider_id": _normalize_for_hash(event.source_message_provider_id), @@ -353,6 +346,23 @@ def _event_identity_hash_payload(event: ParsedSessionEvent, event_index: int) -> } +#: The subset of an event content payload that answers *which event slot is +#: this*, as opposed to *what does it say*: anchoring message plus event +#: type, content-derived and never the array index. Ambiguous within one +#: revision when more than one event shares both (e.g. multiple +#: ``chatgpt_block_metadata`` events on the same message, one per block) -- +#: ``session_revision_projection`` folds the event's own content hash into +#: identity for those specific events, which is STILL content-derived (each +#: block's own content, including any content-intrinsic field such as +#: ``block_index``, already differs), never the array position. +_EVENT_BASE_IDENTITY_FIELDS = ("event_type", "source_message_provider_id") + + +def _event_base_identity_payload(payload: dict[str, JSONValue]) -> dict[str, JSONValue]: + """Project the position-independent base correlation key of one event payload.""" + return {field: payload[field] for field in _EVENT_BASE_IDENTITY_FIELDS} + + def _session_hash_payload( *, title: str | None, @@ -463,13 +473,15 @@ def session_revision_projection(convo: ParsedSession) -> SessionRevisionProjecti content hash and does trigger a re-write. Only the *revision comparison* axes separate identity from acquisition (polylogue-bu1i). - The same holds for message order (polylogue-c429) and provider-reported - generation-duration measurement (polylogue-nuec): ``session_hash`` still - covers the full, order-sensitive message array and the full, - unstripped event payload/timestamp, so a real reorder or a real duration - change still triggers a re-write. Only ``message_contents`` / - ``event_identity_hashes`` -- the *revision comparison* axes -- are - tolerant of the volatility each bug describes. + The same holds for message order (polylogue-c429), attachment identity + presence (polylogue-d8al), and provider-reported generation-duration + measurement (polylogue-nuec): ``session_hash`` still covers the full, + order-sensitive message array, the full attachment payload including + whatever id the provider did or didn't emit, and the full, unstripped + event payload/timestamp, so a real reorder, a real id change, or a real + duration change still triggers a re-write. Only ``message_contents`` / + ``attachment_identities`` / ``attachment_contents`` / ``event_contents`` + -- the *revision comparison* axes -- are content-only (polylogue-aggz). """ messages_payload, attachments_payload, session_events_payload = _session_hash_components(convo) session_hash_hex = _session_tree_hash( @@ -485,40 +497,45 @@ def session_revision_projection(convo: ParsedSession) -> SessionRevisionProjecti content = bytes.fromhex(hash_payload(payload)) message_contents.add((identity, content)) message_hashes.append(content) - attachment_records: list[AttachmentRecord] = [] attachment_identities: set[bytes] = set() attachment_contents: set[tuple[bytes, bytes]] = set() for payload in attachments_payload: identity = bytes.fromhex(hash_payload(_attachment_identity_payload(payload))) - loose_identity = bytes.fromhex(hash_payload(_attachment_loose_identity_payload(payload))) inline_content_hash = payload.get("inline_content_hash") - attachment_content = bytes.fromhex(inline_content_hash) if isinstance(inline_content_hash, str) else None - attachment_records.append((identity, loose_identity, attachment_content)) attachment_identities.add(identity) - if attachment_content is not None: - attachment_contents.add((identity, attachment_content)) + if isinstance(inline_content_hash, str): + attachment_contents.add((identity, bytes.fromhex(inline_content_hash))) event_hashes: list[bytes] = [] - event_identity_hashes: list[bytes] = [] - for event_index, (payload, event) in enumerate(zip(session_events_payload, convo.session_events, strict=True)): + event_base_identities: list[bytes] = [] + event_content_hashes: list[bytes] = [] + for payload, event in zip(session_events_payload, convo.session_events, strict=True): event_hashes.append(bytes.fromhex(hash_payload(payload))) - event_identity_hashes.append(bytes.fromhex(hash_payload(_event_identity_hash_payload(event, event_index)))) - metadata_hash = bytes.fromhex( - hash_payload( - { - "title": _normalize_for_hash(convo.title), - "created_at": _normalize_for_hash(convo.created_at), - "updated_at": _normalize_for_hash(convo.updated_at), - } + content_payload = _event_content_payload(event) + event_base_identities.append(bytes.fromhex(hash_payload(_event_base_identity_payload(content_payload)))) + event_content_hashes.append(bytes.fromhex(hash_payload(content_payload))) + base_identity_counts = Counter(event_base_identities) + event_contents: set[tuple[bytes, bytes]] = set() + for base_identity, content_hash in zip(event_base_identities, event_content_hashes, strict=True): + # A base identity (event type + anchoring message) ambiguous within + # this revision -- more than one event shares it, e.g. one + # chatgpt_block_metadata event per block on a message -- folds the + # event's own content into identity to disambiguate. Still + # content-derived, never the array index: distinct blocks already + # differ in content (e.g. a content-intrinsic block_index), and + # events that are genuine duplicates (same base identity, same + # content) correctly collapse to one set entry either way. + canonical_identity = ( + base_identity + if base_identity_counts[base_identity] == 1 + else bytes.fromhex(hash_payload({"base_identity": base_identity.hex(), "content": content_hash.hex()})) ) - ) + event_contents.add((canonical_identity, content_hash)) return SessionRevisionProjection( session_hash=bytes.fromhex(session_hash_hex), message_hashes=tuple(message_hashes), message_contents=frozenset(message_contents), attachment_identities=frozenset(attachment_identities), attachment_contents=frozenset(attachment_contents), - attachment_records=tuple(attachment_records), event_hashes=tuple(event_hashes), - event_identity_hashes=tuple(event_identity_hashes), - metadata_hash=metadata_hash, + event_contents=frozenset(event_contents), ) diff --git a/tests/unit/archive/test_session_revision_membership.py b/tests/unit/archive/test_session_revision_membership.py index 63e64e3b70..4cd021c009 100644 --- a/tests/unit/archive/test_session_revision_membership.py +++ b/tests/unit/archive/test_session_revision_membership.py @@ -1,9 +1,12 @@ from __future__ import annotations +from itertools import permutations + from polylogue.archive.message.roles import Role from polylogue.archive.session_revision_membership import ( MembershipRevision, - _strictly_dominates, + _maximal_evidence_fallback, + _relation, classify_membership_revisions, ) from polylogue.core.enums import Provider @@ -20,6 +23,24 @@ def _revision(raw_id: str, *texts: str) -> MembershipRevision: return MembershipRevision(raw_id, session_revision_projection(session)) +def _ordered_message_revision(raw_id: str, *id_text_pairs: tuple[str, str]) -> MembershipRevision: + """A session whose messages carry explicit provider ids in a given array order.""" + session = ParsedSession( + source_name=Provider.CLAUDE_AI, + provider_session_id="session", + messages=[ + ParsedMessage(provider_message_id=provider_id, role=Role.USER, text=text) + for provider_id, text in id_text_pairs + ], + ) + return MembershipRevision(raw_id, session_revision_projection(session)) + + +# --------------------------------------------------------------------------- +# Core equal/contains/conflict semantics (polylogue-aggz) +# --------------------------------------------------------------------------- + + def test_classifies_strict_growth_and_semantic_equivalence() -> None: result = classify_membership_revisions( [_revision("raw-b", "one", "two"), _revision("raw-z", "one"), _revision("raw-a", "one")] @@ -30,6 +51,15 @@ def test_classifies_strict_growth_and_semantic_equivalence() -> None: def test_refuses_divergent_maxima() -> None: + """Genuine divergence stays quarantined ambiguous. + + raw-b and raw-c both grew from raw-a in incompatible directions (each + holds a message the other lacks) -- a genuine fork, "conflict" under + ``_relation``. See ``_maximal_evidence_fallback`` for the designed, + unit-tested presence-guarantee alternative to this quarantine, and its + docstring for why it is not wired into ``classify_membership_revisions`` + yet. + """ result = classify_membership_revisions( [_revision("raw-a", "one"), _revision("raw-b", "one", "left"), _revision("raw-c", "one", "right")] ) @@ -37,6 +67,42 @@ def test_refuses_divergent_maxima() -> None: assert result.ambiguous_raw_ids == ("raw-a", "raw-b", "raw-c") +def test_maximal_evidence_fallback_picks_deterministically() -> None: + """The presence-guarantee fallback (not yet wired live) resolves a fork deterministically. + + Frontier tie among raw-b/raw-c falls through to the raw_id tiebreak + ("raw-c" > "raw-b"), same as the frontier-sort used elsewhere in this + module. + """ + representatives = [_revision("raw-a", "one"), _revision("raw-b", "one", "left"), _revision("raw-c", "one", "right")] + assert _maximal_evidence_fallback(representatives).raw_id == "raw-c" + + +def test_maximal_evidence_fallback_is_order_independent() -> None: + """The same genuinely-ambiguous cohort always resolves to the same head. + + A rebuild that processes raws in a different order must not produce a + different archive: every permutation of the same three divergent + revisions must pick the identical fallback head. ``_frontier`` plus the + stable raw_id tiebreak guarantees this -- it depends only on each + revision's own projected content, never on input order. + """ + revisions = [_revision("raw-a", "one"), _revision("raw-b", "one", "left"), _revision("raw-c", "one", "right")] + results = {_maximal_evidence_fallback(list(ordering)).raw_id for ordering in permutations(revisions)} + assert results == {"raw-c"} + + +def test_containment_chain_resolves_without_needing_the_fallback() -> None: + """A clean append-only growth chain resolves via set containment alone -- + accepting the WHOLE chain (both raws), not a single frontier-max pick, + which would silently discard the older revision's own head-of-chain + role. + """ + result = classify_membership_revisions([_revision("raw-old", "one"), _revision("raw-new", "one", "two")]) + assert result.accepted_raw_ids == ("raw-old", "raw-new") + assert result.ambiguous_raw_ids == () + + def test_metadata_only_revision_uses_latest_provider_timestamp() -> None: def revision(raw_id: str, updated_at: str | None, *, title: str = "title") -> MembershipRevision: session = ParsedSession( @@ -59,7 +125,16 @@ def revision(raw_id: str, updated_at: str | None, *, title: str = "title") -> Me assert result.ambiguous_raw_ids == () -def test_metadata_revision_without_complete_provider_time_is_ambiguous() -> None: +def test_metadata_revision_without_complete_provider_time_still_resolves_by_content() -> None: + """A title difference with an unresolvable timestamp no longer blocks presence. + + Both revisions carry the identical single message -- content equality + already proves these are the same revision (title is not part of + ``_relation`` at all); a missing timestamp is no longer a reason to + withhold a revision this comparison has already proven is the same + conversation. Deterministic tiebreak (min raw_id) picks a representative + when no timestamp can settle it. + """ with_timestamp = ParsedSession( source_name=Provider.CHATGPT, provider_session_id="session", @@ -80,11 +155,13 @@ def test_metadata_revision_without_complete_provider_time_is_ambiguous() -> None ] ) - assert result.accepted_raw_ids == () - assert result.ambiguous_raw_ids == ("raw-new", "raw-old") + assert result.accepted_raw_ids == ("raw-new",) + assert result.equivalent_raw_ids == ("raw-old",) + assert result.ambiguous_raw_ids == () -def test_metadata_revisions_with_equal_provider_time_are_ambiguous() -> None: +def test_metadata_revisions_with_equal_provider_time_still_resolve_by_content() -> None: + """A title difference under a TIED timestamp also resolves by content.""" timestamp = "2026-01-02T00:00:00Z" first = ParsedSession( source_name=Provider.CHATGPT, @@ -102,168 +179,94 @@ def test_metadata_revisions_with_equal_provider_time_are_ambiguous() -> None: ] ) - assert result.accepted_raw_ids == () - assert result.ambiguous_raw_ids == ("raw-a", "raw-b") + assert result.accepted_raw_ids == ("raw-a",) + assert result.equivalent_raw_ids == ("raw-b",) + assert result.ambiguous_raw_ids == () -def test_browser_native_snapshot_accepts_later_provider_revision_when_messages_reorder() -> None: - older = _revision("raw-old", "prompt", "attachment context") - newer = _revision("raw-new", "prompt", "tool result", "attachment context") - older = MembershipRevision( - older.raw_id, - older.projection, - "2026-01-01T00:00:00Z", - observed_at_ms=1, - browser_snapshot_fidelity="native", - provider_message_ids=frozenset({"prompt", "attachment"}), - ) - newer = MembershipRevision( - newer.raw_id, - newer.projection, - "2026-01-01T00:01:00Z", - observed_at_ms=2, - browser_snapshot_fidelity="native", - provider_message_ids=frozenset({"prompt", "tool", "attachment"}), - ) +# --------------------------------------------------------------------------- +# Messages: order-insensitive set containment (polylogue-c429) +# --------------------------------------------------------------------------- - result = classify_membership_revisions([newer, older]) +def test_accepts_reordered_message_array_with_identical_content_as_equivalent() -> None: + """Same message ids, byte-identical content per id, different array order. + + Reproduces the exact shape measured on the live archive: Claude.ai's own + export ordering for a conversation is not guaranteed stable across + separate export requests -- 34 of 35 sampled claude-ai-export ambiguous + cohorts had identical message-id sets with zero per-id content + differences (polylogue-c429). A set has no order to violate; this must + resolve as equivalent content. + """ + chronological = _ordered_message_revision("raw-chrono", ("a", "one"), ("b", "two"), ("c", "three")) + resequenced = _ordered_message_revision("raw-resequenced", ("b", "two"), ("a", "one"), ("c", "three")) + + assert chronological.projection.message_hashes != resequenced.projection.message_hashes + assert chronological.projection.message_contents == resequenced.projection.message_contents + assert _relation(chronological.projection, resequenced.projection) == "equal" + + result = classify_membership_revisions([chronological, resequenced]) + + assert result.accepted_raw_ids == ("raw-chrono",) + assert result.equivalent_raw_ids == ("raw-resequenced",) + assert result.ambiguous_raw_ids == () + + +def test_accepts_message_growth_across_a_reordered_prefix() -> None: + """A genuinely appended message is still growth even when the shared ids reorder.""" + older = _ordered_message_revision("raw-old", ("b", "two"), ("a", "one")) + newer = _ordered_message_revision("raw-new", ("a", "one"), ("c", "three"), ("b", "two")) + + assert _relation(older.projection, newer.projection) == "b_contains_a" + + result = classify_membership_revisions([older, newer]) assert result.accepted_raw_ids == ("raw-old", "raw-new") assert result.ambiguous_raw_ids == () -def test_browser_native_snapshot_refuses_later_revision_that_loses_message_identity() -> None: - older = _revision("raw-old", "prompt", "left") - newer = _revision("raw-new", "prompt", "right") - revisions = [ - MembershipRevision( - older.raw_id, - older.projection, - "2026-01-01T00:00:00Z", - observed_at_ms=1, - browser_snapshot_fidelity="native", - provider_message_ids=frozenset({"prompt", "left"}), - ), - MembershipRevision( - newer.raw_id, - newer.projection, - "2026-01-01T00:01:00Z", - observed_at_ms=2, - browser_snapshot_fidelity="native", - provider_message_ids=frozenset({"prompt", "right"}), - ), - ] +def test_refuses_message_content_change_under_a_shared_id_despite_reorder() -> None: + """A real edit under a shared id is a conflict, not laundered by the set model.""" + older = _ordered_message_revision("raw-old", ("a", "one"), ("b", "two")) + newer = _ordered_message_revision("raw-new", ("b", "two"), ("a", "EDITED")) - result = classify_membership_revisions(revisions) + assert _relation(older.projection, newer.projection) == "conflict" + result = classify_membership_revisions([older, newer]) assert result.accepted_raw_ids == () assert result.ambiguous_raw_ids == ("raw-new", "raw-old") -def test_browser_native_upgrade_refuses_any_shrinking_frontier_dimension() -> None: - older = _revision("raw-old", "prompt") - older_projection = older.projection.__class__( - session_hash=b"o" * 32, - message_hashes=older.projection.message_hashes, - message_contents=older.projection.message_contents, - event_hashes=older.projection.event_hashes, - event_identity_hashes=older.projection.event_identity_hashes, - attachment_identities=frozenset({b"attachment"}), - attachment_contents=frozenset(), - attachment_records=((b"attachment", b"attachment-loose", None),), - metadata_hash=older.projection.metadata_hash, - ) - newer = _revision("raw-new", "prompt", "answer") - revisions = [ - MembershipRevision( - older.raw_id, - older_projection, - "2026-01-01T00:00:00Z", - observed_at_ms=1, - browser_snapshot_fidelity="dom", - ), - MembershipRevision( - newer.raw_id, - newer.projection, - "2026-01-01T00:01:00Z", - observed_at_ms=2, - browser_snapshot_fidelity="native", - ), - ] +def test_refuses_message_id_disappearing_despite_reorder() -> None: + """A message id present in older but absent from newer is a real loss (a fork).""" + older = _ordered_message_revision("raw-old", ("a", "one"), ("b", "two")) + newer = _ordered_message_revision("raw-new", ("b", "two"), ("c", "three")) - result = classify_membership_revisions(revisions) + assert _relation(older.projection, newer.projection) == "conflict" + result = classify_membership_revisions([older, newer]) assert result.accepted_raw_ids == () assert result.ambiguous_raw_ids == ("raw-new", "raw-old") -def test_direct_export_outranks_browser_capture_siblings_regardless_of_growth() -> None: - """A genuine non-browser-capture revision always wins over dom/native - browser-capture siblings, even though its content is neither a byte/hash - -growth superset of them nor resolvable by provider-timestamp comparison. - Browser capture exists to backfill a session before its paired direct/ - native provider export shows up, never to compete with or shadow that - export once it arrives (polylogue-z1c6).""" - direct = _revision("raw-direct", "real one", "real two", "real three") - dom = MembershipRevision( - "raw-dom", - _revision("raw-dom", "dom-only-turn").projection, - "2026-07-04T09:55:00Z", - browser_snapshot_fidelity="dom", - ) - native = MembershipRevision( - "raw-native", - _revision("raw-native", "native-turn-a", "native-turn-b").projection, - "2026-07-04T09:54:00Z", - browser_snapshot_fidelity="native", - ) - - result = classify_membership_revisions([dom, native, direct]) - - assert result.accepted_raw_ids == ("raw-direct",) - assert result.equivalent_raw_ids == ("raw-dom", "raw-native") - assert result.ambiguous_raw_ids == () +def test_message_reorder_does_change_the_session_content_hash() -> None: + """Order tolerance must not leak into idempotency. + ``session_hash`` must stay order-sensitive: if two array orderings of the + same messages hashed identically, a real reorder written by the + archive's own writer path would be silently skipped as unchanged on + re-ingest. + """ + chronological = _ordered_message_revision("raw-chrono", ("a", "one"), ("b", "two")) + resequenced = _ordered_message_revision("raw-resequenced", ("b", "two"), ("a", "one")) -def test_browser_snapshot_accepts_later_attachment_enrichment_without_provider_update() -> None: - older = _revision("raw-old", "prompt", "answer") - newer = _revision("raw-new", "prompt", "answer") - newer_projection = newer.projection.__class__( - session_hash=b"n" * 32, - message_hashes=newer.projection.message_hashes, - message_contents=newer.projection.message_contents, - event_hashes=newer.projection.event_hashes, - event_identity_hashes=newer.projection.event_identity_hashes, - attachment_identities=frozenset({b"attachment-v2"}), - attachment_contents=frozenset(), - attachment_records=((b"attachment-v2", b"attachment-v2-loose", None),), - metadata_hash=newer.projection.metadata_hash, - ) - revisions = [ - MembershipRevision( - older.raw_id, - older.projection, - "2026-01-01T00:01:00Z", - observed_at_ms=1, - browser_snapshot_fidelity="native", - provider_message_ids=frozenset({"prompt", "answer"}), - provider_attachment_ids=frozenset({"asset"}), - ), - MembershipRevision( - newer.raw_id, - newer_projection, - "2026-01-01T00:01:00Z", - observed_at_ms=2, - browser_snapshot_fidelity="native", - provider_message_ids=frozenset({"prompt", "answer"}), - provider_attachment_ids=frozenset({"asset"}), - ), - ] + assert chronological.projection.session_hash != resequenced.projection.session_hash - result = classify_membership_revisions(revisions) - assert result.accepted_raw_ids == ("raw-old", "raw-new") - assert result.ambiguous_raw_ids == () +# --------------------------------------------------------------------------- +# Attachments: content-derived identity, acquisition growth (polylogue-bu1i, +# polylogue-d8al, polylogue-hith) +# --------------------------------------------------------------------------- def _attachment_revision( @@ -272,17 +275,15 @@ def _attachment_revision( acquired: bool, provider_attachment_id: str = "drive-file-1", inline: bytes = b"attachment bytes", + name: str = "screenshot.png", + mime_type: str = "image/png", ) -> MembershipRevision: - """A session whose single attachment is referenced, optionally with bytes read. - - Models the shape a lazily-fetched attachment actually produces: the provider - emits a bare reference (no size, no bytes), and a later acquisition pass - resolves the same reference into real bytes. The transcript is untouched - either way -- only the attachment's acquisition state differs. - """ + """A session whose single attachment is referenced, optionally with bytes read.""" attachment = ParsedAttachment( provider_attachment_id=provider_attachment_id, message_provider_id="0", + name=name, + mime_type=mime_type, size_bytes=len(inline) if acquired else None, inline_bytes=inline if acquired else None, ) @@ -298,68 +299,49 @@ def _attachment_revision( def test_accepts_attachment_byte_acquisition_as_growth_not_a_branch() -> None: """Resolving an already-referenced attachment's bytes is a fidelity upgrade. - A Drive/Gemini document acquired twice -- once before its attachment bytes - were fetched, once after -- has an identical transcript and an identical - attachment reference. Folding acquisition state into the attachment identity - hash made the two revisions look like equal-sized disjoint branches, so the - whole cohort was quarantined as ambiguous and neither revision was indexed - (polylogue-bu1i). Measured on the live archive: all 157 two-member - aistudio-drive cohorts had exactly this shape. + A Drive/Gemini document acquired twice -- once before its attachment + bytes were fetched, once after -- has an identical transcript and + attachment reference. All 157 two-member aistudio-drive ambiguous + cohorts in the live archive had exactly this shape (polylogue-bu1i). """ bare = _attachment_revision("raw-bbbb", acquired=False) enriched = _attachment_revision("raw-aaaa", acquired=True) - result = classify_membership_revisions([enriched, bare]) + assert _relation(bare.projection, enriched.projection) == "b_contains_a" - # Ordering matters as much as acceptance: the revision holding the bytes must - # be the head of the chain, or the archive indexes the emptier one. The raw - # ids are chosen so plain lexical ordering would put the enriched revision - # first, which is the direction the old frontier tie fell through to. + result = classify_membership_revisions([enriched, bare]) + # Ordering matters as much as acceptance: the revision holding the bytes + # must be the head of the chain, or the archive indexes the emptier one. assert result.accepted_raw_ids == ("raw-bbbb", "raw-aaaa") assert result.ambiguous_raw_ids == () def test_refuses_attachment_byte_loss_as_a_fidelity_downgrade() -> None: - """Dropping bytes already in hand is never an upgrade, in either direction. - - Without this, an acquisition regression would look like ordinary growth - running backwards and could be accepted, silently re-marking a fetched - attachment as unfetched -- the concrete harm observed on five live sessions. - """ + """Dropping bytes already in hand is never an upgrade, in either direction.""" enriched = _attachment_revision("raw-a", acquired=True) bare = _attachment_revision("raw-b", acquired=False) - result = classify_membership_revisions([enriched, bare]) + assert _relation(enriched.projection, bare.projection) == "a_contains_b" + assert _relation(bare.projection, enriched.projection) == "b_contains_a" + result = classify_membership_revisions([enriched, bare]) assert result.accepted_raw_ids == ("raw-b", "raw-a") - # And the downgrade direction on its own is refused outright. - assert not _strictly_dominates(enriched.projection, bare.projection) - def test_refuses_conflicting_bytes_under_one_attachment_identity() -> None: - """Two sources disagreeing about one attachment's content stays ambiguous. - - Splitting identity from acquisition must not turn a genuine conflict into an - upgrade: when both revisions have read bytes for the same attachment and the - bytes differ, no ordering rule can decide which is authoritative. - """ + """Two sources disagreeing about one attachment's content is a conflict.""" left = _attachment_revision("raw-a", acquired=True, inline=b"left bytes") right = _attachment_revision("raw-b", acquired=True, inline=b"right bytes") - result = classify_membership_revisions([left, right]) + assert _relation(left.projection, right.projection) == "conflict" + result = classify_membership_revisions([left, right]) assert result.accepted_raw_ids == () assert result.ambiguous_raw_ids == ("raw-a", "raw-b") def test_attachment_acquisition_does_change_the_session_content_hash() -> None: - """Acquisition is invisible to *revision comparison*, not to *idempotency*. - - The split must not leak into ``session_hash``: if acquiring bytes did not - change the session's content hash, a re-ingest carrying newly-fetched - attachments would be skipped as unchanged and the bytes would never land. - """ + """Acquisition is invisible to *revision comparison*, not to *idempotency*.""" bare = _attachment_revision("raw-a", acquired=False) enriched = _attachment_revision("raw-b", acquired=True) @@ -369,520 +351,50 @@ def test_attachment_acquisition_does_change_the_session_content_hash() -> None: assert len(enriched.projection.attachment_contents) == 1 -def test_refuses_contradicted_bytes_even_when_the_revision_otherwise_grew() -> None: - """Growth elsewhere must not launder a contradiction about bytes already read. +def test_accepts_real_and_synthetic_id_variants_of_the_same_attachment_as_equivalent() -> None: + """Same physical attachment, a real id on one export vintage, none on the other. - The equal-cardinality conflict case is already refused by the growth test - itself, which leaves the byte-agreement check unexercised and therefore - unproven. This is the shape that genuinely needs it: the newer revision adds - a second attachment -- real growth on the identity axis -- while changing the - bytes it reports for the attachment both revisions share. Accepting that - would overwrite content already in hand with content from a disagreeing - source, under cover of a legitimate-looking frontier advance. + Attachment identity is content-derived (anchoring message, name, media + type) and never the provider's own attachment id (polylogue-d8al, + polylogue-hith): Claude.ai does not consistently emit a real attachment + id across separate export requests of the same conversation. No + id-minting scheme can make a real id and a synthetic hash collide, so + the id was never part of identity to begin with. """ - shared_id = "drive-file-1" - older = _attachment_revision("raw-a", acquired=True, provider_attachment_id=shared_id, inline=b"original bytes") + real_id = _attachment_revision("raw-real", acquired=False, provider_attachment_id="e950263f-51d1-4b7a-9c2e-0") + synthetic_id = _attachment_revision("raw-synthetic", acquired=False, provider_attachment_id="att-ce21cd12d650") - contradicting = ParsedAttachment( - provider_attachment_id=shared_id, - message_provider_id="0", - size_bytes=len(b"rewritten bytes"), - inline_bytes=b"rewritten bytes", - ) - added = ParsedAttachment( - provider_attachment_id="drive-file-2", - message_provider_id="0", - size_bytes=4, - inline_bytes=b"more", - ) - session = ParsedSession( - source_name=Provider.GEMINI, - provider_session_id="session", - messages=[ParsedMessage(provider_message_id="0", role=Role.USER, text="one")], - attachments=[contradicting, added], - ) - newer = MembershipRevision("raw-b", session_revision_projection(session)) + assert real_id.projection.attachment_identities == synthetic_id.projection.attachment_identities + assert _relation(real_id.projection, synthetic_id.projection) == "equal" - # The identity axis really did grow, so the refusal cannot come from there. - assert older.projection.attachment_identities < newer.projection.attachment_identities - assert not _strictly_dominates(older.projection, newer.projection) + result = classify_membership_revisions([real_id, synthetic_id]) - result = classify_membership_revisions([older, newer]) - assert result.accepted_raw_ids == () - assert result.ambiguous_raw_ids == ("raw-a", "raw-b") + assert result.accepted_raw_ids == ("raw-real",) + assert result.equivalent_raw_ids == ("raw-synthetic",) + assert result.ambiguous_raw_ids == () -def _named_attachment_revision( - raw_id: str, - *, - provider_attachment_id: str, - name: str = "screenshot.png", - mime_type: str = "image/png", - message_provider_id: str = "0", - inline: bytes | None = None, -) -> MembershipRevision: - """A session whose single named attachment carries a specific provider id. +def test_refuses_to_guess_when_two_distinct_attachments_share_message_name_and_type() -> None: + """Two distinct attachments sharing (message, name, media type) are indistinguishable. - Unlike ``_attachment_revision`` (whose bare fixture leaves name/mime_type - ``None``, deliberately colliding with any other bare attachment's loose - key), this gives the attachment a real name/mime_type so d8al tests - exercise the id-presence correlation they're meant to, not the - locally-ambiguous-loose-key fallback. + This is the documented, accepted limit (polylogue-d8al, polylogue-aggz): + with no bytes on either side, there is no signal left to tell two + same-metadata attachments apart, so they collapse to ONE identity by + construction. Stated plainly rather than engineered around. """ - attachment = ParsedAttachment( - provider_attachment_id=provider_attachment_id, - message_provider_id=message_provider_id, - name=name, - mime_type=mime_type, - size_bytes=len(inline) if inline is not None else None, - inline_bytes=inline, - ) - session = ParsedSession( - source_name=Provider.CLAUDE_AI, - provider_session_id="session", - messages=[ParsedMessage(provider_message_id="0", role=Role.USER, text="one")], - attachments=[attachment], - ) - return MembershipRevision(raw_id, session_revision_projection(session)) - - -def test_accepts_real_and_synthetic_id_variants_of_the_same_attachment_as_equivalent() -> None: - """Same physical attachment, a real id on one export vintage, none on the other. - - Reproduces the shape measured on the live archive: 268 of 566 - claude-ai-export equal-message-count ambiguous cohorts had byte-identical - messages and events but attachment identity sets that never share a - provider id, because Claude.ai does not consistently emit a real - attachment id across separate export requests of the same conversation -- - one vintage carries a real UUID, the other has none and the parser - synthesizes one (polylogue-d8al). No id-minting scheme can make a real id - and a synthetic hash collide, so the id must be dropped from the - comparison in this shape -- correlating instead by (message, name, media - type), which is unambiguous here (exactly one attachment on this - message). - """ - real_id = _named_attachment_revision("raw-real", provider_attachment_id="e950263f-51d1-4b7a-9c2e-000000000000") - synthetic_id = _named_attachment_revision("raw-synthetic", provider_attachment_id="att-ce21cd12d650") - - assert real_id.projection.attachment_identities != synthetic_id.projection.attachment_identities - assert not _strictly_dominates(real_id.projection, synthetic_id.projection) - assert not _strictly_dominates(synthetic_id.projection, real_id.projection) - - # Both fixtures leave title/created_at/updated_at unset (matching), the - # common live shape for a re-export of an untouched conversation: the - # correlation-based merge gets them into the same content-key group, and - # metadata_hash proves the id-presence mismatch is the ONLY remaining - # difference -- no provider timestamp is needed or consulted (real - # exports commonly carry an IDENTICAL provider updated_at across two - # export requests of unchanged content, so requiring one to differ would - # leave this shape ambiguous forever). - result = classify_membership_revisions([real_id, synthetic_id]) - - assert result.accepted_raw_ids == ("raw-real",) - assert result.equivalent_raw_ids == ("raw-synthetic",) - assert result.ambiguous_raw_ids == () - - -def test_accepts_attachment_growth_despite_id_presence_mismatch_on_the_shared_attachment() -> None: - """Growth must still be recognized when the SHARED attachment's id presence differs. - - The newer revision adds a second attachment (real growth) while also - losing the real id on the FIRST attachment (id-presence mismatch, - polylogue-d8al) -- both must be handled together: the shared attachment - correlates via the loose key, and the new attachment is ordinary growth. - """ - older_attachment = ParsedAttachment( - provider_attachment_id="e950263f-51d1-4b7a-9c2e-000000000000", - message_provider_id="0", - name="screenshot.png", - mime_type="image/png", - ) - newer_shared = ParsedAttachment( - provider_attachment_id="att-ce21cd12d650", - message_provider_id="0", - name="screenshot.png", - mime_type="image/png", - ) - newer_added = ParsedAttachment( - provider_attachment_id="att-fedcba987654", - message_provider_id="0", - name="notes.txt", - mime_type="text/plain", - ) - older_session = ParsedSession( - source_name=Provider.CLAUDE_AI, - provider_session_id="session", - messages=[ParsedMessage(provider_message_id="0", role=Role.USER, text="one")], - attachments=[older_attachment], - ) - newer_session = older_session.model_copy(update={"attachments": [newer_shared, newer_added]}) - - older = MembershipRevision("raw-old", session_revision_projection(older_session)) - newer = MembershipRevision("raw-new", session_revision_projection(newer_session)) - - assert _strictly_dominates(older.projection, newer.projection) - - result = classify_membership_revisions([older, newer]) - assert result.accepted_raw_ids == ("raw-old", "raw-new") - assert result.ambiguous_raw_ids == () - - -def test_attachment_growth_with_id_presence_mismatch_is_a_chain_not_an_equivalence_pick() -> None: - """The equivalence-merge step must not swallow genuine growth as a timestamp pick. - - The test above proves growth is recognized by ``_strictly_dominates`` when - the two revisions are never pre-grouped into one equivalence bucket, but - it does not prove the merge step's own full-correlation requirement - (``unmatched_a or unmatched_b`` in ``_attachments_equivalent``) does - anything -- session_hash always differs when attachment counts differ, so - an over-eager merge just gets re-split by the by_session_hash pass with - the SAME final result. This is the shape that forces it: distinct, - resolvable provider timestamps. If the merge step ignored the extra, - uncorrelated attachment in ``newer`` and merged the groups anyway, the - pre-existing metadata_variants timestamp mechanism would pick ONE - representative and discard the other as merely-equivalent metadata, - instead of recognizing both as an accepted growth chain. - """ - older_attachment = ParsedAttachment( - provider_attachment_id="e950263f-51d1-4b7a-9c2e-000000000000", - message_provider_id="0", - name="screenshot.png", - mime_type="image/png", - ) - newer_shared = ParsedAttachment( - provider_attachment_id="att-ce21cd12d650", - message_provider_id="0", - name="screenshot.png", - mime_type="image/png", - ) - newer_added = ParsedAttachment( - provider_attachment_id="att-fedcba987654", - message_provider_id="0", - name="notes.txt", - mime_type="text/plain", - ) - older_session = ParsedSession( - source_name=Provider.CLAUDE_AI, - provider_session_id="session", - messages=[ParsedMessage(provider_message_id="0", role=Role.USER, text="one")], - attachments=[older_attachment], - ) - newer_session = older_session.model_copy(update={"attachments": [newer_shared, newer_added]}) - - older = MembershipRevision("raw-old", session_revision_projection(older_session), "2026-01-01T00:00:00Z") - newer = MembershipRevision("raw-new", session_revision_projection(newer_session), "2026-01-02T00:00:00Z") - - result = classify_membership_revisions([older, newer]) - assert result.accepted_raw_ids == ("raw-old", "raw-new") - assert result.equivalent_raw_ids == () - assert result.ambiguous_raw_ids == () - - -def test_refuses_contradicted_bytes_despite_id_presence_mismatch() -> None: - """An id-presence mismatch must not launder a genuine byte contradiction. - - Both revisions have acquired bytes for what correlates as the same - attachment (real id vs none, matched by the loose key) -- if the bytes - disagree, that is a real conflict no ordering rule can resolve, exactly - like bu1i's plain-matching-id contradiction case. - """ - older_attachment = ParsedAttachment( - provider_attachment_id="e950263f-51d1-4b7a-9c2e-000000000000", - message_provider_id="0", - name="screenshot.png", - mime_type="image/png", - size_bytes=len(b"original bytes"), - inline_bytes=b"original bytes", - ) - newer_attachment = ParsedAttachment( - provider_attachment_id="att-ce21cd12d650", - message_provider_id="0", - name="screenshot.png", - mime_type="image/png", - size_bytes=len(b"different bytes"), - inline_bytes=b"different bytes", - ) - older_session = ParsedSession( - source_name=Provider.CLAUDE_AI, - provider_session_id="session", - messages=[ParsedMessage(provider_message_id="0", role=Role.USER, text="one")], - attachments=[older_attachment], - ) - newer_session = older_session.model_copy(update={"attachments": [newer_attachment]}) - - older = MembershipRevision("raw-old", session_revision_projection(older_session)) - newer = MembershipRevision("raw-new", session_revision_projection(newer_session)) - - assert not _strictly_dominates(older.projection, newer.projection) - assert not _strictly_dominates(newer.projection, older.projection) - - result = classify_membership_revisions([older, newer]) - assert result.accepted_raw_ids == () - assert result.ambiguous_raw_ids == ("raw-new", "raw-old") - - -def test_refuses_contradicted_bytes_despite_id_presence_mismatch_even_with_resolvable_timestamps() -> None: - """The equivalence-merge content check must not be provably unreachable. - - The test above stays ambiguous even without this specific check, because - dominance's own contradiction guard (``_attachment_evidence_preserved``) - independently refuses the same pair once they fail to merge into one - equivalence group -- so it cannot prove the merge step's OWN content - check (``_attachments_equivalent``) does anything. This is the shape that - forces it: distinct, resolvable provider timestamps on both revisions. If - the merge step ignored the byte contradiction and merged them anyway, the - pre-existing metadata_variants timestamp mechanism would silently pick - the newer-timestamped revision as an "equivalent" upgrade and discard the - other -- overwriting a genuine, unresolvable conflict about the same - attachment's bytes instead of leaving it ambiguous. - """ - older_attachment = ParsedAttachment( - provider_attachment_id="e950263f-51d1-4b7a-9c2e-000000000000", - message_provider_id="0", - name="screenshot.png", - mime_type="image/png", - size_bytes=len(b"original bytes"), - inline_bytes=b"original bytes", - ) - newer_attachment = ParsedAttachment( - provider_attachment_id="att-ce21cd12d650", - message_provider_id="0", - name="screenshot.png", - mime_type="image/png", - size_bytes=len(b"different bytes"), - inline_bytes=b"different bytes", - ) - older_session = ParsedSession( - source_name=Provider.CLAUDE_AI, - provider_session_id="session", - messages=[ParsedMessage(provider_message_id="0", role=Role.USER, text="one")], - attachments=[older_attachment], - ) - newer_session = older_session.model_copy(update={"attachments": [newer_attachment]}) - - older = MembershipRevision("raw-old", session_revision_projection(older_session), "2026-01-01T00:00:00Z") - newer = MembershipRevision("raw-new", session_revision_projection(newer_session), "2026-01-02T00:00:00Z") - - result = classify_membership_revisions([older, newer]) - assert result.accepted_raw_ids == () - assert result.ambiguous_raw_ids == ("raw-new", "raw-old") - - -def test_refuses_to_guess_correlation_when_the_loose_key_is_locally_ambiguous() -> None: - """Two distinct attachments sharing (message, name, media type) on one side - must not be guessed apart by the loose key. - - This is the documented, accepted limit (polylogue-d8al): with no id - agreement and no bytes on either side, there is no signal left to - correlate the attachments, so the strict identity is kept and the - id-presence mismatch stays unresolved (ambiguous) rather than silently - picking a pairing by array position -- the exact mistake this feature - replaces. - """ - ambiguous_older = [ - ParsedAttachment( - provider_attachment_id="uuid-1", message_provider_id="0", name="image.png", mime_type="image/png" - ), - ParsedAttachment( - provider_attachment_id="uuid-2", message_provider_id="0", name="image.png", mime_type="image/png" - ), - ] - single_newer = [ - ParsedAttachment( - provider_attachment_id="att-hash-1", message_provider_id="0", name="image.png", mime_type="image/png" - ), - ] - older_session = ParsedSession( - source_name=Provider.CLAUDE_AI, - provider_session_id="session", - messages=[ParsedMessage(provider_message_id="0", role=Role.USER, text="one")], - attachments=ambiguous_older, - ) - newer_session = older_session.model_copy(update={"attachments": single_newer}) - - older = MembershipRevision("raw-old", session_revision_projection(older_session)) - newer = MembershipRevision("raw-new", session_revision_projection(newer_session)) - - assert not _strictly_dominates(older.projection, newer.projection) - assert not _strictly_dominates(newer.projection, older.projection) - - result = classify_membership_revisions([older, newer]) - assert result.accepted_raw_ids == () - assert result.ambiguous_raw_ids == ("raw-new", "raw-old") - - -def _ordered_message_revision(raw_id: str, *id_text_pairs: tuple[str, str]) -> MembershipRevision: - """A session whose messages carry explicit provider ids in a given array order. - - Unlike ``_revision`` (which derives sequential ids from position), this lets - a test hold the id-to-content mapping fixed while varying only array order. - """ - session = ParsedSession( - source_name=Provider.CLAUDE_AI, - provider_session_id="session", - messages=[ - ParsedMessage(provider_message_id=provider_id, role=Role.USER, text=text) - for provider_id, text in id_text_pairs - ], - ) - return MembershipRevision(raw_id, session_revision_projection(session)) - - -def test_accepts_reordered_message_array_with_identical_content_as_equivalent() -> None: - """Same message ids, byte-identical content per id, different array order. - - Reproduces the exact shape measured on the live archive: Claude.ai's own - export ordering for a conversation is not guaranteed stable across separate - export requests -- 21 of 40 sampled claude-ai-export ambiguous cohorts - (52.5%) had the same 36 message ids with identical {role,text,timestamp} - per id, just resequenced (polylogue-c429). A strict positional-prefix - dominance test refuses both directions and quarantines the whole cohort; - this must instead resolve as equivalent content. - """ - chronological = _ordered_message_revision("raw-chrono", ("a", "one"), ("b", "two"), ("c", "three")) - resequenced = _ordered_message_revision("raw-resequenced", ("b", "two"), ("a", "one"), ("c", "three")) - - assert chronological.projection.message_hashes != resequenced.projection.message_hashes - assert chronological.projection.message_contents == resequenced.projection.message_contents - - # Both fixtures leave title/created_at/updated_at unset (matching), the - # common live shape for a re-export of an untouched conversation: the - # permutation-tolerant content key gets them into the same group, and - # metadata_hash proves the reorder is the ONLY remaining difference -- no - # provider timestamp is needed (Claude.ai's own updated_at commonly - # carries the SAME value across two export requests of unchanged content, - # since re-sequencing an array is not a provider-visible edit). - result = classify_membership_revisions([chronological, resequenced]) - - assert result.accepted_raw_ids == ("raw-chrono",) - assert result.equivalent_raw_ids == ("raw-resequenced",) - assert result.ambiguous_raw_ids == () - - -def test_accepts_message_growth_across_a_reordered_prefix() -> None: - """A genuinely appended message is still growth even when the shared ids reorder. - - The permutation tolerance must not swallow real append-only growth: the - newer revision keeps every id-to-content pair from the older revision (just - resequenced) and adds one new id. - """ - older = _ordered_message_revision("raw-old", ("b", "two"), ("a", "one")) - newer = _ordered_message_revision("raw-new", ("a", "one"), ("c", "three"), ("b", "two")) - - assert _strictly_dominates(older.projection, newer.projection) - - result = classify_membership_revisions([older, newer]) - assert result.accepted_raw_ids == ("raw-old", "raw-new") - assert result.ambiguous_raw_ids == () - - -def test_refuses_message_content_change_under_a_shared_id_despite_reorder() -> None: - """Permutation tolerance must not launder a real content change. - - Same id set, resequenced, but one shared id's text actually differs -- that - is genuine divergence (an edit, not export nondeterminism) and must stay - ambiguous, not be waved through by the order-insensitive comparison. - """ - older = _ordered_message_revision("raw-old", ("a", "one"), ("b", "two")) - newer = _ordered_message_revision("raw-new", ("b", "two"), ("a", "EDITED")) - - assert not _strictly_dominates(older.projection, newer.projection) - assert not _strictly_dominates(newer.projection, older.projection) - - result = classify_membership_revisions([older, newer]) - assert result.accepted_raw_ids == () - assert result.ambiguous_raw_ids == ("raw-new", "raw-old") - - -def test_refuses_message_content_change_even_when_the_revision_otherwise_grew() -> None: - """Growth elsewhere must not launder a contradicted message. - - The equal-count case above is already refused by the growth test itself - (``content_grew`` is ``False`` on both sides, since a same-id edit doesn't - change the message count), which leaves the content-agreement clause in - ``_message_evidence_preserved`` unexercised and therefore unproven -- the - same trap the disappearing-id growth test above closes for the identity - axis. This is the shape that genuinely needs it: the newer revision has - one more message than the older one (real growth) while also rewriting - the text under a shared id. - """ - older = _ordered_message_revision("raw-old", ("a", "one"), ("b", "two")) - newer = _ordered_message_revision("raw-new", ("a", "EDITED"), ("b", "two"), ("c", "three")) - - # The count really did grow, so the refusal cannot come from there. - assert len(newer.projection.message_contents) > len(older.projection.message_contents) - assert not _strictly_dominates(older.projection, newer.projection) - - result = classify_membership_revisions([older, newer]) - assert result.accepted_raw_ids == () - assert result.ambiguous_raw_ids == ("raw-new", "raw-old") - - -def test_refuses_message_id_disappearing_despite_reorder() -> None: - """A message id present in the older revision but absent from the newer one - is a real loss, not a reorder -- must stay ambiguous even though the - remaining ids' content and count could otherwise look like a permutation. - """ - older = _ordered_message_revision("raw-old", ("a", "one"), ("b", "two")) - newer = _ordered_message_revision("raw-new", ("b", "two"), ("c", "three")) - - assert not _strictly_dominates(older.projection, newer.projection) - assert not _strictly_dominates(newer.projection, older.projection) - - result = classify_membership_revisions([older, newer]) - assert result.accepted_raw_ids == () - assert result.ambiguous_raw_ids == ("raw-new", "raw-old") - - -def test_refuses_message_id_disappearing_even_when_the_revision_otherwise_grew() -> None: - """Growth elsewhere must not launder a lost message id. - - The equal-count case above is already refused by the growth test itself - (``content_grew`` is ``False`` on both sides), which leaves the identity- - subset clause in ``_message_evidence_preserved`` unexercised and therefore - unproven -- the same trap ``test_refuses_contradicted_bytes_even_when_the_ - revision_otherwise_grew`` closes for attachments. This is the shape that - genuinely needs it: the newer revision has one more message than the older - one (real growth on the count axis) while dropping an id the older - revision had. Accepting that would silently discard a real message under - cover of a legitimate-looking frontier advance. - """ - older = _ordered_message_revision("raw-old", ("a", "one"), ("b", "two")) - newer = _ordered_message_revision("raw-new", ("a", "one"), ("c", "three"), ("d", "four")) - - # The count really did grow, so the refusal cannot come from there. - assert len(newer.projection.message_contents) > len(older.projection.message_contents) - assert not _strictly_dominates(older.projection, newer.projection) - - result = classify_membership_revisions([older, newer]) - assert result.accepted_raw_ids == () - assert result.ambiguous_raw_ids == ("raw-new", "raw-old") + a = _attachment_revision("raw-a", acquired=False, provider_attachment_id="uuid-1") + b = _attachment_revision("raw-b", acquired=False, provider_attachment_id="uuid-2") + assert a.projection.attachment_identities == b.projection.attachment_identities -def test_message_reorder_does_change_the_session_content_hash() -> None: - """Order tolerance must not leak into idempotency. - - ``session_hash`` must stay order-sensitive: if two array orderings of the - same messages hashed identically, a real reorder written by the archive's - own writer path (a legitimate content change worth re-indexing) would be - silently skipped as unchanged on re-ingest. - """ - chronological = _ordered_message_revision("raw-chrono", ("a", "one"), ("b", "two")) - resequenced = _ordered_message_revision("raw-resequenced", ("b", "two"), ("a", "one")) - - assert chronological.projection.session_hash != resequenced.projection.session_hash - assert {identity for identity, _content in chronological.projection.message_contents} == { - identity for identity, _content in resequenced.projection.message_contents - } +# --------------------------------------------------------------------------- +# Events: order-insensitive, measurement-excluded set containment +# (polylogue-nuec) +# --------------------------------------------------------------------------- def _generation_lifecycle_revision(raw_id: str, elapsed_duration_ms: int) -> MembershipRevision: - """A ChatGPT-shaped session with one `generation_lifecycle` event. - - Mirrors the exact shape ``polylogue/sources/parsers/chatgpt.py`` emits: - ``duration_semantics: "provider_reported_elapsed"`` marking - ``elapsed_duration_ms`` as provider-remeasured evidence, not identity. - """ + """A ChatGPT-shaped session with one `generation_lifecycle` event.""" session = ParsedSession( source_name=Provider.CHATGPT, provider_session_id="session", @@ -909,27 +421,21 @@ def test_accepts_generation_lifecycle_duration_change_as_equivalent() -> None: """Byte-identical transcript, only a provider-remeasured duration differs. Reproduces the shape measured on the live archive: 33 of 35 sampled - chatgpt-export ambiguous cohorts (94%) had identical messages and - attachments but a `generation_lifecycle` event whose `elapsed_duration_ms` - varied non-monotonically between export requests for the SAME generation - (polylogue-nuec). This must resolve as equivalent content, not stay - quarantined as a branch. + chatgpt-export ambiguous cohorts had identical messages and attachments + but a `generation_lifecycle` event whose `elapsed_duration_ms` varied + non-monotonically between export requests for the SAME generation + (polylogue-nuec). Excluded from ``event_contents`` by an explicit + per-event-type ALLOWLIST of content-bearing fields, not a denylist of + fields discovered volatile after the fact. """ first_export = _generation_lifecycle_revision("raw-first", 13000) second_export = _generation_lifecycle_revision("raw-second", 21000) assert first_export.projection.event_hashes != second_export.projection.event_hashes - assert first_export.projection.event_identity_hashes == second_export.projection.event_identity_hashes + assert first_export.projection.event_contents == second_export.projection.event_contents assert first_export.projection.session_hash != second_export.projection.session_hash + assert _relation(first_export.projection, second_export.projection) == "equal" - # Both fixtures leave title/created_at/updated_at unset (matching), the - # common live shape for a re-export of an untouched conversation: the - # measurement-tolerant content key gets them into the same group, and - # metadata_hash proves the duration change is the ONLY remaining - # difference -- no provider timestamp is needed (a remeasured duration is - # not a provider-visible edit to the conversation itself, so ChatGPT's - # own updated_at commonly carries the SAME value across two export - # requests of the same generation). result = classify_membership_revisions([first_export, second_export]) assert result.accepted_raw_ids == ("raw-first",) @@ -937,12 +443,19 @@ def test_accepts_generation_lifecycle_duration_change_as_equivalent() -> None: assert result.ambiguous_raw_ids == () +def test_generation_lifecycle_duration_change_does_change_the_session_content_hash() -> None: + """Measurement tolerance must not leak into idempotency.""" + first_export = _generation_lifecycle_revision("raw-first", 13000) + second_export = _generation_lifecycle_revision("raw-second", 21000) + assert first_export.projection.session_hash != second_export.projection.session_hash + + def test_refuses_generation_lifecycle_state_change_despite_duration_tolerance() -> None: """Duration tolerance must not launder every field of the event as noise. - Only the designated measurement keys (and the timestamp they derive) are - excluded from identity -- a genuinely different ``state`` on the same event - is real divergence and must stay ambiguous. + Only the allowlisted fields (state, evidence_source, fidelity) are + content -- a genuinely different ``state`` on the same event is real + divergence. """ session = ParsedSession( source_name=Provider.CHATGPT, @@ -984,33 +497,18 @@ def test_refuses_generation_lifecycle_state_change_despite_duration_tolerance() older = MembershipRevision("raw-old", session_revision_projection(session)) newer = MembershipRevision("raw-new", session_revision_projection(changed_state)) - assert older.projection.event_identity_hashes != newer.projection.event_identity_hashes - assert not _strictly_dominates(older.projection, newer.projection) - assert not _strictly_dominates(newer.projection, older.projection) + assert older.projection.event_contents != newer.projection.event_contents + assert _relation(older.projection, newer.projection) == "conflict" result = classify_membership_revisions([older, newer]) assert result.accepted_raw_ids == () assert result.ambiguous_raw_ids == ("raw-new", "raw-old") -def test_generation_lifecycle_duration_change_does_change_the_session_content_hash() -> None: - """Measurement tolerance must not leak into idempotency. - - The real provider-reported duration is still archive evidence worth - keeping and re-indexing when it changes -- only *revision comparison* is - tolerant of it, not ``session_hash``. - """ - first_export = _generation_lifecycle_revision("raw-first", 13000) - second_export = _generation_lifecycle_revision("raw-second", 21000) - - assert first_export.projection.session_hash != second_export.projection.session_hash - - -def test_non_provider_reported_event_duration_field_stays_load_bearing() -> None: - """The volatile-key exclusion is scoped to `duration_semantics == - "provider_reported_elapsed"` events only -- an unrelated event type that - happens to carry a same-named field is not touched, so a real difference - there still counts as divergence. +def test_non_allowlisted_event_type_keeps_its_full_payload_as_content() -> None: + """The allowlist is scoped to `generation_lifecycle` -- an unrelated event + type with no registered allowlist compares its FULL payload, so a real + difference there still counts as divergence. """ session = ParsedSession( source_name=Provider.CHATGPT, @@ -1040,23 +538,52 @@ def test_non_provider_reported_event_duration_field_stays_load_bearing() -> None older = MembershipRevision("raw-old", session_revision_projection(session)) newer = MembershipRevision("raw-new", session_revision_projection(other_value)) - assert older.projection.event_identity_hashes != newer.projection.event_identity_hashes + assert older.projection.event_contents != newer.projection.event_contents + assert _relation(older.projection, newer.projection) == "conflict" result = classify_membership_revisions([older, newer]) assert result.accepted_raw_ids == () assert result.ambiguous_raw_ids == ("raw-new", "raw-old") -def test_refuses_event_growth_that_is_not_append_only() -> None: - """Measurement tolerance on ``generation_lifecycle`` must not turn the event - axis's prefix check into a bare count comparison. - - The newer revision has more events than the older one (real growth on the - count axis), but the extra event is *prepended*, not appended -- the - older revision's one event is not a positional prefix of the newer - revision's two. That is a real reshuffle of the event timeline, not - ordinary append growth, and must stay ambiguous. +def test_accepts_reordered_events_with_identical_content_as_equivalent() -> None: + """Same three durations, different array positions -- observed directly on + the live archive for `generation_lifecycle` events across two ChatGPT + export requests of the SAME conversation (polylogue-nuec). """ + + def session_with(order: list[int]) -> ParsedSession: + return ParsedSession( + source_name=Provider.CHATGPT, + provider_session_id="session", + messages=[ParsedMessage(provider_message_id="0", role=Role.ASSISTANT, text="answer")], + session_events=[ + ParsedSessionEvent( + event_type="generation_lifecycle", + timestamp=str(duration / 1000), + source_message_provider_id=f"m{duration}", + payload={ + "state": "completed", + "evidence_source": "provider_native", + "fidelity": "exact", + "duration_semantics": "provider_reported_elapsed", + "elapsed_duration_ms": duration, + }, + ) + for duration in order + ], + ) + + forward = MembershipRevision("raw-forward", session_revision_projection(session_with([161000, 79000, 115000]))) + shuffled = MembershipRevision("raw-shuffled", session_revision_projection(session_with([79000, 115000, 161000]))) + + assert forward.projection.event_hashes != shuffled.projection.event_hashes + assert forward.projection.event_contents == shuffled.projection.event_contents + assert _relation(forward.projection, shuffled.projection) == "equal" + + +def test_refuses_event_growth_that_loses_an_existing_event() -> None: + """A genuinely appended event alongside a genuinely lost one is a fork.""" shared_event = ParsedSessionEvent( event_type="generation_lifecycle", timestamp="13.0", @@ -1069,7 +596,7 @@ def test_refuses_event_growth_that_is_not_append_only() -> None: "elapsed_duration_ms": 13000, }, ) - prepended_event = ParsedSessionEvent( + other_event = ParsedSessionEvent( event_type="chatgpt_block_metadata", timestamp="1.0", source_message_provider_id="0", @@ -1081,14 +608,265 @@ def test_refuses_event_growth_that_is_not_append_only() -> None: messages=[ParsedMessage(provider_message_id="0", role=Role.ASSISTANT, text="answer")], session_events=[shared_event], ) - newer_session = older_session.model_copy(update={"session_events": [prepended_event, shared_event]}) + newer_session = older_session.model_copy(update={"session_events": [other_event]}) older = MembershipRevision("raw-old", session_revision_projection(older_session)) newer = MembershipRevision("raw-new", session_revision_projection(newer_session)) - assert len(newer.projection.event_identity_hashes) > len(older.projection.event_identity_hashes) - assert not _strictly_dominates(older.projection, newer.projection) + assert _relation(older.projection, newer.projection) == "conflict" result = classify_membership_revisions([older, newer]) assert result.accepted_raw_ids == () assert result.ambiguous_raw_ids == ("raw-new", "raw-old") + + +def test_disambiguates_multiple_same_type_events_on_one_message_by_content() -> None: + """Multiple same-type events on one message (e.g. one chatgpt_block_metadata + per block) are distinguished by their OWN content, never by array + position. + """ + + def block_event(index: int) -> ParsedSessionEvent: + return ParsedSessionEvent( + event_type="chatgpt_block_metadata", + timestamp="1.0", + source_message_provider_id="0", + payload={"block_index": index}, + ) + + session = ParsedSession( + source_name=Provider.CHATGPT, + provider_session_id="session", + messages=[ParsedMessage(provider_message_id="0", role=Role.ASSISTANT, text="answer")], + session_events=[block_event(0), block_event(1)], + ) + reordered = session.model_copy(update={"session_events": [block_event(1), block_event(0)]}) + + older = MembershipRevision("raw-old", session_revision_projection(session)) + newer = MembershipRevision("raw-new", session_revision_projection(reordered)) + + assert len(older.projection.event_contents) == 2 + assert older.projection.event_contents == newer.projection.event_contents + assert _relation(older.projection, newer.projection) == "equal" + + +# --------------------------------------------------------------------------- +# Cross-axis fork detection +# --------------------------------------------------------------------------- + + +def test_conflict_on_one_axis_forces_overall_conflict_even_when_others_agree() -> None: + """A single-axis conflict (here: attachments) must not be masked by the + other two axes agreeing. + """ + left = _attachment_revision("raw-a", acquired=True, inline=b"left bytes") + right = _attachment_revision("raw-b", acquired=True, inline=b"right bytes") + # Messages and events are identical (both sessions built the same way); + # only the attachment bytes conflict. + assert left.projection.message_contents == right.projection.message_contents + assert _relation(left.projection, right.projection) == "conflict" + + +def test_mixed_direction_axes_is_a_conflict_not_a_pick() -> None: + """One axis growing forward while another grows backward is a fork. + + raw-a has an extra message but is missing an attachment raw-b has -- + neither strictly contains the other overall, even though each + individual axis alone looks like ordinary growth in ONE direction. + """ + attachment = ParsedAttachment( + provider_attachment_id="uuid-1", + message_provider_id="0", + name="a.png", + mime_type="image/png", + ) + session_a = ParsedSession( + source_name=Provider.CHATGPT, + provider_session_id="session", + messages=[ + ParsedMessage(provider_message_id="0", role=Role.USER, text="one"), + ParsedMessage(provider_message_id="1", role=Role.ASSISTANT, text="two"), + ], + attachments=[], + ) + session_b = session_a.model_copy(update={"messages": session_a.messages[:1], "attachments": [attachment]}) + + a = MembershipRevision("raw-a", session_revision_projection(session_a)) + b = MembershipRevision("raw-b", session_revision_projection(session_b)) + + assert _relation(a.projection, b.projection) == "conflict" + + result = classify_membership_revisions([a, b]) + assert result.accepted_raw_ids == () + assert result.ambiguous_raw_ids == ("raw-a", "raw-b") + + +# --------------------------------------------------------------------------- +# Browser-capture fidelity ordering and direct-export precedence (unchanged +# escape hatches -- see _provider_ordered_browser_snapshots's docstring for +# why this one survives polylogue-aggz's content-only model) +# --------------------------------------------------------------------------- + + +def test_browser_native_snapshot_accepts_later_provider_revision_when_messages_reorder() -> None: + older = _revision("raw-old", "prompt", "attachment context") + newer = _revision("raw-new", "prompt", "tool result", "attachment context") + older = MembershipRevision( + older.raw_id, + older.projection, + "2026-01-01T00:00:00Z", + observed_at_ms=1, + browser_snapshot_fidelity="native", + provider_message_ids=frozenset({"prompt", "attachment"}), + ) + newer = MembershipRevision( + newer.raw_id, + newer.projection, + "2026-01-01T00:01:00Z", + observed_at_ms=2, + browser_snapshot_fidelity="native", + provider_message_ids=frozenset({"prompt", "tool", "attachment"}), + ) + + result = classify_membership_revisions([newer, older]) + + assert result.accepted_raw_ids == ("raw-old", "raw-new") + assert result.ambiguous_raw_ids == () + + +def test_browser_native_snapshot_refuses_later_revision_that_loses_message_identity() -> None: + older = _revision("raw-old", "prompt", "left") + newer = _revision("raw-new", "prompt", "right") + revisions = [ + MembershipRevision( + older.raw_id, + older.projection, + "2026-01-01T00:00:00Z", + observed_at_ms=1, + browser_snapshot_fidelity="native", + provider_message_ids=frozenset({"prompt", "left"}), + ), + MembershipRevision( + newer.raw_id, + newer.projection, + "2026-01-01T00:01:00Z", + observed_at_ms=2, + browser_snapshot_fidelity="native", + provider_message_ids=frozenset({"prompt", "right"}), + ), + ] + + result = classify_membership_revisions(revisions) + + # Genuine divergence (disjoint message identities) stays quarantined + # ambiguous -- neither the browser-fidelity ordering nor direct-export + # precedence can order it. + assert result.accepted_raw_ids == () + assert result.ambiguous_raw_ids == ("raw-new", "raw-old") + + +def test_browser_native_upgrade_refuses_any_shrinking_frontier_dimension() -> None: + older = _revision("raw-old", "prompt") + older_projection = older.projection.__class__( + session_hash=b"o" * 32, + message_hashes=older.projection.message_hashes, + message_contents=older.projection.message_contents, + event_hashes=older.projection.event_hashes, + event_contents=older.projection.event_contents, + attachment_identities=frozenset({b"attachment"}), + attachment_contents=frozenset(), + ) + newer = _revision("raw-new", "prompt", "answer") + revisions = [ + MembershipRevision( + older.raw_id, + older_projection, + "2026-01-01T00:00:00Z", + observed_at_ms=1, + browser_snapshot_fidelity="dom", + ), + MembershipRevision( + newer.raw_id, + newer.projection, + "2026-01-01T00:01:00Z", + observed_at_ms=2, + browser_snapshot_fidelity="native", + ), + ] + + result = classify_membership_revisions(revisions) + + # Mixed-direction axes (newer gained a message, older's-only attachment + # is now missing) is a fork -- neither browser-fidelity ordering (the + # dom->native frontier check requires every dimension to be >=) nor + # direct-export precedence can order it, so it stays quarantined + # ambiguous. + assert result.accepted_raw_ids == () + assert result.ambiguous_raw_ids == ("raw-new", "raw-old") + + +def test_direct_export_outranks_browser_capture_siblings_regardless_of_growth() -> None: + """A genuine non-browser-capture revision always wins over dom/native + browser-capture siblings, even though its content is neither a + growth superset of them nor resolvable by provider-timestamp comparison. + Browser capture exists to backfill a session before its paired direct/ + native provider export shows up, never to compete with or shadow that + export once it arrives (polylogue-z1c6).""" + direct = _revision("raw-direct", "real one", "real two", "real three") + dom = MembershipRevision( + "raw-dom", + _revision("raw-dom", "dom-only-turn").projection, + "2026-07-04T09:55:00Z", + browser_snapshot_fidelity="dom", + ) + native = MembershipRevision( + "raw-native", + _revision("raw-native", "native-turn-a", "native-turn-b").projection, + "2026-07-04T09:54:00Z", + browser_snapshot_fidelity="native", + ) + + result = classify_membership_revisions([dom, native, direct]) + + assert result.accepted_raw_ids == ("raw-direct",) + assert result.equivalent_raw_ids == ("raw-dom", "raw-native") + assert result.ambiguous_raw_ids == () + + +def test_browser_snapshot_accepts_later_attachment_enrichment_without_provider_update() -> None: + older = _revision("raw-old", "prompt", "answer") + newer = _revision("raw-new", "prompt", "answer") + newer_projection = newer.projection.__class__( + session_hash=b"n" * 32, + message_hashes=newer.projection.message_hashes, + message_contents=newer.projection.message_contents, + event_hashes=newer.projection.event_hashes, + event_contents=newer.projection.event_contents, + attachment_identities=frozenset({b"attachment-v2"}), + attachment_contents=frozenset(), + ) + revisions = [ + MembershipRevision( + older.raw_id, + older.projection, + "2026-01-01T00:01:00Z", + observed_at_ms=1, + browser_snapshot_fidelity="native", + provider_message_ids=frozenset({"prompt", "answer"}), + provider_attachment_ids=frozenset({"asset"}), + ), + MembershipRevision( + newer.raw_id, + newer_projection, + "2026-01-01T00:01:00Z", + observed_at_ms=2, + browser_snapshot_fidelity="native", + provider_message_ids=frozenset({"prompt", "answer"}), + provider_attachment_ids=frozenset({"asset"}), + ), + ] + + result = classify_membership_revisions(revisions) + + assert result.accepted_raw_ids == ("raw-old", "raw-new") + assert result.ambiguous_raw_ids == () diff --git a/tests/unit/pipeline/test_pipeline_ids.py b/tests/unit/pipeline/test_pipeline_ids.py index 35c11db15e..25490bd3a2 100644 --- a/tests/unit/pipeline/test_pipeline_ids.py +++ b/tests/unit/pipeline/test_pipeline_ids.py @@ -214,8 +214,12 @@ def test_session_revision_projection_golden_hashes() -> None: "65a99313c3ed8b81e69ecc0b36f314b3bf7848bc10fcea11415ddb9b07188941", "8efbf7b5b70bef4d73ec3550fe97e6f4d456cd724550bc5621a36766fe7f1f8b", ] + # Content-derived identity (message_id, name, mime_type) -- no longer a + # hash of the provider attachment id (polylogue-aggz / polylogue-d8al): + # a provider id is not guaranteed present across export vintages, so it + # is excluded from identity rather than used when present. assert {h.hex() for h in projection.attachment_identities} == { - "c233d41c034109580c7d7e74a96944a55eb0fadcc2564001dcf607f727b48062" + "2f18566179352065740615ea89e60130da5a8e46aae224e36b44ed626722da54" } # The golden attachment declares a size but carries no bytes, so it is # referenced-but-unacquired: identity is known, content is not. @@ -270,13 +274,15 @@ def test_session_revision_projection_matches_independent_recomputation() -> None assert projection.session_hash.hex() == independent_session_hash assert list(projection.message_hashes) == [bytes.fromhex(hash_payload(p)) for p in independent_message_payloads] + # Content-derived identity: message_id/name/mime_type only, never the + # provider attachment id (polylogue-aggz / polylogue-d8al). assert projection.attachment_identities == frozenset( - bytes.fromhex(hash_payload({field: p[field] for field in ("id", "message_id", "name", "mime_type")})) + bytes.fromhex(hash_payload({field: p[field] for field in ("message_id", "name", "mime_type")})) for p in independent_attachment_payloads ) assert projection.attachment_contents == frozenset( ( - bytes.fromhex(hash_payload({field: p[field] for field in ("id", "message_id", "name", "mime_type")})), + bytes.fromhex(hash_payload({field: p[field] for field in ("message_id", "name", "mime_type")})), bytes.fromhex(str(p["inline_content_hash"])), ) for p in independent_attachment_payloads From ce17be0a69d59d5491823baf87c3571d048465fb Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 30 Jul 2026 17:15:50 +0200 Subject: [PATCH 6/8] chore(beads): note c429/nuec/d8al superseded by aggz rewrite Ref polylogue-c429 Ref polylogue-nuec Ref polylogue-d8al Ref polylogue-aggz Co-Authored-By: Claude --- .beads/issues.jsonl | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 748a906128..254dc9c6d8 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -1,3 +1,6 @@ +{"_type":"issue","id":"polylogue-aggz","title":"Collapse the failure taxonomy into three invariants that make the cases unrepresentable","description":"## The problem with the current shape\n\nOne day of investigation produced eleven separately-named defects and found four\nexisting special-case code paths. That is a taxonomy, not an architecture. Every\nnew provider quirk becomes another named category, another branch, another bead,\nand the system's correctness becomes a function of how many cases someone\nremembered. The goal is the opposite: make these failures unrepresentable, so\nthey are not known as anything at all.\n\nAlmost all of it collapses into three invariants.\n\n## Invariant 1 -- comparison identity contains only content\n\n**A conversation is a SET of messages keyed by stable provider identity, each\ncarrying only content-bearing fields. Nothing else may enter the value used to\ncompare two acquisitions of it.**\n\nCollapses, as consequences rather than cases:\n\n- polylogue-bu1i (attachment acquisition state in attachment identity) --\n acquisition state is not content.\n- polylogue-c429 (message array order) -- a set has no order.\n- polylogue-nuec (chatgpt elapsed_duration_ms) -- a measurement is not content.\n- polylogue-hith (synthetic attachment id seeded on position) -- position is not\n identity.\n- polylogue-d8al (real-id presence varies between vintages) -- identity must be\n derivable from content when the provider omits its own.\n- polylogue-oycw (positional-prefix superset test) -- set containment, not\n sequence prefix.\n- `_provider_ordered_browser_snapshots` -- exists only because DOM ordering\n differs from export ordering. Under a set, it has nothing to fix.\n- The `superseded_prefix` / `superseded_equivalent` distinction -- both are just\n \"contained or equal\".\n\nSuperset-ness becomes total and decidable, with no residual category:\n\n equal same id set, equal content per id\n contains A's id set contains B's, equal content on the intersection\n conflict content differs on the intersection\n\nOrdering remains a stored, rendered property of a session. The claim is only\nthat it is not part of the comparison value. `_direct_export_precedence` (a real\nexport outranks a browser capture) probably survives as a genuine provenance\nrule rather than a repair.\n\n## Invariant 2 -- one chokepoint may write a session\n\n**It must be structurally impossible to materialize a session without consulting\nrevision authority.**\n\npolylogue-c737, PR #3397 and PR #3398 all exist because two write paths each\ncarried their own precedence logic, and one of them forgot. #3398 then had to\ncorrect #3397's scope on one path while the other stayed wrong, which is the\nsignature of duplicated semantics rather than a missing check.\n\nThe fix is structural, not another check: one function through which every\nsession write passes, taking authority as a required argument, so a caller\ncannot forget to ask. A predicate copied into two places is a bug that has not\nhappened yet.\n\n## Invariant 3 -- derived state carries the version of the logic that derived it\n\n**Any stored conclusion records which version of which computation produced it,\nso a corrected computation invalidates its own stale outputs automatically.**\n\npolylogue-9dxn is this, and its absence is what made polylogue-bu1i inert on\nexisting data: a persisted `ambiguous` verdict has no version, so a corrected\nclassifier cannot know which verdicts it now disagrees with. The two-component\ndesign already recorded on 9dxn (separate identity and classification\nfingerprints) is the mechanism.\n\nWith this, \"stale verdict\", \"needs re-census\", and \"the fix does not apply to\nexisting rows\" all stop being categories. Correction becomes self-healing by\nconstruction.\n\n## What this does to the current bead set\n\nReframe rather than close -- the individual fixes still ship, but as instances:\n\n bu1i c429 nuec hith d8al oycw -\u003e Invariant 1\n c737 (+ the shape behind #3397/#3398) -\u003e Invariant 2\n 9dxn -\u003e Invariant 3\n ck5v -\u003e not covered; genuinely separate\n (backfill coupled to acquisition\n route -- an availability rule, not\n an identity one)\n ey3r -\u003e a measurement defect, but its cause\n is Invariant 1: it counts\n `superseded_*` as missing because\n the vocabulary has redundant\n categories that Invariant 1 removes\n\n## How to tell whether this worked\n\nNot \"the tests pass\". The observable is that the vocabulary shrinks:\n\n- The membership decision vocabulary loses `superseded_prefix` as distinct from\n `superseded_equivalent`.\n- `_provider_ordered_browser_snapshots` is deleted rather than maintained.\n- `HISTORICAL_NON_PREFIX_GOVERNANCE_DETAIL` and its legacy-detail variants stop\n needing to exist, because non-prefix growth stops being exceptional.\n- No new provider quirk requires a new branch in the classifier.\n\nIf a change adds a case instead of removing one, it is going the wrong way even\nif it makes a test pass. Per this repo's own surgical-renewal rule, the old path\nis deleted in the same change that replaces it -- these special cases must not\nsurvive as dead alternates beside the invariant.\n\n## Acceptance criteria\n\n- The comparison value for a session is constructed from an explicit\n content-only allowlist, so adding a field to a parser cannot silently enter\n identity. Adding a volatile field and observing that comparison is unaffected\n is the test.\n- Exactly one code path can write a session, and it cannot be called without\n authority.\n- Every stored verdict carries a version; changing the logic invalidates the\n affected verdicts without an operator command.\n- At least two existing special-case paths are DELETED, not merely bypassed.\n","status":"open","priority":0,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T14:41:28Z","created_by":"Sinity","updated_at":"2026-07-30T14:41:28Z","labels":["area:ingest"],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-oycw","title":"Coalescing rests on a positional-prefix superset test that real providers violate; 41% of the corpus depends on it","description":"## Scale first: this is the archive's normal condition, not an edge case\n\n logical identities with more than one raw 7,440\n total logical identities 18,228\n -\u003e 41% of the corpus is multi-raw\n\nCohort sizes by origin (raws in multi-member cohorts):\n\n chatgpt-export 3-member 5,817 4-member 551 +tail to 12\n claude-ai-export 4-member 3,592 3-member 258 +tail to 9\n codex-session 2-member 3,544 ... one cohort of 105\n claude-code-session 2-member 2,338 3-member 663 +tail to 25\n hermes-session 2-member 536\n aistudio-drive 2-member 302\n antigravity-session 2-member 232\n browser-capture raws 887 (786 chatgpt, 47 claude-ai, 38 unknown, 16 grok)\n\nCorrectness for nearly half the archive rests on the revision-arbitration layer.\nIt is not a rarely-exercised safety net.\n\n## Where the multiplicity comes from\n\nNot divergence. Repeated whole-account acquisition:\n\n claude-ai-data-2025-10-04 906 raws\n claude-ai-data-2026-04-23 973 raws\n claude-ai-data-2026-06-14 1,998 raws\n chatgpt-data-2025-10-20 2,072 raws\n chatgpt-data-2026-04-23 4,805 raws\n\nEvery GDPR export contains every conversation, so each conversation enters the\narchive once per export vintage. 577 of the 587 claude-ai ambiguous cohorts have\nexactly 4 members for this reason.\n\n## Layer 1 -- identity. This one is sound.\n\n`sessions.session_id` is a generated column, `origin || ':' || native_id`, where\n`native_id` is the parser's `provider_session_id` -- the provider's own\nconversation uuid. Measured: 34 of 35 sampled claude-ai cohorts have an\nIDENTICAL provider_message_id set across all members, and the conversation uuid\nis identical across all four export vintages.\n\nSession identity is stable across acquisitions. The failures found on\n2026-07-30 were narrower and are separately tracked: a dispatch bug appending a\nspurious `-0` (fixed 2026-07-20, polylogue-eqnv), and unstable synthetic\n*attachment* ids (polylogue-hith / polylogue-d8al) -- not session ids.\n\n**Identity is not the problem, and a fix aimed at identity will not help.**\n\n## Layer 2 -- coalescing. Two mechanisms that do not compose.\n\n**(a) Content-hash idempotency** (`pipeline/ids.py:session_content_hash`).\nRe-ingest with a matching hash is skipped. The hash deliberately excludes user\nmetadata, but it INCLUDES: message array order, attachment acquisition state,\nvolatile provider metadata, and synthetic ids. Across two exports of an\nunchanged conversation, at least one of those always differs.\n\nSo idempotency never fires across export vintages -- by construction, not by\naccident. Every re-export falls through to (b).\n\n**(b) Revision membership arbitration.** Decides which raw is authoritative for\na session id when hashes differ. This carries the entire load that (a) fails to\nabsorb, for 41% of the corpus.\n\n## Layer 3 -- superset determination. This is the actual defect.\n\n`_strictly_dominates` (`archive/session_revision_membership.py`) requires:\n\n older.message_hashes == newer.message_hashes[: len(older.message_hashes)]\n\na POSITIONAL PREFIX. Three assumptions are embedded there, and all three are\nviolated by real providers:\n\n1. *Messages keep a stable order across acquisitions.* Violated: 19 of 35\n sampled claude-ai cohorts differ only in array order, same ids, zero content\n differences. Claude.ai does not emit a stable sequence between exports.\n2. *A message's hash is a function of its content alone.* Violated by volatile\n provider metadata (chatgpt `elapsed_duration_ms`, polylogue-nuec) and by\n acquisition state (Drive attachment bytes, polylogue-bu1i).\n3. *Growth is append-only at the tail.* Violated whenever a provider edits or\n inserts mid-conversation, and structurally by browser-capture DOM snapshots.\n\nWhen the test fails in both directions the cohort is quarantined ambiguous and\nNOTHING is indexed -- so a conversation held complete, correct, and in four\nidentical copies is absent from the archive. That is the 1,009-1,027 absence\npopulation.\n\n## What the correct test looks like\n\nPer-message ids are stable (34/35 measured), so superset-ness is decidable on\nevidence we already hold, without ordering:\n\n equal same provider_message_id SET, equal content per id\n -\u003e semantically the same revision; `equivalent_raw_ids`,\n no arbitration needed at all\n dominates A's id set strictly contains B's, content equal on the\n intersection -\u003e A is authoritative\n fork neither contains the other, OR content differs on the\n intersection -\u003e genuinely ambiguous, and rare\n (0 of 35 sampled claude-ai; 1 plausible case archive-wide,\n in grok-export)\n\nOrdering remains a real property of a session and must still be stored and\nrendered -- the claim is only that ordering must not be the DOMINANCE key.\nA conversation is a set of identified messages plus an ordering; which evidence\nexists is a set question, and treating the sequence as identity makes every\nprovider-side reordering look like divergence.\n\nLikewise a message's identity for comparison must exclude provider-volatile\nmeasurement fields and acquisition state, for the same reason bu1i split\nattachment identity from attachment acquisition.\n\n## Browser capture\n\n887 raws, 786 of them chatgpt. A DOM snapshot legitimately carries different\nsynthetic ids and a different ordering from the same conversation's export, so\nit violates assumptions 1 and 3 by design. `_provider_ordered_browser_snapshots`\nand `_direct_export_precedence` exist to special-case it, which is evidence that\nthe general test was already known to be too strict -- the special cases are\npatches over the wrong primitive rather than genuine domain rules. Re-evaluate\nboth once the set-based test lands; `_direct_export_precedence` (a real export\noutranks a browser capture) is probably a genuine rule worth keeping, while the\nordering special-case may become unnecessary.\n\n## Acceptance criteria\n\n- Superset determination is order-independent and decided on stable per-message\n identity plus per-id content equality.\n- Equal-content cohorts resolve as `equivalent`, not `ambiguous`, and index one\n member -- no arbitration for the 34/35 case.\n- Message comparison identity excludes provider-volatile measurement fields and\n acquisition state.\n- Report how many cohorts still reach a genuine-fork verdict; it should be very\n small, and a large number means one of the above is wrong.\n- Re-run `.agent/scripts/corpus-fidelity-audit.py`: absent_documents must fall\n to approximately zero from the 1,027 baseline.\n\nRef polylogue-bu1i, polylogue-c429, polylogue-nuec, polylogue-d8al, polylogue-f1vg\n","status":"open","priority":0,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T14:35:25Z","created_by":"Sinity","updated_at":"2026-07-30T14:35:25Z","labels":["area:ingest"],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-c737","title":"ArchiveStore._write_parsed_precedence_result writes a session for a raw recorded raw_session_memberships.decision='ambiguous'","description":"## What the live archive shows (coordinator measurement, confirmed independently)\n\nLive query against `/realm/db/polylogue` for `origin='aistudio-drive'`:\n34 cohorts now have BOTH members parsed (growing over the course of one\nsession). 28 of those have `raw_session_memberships.decision='ambiguous'`\non BOTH members under the SAME `logical_source_key` -- genuinely arbitrated,\ncorrectly refused a winner -- yet the cohort's session IS present in\nindex.db with zero acquired attachments: 641 attachments total across the\n28, every one `unfetched`, while the enriched sibling in the blob store\nholds the bytes.\n\n## Root cause, traced and reproduced\n\n`polylogue/storage/sqlite/archive_tiers/archive.py`'s\n`apply_raw_membership_classification` (the classify_membership_revisions\nconsumer) is innocent: for a fully-ambiguous cohort (no accepted_raw_ids)\nit explicitly clears `raw_sessions.parsed_at_ms` and never writes to\n`sessions` -- verified by reading its finalization block (`complete` check\nat the end of the function, ~line 3873-3901).\n\nThe actual writer is `_write_parsed_precedence_result` (same file), reached\nvia `write_parsed_for_retained_raw`/`write_parsed_for_retained_raw_result`\nwith `revision_authoritative=False` (the default -- used by the one-shot\nimporter, `pipeline/services/archive_ingest.py`, and by\n`_index_parsed_for_retained_raw`'s other non-membership-governed callers).\nIts ONLY revision-authority awareness before this fix was:\n\n governed = SELECT 1 FROM raw_revision_heads WHERE session_id = ?\n if governed is not None: skip\n\n`raw_revision_heads` is populated ONLY when a cohort has an ACCEPTED\nwinner. A cohort `classify_membership_revisions` genuinely refused to\narbitrate never gets an accepted head, so `governed` stays `None` and the\nfunction falls through to its own browser-capture-precedence/freshness\nlogic and writes the session unconditionally on the raw's next reparse --\nlast-writer-wins, independent of the recorded `ambiguous` verdict.\n`repair.py:1075`/`repair.py:4432-4462` (the two gates the investigation\nstarted from) are both innocent: neither is on this write path at all --\n`repair.py:4432` is a read-only reporting/accounting classifier\n(`_raw_replay_plan_outcome`), and `repair.py:1075` is a narrow inspector\nfor a different (`source-v7`/`quarantined-accepted-raw`) repair scenario.\n\nReproduced directly: a synthetic archive with a raw whose\n`raw_session_memberships` row is `decision='ambiguous'`, then calling\n`archive.write_parsed_for_retained_raw(session, raw_id=..., ...)` (no\n`revision_authoritative`) writes the session anyway pre-fix; the fix makes\nit a no-op (`content_changed=False`).\n\n## Fix landed in polylogue-af059's fast-follow PR\n\n`_write_parsed_precedence_result` now also refuses when the raw's OWN\n`raw_session_memberships.decision = 'ambiguous'`, in addition to the\nexisting `raw_revision_heads` check.\n\n## Known sibling hole, NOT fixed here (different file, different owner)\n\n`polylogue/pipeline/services/ingest_batch/_core.py` (the daemon's default\nbatch-ingest write path, used for most origins that don't go through\n`sources/live/batch.py`'s revision-authority-aware branch) has the SAME\nshape: its own precedence/freshness logic, no `raw_session_memberships`\nconsultation. The coordinator's own measurement\n(`revision_authority='quarantined'` with `parsed_at_ms` set: chatgpt-export\n7,050, codex-session 3,633, claude-code-session 2,450, claude-ai-export\n1,562 -- NOT all necessarily leaked materializations, but the same shape)\nsuggests this is where most of the non-drive volume would leak through, if\nthose origins' raws ever get genuinely `ambiguous`-recorded membership\ndecisions. Needs its own read-only census to confirm before fixing (not\ndone here -- out of file-ownership scope for this PR).\n\n## Live remediation\n\nNOT performed here (code-only fix). The 28 live aistudio-drive sessions\nwith zero-acquired attachments need their own re-materialization pass once\nthis fix (and bu1i's classifier fix) are both deployed.\n\nRef polylogue-eqnv, polylogue-bu1i, polylogue-7ilr, polylogue-9dxn","status":"closed","priority":0,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T13:09:15Z","created_by":"Sinity","updated_at":"2026-07-30T13:20:58Z","closed_at":"2026-07-30T13:20:58Z","close_reason":"Fixed in PR #3397 (feature/fix/ambiguous-membership-precedence-write-leak): ArchiveStore._write_parsed_precedence_result now also refuses to write when the raw's own raw_session_memberships.decision='ambiguous', alongside the pre-existing raw_revision_heads check. Verified with a regression test (anti-vacuity confirmed via temporary guard short-circuit + rerun). Sibling hole in pipeline/services/ingest_batch/_core.py NOT fixed here (different file, out of ownership scope) -- needs its own read-only census before fixing; tracked as residual scope in this same bead's description.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-8249","title":"rebuild parse workers capped at 8 on a 24-thread host; ingest_parse_workers config is inert","design":"Found 2026-07-29 by codebase audit, while checking whether the imminent full\nrebuild honors its parse-worker configuration. TWO defects, one of which\ndirectly caps rebuild throughput.\n\n(1) THE PARSE-WORKER COUNT IS CAPPED AT 8 ON A 24-THREAD HOST\n\npolylogue/pipeline/services/process_pool.py:52\n default = max(1, min(8, (os.cpu_count() or 2) - 1))\n\nOn sinnix-prime (i7-13700K, 16 cores / 24 threads) this resolves to 8, leaving\n16 threads idle. The rebuild path reaches it directly:\n maintenance/rebuild_index.py:543 ingest_workers=None\n maintenance/replay.py:199 resolved = ... else resolve_parse_worker_count()\n\nThe daemon now runs free-threaded 3.14t (GIL disabled) and the GIL parse path\nwas deleted this session, so thread-parallel parse is the only path -- the\n`min(8, ...)` ceiling is the binding constraint on a rebuild we are trying to\nbring from 9.2h down to 1-2h. The cap predates the free-threaded deploy; on a\nGIL build 8 was a reasonable process-pool bound, but that reasoning no longer\napplies.\n\nBEFORE THE REBUILD: either raise/remove the cap, or set\nPOLYLOGUE_INGEST_PARSE_WORKERS explicitly for the rebuild run. Do NOT assume\nhigher is strictly better -- measure. Parse is decode-bound but the apply side\nis a single writer, so beyond some width the writer becomes the bottleneck and\nextra parse threads only add memory pressure. The new RebuildPassCost\ninstrumentation (replay_s / checkpoint_s / mib_per_s / parse_workers, landed\nthis session) is exactly the instrument for choosing the width from one short\nmeasured pass rather than guessing.\n\n(2) THE DOCUMENTED CONFIG KNOB IS INERT\n\nThere are two knobs for parse-worker count and only one works:\n env POLYLOGUE_INGEST_PARSE_WORKERS -- honored (process_pool.py:43,53)\n config `sources.ingest_parse_workers` -- IGNORED\n\nThe config property is defined (config.py:602), given a default\n(config.py:1876), listed in the config inventory twice (config.py:1387,1642),\nand documented (docs/configuration.md:351 \"Parallel parse workers during\ningest (default 1)\") -- but NOTHING reads it. An operator setting it in\n~/.config/polylogue/polylogue.toml gets silence.\n\nThe doc is also wrong independently of the wiring: it says \"default 1\" while\nthe resolver's actual default is min(8, cpus-1) = 8 here.\n\nFIX: make resolve_parse_worker_count read the resolved config, keeping the env\nvar as the override layer the config system already defines -- or delete the\nconfig property and document the env var as the sole knob. Per the standing\ndirective, one of the two must go; a documented knob that does nothing is\nworse than no knob. Whichever survives must be the one the rebuild reads.\n","status":"closed","priority":0,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-29T10:35:27Z","created_by":"Sinity","updated_at":"2026-07-29T17:16:31Z","closed_at":"2026-07-29T17:16:31Z","close_reason":"Fixed. Parse workers now scale to the interpreter: min(16, cpus-2) free-threaded, min(8, cpus-1) under the GIL. Verified under the deployed python3.14t -\u003e 16 workers, up from 8 on this 24-thread host. The module's own control-run measurement (3.9x at w=4 rising to 9.6x at w=16) is the evidence for 16 as the ceiling. Second defect also fixed: the inert sources.ingest_parse_workers config property (defined, defaulted, inventoried twice, documented as 'default 1', read by nothing) is deleted; POLYLOGUE_INGEST_PARSE_WORKERS survives as the single knob with an accurate inventory description. Commit a7945e9fe. Related: the devshell default is now the free-threaded shell (matching the daemon), so local runs no longer silently parse sequentially.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-2qx.4","title":"Field-landing decisions for the unread-wire batch: one index bump, one rebuild","description":"DECIDED. The audit established WHAT is discarded; this fixes WHERE each lands, so the parser work is mechanical and the schema changes batch into a single index-tier bump rather than one per origin.\n\n stop_reason (608,608 on wire)\n -\u003e column on messages. One value per assistant turn, low cardinality,\n feeds terminal_state directly. Replaces three columns that guess at it\n and are 85-99% 'unknown'.\n structuredPatch (105,123) + originalFile (92,313) + oldString/newString/filePath\n -\u003e new file_edits table keyed by tool_use_block_id. It is a RELATION (one\n edit per tool call), not a block attribute. This is what raises\n polylogue-cijx's file-trajectory grading from 'observed' to\n 'checkpointed' -- originalFile is the captured pre-state cijx declares\n unavailable.\n parentToolUseID (842,819 records, 185,982 distinct dispatch ids)\n -\u003e a real join-key column on session_links, plus method. It IS the\n delegation edge; it belongs where edges live. Replaces the positional\n pairing gated on count equality that resolves 12.8%.\n pr-link (20,702)\n -\u003e new session_refs table (kind, url, number, repo). Generalizes to issue\n refs and stays tracker-agnostic -- do not create a github_prs table.\n runSettings (aistudio-drive: temperature, topP, topK, maxOutputTokens,\n thinkingLevel, safetySettings, enable* flags)\n -\u003e JSON column on sessions. Genuinely per-session config; decomposing it\n into columns buys nothing and couples the schema to one provider.\n ai-title (18,422) / threads.title / slug (1,500) / agentId\n -\u003e sessions.title + title_source for the title; slug -\u003e a display_name\n column so subagent rows read 'greedy-squishing-hamming' rather than\n '5ecdb160-...:agent-af4e'.\n outcome-unknown reason\n -\u003e enum column beside blocks.tool_result_is_error. Three causes are\n collapsed into one NULL today (provider emitted nothing / parser\n deliberately distrusts it / parser does not read this provider's\n field), all knowable at parse time.\n tool-results sidecars (12,588 files, 1.34 GB, 3 ingested)\n -\u003e block content, attached to the existing tool_result block by tool_id\n (the filename IS the tool id). NEVER a session -- the hook-inflation\n incident (18,391 -\u003e 83,286 sessions) is the precedent.\n\nBATCHING: all of the above is ONE index-tier bump and ONE rebuild. Splitting by\norigin would mean four bumps and four rebuild windows against a corpus where a\nfull rebuild is the standing performance complaint (polylogue-623q). Do the\nschema change once, then the per-origin parser reads land against it\nincrementally without further bumps.\n\nSCOPE NOTE (operator, 2026-07-29): read everything SEMANTICALLY MEANINGFUL, not\neverything. Some wire fields are genuinely not worth a column -- the\nper-key classification in the OriginSpec fidelity declaration is where that\njudgement is recorded, and 'dropped, because X' is a valid outcome.","acceptance_criteria":"1. One index-tier bump covers every landing above; no second bump for a later origin. 2. Each landing is a typed column/table, not a JSON blob, except runSettings where the blob is the decision. 3. tool-results attachment leaves session count unchanged, asserted by a test. 4. Per-origin parser reads land against the new schema without further migrations. 5. The OriginSpec fidelity declaration records a per-key verdict including deliberate drops with reasons.","status":"closed","priority":0,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-29T04:52:50Z","created_by":"Sinity","updated_at":"2026-07-29T17:16:55Z","closed_at":"2026-07-29T17:16:55Z","close_reason":"Landed. INDEX_SCHEMA_VERSION 45-\u003e46 (SEMANTIC_REPARSE), one bump for the whole batch. The version decision was settled from bootstrap.py's actual code path, not assumed: a same-version reopen only re-applies benign CREATE TABLE/INDEX IF NOT EXISTS DDL and never ALTER TABLE, so staying at v45 would have left existing v45 archives silently missing the new columns. Landed: messages.stop_reason, blocks.tool_result_outcome_unknown_reason, sessions.display_name + run_settings_json, session_links.parent_tool_use_block_id, and the file_edits and session_refs tables. Parsers now populate all of them -- measured coverage: stop_reason 44.7% of main-session messages, file_edit 7,335/44,125 tool_result blocks, display_name 65.0% of subagent sessions, session_refs 1,483 rows, outcome_unknown_reason 19,613 not_reported + 80 distrusted. parent_tool_use_provider_id deliberately left NULL: two independent samples (200 subagent transcripts; 109,853 records) found parentToolUseID appears only on the PARENT's progress records, never on a child's own, so it cannot join parent to child. Delegation resolution instead uses content identity. tool-results sidecars needed no schema change (they attach to the existing tool_result block by tool_id).","labels":["area:ingest","area:sources","delivery:K-interop-origin-export","delivery:ac-patched","horizon:frontier","lane:origin-interop-export","refactor"],"dependencies":[{"issue_id":"polylogue-2qx.4","depends_on_id":"polylogue-2qx","type":"parent-child","created_at":"2026-07-29T06:52:49Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-cijx.4","title":"Repo identity, path normalization and readable labels are ONE batch","description":"DECIDED. These were three separate items; they are one, because the label is unusable until identity is fixed and both fall out of the same normalization.\n\nTHE EVIDENCE, from eight real untitled claude-code sessions:\n repo_name = 'agent-ad682bc849a1cd0f0'\n top path = /realm/project/polylogue/.claude/worktrees/agent-ad682bc849a1cd0f0/\n polylogue/pipeline/services/ingest_batch/_core.py\nA structural label today reads 'agent-ad682bc849a1cd0f0 - 27f - 499m' -- worse\nthan the UUID it replaces. repo_name derives from cwd, the cwd is a worktree\ndirectory, so the agent id becomes the repo name.\n\nNormalize both and the same eight sessions read:\n polylogue - pipeline/services/ingest_batch/_core.py +26 - 499 msgs\n polylogue - daemon/status.py +7 - 322 msgs\n polylogue - api/archive.py +10 - 259 msgs\n polylogue - tests/unit/insights/test_delegation_work_evidence.py +5 - 163 msgs\n polylogue - storage/repair.py +1 - 91 msgs\nFor a coding session, WHICH FILES YOU TOUCHED is the topic. That beats the\nprovider echo titles, which collide 78-way.\n\nDECISION 1 -- REPOSITORY IDENTITY\n A repository is keyed on its normalized remote (all spellings of one remote\n are one repo); where no remote exists, the outermost git root. NOT the cwd.\n A worktree is a CHECKOUT OF a repository, not a repository -- every\n /realm/worktrees/polylogue-* and .claude/worktrees/agent-* is one checkout of\n polylogue. A session with no git evidence resolves to a DIRECTORY and read\n surfaces say so; do not synthesize a repository for it. Measured today:\n polylogue holds 106 distinct repo_ids, sinex 28, sinnix 31; git_branch is\n populated on 15.8% of sessions, git_repository_url on 13.2%, commit_hash on\n 15.9% -- so for ~84% the 'repo' column is really cwd.\n\nDECISION 2 -- PATHS ARE REPO-RELATIVE\n Strip the checkout root prefix (already recorded as repos.root_path) so\n action_pairs.tool_path is comparable across checkouts of one repo. Without\n this, the same file edited in two worktrees is two different paths and no\n cross-session file question works.\n\nDECISION 3 -- THE LABEL IS A PROJECTION, NEVER A COLUMN\n Form: \u003crepo\u003e - \u003cdominant repo-relative path\u003e +N - \u003csize\u003e, substituting the\n provider title for the path clause when a real one exists. Computed at read\n time in the 4p1 Projection. It must not be written to sessions.title: it\n would collide with genuine provider titles (ai-title, threads.title) and\n freeze as the session grows -- '340 msgs' is wrong the moment message 341\n lands. Measured collision rate for the structural form: 3.5% over 4,000\n sessions, max collision 10, mostly pairwise -- acceptable, and far better\n than the echo baseline's 78-way.\n\nDECISION 4 -- RESULT UNIT IS THE TOP-LEVEL SESSION\n All eight sampled sessions above are agent-* subagents; 8,614 of 18,871\n sessions (45.6%) are subagent children. A default list is unreadable because\n it is half fanout. Default unit = top-level session; children reachable\n through an explicit projection, never filling the list. Any count states its\n unit -- '18,871 sessions' unqualified is wrong when 8,614 are children.\n\nSEQUENCE: identity+paths first (write-path change, no schema bump), then the\nlabel projection. Readability cannot land before identity.","acceptance_criteria":"1. One repository per normalized remote; worktrees enumerate underneath as checkouts; polylogue/sinex/sinnix each collapse to one. 2. tool_path is repo-relative; the same file in two worktrees is one path. 3. The display label is computed per request and appears in no table; sessions.title holds only provider-supplied values. 4. Default result unit is the top-level session, proven by re-running 'polylogue find repo:polylogue' and showing named non-fanout rows. 5. Report the label collision rate against the measured 3.5% / max-10 baseline.","status":"open","priority":0,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-29T04:52:49Z","created_by":"Sinity","updated_at":"2026-07-29T04:52:49Z","labels":["area:insights","area:interop","area:substrate","horizon:mid","lane:read-contracts","tech-tree"],"dependencies":[{"issue_id":"polylogue-cijx.4","depends_on_id":"polylogue-cijx","type":"parent-child","created_at":"2026-07-29T06:52:48Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} @@ -68,12 +71,14 @@ {"_type":"issue","id":"polylogue-tf2.1","title":"Rerun forensics on current archive; price origin_reported providers","description":"Rerun scripts/agent_forensics.py against the current archive (v23+); price origin_reported providers via the vendored LiteLLM catalog (match last path segment); all-provider headline or explicitly-labeled per-provenance figures that cannot be misread; record deltas vs 06-27; verify chart SVGs render. Cache-inclusion must be disambiguated (Codex input INCLUDES cached ~96%; see bd memories). Also blocked on logical-session token attribution — the headline must not be double-counted.","notes":"Correction to close_reason monetary values: stored/provider-priced subset was $239,453.14; catalog API-equivalent was $318,650.88; origin_reported catalog estimate was $79,197.74. The original close_reason text lost dollar-prefixed digits due shell expansion, not measurement drift.","status":"closed","priority":0,"issue_type":"task","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-07-03T04:31:33Z","created_by":"Sinity","updated_at":"2026-07-03T09:59:13Z","started_at":"2026-07-03T09:28:10Z","closed_at":"2026-07-03T09:59:02Z","close_reason":"Completed with blocker caveat preserved: scripts/agent_forensics.py now prices origin_reported rows through the shared vendored LiteLLM pricing catalog while preserving stored provenance; report separates stored/provider-priced cost from catalog API-equivalent estimates and carries logical-session/cache caveats instead of claiming final billing reconciliation. Regenerated current artifact at .agent/demos/agent-forensics against /home/sinity/.local/share/polylogue schema v23: 16,498 physical sessions, 4,142,175 messages, 356.5B tokens, ,453.14 stored/provider-priced subset, ,650.88 catalog API-equivalent, and ,197.74 origin_reported catalog estimate. SVG parse check passed for 9 charts; devtools test tests/unit/scripts/test_agent_forensics.py passed; devtools verify --quick passed run 20260703T095718Z-quick-753466-96559776; devloop-review clean. Remaining final-reconciliation blocker stays open as polylogue-4ts.2.","labels":["area:usage","campaign"],"dependencies":[{"issue_id":"polylogue-tf2.1","depends_on_id":"polylogue-4ts.2","type":"blocks","created_at":"2026-07-03T06:32:45Z","created_by":"Sinity","metadata":"{}"},{"issue_id":"polylogue-tf2.1","depends_on_id":"polylogue-sru.7","type":"blocks","created_at":"2026-07-03T06:31:33Z","created_by":"Sinity","metadata":"{}"},{"issue_id":"polylogue-tf2.1","depends_on_id":"polylogue-tf2","type":"parent-child","created_at":"2026-07-03T06:31:33Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":2,"dependent_count":2,"comment_count":0} {"_type":"issue","id":"polylogue-tf2","title":"Campaign: agent-forensics regeneration + all-provider repricing","description":"Regenerate the agent-forensics packet on the current archive with an honest all-provider headline. The 2026-06-27 report (546.6B tokens, $89,368 API-list equivalent, 216x cache amplification) is the most stranger-legible artifact on any shelf, but its numbers are pre-dedup stale and the headline prices only the priced-provenance subset (Claude Code cost_usd rows); Codex/ChatGPT/Gemini are origin_reported token counts with no dollar value (operator estimate ~$150K all-provider). Sequenced after claim-vs-evidence per operator direction 2026-07-02.","design":"Current slice design: turn the existing agent-forensics/cost headline into a product-backed all-provider repricing artifact. First inspect devtools/scripts and polylogue analyze surfaces for agent_forensics/cost code. Use active archive usage headline (detail=headline) for authoritative physical_session and logical_session_model_high_water token totals. Keep priced-provenance dollars and origin-reported token estimates separate: do not multiply every token by one blended price without a labeled lane. Add or reuse a shared pricing/projection helper so the demo artifact is regenerated from Polylogue product code, not ad hoc SQL. Acceptance for this slice: the generated agent-forensics artifact names archive root/schema, includes physical vs logical token grain, separates priced subset from origin-reported estimate lanes, gives reproduction commands, and has focused tests for any new repricing helper/surface.","acceptance_criteria":"Terminal state: regenerated forensics packet on the current archive with an honest all-provider headline (priced subset AND origin-reported estimate lanes separated), agent_forensics.py folded into polylogue analyze (tf2.2), artifact on the demo shelf with reproduction commands, cold-reader gate passed. Epic closes only when that artifact is recorded.","status":"closed","priority":0,"issue_type":"epic","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-07-03T04:31:32Z","created_by":"Sinity","updated_at":"2026-07-03T19:06:44Z","started_at":"2026-07-03T18:47:23Z","closed_at":"2026-07-03T19:06:44Z","close_reason":"Completed: provider usage headline now exposes product-backed pricing lanes in polylogue analyze usage --detail headline, separating stored/provider-priced cost from catalog API-equivalent estimates for origin_reported rows. Regenerated the current .agent/demos/agent-forensics artifact against /home/sinity/.local/share/polylogue schema v23: physical-session tokens 395,320,980,423; logical high-water tokens 288,741,229,728; stored/provider-priced USD 243,392.189328; catalog API-equivalent USD 337,565.031618; priced lane 13,889 rows / 12,331 sessions / 12,650 matched rows; origin_reported lane 2,308 rows / 2,270 sessions / 2,302 matched rows. Verification: live polylogue --plain analyze usage --detail headline --format json --limit 0 wrote /realm/tmp/polylogue-usage-headline-pricing-current.json; devtools test tests/unit/storage/test_provider_usage_report.py tests/unit/cli/test_diagnostics.py passed 23 tests; devtools verify --quick passed run 20260703T190553Z-quick-2226137-d91d4e8f; devtools workspace demo-shelf --json reported ok. Non-claim preserved: this is not final billing reconciliation and physical/logical token grains stay explicitly separated.","labels":["area:usage","campaign","size:M","spine"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-sru","title":"Campaign: claim-vs-evidence report to finding-grade","description":"Terminal state: an externally publishable finding ('how often do coding agents proceed past failed tool calls, by model/tool') with stated sample frame, calibrated markers, benign/consequential split, seeded stranger-runnable reproduction, and a passed cold-reader gate. Slice closure is NOT campaign closure; this epic stays top-of-frame until its terminal state is recorded.\\n\\nState as of 2026-07-03 after calibrated active-archive regeneration: archive root /home/sinity/.local/share/polylogue, index schema v23, 41,886 structured failures total, 5,000 origin-stratified failures inspected (3,746 claude-code-session, 1,247 codex-session, 7 claude-ai-export), 100 unpaired structured failures. Marker vocabulary was tightened to avoid broad issue/fix/block/gitignored false positives. Immediate next-turn totals: acknowledged=420, silent_proceed=1,205, ambiguous=3,375 (2,624 wordless tool continuations; 751 prose without marker). Lower-bound silent rate is 24.1%; among classified immediate next turns, silent rate is 74.2%. Next-3 sensitivity window, stopping before the next user message, finds 302 acknowledgments that appear only after the next turn; window3 silent lower bound is 37.0%. Calibration: 50 hand-labeled immediate-next-turn rows, acknowledged-marker precision=1.0, recall=0.8421052631578947, invalid rows=0. Artifact: .agent/demos/claim-vs-evidence/claim-vs-evidence.report.json.","notes":"2026-07-03 update: methodology package is now cold-read gated. .agent/demos/claim-vs-evidence contains aggregate live evidence, public-summary.json, PUBLIC_REPRODUCTION.md, COLD_READER_GATE.md, and COLD_READ_RESULT.md. Seeded reproduction is meaningful, not empty: 4 structured failures, 2 acknowledged follow-ups, 2 silent-proceed follow-ups, 0 unpaired. Cold-reader subagent PASS recovered claim/non-claim, sample frame, rates, calibration, caveats, and reproduction commands from the artifact directory only. Remaining campaign child: polylogue-sru.1 productizes action-unit outcome/followup_class capability.","status":"closed","priority":0,"issue_type":"epic","owner":"ezo.dev@gmail.com","created_at":"2026-07-03T04:31:26Z","created_by":"Sinity","updated_at":"2026-07-03T09:28:09Z","closed_at":"2026-07-03T09:28:09Z","close_reason":"Completed: all seven campaign children are closed. The claim-vs-evidence finding now has bounded sample-frame reporting, calibrated marker precision/recall, handler-class and next-3 sensitivity splits, meaningful seeded reproduction, cold-reader PASS, and productized action-unit followup_class/followup_message_ref query capability. Current artifact lives under .agent/demos/claim-vs-evidence and was regenerated against /home/sinity/.local/share/polylogue schema v23.","labels":["area:substrate","campaign"],"dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"polylogue-eqnv","title":"Stale pre-fix parser identity lets a same-source_path raw pair silently split into two byte-proven singletons, downgrading fidelity","description":"## What the live archive shows\n\nFor the 5 aistudio-drive sessions Implementing-{066bb070,13ced1c8,37edfeb3,845dd573,d4d7fbab}, the index materialized the SMALLER (attachment-unfetched, \"bare\") raw and never even considered the LARGER (attachment-fetched, \"enriched\") raw. Neither raw has a `raw_session_memberships` row -- they never entered the ambiguous-membership machinery bu1i/9dxn describe. Instead both raws sit in raw_sessions with `revision_kind='full'`, `revision_authority='byte_proven'`, `baseline_raw_id=self` -- i.e. each was independently accepted as an unconditional SINGLETON byte-revision baseline under a DIFFERENT `logical_source_key`:\n\n 0064ddd16c39... (enriched, 967377B) -\u003e logical_source_key = 'gemini:Implementing-066bb070...'\n 13ae07d010bb... (bare, 252347B) -\u003e logical_source_key = 'gemini:Implementing-066bb070...-0'\n\n## Root cause, proven\n\n`raw_authority_parser_census` (source.db) records the census-time parser\nIDENTITY output for both raws:\n\n 0064ddd16c39...: fingerprint=revision-membership-v1, key=[\"gemini:Implementing-066bb070...23810576f616f90fb4254c69\"]\n 13ae07d010bb...: fingerprint=revision-membership-v1, key=[\"gemini:Implementing-066bb070...23810576f616f90fb4254c69-0\"]\n\nBoth under the SAME fingerprint string, yet different identity. Reparsing\nBOTH raw blobs from the live blob store through the CURRENT\n`polylogue/sources/dispatch.py`/`revision_backfill._parse_one` gives the\nIDENTICAL, correct, unsuffixed `provider_session_id` for both (verified with\nproduction code against the real blobs). The \"-0\" suffix is the exact\npre-#3179/z1c6 bug (`_lower_drive_like_payload`'s `_looks_like_chunked_session_list`\nbranch always appended `-{index}` regardless of list length, fixed\n2026-07-20 in b473d9256/#3179). raw_small was acquired+validated 2026-07-16,\nraw_big 2026-07-18 -- both before the fix landed 2026-07-20 -- and their\ncensus (which sets `raw_sessions.logical_source_key`) evidently ran under\nthe pre-fix parser and was never invalidated, because\n`raw_authority_parser_census`'s quiescence gate\n(`uncensused_historical_revision_raw_ids`) treats any row with the SAME\nliteral fingerprint string as \"current parser already observed this\" --\nthere is no version distinction between pre-fix and post-fix identity\noutput. `classify_raw_revision_cohort` (archive.py) then classifies each\nraw against its OWN `logical_source_key` in isolation, has no way to know\nthe two keys describe the same physical document, and unconditionally\naccepts each as a trivial one-member byte-proven chain -- the same\nstructural hole polylogue-52l2/hm2f already document for the RETIRED-SIBLING\ncase, but here the divergence is at the KEY itself, not at retirement\nstate, so the existing `raw_membership_retired_full_revision_siblings` guard\n(keyed on exact logical_source_key match) never fires.\n\n## Relationship to polylogue-9dxn\n\n9dxn's proposed fingerprint-versioning fix (permissive quiescence for any\nKNOWN fingerprint, strict-current-only for the ambiguous TERMINAL gate)\ndoes not by itself heal this case: it is designed to let previously-`ambiguous`\nverdicts be revisited without forcing a blanket re-census, but raw_small's\nstale census here was NOT ambiguous -- it was `status='complete'` with a\nWRONG identity, and 9dxn's design keeps quiescence permissive for any known\nfingerprint, so this raw would stay \"already observed\" forever even after a\nfingerprint bump. This bead's fix is a structural cross-source_path guard in\n`classify_raw_revision_cohort`, independent of fingerprint versioning, that\nalso closes the general case regardless of how two same-document raws ended\nup under different keys (stale census, race, or a future bug of the same\nshape).\n\n## Fix landed in polylogue-af059 (this branch)\n\n- `archive.py`: `classify_raw_revision_cohort` refuses unconditional\n singleton acceptance when another 'full' raw shares the same source_path\n under a different (or already-retired) logical_source_key -- forces both\n into membership governance instead of letting either become an\n unconditionally-accepted baseline.\n- `revision_backfill.py`: the retire-to-membership-governance fallback now\n buckets `membership_candidates`/`membership_keys` by the FRESHLY re-parsed\n identity (`session.provider_session_id`) instead of the stale outer-loop\n `logical_source_key`, so two same-document raws retired under different\n stale keys land in ONE membership cohort and get jointly arbitrated\n instead of each being accepted as an independent membership singleton.\n\n## Residual / follow-up\n\n- The live archive's 5 already-downgraded sessions are NOT repaired by this\n code fix (need a live remediation pass, out of scope for this PR).\n- A full census-fingerprint bump (9dxn) is still needed to catch every OTHER\n raw whose identity was assigned by pre-#3179 dispatch.py, if any exist\n beyond aistudio-drive.\n\nRef polylogue-bu1i, polylogue-7ilr, polylogue-9dxn","status":"open","priority":1,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T12:22:33Z","created_by":"Sinity","updated_at":"2026-07-30T12:22:33Z","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"polylogue-nuec","title":"chatgpt-export: provider-reported generation-duration metadata volatile, contaminates session-event identity hash","description":"## What the data says\n\nSampled 35 of 129 chatgpt-export \"ambiguous\" equal-message-count membership\ncohorts (27%; read-only against /realm/db/polylogue). Reproduced with\nproduction code identically to polylogue-c429/polylogue-c42a: parsed both\ndistinct-content raw revisions per cohort via\n`polylogue.sources.dispatch.parse_payload`, projected each with\n`polylogue.pipeline.ids.session_revision_projection`.\n\n33 of 35 sampled cohorts (94%) have this exact shape:\n\n message_hashes equal (messages byte-identical, same order)\n attachment_hashes equal\n event_hashes DIFFER, with event COUNT equal on both sides\n\nFor every sampled case, the first differing `session_events` entry is\n`event_type == \"generation_lifecycle\"` with an identical payload key set\n(`duration_semantics`, `elapsed_duration_ms`, `evidence_source`,\n`fidelity`, `state`) but a DIFFERENT `elapsed_duration_ms` value (e.g.\n13000 vs 21000; 52000 vs 107000; 123000 vs 33000 -- no consistent\ndirection, ruling out simple clock skew). In several cases the event's\n`source_message_provider_id` also differs at the same array index, evidence\nthat the generation-lifecycle event LIST itself may reorder alongside the\nduration values, though message order (which correlates with these events)\nwas independently confirmed stable.\n\n## Root cause\n\n`polylogue/sources/parsers/chatgpt.py` (`_resolve_generation_timings` /\n`~line 1069`, `duration_semantics=\"provider_reported_elapsed\"`) derives a\nsynthetic `generation_lifecycle` session event per assistant/tool message,\nwith `elapsed_duration_ms` computed from the RAW EXPORT's own\n`finished_duration_sec` or `reasoning_start_time`/`reasoning_end_time`\nmetadata fields on that message's mapping node (not something Polylogue\ninvents -- traced to `raw_metadata.get(\"finished_duration_sec\")` and the\n`reasoning_start_time`/`reasoning_end_time` delta). This value is folded into\n`session_events` and hashed via `session_revision_projection`'s\n`event_hashes` (`polylogue/pipeline/ids.py`), which\n`_strictly_dominates`/`classify_membership_revisions`\n(`polylogue/archive/session_revision_membership.py`) treats as part of\ncontent identity.\n\nThe underlying provider-reported duration values are not stable across\nseparate ChatGPT export requests for the SAME generation -- 33 of 35 sampled\ncohorts have message and attachment content that is byte-identical across\ntwo export vintages, yet the reported generation timing differs, sometimes\nsubstantially (e.g. 2s vs 27s; 794s vs 445s), with no consistent\nincrease/decrease pattern that would suggest a benign refinement. This reads\nas either non-deterministic export-time re-derivation on OpenAI's side, or a\nmetric that legitimately varies by measurement context and was never meant\nto be a durable per-generation identity value. Either way, folding it into\nsession identity hash makes byte-identical conversations look like divergent\nbranches on every re-export.\n\n## Reproduction recipe (production code, no archive mutation)\n\nSame harness pattern as polylogue-c429, with `origin='chatgpt-export'`;\nafter loading both `ParsedSession`s for a cohort:\n\n```python\nfrom polylogue.pipeline.ids import session_revision_projection\npa, pb = session_revision_projection(a), session_revision_projection(b)\nassert pa.message_hashes == pb.message_hashes\nassert pa.attachment_hashes == pb.attachment_hashes\nassert pa.event_hashes != pb.event_hashes\nassert len(a.session_events) == len(b.session_events)\n# first differing pair:\nfor ea, eb in zip(a.session_events, b.session_events):\n if ea.payload != eb.payload:\n assert ea.event_type == eb.event_type == \"generation_lifecycle\"\n assert ea.payload[\"elapsed_duration_ms\"] != eb.payload[\"elapsed_duration_ms\"]\n break\n```\n\n## Extrapolation honesty\n\n35 of 129 sampled (27%, the largest sample fraction of any origin in this\ncensus). 33/35 = 94% match this exact shape (message+attachment hashes\nequal, event hashes differ, dominant delta traced to\n`generation_lifecycle.elapsed_duration_ms`). 1/35 has both message and\nevent differences (a separate, unexamined cause). 1/35 is now identical\nunder the current classifier (message/event/attachment hashes all equal) --\nits recorded 'ambiguous' decision appears stale relative to current\nevidence; see polylogue-9dxn for the general \"persisted ambiguous verdicts\nnever get re-derived\" defect that would explain this. Extrapolating 94% to\nthe full 129-cohort population suggests roughly 120 of 129 cohorts, but this\nis an estimate from a 27% sample, not a full census.\n\n## Proposed fix direction (for the classifier/parser-owning lane, not this bead)\n\nThis is the clearest case in the whole census for excluding a field from\nidentity rather than relaxing dominance comparison: `elapsed_duration_ms` is\nexplicitly labeled a measurement (`duration_semantics:\n\"provider_reported_elapsed\"`), not a content field, and doesn't belong in a\ncontent-identity hash at all. Either exclude `generation_lifecycle` event\npayloads (or just the `elapsed_duration_ms` field within them) from\n`_session_hash_components`'s `session_events_payload` in\n`polylogue/pipeline/ids.py`, or store/compare `session_events` with a\ntolerant equality that ignores this specific volatile field. Narrower and\nlower-risk than the message-order or attachment-identity fixes in\npolylogue-c429/polylogue-c42a because it doesn't touch dominance logic at\nall -- it just stops hashing a value the parser itself already documents as\nnon-durable measurement evidence.\n\nRef polylogue-bu1i\n","status":"open","priority":1,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T12:16:49Z","created_by":"Sinity","updated_at":"2026-07-30T12:17:41Z","labels":["area:ingest"],"dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"polylogue-hith","title":"claude-ai-export: synthetic attachment id keyed on positional index is unstable across export vintages","description":"## What the data says\n\nSame sample as polylogue-c429 (40 of 566 claude-ai-export ambiguous\nequal-message-count cohorts, read-only against /realm/db/polylogue,\nreproduced with production `parse_payload` + `session_revision_projection` +\n`classify_membership_revisions`). Of the 40, 16 (40%) have this exact shape,\ndisjoint from the message-order cause in polylogue-c429:\n\n message id set equal, message array order equal, 0 content diffs\n len(attachments_a) == len(attachments_b)\n set of (provider_attachment_id, message_provider_id) DISJOINT or partially disjoint\n between the two revisions, for attachments anchored to the SAME message\n\nExample (cohort with 4 attachments, 2 anchor messages, `key` starting\n`claude-a...`):\n\n A: att_id=e950263f-063d-495d-b0c0-61e9330d3a14 msg=7f1cf6ff-...\n B: att_id=att-ce21cd12d650 msg=7f1cf6ff-... (same message anchor)\n\n A: att_id=66a7a163-d488-4320-8129-19ad43f64a43 msg=c38e86ac-...\n B: att_id=att-cd01a39eb65c msg=0d3a13b8-... (different message anchor too)\n\nmime_type/size_bytes/inline-presence/name-length are identical between the\npaired attachments in every sampled case -- this is not\npolylogue-bu1i's acquisition-state pattern (`inline_bytes`/`size_bytes`\nflipping None-\u003ereal). The IDENTITY STRING itself differs, and sometimes so\ndoes the message it's anchored to.\n\n## Root cause\n\n`polylogue/sources/parsers/base_support.py:152-197`\n(`attachment_from_meta`/`_make_attachment_id`), used by the Claude.ai parser\nvia `attachment_from_meta` (`polylogue/sources/parsers/claude/ai_parser.py`,\n`_merge_session_attachments` at line ~191, iterating `(\"attachments\",\n\"files\")`):\n\n```python\ndef _make_attachment_id(seed: str) -\u003e str:\n return f\"att-{hash_text(seed)[:12]}\"\n\ndef attachment_from_meta(meta, message_id, index):\n attachment_id = (\n meta.get(\"id\") or meta.get(\"file_id\") or meta.get(\"fileId\")\n or meta.get(\"uuid\") or meta.get(\"file_uuid\")\n )\n ...\n if not attachment_id:\n if not name:\n return None\n seed = f\"{message_id or 'msg'}:{name}:{index}\"\n attachment_id = _make_attachment_id(seed)\n```\n\nTwo independent failure modes both traced in the sample:\n\n1. **Real-id presence is inconsistent across export vintages.** When\n Claude.ai's own export payload carries a real `id`/`file_id`/`uuid` for an\n attachment, that string is used directly (stable). When it's absent, the\n parser falls back to a SYNTHETIC id hashed from\n `f\"{message_id}:{name}:{index}\"`. The two export vintages of the same\n conversation don't consistently include the real id -- one carries it,\n the other doesn't -- so the same physical attachment gets a real UUID in\n one revision and a synthetic `att-...` id in the other.\n2. **`index` is positional, and attachment order is not guaranteed stable.**\n Even when BOTH revisions fall back to synthesis, `index` is the\n attachment's position in the merged `attachments`+`files` iteration for\n that message. If that per-message ordering shifts between export\n vintages (plausible given polylogue-c429's proof that the surrounding\n MESSAGE array order is itself unstable across Claude.ai exports), the\n synthesized id changes even though the underlying attachment didn't.\n\nEither way, attachment identity is accidentally keyed on transient\nexport-shape details (real-id presence, list order) rather than a property\nof the attachment itself, so `_attachment_hash_payload`\n(`polylogue/pipeline/ids.py:152`) hashes the same physical attachment to two\ndifferent identities across export vintages -- the same general shape as\npolylogue-bu1i (acquisition/export-time noise contaminating an identity\nhash), but a DIFFERENT concrete defect (id synthesis, not acquisition-state\nflip) requiring a different fix.\n\n## Reproduction recipe (production code, no archive mutation)\n\nSame harness as polylogue-c429's reproduction recipe; after loading both\n`ParsedSession`s for a cohort where message ids/order/content are identical:\n\n```python\natts_a = {(at.provider_attachment_id, at.message_provider_id): at for at in a.attachments}\natts_b = {(at.provider_attachment_id, at.message_provider_id): at for at in b.attachments}\nassert len(atts_a) == len(atts_b)\nassert set(atts_a) != set(atts_b) # disjoint identity despite same count\n```\n\n## Extrapolation honesty\n\n40 of 566 sampled (7%). 16/40 = 40% match this exact shape (message\ncontent/order fully identical, attachment key sets disjoint at equal\ncount). Extrapolating to the full population suggests roughly 220-230 of the\n566 cohorts, but this is an estimate from a 7% sample, not a census.\n\n## Proposed fix direction (for the classifier/parser-owning lane, not this bead)\n\nTwo independent levers, either alone reduces the blast radius:\n\n- Parser-side: derive the synthetic attachment id from content-stable\n material only (e.g. a hash of `(message_provider_id, name, mime_type,\n size_bytes)` without positional `index`), so re-ordering the export's\n attachment list doesn't change identity. Does not fix mode (1)\n (real-id-present-in-one-export-only).\n- Classifier-side (in the files this investigation lane does not edit):\n compare attachments by a looser key (e.g. `(message_provider_id, name,\n mime_type, size_bytes)`) when testing dominance, falling back to id\n equality only when that tuple is ambiguous -- the same class of relaxation\n polylogue-bu1i proposes for acquisition-state, generalized.\n\nRef polylogue-bu1i\nRef polylogue-c429\n","status":"open","priority":1,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T12:16:40Z","created_by":"Sinity","updated_at":"2026-07-30T12:17:40Z","labels":["area:ingest"],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-f1vg","title":"Corpus acceptance gate: no absences and maximum fidelity, with the 2026-07-30 frozen baseline","description":"## What the operator asked for\n\n\"Ensure max fidelity as well as no absences, through the entire corpus.\" That is\na stronger bar than any existing check enforces, and nothing measured either\nhalf until now.\n\n## Why the existing checks cannot serve\n\n`verify-archive`'s `source-index-coverage` counts superseded revisions as\nmissing work (polylogue-ey3r), so it cannot reach zero on any archive that ever\ningested a conversation twice, and therefore cannot gate a rebuild. Nothing at\nall measures fidelity: an archive can report perfect coverage while every\nattachment whose bytes it holds is recorded `unfetched`, which is precisely the\nstate measured on 2026-07-30.\n\n## The gate\n\n`.agent/scripts/corpus-fidelity-audit.py` (read-only, `mode=ro` throughout,\nexits 1 on failure so it can gate a rebuild). Three measures:\n\n1. **Absences** -- logical documents (origin + provider_session_id) the archive\n holds evidence for but does not surface, bucketed by cause so a fix's effect\n is attributable rather than a single number moving for unknown reasons.\n2. **Attachment fidelity** -- acquired vs not-acquired refs, split by origin and\n upload_origin, because a Drive-hosted reference never fetched is actionable\n while a genuinely byte-less attachment kind is not.\n3. **Revision fidelity** -- documents whose indexed evidence is smaller than the\n largest revision recorded for them.\n\n## Baseline, live archive frozen 2026-07-30 (daemon stopped)\n\n ABSENCES 1,009 of 18,248 known documents\n 587 claude-ai-export/ambiguous-only\n 184 claude-code-session/ambiguous-only\n 135 chatgpt-export/ambiguous-only\n 71 aistudio-drive/settled-yet-absent\n 20 unknown-export/settled-yet-absent\n 12 gemini-cli / hermes / unknown / grok / codex\n\n ATTACHMENT FIDELITY acquired=2,118 not-acquired=7,655\n 3,684 chatgpt-export/oauth/unfetched\n 2,391 chatgpt-export/\u003cnone\u003e/unfetched\n 1,975 aistudio-drive/drive/acquired\n 1,119 aistudio-drive/drive/unfetched\n 398 claude-ai-export/oauth/unfetched\n\n REVISION FIDELITY 94 documents below best recorded evidence\n 76 hermes-session\n 16 claude-code-session\n 2 chatgpt-export\n\n VERDICT: FAIL\n\nThe `settled-yet-absent` buckets (71 drive, 20 unknown-export, 1 codex) are not\nexplained by any currently-tracked cause and want their own investigation --\nthese are documents with no ambiguous decision anywhere that are nonetheless\nmissing.\n\nThe 94 revision-fidelity documents are a residue after correcting a false\npositive, and should be treated as a prompt to investigate rather than proof of\nloss (see below).\n\n## Measurement trap this already caught\n\nThe first version compared indexed *messages* against\n`raw_session_memberships.message_count` and reported **474** shortfalls, 294 of\nthem codex-session. All false. `message_count` was recorded by whichever parser\ncensused that raw, and index v46 deliberately reclassified a large share of\nCodex/Claude Code rows from chat turns into typed `session_events`. One codex\nsession read as \"15 indexed vs 68,553 recorded\" when it actually holds 15\nmessages plus 84,612 events. Counting `messages + session_events` drops the\nfigure to 94.\n\nAnyone extending this must keep that in mind: cross-generation counts are only\napproximately comparable, so a metric built on them needs its assumption stated\nand checked against a real sample before its number is believed.\n\n## Follow-up\n\nPromote this into `devtools` as a first-class command with a `CommandSpec` (plus\n`devtools render devtools-reference`) so it is an enforced gate rather than a\nscript, and wire it into the post-rebuild acceptance path alongside\n`verify-archive`. Kept as a script for now because the fixes it measures are\nstill in flight and its thresholds will move as they land.\n\n## Acceptance criteria\n\n- Absences reach 0, or every residual is individually justified in writing.\n- Attachment refs marked not-acquired are either acquired or shown to be\n genuinely unfetchable (deleted upstream, over the size cap, byte-less kind).\n- Revision-fidelity residue is explained rather than merely small.\n","status":"open","priority":1,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T14:04:01Z","created_by":"Sinity","updated_at":"2026-07-30T14:04:01Z","labels":["area:ingest"],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-d8al","title":"claude-ai-export: attachment real-id presence is inconsistent across export vintages, needs comparison-layer relaxation","description":"## What the data says\n\nCensus (full population, not a sample): replayed the production classifier\n(polylogue.sources.dispatch.parse_payload -\u003e session_revision_projection -\u003e\nclassify_membership_revisions) over all 566 claude-ai-export\nequal-message-count ambiguous cohorts in the live archive (read-only,\n/realm/db/polylogue), with polylogue-hith's parser-side fix (drop the\npositional-index seed for synthetic attachment ids) already applied.\n\n 566 claude-ai-export equal-message-count ambiguous cohorts (full census)\n 297 still ambiguous because message_hashes differ (polylogue-c429 /\n message-order-not-stable territory, or genuine content divergence)\n 268 still ambiguous with message_hashes EQUAL (0 content diffs) but\n attachment identity axis mismatched -- the exact shape hith\n targeted\n 0 of those 268 resolved by hith's fix\n 268 of those 268 are \"mixed real/synthetic\": one export vintage of the\n SAME conversation carries a real id (id/file_id/fileId/uuid/\n file_uuid) for an attachment; the OTHER vintage of the same\n conversation has no real id for the physically-same attachment and\n synthesizes one instead\n 0 are \"pure synthetic on both sides\" (the positional-index shape\n hith's fix targets and fully resolves when it occurs)\n\nIn other words: in the population currently persisted as ambiguous, 100% of\nthe identity-mismatch cases are this real-id-presence axis, not the\npositional-index axis. hith's fix is verified correct and regression-safe\n(250-cohort replay of already-resolved cohorts: 249/250 agree old vs new\nlogic, 1 improvement, 0 regressions) but resolves 0 of the currently-measured\n566-cohort population by itself, because no synthetic-minting scheme can ever\nmake a real UUID and a hash of (message, name, mime_type) collide.\n\n## Root cause\n\n`polylogue/sources/parsers/base_support.py:attachment_from_meta` uses the\nexport's own `id`/`file_id`/`fileId`/`uuid`/`file_uuid` field when present,\nand only falls back to synthesis when absent. Claude.ai does not consistently\nemit this field for the same attachment across export vintages of the same\nconversation -- verified directly against blob content for 6 sampled\ncohorts, all showing exactly this shape (one blob's attachment has a real\nUUID-shaped id, the other blob's attachment for the same message has no id\nfield and synthesizes `att-\u003chash\u003e`).\n\nNo id-minting scheme at the parser layer can reconcile this: a real id and a\nsynthetic hash will never be equal strings by construction, regardless of\nwhat the synthetic hash is seeded from.\n\n## Proposed fix (comparison layer, NOT parser layer)\n\nIn `polylogue/archive/session_revision_membership.py` (and/or\n`polylogue/pipeline/ids.py`'s `SessionRevisionProjection` /\n`_attachment_identity_payload`), the dominance/equivalence test should\ncompare attachments by a looser key when testing dominance -- e.g.\n`(message_provider_id, name, mime_type)` without the `id` field -- falling\nback to strict id equality only when that looser key is itself ambiguous\n(more than one attachment sharing the tuple on one side). This is the same\nclass of relaxation polylogue-bu1i introduced for acquisition state\n(`attachment_identities` vs `attachment_contents`), generalized to a third\naxis: \"same attachment referenced with and without a stable provider id\".\n\nThis bead deliberately does NOT propose an implementation in those files --\npolylogue-hith's owning lane was scoped away from\n`session_revision_membership.py`/`ids.py` because another lane owns them\nconcurrently. Whoever picks this up should re-run the census harness\ndescribed in polylogue-hith (or the updated one referenced in its closing\nnote) against the classifier change to prove the 268-cohort population above\nactually resolves, the same way polylogue-bu1i's PR proved 157/157.\n\n## Verification recipe\n\nSame read-only harness as polylogue-hith / polylogue-bu1i: parse both blobs\nof a cohort with production `parse_payload`, project with\n`session_revision_projection`, and diff the resulting\n`attachment_identities` sets. For the 268-cohort population, at least one\nattachment identity differs solely because one side has a real id string and\nthe other has a synthetic hash string for what is, by every other field\n(message anchor, name, mime_type), the same attachment.\n\nRef polylogue-hith\nRef polylogue-bu1i","notes":"Superseded by polylogue-aggz's architecture: attachment identity now unconditionally drops the provider id (content-derived: message_id+name+mime_type only), eliminating the strict/loose duality and its pairwise correlation machinery entirely rather than adding a fallback. See PR.","status":"open","priority":1,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T13:02:14Z","created_by":"Sinity","updated_at":"2026-07-30T15:15:31Z","labels":["area:ingest"],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-eqnv","title":"Stale pre-fix parser identity lets a same-source_path raw pair silently split into two byte-proven singletons, downgrading fidelity","description":"## What the live archive shows\n\nFor the 5 aistudio-drive sessions Implementing-{066bb070,13ced1c8,37edfeb3,845dd573,d4d7fbab}, the index materialized the SMALLER (attachment-unfetched, \"bare\") raw and never even considered the LARGER (attachment-fetched, \"enriched\") raw. Neither raw has a `raw_session_memberships` row -- they never entered the ambiguous-membership machinery bu1i/9dxn describe. Instead both raws sit in raw_sessions with `revision_kind='full'`, `revision_authority='byte_proven'`, `baseline_raw_id=self` -- i.e. each was independently accepted as an unconditional SINGLETON byte-revision baseline under a DIFFERENT `logical_source_key`:\n\n 0064ddd16c39... (enriched, 967377B) -\u003e logical_source_key = 'gemini:Implementing-066bb070...'\n 13ae07d010bb... (bare, 252347B) -\u003e logical_source_key = 'gemini:Implementing-066bb070...-0'\n\n## Root cause, proven\n\n`raw_authority_parser_census` (source.db) records the census-time parser\nIDENTITY output for both raws:\n\n 0064ddd16c39...: fingerprint=revision-membership-v1, key=[\"gemini:Implementing-066bb070...23810576f616f90fb4254c69\"]\n 13ae07d010bb...: fingerprint=revision-membership-v1, key=[\"gemini:Implementing-066bb070...23810576f616f90fb4254c69-0\"]\n\nBoth under the SAME fingerprint string, yet different identity. Reparsing\nBOTH raw blobs from the live blob store through the CURRENT\n`polylogue/sources/dispatch.py`/`revision_backfill._parse_one` gives the\nIDENTICAL, correct, unsuffixed `provider_session_id` for both (verified with\nproduction code against the real blobs). The \"-0\" suffix is the exact\npre-#3179/z1c6 bug (`_lower_drive_like_payload`'s `_looks_like_chunked_session_list`\nbranch always appended `-{index}` regardless of list length, fixed\n2026-07-20 in b473d9256/#3179). raw_small was acquired+validated 2026-07-16,\nraw_big 2026-07-18 -- both before the fix landed 2026-07-20 -- and their\ncensus (which sets `raw_sessions.logical_source_key`) evidently ran under\nthe pre-fix parser and was never invalidated, because\n`raw_authority_parser_census`'s quiescence gate\n(`uncensused_historical_revision_raw_ids`) treats any row with the SAME\nliteral fingerprint string as \"current parser already observed this\" --\nthere is no version distinction between pre-fix and post-fix identity\noutput. `classify_raw_revision_cohort` (archive.py) then classifies each\nraw against its OWN `logical_source_key` in isolation, has no way to know\nthe two keys describe the same physical document, and unconditionally\naccepts each as a trivial one-member byte-proven chain -- the same\nstructural hole polylogue-52l2/hm2f already document for the RETIRED-SIBLING\ncase, but here the divergence is at the KEY itself, not at retirement\nstate, so the existing `raw_membership_retired_full_revision_siblings` guard\n(keyed on exact logical_source_key match) never fires.\n\n## Relationship to polylogue-9dxn\n\n9dxn's proposed fingerprint-versioning fix (permissive quiescence for any\nKNOWN fingerprint, strict-current-only for the ambiguous TERMINAL gate)\ndoes not by itself heal this case: it is designed to let previously-`ambiguous`\nverdicts be revisited without forcing a blanket re-census, but raw_small's\nstale census here was NOT ambiguous -- it was `status='complete'` with a\nWRONG identity, and 9dxn's design keeps quiescence permissive for any known\nfingerprint, so this raw would stay \"already observed\" forever even after a\nfingerprint bump. This bead's fix is a structural cross-source_path guard in\n`classify_raw_revision_cohort`, independent of fingerprint versioning, that\nalso closes the general case regardless of how two same-document raws ended\nup under different keys (stale census, race, or a future bug of the same\nshape).\n\n## Fix landed in polylogue-af059 (this branch)\n\n- `archive.py`: `classify_raw_revision_cohort` refuses unconditional\n singleton acceptance when another 'full' raw shares the same source_path\n under a different (or already-retired) logical_source_key -- forces both\n into membership governance instead of letting either become an\n unconditionally-accepted baseline.\n- `revision_backfill.py`: the retire-to-membership-governance fallback now\n buckets `membership_candidates`/`membership_keys` by the FRESHLY re-parsed\n identity (`session.provider_session_id`) instead of the stale outer-loop\n `logical_source_key`, so two same-document raws retired under different\n stale keys land in ONE membership cohort and get jointly arbitrated\n instead of each being accepted as an independent membership singleton.\n\n## Residual / follow-up\n\n- The live archive's 5 already-downgraded sessions are NOT repaired by this\n code fix (need a live remediation pass, out of scope for this PR).\n- A full census-fingerprint bump (9dxn) is still needed to catch every OTHER\n raw whose identity was assigned by pre-#3179 dispatch.py, if any exist\n beyond aistudio-drive.\n\nRef polylogue-bu1i, polylogue-7ilr, polylogue-9dxn","status":"closed","priority":1,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T12:22:33Z","created_by":"Sinity","updated_at":"2026-07-30T12:45:58Z","started_at":"2026-07-30T12:45:56Z","closed_at":"2026-07-30T12:45:58Z","close_reason":"Fixed in PR #3396 (feature/fix/ambiguous-raw-materialization-leak): ArchiveStore.classify_raw_revision_cohort gains an opt-in check_source_path_identity_split guard (used only by the offline backfill/rebuild replay loop, not the live watcher), plus revision_backfill.py's retire-to-membership-governance fallback now buckets by the freshly re-derived identity instead of the stale outer-loop key. Verified with two new regression tests (anti-vacuity confirmed both ways via direct revert+rerun). The 5 already-downgraded live sessions are NOT repaired by this fix; live remediation is a separate, explicitly out-of-scope lane.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-nuec","title":"chatgpt-export: provider-reported generation-duration metadata volatile, contaminates session-event identity hash","description":"## What the data says\n\nSampled 35 of 129 chatgpt-export \"ambiguous\" equal-message-count membership\ncohorts (27%; read-only against /realm/db/polylogue). Reproduced with\nproduction code identically to polylogue-c429/polylogue-c42a: parsed both\ndistinct-content raw revisions per cohort via\n`polylogue.sources.dispatch.parse_payload`, projected each with\n`polylogue.pipeline.ids.session_revision_projection`.\n\n33 of 35 sampled cohorts (94%) have this exact shape:\n\n message_hashes equal (messages byte-identical, same order)\n attachment_hashes equal\n event_hashes DIFFER, with event COUNT equal on both sides\n\nFor every sampled case, the first differing `session_events` entry is\n`event_type == \"generation_lifecycle\"` with an identical payload key set\n(`duration_semantics`, `elapsed_duration_ms`, `evidence_source`,\n`fidelity`, `state`) but a DIFFERENT `elapsed_duration_ms` value (e.g.\n13000 vs 21000; 52000 vs 107000; 123000 vs 33000 -- no consistent\ndirection, ruling out simple clock skew). In several cases the event's\n`source_message_provider_id` also differs at the same array index, evidence\nthat the generation-lifecycle event LIST itself may reorder alongside the\nduration values, though message order (which correlates with these events)\nwas independently confirmed stable.\n\n## Root cause\n\n`polylogue/sources/parsers/chatgpt.py` (`_resolve_generation_timings` /\n`~line 1069`, `duration_semantics=\"provider_reported_elapsed\"`) derives a\nsynthetic `generation_lifecycle` session event per assistant/tool message,\nwith `elapsed_duration_ms` computed from the RAW EXPORT's own\n`finished_duration_sec` or `reasoning_start_time`/`reasoning_end_time`\nmetadata fields on that message's mapping node (not something Polylogue\ninvents -- traced to `raw_metadata.get(\"finished_duration_sec\")` and the\n`reasoning_start_time`/`reasoning_end_time` delta). This value is folded into\n`session_events` and hashed via `session_revision_projection`'s\n`event_hashes` (`polylogue/pipeline/ids.py`), which\n`_strictly_dominates`/`classify_membership_revisions`\n(`polylogue/archive/session_revision_membership.py`) treats as part of\ncontent identity.\n\nThe underlying provider-reported duration values are not stable across\nseparate ChatGPT export requests for the SAME generation -- 33 of 35 sampled\ncohorts have message and attachment content that is byte-identical across\ntwo export vintages, yet the reported generation timing differs, sometimes\nsubstantially (e.g. 2s vs 27s; 794s vs 445s), with no consistent\nincrease/decrease pattern that would suggest a benign refinement. This reads\nas either non-deterministic export-time re-derivation on OpenAI's side, or a\nmetric that legitimately varies by measurement context and was never meant\nto be a durable per-generation identity value. Either way, folding it into\nsession identity hash makes byte-identical conversations look like divergent\nbranches on every re-export.\n\n## Reproduction recipe (production code, no archive mutation)\n\nSame harness pattern as polylogue-c429, with `origin='chatgpt-export'`;\nafter loading both `ParsedSession`s for a cohort:\n\n```python\nfrom polylogue.pipeline.ids import session_revision_projection\npa, pb = session_revision_projection(a), session_revision_projection(b)\nassert pa.message_hashes == pb.message_hashes\nassert pa.attachment_hashes == pb.attachment_hashes\nassert pa.event_hashes != pb.event_hashes\nassert len(a.session_events) == len(b.session_events)\n# first differing pair:\nfor ea, eb in zip(a.session_events, b.session_events):\n if ea.payload != eb.payload:\n assert ea.event_type == eb.event_type == \"generation_lifecycle\"\n assert ea.payload[\"elapsed_duration_ms\"] != eb.payload[\"elapsed_duration_ms\"]\n break\n```\n\n## Extrapolation honesty\n\n35 of 129 sampled (27%, the largest sample fraction of any origin in this\ncensus). 33/35 = 94% match this exact shape (message+attachment hashes\nequal, event hashes differ, dominant delta traced to\n`generation_lifecycle.elapsed_duration_ms`). 1/35 has both message and\nevent differences (a separate, unexamined cause). 1/35 is now identical\nunder the current classifier (message/event/attachment hashes all equal) --\nits recorded 'ambiguous' decision appears stale relative to current\nevidence; see polylogue-9dxn for the general \"persisted ambiguous verdicts\nnever get re-derived\" defect that would explain this. Extrapolating 94% to\nthe full 129-cohort population suggests roughly 120 of 129 cohorts, but this\nis an estimate from a 27% sample, not a full census.\n\n## Proposed fix direction (for the classifier/parser-owning lane, not this bead)\n\nThis is the clearest case in the whole census for excluding a field from\nidentity rather than relaxing dominance comparison: `elapsed_duration_ms` is\nexplicitly labeled a measurement (`duration_semantics:\n\"provider_reported_elapsed\"`), not a content field, and doesn't belong in a\ncontent-identity hash at all. Either exclude `generation_lifecycle` event\npayloads (or just the `elapsed_duration_ms` field within them) from\n`_session_hash_components`'s `session_events_payload` in\n`polylogue/pipeline/ids.py`, or store/compare `session_events` with a\ntolerant equality that ignores this specific volatile field. Narrower and\nlower-risk than the message-order or attachment-identity fixes in\npolylogue-c429/polylogue-c42a because it doesn't touch dominance logic at\nall -- it just stops hashing a value the parser itself already documents as\nnon-durable measurement evidence.\n\nRef polylogue-bu1i\n","notes":"Superseded by polylogue-aggz's architecture: chatgpt-export generation_lifecycle duration volatility is now handled via an explicit content-only ALLOWLIST (_EVENT_CONTENT_PAYLOAD_ALLOWLIST) rather than a denylist strip of known-volatile fields. Live census: 119/135 (88.1%) chatgpt-export ambiguous cohorts now resolve, 0 regressions. See PR.","status":"open","priority":1,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T12:16:49Z","created_by":"Sinity","updated_at":"2026-07-30T15:15:30Z","labels":["area:ingest"],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-hith","title":"claude-ai-export: synthetic attachment id keyed on positional index is unstable across export vintages","description":"## What the data says\n\nSame sample as polylogue-c429 (40 of 566 claude-ai-export ambiguous\nequal-message-count cohorts, read-only against /realm/db/polylogue,\nreproduced with production `parse_payload` + `session_revision_projection` +\n`classify_membership_revisions`). Of the 40, 16 (40%) have this exact shape,\ndisjoint from the message-order cause in polylogue-c429:\n\n message id set equal, message array order equal, 0 content diffs\n len(attachments_a) == len(attachments_b)\n set of (provider_attachment_id, message_provider_id) DISJOINT or partially disjoint\n between the two revisions, for attachments anchored to the SAME message\n\nExample (cohort with 4 attachments, 2 anchor messages, `key` starting\n`claude-a...`):\n\n A: att_id=e950263f-063d-495d-b0c0-61e9330d3a14 msg=7f1cf6ff-...\n B: att_id=att-ce21cd12d650 msg=7f1cf6ff-... (same message anchor)\n\n A: att_id=66a7a163-d488-4320-8129-19ad43f64a43 msg=c38e86ac-...\n B: att_id=att-cd01a39eb65c msg=0d3a13b8-... (different message anchor too)\n\nmime_type/size_bytes/inline-presence/name-length are identical between the\npaired attachments in every sampled case -- this is not\npolylogue-bu1i's acquisition-state pattern (`inline_bytes`/`size_bytes`\nflipping None-\u003ereal). The IDENTITY STRING itself differs, and sometimes so\ndoes the message it's anchored to.\n\n## Root cause\n\n`polylogue/sources/parsers/base_support.py:152-197`\n(`attachment_from_meta`/`_make_attachment_id`), used by the Claude.ai parser\nvia `attachment_from_meta` (`polylogue/sources/parsers/claude/ai_parser.py`,\n`_merge_session_attachments` at line ~191, iterating `(\"attachments\",\n\"files\")`):\n\n```python\ndef _make_attachment_id(seed: str) -\u003e str:\n return f\"att-{hash_text(seed)[:12]}\"\n\ndef attachment_from_meta(meta, message_id, index):\n attachment_id = (\n meta.get(\"id\") or meta.get(\"file_id\") or meta.get(\"fileId\")\n or meta.get(\"uuid\") or meta.get(\"file_uuid\")\n )\n ...\n if not attachment_id:\n if not name:\n return None\n seed = f\"{message_id or 'msg'}:{name}:{index}\"\n attachment_id = _make_attachment_id(seed)\n```\n\nTwo independent failure modes both traced in the sample:\n\n1. **Real-id presence is inconsistent across export vintages.** When\n Claude.ai's own export payload carries a real `id`/`file_id`/`uuid` for an\n attachment, that string is used directly (stable). When it's absent, the\n parser falls back to a SYNTHETIC id hashed from\n `f\"{message_id}:{name}:{index}\"`. The two export vintages of the same\n conversation don't consistently include the real id -- one carries it,\n the other doesn't -- so the same physical attachment gets a real UUID in\n one revision and a synthetic `att-...` id in the other.\n2. **`index` is positional, and attachment order is not guaranteed stable.**\n Even when BOTH revisions fall back to synthesis, `index` is the\n attachment's position in the merged `attachments`+`files` iteration for\n that message. If that per-message ordering shifts between export\n vintages (plausible given polylogue-c429's proof that the surrounding\n MESSAGE array order is itself unstable across Claude.ai exports), the\n synthesized id changes even though the underlying attachment didn't.\n\nEither way, attachment identity is accidentally keyed on transient\nexport-shape details (real-id presence, list order) rather than a property\nof the attachment itself, so `_attachment_hash_payload`\n(`polylogue/pipeline/ids.py:152`) hashes the same physical attachment to two\ndifferent identities across export vintages -- the same general shape as\npolylogue-bu1i (acquisition/export-time noise contaminating an identity\nhash), but a DIFFERENT concrete defect (id synthesis, not acquisition-state\nflip) requiring a different fix.\n\n## Reproduction recipe (production code, no archive mutation)\n\nSame harness as polylogue-c429's reproduction recipe; after loading both\n`ParsedSession`s for a cohort where message ids/order/content are identical:\n\n```python\natts_a = {(at.provider_attachment_id, at.message_provider_id): at for at in a.attachments}\natts_b = {(at.provider_attachment_id, at.message_provider_id): at for at in b.attachments}\nassert len(atts_a) == len(atts_b)\nassert set(atts_a) != set(atts_b) # disjoint identity despite same count\n```\n\n## Extrapolation honesty\n\n40 of 566 sampled (7%). 16/40 = 40% match this exact shape (message\ncontent/order fully identical, attachment key sets disjoint at equal\ncount). Extrapolating to the full population suggests roughly 220-230 of the\n566 cohorts, but this is an estimate from a 7% sample, not a census.\n\n## Proposed fix direction (for the classifier/parser-owning lane, not this bead)\n\nTwo independent levers, either alone reduces the blast radius:\n\n- Parser-side: derive the synthetic attachment id from content-stable\n material only (e.g. a hash of `(message_provider_id, name, mime_type,\n size_bytes)` without positional `index`), so re-ordering the export's\n attachment list doesn't change identity. Does not fix mode (1)\n (real-id-present-in-one-export-only).\n- Classifier-side (in the files this investigation lane does not edit):\n compare attachments by a looser key (e.g. `(message_provider_id, name,\n mime_type, size_bytes)`) when testing dominance, falling back to id\n equality only when that tuple is ambiguous -- the same class of relaxation\n polylogue-bu1i proposes for acquisition-state, generalized.\n\nRef polylogue-bu1i\nRef polylogue-c429\n","status":"open","priority":1,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T12:16:40Z","created_by":"Sinity","updated_at":"2026-07-30T12:17:40Z","labels":["area:ingest"],"comments":[{"id":"019fb31e-ef07-7bef-a785-41e5de19372f","issue_id":"polylogue-hith","author":"Sinity","text":"Parser-side fix landed (PR pending, branch feature/fix/synthetic-attachment-id-stability):\nattachment_from_meta's synthetic-id seed no longer includes the positional\n`index`; mime_type is used as the one extra structural disambiguator instead\n(id/name/mime_type). The now-unused `index` param was removed from\nattachment_from_meta and all 3 call sites (ai_parser.py x2,\nclaude/common.py's _message_attachments).\n\nVerified: 250-cohort regression replay (old vs new minting logic) over\nalready-resolved claude-ai-export cohorts -- 249/250 agree, 1 improvement,\n0 regressions.\n\nHonest disposition on the 566-cohort measured population: 0 resolved by this\nfix alone. Full census (not sample) shows all 268 message-hashes-equal\nambiguous cohorts are \"mixed real/synthetic\" (failure mode 1: real-id\npresence varies across export vintages of the same conversation) -- 0 are\n\"pure synthetic on both sides\" (the positional-index shape this fix\ntargets). Failure mode 1 needs a comparison-layer relaxation in\nsession_revision_membership.py/ids.py, which this lane was scoped away\nfrom. Filed as polylogue-d8al with the full census breakdown and a proposed\ndesign (loosen dominance comparison to (message_id, name, mime_type) when\nprovider ids disagree, id-equality fallback only when that's itself\nambiguous). polylogue-c429 (message order) accounts for the other 297.\n\nLeaving this bead open pending the comparison-layer fix -- the fix in this\nPR is real and durable (protects any future/other-origin case of the\npositional-index shape) but does not itself resolve the currently-measured\npopulation; polylogue-d8al is the actionable remainder.\n","created_at":"2026-07-30T13:02:57Z"}],"dependency_count":0,"dependent_count":0,"comment_count":1} {"_type":"issue","id":"polylogue-qkuq","title":"claude-ai-export: synthetic attachment id keyed on positional index is unstable across export vintages","status":"open","priority":1,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T12:16:33Z","created_by":"Sinity","updated_at":"2026-07-30T12:16:33Z","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"polylogue-c429","title":"claude-ai-export: message array order is not stable across export vintages, breaking prefix-dominance","description":"## What the data says\n\nSampled 40 of 566 claude-ai-export \"ambiguous\" equal-message-count membership\ncohorts (7%; read-only against /realm/db/polylogue). Reproduced with\nproduction code: parsed both distinct-content raw revisions of each cohort\nvia `polylogue.sources.dispatch.parse_payload` (routed through\n`provider_from_origin`/`capture_mode` exactly as `_parse_one` does in\n`polylogue/sources/revision_backfill.py`), projected each with\n`polylogue.pipeline.ids.session_revision_projection`, and ran the production\n`classify_membership_revisions`.\n\n21 of 40 sampled cohorts (52.5%) have this exact shape:\n\n n_messages_a == n_messages_b\n set(provider_message_id for a.messages) == set(provider_message_id for b.messages)\n [a.provider_message_id for a in a.messages] != [... for b.messages] # order differs\n for every shared id: (role, text, timestamp) identical between a and b\n\nThat is: the SAME messages, byte-identical per-message content, just in a\nDIFFERENT SEQUENCE in the two exports. Concretely reproduced on cohort\n`claude-ai:944d1095-51ea-4063-abe9-719d9971e281` (raws\n`aa0990572bb833d4...` vs `ebe3a4f95f45b235...`): 36/36 messages, identical\n`{role,text,timestamp}` for every one of the 36 shared `provider_message_id`s,\n0 content diffs, but `ids_a != ids_b`. One revision's message array is sorted\nchronologically; the other is not (its role sequence pairs adjacent\nuser/user, assistant/assistant messages -- looks like Claude.ai's own tree\nflattening interleaving edited-message siblings rather than a strict\ntimestamp sort).\n\n## Root cause\n\n`polylogue/pipeline/ids.py:session_revision_projection` builds\n`message_hashes` as an ORDER-SENSITIVE tuple (`_message_hash_payload` per\nmessage, in array order). `polylogue/archive/session_revision_membership.py`\n`_strictly_dominates` requires\n`older.message_hashes == newer.message_hashes[: len(older.message_hashes)]`\n-- an exact positional prefix match. When Claude.ai's own export emits the\nsame conversation's message array in a different sequence across two export\nrequests (same message set, same content, different order), this prefix\ncheck fails in BOTH directions even though there is no real content\ndivergence, and the cohort is quarantined ambiguous.\n\nNo parser code sorts messages by timestamp before this hash is computed\n(`polylogue/sources/parsers/claude/ai_parser.py` preserves whatever order the\nexport's `chat_messages` array carries; see `_merge_session_attachments`\niterating `(\"attachments\", \"files\")` for the analogous merge-order case in\nattachments). Claude.ai's own export ordering for a given conversation is\napparently NOT guaranteed stable across separate export requests -- this is\nupstream non-determinism polylogue must tolerate, not something polylogue's\nown acquisition controls.\n\n## Reproduction recipe (production code, no archive mutation)\n\n```python\nfrom pathlib import Path\nfrom polylogue.sources.decoders import _iter_json_stream\nfrom polylogue.sources.dispatch import parse_payload\nfrom polylogue.core.enums import Origin\nfrom polylogue.core.sources import provider_from_origin\nfrom polylogue.pipeline.ids import session_revision_projection\nimport io, sqlite3\n\ncon = sqlite3.connect(\"file:/realm/db/polylogue/source.db?mode=ro\", uri=True)\ncon.row_factory = sqlite3.Row\nrows = con.execute(\n \"select rs.raw_id, rs.source_path, rs.blob_hash, rs.capture_mode \"\n \"from raw_session_memberships m join raw_sessions rs on rs.raw_id = m.raw_id \"\n \"where m.decision='ambiguous' and rs.origin='claude-ai-export' \"\n \"and m.logical_source_key = ?\",\n (\"claude-ai:944d1095-51ea-4063-abe9-719d9971e281\",),\n).fetchall()\n\ndef load(row):\n h = row[\"blob_hash\"].hex()\n raw = (Path(\"/realm/db/polylogue/blob\") / h[:2] / h[2:]).read_bytes()\n provider = provider_from_origin(Origin.CLAUDE_AI_EXPORT, family_hint=row[\"capture_mode\"])\n fallback_id = Path(row[\"source_path\"].split(\":\")[-1]).stem\n name = Path(row[\"source_path\"].split(\":\")[-1]).name\n records = list(_iter_json_stream(io.BytesIO(raw), name))\n return parse_payload(str(provider), records, fallback_id, source_path=row[\"source_path\"])\n\nsessions = {r[\"raw_id\"]: load(r)[0] for r in rows} # cohort has exactly 1 session per raw here\nids = list(sessions)\na, b = sessions[ids[0]], sessions[ids[1]]\nassert {m.provider_message_id for m in a.messages} == {m.provider_message_id for m in b.messages}\nassert [m.provider_message_id for m in a.messages] != [m.provider_message_id for m in b.messages]\n```\n\n## Extrapolation honesty\n\n40 of 566 sampled (7%), stratified randomly (seed fixed). 21/40 = 52.5% match\nthis exact shape; 3 more sampled cohorts show this pattern layered with a\nsecond delta (attachment count or session-event differences) in addition.\nExtrapolating the 52.5% rate to the full population suggests roughly 280-300\nof the 566 cohorts, but this is an ESTIMATE from a 7% sample, not a census --\nunlike polylogue-bu1i's 100%-verified aistudio-drive population, this has not\nbeen checked against every cohort.\n\n## Proposed fix direction (for the classifier-owning lane, not this bead)\n\n`_strictly_dominates` and `session_revision_projection` currently treat\nmessage sequence as part of content identity. A safe fix compares the\nmessage SET (by `provider_message_id` + content) rather than requiring an\nexact positional prefix when a provider's export ordering is not\nauthoritative -- i.e. treat \"same message ids/content, different array\norder\" as equivalent, not as a branch. This is a distinct code path from\npolylogue-bu1i's attachment-acquisition-state fix (different failure\nmode, different field: sequence vs. attachment identity) and should not be\nfolded into the same patch without separate verification, since a naive\norder-insensitive compare would also need to preserve real append-order\ndetection (`older.message_hashes == newer.message_hashes[:len(older)]`) for\ngenuinely growing sessions.\n\nRef polylogue-bu1i\nRef polylogue-hith\nRef polylogue-nuec\n","status":"open","priority":1,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T12:14:22Z","created_by":"Sinity","updated_at":"2026-07-30T12:17:41Z","labels":["area:ingest"],"dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"polylogue-9dxn","title":"A persisted 'ambiguous' verdict is terminal with no classifier version, so classifier corrections are inert on existing data","description":"## Problem\n\n`polylogue-bu1i` fixes the classifier so that acquiring an attachment's bytes is\nread as a fidelity upgrade rather than a branch. Verified: all 157 live\naistudio-drive cohorts now resolve to an accepted chain with the enriched\nrevision at its head, where previously 157/157 were ambiguous.\n\nThat fix cannot heal the archive it was written for. The verdicts it corrects are\nalready persisted, and a persisted `ambiguous` verdict is TERMINAL:\n\n polylogue/storage/repair.py:4432-4462\n SELECT 1 FROM raw_session_memberships\n WHERE raw_id IN (...) AND decision = 'ambiguous'\n -\u003e RawReplayPlanStatus.TERMINAL,\n \"component ended in explicit ambiguous or parse-terminal authority state\",\n \"inspect durable authority debt; do not replay without new evidence\"\n\n`raw_session_memberships` has no fingerprint column, so nothing distinguishes\n\"ambiguous under the current classifier\" from \"ambiguous under a classifier we\nhave since corrected\". Every improvement to `classify_membership_revisions` is\ntherefore inert on existing data and only affects newly-acquired raws, while the\nexisting debt sits terminal forever and reads as though it needed operator\njudgment.\n\nLive scale of the inert-fix problem: 3,875 ambiguous membership rows across\n~1,079 cohorts (587 claude-ai-export, 191 claude-code-session, 151\naistudio-drive, 136 chatgpt-export, and a tail).\n\n## Second defect: a bump would not propagate\n\n`RAW_AUTHORITY_PARSER_FINGERPRINT = \"revision-membership-v1\"` exists as a proper\nconstant in `polylogue/storage/raw_authority.py:27`, but\n`polylogue/sources/revision_backfill.py` hardcodes the literal string eight\ntimes instead of importing it (lines 318, 348, 438, 475, 552, 565, 596, 919),\nincluding inside an f-string. Bumping the constant today would half-apply: the\nwriter would stamp the new value while the quiescence gate still matched the old\none. The constant is not load-bearing, which makes the versioning mechanism\nnon-functional exactly when it is first needed.\n\n## Proposed fix\n\n1. Make the constant load-bearing: `revision_backfill.py` imports\n `RAW_AUTHORITY_PARSER_FINGERPRINT` rather than repeating the literal.\n2. Separate two questions the single fingerprint currently conflates:\n - *Was this raw ever observed by a real parser?* -- the quiescence gate\n (`uncensused_historical_revision_raw_ids`, `revision_backfill.py:321`).\n Any known fingerprint should satisfy this, so a bump does NOT trigger an\n archive-wide re-census of all 41,363 raws.\n - *Is this verdict still authoritative under current semantics?* -- the\n terminal gate. Only the CURRENT fingerprint should satisfy this.\n Concretely: keep a `SUPERSEDED_MEMBERSHIP_FINGERPRINTS` set alongside the\n current one, and have the terminal check treat an `ambiguous` decision as\n stale (replayable) when the raw's census fingerprint is superseded rather\n than current. Absent census row -\u003e treat as current, i.e. stay conservative.\n `index_tier.raw_revision_applications` carries the same `decision='ambiguous'`\n check and needs the same treatment.\n3. Bump `RAW_AUTHORITY_PARSER_FINGERPRINT` to `revision-membership-v2`, because\n `polylogue-bu1i` genuinely changed classification semantics.\n\nWith (2) in place the healing is targeted: roughly 3,875 raws re-derive their\nverdict, instead of re-censusing the whole 99 GB archive. Without (2), a bump\nis correct but costs a full reparse (~4h20m measured on this archive).\n\n## Why this is the general fix, not a one-off\n\nThe value here is not unblocking one origin. It is that a classifier correction\nbecomes self-healing: today, improving `classify_membership_revisions` requires\nmanual archive surgery to have any effect on existing data, which is precisely\nthe shape that leaves corrected logic silently inert and debt looking legitimate.\n\n## Acceptance criteria\n\n- `RAW_AUTHORITY_PARSER_FINGERPRINT` is the single source of the fingerprint\n string; no module hardcodes it.\n- An `ambiguous` verdict recorded under a superseded fingerprint is replayable,\n and one recorded under the current fingerprint remains terminal. Both\n directions covered by tests.\n- A bump does not force re-census of raws whose verdict is unaffected; assert\n this against a fixture archive rather than by reasoning.\n- Anti-vacuity: state the production line mutated and the resulting failure.\n\nRef polylogue-bu1i\n","status":"open","priority":1,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T12:13:46Z","created_by":"Sinity","updated_at":"2026-07-30T12:13:46Z","labels":["area:ingest"],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-c429","title":"claude-ai-export: message array order is not stable across export vintages, breaking prefix-dominance","description":"## What the data says\n\nSampled 40 of 566 claude-ai-export \"ambiguous\" equal-message-count membership\ncohorts (7%; read-only against /realm/db/polylogue). Reproduced with\nproduction code: parsed both distinct-content raw revisions of each cohort\nvia `polylogue.sources.dispatch.parse_payload` (routed through\n`provider_from_origin`/`capture_mode` exactly as `_parse_one` does in\n`polylogue/sources/revision_backfill.py`), projected each with\n`polylogue.pipeline.ids.session_revision_projection`, and ran the production\n`classify_membership_revisions`.\n\n21 of 40 sampled cohorts (52.5%) have this exact shape:\n\n n_messages_a == n_messages_b\n set(provider_message_id for a.messages) == set(provider_message_id for b.messages)\n [a.provider_message_id for a in a.messages] != [... for b.messages] # order differs\n for every shared id: (role, text, timestamp) identical between a and b\n\nThat is: the SAME messages, byte-identical per-message content, just in a\nDIFFERENT SEQUENCE in the two exports. Concretely reproduced on cohort\n`claude-ai:944d1095-51ea-4063-abe9-719d9971e281` (raws\n`aa0990572bb833d4...` vs `ebe3a4f95f45b235...`): 36/36 messages, identical\n`{role,text,timestamp}` for every one of the 36 shared `provider_message_id`s,\n0 content diffs, but `ids_a != ids_b`. One revision's message array is sorted\nchronologically; the other is not (its role sequence pairs adjacent\nuser/user, assistant/assistant messages -- looks like Claude.ai's own tree\nflattening interleaving edited-message siblings rather than a strict\ntimestamp sort).\n\n## Root cause\n\n`polylogue/pipeline/ids.py:session_revision_projection` builds\n`message_hashes` as an ORDER-SENSITIVE tuple (`_message_hash_payload` per\nmessage, in array order). `polylogue/archive/session_revision_membership.py`\n`_strictly_dominates` requires\n`older.message_hashes == newer.message_hashes[: len(older.message_hashes)]`\n-- an exact positional prefix match. When Claude.ai's own export emits the\nsame conversation's message array in a different sequence across two export\nrequests (same message set, same content, different order), this prefix\ncheck fails in BOTH directions even though there is no real content\ndivergence, and the cohort is quarantined ambiguous.\n\nNo parser code sorts messages by timestamp before this hash is computed\n(`polylogue/sources/parsers/claude/ai_parser.py` preserves whatever order the\nexport's `chat_messages` array carries; see `_merge_session_attachments`\niterating `(\"attachments\", \"files\")` for the analogous merge-order case in\nattachments). Claude.ai's own export ordering for a given conversation is\napparently NOT guaranteed stable across separate export requests -- this is\nupstream non-determinism polylogue must tolerate, not something polylogue's\nown acquisition controls.\n\n## Reproduction recipe (production code, no archive mutation)\n\n```python\nfrom pathlib import Path\nfrom polylogue.sources.decoders import _iter_json_stream\nfrom polylogue.sources.dispatch import parse_payload\nfrom polylogue.core.enums import Origin\nfrom polylogue.core.sources import provider_from_origin\nfrom polylogue.pipeline.ids import session_revision_projection\nimport io, sqlite3\n\ncon = sqlite3.connect(\"file:/realm/db/polylogue/source.db?mode=ro\", uri=True)\ncon.row_factory = sqlite3.Row\nrows = con.execute(\n \"select rs.raw_id, rs.source_path, rs.blob_hash, rs.capture_mode \"\n \"from raw_session_memberships m join raw_sessions rs on rs.raw_id = m.raw_id \"\n \"where m.decision='ambiguous' and rs.origin='claude-ai-export' \"\n \"and m.logical_source_key = ?\",\n (\"claude-ai:944d1095-51ea-4063-abe9-719d9971e281\",),\n).fetchall()\n\ndef load(row):\n h = row[\"blob_hash\"].hex()\n raw = (Path(\"/realm/db/polylogue/blob\") / h[:2] / h[2:]).read_bytes()\n provider = provider_from_origin(Origin.CLAUDE_AI_EXPORT, family_hint=row[\"capture_mode\"])\n fallback_id = Path(row[\"source_path\"].split(\":\")[-1]).stem\n name = Path(row[\"source_path\"].split(\":\")[-1]).name\n records = list(_iter_json_stream(io.BytesIO(raw), name))\n return parse_payload(str(provider), records, fallback_id, source_path=row[\"source_path\"])\n\nsessions = {r[\"raw_id\"]: load(r)[0] for r in rows} # cohort has exactly 1 session per raw here\nids = list(sessions)\na, b = sessions[ids[0]], sessions[ids[1]]\nassert {m.provider_message_id for m in a.messages} == {m.provider_message_id for m in b.messages}\nassert [m.provider_message_id for m in a.messages] != [m.provider_message_id for m in b.messages]\n```\n\n## Extrapolation honesty\n\n40 of 566 sampled (7%), stratified randomly (seed fixed). 21/40 = 52.5% match\nthis exact shape; 3 more sampled cohorts show this pattern layered with a\nsecond delta (attachment count or session-event differences) in addition.\nExtrapolating the 52.5% rate to the full population suggests roughly 280-300\nof the 566 cohorts, but this is an ESTIMATE from a 7% sample, not a census --\nunlike polylogue-bu1i's 100%-verified aistudio-drive population, this has not\nbeen checked against every cohort.\n\n## Proposed fix direction (for the classifier-owning lane, not this bead)\n\n`_strictly_dominates` and `session_revision_projection` currently treat\nmessage sequence as part of content identity. A safe fix compares the\nmessage SET (by `provider_message_id` + content) rather than requiring an\nexact positional prefix when a provider's export ordering is not\nauthoritative -- i.e. treat \"same message ids/content, different array\norder\" as equivalent, not as a branch. This is a distinct code path from\npolylogue-bu1i's attachment-acquisition-state fix (different failure\nmode, different field: sequence vs. attachment identity) and should not be\nfolded into the same patch without separate verification, since a naive\norder-insensitive compare would also need to preserve real append-order\ndetection (`older.message_hashes == newer.message_hashes[:len(older)]`) for\ngenuinely growing sessions.\n\nRef polylogue-bu1i\nRef polylogue-hith\nRef polylogue-nuec\n","notes":"Superseded by polylogue-aggz's architecture: message array order is now handled as a byproduct of set-based (identity, content) comparison (message_contents), not a dedicated positional-prefix fix. Live census (full population): 554/587 (94.4%) claude-ai-export ambiguous cohorts now resolve, 0 regressions against previously-resolved cohorts. See PR.","status":"open","priority":1,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T12:14:22Z","created_by":"Sinity","updated_at":"2026-07-30T15:15:29Z","labels":["area:ingest"],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-9dxn","title":"A persisted 'ambiguous' verdict is terminal with no classifier version, so classifier corrections are inert on existing data","description":"## Problem\n\n`polylogue-bu1i` fixes the classifier so that acquiring an attachment's bytes is\nread as a fidelity upgrade rather than a branch. Verified: all 157 live\naistudio-drive cohorts now resolve to an accepted chain with the enriched\nrevision at its head, where previously 157/157 were ambiguous.\n\nThat fix cannot heal the archive it was written for. The verdicts it corrects are\nalready persisted, and a persisted `ambiguous` verdict is TERMINAL:\n\n polylogue/storage/repair.py:4432-4462\n SELECT 1 FROM raw_session_memberships\n WHERE raw_id IN (...) AND decision = 'ambiguous'\n -\u003e RawReplayPlanStatus.TERMINAL,\n \"component ended in explicit ambiguous or parse-terminal authority state\",\n \"inspect durable authority debt; do not replay without new evidence\"\n\n`raw_session_memberships` has no fingerprint column, so nothing distinguishes\n\"ambiguous under the current classifier\" from \"ambiguous under a classifier we\nhave since corrected\". Every improvement to `classify_membership_revisions` is\ntherefore inert on existing data and only affects newly-acquired raws, while the\nexisting debt sits terminal forever and reads as though it needed operator\njudgment.\n\nLive scale of the inert-fix problem: 3,875 ambiguous membership rows across\n~1,079 cohorts (587 claude-ai-export, 191 claude-code-session, 151\naistudio-drive, 136 chatgpt-export, and a tail).\n\n## Second defect: a bump would not propagate\n\n`RAW_AUTHORITY_PARSER_FINGERPRINT = \"revision-membership-v1\"` exists as a proper\nconstant in `polylogue/storage/raw_authority.py:27`, but\n`polylogue/sources/revision_backfill.py` hardcodes the literal string eight\ntimes instead of importing it (lines 318, 348, 438, 475, 552, 565, 596, 919),\nincluding inside an f-string. Bumping the constant today would half-apply: the\nwriter would stamp the new value while the quiescence gate still matched the old\none. The constant is not load-bearing, which makes the versioning mechanism\nnon-functional exactly when it is first needed.\n\n## Proposed fix\n\n1. Make the constant load-bearing: `revision_backfill.py` imports\n `RAW_AUTHORITY_PARSER_FINGERPRINT` rather than repeating the literal.\n2. Separate two questions the single fingerprint currently conflates:\n - *Was this raw ever observed by a real parser?* -- the quiescence gate\n (`uncensused_historical_revision_raw_ids`, `revision_backfill.py:321`).\n Any known fingerprint should satisfy this, so a bump does NOT trigger an\n archive-wide re-census of all 41,363 raws.\n - *Is this verdict still authoritative under current semantics?* -- the\n terminal gate. Only the CURRENT fingerprint should satisfy this.\n Concretely: keep a `SUPERSEDED_MEMBERSHIP_FINGERPRINTS` set alongside the\n current one, and have the terminal check treat an `ambiguous` decision as\n stale (replayable) when the raw's census fingerprint is superseded rather\n than current. Absent census row -\u003e treat as current, i.e. stay conservative.\n `index_tier.raw_revision_applications` carries the same `decision='ambiguous'`\n check and needs the same treatment.\n3. Bump `RAW_AUTHORITY_PARSER_FINGERPRINT` to `revision-membership-v2`, because\n `polylogue-bu1i` genuinely changed classification semantics.\n\nWith (2) in place the healing is targeted: roughly 3,875 raws re-derive their\nverdict, instead of re-censusing the whole 99 GB archive. Without (2), a bump\nis correct but costs a full reparse (~4h20m measured on this archive).\n\n## Why this is the general fix, not a one-off\n\nThe value here is not unblocking one origin. It is that a classifier correction\nbecomes self-healing: today, improving `classify_membership_revisions` requires\nmanual archive surgery to have any effect on existing data, which is precisely\nthe shape that leaves corrected logic silently inert and debt looking legitimate.\n\n## Acceptance criteria\n\n- `RAW_AUTHORITY_PARSER_FINGERPRINT` is the single source of the fingerprint\n string; no module hardcodes it.\n- An `ambiguous` verdict recorded under a superseded fingerprint is replayable,\n and one recorded under the current fingerprint remains terminal. Both\n directions covered by tests.\n- A bump does not force re-census of raws whose verdict is unaffected; assert\n this against a fixture archive rather than by reasoning.\n- Anti-vacuity: state the production line mutated and the resulting failure.\n\nRef polylogue-bu1i\n","notes":"CORRECTION 2026-07-30, from the lane that traced polylogue-eqnv: the 'Proposed fix' item (2) above is wrong for identity-class staleness, and I am recording that before anyone implements it.\n\nI proposed splitting the fingerprint's two jobs so that the QUIESCENCE gate accepts any *known* fingerprint (avoiding an archive-wide re-census on a bump) while only the TERMINAL gate requires the current one. The motive was cost: targeted healing of ~3,900 raws instead of reparsing 99 GB.\n\nThat does not work when the stale thing is the raw's derived IDENTITY rather than its verdict. polylogue-eqnv is the concrete counterexample: two raws of one document were censused under the same fingerprint string but recorded different logical_source_key values, one carrying a pre-#3179 '-0' suffix from a dispatch bug fixed 2026-07-20 (b473d9256) that their 2026-07-16/18 acquisition predates. Reparsing both blobs through current dispatch yields the identical correct key. Permissive quiescence is exactly what keeps that raw from ever being re-derived, so it preserves the corruption it was meant to be cheap about.\n\nConsequence for this bead's scope: re-census (a reparse) is the honest price for any change that alters derived identity, and the cost cannot be engineered away by making the gate permissive. The split between 'was this observed' and 'is this verdict current' may still be worth having for pure VERDICT changes, where the recorded identity is unaffected -- polylogue-bu1i is that shape, since it changed only how revisions are COMPARED. State which class a change falls in before choosing the cheap path.\n\nPossible middle path, not yet evaluated: re-census only raws whose recorded identity disagrees with a cheap re-derivation, which needs a parse but not a full projection/materialization. Whether that is meaningfully cheaper than the full reparse is unmeasured -- do not assume it is.\nDESIGN 2026-07-30, from the polylogue-eqnv/c737 lane, supersedes the correction note above with something actionable.\n\nSplit the single parser fingerprint into two independently-versioned components:\n\n identity fingerprint -- covers dispatch.py's provider_session_id /\n logical_source_key derivation\n classification fingerprint -- covers session_revision_membership.py's\n dominance rules\n\nThen each class of fix pays only its own price:\n\n- A CLASSIFICATION fix (polylogue-bu1i's shape: dominance rules changed, the\n stored identity is unaffected) bumps only the classification component.\n Quiescence stays permissive on identity, so no reparse is forced, and the\n terminal-ambiguous gate re-runs classification against the already-known\n identity. Cheap, and it makes classifier corrections self-healing, which is\n this bead's original ask.\n- An IDENTITY fix (polylogue-eqnv's shape and the z1c6 dispatch bug: the stored\n logical_source_key itself was wrong) bumps the identity component. Quiescence\n goes strict for it, forcing exactly the reparse that is unavoidably the honest\n price -- you cannot know an identity is still correct without recomputing it,\n since recomputing IS how you discover it changed.\n\nThis is strictly better than the single fingerprint in both directions: today a\nclassification fix cannot heal existing data at all (the terminal gate has no\nversion to compare), and an identity fix would force a full 99 GB reparse even\nwhen only classification changed.\n\nImplementation note carried over: RAW_AUTHORITY_PARSER_FINGERPRINT must first\nbecome load-bearing -- sources/revision_backfill.py still hardcodes\n'revision-membership-v1' at eight sites (318, 348, 438, 475, 552, 565, 596,\n919) instead of importing the constant, so any bump half-applies until that is\nfixed.\nDESIGN (re-recorded 2026-07-30 after a bd reimport dropped the first append), from the polylogue-eqnv/c737 lane.\n\nSplit the single parser fingerprint into two independently-versioned components:\n\n identity fingerprint -- covers dispatch.py's provider_session_id /\n logical_source_key derivation\n classification fingerprint -- covers session_revision_membership.py's\n dominance rules\n\nEach class of fix then pays only its own price:\n\n- A CLASSIFICATION fix (polylogue-bu1i's shape: dominance rules changed, stored\n identity unaffected) bumps only the classification component. Quiescence stays\n permissive on identity so no reparse is forced, and the terminal-ambiguous\n gate re-runs classification against the already-known identity. Cheap, and it\n makes classifier corrections self-healing -- this bead's original ask.\n- An IDENTITY fix (polylogue-eqnv's shape, and the z1c6 dispatch bug: the stored\n logical_source_key itself was wrong) bumps the identity component. Quiescence\n goes strict for it, forcing exactly the reparse that is unavoidably the honest\n price -- you cannot know an identity is still correct without recomputing it,\n because recomputing IS how you discover it changed.\n\nStrictly better than one fingerprint in both directions: today a classification\nfix cannot heal existing data at all (the terminal gate has no version to\ncompare against), while an identity fix would force a full 99 GB reparse even\nwhen only classification changed.\n\nPrerequisite: RAW_AUTHORITY_PARSER_FINGERPRINT must become load-bearing first --\nsources/revision_backfill.py hardcodes 'revision-membership-v1' at eight sites\n(318, 348, 438, 475, 552, 565, 596, 919) instead of importing the constant, so\nany bump half-applies until that is fixed.","status":"open","priority":1,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T12:13:46Z","created_by":"Sinity","updated_at":"2026-07-30T13:32:35Z","labels":["area:ingest"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-bu1i","title":"aistudio-drive 'ambiguous' revision pairs are not branches: attachment acquisition state contaminates attachment identity hash","description":"## What the data says\n\n151 of 151 aistudio-drive ambiguous membership cohorts (100%) are the SAME Drive\ndocument acquired twice, where the later acquisition merely resolved\nDrive-hosted attachment bytes. There is no branch and nothing to judge.\n\nVerified across all 157 two-member source_path cohorts in the live archive\n(/realm/db/polylogue), by loading both blobs and comparing:\n\n 157/157 file_mtime_ms IDENTICAL (both carry Drive modifiedTime)\n 157/157 earlier blob has NO _polylogue_drive_live_bytes_b64\n 157/157 later blob HAS it\n 157/157 the two payloads are byte-equal after stripping that key\n 157/157 later blob is larger (median ~5-80x)\n\nReproduced deterministically with production code on the pair\n30-12-2025-SINEX-IDEAS.json (raws f6b63f0b / d0715a7f):\n\n bare 604,853 B msgs=60 events=61 atts=4 all inline=None, size_bytes=None\n enriched 5,602,664 B msgs=60 events=61 atts=4 same 4 Drive file ids, bytes fetched\n\n message_hashes equal: True\n event_hashes equal: True\n attachment sets: n1=4 n2=4 intersection=0 subset=False\n _strictly_dominates(bare-\u003eenriched) = False\n classify_membership_revisions -\u003e ambiguous ['d0715a7f','f6b63f0b']\n\n## Root cause (two independent contributors)\n\n1. `_attachment_hash_payload` (polylogue/pipeline/ids.py:152) folds\n ACQUISITION STATE into attachment IDENTITY: it appends\n `inline_content_hash` only when `inline_bytes is not None`, and\n `size_bytes` flips None -\u003e real once bytes are fetched. So the same\n attachment (same Drive file id, same message anchor) hashes differently\n before and after acquisition. The two revisions' attachment_hashes end up\n equal-cardinality and DISJOINT.\n\n2. `_strictly_dominates` (archive/session_revision_membership.py:188) then\n fails both of its conditions: `content_grew` is False (equal message and\n event counts, no proper attachment superset) and\n `older.attachment_hashes \u003c= newer.attachment_hashes` is False (disjoint,\n not subset). Neither escape hatch applies: both revisions have\n `browser_snapshot_fidelity=None` so `_provider_ordered_browser_snapshots`\n bails, and `_direct_export_precedence` needs a browser-capture sibling.\n -\u003e ambiguous, both quarantined.\n\nSeparately, `raw_sessions.revision_kind='unknown'` / `logical_source_key IS NULL`\nbecause the byte-prefix chain check cannot hold: the injector splices base64\nmid-document and re-serializes the whole JSON\n(`json.dumps(resolved, ensure_ascii=False)`, sources/drive/__init__.py:173),\nso the later bytes are not a byte-prefix extension of the earlier.\n\n## Where the second scrape came from\n\nNot two Drive versions. Both acquisitions read the SAME local cache file under\n`~/.local/share/polylogue/drive-cache/gemini/` (240 documents). The 2026-06-29\npass wrote the cache with attachments unresolved. The 2026-07-18 pass took the\ncache-hit branch (no Drive re-download at all) and ran\n`_inject_live_drive_attachment_bytes` -- which by design runs on EVERY read,\ncache hit or not, precisely to backfill caches written before the feature\nexisted (sources/drive/__init__.py:242-256). It mutated the bytes, rewrote the\ncache in place, and hashed the mutated payload -\u003e a second, distinct raw row.\nDrive modifiedTime never changed, which is why file_mtime_ms is identical.\n\nThe 83 single-row cohorts corroborate this: 72 have no driveDocument/Image/\nAudio/Video reference at all, and 11 have references the injector could not\nresolve -- in both cases the injector returns bytes unchanged, the blob hash is\nstable, and no second raw row is created.\n\n## Concrete harm already in the index\n\nPost-promotion convergence materialized these ambiguous raws anyway, arbitrarily\nand last-writer-wins. 6 cohorts got BOTH members materialized; in 5 of the 6 the\nBARE revision was written last, so the index now reports those sessions'\nattachments as `unfetched` even though the bytes were successfully fetched and\nare sitting in the blob store:\n\n aistudio-drive:Implementing-066bb070... atts=1 acquired=0\n aistudio-drive:Implementing-13ced1c8... atts=1 acquired=0\n aistudio-drive:Implementing-37edfeb3... atts=1 acquired=0\n aistudio-drive:Implementing-845dd573... atts=1 acquired=0\n aistudio-drive:Implementing-d4d7fbab... atts=1 acquired=0\n\nThat is a silent fidelity DOWNGRADE, and it is the exact failure mode the\n'never choose between branches' invariant exists to prevent -- it happened\nbecause a non-branch was labelled a branch, and then something picked anyway.\nWhich stage performed that pick is not yet traced: `repair.py:1075` does\nquarantine ambiguous membership, yet 135 of the 151 cohorts acquired a\n`parsed_at_ms` between 06:57 and 13:12 local on 2026-07-30, after the\n`decided_at_ms` of 07:00 that recorded them ambiguous. That gap needs its own\ntrace and may be a second, separate defect.\n\n## Proposed fix\n\nTreat 'same attachment identity, bytes now acquired' as a fidelity upgrade, the\ndirect analogue of the documented DOM-\u003enative rule. Concretely: compare\nattachments by provider identity (provider_attachment_id + message_provider_id\n+ name + mime_type) when testing dominance, and allow a differing hash when the\nonly delta is that the newer side has inline_bytes where the older did not.\nEquivalently, split attachment identity from attachment acquisition state so\nacquisition can never fabricate a branch.\n\nPrefer this over adding a Drive-specific escape hatch: the shape is generic\n(any origin whose attachments are fetched lazily), and the classifier already\nhas two precedents for 'this is an upgrade, not a branch'.\n\n## Blast radius beyond drive\n\nEqual-message-count ambiguous cohorts by origin (same shape; needs its own\nverification per origin before claiming the same cause):\n\n claude-ai-export 566 / 587 cohorts\n chatgpt-export 128 / 136\n aistudio-drive 151 / 151 \u003c- proven, this bead\n hermes-session 3 / 4\n claude-code-session 6 / 191 \u003c- different shape, not this\n gemini-cli-session 0 / 3\n\n## Measurement notes for whoever picks this up\n\n- Live aistudio-drive state at filing: index 225 sessions / 95,823 blocks\n (retired generation had 239 / 106,178); source has 173 unparsed raws, of\n which 129 are correctly superseded (their enriched sibling IS materialized)\n and 44 are the 22 both-unparsed cohorts. 14 documents are absent from the\n index entirely -- exactly the 239-225 gap.\n- The earlier claim '0 correctly superseded, all 302 genuinely unmaterialized'\n was a measurement artifact: it checked `raw_sessions.logical_source_key`,\n which governance deliberately NULLs on transition to semantic membership\n (archive.py:2710). The key survives on\n `raw_session_memberships.logical_source_key` -- join that table instead.\n- Attachment acquisition overall improved enormously in this generation:\n acquired 26 -\u003e 2,849 (unfetched 3,120 -\u003e 177). This bead is a narrow\n regression channel inside a large win, not a verdict on the rebuild.\n\nRef polylogue-7ilr (which framed this residue as genuine authority debt\nrequiring operator judgment; for aistudio-drive that framing is wrong).\n","status":"open","priority":1,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T11:34:40Z","created_by":"Sinity","updated_at":"2026-07-30T11:34:40Z","labels":["area:ingest"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-ne6k","title":"repair_empty_sessions would delete the 832 genuinely-empty sessions the hook-inflation postmortem chose to retain","design":"Found 2026-07-29 by the pre-rebuild deletion audit. NOT on the rebuild path.\n\nrepair_empty_sessions / count_empty_sessions_sync (polylogue/storage/repair.py)\nselect with a blanket predicate:\n\n sessions LEFT JOIN messages ... WHERE m.session_id IS NULL\n\nIt cannot distinguish a legitimately-empty session from corruption debris.\nThat distinction is not hypothetical: the 2026-07-22 hook-inflation\npostmortem explicitly decided to RETAIN ~832 genuinely-empty sessions after\nthe de-inflation (index sessions went 83,286 -\u003e 18,391 = 17,559 real + 832\ngenuinely-empty). Browser-capture stubs are a second legitimate source.\nRunning this repair would delete exactly the rows that postmortem chose to\nkeep.\n\nWHY IT IS NOT A REBUILD BLOCKER: the target is MaintenanceTargetMode.CLEANUP\nwith destructive=True, and resolve_selected_maintenance_targets\n(cli/shared/check_maintenance.py) only includes CLEANUP targets when the\noperator explicitly passes --cleanup or names the target. Neither\nmaintenance/rebuild_index.py nor daemon/bulk_rebuild.py ever calls it. So the\nrebuild pipeline cannot trigger it.\n\nTHE REAL RISK IS OPERATIONAL: someone running `polylogue check --cleanup` as\nhousekeeping around the big rebuild would silently delete the retained\nsessions. DO NOT RUN --cleanup against the live archive until this is fixed.\n\nFIX: give the predicate a distinguishing signal -- e.g. require raw_id IS NULL\n(no acquired bytes behind it) or an explicit acquisition-status check -- so a\nsession that was legitimately acquired and legitimately has no messages is\nretained, and only rows with no provenance at all are candidates.\n","status":"open","priority":1,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-29T20:21:06Z","created_by":"Sinity","updated_at":"2026-07-29T20:21:06Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-ic5i","title":"three modules (~800 loc) are unreachable from production, including an unenforced holdout guard","design":"Found 2026-07-29 by a systematic sweep for code that exists, imports cleanly,\ntype-checks, has tests -- and is reachable from nothing in production. This is\na distinct failure mode from unfinished work and is invisible to every gate the\nrepo has.\n\nTHREE MODULES, ~800 LOC, 22 PUBLIC EXPORTS, ZERO PRODUCTION REFERENCES\n(each referenced only by its own test file; verified with a full-tree grep for\nthe module name AND for every public symbol it exports):\n\n polylogue/storage/sqlite/holdout_cohorts.py 260 loc, 10 exports\n HoldoutPolicy, HoldoutAccessError, HoldoutAccessReceipt, mark_holdout,\n get_holdout_policy, is_holdout, record_holdout_access,\n list_holdout_access_receipts, has_holdout_contamination,\n require_non_holdout_access\n THE SHARPEST ONE: this is an evaluation-integrity guard. Nothing calls\n require_non_holdout_access or has_holdout_contamination, so holdout\n protection is not enforced on any path. A guard that guards nothing is\n worse than no guard -- it reads, in review and in the module list, as\n though the protection exists.\n\n polylogue/insights/fable_packet.py 306 loc, 6 exports\n compile_private_fable_packet, regenerate_private_fable_packet,\n FableDelegationPacket, DelegationPacketRow, DelegationPacketLabel,\n DescriptiveDistribution\n\n polylogue/storage/block_anchor.py 231 loc, 6 exports\n parse_block_anchor, resolve_block_anchor, format_block_anchor,\n BlockAnchor, BlockAnchorResolution, InvalidBlockAnchorError\n Block content-hash citation anchors (svfj). If nothing resolves an\n anchor, a stored citation cannot be followed back to its block.\n\nDISPOSITION NEEDED PER MODULE, not a blanket answer: wire it (the capability\nis wanted and was simply never connected -- the right answer for\nsession_agent_policies earlier today), or delete it (nothing needs it, and it\nis costing review attention and a false sense of coverage). Do not leave a\nthird state.\n\nMETHOD, so this is repeatable:\n - modules whose name and whose every public symbol appear nowhere outside\n their own file and tests\n - config properties with no consumer\n - tables written but never read\n - enum members never constructed\n - repository/service public methods no surface calls\n\nFALSE POSITIVES THIS SWEEP PRODUCED -- record them so the next run does not\nre-raise them:\n - session_events kinds \"written but never read\" (31 of 54): WRONG. A generic\n reader exists (storage/sqlite/queries/session_events.py -\u003e\n repository/archive/sessions.py), they land on the domain Session model,\n a CLI surface renders them with an --event-type filter, and they drive\n session timestamp derivation for providers whose messages lack timestamps.\n - surfaces/projection_spec enums (RenderFormat, BodyPolicy, ...): WRONG as\n \"dead\" -- they are Pydantic field types, so they are live as validators.\n The real (narrower) defect is that nothing DISPATCHES on RenderFormat.\n - repository methods with no surface caller (51): TOO NOISY to act on as a\n list. traverse_work_evidence looked orphaned but its subsystem is\n referenced by 18 files; some others were added hours earlier and their\n surface is a known follow-up. Individual-method orphanhood is weak\n evidence; module-level orphanhood is strong.\n - cli/commands/maintenance/_blob_integrity.py: WRONG. Its five *_command\n functions are each registered elsewhere.\n\nA standing detector is worth building AFTER the imminent rebuild, but only in\nthe module-level form -- that is the form that produced true positives every\ntime. The method-level and event-level forms produced only noise.\n","status":"open","priority":1,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-29T18:21:54Z","created_by":"Sinity","updated_at":"2026-07-29T18:21:54Z","dependency_count":0,"dependent_count":0,"comment_count":0} @@ -447,6 +452,7 @@ {"_type":"issue","id":"polylogue-sru.2","title":"Characterize ambiguous bucket: wordless continuation vs prose-without-markers","description":"Split next-turn-is-tool-call (wordless continuation) from prose-lacking-ack-markers; state counts for both. Opus-4-7 74% ambiguous vs deepseek 17% is likely turn-structure variance, not behavior — this split disambiguates.","design":"Implementation home: the claim-vs-evidence classifier in devtools (devtools/ module behind `devtools workspace claim-vs-evidence`; tests tests/unit/devtools/test_claim_vs_evidence.py). Wordless-continuation detection: for each failure's paired next assistant message, check whether its blocks contain tool_use and no text block with \u003eN chars before the first tool_use — that is 'wordless continuation'; prose without matched ack markers stays 'ambiguous-prose'. Emit both as classification_reason variants (field already exists) and add the two counts to the report summary + by_model/by_tool cuts. Regen: `devtools workspace claim-vs-evidence --limit 5000 --out-dir .agent/demos/claim-vs-evidence --json`. Acceptance: report shows ambiguous split into wordless_continuation vs prose_no_marker with counts; per-model ambiguous variance (opus-4-7 74% vs deepseek 17%) re-examined after the split.","notes":"2026-07-03 Codex WIP: unit implementation for ambiguous split passes focused tests, but live regeneration with --limit 5000 became too slow and had to be killed twice. First attempt used correlated subqueries for next-message block shape; second used set-based CTE; third used chunked second query after sampled rows, but the full command still exceeded 90s on active archive and ignored SIGINT while inside SQLite. Do not close or commit this slice until the live regeneration path is profiled/fixed. Dirty files currently show the WIP implementation: devtools/claim_vs_evidence.py and tests/unit/devtools/test_claim_vs_evidence.py. Last passing focused proof: python -m py_compile + ruff check + devtools test tests/unit/devtools/test_claim_vs_evidence.py -\u003e 3 passed.","status":"closed","priority":1,"issue_type":"task","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-07-03T04:31:28Z","created_by":"Sinity","updated_at":"2026-07-03T07:45:10Z","started_at":"2026-07-03T07:09:21Z","closed_at":"2026-07-03T07:45:10Z","close_reason":"Completed: claim-vs-evidence now splits ambiguous follow-ups into wordless tool continuations and prose-without-marker buckets, reports the counts in JSON/README summaries, and regenerates the current demo on the active archive. Focused tests pass; live regen/check completed.","labels":["area:substrate","campaign"],"dependencies":[{"issue_id":"polylogue-sru.2","depends_on_id":"polylogue-sru","type":"parent-child","created_at":"2026-07-03T06:31:27Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":1,"comment_count":0} {"_type":"issue","id":"polylogue-sru.3","title":"Benign-recovery vs consequential-silence split by handler kind","description":"Read failures are ~94% silent but 'tried another path' is usually benign; Bash/test failures are the consequential class. Scope the headline to consequential handler kinds or add an explicit split — credibility depends on not inflating with trivial recoveries.","design":"Handler kind is already available on the paired failure row (actions lane exposes handler/tool). Define the consequential set explicitly in code (Bash/test/build/write-class handlers) and the benign-recovery set (Read/Glob/Grep-class 'tried another path'), emit split headline rows: silent-proceed among consequential vs among all. Keep the mapping a named constant with a rationale comment so reviewers can argue with it. Report both; never let the headline mix classes silently. Same regen/tests as the other methodology children.","status":"closed","priority":1,"issue_type":"task","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-07-03T04:31:28Z","created_by":"Sinity","updated_at":"2026-07-03T07:58:08Z","started_at":"2026-07-03T07:55:37Z","closed_at":"2026-07-03T07:58:08Z","close_reason":"Completed: claim-vs-evidence now reports a first-class handler-class split separating consequential shell/edit/write-class tool failures from benign read/search/path-discovery failures and other tools. The regenerated active-archive artifact shows consequential=4,177 failures with 921 silent-proceed (22.0% lower bound), benign_recovery=633 with 166 silent-proceed (26.2%), and other=190 with 92 silent-proceed (48.4%). Focused tests and demo shelf checks passed.","labels":["area:substrate","campaign"],"dependencies":[{"issue_id":"polylogue-sru.3","depends_on_id":"polylogue-sru","type":"parent-child","created_at":"2026-07-03T06:31:28Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":1,"comment_count":0} {"_type":"issue","id":"polylogue-sru.1","title":"Expose action-unit outcome fields + followup_class as product capability","description":"Capabilities-may-not-be-silos gate for the campaign: the facts the report needs must become composable query capability. After this, the whole report is `actions where is_error:true | group by session.origin, followup_class | count` and every future cut (model/tool/repo/time) is free.","design":"1) is_error/exit_code are normalized at parse time (sources/parsers/base_models.py:74-75) but ActionQueryRowPayload (surfaces/payloads.py:~1298) carries neither — add as filterable/groupable action-unit fields. 2) Add derived followup_class (acknowledged|silent_proceed|wordless_continuation|ambiguous) + followup_message_ref computed in the source-derived lowering (no cache tables). 3) Reduce devtools workspace claim-vs-evidence to a render preset over these query strings, or retire it. Touchpoint chain: stage parser -\u003e AST to_payload -\u003e executor -\u003e metadata.py aggregate_group_fields -\u003e shell_completion_values.py -\u003e devtools render openapi + cli-output-schemas + cli-reference. Line refs pre-07-03; re-locate.","acceptance_criteria":"Fixture session with known unacknowledged failure fires via pure query strings; report README numbers reproducible from the printed queries.","notes":"Completed: action-unit outcome follow-up classification is now shared query capability. is_error/exit_code were already wired; this slice added source-derived followup_class and followup_message_ref over existing actions/messages/blocks, exposed followup_class as filterable/groupable action metadata, added action row payload fields, routed root CLI terminal-unit aggregate expressions before session-selector compilation, and moved the report classifier from scripts into polylogue.archive.actions.followup. Reproduction/query forms are now printed in .agent/demos/claim-vs-evidence/PUBLIC_REPRODUCTION.md: actions where is_error:true | group by followup_class | count; actions where followup_class:silent_proceed. Verification: focused DSL/report/CLI tests passed; active demo packet regenerated over archive root /home/sinity/.local/share/polylogue schema v23 with 41,886 structured failures and 5,000 inspected; devtools verify --quick passed run 20260703T092510Z-quick-718233-46e8b587.","status":"closed","priority":1,"issue_type":"feature","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-07-03T04:31:27Z","created_by":"Sinity","updated_at":"2026-07-03T09:25:36Z","started_at":"2026-07-03T09:05:37Z","closed_at":"2026-07-03T09:25:36Z","close_reason":"Completed","labels":["area:query","area:substrate","campaign"],"dependencies":[{"issue_id":"polylogue-sru.1","depends_on_id":"polylogue-sru","type":"parent-child","created_at":"2026-07-03T06:31:26Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-ey3r","title":"verify-archive source-index-coverage counts superseded revisions as missing work, so its blocking error is ~72% by-design noise","description":"## Problem\n\n`polylogue ops maintenance verify-archive` reports `source-index-coverage` as a\nblocking **error** on the live archive:\n\n 29,992 complete-census raw(s), 23,036 raw-backed session(s);\n missing_work=12,976 orphans=0\n\nThe majority of that number is a by-design state, not missing work. A raw whose\nmembership decision is `superseded_equivalent` or `superseded_prefix` is a\nrevision whose content is represented in the index through its *accepted*\nsibling; it is not supposed to own a session row. Counting it as missing work\nmakes the metric unable to reach zero on any archive that has ever ingested the\nsame conversation twice -- which is every real archive.\n\nMeasured on the live archive (complete-census raws, joined to\n`raw_session_memberships`):\n\n superseded_equivalent / superseded_prefix 9,411 \u003c- by design, no own session\n ambiguous 3,920 \u003c- genuine authority debt\n applied / \u003cnone\u003e 28,593\n\nHand-inspecting the check's own `missing_work_sample` (10 ids) splits the same\nway: 7 are `ambiguous` + unparsed + genuinely absent from the index, and 3 are\n`superseded_equivalent`/`superseded_prefix`, already parsed, and **present in\nthe index** via their accepted sibling. Those 3 are counted as missing work\nanyway.\n\n## Why it matters beyond tidiness\n\nThis is the archive's own coherence gate, the thing meant to answer \"did the\nrebuild land correctly\". A blocking error that includes states the design\nrequires trains an operator to ignore it, which is worse than not having the\ncheck: the 3,920 rows of real debt hide inside a number that is ~72% noise. It\nalso means the check cannot be used as an acceptance criterion for a rebuild or\nrestore, which is exactly what it exists for.\n\n## Proposed fix\n\nExclude raws whose membership decision is `superseded_*` from `missing_work`,\nand report them as their own evidence bucket (`superseded_count`) so coverage\nstays auditable without being conflated. Keep `ambiguous` in a distinct bucket\ntoo -- it is real debt, but it is *known, recorded* debt with an owner\n(polylogue-9dxn / polylogue-bu1i and the per-origin causes), so it should be\nreportable separately from \"we cannot account for this raw at all\", which is the\nonly thing that deserves to block.\n\nSuggested shape:\n\n missing_work_count raws with no session and no explanation\n superseded_count content represented via an accepted sibling\n ambiguous_debt_count recorded authority debt\n orphan_count (unchanged)\n\nwith `error` reserved for `missing_work_count \u003e 0` and `warning` for a nonzero\n`ambiguous_debt_count`.\n\n## Related, observed in the same run, NOT this bead\n\n`fts-parity` also errors: `messages_fts gap=36757`,\n`blocks_command_trigram gap=13235`. This one looks like genuine convergence\nbacklog rather than a measurement artifact -- every worst-offender session has\n`indexed=0` and they are all subagent sessions ingested the same day, i.e. the\nFTS repair stage had not caught up when the daemon was stopped. Re-verify after\nthe next full rebuild before filing anything; if a gap survives a rebuild, that\nis a real defect and deserves its own bead.\n\n## Acceptance criteria\n\n- `source-index-coverage` distinguishes unexplained-missing from\n superseded-by-sibling from recorded-ambiguous, with counts for each.\n- On an archive whose only residue is superseded revisions, the check does not\n report `error`.\n- A raw that is genuinely absent and unexplained still errors.\n- Anti-vacuity: state the production line mutated and the resulting failure.\n","status":"open","priority":2,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T13:14:36Z","created_by":"Sinity","updated_at":"2026-07-30T13:14:36Z","labels":["area:ingest"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-ck5v","title":"Attachment byte backfill is coupled to one acquisition route, so payloads from any other route are never backfilled","description":"## Problem\n\nResolving a Drive-hosted attachment's bytes happens in exactly one place:\n`_inject_live_drive_attachment_bytes`, called from inside\n`iter_drive_raw_data` (`polylogue/sources/drive/__init__.py:253`). Its docstring\nis explicit that this placement is deliberate -- it is the only scope where the\nlive authenticated client and the raw JSON coexist -- and that it runs on every\nread, cache hit included, so a cache file written before the feature existed\nstill gets backfilled rather than being skipped forever.\n\nThat guarantee only holds for payloads the Drive iterator enumerates. A Drive\npayload that entered the archive by any other route is structurally outside it\nand can never be backfilled, no matter how many times the daemon converges.\n\nMeasured on the live archive: 11 of 397 `aistudio-drive` raws came from a legacy\nzip backfill under `inbox/polylogue-aistudio-legacy-backfill-sha256-*.zip:members/…`\nrather than from `drive-cache/gemini/`. Those payloads carry Drive-hosted\nattachment references in the ordinary `{\"id\": \"\u003cdrive-file-id\u003e\"}` shape the\ninjector resolves successfully elsewhere, but nothing will ever visit them,\nbecause the Drive iterator enumerates the live Drive folder and a zip member is\nnot in it.\n\nUnfetched attachment counts, live archive (still moving -- convergence was\nactively writing when these were taken, so re-derive before acting):\n\n chatgpt-export 6,075\n claude-ai-export 398\n aistudio-drive 360 \u003c- of which 334 upload_origin='drive'\n grok-export 37\n\nThe 334 drive-hosted ones are fetchable in principle: a live client and a file id\nare all that is required. Some fraction will be genuinely unfetchable (deleted\nDrive files, revoked access, over the 50 MB cap) and must stay honestly\nunfetched -- that distinction is part of the work, not an inconvenience.\n\n## Why this is an invariant, not a command\n\nPer the project's automagic-invariants principle, a condition Polylogue can\nmaintain automatically belongs in daemon convergence, not in an operator\ncommand. \"Every attachment whose bytes are fetchable has been fetched\" is\nexactly such a condition, and it is currently a side effect of one acquisition\nroute instead of a maintained property of the archive.\n\nCoupling it to acquisition also has a second cost: it makes attachment fidelity\ndepend on how a payload happened to arrive. Two identical documents, one synced\nfrom Drive and one restored from a zip, end up with different evidence.\n\n## Proposed direction\n\nA convergence stage that selects attachments with `acquisition_status \u003c\u003e\n'acquired'` and a resolvable provider handle, fetches them through the owning\nsource's client in bounded windows, and records terminal failures so a\npermanently-gone Drive file is not retried forever. The existing\n`ConvergenceStage` shape fits: bounded work per pass returning `False` to push\nthe remainder into `convergence_debt` as retryable is the documented pattern for\nexactly this.\n\nNote the interaction with `polylogue-bu1i`: backfilling bytes for an\nalready-indexed session changes its content hash and therefore produces a new\nrevision to reconcile. That is now safe -- acquisition is read as a fidelity\nupgrade rather than a branch -- but it means this stage must land after bu1i,\nnot before, or it will manufacture ambiguous cohorts at scale.\n\n## Acceptance criteria\n\n- An attachment referenced by a payload that did NOT arrive through its source's\n live iterator is still backfilled. Cover the legacy-zip route specifically,\n since that is the observed miss.\n- A genuinely unfetchable attachment reaches a terminal state and stops being\n retried; nothing fabricates a hash or size for bytes never read.\n- Bounded per-pass work with the remainder in `convergence_debt`.\n- Anti-vacuity: state the production line mutated and the resulting failure.\n\nRef polylogue-bu1i\n","status":"open","priority":2,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T12:16:42Z","created_by":"Sinity","updated_at":"2026-07-30T12:16:42Z","labels":["area:ingest"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-7ilr","title":"Surface why a raw failed to materialize (ambiguous/deferred membership authority) on operator-visible surfaces","description":"Context: the 2026-07-30 full index rebuild (41,363 raws / 99GB, 4h20m,\npromoted as gen-1785377665711-06297b00) left 3,884 raws genuinely\nunmaterialized (parsed_at_ms IS NULL, no materialized logical_source_key\nsibling). Root-caused via read-only reflink-copy probe against source.db +\nsymlinked blob dir (never touched the live archive):\n\n- select_rebuild_raw_ids/all_index_rebuild_raw_ids/next_raw_page (rebuild_index.py,\n storage/index_generation.py:431) DO enumerate every raw unconditionally --\n scheduling is not the bug.\n- ~3,728 of the 3,884 are raws whose full-byte cohort was NOT a unique\n byte-prefix chain, so replace_raw_membership_census(...,\n retire_full_revision_governance=True) (storage/sqlite/archive_tiers/archive.py:2666)\n moved them to semantic membership governance, and\n classify_membership_revisions (archive/session_revision_membership.py:29)\n then correctly refused to pick a winner (no strict hash-domination, e.g.\n aistudio-drive Gemini/Drive re-scrapes where message_count is identical\n but attachment/event hashes diverge non-monotonically -- same message\n content, differently-encoded/refetched attachments).\n- This IS recorded: raw_session_memberships.decision='ambiguous' and a\n raw_authority_plans/raw_authority_blockers row\n (storage/sqlite/archive_tiers/archive.py:3760-3790, decisions dict) --\n but raw_sessions.parse_error stays NULL, so grepping the one column an\n operator would naturally check finds nothing. No CLI/devtools surface\n summarizes \"N raws are durable authority debt, here is why\" in one place;\n discovering this required manual cross-referencing of raw_sessions,\n raw_session_memberships, and raw_authority_blockers by hand.\n- Confirmed via a probe-archive reflink copy that re-parsing works fine\n (3,869/3,875 parse cleanly with the production parser incl.\n parse_retained_raw_sessions); only 6 unknown-export raws throw\n JSONDecodeError (genuinely corrupt/truncated, likely the known\n pre-#2823 Chrome-truncation captures).\n- Residue breakdown of the 3,884: ~3,294 retired-full-cohort ambiguous +\n ~434 multi-session-unknown ambiguous (both need real frontier judgment,\n `polylogue ops maintenance raw-authority-frontier --apply-plan --yes`,\n not automation -- picking a side would violate the \"never silently choose\n between branches\" invariant) + 83 non_session (legitimately empty parse,\n nothing to materialize) + 64 append-fragments blocked behind a\n quarantined head (auto-resolves once/if the head cohort is judged) + 6\n genuinely corrupt (unparseable).\n\nProposed fix: a devtools/CLI surface (e.g.\n`polylogue ops maintenance raw-authority-debt-summary` or an addition to\n`raw_materialization_replay_backlog`, storage/repair.py:4583) that joins\nraw_sessions + raw_session_memberships.decision + raw_authority_blockers\ninto one counted, origin-bucketed summary (\"ambiguous: N, non_session: N,\nappend-blocked: N, corrupt: N, resource-blocked: N\") so a future rebuild's\ncompletion receipt (or a post-rebuild doctor check) can print this instead\nof requiring hand cross-referencing three tables.\n\nSeparately (already fixed live, no code change needed): two pre-existing\nstale-plan raw-authority blockers\n(raw-authority-blocker:79ce004f... and ...1b51ed69...) were fail-closing\nrepair_raw_materialization ARCHIVE-WIDE (storage/repair.py:6151\nunresolved_raw_replay_blockers gate). Resolved both via\n`polylogue ops maintenance raw-authority-blocker-resolve --yes` (the\nexisting, safe, no-judgment-required stale-plan resolution path) so the\ndaemon's ordinary convergence loop is unblocked for any future\nnon-ambiguous backlog. Made zero difference to the 3,884 (confirmed\nunchanged before/after), since virtually all of it is genuinely-ambiguous\nauthority debt, not stale-plan debt.","notes":"CORRECTION 2026-07-30 (see polylogue-bu1i): this bead's root-cause paragraph describes the aistudio-drive residue as 'same message content, differently-encoded/refetched attachments' and treats it as genuine ambiguity needing operator judgment. That framing is wrong for aistudio-drive, verified on all 157 two-member cohorts: the pairs are byte-identical documents differing only by the injected _polylogue_drive_live_bytes_b64 attachment payload, so the later revision is a strict fidelity upgrade with nothing to judge. 151/151 drive cohorts, 100%. The classifier calls them ambiguous only because _attachment_hash_payload folds acquisition state (inline_content_hash, size_bytes) into attachment identity, making the enriched revision's attachment_hashes disjoint from -- rather than a superset of -- the bare one's.\n\nThis bead's own ask (surface WHY a raw failed to materialize) remains valid and is unaffected. What changes is the expected residue after bu1i lands: the ~3,294 'needs real frontier judgment' figure is an overcount by at least the drive share, and the same equal-message-count shape covers 566/587 claude-ai-export and 128/136 chatgpt-export cohorts, which need their own per-origin verification before being counted as judgment debt.","status":"open","priority":2,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T07:25:36Z","created_by":"Sinity","updated_at":"2026-07-30T11:35:00Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-9soj","title":"Selective index deferral for bulk index rebuild (scoped follow-up to polylogue-623q)","description":"polylogue-623q found the single SQLite writer (apply_s) dominates full-corpus rebuild wall-clock (54-77%). The most obvious lever -- build all secondary B-tree indexes AFTER the bulk insert instead of maintaining them during -- was measured via a real 1.2GB/1298-raw subset benchmark (drop all 72 non-unique CREATE INDEX statements before backfill_historical_revision_evidence, recreate after) and REJECTED: apply_s got WORSE (187.6s -\u003e 317.7s incl. rebuild, +69%). Root cause: write_parsed_session_to_archive's per-session 'full replace' path (_clear_session_projection_rows + DELETE FROM messages WHERE session_id=?) issues point-DELETEs against session_id on ~14 tables (messages, blocks, action_pairs, session_events, session_links, attachment_refs, paste_spans, session_provider_usage_events, session_agent_policies, session_working_dirs, session_repos, session_commits, session_model_usage, session_refs) for EVERY replayed session, even on a bulk-build-from-empty generation where every DELETE matches zero rows. Without indexes, each of those becomes an O(table_size) full scan instead of an O(log n) point lookup -- clear_projection_rows alone went 8.3s -\u003e 57.9s (7x) in the sample. A SELECTIVE variant is the real follow-up: keep the session_id/src_session_id-scoped indexes each full-replace DELETE needs (idx_messages_session_position, idx_blocks_session_position, and equivalents on the other ~12 tables -- audit which currently HAVE a session-scoped index at all), defer only the remaining query-serving indexes (role/type/tool/content-hash/profile/latency/rollup indexes -- roughly 50-60 of the 72) that full_replace never touches. Requires: (1) auditing all 72 non-unique CREATE INDEX statements in storage/sqlite/archive_tiers/index.py against the ~14-table DELETE cascade in _clear_session_projection_rows + the direct 'DELETE FROM messages'/'DELETE FROM blocks' calls to classify safe-to-defer vs must-keep, (2) splitting INDEX_DDL into an eager (tables + must-keep indexes) and deferred (query-serving indexes) script, (3) threading a defer flag through IndexGenerationStore.create()/create_transaction() -\u003e initialize_archive_database(..., defer_secondary_indexes=True) for the offline-rebuild caller only, (4) a new terminal stage in maintenance/rebuild_index.py that creates the deferred indexes once, before _repopulate_bulk_build_derived_state (which itself reads blocks/messages and likely benefits from indexes existing already). Re-measure on the same subset methodology (build_subset.py-style real corpus copy) before shipping -- do not ship on theory.","status":"open","priority":2,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-29T21:02:13Z","created_by":"Sinity","updated_at":"2026-07-29T21:02:13Z","dependency_count":0,"dependent_count":0,"comment_count":0} From 37e30b9ef04462fd98664fba82c25c5c00c0f1c9 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 30 Jul 2026 17:31:17 +0200 Subject: [PATCH 7/8] fix(archive): repair four identity-invariant defects in aggz's revision relation Problem: PR #3401 review (chatgpt-codex-connector) found the content-only comparison relation from polylogue-aggz did not yet fully hold the invariant it claims in four concrete ways. What changed: - session_revision_membership.py: equal-content collapse now decides its representative by source authority (direct export over browser capture, native over DOM) BEFORE falling back to provider timestamp / raw_id, via new _equal_content_representative -- mirrors the ordering _direct_export_precedence/_browser_snapshot_dominates already apply to the non-equal growth-chain case, so equal content no longer lets a lower- fidelity capture supersede its authoritative export (P1). - session_revision_membership.py: _axis_relation now groups content hashes by identity into a set (_content_by_identity) instead of a plain dict, so two colliding attachment identities (same message/name/mime, different bytes) retain every content hash instead of one silently overwriting the other -- a real collision now degrades to conflict, never to equal (P2). - pipeline/ids.py: an event's canonical identity is now always hash(base_identity, content), never conditionally base_identity alone depending on whether a sibling happens to share that base identity in the SAME revision -- an item's identity must not depend on what else is in the set (P2, the most serious of the four: it reintroduced exactly the bug class aggz exists to eliminate). - pipeline/ids.py: the generation_lifecycle duration-stripping allowlist now applies only when the event's own payload declares duration_semantics == "provider_reported_elapsed" (the ChatGPT-parser shape polylogue-nuec targeted), not to every event of that type -- the browser-capture parser's own DOM/UI generation observations (different duration_semantics values) keep their observation id, timestamps, and duration/label/trigger fields as real content (P2). Wires up the previously-dead _PROVIDER_REPORTED_ELAPSED_MARKER_KEY/_VALUE constants. Not touched, per explicit instruction: _provider_ordered_browser_snapshots stays (kept, not deleted); _maximal_evidence_fallback stays designed/tested but unwired (archive.py write-back invariant, out of this lane's scope). Verification: devtools test tests/unit/archive/test_session_revision_membership.py tests/unit/pipeline/test_pipeline_ids.py tests/unit/sources/test_revision_backfill.py tests/unit/storage/test_revision_replay.py tests/unit/storage/test_browser_capture_origin_repair.py -- 133 passed. Ref polylogue-aggz Co-Authored-By: Claude --- .../archive/session_revision_membership.py | 97 ++++++++++++++----- polylogue/pipeline/ids.py | 65 +++++++++---- 2 files changed, 118 insertions(+), 44 deletions(-) diff --git a/polylogue/archive/session_revision_membership.py b/polylogue/archive/session_revision_membership.py index 608d6b78c2..d0e3c503d1 100644 --- a/polylogue/archive/session_revision_membership.py +++ b/polylogue/archive/session_revision_membership.py @@ -62,6 +62,25 @@ def _identities(contents: frozenset[tuple[bytes, bytes]]) -> frozenset[bytes]: return frozenset(identity for identity, _content in contents) +def _content_by_identity(contents: frozenset[tuple[bytes, bytes]]) -> dict[bytes, frozenset[bytes]]: + """Group content hashes by identity, retaining EVERY value under a colliding identity. + + Identity is not always injective: two acquired attachments on one + message can share one identity (same ``message_id``/``name``/ + ``mime_type``, different bytes -- the accepted limit documented on + ``_ATTACHMENT_IDENTITY_FIELDS``, not a new one). Collapsing such a + collision to a single arbitrary content hash (a plain ``dict``, last + value wins) would let a real content conflict compare as equal instead. + Grouping into a set per identity means a collision always degrades to + ``conflict`` when compared against a revision that disagrees, never to + ``equal``. + """ + grouped: dict[bytes, set[bytes]] = {} + for identity, content in contents: + grouped.setdefault(identity, set()).add(content) + return {identity: frozenset(values) for identity, values in grouped.items()} + + def _axis_relation( identities_a: frozenset[bytes], contents_a: frozenset[tuple[bytes, bytes]], @@ -77,8 +96,8 @@ def _axis_relation( events every identity always carries content, so this degrades to plain set equality/containment for those two axes. """ - content_a = dict(contents_a) - content_b = dict(contents_b) + content_a = _content_by_identity(contents_a) + content_b = _content_by_identity(contents_b) shared = identities_a & identities_b for identity in shared: value_a, value_b = content_a.get(identity), content_b.get(identity) @@ -146,6 +165,58 @@ def _frontier(projection: SessionRevisionProjection) -> tuple[int, int, int, int ) +def _equal_content_representative( + incumbent: MembershipRevision, candidate: MembershipRevision +) -> tuple[MembershipRevision, MembershipRevision]: + """Pick (winner, loser) between two revisions already proven ``equal`` in content. + + Equal content is not equal authority: a direct/native export and a + browser DOM scrape can project to byte-identical messages while one is + first-class provenance and the other is a lossy capture, and which one + survives as the representative determines what future re-acquisitions + are compared against. Source authority is therefore decided BEFORE any + timestamp tiebreak -- mirroring, for the equal case, the same + ``_direct_export_precedence`` / browser-fidelity ordering + (``_browser_snapshot_dominates``) already applied to the non-equal + growth-chain case below: a direct export always outranks a + browser-capture sibling, and a native browser snapshot always outranks a + DOM snapshot of the same underlying session. Only when neither + revision's provenance outranks the other's does provider timestamp (and + finally a stable raw_id) decide, exactly as before this function + existed. + """ + incumbent_direct = incumbent.browser_snapshot_fidelity is None + candidate_direct = candidate.browser_snapshot_fidelity is None + if incumbent_direct != candidate_direct: + return (incumbent, candidate) if incumbent_direct else (candidate, incumbent) + if ( + not incumbent_direct + and not candidate_direct + and incumbent.browser_snapshot_fidelity != candidate.browser_snapshot_fidelity + ): + return (incumbent, candidate) if incumbent.browser_snapshot_fidelity == "native" else (candidate, incumbent) + incumbent_time = parse_timestamp(incumbent.provider_updated_at) + candidate_time = parse_timestamp(candidate.provider_updated_at) + if ( + incumbent_time is not None + and candidate_time is not None + and candidate_time.timestamp() != incumbent_time.timestamp() + ): + return ( + (candidate, incumbent) + if candidate_time.timestamp() > incumbent_time.timestamp() + else (incumbent, candidate) + ) + # No distinguishing provenance or provider timestamp -- these two are + # already proven identical content, so which raw_id represents them does + # not matter for correctness; pick deterministically rather than + # requiring a timestamp that a re-export of an untouched conversation + # may never carry (its own provider updated_at legitimately does not + # move when nothing provider-visible changed). + winner, loser = sorted((incumbent, candidate), key=lambda item: item.raw_id) + return winner, loser + + def classify_membership_revisions(revisions: list[MembershipRevision]) -> MembershipClassification: """Accept one total growth chain by set containment; never choose a branch silently. @@ -182,27 +253,7 @@ def classify_membership_revisions(revisions: list[MembershipRevision]) -> Member representatives.append(revision) continue incumbent = representatives[match_index] - incumbent_time = parse_timestamp(incumbent.provider_updated_at) - candidate_time = parse_timestamp(revision.provider_updated_at) - if ( - incumbent_time is not None - and candidate_time is not None - and candidate_time.timestamp() != incumbent_time.timestamp() - ): - winner, loser = ( - (revision, incumbent) - if candidate_time.timestamp() > incumbent_time.timestamp() - else (incumbent, revision) - ) - else: - # No distinguishing provider timestamp -- these two are already - # proven identical content, so which raw_id represents them does - # not matter for correctness; pick deterministically rather than - # requiring a timestamp that a re-export of an untouched - # conversation may never carry (its own provider updated_at - # legitimately does not move when nothing provider-visible - # changed). - winner, loser = sorted((incumbent, revision), key=lambda item: item.raw_id) + winner, loser = _equal_content_representative(incumbent, revision) representatives[match_index] = winner equivalents.append(loser.raw_id) diff --git a/polylogue/pipeline/ids.py b/polylogue/pipeline/ids.py index 208b08af3c..a34fcdb421 100644 --- a/polylogue/pipeline/ids.py +++ b/polylogue/pipeline/ids.py @@ -3,7 +3,6 @@ from __future__ import annotations import unicodedata -from collections import Counter from collections.abc import Mapping from dataclasses import dataclass from typing import TYPE_CHECKING, TypeAlias @@ -325,9 +324,25 @@ def _event_content_payload(event: ParsedSessionEvent) -> dict[str, JSONValue]: messages, on a different axis (polylogue-nuec). This never includes ``event_index``; ``session_revision_projection`` builds identity purely from this content plus the event's own type and anchoring message. + + The allowlist strip is narrowed to the specific provider-remeasured + shape it targets: it applies only when the event's own payload declares + itself non-durable via ``duration_semantics == + "provider_reported_elapsed"`` (set by the ChatGPT parser). The + browser-capture parser emits the SAME ``event_type`` for its own DOM/UI + generation observations, tagged with a different ``duration_semantics`` + (e.g. ``dom_observed_wall``, ``provider_ui_elapsed``) -- those are a real + first-party measurement this projection has no other record of, not the + re-derived-on-every-export ChatGPT value nuec exists for, so their + observation id, timestamp, and duration/label/trigger fields remain + content rather than being silently stripped. """ allowlist = _EVENT_CONTENT_PAYLOAD_ALLOWLIST.get(event.event_type) - if allowlist is None: + provider_reported_elapsed = ( + allowlist is not None + and event.payload.get(_PROVIDER_REPORTED_ELAPSED_MARKER_KEY) == _PROVIDER_REPORTED_ELAPSED_MARKER_VALUE + ) + if allowlist is None or not provider_reported_elapsed: payload = event.payload timestamp = event.timestamp else: @@ -348,12 +363,14 @@ def _event_content_payload(event: ParsedSessionEvent) -> dict[str, JSONValue]: #: The subset of an event content payload that answers *which event slot is #: this*, as opposed to *what does it say*: anchoring message plus event -#: type, content-derived and never the array index. Ambiguous within one -#: revision when more than one event shares both (e.g. multiple -#: ``chatgpt_block_metadata`` events on the same message, one per block) -- -#: ``session_revision_projection`` folds the event's own content hash into -#: identity for those specific events, which is STILL content-derived (each -#: block's own content, including any content-intrinsic field such as +#: type, content-derived and never the array index. Can be shared by more +#: than one event within one revision (e.g. multiple +#: ``chatgpt_block_metadata`` events on the same message, one per block), so +#: ``session_revision_projection`` always folds the event's own content hash +#: into the FINAL identity on top of this base -- unconditionally, not only +#: when a sibling is present in that particular revision, so identity never +#: depends on what else happens to be in the set. Still content-derived +#: (each block's own content, including any content-intrinsic field such as #: ``block_index``, already differs), never the array position. _EVENT_BASE_IDENTITY_FIELDS = ("event_type", "source_message_provider_id") @@ -513,21 +530,27 @@ def session_revision_projection(convo: ParsedSession) -> SessionRevisionProjecti content_payload = _event_content_payload(event) event_base_identities.append(bytes.fromhex(hash_payload(_event_base_identity_payload(content_payload)))) event_content_hashes.append(bytes.fromhex(hash_payload(content_payload))) - base_identity_counts = Counter(event_base_identities) event_contents: set[tuple[bytes, bytes]] = set() for base_identity, content_hash in zip(event_base_identities, event_content_hashes, strict=True): - # A base identity (event type + anchoring message) ambiguous within - # this revision -- more than one event shares it, e.g. one - # chatgpt_block_metadata event per block on a message -- folds the - # event's own content into identity to disambiguate. Still - # content-derived, never the array index: distinct blocks already - # differ in content (e.g. a content-intrinsic block_index), and - # events that are genuine duplicates (same base identity, same - # content) correctly collapse to one set entry either way. - canonical_identity = ( - base_identity - if base_identity_counts[base_identity] == 1 - else bytes.fromhex(hash_payload({"base_identity": base_identity.hex(), "content": content_hash.hex()})) + # A base identity (event type + anchoring message) is ambiguous + # whenever it is EVER possible for more than one event to share it + # (e.g. one chatgpt_block_metadata event per block on a message), so + # the event's own content is always folded into identity here -- + # unconditionally, not only when a sibling happens to be present in + # THIS revision. An item's identity must not depend on what else is + # in the set: computing it from this revision's own sibling count + # made the same event's identity shift between `base_identity` (one + # instance) and `hash(base_identity, content)` (two or more) purely + # because a sibling appeared in a later revision, which made an + # ordinary event-growth revision compare as a disjoint conflict + # instead of containment. Folding content in always keeps identity + # intrinsic to the event itself: still content-derived, never the + # array index (distinct blocks already differ in content, e.g. a + # content-intrinsic block_index), and true duplicates (same base + # identity, same content, whether or not any sibling exists) + # correctly collapse to one set entry either way. + canonical_identity = bytes.fromhex( + hash_payload({"base_identity": base_identity.hex(), "content": content_hash.hex()}) ) event_contents.add((canonical_identity, content_hash)) return SessionRevisionProjection( From 197a9fdfab23ef00eebd0824b4b3f15818be6e5d Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 30 Jul 2026 17:41:39 +0200 Subject: [PATCH 8/8] test(archive): cover the four aggz identity-invariant defects with mutation-proof tests Adds a regression test per review finding, each verified by manually mutating the corresponding production line back to the buggy behavior and observing the exact expected failure (reverted afterward, not part of this commit): - P1 (source authority in equal-content collapse): two tests where the direct/native revision loses on BOTH timestamp and raw_id tiebreaks, so only source-authority ordering can make it win. Mutating out the new authority-first branches in _equal_content_representative flips both assertions (raw-zzzz-dom/raw-aaaa-capture win instead of the authoritative revision). - P2 (attachment identity collision): a direct unit test on _content_by_identity proving type-mismatch-guaranteed failure against the old dict()-based grouping, plus an end-to-end test asserting two acquired attachments colliding on identity with different bytes classify as conflict. Reverting _content_by_identity's call site to dict() flips the end-to-end assertion to "equal". - P2 (event identity sibling instability): a growth-chain test (one block event, then a second appended) asserting b_contains_a and full containment. Reintroducing the base_identity_counts-conditional identity computation flips this to a set-subset assertion failure (disjoint identities). - P2 (duration-stripping scope): a browser-capture generation_lifecycle test with differing observation ids/durations but IDENTICAL state/evidence_source/fidelity, asserting conflict. Removing the duration_semantics marker gate flips this to "equal" (both collapse to the same stripped payload). Verification: devtools test tests/unit/archive/test_session_revision_membership.py tests/unit/pipeline/test_pipeline_ids.py tests/unit/sources/test_revision_backfill.py tests/unit/storage/test_revision_replay.py tests/unit/storage/test_browser_capture_origin_repair.py -- 139 passed. devtools verify --quick -- exit_code 0. Ref polylogue-aggz Co-Authored-By: Claude --- .../test_session_revision_membership.py | 222 ++++++++++++++++++ 1 file changed, 222 insertions(+) diff --git a/tests/unit/archive/test_session_revision_membership.py b/tests/unit/archive/test_session_revision_membership.py index 4cd021c009..38d06fec2f 100644 --- a/tests/unit/archive/test_session_revision_membership.py +++ b/tests/unit/archive/test_session_revision_membership.py @@ -5,6 +5,7 @@ from polylogue.archive.message.roles import Role from polylogue.archive.session_revision_membership import ( MembershipRevision, + _content_by_identity, _maximal_evidence_fallback, _relation, classify_membership_revisions, @@ -387,6 +388,70 @@ def test_refuses_to_guess_when_two_distinct_attachments_share_message_name_and_t assert a.projection.attachment_identities == b.projection.attachment_identities +def _colliding_attachments_revision(raw_id: str, *contents: bytes) -> MembershipRevision: + """A session whose attachments all share (message, name, mime) but carry distinct bytes.""" + attachments = [ + ParsedAttachment( + provider_attachment_id=f"att-{i}", + message_provider_id="0", + name="screenshot.png", + mime_type="image/png", + size_bytes=len(content), + inline_bytes=content, + ) + for i, content in enumerate(contents) + ] + session = ParsedSession( + source_name=Provider.GEMINI, + provider_session_id="session", + messages=[ParsedMessage(provider_message_id="0", role=Role.USER, text="one")], + attachments=attachments, + ) + return MembershipRevision(raw_id, session_revision_projection(session)) + + +def test_content_by_identity_retains_every_value_under_a_colliding_identity() -> None: + """Grouping must be a set per identity, not a dict entry that silently overwrites. + + Directly exercises the review-found bug: a plain ``dict(contents)`` maps + a colliding identity to whichever (identity, content) pair happened to + be visited last, discarding the other content hash entirely. The fixed + grouping must retain BOTH. + """ + contents = frozenset({(b"identity-1", b"content-a"), (b"identity-1", b"content-b"), (b"identity-2", b"content-c")}) + + grouped = _content_by_identity(contents) + + assert grouped == { + b"identity-1": frozenset({b"content-a", b"content-b"}), + b"identity-2": frozenset({b"content-c"}), + } + + +def test_refuses_to_equate_colliding_attachment_identities_with_different_bytes() -> None: + """Two acquired attachments sharing one identity but different bytes on one + message must degrade to conflict, never silently compare as equal or + let the second collapse invisibly into the first. + + Reproduces the review-found bug where `_axis_relation` built the + identity->content mapping via plain `dict()`, so the second colliding + attachment's content hash silently overwrote the first -- a real + conflict could then compare as equal, strictly worse than either honest + outcome (equal or conflict), and beyond the documented limitation (which + only claims byte-LESS duplicates are indistinguishable). + """ + both = _colliding_attachments_revision("raw-both", b"left bytes", b"right bytes") + left_only = _colliding_attachments_revision("raw-left-only", b"left bytes") + + assert both.projection.attachment_identities == left_only.projection.attachment_identities + assert len(both.projection.attachment_contents) == 2 + assert _relation(both.projection, left_only.projection) == "conflict" + + result = classify_membership_revisions([both, left_only]) + assert result.accepted_raw_ids == () + assert result.ambiguous_raw_ids == ("raw-both", "raw-left-only") + + # --------------------------------------------------------------------------- # Events: order-insensitive, measurement-excluded set containment # (polylogue-nuec) @@ -650,6 +715,98 @@ def block_event(index: int) -> ParsedSessionEvent: assert _relation(older.projection, newer.projection) == "equal" +def test_event_identity_is_stable_when_a_sibling_appears_later() -> None: + """An event's identity must not depend on what else is in the set. + + Reproduces the review-found bug where an event's canonical identity + shifted from `base_identity` (unique within the revision) to + `hash(base_identity, content)` purely because a SECOND event later + shared its base identity -- so an ordinary event-growth revision (one + block event, then a second block appended in a later export) compared + as a disjoint conflict instead of containment, reintroducing exactly the + identity-instability bug class polylogue-aggz exists to eliminate. + """ + + def block_event(index: int) -> ParsedSessionEvent: + return ParsedSessionEvent( + event_type="chatgpt_block_metadata", + timestamp="1.0", + source_message_provider_id="0", + payload={"block_index": index}, + ) + + single_session = ParsedSession( + source_name=Provider.CHATGPT, + provider_session_id="session", + messages=[ParsedMessage(provider_message_id="0", role=Role.ASSISTANT, text="answer")], + session_events=[block_event(0)], + ) + grown_session = single_session.model_copy(update={"session_events": [block_event(0), block_event(1)]}) + + older = MembershipRevision("raw-old", session_revision_projection(single_session)) + newer = MembershipRevision("raw-new", session_revision_projection(grown_session)) + + assert older.projection.event_contents <= newer.projection.event_contents + assert _relation(older.projection, newer.projection) == "b_contains_a" + + result = classify_membership_revisions([newer, older]) + assert result.accepted_raw_ids == ("raw-old", "raw-new") + assert result.ambiguous_raw_ids == () + + +def _browser_generation_lifecycle_revision( + raw_id: str, observation_id: str, wall_elapsed_ms: int +) -> MembershipRevision: + """A browser-capture-shaped session with one DOM-observed `generation_lifecycle` event.""" + session = ParsedSession( + source_name=Provider.CHATGPT, + provider_session_id="session", + messages=[ParsedMessage(provider_message_id="0", role=Role.ASSISTANT, text="answer")], + session_events=[ + ParsedSessionEvent( + event_type="generation_lifecycle", + timestamp="2026-01-01T00:00:00Z", + source_message_provider_id="0", + payload={ + "observation_id": observation_id, + "state": "completed", + "evidence_source": "dom_observation", + "fidelity": "observed", + "duration_semantics": "dom_observed_wall", + "wall_elapsed_ms": wall_elapsed_ms, + }, + ) + ], + ) + return MembershipRevision(raw_id, session_revision_projection(session)) + + +def test_browser_observed_generation_lifecycle_duration_is_real_content_not_stripped() -> None: + """The duration-stripping allowlist targets ChatGPT's own re-derived + measurement (`duration_semantics == "provider_reported_elapsed"`), not + every `generation_lifecycle` event regardless of source. + + Reproduces the review-found bug where the allowlist strip was keyed on + `event_type` alone, so it also stripped browser-capture's own DOM/UI + generation observations -- this projection's only record of a real + first-party measurement, not the provider-remeasured-on-every-export + value polylogue-nuec targeted. Two observations with different + observation ids and wall-clock durations, but IDENTICAL + state/evidence_source/fidelity (the fields the allowlist would keep), + must compare as a genuine conflict, not equivalent -- if the allowlist + strip fired here, both would collapse to the same stripped payload and + compare equal. + """ + left = _browser_generation_lifecycle_revision("raw-a", "obs-1", 4200) + right = _browser_generation_lifecycle_revision("raw-b", "obs-2", 9100) + + assert _relation(left.projection, right.projection) == "conflict" + + result = classify_membership_revisions([left, right]) + assert result.accepted_raw_ids == () + assert result.ambiguous_raw_ids == ("raw-a", "raw-b") + + # --------------------------------------------------------------------------- # Cross-axis fork detection # --------------------------------------------------------------------------- @@ -833,6 +990,71 @@ def test_direct_export_outranks_browser_capture_siblings_regardless_of_growth() assert result.ambiguous_raw_ids == () +def test_equal_content_collapse_prefers_direct_export_over_browser_capture() -> None: + """Equal content is not equal authority: a direct export must survive over + a browser capture that happens to project to identical content, even + when timestamp and raw_id both favor the capture. + + Reproduces the review-found bug where the representative for two + revisions already proven `equal` was picked by provider timestamp (and, + failing that, raw_id) alone, before any source-authority ordering ran -- + so a later-timestamped, lexically-earlier-raw_id browser DOM scrape + could supersede an earlier direct/native export projecting to the exact + same content, even though a direct export always outranks a browser + capture for authority (`_direct_export_precedence`). + """ + export_projection = _revision("raw-export", "one", "two").projection + capture_projection = _revision("raw-capture", "one", "two").projection + export = MembershipRevision( + "raw-zzzz-export", export_projection, provider_updated_at="2024-01-01T00:00:00Z", browser_snapshot_fidelity=None + ) + capture = MembershipRevision( + "raw-aaaa-capture", + capture_projection, + provider_updated_at="2024-06-01T00:00:00Z", + browser_snapshot_fidelity="dom", + ) + + assert _relation(export.projection, capture.projection) == "equal" + + # Both the (later) timestamp and the (lexically smaller) raw_id favor + # the capture -- only source-authority ordering can make the export win. + result = classify_membership_revisions([export, capture]) + + assert result.accepted_raw_ids == ("raw-zzzz-export",) + assert result.equivalent_raw_ids == ("raw-aaaa-capture",) + assert result.ambiguous_raw_ids == () + + +def test_equal_content_collapse_prefers_native_snapshot_over_dom_snapshot() -> None: + """Among two equal-content browser captures, native outranks DOM. + + Same defect as the direct-export case above, one authority tier down: + fidelity ordering must also run before the timestamp/raw_id tiebreak. + """ + dom_projection = _revision("raw-dom", "one").projection + native_projection = _revision("raw-native", "one").projection + dom = MembershipRevision( + "raw-zzzz-dom", dom_projection, provider_updated_at="2024-06-01T00:00:00Z", browser_snapshot_fidelity="dom" + ) + native = MembershipRevision( + "raw-aaaa-native", + native_projection, + provider_updated_at="2024-01-01T00:00:00Z", + browser_snapshot_fidelity="native", + ) + + assert _relation(dom.projection, native.projection) == "equal" + + # Both the (later) timestamp and the (lexically smaller) raw_id favor + # the dom snapshot -- only fidelity ordering can make native win. + result = classify_membership_revisions([dom, native]) + + assert result.accepted_raw_ids == ("raw-aaaa-native",) + assert result.equivalent_raw_ids == ("raw-zzzz-dom",) + assert result.ambiguous_raw_ids == () + + def test_browser_snapshot_accepts_later_attachment_enrichment_without_provider_update() -> None: older = _revision("raw-old", "prompt", "answer") newer = _revision("raw-new", "prompt", "answer")