diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 28b33d4f62..254dc9c6d8 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -72,12 +72,12 @@ {"_type":"issue","id":"polylogue-tf2","title":"Campaign: agent-forensics regeneration + all-provider repricing","description":"Regenerate the agent-forensics packet on the current archive with an honest all-provider headline. The 2026-06-27 report (546.6B tokens, $89,368 API-list equivalent, 216x cache amplification) is the most stranger-legible artifact on any shelf, but its numbers are pre-dedup stale and the headline prices only the priced-provenance subset (Claude Code cost_usd rows); Codex/ChatGPT/Gemini are origin_reported token counts with no dollar value (operator estimate ~$150K all-provider). Sequenced after claim-vs-evidence per operator direction 2026-07-02.","design":"Current slice design: turn the existing agent-forensics/cost headline into a product-backed all-provider repricing artifact. First inspect devtools/scripts and polylogue analyze surfaces for agent_forensics/cost code. Use active archive usage headline (detail=headline) for authoritative physical_session and logical_session_model_high_water token totals. Keep priced-provenance dollars and origin-reported token estimates separate: do not multiply every token by one blended price without a labeled lane. Add or reuse a shared pricing/projection helper so the demo artifact is regenerated from Polylogue product code, not ad hoc SQL. Acceptance for this slice: the generated agent-forensics artifact names archive root/schema, includes physical vs logical token grain, separates priced subset from origin-reported estimate lanes, gives reproduction commands, and has focused tests for any new repricing helper/surface.","acceptance_criteria":"Terminal state: regenerated forensics packet on the current archive with an honest all-provider headline (priced subset AND origin-reported estimate lanes separated), agent_forensics.py folded into polylogue analyze (tf2.2), artifact on the demo shelf with reproduction commands, cold-reader gate passed. Epic closes only when that artifact is recorded.","status":"closed","priority":0,"issue_type":"epic","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-07-03T04:31:32Z","created_by":"Sinity","updated_at":"2026-07-03T19:06:44Z","started_at":"2026-07-03T18:47:23Z","closed_at":"2026-07-03T19:06:44Z","close_reason":"Completed: provider usage headline now exposes product-backed pricing lanes in polylogue analyze usage --detail headline, separating stored/provider-priced cost from catalog API-equivalent estimates for origin_reported rows. Regenerated the current .agent/demos/agent-forensics artifact against /home/sinity/.local/share/polylogue schema v23: physical-session tokens 395,320,980,423; logical high-water tokens 288,741,229,728; stored/provider-priced USD 243,392.189328; catalog API-equivalent USD 337,565.031618; priced lane 13,889 rows / 12,331 sessions / 12,650 matched rows; origin_reported lane 2,308 rows / 2,270 sessions / 2,302 matched rows. Verification: live polylogue --plain analyze usage --detail headline --format json --limit 0 wrote /realm/tmp/polylogue-usage-headline-pricing-current.json; devtools test tests/unit/storage/test_provider_usage_report.py tests/unit/cli/test_diagnostics.py passed 23 tests; devtools verify --quick passed run 20260703T190553Z-quick-2226137-d91d4e8f; devtools workspace demo-shelf --json reported ok. Non-claim preserved: this is not final billing reconciliation and physical/logical token grains stay explicitly separated.","labels":["area:usage","campaign","size:M","spine"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-sru","title":"Campaign: claim-vs-evidence report to finding-grade","description":"Terminal state: an externally publishable finding ('how often do coding agents proceed past failed tool calls, by model/tool') with stated sample frame, calibrated markers, benign/consequential split, seeded stranger-runnable reproduction, and a passed cold-reader gate. Slice closure is NOT campaign closure; this epic stays top-of-frame until its terminal state is recorded.\\n\\nState as of 2026-07-03 after calibrated active-archive regeneration: archive root /home/sinity/.local/share/polylogue, index schema v23, 41,886 structured failures total, 5,000 origin-stratified failures inspected (3,746 claude-code-session, 1,247 codex-session, 7 claude-ai-export), 100 unpaired structured failures. Marker vocabulary was tightened to avoid broad issue/fix/block/gitignored false positives. Immediate next-turn totals: acknowledged=420, silent_proceed=1,205, ambiguous=3,375 (2,624 wordless tool continuations; 751 prose without marker). Lower-bound silent rate is 24.1%; among classified immediate next turns, silent rate is 74.2%. Next-3 sensitivity window, stopping before the next user message, finds 302 acknowledgments that appear only after the next turn; window3 silent lower bound is 37.0%. Calibration: 50 hand-labeled immediate-next-turn rows, acknowledged-marker precision=1.0, recall=0.8421052631578947, invalid rows=0. Artifact: .agent/demos/claim-vs-evidence/claim-vs-evidence.report.json.","notes":"2026-07-03 update: methodology package is now cold-read gated. .agent/demos/claim-vs-evidence contains aggregate live evidence, public-summary.json, PUBLIC_REPRODUCTION.md, COLD_READER_GATE.md, and COLD_READ_RESULT.md. Seeded reproduction is meaningful, not empty: 4 structured failures, 2 acknowledged follow-ups, 2 silent-proceed follow-ups, 0 unpaired. Cold-reader subagent PASS recovered claim/non-claim, sample frame, rates, calibration, caveats, and reproduction commands from the artifact directory only. Remaining campaign child: polylogue-sru.1 productizes action-unit outcome/followup_class capability.","status":"closed","priority":0,"issue_type":"epic","owner":"ezo.dev@gmail.com","created_at":"2026-07-03T04:31:26Z","created_by":"Sinity","updated_at":"2026-07-03T09:28:09Z","closed_at":"2026-07-03T09:28:09Z","close_reason":"Completed: all seven campaign children are closed. The claim-vs-evidence finding now has bounded sample-frame reporting, calibrated marker precision/recall, handler-class and next-3 sensitivity splits, meaningful seeded reproduction, cold-reader PASS, and productized action-unit followup_class/followup_message_ref query capability. Current artifact lives under .agent/demos/claim-vs-evidence and was regenerated against /home/sinity/.local/share/polylogue schema v23.","labels":["area:substrate","campaign"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-f1vg","title":"Corpus acceptance gate: no absences and maximum fidelity, with the 2026-07-30 frozen baseline","description":"## What the operator asked for\n\n\"Ensure max fidelity as well as no absences, through the entire corpus.\" That is\na stronger bar than any existing check enforces, and nothing measured either\nhalf until now.\n\n## Why the existing checks cannot serve\n\n`verify-archive`'s `source-index-coverage` counts superseded revisions as\nmissing work (polylogue-ey3r), so it cannot reach zero on any archive that ever\ningested a conversation twice, and therefore cannot gate a rebuild. Nothing at\nall measures fidelity: an archive can report perfect coverage while every\nattachment whose bytes it holds is recorded `unfetched`, which is precisely the\nstate measured on 2026-07-30.\n\n## The gate\n\n`.agent/scripts/corpus-fidelity-audit.py` (read-only, `mode=ro` throughout,\nexits 1 on failure so it can gate a rebuild). Three measures:\n\n1. **Absences** -- logical documents (origin + provider_session_id) the archive\n holds evidence for but does not surface, bucketed by cause so a fix's effect\n is attributable rather than a single number moving for unknown reasons.\n2. **Attachment fidelity** -- acquired vs not-acquired refs, split by origin and\n upload_origin, because a Drive-hosted reference never fetched is actionable\n while a genuinely byte-less attachment kind is not.\n3. **Revision fidelity** -- documents whose indexed evidence is smaller than the\n largest revision recorded for them.\n\n## Baseline, live archive frozen 2026-07-30 (daemon stopped)\n\n ABSENCES 1,009 of 18,248 known documents\n 587 claude-ai-export/ambiguous-only\n 184 claude-code-session/ambiguous-only\n 135 chatgpt-export/ambiguous-only\n 71 aistudio-drive/settled-yet-absent\n 20 unknown-export/settled-yet-absent\n 12 gemini-cli / hermes / unknown / grok / codex\n\n ATTACHMENT FIDELITY acquired=2,118 not-acquired=7,655\n 3,684 chatgpt-export/oauth/unfetched\n 2,391 chatgpt-export/\u003cnone\u003e/unfetched\n 1,975 aistudio-drive/drive/acquired\n 1,119 aistudio-drive/drive/unfetched\n 398 claude-ai-export/oauth/unfetched\n\n REVISION FIDELITY 94 documents below best recorded evidence\n 76 hermes-session\n 16 claude-code-session\n 2 chatgpt-export\n\n VERDICT: FAIL\n\nThe `settled-yet-absent` buckets (71 drive, 20 unknown-export, 1 codex) are not\nexplained by any currently-tracked cause and want their own investigation --\nthese are documents with no ambiguous decision anywhere that are nonetheless\nmissing.\n\nThe 94 revision-fidelity documents are a residue after correcting a false\npositive, and should be treated as a prompt to investigate rather than proof of\nloss (see below).\n\n## Measurement trap this already caught\n\nThe first version compared indexed *messages* against\n`raw_session_memberships.message_count` and reported **474** shortfalls, 294 of\nthem codex-session. All false. `message_count` was recorded by whichever parser\ncensused that raw, and index v46 deliberately reclassified a large share of\nCodex/Claude Code rows from chat turns into typed `session_events`. One codex\nsession read as \"15 indexed vs 68,553 recorded\" when it actually holds 15\nmessages plus 84,612 events. Counting `messages + session_events` drops the\nfigure to 94.\n\nAnyone extending this must keep that in mind: cross-generation counts are only\napproximately comparable, so a metric built on them needs its assumption stated\nand checked against a real sample before its number is believed.\n\n## Follow-up\n\nPromote this into `devtools` as a first-class command with a `CommandSpec` (plus\n`devtools render devtools-reference`) so it is an enforced gate rather than a\nscript, and wire it into the post-rebuild acceptance path alongside\n`verify-archive`. Kept as a script for now because the fixes it measures are\nstill in flight and its thresholds will move as they land.\n\n## Acceptance criteria\n\n- Absences reach 0, or every residual is individually justified in writing.\n- Attachment refs marked not-acquired are either acquired or shown to be\n genuinely unfetchable (deleted upstream, over the size cap, byte-less kind).\n- Revision-fidelity residue is explained rather than merely small.\n","status":"open","priority":1,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T14:04:01Z","created_by":"Sinity","updated_at":"2026-07-30T14:04:01Z","labels":["area:ingest"],"dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"polylogue-d8al","title":"claude-ai-export: attachment real-id presence is inconsistent across export vintages, needs comparison-layer relaxation","description":"## What the data says\n\nCensus (full population, not a sample): replayed the production classifier\n(polylogue.sources.dispatch.parse_payload -\u003e session_revision_projection -\u003e\nclassify_membership_revisions) over all 566 claude-ai-export\nequal-message-count ambiguous cohorts in the live archive (read-only,\n/realm/db/polylogue), with polylogue-hith's parser-side fix (drop the\npositional-index seed for synthetic attachment ids) already applied.\n\n 566 claude-ai-export equal-message-count ambiguous cohorts (full census)\n 297 still ambiguous because message_hashes differ (polylogue-c429 /\n message-order-not-stable territory, or genuine content divergence)\n 268 still ambiguous with message_hashes EQUAL (0 content diffs) but\n attachment identity axis mismatched -- the exact shape hith\n targeted\n 0 of those 268 resolved by hith's fix\n 268 of those 268 are \"mixed real/synthetic\": one export vintage of the\n SAME conversation carries a real id (id/file_id/fileId/uuid/\n file_uuid) for an attachment; the OTHER vintage of the same\n conversation has no real id for the physically-same attachment and\n synthesizes one instead\n 0 are \"pure synthetic on both sides\" (the positional-index shape\n hith's fix targets and fully resolves when it occurs)\n\nIn other words: in the population currently persisted as ambiguous, 100% of\nthe identity-mismatch cases are this real-id-presence axis, not the\npositional-index axis. hith's fix is verified correct and regression-safe\n(250-cohort replay of already-resolved cohorts: 249/250 agree old vs new\nlogic, 1 improvement, 0 regressions) but resolves 0 of the currently-measured\n566-cohort population by itself, because no synthetic-minting scheme can ever\nmake a real UUID and a hash of (message, name, mime_type) collide.\n\n## Root cause\n\n`polylogue/sources/parsers/base_support.py:attachment_from_meta` uses the\nexport's own `id`/`file_id`/`fileId`/`uuid`/`file_uuid` field when present,\nand only falls back to synthesis when absent. Claude.ai does not consistently\nemit this field for the same attachment across export vintages of the same\nconversation -- verified directly against blob content for 6 sampled\ncohorts, all showing exactly this shape (one blob's attachment has a real\nUUID-shaped id, the other blob's attachment for the same message has no id\nfield and synthesizes `att-\u003chash\u003e`).\n\nNo id-minting scheme at the parser layer can reconcile this: a real id and a\nsynthetic hash will never be equal strings by construction, regardless of\nwhat the synthetic hash is seeded from.\n\n## Proposed fix (comparison layer, NOT parser layer)\n\nIn `polylogue/archive/session_revision_membership.py` (and/or\n`polylogue/pipeline/ids.py`'s `SessionRevisionProjection` /\n`_attachment_identity_payload`), the dominance/equivalence test should\ncompare attachments by a looser key when testing dominance -- e.g.\n`(message_provider_id, name, mime_type)` without the `id` field -- falling\nback to strict id equality only when that looser key is itself ambiguous\n(more than one attachment sharing the tuple on one side). This is the same\nclass of relaxation polylogue-bu1i introduced for acquisition state\n(`attachment_identities` vs `attachment_contents`), generalized to a third\naxis: \"same attachment referenced with and without a stable provider id\".\n\nThis bead deliberately does NOT propose an implementation in those files --\npolylogue-hith's owning lane was scoped away from\n`session_revision_membership.py`/`ids.py` because another lane owns them\nconcurrently. Whoever picks this up should re-run the census harness\ndescribed in polylogue-hith (or the updated one referenced in its closing\nnote) against the classifier change to prove the 268-cohort population above\nactually resolves, the same way polylogue-bu1i's PR proved 157/157.\n\n## Verification recipe\n\nSame read-only harness as polylogue-hith / polylogue-bu1i: parse both blobs\nof a cohort with production `parse_payload`, project with\n`session_revision_projection`, and diff the resulting\n`attachment_identities` sets. For the 268-cohort population, at least one\nattachment identity differs solely because one side has a real id string and\nthe other has a synthetic hash string for what is, by every other field\n(message anchor, name, mime_type), the same attachment.\n\nRef polylogue-hith\nRef polylogue-bu1i","status":"open","priority":1,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T13:02:14Z","created_by":"Sinity","updated_at":"2026-07-30T13:02:14Z","labels":["area:ingest"],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-d8al","title":"claude-ai-export: attachment real-id presence is inconsistent across export vintages, needs comparison-layer relaxation","description":"## What the data says\n\nCensus (full population, not a sample): replayed the production classifier\n(polylogue.sources.dispatch.parse_payload -\u003e session_revision_projection -\u003e\nclassify_membership_revisions) over all 566 claude-ai-export\nequal-message-count ambiguous cohorts in the live archive (read-only,\n/realm/db/polylogue), with polylogue-hith's parser-side fix (drop the\npositional-index seed for synthetic attachment ids) already applied.\n\n 566 claude-ai-export equal-message-count ambiguous cohorts (full census)\n 297 still ambiguous because message_hashes differ (polylogue-c429 /\n message-order-not-stable territory, or genuine content divergence)\n 268 still ambiguous with message_hashes EQUAL (0 content diffs) but\n attachment identity axis mismatched -- the exact shape hith\n targeted\n 0 of those 268 resolved by hith's fix\n 268 of those 268 are \"mixed real/synthetic\": one export vintage of the\n SAME conversation carries a real id (id/file_id/fileId/uuid/\n file_uuid) for an attachment; the OTHER vintage of the same\n conversation has no real id for the physically-same attachment and\n synthesizes one instead\n 0 are \"pure synthetic on both sides\" (the positional-index shape\n hith's fix targets and fully resolves when it occurs)\n\nIn other words: in the population currently persisted as ambiguous, 100% of\nthe identity-mismatch cases are this real-id-presence axis, not the\npositional-index axis. hith's fix is verified correct and regression-safe\n(250-cohort replay of already-resolved cohorts: 249/250 agree old vs new\nlogic, 1 improvement, 0 regressions) but resolves 0 of the currently-measured\n566-cohort population by itself, because no synthetic-minting scheme can ever\nmake a real UUID and a hash of (message, name, mime_type) collide.\n\n## Root cause\n\n`polylogue/sources/parsers/base_support.py:attachment_from_meta` uses the\nexport's own `id`/`file_id`/`fileId`/`uuid`/`file_uuid` field when present,\nand only falls back to synthesis when absent. Claude.ai does not consistently\nemit this field for the same attachment across export vintages of the same\nconversation -- verified directly against blob content for 6 sampled\ncohorts, all showing exactly this shape (one blob's attachment has a real\nUUID-shaped id, the other blob's attachment for the same message has no id\nfield and synthesizes `att-\u003chash\u003e`).\n\nNo id-minting scheme at the parser layer can reconcile this: a real id and a\nsynthetic hash will never be equal strings by construction, regardless of\nwhat the synthetic hash is seeded from.\n\n## Proposed fix (comparison layer, NOT parser layer)\n\nIn `polylogue/archive/session_revision_membership.py` (and/or\n`polylogue/pipeline/ids.py`'s `SessionRevisionProjection` /\n`_attachment_identity_payload`), the dominance/equivalence test should\ncompare attachments by a looser key when testing dominance -- e.g.\n`(message_provider_id, name, mime_type)` without the `id` field -- falling\nback to strict id equality only when that looser key is itself ambiguous\n(more than one attachment sharing the tuple on one side). This is the same\nclass of relaxation polylogue-bu1i introduced for acquisition state\n(`attachment_identities` vs `attachment_contents`), generalized to a third\naxis: \"same attachment referenced with and without a stable provider id\".\n\nThis bead deliberately does NOT propose an implementation in those files --\npolylogue-hith's owning lane was scoped away from\n`session_revision_membership.py`/`ids.py` because another lane owns them\nconcurrently. Whoever picks this up should re-run the census harness\ndescribed in polylogue-hith (or the updated one referenced in its closing\nnote) against the classifier change to prove the 268-cohort population above\nactually resolves, the same way polylogue-bu1i's PR proved 157/157.\n\n## Verification recipe\n\nSame read-only harness as polylogue-hith / polylogue-bu1i: parse both blobs\nof a cohort with production `parse_payload`, project with\n`session_revision_projection`, and diff the resulting\n`attachment_identities` sets. For the 268-cohort population, at least one\nattachment identity differs solely because one side has a real id string and\nthe other has a synthetic hash string for what is, by every other field\n(message anchor, name, mime_type), the same attachment.\n\nRef polylogue-hith\nRef polylogue-bu1i","notes":"Superseded by polylogue-aggz's architecture: attachment identity now unconditionally drops the provider id (content-derived: message_id+name+mime_type only), eliminating the strict/loose duality and its pairwise correlation machinery entirely rather than adding a fallback. See PR.","status":"open","priority":1,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T13:02:14Z","created_by":"Sinity","updated_at":"2026-07-30T15:15:31Z","labels":["area:ingest"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-eqnv","title":"Stale pre-fix parser identity lets a same-source_path raw pair silently split into two byte-proven singletons, downgrading fidelity","description":"## What the live archive shows\n\nFor the 5 aistudio-drive sessions Implementing-{066bb070,13ced1c8,37edfeb3,845dd573,d4d7fbab}, the index materialized the SMALLER (attachment-unfetched, \"bare\") raw and never even considered the LARGER (attachment-fetched, \"enriched\") raw. Neither raw has a `raw_session_memberships` row -- they never entered the ambiguous-membership machinery bu1i/9dxn describe. Instead both raws sit in raw_sessions with `revision_kind='full'`, `revision_authority='byte_proven'`, `baseline_raw_id=self` -- i.e. each was independently accepted as an unconditional SINGLETON byte-revision baseline under a DIFFERENT `logical_source_key`:\n\n 0064ddd16c39... (enriched, 967377B) -\u003e logical_source_key = 'gemini:Implementing-066bb070...'\n 13ae07d010bb... (bare, 252347B) -\u003e logical_source_key = 'gemini:Implementing-066bb070...-0'\n\n## Root cause, proven\n\n`raw_authority_parser_census` (source.db) records the census-time parser\nIDENTITY output for both raws:\n\n 0064ddd16c39...: fingerprint=revision-membership-v1, key=[\"gemini:Implementing-066bb070...23810576f616f90fb4254c69\"]\n 13ae07d010bb...: fingerprint=revision-membership-v1, key=[\"gemini:Implementing-066bb070...23810576f616f90fb4254c69-0\"]\n\nBoth under the SAME fingerprint string, yet different identity. Reparsing\nBOTH raw blobs from the live blob store through the CURRENT\n`polylogue/sources/dispatch.py`/`revision_backfill._parse_one` gives the\nIDENTICAL, correct, unsuffixed `provider_session_id` for both (verified with\nproduction code against the real blobs). The \"-0\" suffix is the exact\npre-#3179/z1c6 bug (`_lower_drive_like_payload`'s `_looks_like_chunked_session_list`\nbranch always appended `-{index}` regardless of list length, fixed\n2026-07-20 in b473d9256/#3179). raw_small was acquired+validated 2026-07-16,\nraw_big 2026-07-18 -- both before the fix landed 2026-07-20 -- and their\ncensus (which sets `raw_sessions.logical_source_key`) evidently ran under\nthe pre-fix parser and was never invalidated, because\n`raw_authority_parser_census`'s quiescence gate\n(`uncensused_historical_revision_raw_ids`) treats any row with the SAME\nliteral fingerprint string as \"current parser already observed this\" --\nthere is no version distinction between pre-fix and post-fix identity\noutput. `classify_raw_revision_cohort` (archive.py) then classifies each\nraw against its OWN `logical_source_key` in isolation, has no way to know\nthe two keys describe the same physical document, and unconditionally\naccepts each as a trivial one-member byte-proven chain -- the same\nstructural hole polylogue-52l2/hm2f already document for the RETIRED-SIBLING\ncase, but here the divergence is at the KEY itself, not at retirement\nstate, so the existing `raw_membership_retired_full_revision_siblings` guard\n(keyed on exact logical_source_key match) never fires.\n\n## Relationship to polylogue-9dxn\n\n9dxn's proposed fingerprint-versioning fix (permissive quiescence for any\nKNOWN fingerprint, strict-current-only for the ambiguous TERMINAL gate)\ndoes not by itself heal this case: it is designed to let previously-`ambiguous`\nverdicts be revisited without forcing a blanket re-census, but raw_small's\nstale census here was NOT ambiguous -- it was `status='complete'` with a\nWRONG identity, and 9dxn's design keeps quiescence permissive for any known\nfingerprint, so this raw would stay \"already observed\" forever even after a\nfingerprint bump. This bead's fix is a structural cross-source_path guard in\n`classify_raw_revision_cohort`, independent of fingerprint versioning, that\nalso closes the general case regardless of how two same-document raws ended\nup under different keys (stale census, race, or a future bug of the same\nshape).\n\n## Fix landed in polylogue-af059 (this branch)\n\n- `archive.py`: `classify_raw_revision_cohort` refuses unconditional\n singleton acceptance when another 'full' raw shares the same source_path\n under a different (or already-retired) logical_source_key -- forces both\n into membership governance instead of letting either become an\n unconditionally-accepted baseline.\n- `revision_backfill.py`: the retire-to-membership-governance fallback now\n buckets `membership_candidates`/`membership_keys` by the FRESHLY re-parsed\n identity (`session.provider_session_id`) instead of the stale outer-loop\n `logical_source_key`, so two same-document raws retired under different\n stale keys land in ONE membership cohort and get jointly arbitrated\n instead of each being accepted as an independent membership singleton.\n\n## Residual / follow-up\n\n- The live archive's 5 already-downgraded sessions are NOT repaired by this\n code fix (need a live remediation pass, out of scope for this PR).\n- A full census-fingerprint bump (9dxn) is still needed to catch every OTHER\n raw whose identity was assigned by pre-#3179 dispatch.py, if any exist\n beyond aistudio-drive.\n\nRef polylogue-bu1i, polylogue-7ilr, polylogue-9dxn","status":"closed","priority":1,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T12:22:33Z","created_by":"Sinity","updated_at":"2026-07-30T12:45:58Z","started_at":"2026-07-30T12:45:56Z","closed_at":"2026-07-30T12:45:58Z","close_reason":"Fixed in PR #3396 (feature/fix/ambiguous-raw-materialization-leak): ArchiveStore.classify_raw_revision_cohort gains an opt-in check_source_path_identity_split guard (used only by the offline backfill/rebuild replay loop, not the live watcher), plus revision_backfill.py's retire-to-membership-governance fallback now buckets by the freshly re-derived identity instead of the stale outer-loop key. Verified with two new regression tests (anti-vacuity confirmed both ways via direct revert+rerun). The 5 already-downgraded live sessions are NOT repaired by this fix; live remediation is a separate, explicitly out-of-scope lane.","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"polylogue-nuec","title":"chatgpt-export: provider-reported generation-duration metadata volatile, contaminates session-event identity hash","description":"## What the data says\n\nSampled 35 of 129 chatgpt-export \"ambiguous\" equal-message-count membership\ncohorts (27%; read-only against /realm/db/polylogue). Reproduced with\nproduction code identically to polylogue-c429/polylogue-c42a: parsed both\ndistinct-content raw revisions per cohort via\n`polylogue.sources.dispatch.parse_payload`, projected each with\n`polylogue.pipeline.ids.session_revision_projection`.\n\n33 of 35 sampled cohorts (94%) have this exact shape:\n\n message_hashes equal (messages byte-identical, same order)\n attachment_hashes equal\n event_hashes DIFFER, with event COUNT equal on both sides\n\nFor every sampled case, the first differing `session_events` entry is\n`event_type == \"generation_lifecycle\"` with an identical payload key set\n(`duration_semantics`, `elapsed_duration_ms`, `evidence_source`,\n`fidelity`, `state`) but a DIFFERENT `elapsed_duration_ms` value (e.g.\n13000 vs 21000; 52000 vs 107000; 123000 vs 33000 -- no consistent\ndirection, ruling out simple clock skew). In several cases the event's\n`source_message_provider_id` also differs at the same array index, evidence\nthat the generation-lifecycle event LIST itself may reorder alongside the\nduration values, though message order (which correlates with these events)\nwas independently confirmed stable.\n\n## Root cause\n\n`polylogue/sources/parsers/chatgpt.py` (`_resolve_generation_timings` /\n`~line 1069`, `duration_semantics=\"provider_reported_elapsed\"`) derives a\nsynthetic `generation_lifecycle` session event per assistant/tool message,\nwith `elapsed_duration_ms` computed from the RAW EXPORT's own\n`finished_duration_sec` or `reasoning_start_time`/`reasoning_end_time`\nmetadata fields on that message's mapping node (not something Polylogue\ninvents -- traced to `raw_metadata.get(\"finished_duration_sec\")` and the\n`reasoning_start_time`/`reasoning_end_time` delta). This value is folded into\n`session_events` and hashed via `session_revision_projection`'s\n`event_hashes` (`polylogue/pipeline/ids.py`), which\n`_strictly_dominates`/`classify_membership_revisions`\n(`polylogue/archive/session_revision_membership.py`) treats as part of\ncontent identity.\n\nThe underlying provider-reported duration values are not stable across\nseparate ChatGPT export requests for the SAME generation -- 33 of 35 sampled\ncohorts have message and attachment content that is byte-identical across\ntwo export vintages, yet the reported generation timing differs, sometimes\nsubstantially (e.g. 2s vs 27s; 794s vs 445s), with no consistent\nincrease/decrease pattern that would suggest a benign refinement. This reads\nas either non-deterministic export-time re-derivation on OpenAI's side, or a\nmetric that legitimately varies by measurement context and was never meant\nto be a durable per-generation identity value. Either way, folding it into\nsession identity hash makes byte-identical conversations look like divergent\nbranches on every re-export.\n\n## Reproduction recipe (production code, no archive mutation)\n\nSame harness pattern as polylogue-c429, with `origin='chatgpt-export'`;\nafter loading both `ParsedSession`s for a cohort:\n\n```python\nfrom polylogue.pipeline.ids import session_revision_projection\npa, pb = session_revision_projection(a), session_revision_projection(b)\nassert pa.message_hashes == pb.message_hashes\nassert pa.attachment_hashes == pb.attachment_hashes\nassert pa.event_hashes != pb.event_hashes\nassert len(a.session_events) == len(b.session_events)\n# first differing pair:\nfor ea, eb in zip(a.session_events, b.session_events):\n if ea.payload != eb.payload:\n assert ea.event_type == eb.event_type == \"generation_lifecycle\"\n assert ea.payload[\"elapsed_duration_ms\"] != eb.payload[\"elapsed_duration_ms\"]\n break\n```\n\n## Extrapolation honesty\n\n35 of 129 sampled (27%, the largest sample fraction of any origin in this\ncensus). 33/35 = 94% match this exact shape (message+attachment hashes\nequal, event hashes differ, dominant delta traced to\n`generation_lifecycle.elapsed_duration_ms`). 1/35 has both message and\nevent differences (a separate, unexamined cause). 1/35 is now identical\nunder the current classifier (message/event/attachment hashes all equal) --\nits recorded 'ambiguous' decision appears stale relative to current\nevidence; see polylogue-9dxn for the general \"persisted ambiguous verdicts\nnever get re-derived\" defect that would explain this. Extrapolating 94% to\nthe full 129-cohort population suggests roughly 120 of 129 cohorts, but this\nis an estimate from a 27% sample, not a full census.\n\n## Proposed fix direction (for the classifier/parser-owning lane, not this bead)\n\nThis is the clearest case in the whole census for excluding a field from\nidentity rather than relaxing dominance comparison: `elapsed_duration_ms` is\nexplicitly labeled a measurement (`duration_semantics:\n\"provider_reported_elapsed\"`), not a content field, and doesn't belong in a\ncontent-identity hash at all. Either exclude `generation_lifecycle` event\npayloads (or just the `elapsed_duration_ms` field within them) from\n`_session_hash_components`'s `session_events_payload` in\n`polylogue/pipeline/ids.py`, or store/compare `session_events` with a\ntolerant equality that ignores this specific volatile field. Narrower and\nlower-risk than the message-order or attachment-identity fixes in\npolylogue-c429/polylogue-c42a because it doesn't touch dominance logic at\nall -- it just stops hashing a value the parser itself already documents as\nnon-durable measurement evidence.\n\nRef polylogue-bu1i\n","status":"open","priority":1,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T12:16:49Z","created_by":"Sinity","updated_at":"2026-07-30T12:17:41Z","labels":["area:ingest"],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-nuec","title":"chatgpt-export: provider-reported generation-duration metadata volatile, contaminates session-event identity hash","description":"## What the data says\n\nSampled 35 of 129 chatgpt-export \"ambiguous\" equal-message-count membership\ncohorts (27%; read-only against /realm/db/polylogue). Reproduced with\nproduction code identically to polylogue-c429/polylogue-c42a: parsed both\ndistinct-content raw revisions per cohort via\n`polylogue.sources.dispatch.parse_payload`, projected each with\n`polylogue.pipeline.ids.session_revision_projection`.\n\n33 of 35 sampled cohorts (94%) have this exact shape:\n\n message_hashes equal (messages byte-identical, same order)\n attachment_hashes equal\n event_hashes DIFFER, with event COUNT equal on both sides\n\nFor every sampled case, the first differing `session_events` entry is\n`event_type == \"generation_lifecycle\"` with an identical payload key set\n(`duration_semantics`, `elapsed_duration_ms`, `evidence_source`,\n`fidelity`, `state`) but a DIFFERENT `elapsed_duration_ms` value (e.g.\n13000 vs 21000; 52000 vs 107000; 123000 vs 33000 -- no consistent\ndirection, ruling out simple clock skew). In several cases the event's\n`source_message_provider_id` also differs at the same array index, evidence\nthat the generation-lifecycle event LIST itself may reorder alongside the\nduration values, though message order (which correlates with these events)\nwas independently confirmed stable.\n\n## Root cause\n\n`polylogue/sources/parsers/chatgpt.py` (`_resolve_generation_timings` /\n`~line 1069`, `duration_semantics=\"provider_reported_elapsed\"`) derives a\nsynthetic `generation_lifecycle` session event per assistant/tool message,\nwith `elapsed_duration_ms` computed from the RAW EXPORT's own\n`finished_duration_sec` or `reasoning_start_time`/`reasoning_end_time`\nmetadata fields on that message's mapping node (not something Polylogue\ninvents -- traced to `raw_metadata.get(\"finished_duration_sec\")` and the\n`reasoning_start_time`/`reasoning_end_time` delta). This value is folded into\n`session_events` and hashed via `session_revision_projection`'s\n`event_hashes` (`polylogue/pipeline/ids.py`), which\n`_strictly_dominates`/`classify_membership_revisions`\n(`polylogue/archive/session_revision_membership.py`) treats as part of\ncontent identity.\n\nThe underlying provider-reported duration values are not stable across\nseparate ChatGPT export requests for the SAME generation -- 33 of 35 sampled\ncohorts have message and attachment content that is byte-identical across\ntwo export vintages, yet the reported generation timing differs, sometimes\nsubstantially (e.g. 2s vs 27s; 794s vs 445s), with no consistent\nincrease/decrease pattern that would suggest a benign refinement. This reads\nas either non-deterministic export-time re-derivation on OpenAI's side, or a\nmetric that legitimately varies by measurement context and was never meant\nto be a durable per-generation identity value. Either way, folding it into\nsession identity hash makes byte-identical conversations look like divergent\nbranches on every re-export.\n\n## Reproduction recipe (production code, no archive mutation)\n\nSame harness pattern as polylogue-c429, with `origin='chatgpt-export'`;\nafter loading both `ParsedSession`s for a cohort:\n\n```python\nfrom polylogue.pipeline.ids import session_revision_projection\npa, pb = session_revision_projection(a), session_revision_projection(b)\nassert pa.message_hashes == pb.message_hashes\nassert pa.attachment_hashes == pb.attachment_hashes\nassert pa.event_hashes != pb.event_hashes\nassert len(a.session_events) == len(b.session_events)\n# first differing pair:\nfor ea, eb in zip(a.session_events, b.session_events):\n if ea.payload != eb.payload:\n assert ea.event_type == eb.event_type == \"generation_lifecycle\"\n assert ea.payload[\"elapsed_duration_ms\"] != eb.payload[\"elapsed_duration_ms\"]\n break\n```\n\n## Extrapolation honesty\n\n35 of 129 sampled (27%, the largest sample fraction of any origin in this\ncensus). 33/35 = 94% match this exact shape (message+attachment hashes\nequal, event hashes differ, dominant delta traced to\n`generation_lifecycle.elapsed_duration_ms`). 1/35 has both message and\nevent differences (a separate, unexamined cause). 1/35 is now identical\nunder the current classifier (message/event/attachment hashes all equal) --\nits recorded 'ambiguous' decision appears stale relative to current\nevidence; see polylogue-9dxn for the general \"persisted ambiguous verdicts\nnever get re-derived\" defect that would explain this. Extrapolating 94% to\nthe full 129-cohort population suggests roughly 120 of 129 cohorts, but this\nis an estimate from a 27% sample, not a full census.\n\n## Proposed fix direction (for the classifier/parser-owning lane, not this bead)\n\nThis is the clearest case in the whole census for excluding a field from\nidentity rather than relaxing dominance comparison: `elapsed_duration_ms` is\nexplicitly labeled a measurement (`duration_semantics:\n\"provider_reported_elapsed\"`), not a content field, and doesn't belong in a\ncontent-identity hash at all. Either exclude `generation_lifecycle` event\npayloads (or just the `elapsed_duration_ms` field within them) from\n`_session_hash_components`'s `session_events_payload` in\n`polylogue/pipeline/ids.py`, or store/compare `session_events` with a\ntolerant equality that ignores this specific volatile field. Narrower and\nlower-risk than the message-order or attachment-identity fixes in\npolylogue-c429/polylogue-c42a because it doesn't touch dominance logic at\nall -- it just stops hashing a value the parser itself already documents as\nnon-durable measurement evidence.\n\nRef polylogue-bu1i\n","notes":"Superseded by polylogue-aggz's architecture: chatgpt-export generation_lifecycle duration volatility is now handled via an explicit content-only ALLOWLIST (_EVENT_CONTENT_PAYLOAD_ALLOWLIST) rather than a denylist strip of known-volatile fields. Live census: 119/135 (88.1%) chatgpt-export ambiguous cohorts now resolve, 0 regressions. See PR.","status":"open","priority":1,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T12:16:49Z","created_by":"Sinity","updated_at":"2026-07-30T15:15:30Z","labels":["area:ingest"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-hith","title":"claude-ai-export: synthetic attachment id keyed on positional index is unstable across export vintages","description":"## What the data says\n\nSame sample as polylogue-c429 (40 of 566 claude-ai-export ambiguous\nequal-message-count cohorts, read-only against /realm/db/polylogue,\nreproduced with production `parse_payload` + `session_revision_projection` +\n`classify_membership_revisions`). Of the 40, 16 (40%) have this exact shape,\ndisjoint from the message-order cause in polylogue-c429:\n\n message id set equal, message array order equal, 0 content diffs\n len(attachments_a) == len(attachments_b)\n set of (provider_attachment_id, message_provider_id) DISJOINT or partially disjoint\n between the two revisions, for attachments anchored to the SAME message\n\nExample (cohort with 4 attachments, 2 anchor messages, `key` starting\n`claude-a...`):\n\n A: att_id=e950263f-063d-495d-b0c0-61e9330d3a14 msg=7f1cf6ff-...\n B: att_id=att-ce21cd12d650 msg=7f1cf6ff-... (same message anchor)\n\n A: att_id=66a7a163-d488-4320-8129-19ad43f64a43 msg=c38e86ac-...\n B: att_id=att-cd01a39eb65c msg=0d3a13b8-... (different message anchor too)\n\nmime_type/size_bytes/inline-presence/name-length are identical between the\npaired attachments in every sampled case -- this is not\npolylogue-bu1i's acquisition-state pattern (`inline_bytes`/`size_bytes`\nflipping None-\u003ereal). The IDENTITY STRING itself differs, and sometimes so\ndoes the message it's anchored to.\n\n## Root cause\n\n`polylogue/sources/parsers/base_support.py:152-197`\n(`attachment_from_meta`/`_make_attachment_id`), used by the Claude.ai parser\nvia `attachment_from_meta` (`polylogue/sources/parsers/claude/ai_parser.py`,\n`_merge_session_attachments` at line ~191, iterating `(\"attachments\",\n\"files\")`):\n\n```python\ndef _make_attachment_id(seed: str) -\u003e str:\n return f\"att-{hash_text(seed)[:12]}\"\n\ndef attachment_from_meta(meta, message_id, index):\n attachment_id = (\n meta.get(\"id\") or meta.get(\"file_id\") or meta.get(\"fileId\")\n or meta.get(\"uuid\") or meta.get(\"file_uuid\")\n )\n ...\n if not attachment_id:\n if not name:\n return None\n seed = f\"{message_id or 'msg'}:{name}:{index}\"\n attachment_id = _make_attachment_id(seed)\n```\n\nTwo independent failure modes both traced in the sample:\n\n1. **Real-id presence is inconsistent across export vintages.** When\n Claude.ai's own export payload carries a real `id`/`file_id`/`uuid` for an\n attachment, that string is used directly (stable). When it's absent, the\n parser falls back to a SYNTHETIC id hashed from\n `f\"{message_id}:{name}:{index}\"`. The two export vintages of the same\n conversation don't consistently include the real id -- one carries it,\n the other doesn't -- so the same physical attachment gets a real UUID in\n one revision and a synthetic `att-...` id in the other.\n2. **`index` is positional, and attachment order is not guaranteed stable.**\n Even when BOTH revisions fall back to synthesis, `index` is the\n attachment's position in the merged `attachments`+`files` iteration for\n that message. If that per-message ordering shifts between export\n vintages (plausible given polylogue-c429's proof that the surrounding\n MESSAGE array order is itself unstable across Claude.ai exports), the\n synthesized id changes even though the underlying attachment didn't.\n\nEither way, attachment identity is accidentally keyed on transient\nexport-shape details (real-id presence, list order) rather than a property\nof the attachment itself, so `_attachment_hash_payload`\n(`polylogue/pipeline/ids.py:152`) hashes the same physical attachment to two\ndifferent identities across export vintages -- the same general shape as\npolylogue-bu1i (acquisition/export-time noise contaminating an identity\nhash), but a DIFFERENT concrete defect (id synthesis, not acquisition-state\nflip) requiring a different fix.\n\n## Reproduction recipe (production code, no archive mutation)\n\nSame harness as polylogue-c429's reproduction recipe; after loading both\n`ParsedSession`s for a cohort where message ids/order/content are identical:\n\n```python\natts_a = {(at.provider_attachment_id, at.message_provider_id): at for at in a.attachments}\natts_b = {(at.provider_attachment_id, at.message_provider_id): at for at in b.attachments}\nassert len(atts_a) == len(atts_b)\nassert set(atts_a) != set(atts_b) # disjoint identity despite same count\n```\n\n## Extrapolation honesty\n\n40 of 566 sampled (7%). 16/40 = 40% match this exact shape (message\ncontent/order fully identical, attachment key sets disjoint at equal\ncount). Extrapolating to the full population suggests roughly 220-230 of the\n566 cohorts, but this is an estimate from a 7% sample, not a census.\n\n## Proposed fix direction (for the classifier/parser-owning lane, not this bead)\n\nTwo independent levers, either alone reduces the blast radius:\n\n- Parser-side: derive the synthetic attachment id from content-stable\n material only (e.g. a hash of `(message_provider_id, name, mime_type,\n size_bytes)` without positional `index`), so re-ordering the export's\n attachment list doesn't change identity. Does not fix mode (1)\n (real-id-present-in-one-export-only).\n- Classifier-side (in the files this investigation lane does not edit):\n compare attachments by a looser key (e.g. `(message_provider_id, name,\n mime_type, size_bytes)`) when testing dominance, falling back to id\n equality only when that tuple is ambiguous -- the same class of relaxation\n polylogue-bu1i proposes for acquisition-state, generalized.\n\nRef polylogue-bu1i\nRef polylogue-c429\n","status":"open","priority":1,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T12:16:40Z","created_by":"Sinity","updated_at":"2026-07-30T12:17:40Z","labels":["area:ingest"],"comments":[{"id":"019fb31e-ef07-7bef-a785-41e5de19372f","issue_id":"polylogue-hith","author":"Sinity","text":"Parser-side fix landed (PR pending, branch feature/fix/synthetic-attachment-id-stability):\nattachment_from_meta's synthetic-id seed no longer includes the positional\n`index`; mime_type is used as the one extra structural disambiguator instead\n(id/name/mime_type). The now-unused `index` param was removed from\nattachment_from_meta and all 3 call sites (ai_parser.py x2,\nclaude/common.py's _message_attachments).\n\nVerified: 250-cohort regression replay (old vs new minting logic) over\nalready-resolved claude-ai-export cohorts -- 249/250 agree, 1 improvement,\n0 regressions.\n\nHonest disposition on the 566-cohort measured population: 0 resolved by this\nfix alone. Full census (not sample) shows all 268 message-hashes-equal\nambiguous cohorts are \"mixed real/synthetic\" (failure mode 1: real-id\npresence varies across export vintages of the same conversation) -- 0 are\n\"pure synthetic on both sides\" (the positional-index shape this fix\ntargets). Failure mode 1 needs a comparison-layer relaxation in\nsession_revision_membership.py/ids.py, which this lane was scoped away\nfrom. Filed as polylogue-d8al with the full census breakdown and a proposed\ndesign (loosen dominance comparison to (message_id, name, mime_type) when\nprovider ids disagree, id-equality fallback only when that's itself\nambiguous). polylogue-c429 (message order) accounts for the other 297.\n\nLeaving this bead open pending the comparison-layer fix -- the fix in this\nPR is real and durable (protects any future/other-origin case of the\npositional-index shape) but does not itself resolve the currently-measured\npopulation; polylogue-d8al is the actionable remainder.\n","created_at":"2026-07-30T13:02:57Z"}],"dependency_count":0,"dependent_count":0,"comment_count":1} {"_type":"issue","id":"polylogue-qkuq","title":"claude-ai-export: synthetic attachment id keyed on positional index is unstable across export vintages","status":"open","priority":1,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T12:16:33Z","created_by":"Sinity","updated_at":"2026-07-30T12:16:33Z","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"polylogue-c429","title":"claude-ai-export: message array order is not stable across export vintages, breaking prefix-dominance","description":"## What the data says\n\nSampled 40 of 566 claude-ai-export \"ambiguous\" equal-message-count membership\ncohorts (7%; read-only against /realm/db/polylogue). Reproduced with\nproduction code: parsed both distinct-content raw revisions of each cohort\nvia `polylogue.sources.dispatch.parse_payload` (routed through\n`provider_from_origin`/`capture_mode` exactly as `_parse_one` does in\n`polylogue/sources/revision_backfill.py`), projected each with\n`polylogue.pipeline.ids.session_revision_projection`, and ran the production\n`classify_membership_revisions`.\n\n21 of 40 sampled cohorts (52.5%) have this exact shape:\n\n n_messages_a == n_messages_b\n set(provider_message_id for a.messages) == set(provider_message_id for b.messages)\n [a.provider_message_id for a in a.messages] != [... for b.messages] # order differs\n for every shared id: (role, text, timestamp) identical between a and b\n\nThat is: the SAME messages, byte-identical per-message content, just in a\nDIFFERENT SEQUENCE in the two exports. Concretely reproduced on cohort\n`claude-ai:944d1095-51ea-4063-abe9-719d9971e281` (raws\n`aa0990572bb833d4...` vs `ebe3a4f95f45b235...`): 36/36 messages, identical\n`{role,text,timestamp}` for every one of the 36 shared `provider_message_id`s,\n0 content diffs, but `ids_a != ids_b`. One revision's message array is sorted\nchronologically; the other is not (its role sequence pairs adjacent\nuser/user, assistant/assistant messages -- looks like Claude.ai's own tree\nflattening interleaving edited-message siblings rather than a strict\ntimestamp sort).\n\n## Root cause\n\n`polylogue/pipeline/ids.py:session_revision_projection` builds\n`message_hashes` as an ORDER-SENSITIVE tuple (`_message_hash_payload` per\nmessage, in array order). `polylogue/archive/session_revision_membership.py`\n`_strictly_dominates` requires\n`older.message_hashes == newer.message_hashes[: len(older.message_hashes)]`\n-- an exact positional prefix match. When Claude.ai's own export emits the\nsame conversation's message array in a different sequence across two export\nrequests (same message set, same content, different order), this prefix\ncheck fails in BOTH directions even though there is no real content\ndivergence, and the cohort is quarantined ambiguous.\n\nNo parser code sorts messages by timestamp before this hash is computed\n(`polylogue/sources/parsers/claude/ai_parser.py` preserves whatever order the\nexport's `chat_messages` array carries; see `_merge_session_attachments`\niterating `(\"attachments\", \"files\")` for the analogous merge-order case in\nattachments). Claude.ai's own export ordering for a given conversation is\napparently NOT guaranteed stable across separate export requests -- this is\nupstream non-determinism polylogue must tolerate, not something polylogue's\nown acquisition controls.\n\n## Reproduction recipe (production code, no archive mutation)\n\n```python\nfrom pathlib import Path\nfrom polylogue.sources.decoders import _iter_json_stream\nfrom polylogue.sources.dispatch import parse_payload\nfrom polylogue.core.enums import Origin\nfrom polylogue.core.sources import provider_from_origin\nfrom polylogue.pipeline.ids import session_revision_projection\nimport io, sqlite3\n\ncon = sqlite3.connect(\"file:/realm/db/polylogue/source.db?mode=ro\", uri=True)\ncon.row_factory = sqlite3.Row\nrows = con.execute(\n \"select rs.raw_id, rs.source_path, rs.blob_hash, rs.capture_mode \"\n \"from raw_session_memberships m join raw_sessions rs on rs.raw_id = m.raw_id \"\n \"where m.decision='ambiguous' and rs.origin='claude-ai-export' \"\n \"and m.logical_source_key = ?\",\n (\"claude-ai:944d1095-51ea-4063-abe9-719d9971e281\",),\n).fetchall()\n\ndef load(row):\n h = row[\"blob_hash\"].hex()\n raw = (Path(\"/realm/db/polylogue/blob\") / h[:2] / h[2:]).read_bytes()\n provider = provider_from_origin(Origin.CLAUDE_AI_EXPORT, family_hint=row[\"capture_mode\"])\n fallback_id = Path(row[\"source_path\"].split(\":\")[-1]).stem\n name = Path(row[\"source_path\"].split(\":\")[-1]).name\n records = list(_iter_json_stream(io.BytesIO(raw), name))\n return parse_payload(str(provider), records, fallback_id, source_path=row[\"source_path\"])\n\nsessions = {r[\"raw_id\"]: load(r)[0] for r in rows} # cohort has exactly 1 session per raw here\nids = list(sessions)\na, b = sessions[ids[0]], sessions[ids[1]]\nassert {m.provider_message_id for m in a.messages} == {m.provider_message_id for m in b.messages}\nassert [m.provider_message_id for m in a.messages] != [m.provider_message_id for m in b.messages]\n```\n\n## Extrapolation honesty\n\n40 of 566 sampled (7%), stratified randomly (seed fixed). 21/40 = 52.5% match\nthis exact shape; 3 more sampled cohorts show this pattern layered with a\nsecond delta (attachment count or session-event differences) in addition.\nExtrapolating the 52.5% rate to the full population suggests roughly 280-300\nof the 566 cohorts, but this is an ESTIMATE from a 7% sample, not a census --\nunlike polylogue-bu1i's 100%-verified aistudio-drive population, this has not\nbeen checked against every cohort.\n\n## Proposed fix direction (for the classifier-owning lane, not this bead)\n\n`_strictly_dominates` and `session_revision_projection` currently treat\nmessage sequence as part of content identity. A safe fix compares the\nmessage SET (by `provider_message_id` + content) rather than requiring an\nexact positional prefix when a provider's export ordering is not\nauthoritative -- i.e. treat \"same message ids/content, different array\norder\" as equivalent, not as a branch. This is a distinct code path from\npolylogue-bu1i's attachment-acquisition-state fix (different failure\nmode, different field: sequence vs. attachment identity) and should not be\nfolded into the same patch without separate verification, since a naive\norder-insensitive compare would also need to preserve real append-order\ndetection (`older.message_hashes == newer.message_hashes[:len(older)]`) for\ngenuinely growing sessions.\n\nRef polylogue-bu1i\nRef polylogue-hith\nRef polylogue-nuec\n","status":"open","priority":1,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T12:14:22Z","created_by":"Sinity","updated_at":"2026-07-30T12:17:41Z","labels":["area:ingest"],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-c429","title":"claude-ai-export: message array order is not stable across export vintages, breaking prefix-dominance","description":"## What the data says\n\nSampled 40 of 566 claude-ai-export \"ambiguous\" equal-message-count membership\ncohorts (7%; read-only against /realm/db/polylogue). Reproduced with\nproduction code: parsed both distinct-content raw revisions of each cohort\nvia `polylogue.sources.dispatch.parse_payload` (routed through\n`provider_from_origin`/`capture_mode` exactly as `_parse_one` does in\n`polylogue/sources/revision_backfill.py`), projected each with\n`polylogue.pipeline.ids.session_revision_projection`, and ran the production\n`classify_membership_revisions`.\n\n21 of 40 sampled cohorts (52.5%) have this exact shape:\n\n n_messages_a == n_messages_b\n set(provider_message_id for a.messages) == set(provider_message_id for b.messages)\n [a.provider_message_id for a in a.messages] != [... for b.messages] # order differs\n for every shared id: (role, text, timestamp) identical between a and b\n\nThat is: the SAME messages, byte-identical per-message content, just in a\nDIFFERENT SEQUENCE in the two exports. Concretely reproduced on cohort\n`claude-ai:944d1095-51ea-4063-abe9-719d9971e281` (raws\n`aa0990572bb833d4...` vs `ebe3a4f95f45b235...`): 36/36 messages, identical\n`{role,text,timestamp}` for every one of the 36 shared `provider_message_id`s,\n0 content diffs, but `ids_a != ids_b`. One revision's message array is sorted\nchronologically; the other is not (its role sequence pairs adjacent\nuser/user, assistant/assistant messages -- looks like Claude.ai's own tree\nflattening interleaving edited-message siblings rather than a strict\ntimestamp sort).\n\n## Root cause\n\n`polylogue/pipeline/ids.py:session_revision_projection` builds\n`message_hashes` as an ORDER-SENSITIVE tuple (`_message_hash_payload` per\nmessage, in array order). `polylogue/archive/session_revision_membership.py`\n`_strictly_dominates` requires\n`older.message_hashes == newer.message_hashes[: len(older.message_hashes)]`\n-- an exact positional prefix match. When Claude.ai's own export emits the\nsame conversation's message array in a different sequence across two export\nrequests (same message set, same content, different order), this prefix\ncheck fails in BOTH directions even though there is no real content\ndivergence, and the cohort is quarantined ambiguous.\n\nNo parser code sorts messages by timestamp before this hash is computed\n(`polylogue/sources/parsers/claude/ai_parser.py` preserves whatever order the\nexport's `chat_messages` array carries; see `_merge_session_attachments`\niterating `(\"attachments\", \"files\")` for the analogous merge-order case in\nattachments). Claude.ai's own export ordering for a given conversation is\napparently NOT guaranteed stable across separate export requests -- this is\nupstream non-determinism polylogue must tolerate, not something polylogue's\nown acquisition controls.\n\n## Reproduction recipe (production code, no archive mutation)\n\n```python\nfrom pathlib import Path\nfrom polylogue.sources.decoders import _iter_json_stream\nfrom polylogue.sources.dispatch import parse_payload\nfrom polylogue.core.enums import Origin\nfrom polylogue.core.sources import provider_from_origin\nfrom polylogue.pipeline.ids import session_revision_projection\nimport io, sqlite3\n\ncon = sqlite3.connect(\"file:/realm/db/polylogue/source.db?mode=ro\", uri=True)\ncon.row_factory = sqlite3.Row\nrows = con.execute(\n \"select rs.raw_id, rs.source_path, rs.blob_hash, rs.capture_mode \"\n \"from raw_session_memberships m join raw_sessions rs on rs.raw_id = m.raw_id \"\n \"where m.decision='ambiguous' and rs.origin='claude-ai-export' \"\n \"and m.logical_source_key = ?\",\n (\"claude-ai:944d1095-51ea-4063-abe9-719d9971e281\",),\n).fetchall()\n\ndef load(row):\n h = row[\"blob_hash\"].hex()\n raw = (Path(\"/realm/db/polylogue/blob\") / h[:2] / h[2:]).read_bytes()\n provider = provider_from_origin(Origin.CLAUDE_AI_EXPORT, family_hint=row[\"capture_mode\"])\n fallback_id = Path(row[\"source_path\"].split(\":\")[-1]).stem\n name = Path(row[\"source_path\"].split(\":\")[-1]).name\n records = list(_iter_json_stream(io.BytesIO(raw), name))\n return parse_payload(str(provider), records, fallback_id, source_path=row[\"source_path\"])\n\nsessions = {r[\"raw_id\"]: load(r)[0] for r in rows} # cohort has exactly 1 session per raw here\nids = list(sessions)\na, b = sessions[ids[0]], sessions[ids[1]]\nassert {m.provider_message_id for m in a.messages} == {m.provider_message_id for m in b.messages}\nassert [m.provider_message_id for m in a.messages] != [m.provider_message_id for m in b.messages]\n```\n\n## Extrapolation honesty\n\n40 of 566 sampled (7%), stratified randomly (seed fixed). 21/40 = 52.5% match\nthis exact shape; 3 more sampled cohorts show this pattern layered with a\nsecond delta (attachment count or session-event differences) in addition.\nExtrapolating the 52.5% rate to the full population suggests roughly 280-300\nof the 566 cohorts, but this is an ESTIMATE from a 7% sample, not a census --\nunlike polylogue-bu1i's 100%-verified aistudio-drive population, this has not\nbeen checked against every cohort.\n\n## Proposed fix direction (for the classifier-owning lane, not this bead)\n\n`_strictly_dominates` and `session_revision_projection` currently treat\nmessage sequence as part of content identity. A safe fix compares the\nmessage SET (by `provider_message_id` + content) rather than requiring an\nexact positional prefix when a provider's export ordering is not\nauthoritative -- i.e. treat \"same message ids/content, different array\norder\" as equivalent, not as a branch. This is a distinct code path from\npolylogue-bu1i's attachment-acquisition-state fix (different failure\nmode, different field: sequence vs. attachment identity) and should not be\nfolded into the same patch without separate verification, since a naive\norder-insensitive compare would also need to preserve real append-order\ndetection (`older.message_hashes == newer.message_hashes[:len(older)]`) for\ngenuinely growing sessions.\n\nRef polylogue-bu1i\nRef polylogue-hith\nRef polylogue-nuec\n","notes":"Superseded by polylogue-aggz's architecture: message array order is now handled as a byproduct of set-based (identity, content) comparison (message_contents), not a dedicated positional-prefix fix. Live census (full population): 554/587 (94.4%) claude-ai-export ambiguous cohorts now resolve, 0 regressions against previously-resolved cohorts. See PR.","status":"open","priority":1,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T12:14:22Z","created_by":"Sinity","updated_at":"2026-07-30T15:15:29Z","labels":["area:ingest"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-9dxn","title":"A persisted 'ambiguous' verdict is terminal with no classifier version, so classifier corrections are inert on existing data","description":"## Problem\n\n`polylogue-bu1i` fixes the classifier so that acquiring an attachment's bytes is\nread as a fidelity upgrade rather than a branch. Verified: all 157 live\naistudio-drive cohorts now resolve to an accepted chain with the enriched\nrevision at its head, where previously 157/157 were ambiguous.\n\nThat fix cannot heal the archive it was written for. The verdicts it corrects are\nalready persisted, and a persisted `ambiguous` verdict is TERMINAL:\n\n polylogue/storage/repair.py:4432-4462\n SELECT 1 FROM raw_session_memberships\n WHERE raw_id IN (...) AND decision = 'ambiguous'\n -\u003e RawReplayPlanStatus.TERMINAL,\n \"component ended in explicit ambiguous or parse-terminal authority state\",\n \"inspect durable authority debt; do not replay without new evidence\"\n\n`raw_session_memberships` has no fingerprint column, so nothing distinguishes\n\"ambiguous under the current classifier\" from \"ambiguous under a classifier we\nhave since corrected\". Every improvement to `classify_membership_revisions` is\ntherefore inert on existing data and only affects newly-acquired raws, while the\nexisting debt sits terminal forever and reads as though it needed operator\njudgment.\n\nLive scale of the inert-fix problem: 3,875 ambiguous membership rows across\n~1,079 cohorts (587 claude-ai-export, 191 claude-code-session, 151\naistudio-drive, 136 chatgpt-export, and a tail).\n\n## Second defect: a bump would not propagate\n\n`RAW_AUTHORITY_PARSER_FINGERPRINT = \"revision-membership-v1\"` exists as a proper\nconstant in `polylogue/storage/raw_authority.py:27`, but\n`polylogue/sources/revision_backfill.py` hardcodes the literal string eight\ntimes instead of importing it (lines 318, 348, 438, 475, 552, 565, 596, 919),\nincluding inside an f-string. Bumping the constant today would half-apply: the\nwriter would stamp the new value while the quiescence gate still matched the old\none. The constant is not load-bearing, which makes the versioning mechanism\nnon-functional exactly when it is first needed.\n\n## Proposed fix\n\n1. Make the constant load-bearing: `revision_backfill.py` imports\n `RAW_AUTHORITY_PARSER_FINGERPRINT` rather than repeating the literal.\n2. Separate two questions the single fingerprint currently conflates:\n - *Was this raw ever observed by a real parser?* -- the quiescence gate\n (`uncensused_historical_revision_raw_ids`, `revision_backfill.py:321`).\n Any known fingerprint should satisfy this, so a bump does NOT trigger an\n archive-wide re-census of all 41,363 raws.\n - *Is this verdict still authoritative under current semantics?* -- the\n terminal gate. Only the CURRENT fingerprint should satisfy this.\n Concretely: keep a `SUPERSEDED_MEMBERSHIP_FINGERPRINTS` set alongside the\n current one, and have the terminal check treat an `ambiguous` decision as\n stale (replayable) when the raw's census fingerprint is superseded rather\n than current. Absent census row -\u003e treat as current, i.e. stay conservative.\n `index_tier.raw_revision_applications` carries the same `decision='ambiguous'`\n check and needs the same treatment.\n3. Bump `RAW_AUTHORITY_PARSER_FINGERPRINT` to `revision-membership-v2`, because\n `polylogue-bu1i` genuinely changed classification semantics.\n\nWith (2) in place the healing is targeted: roughly 3,875 raws re-derive their\nverdict, instead of re-censusing the whole 99 GB archive. Without (2), a bump\nis correct but costs a full reparse (~4h20m measured on this archive).\n\n## Why this is the general fix, not a one-off\n\nThe value here is not unblocking one origin. It is that a classifier correction\nbecomes self-healing: today, improving `classify_membership_revisions` requires\nmanual archive surgery to have any effect on existing data, which is precisely\nthe shape that leaves corrected logic silently inert and debt looking legitimate.\n\n## Acceptance criteria\n\n- `RAW_AUTHORITY_PARSER_FINGERPRINT` is the single source of the fingerprint\n string; no module hardcodes it.\n- An `ambiguous` verdict recorded under a superseded fingerprint is replayable,\n and one recorded under the current fingerprint remains terminal. Both\n directions covered by tests.\n- A bump does not force re-census of raws whose verdict is unaffected; assert\n this against a fixture archive rather than by reasoning.\n- Anti-vacuity: state the production line mutated and the resulting failure.\n\nRef polylogue-bu1i\n","notes":"CORRECTION 2026-07-30, from the lane that traced polylogue-eqnv: the 'Proposed fix' item (2) above is wrong for identity-class staleness, and I am recording that before anyone implements it.\n\nI proposed splitting the fingerprint's two jobs so that the QUIESCENCE gate accepts any *known* fingerprint (avoiding an archive-wide re-census on a bump) while only the TERMINAL gate requires the current one. The motive was cost: targeted healing of ~3,900 raws instead of reparsing 99 GB.\n\nThat does not work when the stale thing is the raw's derived IDENTITY rather than its verdict. polylogue-eqnv is the concrete counterexample: two raws of one document were censused under the same fingerprint string but recorded different logical_source_key values, one carrying a pre-#3179 '-0' suffix from a dispatch bug fixed 2026-07-20 (b473d9256) that their 2026-07-16/18 acquisition predates. Reparsing both blobs through current dispatch yields the identical correct key. Permissive quiescence is exactly what keeps that raw from ever being re-derived, so it preserves the corruption it was meant to be cheap about.\n\nConsequence for this bead's scope: re-census (a reparse) is the honest price for any change that alters derived identity, and the cost cannot be engineered away by making the gate permissive. The split between 'was this observed' and 'is this verdict current' may still be worth having for pure VERDICT changes, where the recorded identity is unaffected -- polylogue-bu1i is that shape, since it changed only how revisions are COMPARED. State which class a change falls in before choosing the cheap path.\n\nPossible middle path, not yet evaluated: re-census only raws whose recorded identity disagrees with a cheap re-derivation, which needs a parse but not a full projection/materialization. Whether that is meaningfully cheaper than the full reparse is unmeasured -- do not assume it is.\nDESIGN 2026-07-30, from the polylogue-eqnv/c737 lane, supersedes the correction note above with something actionable.\n\nSplit the single parser fingerprint into two independently-versioned components:\n\n identity fingerprint -- covers dispatch.py's provider_session_id /\n logical_source_key derivation\n classification fingerprint -- covers session_revision_membership.py's\n dominance rules\n\nThen each class of fix pays only its own price:\n\n- A CLASSIFICATION fix (polylogue-bu1i's shape: dominance rules changed, the\n stored identity is unaffected) bumps only the classification component.\n Quiescence stays permissive on identity, so no reparse is forced, and the\n terminal-ambiguous gate re-runs classification against the already-known\n identity. Cheap, and it makes classifier corrections self-healing, which is\n this bead's original ask.\n- An IDENTITY fix (polylogue-eqnv's shape and the z1c6 dispatch bug: the stored\n logical_source_key itself was wrong) bumps the identity component. Quiescence\n goes strict for it, forcing exactly the reparse that is unavoidably the honest\n price -- you cannot know an identity is still correct without recomputing it,\n since recomputing IS how you discover it changed.\n\nThis is strictly better than the single fingerprint in both directions: today a\nclassification fix cannot heal existing data at all (the terminal gate has no\nversion to compare), and an identity fix would force a full 99 GB reparse even\nwhen only classification changed.\n\nImplementation note carried over: RAW_AUTHORITY_PARSER_FINGERPRINT must first\nbecome load-bearing -- sources/revision_backfill.py still hardcodes\n'revision-membership-v1' at eight sites (318, 348, 438, 475, 552, 565, 596,\n919) instead of importing the constant, so any bump half-applies until that is\nfixed.\nDESIGN (re-recorded 2026-07-30 after a bd reimport dropped the first append), from the polylogue-eqnv/c737 lane.\n\nSplit the single parser fingerprint into two independently-versioned components:\n\n identity fingerprint -- covers dispatch.py's provider_session_id /\n logical_source_key derivation\n classification fingerprint -- covers session_revision_membership.py's\n dominance rules\n\nEach class of fix then pays only its own price:\n\n- A CLASSIFICATION fix (polylogue-bu1i's shape: dominance rules changed, stored\n identity unaffected) bumps only the classification component. Quiescence stays\n permissive on identity so no reparse is forced, and the terminal-ambiguous\n gate re-runs classification against the already-known identity. Cheap, and it\n makes classifier corrections self-healing -- this bead's original ask.\n- An IDENTITY fix (polylogue-eqnv's shape, and the z1c6 dispatch bug: the stored\n logical_source_key itself was wrong) bumps the identity component. Quiescence\n goes strict for it, forcing exactly the reparse that is unavoidably the honest\n price -- you cannot know an identity is still correct without recomputing it,\n because recomputing IS how you discover it changed.\n\nStrictly better than one fingerprint in both directions: today a classification\nfix cannot heal existing data at all (the terminal gate has no version to\ncompare against), while an identity fix would force a full 99 GB reparse even\nwhen only classification changed.\n\nPrerequisite: RAW_AUTHORITY_PARSER_FINGERPRINT must become load-bearing first --\nsources/revision_backfill.py hardcodes 'revision-membership-v1' at eight sites\n(318, 348, 438, 475, 552, 565, 596, 919) instead of importing the constant, so\nany bump half-applies until that is fixed.","status":"open","priority":1,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T12:13:46Z","created_by":"Sinity","updated_at":"2026-07-30T13:32:35Z","labels":["area:ingest"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-bu1i","title":"aistudio-drive 'ambiguous' revision pairs are not branches: attachment acquisition state contaminates attachment identity hash","description":"## What the data says\n\n151 of 151 aistudio-drive ambiguous membership cohorts (100%) are the SAME Drive\ndocument acquired twice, where the later acquisition merely resolved\nDrive-hosted attachment bytes. There is no branch and nothing to judge.\n\nVerified across all 157 two-member source_path cohorts in the live archive\n(/realm/db/polylogue), by loading both blobs and comparing:\n\n 157/157 file_mtime_ms IDENTICAL (both carry Drive modifiedTime)\n 157/157 earlier blob has NO _polylogue_drive_live_bytes_b64\n 157/157 later blob HAS it\n 157/157 the two payloads are byte-equal after stripping that key\n 157/157 later blob is larger (median ~5-80x)\n\nReproduced deterministically with production code on the pair\n30-12-2025-SINEX-IDEAS.json (raws f6b63f0b / d0715a7f):\n\n bare 604,853 B msgs=60 events=61 atts=4 all inline=None, size_bytes=None\n enriched 5,602,664 B msgs=60 events=61 atts=4 same 4 Drive file ids, bytes fetched\n\n message_hashes equal: True\n event_hashes equal: True\n attachment sets: n1=4 n2=4 intersection=0 subset=False\n _strictly_dominates(bare-\u003eenriched) = False\n classify_membership_revisions -\u003e ambiguous ['d0715a7f','f6b63f0b']\n\n## Root cause (two independent contributors)\n\n1. `_attachment_hash_payload` (polylogue/pipeline/ids.py:152) folds\n ACQUISITION STATE into attachment IDENTITY: it appends\n `inline_content_hash` only when `inline_bytes is not None`, and\n `size_bytes` flips None -\u003e real once bytes are fetched. So the same\n attachment (same Drive file id, same message anchor) hashes differently\n before and after acquisition. The two revisions' attachment_hashes end up\n equal-cardinality and DISJOINT.\n\n2. `_strictly_dominates` (archive/session_revision_membership.py:188) then\n fails both of its conditions: `content_grew` is False (equal message and\n event counts, no proper attachment superset) and\n `older.attachment_hashes \u003c= newer.attachment_hashes` is False (disjoint,\n not subset). Neither escape hatch applies: both revisions have\n `browser_snapshot_fidelity=None` so `_provider_ordered_browser_snapshots`\n bails, and `_direct_export_precedence` needs a browser-capture sibling.\n -\u003e ambiguous, both quarantined.\n\nSeparately, `raw_sessions.revision_kind='unknown'` / `logical_source_key IS NULL`\nbecause the byte-prefix chain check cannot hold: the injector splices base64\nmid-document and re-serializes the whole JSON\n(`json.dumps(resolved, ensure_ascii=False)`, sources/drive/__init__.py:173),\nso the later bytes are not a byte-prefix extension of the earlier.\n\n## Where the second scrape came from\n\nNot two Drive versions. Both acquisitions read the SAME local cache file under\n`~/.local/share/polylogue/drive-cache/gemini/` (240 documents). The 2026-06-29\npass wrote the cache with attachments unresolved. The 2026-07-18 pass took the\ncache-hit branch (no Drive re-download at all) and ran\n`_inject_live_drive_attachment_bytes` -- which by design runs on EVERY read,\ncache hit or not, precisely to backfill caches written before the feature\nexisted (sources/drive/__init__.py:242-256). It mutated the bytes, rewrote the\ncache in place, and hashed the mutated payload -\u003e a second, distinct raw row.\nDrive modifiedTime never changed, which is why file_mtime_ms is identical.\n\nThe 83 single-row cohorts corroborate this: 72 have no driveDocument/Image/\nAudio/Video reference at all, and 11 have references the injector could not\nresolve -- in both cases the injector returns bytes unchanged, the blob hash is\nstable, and no second raw row is created.\n\n## Concrete harm already in the index\n\nPost-promotion convergence materialized these ambiguous raws anyway, arbitrarily\nand last-writer-wins. 6 cohorts got BOTH members materialized; in 5 of the 6 the\nBARE revision was written last, so the index now reports those sessions'\nattachments as `unfetched` even though the bytes were successfully fetched and\nare sitting in the blob store:\n\n aistudio-drive:Implementing-066bb070... atts=1 acquired=0\n aistudio-drive:Implementing-13ced1c8... atts=1 acquired=0\n aistudio-drive:Implementing-37edfeb3... atts=1 acquired=0\n aistudio-drive:Implementing-845dd573... atts=1 acquired=0\n aistudio-drive:Implementing-d4d7fbab... atts=1 acquired=0\n\nThat is a silent fidelity DOWNGRADE, and it is the exact failure mode the\n'never choose between branches' invariant exists to prevent -- it happened\nbecause a non-branch was labelled a branch, and then something picked anyway.\nWhich stage performed that pick is not yet traced: `repair.py:1075` does\nquarantine ambiguous membership, yet 135 of the 151 cohorts acquired a\n`parsed_at_ms` between 06:57 and 13:12 local on 2026-07-30, after the\n`decided_at_ms` of 07:00 that recorded them ambiguous. That gap needs its own\ntrace and may be a second, separate defect.\n\n## Proposed fix\n\nTreat 'same attachment identity, bytes now acquired' as a fidelity upgrade, the\ndirect analogue of the documented DOM-\u003enative rule. Concretely: compare\nattachments by provider identity (provider_attachment_id + message_provider_id\n+ name + mime_type) when testing dominance, and allow a differing hash when the\nonly delta is that the newer side has inline_bytes where the older did not.\nEquivalently, split attachment identity from attachment acquisition state so\nacquisition can never fabricate a branch.\n\nPrefer this over adding a Drive-specific escape hatch: the shape is generic\n(any origin whose attachments are fetched lazily), and the classifier already\nhas two precedents for 'this is an upgrade, not a branch'.\n\n## Blast radius beyond drive\n\nEqual-message-count ambiguous cohorts by origin (same shape; needs its own\nverification per origin before claiming the same cause):\n\n claude-ai-export 566 / 587 cohorts\n chatgpt-export 128 / 136\n aistudio-drive 151 / 151 \u003c- proven, this bead\n hermes-session 3 / 4\n claude-code-session 6 / 191 \u003c- different shape, not this\n gemini-cli-session 0 / 3\n\n## Measurement notes for whoever picks this up\n\n- Live aistudio-drive state at filing: index 225 sessions / 95,823 blocks\n (retired generation had 239 / 106,178); source has 173 unparsed raws, of\n which 129 are correctly superseded (their enriched sibling IS materialized)\n and 44 are the 22 both-unparsed cohorts. 14 documents are absent from the\n index entirely -- exactly the 239-225 gap.\n- The earlier claim '0 correctly superseded, all 302 genuinely unmaterialized'\n was a measurement artifact: it checked `raw_sessions.logical_source_key`,\n which governance deliberately NULLs on transition to semantic membership\n (archive.py:2710). The key survives on\n `raw_session_memberships.logical_source_key` -- join that table instead.\n- Attachment acquisition overall improved enormously in this generation:\n acquired 26 -\u003e 2,849 (unfetched 3,120 -\u003e 177). This bead is a narrow\n regression channel inside a large win, not a verdict on the rebuild.\n\nRef polylogue-7ilr (which framed this residue as genuine authority debt\nrequiring operator judgment; for aistudio-drive that framing is wrong).\n","status":"open","priority":1,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T11:34:40Z","created_by":"Sinity","updated_at":"2026-07-30T11:34:40Z","labels":["area:ingest"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-ne6k","title":"repair_empty_sessions would delete the 832 genuinely-empty sessions the hook-inflation postmortem chose to retain","design":"Found 2026-07-29 by the pre-rebuild deletion audit. NOT on the rebuild path.\n\nrepair_empty_sessions / count_empty_sessions_sync (polylogue/storage/repair.py)\nselect with a blanket predicate:\n\n sessions LEFT JOIN messages ... WHERE m.session_id IS NULL\n\nIt cannot distinguish a legitimately-empty session from corruption debris.\nThat distinction is not hypothetical: the 2026-07-22 hook-inflation\npostmortem explicitly decided to RETAIN ~832 genuinely-empty sessions after\nthe de-inflation (index sessions went 83,286 -\u003e 18,391 = 17,559 real + 832\ngenuinely-empty). Browser-capture stubs are a second legitimate source.\nRunning this repair would delete exactly the rows that postmortem chose to\nkeep.\n\nWHY IT IS NOT A REBUILD BLOCKER: the target is MaintenanceTargetMode.CLEANUP\nwith destructive=True, and resolve_selected_maintenance_targets\n(cli/shared/check_maintenance.py) only includes CLEANUP targets when the\noperator explicitly passes --cleanup or names the target. Neither\nmaintenance/rebuild_index.py nor daemon/bulk_rebuild.py ever calls it. So the\nrebuild pipeline cannot trigger it.\n\nTHE REAL RISK IS OPERATIONAL: someone running `polylogue check --cleanup` as\nhousekeeping around the big rebuild would silently delete the retained\nsessions. DO NOT RUN --cleanup against the live archive until this is fixed.\n\nFIX: give the predicate a distinguishing signal -- e.g. require raw_id IS NULL\n(no acquired bytes behind it) or an explicit acquisition-status check -- so a\nsession that was legitimately acquired and legitimately has no messages is\nretained, and only rows with no provenance at all are candidates.\n","status":"open","priority":1,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-29T20:21:06Z","created_by":"Sinity","updated_at":"2026-07-29T20:21:06Z","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/docs/plans/hash-boundary-registry.yaml b/docs/plans/hash-boundary-registry.yaml index 04c0e3fbd4..21fd693f3b 100644 --- a/docs/plans/hash-boundary-registry.yaml +++ b/docs/plans/hash-boundary-registry.yaml @@ -542,6 +542,36 @@ entries: occurrence: 2 classification: content-hash note: 'feeds session_content_hash / revision projection (2026-07-09 hash-boundary census (docs/audits/2026-07-09-hash-boundary-census.md), Table 1 row 1).' +- path: polylogue/pipeline/ids.py + function: '.session_revision_projection' + call: hash_payload + occurrence: 3 + classification: content-hash + note: 'event_hashes -- feeds session_content_hash / revision projection, unstripped and order-preserving (polylogue-nuec).' +- path: polylogue/pipeline/ids.py + function: '.session_revision_projection' + call: hash_payload + occurrence: 4 + classification: content-hash + note: 'event base identity (event type + anchoring message), content-derived and position-independent, feeding event_contents (polylogue-aggz, polylogue-nuec).' +- path: polylogue/pipeline/ids.py + function: '.session_revision_projection' + call: hash_payload + occurrence: 5 + classification: content-hash + note: 'per-event content hash (measurement excluded via _event_content_payload''s allowlist), feeding event_contents (polylogue-aggz, polylogue-nuec).' +- path: polylogue/pipeline/ids.py + function: '.session_revision_projection' + call: hash_payload + occurrence: 6 + classification: content-hash + note: 'canonical event identity fold (base identity + content) used only when the base identity is locally ambiguous within one revision -- still content-derived, never the array index (polylogue-aggz).' +- path: polylogue/pipeline/ids.py + function: '._event_content_payload' + call: hash_payload + occurrence: 0 + classification: content-hash + note: 'per-event content payload feeding event_contents, an explicit per-event-type ALLOWLIST of content-bearing fields rather than a denylist of fields discovered volatile after the fact (polylogue-aggz, polylogue-nuec).' - path: polylogue/scenarios/workload.py function: '._identity' call: hashlib.sha256 diff --git a/polylogue/archive/session_revision_membership.py b/polylogue/archive/session_revision_membership.py index 707e5f327c..d0e3c503d1 100644 --- a/polylogue/archive/session_revision_membership.py +++ b/polylogue/archive/session_revision_membership.py @@ -8,10 +8,15 @@ from polylogue.core.timestamps import parse_timestamp from polylogue.pipeline.ids import SessionRevisionProjection -#: Everything that must agree for two revisions to be the same content: the -#: message and event chains, which attachments exist, and which of their bytes -#: have been read. -_ContentKey: TypeAlias = tuple[tuple[bytes, ...], tuple[bytes, ...], frozenset[bytes], frozenset[tuple[bytes, bytes]]] +#: One content axis's relation between two revisions: total and decidable, +#: no residual category (polylogue-aggz). ``equal`` is the same identity set +#: with equal content per identity. ``a_contains_b``/``b_contains_a`` is one +#: side's identity set containing the other's with equal content on the +#: intersection -- set containment, not sequence prefix, so this subsumes +#: append-only growth without caring about array order. ``conflict`` is +#: either side disagreeing about content under a shared identity, or each +#: side holding an identity the other lacks (a genuine fork). +_Relation: TypeAlias = Literal["equal", "a_contains_b", "b_contains_a", "conflict"] @dataclass(frozen=True, slots=True) @@ -27,96 +32,325 @@ class MembershipRevision: @dataclass(frozen=True, slots=True) class MembershipClassification: + #: Materialized head(s): an append-only growth chain (oldest to newest, + #: by set containment). Empty only when the cohort is a genuine, + #: irreducible conflict (see ``ambiguous_raw_ids``) -- a presence- + #: guarantee fallback that would deterministically pick a maximal- + #: evidence head even then is designed and unit-tested + #: (``_maximal_evidence_fallback``) but not yet wired into this return + #: path; see that function's docstring for why. accepted_raw_ids: tuple[str, ...] + #: Raws proven to carry the SAME content as an accepted head (``equal`` + #: per ``_relation``) -- legitimately superseded, not debt. equivalent_raw_ids: tuple[str, ...] + #: Raws NOT reflected in ``accepted_raw_ids`` because no containment + #: relation could order them against it: unresolved conflict debt + #: attached to the session, retained as raw provenance only (never + #: deleted, never silently discarded). ambiguous_raw_ids: tuple[str, ...] +def _identities(contents: frozenset[tuple[bytes, bytes]]) -> frozenset[bytes]: + """Project the identity half of a (identity, content) pair set. + + Messages and events always carry content for every identity they have -- + unlike attachments, which project identity and content as two separate + fields because an attachment can be referenced before its bytes are + acquired (polylogue-bu1i) -- so their identity set is always exactly + this projection of their content set. + """ + return frozenset(identity for identity, _content in contents) + + +def _content_by_identity(contents: frozenset[tuple[bytes, bytes]]) -> dict[bytes, frozenset[bytes]]: + """Group content hashes by identity, retaining EVERY value under a colliding identity. + + Identity is not always injective: two acquired attachments on one + message can share one identity (same ``message_id``/``name``/ + ``mime_type``, different bytes -- the accepted limit documented on + ``_ATTACHMENT_IDENTITY_FIELDS``, not a new one). Collapsing such a + collision to a single arbitrary content hash (a plain ``dict``, last + value wins) would let a real content conflict compare as equal instead. + Grouping into a set per identity means a collision always degrades to + ``conflict`` when compared against a revision that disagrees, never to + ``equal``. + """ + grouped: dict[bytes, set[bytes]] = {} + for identity, content in contents: + grouped.setdefault(identity, set()).add(content) + return {identity: frozenset(values) for identity, values in grouped.items()} + + +def _axis_relation( + identities_a: frozenset[bytes], + contents_a: frozenset[tuple[bytes, bytes]], + identities_b: frozenset[bytes], + contents_b: frozenset[tuple[bytes, bytes]], +) -> _Relation: + """Compare one content axis (messages, attachments, or events) between two revisions. + + An identity present without content on either side (an attachment + referenced but not yet acquired) is compatible with, never in conflict + with, the SAME identity carrying content on the other side -- unknown is + less evidence, not a disagreement (polylogue-bu1i). For messages and + events every identity always carries content, so this degrades to plain + set equality/containment for those two axes. + """ + content_a = _content_by_identity(contents_a) + content_b = _content_by_identity(contents_b) + shared = identities_a & identities_b + for identity in shared: + value_a, value_b = content_a.get(identity), content_b.get(identity) + if value_a is not None and value_b is not None and value_a != value_b: + return "conflict" + a_richer = bool(identities_a - identities_b) or any( + content_a.get(identity) is not None and content_b.get(identity) is None for identity in shared + ) + b_richer = bool(identities_b - identities_a) or any( + content_b.get(identity) is not None and content_a.get(identity) is None for identity in shared + ) + if a_richer and b_richer: + return "conflict" + if a_richer: + return "a_contains_b" + if b_richer: + return "b_contains_a" + return "equal" + + +def _relation(a: SessionRevisionProjection, b: SessionRevisionProjection) -> _Relation: + """Combine the message/attachment/event axes into one overall relation. + + Each axis is independently equal/contains/conflict; the whole document is + ``equal`` only if every axis is, ``a_contains_b``/``b_contains_a`` only if + every non-equal axis agrees on the SAME direction (a mix -- one axis grew + forward, another backward -- is exactly a fork: each side holds + something the other lacks, so it is a conflict too), and ``conflict`` if + any single axis already is. + """ + axes = ( + _axis_relation( + _identities(a.message_contents), a.message_contents, _identities(b.message_contents), b.message_contents + ), + _axis_relation(a.attachment_identities, a.attachment_contents, b.attachment_identities, b.attachment_contents), + _axis_relation( + _identities(a.event_contents), a.event_contents, _identities(b.event_contents), b.event_contents + ), + ) + if "conflict" in axes: + return "conflict" + directions = {axis for axis in axes if axis != "equal"} + if not directions: + return "equal" + if directions == {"a_contains_b"}: + return "a_contains_b" + if directions == {"b_contains_a"}: + return "b_contains_a" + return "conflict" + + +def _frontier(projection: SessionRevisionProjection) -> tuple[int, int, int, int]: + """Rank a revision along every axis on which it can only grow. + + Used only to order candidates for the pairwise ``_relation`` chain check + and to break ties deterministically in the presence-guarantee fallback + (stable regardless of processing order -- a rebuild must always resolve + the same cohort to the same head). + """ + return ( + len(projection.message_contents), + len(projection.event_contents), + len(projection.attachment_identities), + len(projection.attachment_contents), + ) + + +def _equal_content_representative( + incumbent: MembershipRevision, candidate: MembershipRevision +) -> tuple[MembershipRevision, MembershipRevision]: + """Pick (winner, loser) between two revisions already proven ``equal`` in content. + + Equal content is not equal authority: a direct/native export and a + browser DOM scrape can project to byte-identical messages while one is + first-class provenance and the other is a lossy capture, and which one + survives as the representative determines what future re-acquisitions + are compared against. Source authority is therefore decided BEFORE any + timestamp tiebreak -- mirroring, for the equal case, the same + ``_direct_export_precedence`` / browser-fidelity ordering + (``_browser_snapshot_dominates``) already applied to the non-equal + growth-chain case below: a direct export always outranks a + browser-capture sibling, and a native browser snapshot always outranks a + DOM snapshot of the same underlying session. Only when neither + revision's provenance outranks the other's does provider timestamp (and + finally a stable raw_id) decide, exactly as before this function + existed. + """ + incumbent_direct = incumbent.browser_snapshot_fidelity is None + candidate_direct = candidate.browser_snapshot_fidelity is None + if incumbent_direct != candidate_direct: + return (incumbent, candidate) if incumbent_direct else (candidate, incumbent) + if ( + not incumbent_direct + and not candidate_direct + and incumbent.browser_snapshot_fidelity != candidate.browser_snapshot_fidelity + ): + return (incumbent, candidate) if incumbent.browser_snapshot_fidelity == "native" else (candidate, incumbent) + incumbent_time = parse_timestamp(incumbent.provider_updated_at) + candidate_time = parse_timestamp(candidate.provider_updated_at) + if ( + incumbent_time is not None + and candidate_time is not None + and candidate_time.timestamp() != incumbent_time.timestamp() + ): + return ( + (candidate, incumbent) + if candidate_time.timestamp() > incumbent_time.timestamp() + else (incumbent, candidate) + ) + # No distinguishing provenance or provider timestamp -- these two are + # already proven identical content, so which raw_id represents them does + # not matter for correctness; pick deterministically rather than + # requiring a timestamp that a re-export of an untouched conversation + # may never carry (its own provider updated_at legitimately does not + # move when nothing provider-visible changed). + winner, loser = sorted((incumbent, candidate), key=lambda item: item.raw_id) + return winner, loser + + def classify_membership_revisions(revisions: list[MembershipRevision]) -> MembershipClassification: - """Accept one total strict-growth chain; never choose between branches.""" + """Accept one total growth chain by set containment; never choose a branch silently. + + Two revisions are the same content, contain each other, or conflict -- + total and decidable, with no residual category (polylogue-aggz). This + replaces what used to be four separate mechanisms (a positional-prefix + message dominance test, a denylist-stripped ordered event-hash prefix + test, a strict-vs-loose attachment id correlation pass, and a + session-hash/metadata-timestamp tiebreak layered on top of all three) + with one relation, applied uniformly to every axis. + + A genuine, irreducible conflict -- no containment chain exists at all, + not export-vintage noise -- still quarantines every representative + (``ambiguous_raw_ids``) with nothing accepted, after browser-fidelity and + direct-export precedence have also failed to order the cohort. A + presence-guarantee alternative to that quarantine -- deterministically + materializing the maximal-evidence representative instead of withholding + every revision, with the conflict recorded as debt rather than absence + -- is designed and unit-tested (``_maximal_evidence_fallback``) but not + wired in here; see that function's docstring for the concrete, + proven reason (a real archive.py write-back invariant it would violate). + """ if not revisions: return MembershipClassification((), (), ()) - # Keyed on acquisition state as well as identity: two revisions are the same - # content only when they also agree on which attachment bytes are in hand, - # otherwise collapsing them as equivalents could discard the one that - # actually carries the bytes. - by_content: dict[_ContentKey, list[MembershipRevision]] = {} - for revision in revisions: - projection = revision.projection - key = ( - projection.message_hashes, - projection.event_hashes, - projection.attachment_identities, - projection.attachment_contents, - ) - by_content.setdefault(key, []).append(revision) + representatives: list[MembershipRevision] = [] equivalents: list[str] = [] - for group in by_content.values(): - by_session_hash: dict[bytes, list[MembershipRevision]] = {} - for item in group: - by_session_hash.setdefault(item.projection.session_hash, []).append(item) - metadata_variants: list[MembershipRevision] = [] - for hash_group in by_session_hash.values(): - representative = min(hash_group, key=lambda item: item.raw_id) - metadata_variants.append(representative) - equivalents.extend(item.raw_id for item in hash_group if item.raw_id != representative.raw_id) - if len(metadata_variants) == 1: - representatives.extend(metadata_variants) + for revision in revisions: + match_index = next( + (i for i, rep in enumerate(representatives) if _relation(rep.projection, revision.projection) == "equal"), + None, + ) + if match_index is None: + representatives.append(revision) continue - timestamped = [ - (parsed.timestamp(), item) - for item in metadata_variants - if (parsed := parse_timestamp(item.provider_updated_at)) is not None - ] - timestamps = [timestamp for timestamp, _item in timestamped] - if len(timestamped) == len(metadata_variants) and len(set(timestamps)) == len(timestamps): - representative = max(timestamped, key=lambda pair: pair[0])[1] - representatives.append(representative) - equivalents.extend(item.raw_id for item in metadata_variants if item.raw_id != representative.raw_id) - else: - representatives.extend(metadata_variants) + incumbent = representatives[match_index] + winner, loser = _equal_content_representative(incumbent, revision) + representatives[match_index] = winner + equivalents.append(loser.raw_id) + representatives.sort(key=lambda item: (_frontier(item.projection), item.raw_id)) - if any( - not _strictly_dominates(older.projection, newer.projection) + conflict = any( + _relation(older.projection, newer.projection) != "b_contains_a" for older, newer in zip(representatives, representatives[1:], strict=False) - ): - browser_order = _provider_ordered_browser_snapshots(representatives) - if browser_order is None: - direct_export = _direct_export_precedence(representatives) - if direct_export is None: - return MembershipClassification( - (), - tuple(sorted(equivalents)), - tuple(sorted(item.raw_id for item in representatives)), - ) - accepted, browser_capture_raw_ids = direct_export - return MembershipClassification( - (accepted.raw_id,), - tuple(sorted((*equivalents, *browser_capture_raw_ids))), - (), - ) - representatives = browser_order + ) + if not conflict: + return MembershipClassification( + tuple(item.raw_id for item in representatives), + tuple(sorted(equivalents)), + (), + ) + + browser_order = _provider_ordered_browser_snapshots(representatives) + if browser_order is not None: + return MembershipClassification( + tuple(item.raw_id for item in browser_order), + tuple(sorted(equivalents)), + (), + ) + direct_export = _direct_export_precedence(representatives) + if direct_export is not None: + accepted, browser_capture_raw_ids = direct_export + return MembershipClassification( + (accepted.raw_id,), + tuple(sorted((*equivalents, *browser_capture_raw_ids))), + (), + ) + # _maximal_evidence_fallback (below) is the intended long-term behavior + # for a genuine, irreducible conflict -- see its docstring for why it is + # not called here yet. Until it is wired in, an irreducible conflict is + # quarantined exactly as it always was: every representative in + # `ambiguous_raw_ids`, nothing accepted. return MembershipClassification( - tuple(item.raw_id for item in representatives), - tuple(sorted(equivalents)), (), + tuple(sorted(equivalents)), + tuple(sorted(item.raw_id for item in representatives)), ) +def _maximal_evidence_fallback(representatives: list[MembershipRevision]) -> MembershipRevision: + """Deterministically select the maximal-evidence revision for an irreducible conflict. + + Designed and unit-tested as the presence-guarantee alternative to + quarantining every representative with nothing accepted: a document + should never simply vanish from the archive just because arbitration + could not order its revisions. Frontier order (most messages, most + events, most attachments, most acquired bytes) with a stable raw_id + tiebreak means the same cohort always resolves to the same head + regardless of processing order -- verified directly by + ``test_presence_guarantee_fallback_is_order_independent``. + + NOT called by ``classify_membership_revisions`` yet. Wiring it in would + make ``accepted_raw_ids`` non-empty for every non-empty input, moving the + other representatives into ``ambiguous_raw_ids`` as recorded conflict + debt instead of leaving the whole cohort headless. That trips a real, + carefully-designed invariant in + ``archive.py``'s ``apply_raw_membership_classification`` write-back, with + documented production incident history behind it (polylogue-miwv, PR + #3211): once a logical source has an accepted head, a later membership + pass may not silently retire it in favor of an unrelated raw unless that + raw is part of the new accepted/equivalent set. A fallback pick can + legitimately differ from an already-established head -- proven by two + real integration-test failures during verification + (``test_divergent_bundle_member_preserves_last_accepted_session`` and + its sibling), not a theoretical concern. Landing this safely needs + either the classifier or its caller to carry the existing head's raw_id + into this decision, or the write-back guard to accept a + re-affirmed-quarantined outcome explicitly -- both changes belong to + archive.py's write path, out of this lane's scope. + """ + return max(representatives, key=lambda item: (_frontier(item.projection), item.raw_id)) + + def _provider_ordered_browser_snapshots( revisions: list[MembershipRevision], ) -> list[MembershipRevision] | None: """Order compatible mutable browser snapshots by provider authority. - Browser-native payloads are complete snapshots, not append logs. ChatGPT - can move an already-present context/tool node when later work appears, and - can complete text in place under the same provider message id. A strict - serialized-content prefix therefore rejects ordinary provider progress. - Provider timestamps may resolve that progress only when stable message and - attachment identities are preserved. DOM-to-native is the sole fidelity - upgrade and may use different synthetic ids; a native-to-DOM downgrade is - never selected. + Kept as a genuine exception to the content-only relation above, not a + dead alternate path: browser-captured DOM and native snapshots synthesize + their OWN local message/attachment ids from DOM structure, not a stable + provider identity, so the same underlying message can carry a different + synthetic id in a DOM capture than in a native one -- the content-only + ``_relation`` genuinely cannot correlate them (it would read a + DOM-to-native fidelity upgrade as two disjoint, conflicting message id + sets). ``provider_message_ids``/``provider_attachment_ids`` on + ``MembershipRevision`` carry the browser-capture layer's own identity + evidence for exactly this reason. ChatGPT can also move an already-present + context/tool node when later work appears, and can complete text in place + under the same provider message id, so a strict serialized-content + comparison would reject ordinary provider progress; provider timestamps + resolve that progress only when stable message and attachment identities + are preserved. DOM-to-native is the sole fidelity upgrade and may use + different synthetic ids; a native-to-DOM downgrade is never selected. """ if not revisions or any(item.browser_snapshot_fidelity is None for item in revisions): @@ -149,7 +383,7 @@ def _direct_export_precedence( membership group carries no browser-capture provenance at all (``browser_snapshot_fidelity is None``) and at least one sibling is browser-capture-sourced, the non-browser candidate is authoritative for - session content regardless of byte-growth or provider-timestamp + session content regardless of set-containment or provider-timestamp comparisons; the browser-capture siblings are retained as raw provenance only, not indexed content (polylogue-z1c6). Returns ``None`` when this specific shape does not apply (zero or multiple non-browser @@ -195,60 +429,4 @@ def _browser_snapshot_dominates(older: MembershipRevision, newer: MembershipRevi ) -def _frontier(projection: SessionRevisionProjection) -> tuple[int, int, int, int]: - """Rank a revision along every axis on which it can only grow. - - Attachment acquisition is its own axis. Without it, a revision that added - nothing but the bytes of attachments it already referenced tied with its - predecessor on every dimension, the sort fell through to ``raw_id``, and the - dominance test was then run in whichever direction the hex happened to - order -- half the time backwards, against a revision that genuinely does - dominate (polylogue-bu1i). - """ - return ( - len(projection.message_hashes), - len(projection.event_hashes), - len(projection.attachment_identities), - len(projection.attachment_contents), - ) - - -def _attachment_evidence_preserved( - older: SessionRevisionProjection, - newer: SessionRevisionProjection, -) -> bool: - """True when ``newer`` loses no attachment and contradicts no fetched bytes. - - Two distinct regressions are refused here. An attachment whose bytes were - read and are no longer present in the newer revision is a fidelity - *downgrade*: the newer revision knows strictly less, so accepting it would - silently mark an acquired attachment unfetched. An attachment present in - both but carrying *different* bytes is a genuine conflict -- two sources - disagree about content under one identity -- and no ordering rule can - resolve that, so the cohort stays ambiguous. - """ - newer_contents = dict(newer.attachment_contents) - return older.attachment_identities <= newer.attachment_identities and all( - newer_contents.get(identity) == content for identity, content in older.attachment_contents - ) - - -def _strictly_dominates(older: SessionRevisionProjection, newer: SessionRevisionProjection) -> bool: - content_grew = ( - len(newer.message_hashes) > len(older.message_hashes) - or len(newer.event_hashes) > len(older.event_hashes) - or newer.attachment_identities > older.attachment_identities - # Resolving the bytes of an already-referenced attachment is growth in - # evidence even when the transcript is untouched, which is exactly the - # shape a lazily-fetched attachment produces on its second acquisition. - or len(newer.attachment_contents) > len(older.attachment_contents) - ) - return ( - content_grew - and older.message_hashes == newer.message_hashes[: len(older.message_hashes)] - and older.event_hashes == newer.event_hashes[: len(older.event_hashes)] - and _attachment_evidence_preserved(older, newer) - ) - - __all__ = ["MembershipClassification", "MembershipRevision", "classify_membership_revisions"] diff --git a/polylogue/pipeline/ids.py b/polylogue/pipeline/ids.py index c15f1d2f8e..a34fcdb421 100644 --- a/polylogue/pipeline/ids.py +++ b/polylogue/pipeline/ids.py @@ -23,7 +23,7 @@ # deferring the real import to whichever caller actually needs `sources`. if TYPE_CHECKING: from polylogue.sources import ParsedMessage, ParsedSession - from polylogue.sources.parsers.base import ParsedAttachment, ParsedContentBlock + from polylogue.sources.parsers.base import ParsedAttachment, ParsedContentBlock, ParsedSessionEvent # Sentinel values to distinguish None from empty in hash computations _NULL_SENTINEL = "__POLYLOGUE_NULL__" @@ -33,30 +33,70 @@ @dataclass(frozen=True, slots=True) class SessionRevisionProjection: - """Canonical content hashes used to prove append-only session growth. - - Attachments are projected on two axes deliberately, because folding them - into one hash makes acquisition look like divergence. ``attachment_identities`` - answers *which attachment is this* (provider id, anchoring message, name, - media type) and never changes once a provider has emitted the reference. - ``attachment_contents`` answers *have we read its bytes yet, and which bytes* - -- it carries an ``(identity, content)`` pair only for attachments whose - bytes are actually in hand. - - A single hash over both axes made an ordinary lazy fetch indistinguishable - from a branch: a Drive/Gemini document acquired before and after its - attachment bytes were resolved produced equal-cardinality *disjoint* hash - sets, so the later revision was neither a superset nor a prefix of the - earlier one and the whole cohort was quarantined as ambiguous. Splitting the - axes lets a dominance test say what is actually true -- same attachments, - strictly more of their bytes now known (polylogue-bu1i). + """Canonical content-only comparison value for a session revision. + + One invariant governs every field below: *a conversation is a SET of + items keyed by content-derived identity, each carrying only + content-bearing fields -- nothing else may enter the value used to + compare two acquisitions of it* (polylogue-aggz). Concretely: + + - Identity is never array position. ``message_contents``, + ``attachment_contents``, and ``event_contents`` are ``frozenset``s, not + ordered tuples -- a provider's export can replay an unchanged item set + in a different array sequence across separate export requests (proven + for Claude.ai messages and ChatGPT ``generation_lifecycle`` events: + same items, different sequence, every re-export), and a set has no + order to violate (polylogue-c429, polylogue-nuec). + - Identity is derived from content, never a provider id whose PRESENCE + is itself unstable. ``attachment_contents``' key is + ``(anchoring message, name, media type)`` -- never the provider's own + attachment id, which Claude.ai does not consistently emit for the same + attachment across export vintages (one vintage carries a real UUID, + the other has none) -- no id-minting scheme can make a real id and a + synthetic one collide, so the id is simply not part of identity + (polylogue-d8al, polylogue-hith). The same reasoning selects + ``(event type, anchoring message)`` for events, folding in the event's + own content hash only when that pair is ambiguous within one revision + (e.g. more than one ``chatgpt_block_metadata`` event on one message, + one per block) -- still content-derived, never the array index. + - Acquisition state and provider-reported measurement are not content. + An attachment's bytes may be known or not (``attachment_contents`` + omits an identity until its bytes are read, while + ``attachment_identities`` already knows the reference exists) -- + resolving them is evidence *growing*, not the attachment becoming a + different one (polylogue-bu1i). ChatGPT's ``generation_lifecycle`` + event re-derives ``elapsed_duration_ms`` from the raw export's own + timing metadata on every export request, and the value is not stable + across requests for the SAME generation even when the transcript is + byte-identical -- excluded from ``event_contents`` by + ``_EVENT_CONTENT_PAYLOAD_ALLOWLIST``, an explicit per-event-type + ALLOWLIST of content-bearing payload fields rather than a denylist of + fields discovered volatile after the fact (three volatility axes were + each found only after shipping: acquisition state, array order, + provider-reported duration -- a denylist means the next provider + quirk is silently invisible to comparison until someone notices and + files a bead; an allowlist means a NEW field a parser adds later + cannot silently enter identity without an explicit decision to add it). + + ``session_hash``, ``message_hashes`` (ordered), and ``event_hashes`` + (ordered, unstripped) are UNCHANGED by any of this: they still cover the + full, order-sensitive, unstripped payload, so a real reorder, a real + duration change, or newly-acquired bytes still change the session's + content hash and still trigger a re-write (idempotency is a different + question from revision *comparison*, and only the latter is + content-only). ``attachment_identities`` is kept as a plain + ``frozenset[bytes]`` of the same content-derived keys, read by + ``storage/repair.py``/``archive.py`` only via ``len()`` for a frontier + count. """ session_hash: bytes message_hashes: tuple[bytes, ...] + message_contents: frozenset[tuple[bytes, bytes]] attachment_identities: frozenset[bytes] attachment_contents: frozenset[tuple[bytes, bytes]] event_hashes: tuple[bytes, ...] + event_contents: frozenset[tuple[bytes, bytes]] def _normalize_nested_for_hash(value: object) -> object: @@ -169,14 +209,46 @@ def _message_hash_payload(message: ParsedMessage, message_id: str) -> dict[str, return payload -#: Fields of an attachment hash payload that answer *which attachment is this*, -#: as opposed to *what have we managed to read about it*. ``size_bytes`` is -#: excluded on purpose: for lazily-fetched attachments (Drive/Gemini references, -#: browser capture) the provider states no size until the bytes are actually -#: read, so treating it as identity makes acquisition look like a different -#: attachment. ``inline_content_hash`` is excluded for the same reason and is -#: recovered separately as acquisition evidence. -_ATTACHMENT_IDENTITY_FIELDS = ("id", "message_id", "name", "mime_type") +#: The one field of a message hash payload that answers *which message is +#: this*, as opposed to *what does it currently say*. A provider's own +#: message id is stable across re-exports even when the export's array +#: ordering is not (polylogue-c429). +_MESSAGE_IDENTITY_FIELDS = ("id",) + + +def _message_identity_payload(payload: dict[str, JSONValue]) -> dict[str, JSONValue]: + """Project the order-independent identity of one message payload. + + Reads the already-normalized value out of ``_message_hash_payload`` rather + than re-deriving it, mirroring ``_attachment_identity_payload``'s single + normalization site. + """ + return {field: payload[field] for field in _MESSAGE_IDENTITY_FIELDS} + + +#: Fields of an attachment hash payload that answer *which attachment is +#: this*, as opposed to *what have we managed to read about it* -- the +#: anchoring message plus content descriptors, content-derived and never the +#: provider's own attachment id. Claude.ai does not consistently emit +#: ``id``/``file_id``/``fileId``/``uuid``/``file_uuid`` for the same +#: attachment across separate export requests of the same conversation: one +#: vintage carries a real UUID-shaped id, the other has none (a positionally +#: -seeded synthetic id is not identity either -- polylogue-hith). No +#: id-minting scheme can make a real id and any synthetic value collide by +#: construction, so the id is excluded from identity altogether rather than +#: used when present (polylogue-d8al). ``size_bytes`` is excluded because for +#: lazily-fetched attachments (Drive/Gemini references, browser capture) the +#: provider states no size until the bytes are actually read, so treating it +#: as identity would make acquisition look like a different attachment +#: (polylogue-bu1i). ``inline_content_hash`` is excluded for the same reason +#: and is recovered separately as acquisition evidence in +#: ``attachment_contents``. +#: +#: Known, accepted limit stated explicitly rather than engineered around: two +#: genuinely distinct attachments that share one message/name/media-type and +#: carry no bytes on either side of a comparison are indistinguishable by any +#: signal this projection can offer. +_ATTACHMENT_IDENTITY_FIELDS = ("message_id", "name", "mime_type") def _attachment_hash_payload(attachment: ParsedAttachment) -> dict[str, JSONValue]: @@ -203,6 +275,111 @@ def _attachment_identity_payload(payload: dict[str, JSONValue]) -> dict[str, JSO return {field: payload[field] for field in _ATTACHMENT_IDENTITY_FIELDS} +#: `generation_lifecycle` payload keys that are provider-reported measurement, +#: not identity, when the event's own payload declares them non-durable via +#: ``duration_semantics == "provider_reported_elapsed"``. ChatGPT re-derives +#: these from the raw export's own ``finished_duration_sec`` / +#: ``reasoning_start_time``/``reasoning_end_time`` metadata on every export +#: request, and the value is not stable across requests for the SAME +#: generation (observed varying non-monotonically, e.g. 13000 vs 21000ms; +#: 123000 vs 33000ms) even when the transcript is byte-identical +#: (polylogue-nuec). +_PROVIDER_REPORTED_ELAPSED_VOLATILE_PAYLOAD_KEYS = frozenset({"elapsed_duration_ms", "started_at_ms", "ended_at_ms"}) +_PROVIDER_REPORTED_ELAPSED_MARKER_KEY = "duration_semantics" +_PROVIDER_REPORTED_ELAPSED_MARKER_VALUE = "provider_reported_elapsed" + +#: Event payload ALLOWLIST by event type: only these fields, per type, ever +#: enter ``event_contents``. This is deliberately an allowlist, not a +#: denylist of fields discovered volatile after the fact -- three separate +#: volatility axes (attachment acquisition state, message array order, +#: provider-reported generation duration) were each found only after +#: shipping, one bead and one branch at a time, because a denylist design +#: means a NEW field silently enters identity the moment a parser starts +#: emitting it, until someone notices and adds it to the strip-list. Under +#: an allowlist, a field a parser adds later is excluded from comparison by +#: construction -- it takes an explicit decision to add it here before it +#: can affect identity, not an explicit decision to exclude it. +#: +#: Event types with no entry here compare their FULL payload (today's +#: behavior for every type except ``generation_lifecycle``, which is the +#: only one with proven volatility -- polylogue-nuec; the raw export's own +#: ``finished_duration_sec``/``reasoning_start_time``/``reasoning_end_time`` +#: metadata is not stable across separate export requests for the SAME +#: generation, so ``elapsed_duration_ms``/``started_at_ms``/``ended_at_ms`` +#: and the derived ``timestamp`` are excluded; ``duration_semantics`` merely +#: documents that fact and carries no content of its own). +_EVENT_CONTENT_PAYLOAD_ALLOWLIST: dict[str, frozenset[str]] = { + "generation_lifecycle": frozenset({"state", "evidence_source", "fidelity"}), +} + + +def _event_content_payload(event: ParsedSessionEvent) -> dict[str, JSONValue]: + """Build the position- and measurement-independent CONTENT payload for one event. + + Array position is not identity for events any more than for messages: + ChatGPT's ``generation_lifecycle`` events were independently observed to + reorder alongside their duration values across separate export requests + of the SAME conversation (same three durations, different array + positions each time) -- the same volatility polylogue-c429 found for + messages, on a different axis (polylogue-nuec). This never includes + ``event_index``; ``session_revision_projection`` builds identity purely + from this content plus the event's own type and anchoring message. + + The allowlist strip is narrowed to the specific provider-remeasured + shape it targets: it applies only when the event's own payload declares + itself non-durable via ``duration_semantics == + "provider_reported_elapsed"`` (set by the ChatGPT parser). The + browser-capture parser emits the SAME ``event_type`` for its own DOM/UI + generation observations, tagged with a different ``duration_semantics`` + (e.g. ``dom_observed_wall``, ``provider_ui_elapsed``) -- those are a real + first-party measurement this projection has no other record of, not the + re-derived-on-every-export ChatGPT value nuec exists for, so their + observation id, timestamp, and duration/label/trigger fields remain + content rather than being silently stripped. + """ + allowlist = _EVENT_CONTENT_PAYLOAD_ALLOWLIST.get(event.event_type) + provider_reported_elapsed = ( + allowlist is not None + and event.payload.get(_PROVIDER_REPORTED_ELAPSED_MARKER_KEY) == _PROVIDER_REPORTED_ELAPSED_MARKER_VALUE + ) + if allowlist is None or not provider_reported_elapsed: + payload = event.payload + timestamp = event.timestamp + else: + payload = {key: value for key, value in event.payload.items() if key in allowlist} + # An event type with a registered allowlist also has its own + # provider-remeasured timestamp excluded: for generation_lifecycle, + # ChatGPT sets it from the same reasoning_end_time value the + # duration is derived from, so it varies in tandem and is + # measurement too, not content. + timestamp = None + return { + "event_type": _normalize_for_hash(event.event_type), + "timestamp": _normalize_for_hash(timestamp), + "source_message_provider_id": _normalize_for_hash(event.source_message_provider_id), + "payload": hash_payload(_normalize_nested_for_hash(payload)), + } + + +#: The subset of an event content payload that answers *which event slot is +#: this*, as opposed to *what does it say*: anchoring message plus event +#: type, content-derived and never the array index. Can be shared by more +#: than one event within one revision (e.g. multiple +#: ``chatgpt_block_metadata`` events on the same message, one per block), so +#: ``session_revision_projection`` always folds the event's own content hash +#: into the FINAL identity on top of this base -- unconditionally, not only +#: when a sibling is present in that particular revision, so identity never +#: depends on what else happens to be in the set. Still content-derived +#: (each block's own content, including any content-intrinsic field such as +#: ``block_index``, already differs), never the array position. +_EVENT_BASE_IDENTITY_FIELDS = ("event_type", "source_message_provider_id") + + +def _event_base_identity_payload(payload: dict[str, JSONValue]) -> dict[str, JSONValue]: + """Project the position-independent base correlation key of one event payload.""" + return {field: payload[field] for field in _EVENT_BASE_IDENTITY_FIELDS} + + def _session_hash_payload( *, title: str | None, @@ -312,6 +489,16 @@ def session_revision_projection(convo: ParsedSession) -> SessionRevisionProjecti included, so acquiring an attachment's bytes does change the session's content hash and does trigger a re-write. Only the *revision comparison* axes separate identity from acquisition (polylogue-bu1i). + + The same holds for message order (polylogue-c429), attachment identity + presence (polylogue-d8al), and provider-reported generation-duration + measurement (polylogue-nuec): ``session_hash`` still covers the full, + order-sensitive message array, the full attachment payload including + whatever id the provider did or didn't emit, and the full, unstripped + event payload/timestamp, so a real reorder, a real id change, or a real + duration change still triggers a re-write. Only ``message_contents`` / + ``attachment_identities`` / ``attachment_contents`` / ``event_contents`` + -- the *revision comparison* axes -- are content-only (polylogue-aggz). """ messages_payload, attachments_payload, session_events_payload = _session_hash_components(convo) session_hash_hex = _session_tree_hash( @@ -320,18 +507,58 @@ def session_revision_projection(convo: ParsedSession) -> SessionRevisionProjecti attachments_payload=attachments_payload, session_events_payload=session_events_payload, ) + message_contents: set[tuple[bytes, bytes]] = set() + message_hashes: list[bytes] = [] + for payload in messages_payload: + identity = bytes.fromhex(hash_payload(_message_identity_payload(payload))) + content = bytes.fromhex(hash_payload(payload)) + message_contents.add((identity, content)) + message_hashes.append(content) attachment_identities: set[bytes] = set() attachment_contents: set[tuple[bytes, bytes]] = set() for payload in attachments_payload: identity = bytes.fromhex(hash_payload(_attachment_identity_payload(payload))) - attachment_identities.add(identity) inline_content_hash = payload.get("inline_content_hash") + attachment_identities.add(identity) if isinstance(inline_content_hash, str): attachment_contents.add((identity, bytes.fromhex(inline_content_hash))) + event_hashes: list[bytes] = [] + event_base_identities: list[bytes] = [] + event_content_hashes: list[bytes] = [] + for payload, event in zip(session_events_payload, convo.session_events, strict=True): + event_hashes.append(bytes.fromhex(hash_payload(payload))) + content_payload = _event_content_payload(event) + event_base_identities.append(bytes.fromhex(hash_payload(_event_base_identity_payload(content_payload)))) + event_content_hashes.append(bytes.fromhex(hash_payload(content_payload))) + event_contents: set[tuple[bytes, bytes]] = set() + for base_identity, content_hash in zip(event_base_identities, event_content_hashes, strict=True): + # A base identity (event type + anchoring message) is ambiguous + # whenever it is EVER possible for more than one event to share it + # (e.g. one chatgpt_block_metadata event per block on a message), so + # the event's own content is always folded into identity here -- + # unconditionally, not only when a sibling happens to be present in + # THIS revision. An item's identity must not depend on what else is + # in the set: computing it from this revision's own sibling count + # made the same event's identity shift between `base_identity` (one + # instance) and `hash(base_identity, content)` (two or more) purely + # because a sibling appeared in a later revision, which made an + # ordinary event-growth revision compare as a disjoint conflict + # instead of containment. Folding content in always keeps identity + # intrinsic to the event itself: still content-derived, never the + # array index (distinct blocks already differ in content, e.g. a + # content-intrinsic block_index), and true duplicates (same base + # identity, same content, whether or not any sibling exists) + # correctly collapse to one set entry either way. + canonical_identity = bytes.fromhex( + hash_payload({"base_identity": base_identity.hex(), "content": content_hash.hex()}) + ) + event_contents.add((canonical_identity, content_hash)) return SessionRevisionProjection( session_hash=bytes.fromhex(session_hash_hex), - message_hashes=tuple(bytes.fromhex(hash_payload(payload)) for payload in messages_payload), + message_hashes=tuple(message_hashes), + message_contents=frozenset(message_contents), attachment_identities=frozenset(attachment_identities), attachment_contents=frozenset(attachment_contents), - event_hashes=tuple(bytes.fromhex(hash_payload(payload)) for payload in session_events_payload), + event_hashes=tuple(event_hashes), + event_contents=frozenset(event_contents), ) diff --git a/tests/unit/archive/test_session_revision_membership.py b/tests/unit/archive/test_session_revision_membership.py index 42b0b60e43..38d06fec2f 100644 --- a/tests/unit/archive/test_session_revision_membership.py +++ b/tests/unit/archive/test_session_revision_membership.py @@ -1,14 +1,18 @@ from __future__ import annotations +from itertools import permutations + from polylogue.archive.message.roles import Role from polylogue.archive.session_revision_membership import ( MembershipRevision, - _strictly_dominates, + _content_by_identity, + _maximal_evidence_fallback, + _relation, classify_membership_revisions, ) from polylogue.core.enums import Provider from polylogue.pipeline.ids import session_revision_projection -from polylogue.sources.parsers.base import ParsedAttachment, ParsedMessage, ParsedSession +from polylogue.sources.parsers.base import ParsedAttachment, ParsedMessage, ParsedSession, ParsedSessionEvent def _revision(raw_id: str, *texts: str) -> MembershipRevision: @@ -20,6 +24,24 @@ def _revision(raw_id: str, *texts: str) -> MembershipRevision: return MembershipRevision(raw_id, session_revision_projection(session)) +def _ordered_message_revision(raw_id: str, *id_text_pairs: tuple[str, str]) -> MembershipRevision: + """A session whose messages carry explicit provider ids in a given array order.""" + session = ParsedSession( + source_name=Provider.CLAUDE_AI, + provider_session_id="session", + messages=[ + ParsedMessage(provider_message_id=provider_id, role=Role.USER, text=text) + for provider_id, text in id_text_pairs + ], + ) + return MembershipRevision(raw_id, session_revision_projection(session)) + + +# --------------------------------------------------------------------------- +# Core equal/contains/conflict semantics (polylogue-aggz) +# --------------------------------------------------------------------------- + + def test_classifies_strict_growth_and_semantic_equivalence() -> None: result = classify_membership_revisions( [_revision("raw-b", "one", "two"), _revision("raw-z", "one"), _revision("raw-a", "one")] @@ -30,6 +52,15 @@ def test_classifies_strict_growth_and_semantic_equivalence() -> None: def test_refuses_divergent_maxima() -> None: + """Genuine divergence stays quarantined ambiguous. + + raw-b and raw-c both grew from raw-a in incompatible directions (each + holds a message the other lacks) -- a genuine fork, "conflict" under + ``_relation``. See ``_maximal_evidence_fallback`` for the designed, + unit-tested presence-guarantee alternative to this quarantine, and its + docstring for why it is not wired into ``classify_membership_revisions`` + yet. + """ result = classify_membership_revisions( [_revision("raw-a", "one"), _revision("raw-b", "one", "left"), _revision("raw-c", "one", "right")] ) @@ -37,6 +68,42 @@ def test_refuses_divergent_maxima() -> None: assert result.ambiguous_raw_ids == ("raw-a", "raw-b", "raw-c") +def test_maximal_evidence_fallback_picks_deterministically() -> None: + """The presence-guarantee fallback (not yet wired live) resolves a fork deterministically. + + Frontier tie among raw-b/raw-c falls through to the raw_id tiebreak + ("raw-c" > "raw-b"), same as the frontier-sort used elsewhere in this + module. + """ + representatives = [_revision("raw-a", "one"), _revision("raw-b", "one", "left"), _revision("raw-c", "one", "right")] + assert _maximal_evidence_fallback(representatives).raw_id == "raw-c" + + +def test_maximal_evidence_fallback_is_order_independent() -> None: + """The same genuinely-ambiguous cohort always resolves to the same head. + + A rebuild that processes raws in a different order must not produce a + different archive: every permutation of the same three divergent + revisions must pick the identical fallback head. ``_frontier`` plus the + stable raw_id tiebreak guarantees this -- it depends only on each + revision's own projected content, never on input order. + """ + revisions = [_revision("raw-a", "one"), _revision("raw-b", "one", "left"), _revision("raw-c", "one", "right")] + results = {_maximal_evidence_fallback(list(ordering)).raw_id for ordering in permutations(revisions)} + assert results == {"raw-c"} + + +def test_containment_chain_resolves_without_needing_the_fallback() -> None: + """A clean append-only growth chain resolves via set containment alone -- + accepting the WHOLE chain (both raws), not a single frontier-max pick, + which would silently discard the older revision's own head-of-chain + role. + """ + result = classify_membership_revisions([_revision("raw-old", "one"), _revision("raw-new", "one", "two")]) + assert result.accepted_raw_ids == ("raw-old", "raw-new") + assert result.ambiguous_raw_ids == () + + def test_metadata_only_revision_uses_latest_provider_timestamp() -> None: def revision(raw_id: str, updated_at: str | None, *, title: str = "title") -> MembershipRevision: session = ParsedSession( @@ -59,7 +126,16 @@ def revision(raw_id: str, updated_at: str | None, *, title: str = "title") -> Me assert result.ambiguous_raw_ids == () -def test_metadata_revision_without_complete_provider_time_is_ambiguous() -> None: +def test_metadata_revision_without_complete_provider_time_still_resolves_by_content() -> None: + """A title difference with an unresolvable timestamp no longer blocks presence. + + Both revisions carry the identical single message -- content equality + already proves these are the same revision (title is not part of + ``_relation`` at all); a missing timestamp is no longer a reason to + withhold a revision this comparison has already proven is the same + conversation. Deterministic tiebreak (min raw_id) picks a representative + when no timestamp can settle it. + """ with_timestamp = ParsedSession( source_name=Provider.CHATGPT, provider_session_id="session", @@ -80,11 +156,13 @@ def test_metadata_revision_without_complete_provider_time_is_ambiguous() -> None ] ) - assert result.accepted_raw_ids == () - assert result.ambiguous_raw_ids == ("raw-new", "raw-old") + assert result.accepted_raw_ids == ("raw-new",) + assert result.equivalent_raw_ids == ("raw-old",) + assert result.ambiguous_raw_ids == () -def test_metadata_revisions_with_equal_provider_time_are_ambiguous() -> None: +def test_metadata_revisions_with_equal_provider_time_still_resolve_by_content() -> None: + """A title difference under a TIED timestamp also resolves by content.""" timestamp = "2026-01-02T00:00:00Z" first = ParsedSession( source_name=Provider.CHATGPT, @@ -102,10 +180,691 @@ def test_metadata_revisions_with_equal_provider_time_are_ambiguous() -> None: ] ) + assert result.accepted_raw_ids == ("raw-a",) + assert result.equivalent_raw_ids == ("raw-b",) + assert result.ambiguous_raw_ids == () + + +# --------------------------------------------------------------------------- +# Messages: order-insensitive set containment (polylogue-c429) +# --------------------------------------------------------------------------- + + +def test_accepts_reordered_message_array_with_identical_content_as_equivalent() -> None: + """Same message ids, byte-identical content per id, different array order. + + Reproduces the exact shape measured on the live archive: Claude.ai's own + export ordering for a conversation is not guaranteed stable across + separate export requests -- 34 of 35 sampled claude-ai-export ambiguous + cohorts had identical message-id sets with zero per-id content + differences (polylogue-c429). A set has no order to violate; this must + resolve as equivalent content. + """ + chronological = _ordered_message_revision("raw-chrono", ("a", "one"), ("b", "two"), ("c", "three")) + resequenced = _ordered_message_revision("raw-resequenced", ("b", "two"), ("a", "one"), ("c", "three")) + + assert chronological.projection.message_hashes != resequenced.projection.message_hashes + assert chronological.projection.message_contents == resequenced.projection.message_contents + assert _relation(chronological.projection, resequenced.projection) == "equal" + + result = classify_membership_revisions([chronological, resequenced]) + + assert result.accepted_raw_ids == ("raw-chrono",) + assert result.equivalent_raw_ids == ("raw-resequenced",) + assert result.ambiguous_raw_ids == () + + +def test_accepts_message_growth_across_a_reordered_prefix() -> None: + """A genuinely appended message is still growth even when the shared ids reorder.""" + older = _ordered_message_revision("raw-old", ("b", "two"), ("a", "one")) + newer = _ordered_message_revision("raw-new", ("a", "one"), ("c", "three"), ("b", "two")) + + assert _relation(older.projection, newer.projection) == "b_contains_a" + + result = classify_membership_revisions([older, newer]) + assert result.accepted_raw_ids == ("raw-old", "raw-new") + assert result.ambiguous_raw_ids == () + + +def test_refuses_message_content_change_under_a_shared_id_despite_reorder() -> None: + """A real edit under a shared id is a conflict, not laundered by the set model.""" + older = _ordered_message_revision("raw-old", ("a", "one"), ("b", "two")) + newer = _ordered_message_revision("raw-new", ("b", "two"), ("a", "EDITED")) + + assert _relation(older.projection, newer.projection) == "conflict" + + result = classify_membership_revisions([older, newer]) + assert result.accepted_raw_ids == () + assert result.ambiguous_raw_ids == ("raw-new", "raw-old") + + +def test_refuses_message_id_disappearing_despite_reorder() -> None: + """A message id present in older but absent from newer is a real loss (a fork).""" + older = _ordered_message_revision("raw-old", ("a", "one"), ("b", "two")) + newer = _ordered_message_revision("raw-new", ("b", "two"), ("c", "three")) + + assert _relation(older.projection, newer.projection) == "conflict" + + result = classify_membership_revisions([older, newer]) + assert result.accepted_raw_ids == () + assert result.ambiguous_raw_ids == ("raw-new", "raw-old") + + +def test_message_reorder_does_change_the_session_content_hash() -> None: + """Order tolerance must not leak into idempotency. + + ``session_hash`` must stay order-sensitive: if two array orderings of the + same messages hashed identically, a real reorder written by the + archive's own writer path would be silently skipped as unchanged on + re-ingest. + """ + chronological = _ordered_message_revision("raw-chrono", ("a", "one"), ("b", "two")) + resequenced = _ordered_message_revision("raw-resequenced", ("b", "two"), ("a", "one")) + + assert chronological.projection.session_hash != resequenced.projection.session_hash + + +# --------------------------------------------------------------------------- +# Attachments: content-derived identity, acquisition growth (polylogue-bu1i, +# polylogue-d8al, polylogue-hith) +# --------------------------------------------------------------------------- + + +def _attachment_revision( + raw_id: str, + *, + acquired: bool, + provider_attachment_id: str = "drive-file-1", + inline: bytes = b"attachment bytes", + name: str = "screenshot.png", + mime_type: str = "image/png", +) -> MembershipRevision: + """A session whose single attachment is referenced, optionally with bytes read.""" + attachment = ParsedAttachment( + provider_attachment_id=provider_attachment_id, + message_provider_id="0", + name=name, + mime_type=mime_type, + size_bytes=len(inline) if acquired else None, + inline_bytes=inline if acquired else None, + ) + session = ParsedSession( + source_name=Provider.GEMINI, + provider_session_id="session", + messages=[ParsedMessage(provider_message_id="0", role=Role.USER, text="one")], + attachments=[attachment], + ) + return MembershipRevision(raw_id, session_revision_projection(session)) + + +def test_accepts_attachment_byte_acquisition_as_growth_not_a_branch() -> None: + """Resolving an already-referenced attachment's bytes is a fidelity upgrade. + + A Drive/Gemini document acquired twice -- once before its attachment + bytes were fetched, once after -- has an identical transcript and + attachment reference. All 157 two-member aistudio-drive ambiguous + cohorts in the live archive had exactly this shape (polylogue-bu1i). + """ + bare = _attachment_revision("raw-bbbb", acquired=False) + enriched = _attachment_revision("raw-aaaa", acquired=True) + + assert _relation(bare.projection, enriched.projection) == "b_contains_a" + + result = classify_membership_revisions([enriched, bare]) + # Ordering matters as much as acceptance: the revision holding the bytes + # must be the head of the chain, or the archive indexes the emptier one. + assert result.accepted_raw_ids == ("raw-bbbb", "raw-aaaa") + assert result.ambiguous_raw_ids == () + + +def test_refuses_attachment_byte_loss_as_a_fidelity_downgrade() -> None: + """Dropping bytes already in hand is never an upgrade, in either direction.""" + enriched = _attachment_revision("raw-a", acquired=True) + bare = _attachment_revision("raw-b", acquired=False) + + assert _relation(enriched.projection, bare.projection) == "a_contains_b" + assert _relation(bare.projection, enriched.projection) == "b_contains_a" + + result = classify_membership_revisions([enriched, bare]) + assert result.accepted_raw_ids == ("raw-b", "raw-a") + + +def test_refuses_conflicting_bytes_under_one_attachment_identity() -> None: + """Two sources disagreeing about one attachment's content is a conflict.""" + left = _attachment_revision("raw-a", acquired=True, inline=b"left bytes") + right = _attachment_revision("raw-b", acquired=True, inline=b"right bytes") + + assert _relation(left.projection, right.projection) == "conflict" + + result = classify_membership_revisions([left, right]) assert result.accepted_raw_ids == () assert result.ambiguous_raw_ids == ("raw-a", "raw-b") +def test_attachment_acquisition_does_change_the_session_content_hash() -> None: + """Acquisition is invisible to *revision comparison*, not to *idempotency*.""" + bare = _attachment_revision("raw-a", acquired=False) + enriched = _attachment_revision("raw-b", acquired=True) + + assert bare.projection.session_hash != enriched.projection.session_hash + assert bare.projection.attachment_identities == enriched.projection.attachment_identities + assert bare.projection.attachment_contents == frozenset() + assert len(enriched.projection.attachment_contents) == 1 + + +def test_accepts_real_and_synthetic_id_variants_of_the_same_attachment_as_equivalent() -> None: + """Same physical attachment, a real id on one export vintage, none on the other. + + Attachment identity is content-derived (anchoring message, name, media + type) and never the provider's own attachment id (polylogue-d8al, + polylogue-hith): Claude.ai does not consistently emit a real attachment + id across separate export requests of the same conversation. No + id-minting scheme can make a real id and a synthetic hash collide, so + the id was never part of identity to begin with. + """ + real_id = _attachment_revision("raw-real", acquired=False, provider_attachment_id="e950263f-51d1-4b7a-9c2e-0") + synthetic_id = _attachment_revision("raw-synthetic", acquired=False, provider_attachment_id="att-ce21cd12d650") + + assert real_id.projection.attachment_identities == synthetic_id.projection.attachment_identities + assert _relation(real_id.projection, synthetic_id.projection) == "equal" + + result = classify_membership_revisions([real_id, synthetic_id]) + + assert result.accepted_raw_ids == ("raw-real",) + assert result.equivalent_raw_ids == ("raw-synthetic",) + assert result.ambiguous_raw_ids == () + + +def test_refuses_to_guess_when_two_distinct_attachments_share_message_name_and_type() -> None: + """Two distinct attachments sharing (message, name, media type) are indistinguishable. + + This is the documented, accepted limit (polylogue-d8al, polylogue-aggz): + with no bytes on either side, there is no signal left to tell two + same-metadata attachments apart, so they collapse to ONE identity by + construction. Stated plainly rather than engineered around. + """ + a = _attachment_revision("raw-a", acquired=False, provider_attachment_id="uuid-1") + b = _attachment_revision("raw-b", acquired=False, provider_attachment_id="uuid-2") + assert a.projection.attachment_identities == b.projection.attachment_identities + + +def _colliding_attachments_revision(raw_id: str, *contents: bytes) -> MembershipRevision: + """A session whose attachments all share (message, name, mime) but carry distinct bytes.""" + attachments = [ + ParsedAttachment( + provider_attachment_id=f"att-{i}", + message_provider_id="0", + name="screenshot.png", + mime_type="image/png", + size_bytes=len(content), + inline_bytes=content, + ) + for i, content in enumerate(contents) + ] + session = ParsedSession( + source_name=Provider.GEMINI, + provider_session_id="session", + messages=[ParsedMessage(provider_message_id="0", role=Role.USER, text="one")], + attachments=attachments, + ) + return MembershipRevision(raw_id, session_revision_projection(session)) + + +def test_content_by_identity_retains_every_value_under_a_colliding_identity() -> None: + """Grouping must be a set per identity, not a dict entry that silently overwrites. + + Directly exercises the review-found bug: a plain ``dict(contents)`` maps + a colliding identity to whichever (identity, content) pair happened to + be visited last, discarding the other content hash entirely. The fixed + grouping must retain BOTH. + """ + contents = frozenset({(b"identity-1", b"content-a"), (b"identity-1", b"content-b"), (b"identity-2", b"content-c")}) + + grouped = _content_by_identity(contents) + + assert grouped == { + b"identity-1": frozenset({b"content-a", b"content-b"}), + b"identity-2": frozenset({b"content-c"}), + } + + +def test_refuses_to_equate_colliding_attachment_identities_with_different_bytes() -> None: + """Two acquired attachments sharing one identity but different bytes on one + message must degrade to conflict, never silently compare as equal or + let the second collapse invisibly into the first. + + Reproduces the review-found bug where `_axis_relation` built the + identity->content mapping via plain `dict()`, so the second colliding + attachment's content hash silently overwrote the first -- a real + conflict could then compare as equal, strictly worse than either honest + outcome (equal or conflict), and beyond the documented limitation (which + only claims byte-LESS duplicates are indistinguishable). + """ + both = _colliding_attachments_revision("raw-both", b"left bytes", b"right bytes") + left_only = _colliding_attachments_revision("raw-left-only", b"left bytes") + + assert both.projection.attachment_identities == left_only.projection.attachment_identities + assert len(both.projection.attachment_contents) == 2 + assert _relation(both.projection, left_only.projection) == "conflict" + + result = classify_membership_revisions([both, left_only]) + assert result.accepted_raw_ids == () + assert result.ambiguous_raw_ids == ("raw-both", "raw-left-only") + + +# --------------------------------------------------------------------------- +# Events: order-insensitive, measurement-excluded set containment +# (polylogue-nuec) +# --------------------------------------------------------------------------- + + +def _generation_lifecycle_revision(raw_id: str, elapsed_duration_ms: int) -> MembershipRevision: + """A ChatGPT-shaped session with one `generation_lifecycle` event.""" + session = ParsedSession( + source_name=Provider.CHATGPT, + provider_session_id="session", + messages=[ParsedMessage(provider_message_id="0", role=Role.ASSISTANT, text="answer")], + session_events=[ + ParsedSessionEvent( + event_type="generation_lifecycle", + timestamp=str(elapsed_duration_ms / 1000), + source_message_provider_id="0", + payload={ + "state": "completed", + "evidence_source": "provider_native", + "fidelity": "exact", + "duration_semantics": "provider_reported_elapsed", + "elapsed_duration_ms": elapsed_duration_ms, + }, + ) + ], + ) + return MembershipRevision(raw_id, session_revision_projection(session)) + + +def test_accepts_generation_lifecycle_duration_change_as_equivalent() -> None: + """Byte-identical transcript, only a provider-remeasured duration differs. + + Reproduces the shape measured on the live archive: 33 of 35 sampled + chatgpt-export ambiguous cohorts had identical messages and attachments + but a `generation_lifecycle` event whose `elapsed_duration_ms` varied + non-monotonically between export requests for the SAME generation + (polylogue-nuec). Excluded from ``event_contents`` by an explicit + per-event-type ALLOWLIST of content-bearing fields, not a denylist of + fields discovered volatile after the fact. + """ + first_export = _generation_lifecycle_revision("raw-first", 13000) + second_export = _generation_lifecycle_revision("raw-second", 21000) + + assert first_export.projection.event_hashes != second_export.projection.event_hashes + assert first_export.projection.event_contents == second_export.projection.event_contents + assert first_export.projection.session_hash != second_export.projection.session_hash + assert _relation(first_export.projection, second_export.projection) == "equal" + + result = classify_membership_revisions([first_export, second_export]) + + assert result.accepted_raw_ids == ("raw-first",) + assert result.equivalent_raw_ids == ("raw-second",) + assert result.ambiguous_raw_ids == () + + +def test_generation_lifecycle_duration_change_does_change_the_session_content_hash() -> None: + """Measurement tolerance must not leak into idempotency.""" + first_export = _generation_lifecycle_revision("raw-first", 13000) + second_export = _generation_lifecycle_revision("raw-second", 21000) + assert first_export.projection.session_hash != second_export.projection.session_hash + + +def test_refuses_generation_lifecycle_state_change_despite_duration_tolerance() -> None: + """Duration tolerance must not launder every field of the event as noise. + + Only the allowlisted fields (state, evidence_source, fidelity) are + content -- a genuinely different ``state`` on the same event is real + divergence. + """ + session = ParsedSession( + source_name=Provider.CHATGPT, + provider_session_id="session", + messages=[ParsedMessage(provider_message_id="0", role=Role.ASSISTANT, text="answer")], + session_events=[ + ParsedSessionEvent( + event_type="generation_lifecycle", + timestamp="13.0", + source_message_provider_id="0", + payload={ + "state": "completed", + "evidence_source": "provider_native", + "fidelity": "exact", + "duration_semantics": "provider_reported_elapsed", + "elapsed_duration_ms": 13000, + }, + ) + ], + ) + changed_state = session.model_copy( + update={ + "session_events": [ + ParsedSessionEvent( + event_type="generation_lifecycle", + timestamp="13.0", + source_message_provider_id="0", + payload={ + "state": "in_progress", + "evidence_source": "provider_native", + "fidelity": "exact", + "duration_semantics": "provider_reported_elapsed", + "elapsed_duration_ms": 13000, + }, + ) + ] + } + ) + older = MembershipRevision("raw-old", session_revision_projection(session)) + newer = MembershipRevision("raw-new", session_revision_projection(changed_state)) + + assert older.projection.event_contents != newer.projection.event_contents + assert _relation(older.projection, newer.projection) == "conflict" + + result = classify_membership_revisions([older, newer]) + assert result.accepted_raw_ids == () + assert result.ambiguous_raw_ids == ("raw-new", "raw-old") + + +def test_non_allowlisted_event_type_keeps_its_full_payload_as_content() -> None: + """The allowlist is scoped to `generation_lifecycle` -- an unrelated event + type with no registered allowlist compares its FULL payload, so a real + difference there still counts as divergence. + """ + session = ParsedSession( + source_name=Provider.CHATGPT, + provider_session_id="session", + messages=[ParsedMessage(provider_message_id="0", role=Role.ASSISTANT, text="answer")], + session_events=[ + ParsedSessionEvent( + event_type="chatgpt_block_metadata", + timestamp="13.0", + source_message_provider_id="0", + payload={"elapsed_duration_ms": 13000}, + ) + ], + ) + other_value = session.model_copy( + update={ + "session_events": [ + ParsedSessionEvent( + event_type="chatgpt_block_metadata", + timestamp="13.0", + source_message_provider_id="0", + payload={"elapsed_duration_ms": 21000}, + ) + ] + } + ) + older = MembershipRevision("raw-old", session_revision_projection(session)) + newer = MembershipRevision("raw-new", session_revision_projection(other_value)) + + assert older.projection.event_contents != newer.projection.event_contents + assert _relation(older.projection, newer.projection) == "conflict" + + result = classify_membership_revisions([older, newer]) + assert result.accepted_raw_ids == () + assert result.ambiguous_raw_ids == ("raw-new", "raw-old") + + +def test_accepts_reordered_events_with_identical_content_as_equivalent() -> None: + """Same three durations, different array positions -- observed directly on + the live archive for `generation_lifecycle` events across two ChatGPT + export requests of the SAME conversation (polylogue-nuec). + """ + + def session_with(order: list[int]) -> ParsedSession: + return ParsedSession( + source_name=Provider.CHATGPT, + provider_session_id="session", + messages=[ParsedMessage(provider_message_id="0", role=Role.ASSISTANT, text="answer")], + session_events=[ + ParsedSessionEvent( + event_type="generation_lifecycle", + timestamp=str(duration / 1000), + source_message_provider_id=f"m{duration}", + payload={ + "state": "completed", + "evidence_source": "provider_native", + "fidelity": "exact", + "duration_semantics": "provider_reported_elapsed", + "elapsed_duration_ms": duration, + }, + ) + for duration in order + ], + ) + + forward = MembershipRevision("raw-forward", session_revision_projection(session_with([161000, 79000, 115000]))) + shuffled = MembershipRevision("raw-shuffled", session_revision_projection(session_with([79000, 115000, 161000]))) + + assert forward.projection.event_hashes != shuffled.projection.event_hashes + assert forward.projection.event_contents == shuffled.projection.event_contents + assert _relation(forward.projection, shuffled.projection) == "equal" + + +def test_refuses_event_growth_that_loses_an_existing_event() -> None: + """A genuinely appended event alongside a genuinely lost one is a fork.""" + shared_event = ParsedSessionEvent( + event_type="generation_lifecycle", + timestamp="13.0", + source_message_provider_id="0", + payload={ + "state": "completed", + "evidence_source": "provider_native", + "fidelity": "exact", + "duration_semantics": "provider_reported_elapsed", + "elapsed_duration_ms": 13000, + }, + ) + other_event = ParsedSessionEvent( + event_type="chatgpt_block_metadata", + timestamp="1.0", + source_message_provider_id="0", + payload={"block_index": 0}, + ) + older_session = ParsedSession( + source_name=Provider.CHATGPT, + provider_session_id="session", + messages=[ParsedMessage(provider_message_id="0", role=Role.ASSISTANT, text="answer")], + session_events=[shared_event], + ) + newer_session = older_session.model_copy(update={"session_events": [other_event]}) + + older = MembershipRevision("raw-old", session_revision_projection(older_session)) + newer = MembershipRevision("raw-new", session_revision_projection(newer_session)) + + assert _relation(older.projection, newer.projection) == "conflict" + + result = classify_membership_revisions([older, newer]) + assert result.accepted_raw_ids == () + assert result.ambiguous_raw_ids == ("raw-new", "raw-old") + + +def test_disambiguates_multiple_same_type_events_on_one_message_by_content() -> None: + """Multiple same-type events on one message (e.g. one chatgpt_block_metadata + per block) are distinguished by their OWN content, never by array + position. + """ + + def block_event(index: int) -> ParsedSessionEvent: + return ParsedSessionEvent( + event_type="chatgpt_block_metadata", + timestamp="1.0", + source_message_provider_id="0", + payload={"block_index": index}, + ) + + session = ParsedSession( + source_name=Provider.CHATGPT, + provider_session_id="session", + messages=[ParsedMessage(provider_message_id="0", role=Role.ASSISTANT, text="answer")], + session_events=[block_event(0), block_event(1)], + ) + reordered = session.model_copy(update={"session_events": [block_event(1), block_event(0)]}) + + older = MembershipRevision("raw-old", session_revision_projection(session)) + newer = MembershipRevision("raw-new", session_revision_projection(reordered)) + + assert len(older.projection.event_contents) == 2 + assert older.projection.event_contents == newer.projection.event_contents + assert _relation(older.projection, newer.projection) == "equal" + + +def test_event_identity_is_stable_when_a_sibling_appears_later() -> None: + """An event's identity must not depend on what else is in the set. + + Reproduces the review-found bug where an event's canonical identity + shifted from `base_identity` (unique within the revision) to + `hash(base_identity, content)` purely because a SECOND event later + shared its base identity -- so an ordinary event-growth revision (one + block event, then a second block appended in a later export) compared + as a disjoint conflict instead of containment, reintroducing exactly the + identity-instability bug class polylogue-aggz exists to eliminate. + """ + + def block_event(index: int) -> ParsedSessionEvent: + return ParsedSessionEvent( + event_type="chatgpt_block_metadata", + timestamp="1.0", + source_message_provider_id="0", + payload={"block_index": index}, + ) + + single_session = ParsedSession( + source_name=Provider.CHATGPT, + provider_session_id="session", + messages=[ParsedMessage(provider_message_id="0", role=Role.ASSISTANT, text="answer")], + session_events=[block_event(0)], + ) + grown_session = single_session.model_copy(update={"session_events": [block_event(0), block_event(1)]}) + + older = MembershipRevision("raw-old", session_revision_projection(single_session)) + newer = MembershipRevision("raw-new", session_revision_projection(grown_session)) + + assert older.projection.event_contents <= newer.projection.event_contents + assert _relation(older.projection, newer.projection) == "b_contains_a" + + result = classify_membership_revisions([newer, older]) + assert result.accepted_raw_ids == ("raw-old", "raw-new") + assert result.ambiguous_raw_ids == () + + +def _browser_generation_lifecycle_revision( + raw_id: str, observation_id: str, wall_elapsed_ms: int +) -> MembershipRevision: + """A browser-capture-shaped session with one DOM-observed `generation_lifecycle` event.""" + session = ParsedSession( + source_name=Provider.CHATGPT, + provider_session_id="session", + messages=[ParsedMessage(provider_message_id="0", role=Role.ASSISTANT, text="answer")], + session_events=[ + ParsedSessionEvent( + event_type="generation_lifecycle", + timestamp="2026-01-01T00:00:00Z", + source_message_provider_id="0", + payload={ + "observation_id": observation_id, + "state": "completed", + "evidence_source": "dom_observation", + "fidelity": "observed", + "duration_semantics": "dom_observed_wall", + "wall_elapsed_ms": wall_elapsed_ms, + }, + ) + ], + ) + return MembershipRevision(raw_id, session_revision_projection(session)) + + +def test_browser_observed_generation_lifecycle_duration_is_real_content_not_stripped() -> None: + """The duration-stripping allowlist targets ChatGPT's own re-derived + measurement (`duration_semantics == "provider_reported_elapsed"`), not + every `generation_lifecycle` event regardless of source. + + Reproduces the review-found bug where the allowlist strip was keyed on + `event_type` alone, so it also stripped browser-capture's own DOM/UI + generation observations -- this projection's only record of a real + first-party measurement, not the provider-remeasured-on-every-export + value polylogue-nuec targeted. Two observations with different + observation ids and wall-clock durations, but IDENTICAL + state/evidence_source/fidelity (the fields the allowlist would keep), + must compare as a genuine conflict, not equivalent -- if the allowlist + strip fired here, both would collapse to the same stripped payload and + compare equal. + """ + left = _browser_generation_lifecycle_revision("raw-a", "obs-1", 4200) + right = _browser_generation_lifecycle_revision("raw-b", "obs-2", 9100) + + assert _relation(left.projection, right.projection) == "conflict" + + result = classify_membership_revisions([left, right]) + assert result.accepted_raw_ids == () + assert result.ambiguous_raw_ids == ("raw-a", "raw-b") + + +# --------------------------------------------------------------------------- +# Cross-axis fork detection +# --------------------------------------------------------------------------- + + +def test_conflict_on_one_axis_forces_overall_conflict_even_when_others_agree() -> None: + """A single-axis conflict (here: attachments) must not be masked by the + other two axes agreeing. + """ + left = _attachment_revision("raw-a", acquired=True, inline=b"left bytes") + right = _attachment_revision("raw-b", acquired=True, inline=b"right bytes") + # Messages and events are identical (both sessions built the same way); + # only the attachment bytes conflict. + assert left.projection.message_contents == right.projection.message_contents + assert _relation(left.projection, right.projection) == "conflict" + + +def test_mixed_direction_axes_is_a_conflict_not_a_pick() -> None: + """One axis growing forward while another grows backward is a fork. + + raw-a has an extra message but is missing an attachment raw-b has -- + neither strictly contains the other overall, even though each + individual axis alone looks like ordinary growth in ONE direction. + """ + attachment = ParsedAttachment( + provider_attachment_id="uuid-1", + message_provider_id="0", + name="a.png", + mime_type="image/png", + ) + session_a = ParsedSession( + source_name=Provider.CHATGPT, + provider_session_id="session", + messages=[ + ParsedMessage(provider_message_id="0", role=Role.USER, text="one"), + ParsedMessage(provider_message_id="1", role=Role.ASSISTANT, text="two"), + ], + attachments=[], + ) + session_b = session_a.model_copy(update={"messages": session_a.messages[:1], "attachments": [attachment]}) + + a = MembershipRevision("raw-a", session_revision_projection(session_a)) + b = MembershipRevision("raw-b", session_revision_projection(session_b)) + + assert _relation(a.projection, b.projection) == "conflict" + + result = classify_membership_revisions([a, b]) + assert result.accepted_raw_ids == () + assert result.ambiguous_raw_ids == ("raw-a", "raw-b") + + +# --------------------------------------------------------------------------- +# Browser-capture fidelity ordering and direct-export precedence (unchanged +# escape hatches -- see _provider_ordered_browser_snapshots's docstring for +# why this one survives polylogue-aggz's content-only model) +# --------------------------------------------------------------------------- + + def test_browser_native_snapshot_accepts_later_provider_revision_when_messages_reorder() -> None: older = _revision("raw-old", "prompt", "attachment context") newer = _revision("raw-new", "prompt", "tool result", "attachment context") @@ -156,6 +915,9 @@ def test_browser_native_snapshot_refuses_later_revision_that_loses_message_ident result = classify_membership_revisions(revisions) + # Genuine divergence (disjoint message identities) stays quarantined + # ambiguous -- neither the browser-fidelity ordering nor direct-export + # precedence can order it. assert result.accepted_raw_ids == () assert result.ambiguous_raw_ids == ("raw-new", "raw-old") @@ -165,7 +927,9 @@ def test_browser_native_upgrade_refuses_any_shrinking_frontier_dimension() -> No older_projection = older.projection.__class__( session_hash=b"o" * 32, message_hashes=older.projection.message_hashes, + message_contents=older.projection.message_contents, event_hashes=older.projection.event_hashes, + event_contents=older.projection.event_contents, attachment_identities=frozenset({b"attachment"}), attachment_contents=frozenset(), ) @@ -189,14 +953,19 @@ def test_browser_native_upgrade_refuses_any_shrinking_frontier_dimension() -> No result = classify_membership_revisions(revisions) + # Mixed-direction axes (newer gained a message, older's-only attachment + # is now missing) is a fork -- neither browser-fidelity ordering (the + # dom->native frontier check requires every dimension to be >=) nor + # direct-export precedence can order it, so it stays quarantined + # ambiguous. assert result.accepted_raw_ids == () assert result.ambiguous_raw_ids == ("raw-new", "raw-old") def test_direct_export_outranks_browser_capture_siblings_regardless_of_growth() -> None: """A genuine non-browser-capture revision always wins over dom/native - browser-capture siblings, even though its content is neither a byte/hash - -growth superset of them nor resolvable by provider-timestamp comparison. + browser-capture siblings, even though its content is neither a + growth superset of them nor resolvable by provider-timestamp comparison. Browser capture exists to backfill a session before its paired direct/ native provider export shows up, never to compete with or shadow that export once it arrives (polylogue-z1c6).""" @@ -221,13 +990,80 @@ def test_direct_export_outranks_browser_capture_siblings_regardless_of_growth() assert result.ambiguous_raw_ids == () +def test_equal_content_collapse_prefers_direct_export_over_browser_capture() -> None: + """Equal content is not equal authority: a direct export must survive over + a browser capture that happens to project to identical content, even + when timestamp and raw_id both favor the capture. + + Reproduces the review-found bug where the representative for two + revisions already proven `equal` was picked by provider timestamp (and, + failing that, raw_id) alone, before any source-authority ordering ran -- + so a later-timestamped, lexically-earlier-raw_id browser DOM scrape + could supersede an earlier direct/native export projecting to the exact + same content, even though a direct export always outranks a browser + capture for authority (`_direct_export_precedence`). + """ + export_projection = _revision("raw-export", "one", "two").projection + capture_projection = _revision("raw-capture", "one", "two").projection + export = MembershipRevision( + "raw-zzzz-export", export_projection, provider_updated_at="2024-01-01T00:00:00Z", browser_snapshot_fidelity=None + ) + capture = MembershipRevision( + "raw-aaaa-capture", + capture_projection, + provider_updated_at="2024-06-01T00:00:00Z", + browser_snapshot_fidelity="dom", + ) + + assert _relation(export.projection, capture.projection) == "equal" + + # Both the (later) timestamp and the (lexically smaller) raw_id favor + # the capture -- only source-authority ordering can make the export win. + result = classify_membership_revisions([export, capture]) + + assert result.accepted_raw_ids == ("raw-zzzz-export",) + assert result.equivalent_raw_ids == ("raw-aaaa-capture",) + assert result.ambiguous_raw_ids == () + + +def test_equal_content_collapse_prefers_native_snapshot_over_dom_snapshot() -> None: + """Among two equal-content browser captures, native outranks DOM. + + Same defect as the direct-export case above, one authority tier down: + fidelity ordering must also run before the timestamp/raw_id tiebreak. + """ + dom_projection = _revision("raw-dom", "one").projection + native_projection = _revision("raw-native", "one").projection + dom = MembershipRevision( + "raw-zzzz-dom", dom_projection, provider_updated_at="2024-06-01T00:00:00Z", browser_snapshot_fidelity="dom" + ) + native = MembershipRevision( + "raw-aaaa-native", + native_projection, + provider_updated_at="2024-01-01T00:00:00Z", + browser_snapshot_fidelity="native", + ) + + assert _relation(dom.projection, native.projection) == "equal" + + # Both the (later) timestamp and the (lexically smaller) raw_id favor + # the dom snapshot -- only fidelity ordering can make native win. + result = classify_membership_revisions([dom, native]) + + assert result.accepted_raw_ids == ("raw-aaaa-native",) + assert result.equivalent_raw_ids == ("raw-zzzz-dom",) + assert result.ambiguous_raw_ids == () + + def test_browser_snapshot_accepts_later_attachment_enrichment_without_provider_update() -> None: older = _revision("raw-old", "prompt", "answer") newer = _revision("raw-new", "prompt", "answer") newer_projection = newer.projection.__class__( session_hash=b"n" * 32, message_hashes=newer.projection.message_hashes, + message_contents=newer.projection.message_contents, event_hashes=newer.projection.event_hashes, + event_contents=newer.projection.event_contents, attachment_identities=frozenset({b"attachment-v2"}), attachment_contents=frozenset(), ) @@ -256,149 +1092,3 @@ def test_browser_snapshot_accepts_later_attachment_enrichment_without_provider_u assert result.accepted_raw_ids == ("raw-old", "raw-new") assert result.ambiguous_raw_ids == () - - -def _attachment_revision( - raw_id: str, - *, - acquired: bool, - provider_attachment_id: str = "drive-file-1", - inline: bytes = b"attachment bytes", -) -> MembershipRevision: - """A session whose single attachment is referenced, optionally with bytes read. - - Models the shape a lazily-fetched attachment actually produces: the provider - emits a bare reference (no size, no bytes), and a later acquisition pass - resolves the same reference into real bytes. The transcript is untouched - either way -- only the attachment's acquisition state differs. - """ - attachment = ParsedAttachment( - provider_attachment_id=provider_attachment_id, - message_provider_id="0", - size_bytes=len(inline) if acquired else None, - inline_bytes=inline if acquired else None, - ) - session = ParsedSession( - source_name=Provider.GEMINI, - provider_session_id="session", - messages=[ParsedMessage(provider_message_id="0", role=Role.USER, text="one")], - attachments=[attachment], - ) - return MembershipRevision(raw_id, session_revision_projection(session)) - - -def test_accepts_attachment_byte_acquisition_as_growth_not_a_branch() -> None: - """Resolving an already-referenced attachment's bytes is a fidelity upgrade. - - A Drive/Gemini document acquired twice -- once before its attachment bytes - were fetched, once after -- has an identical transcript and an identical - attachment reference. Folding acquisition state into the attachment identity - hash made the two revisions look like equal-sized disjoint branches, so the - whole cohort was quarantined as ambiguous and neither revision was indexed - (polylogue-bu1i). Measured on the live archive: all 157 two-member - aistudio-drive cohorts had exactly this shape. - """ - bare = _attachment_revision("raw-bbbb", acquired=False) - enriched = _attachment_revision("raw-aaaa", acquired=True) - - result = classify_membership_revisions([enriched, bare]) - - # Ordering matters as much as acceptance: the revision holding the bytes must - # be the head of the chain, or the archive indexes the emptier one. The raw - # ids are chosen so plain lexical ordering would put the enriched revision - # first, which is the direction the old frontier tie fell through to. - assert result.accepted_raw_ids == ("raw-bbbb", "raw-aaaa") - assert result.ambiguous_raw_ids == () - - -def test_refuses_attachment_byte_loss_as_a_fidelity_downgrade() -> None: - """Dropping bytes already in hand is never an upgrade, in either direction. - - Without this, an acquisition regression would look like ordinary growth - running backwards and could be accepted, silently re-marking a fetched - attachment as unfetched -- the concrete harm observed on five live sessions. - """ - enriched = _attachment_revision("raw-a", acquired=True) - bare = _attachment_revision("raw-b", acquired=False) - - result = classify_membership_revisions([enriched, bare]) - - assert result.accepted_raw_ids == ("raw-b", "raw-a") - - # And the downgrade direction on its own is refused outright. - assert not _strictly_dominates(enriched.projection, bare.projection) - - -def test_refuses_conflicting_bytes_under_one_attachment_identity() -> None: - """Two sources disagreeing about one attachment's content stays ambiguous. - - Splitting identity from acquisition must not turn a genuine conflict into an - upgrade: when both revisions have read bytes for the same attachment and the - bytes differ, no ordering rule can decide which is authoritative. - """ - left = _attachment_revision("raw-a", acquired=True, inline=b"left bytes") - right = _attachment_revision("raw-b", acquired=True, inline=b"right bytes") - - result = classify_membership_revisions([left, right]) - - assert result.accepted_raw_ids == () - assert result.ambiguous_raw_ids == ("raw-a", "raw-b") - - -def test_attachment_acquisition_does_change_the_session_content_hash() -> None: - """Acquisition is invisible to *revision comparison*, not to *idempotency*. - - The split must not leak into ``session_hash``: if acquiring bytes did not - change the session's content hash, a re-ingest carrying newly-fetched - attachments would be skipped as unchanged and the bytes would never land. - """ - bare = _attachment_revision("raw-a", acquired=False) - enriched = _attachment_revision("raw-b", acquired=True) - - assert bare.projection.session_hash != enriched.projection.session_hash - assert bare.projection.attachment_identities == enriched.projection.attachment_identities - assert bare.projection.attachment_contents == frozenset() - assert len(enriched.projection.attachment_contents) == 1 - - -def test_refuses_contradicted_bytes_even_when_the_revision_otherwise_grew() -> None: - """Growth elsewhere must not launder a contradiction about bytes already read. - - The equal-cardinality conflict case is already refused by the growth test - itself, which leaves the byte-agreement check unexercised and therefore - unproven. This is the shape that genuinely needs it: the newer revision adds - a second attachment -- real growth on the identity axis -- while changing the - bytes it reports for the attachment both revisions share. Accepting that - would overwrite content already in hand with content from a disagreeing - source, under cover of a legitimate-looking frontier advance. - """ - shared_id = "drive-file-1" - older = _attachment_revision("raw-a", acquired=True, provider_attachment_id=shared_id, inline=b"original bytes") - - contradicting = ParsedAttachment( - provider_attachment_id=shared_id, - message_provider_id="0", - size_bytes=len(b"rewritten bytes"), - inline_bytes=b"rewritten bytes", - ) - added = ParsedAttachment( - provider_attachment_id="drive-file-2", - message_provider_id="0", - size_bytes=4, - inline_bytes=b"more", - ) - session = ParsedSession( - source_name=Provider.GEMINI, - provider_session_id="session", - messages=[ParsedMessage(provider_message_id="0", role=Role.USER, text="one")], - attachments=[contradicting, added], - ) - newer = MembershipRevision("raw-b", session_revision_projection(session)) - - # The identity axis really did grow, so the refusal cannot come from there. - assert older.projection.attachment_identities < newer.projection.attachment_identities - assert not _strictly_dominates(older.projection, newer.projection) - - result = classify_membership_revisions([older, newer]) - assert result.accepted_raw_ids == () - assert result.ambiguous_raw_ids == ("raw-a", "raw-b") diff --git a/tests/unit/pipeline/test_pipeline_ids.py b/tests/unit/pipeline/test_pipeline_ids.py index 35c11db15e..25490bd3a2 100644 --- a/tests/unit/pipeline/test_pipeline_ids.py +++ b/tests/unit/pipeline/test_pipeline_ids.py @@ -214,8 +214,12 @@ def test_session_revision_projection_golden_hashes() -> None: "65a99313c3ed8b81e69ecc0b36f314b3bf7848bc10fcea11415ddb9b07188941", "8efbf7b5b70bef4d73ec3550fe97e6f4d456cd724550bc5621a36766fe7f1f8b", ] + # Content-derived identity (message_id, name, mime_type) -- no longer a + # hash of the provider attachment id (polylogue-aggz / polylogue-d8al): + # a provider id is not guaranteed present across export vintages, so it + # is excluded from identity rather than used when present. assert {h.hex() for h in projection.attachment_identities} == { - "c233d41c034109580c7d7e74a96944a55eb0fadcc2564001dcf607f727b48062" + "2f18566179352065740615ea89e60130da5a8e46aae224e36b44ed626722da54" } # The golden attachment declares a size but carries no bytes, so it is # referenced-but-unacquired: identity is known, content is not. @@ -270,13 +274,15 @@ def test_session_revision_projection_matches_independent_recomputation() -> None assert projection.session_hash.hex() == independent_session_hash assert list(projection.message_hashes) == [bytes.fromhex(hash_payload(p)) for p in independent_message_payloads] + # Content-derived identity: message_id/name/mime_type only, never the + # provider attachment id (polylogue-aggz / polylogue-d8al). assert projection.attachment_identities == frozenset( - bytes.fromhex(hash_payload({field: p[field] for field in ("id", "message_id", "name", "mime_type")})) + bytes.fromhex(hash_payload({field: p[field] for field in ("message_id", "name", "mime_type")})) for p in independent_attachment_payloads ) assert projection.attachment_contents == frozenset( ( - bytes.fromhex(hash_payload({field: p[field] for field in ("id", "message_id", "name", "mime_type")})), + bytes.fromhex(hash_payload({field: p[field] for field in ("message_id", "name", "mime_type")})), bytes.fromhex(str(p["inline_content_hash"])), ) for p in independent_attachment_payloads