From 848c7ff641578c910948a4bf230ae8da41c26684 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sat, 8 Aug 2026 23:49:45 +0200 Subject: [PATCH 01/19] fix: stabilize idless message revision identity Problem:\nId-less provider messages still depended on positional comparison fallbacks, so reordering could manufacture revision changes and same-timestamp Drive messages could collapse attachment ownership. Tagged message IDs also confused reader session ownership.\n\nWhat changed:\n- derive every id-less revision anchor from role, timestamp, and text\n- keep parser-local coordinates separate from provider message identity\n- resolve tagged message IDs back to their owning session\n- cover Drive reorder, parent-chain, and attachment-owner behavior\n\nAlternatives:\n- Retaining timestamp-only or positional fallbacks would lose multiplicity or make attachment moves invisible.\n\nCompatibility:\n- Native provider IDs remain unchanged; synthetic parser IDs continue using the shared content-derived constructor.\n --- polylogue/pipeline/ids.py | 26 ++++----- polylogue/sources/origin_specs.py | 2 +- polylogue/sources/parsers/base_support.py | 8 +-- .../storage/sqlite/archive_tiers/archive.py | 8 +++ tests/unit/daemon/test_web_reader.py | 56 ++++++++++--------- tests/unit/sources/test_parsers_drive.py | 4 +- 6 files changed, 59 insertions(+), 45 deletions(-) diff --git a/polylogue/pipeline/ids.py b/polylogue/pipeline/ids.py index d8a2bf4f85..200a5dc23e 100644 --- a/polylogue/pipeline/ids.py +++ b/polylogue/pipeline/ids.py @@ -235,15 +235,14 @@ def _message_hash_payload(message: ParsedMessage, message_id: str) -> dict[str, return payload -#: Marker prefix for a content-derived (role + timestamp) message identity -#: anchor, used only when a message carries no native ``provider_message_id``. -#: Namespaced so it can never collide with a real provider id string (a -#: provider id never contains this literal token by construction). -_TIMESTAMP_ANCHOR_PREFIX = "__polylogue_msg_ts_anchor__" +#: Marker prefix for a content-derived message identity anchor, used only +#: when a message carries no native ``provider_message_id``. Namespaced so it +#: can never collide with a real provider id string (a provider id never +#: contains this literal token by construction). _CONTENT_ANCHOR_PREFIX = "__polylogue_msg_content_anchor__" -def _message_comparison_id(message: ParsedMessage, index: int) -> str: +def _message_comparison_id(message: ParsedMessage) -> str: """Resolve the id used as both a message's content-payload id and its comparison identity (``message_identity_hash``'s sole input). @@ -256,10 +255,11 @@ class the attachment identity fix (polylogue-hith/-d8al) removed for a array position would get two different fallback "ids" and compare as a conflict instead of the same message (polylogue-gysk3). - The fix: fall back to a content-derived anchor -- role plus timestamp, - or role plus message text when no timestamp exists -- instead of array - position, so reordering an otherwise-unchanged id-less message never - changes its comparison identity. + The fix: fall back to one content-derived anchor over role, timestamp, + and message text instead of array position. Including text even when a + timestamp exists keeps two same-timestamp id-less messages distinct and + gives attachment ownership a stable non-positional anchor when a Drive + attachment moves between them. Parser normalization maintains the complementary invariant: a missing native id is never replaced with an array-position-derived value before @@ -268,9 +268,7 @@ class the attachment identity fix (polylogue-hith/-d8al) removed for a """ if message.provider_message_id: return message.provider_message_id - if message.timestamp: - return f"{_TIMESTAMP_ANCHOR_PREFIX}:{message.role}:{message.timestamp}" - return f"{_CONTENT_ANCHOR_PREFIX}:{hash_payload({'role': str(message.role), 'text': _normalize_for_hash(message.text)})}" + return f"{_CONTENT_ANCHOR_PREFIX}:{hash_payload({'role': str(message.role), 'timestamp': _normalize_for_hash(message.timestamp), 'text': _normalize_for_hash(message.text)})}" def message_identity_hash(*, id: str) -> bytes: @@ -516,7 +514,7 @@ def _session_hash_components( caller re-deriving its own copy. Byte-identical to computing each payload independently -- pure sharing of an already-pure computation. """ - message_comparison_ids = [_message_comparison_id(msg, idx) for idx, msg in enumerate(convo.messages, start=1)] + message_comparison_ids = [_message_comparison_id(msg) for msg in convo.messages] messages_payload = [ _message_hash_payload(message, comparison_id) for message, comparison_id in zip(convo.messages, message_comparison_ids, strict=True) diff --git a/polylogue/sources/origin_specs.py b/polylogue/sources/origin_specs.py index 23a03f3b84..99cf71ead4 100644 --- a/polylogue/sources/origin_specs.py +++ b/polylogue/sources/origin_specs.py @@ -835,7 +835,7 @@ def _grok_spec() -> OriginSpec: assembly_paths=("polylogue/sources/dispatch.py:_lower_grok_export_payload",), fidelity_notes=( "No native conversation or response id is present in any confirmed export shape; " - "provider_session_id/provider_message_id are synthesized from file-relative position.", + "provider_session_id is derived from file identity and provider_message_id from response content.", "The export drops attachments/images by xAI's own documentation; only text turns are recoverable.", ), display_description="Grok account-data exports (lab: xAI)", diff --git a/polylogue/sources/parsers/base_support.py b/polylogue/sources/parsers/base_support.py index 99b98ce846..73a7e8d7c4 100644 --- a/polylogue/sources/parsers/base_support.py +++ b/polylogue/sources/parsers/base_support.py @@ -78,7 +78,7 @@ def synthetic_message_id( This is reserved for parser-produced rows that are inherently synthetic, such as an exported summary or a transcript section. Native-id fallback paths must pass an empty string instead, so ``pipeline.ids`` can use its - role/timestamp comparison anchor. + role/timestamp/text comparison anchor. """ seed = "\x1f".join((namespace, str(role), timestamp or "", text or "", kind)) return f"synthetic-{hash_text(seed)[:24]}" @@ -440,9 +440,9 @@ def extract_messages_from_list(items: Sequence[object]) -> list[ParsedMessage]: if text: # polylogue-slshy: no positional fallback -- empty id lets - # _message_comparison_id's content-anchor (role + timestamp) - # fallback run instead of a position-derived string that would - # change identity when array order shifts across re-acquisitions. + # _message_comparison_id's content-derived anchor fallback run + # instead of a position-derived string that would change identity + # when array order shifts across re-acquisitions. msg_id = str(payload.get("id") or payload.get("uuid") or item.get("uuid") or item.get("id") or "") messages.append( ParsedMessage( diff --git a/polylogue/storage/sqlite/archive_tiers/archive.py b/polylogue/storage/sqlite/archive_tiers/archive.py index e86b804d46..fb15fb971b 100644 --- a/polylogue/storage/sqlite/archive_tiers/archive.py +++ b/polylogue/storage/sqlite/archive_tiers/archive.py @@ -9338,6 +9338,14 @@ def _user_mark_session_id(target_type: str, target_id: str) -> str: if target_type == "session": return target_id if target_type == "message": + # Message identities use tagged local components (``:n:`` or + # ``:p:.``). Splitting at the final colon used to + # work only while native IDs were untagged; with ``session:n:native`` + # it incorrectly reports ``session:n`` as the owning session. + for marker in (":n:", ":p:"): + marker_index = target_id.find(marker) + if marker_index > 0: + return target_id[:marker_index] session_id, _sep, _message_native_id = target_id.rpartition(":") return session_id return "" diff --git a/tests/unit/daemon/test_web_reader.py b/tests/unit/daemon/test_web_reader.py index b3125c162f..5a70c47613 100644 --- a/tests/unit/daemon/test_web_reader.py +++ b/tests/unit/daemon/test_web_reader.py @@ -23,6 +23,8 @@ import pytest +from polylogue.surfaces.payloads import reader_anchor + pytestmark = pytest.mark.xdist_group("web-reader") import html as html_module @@ -333,7 +335,7 @@ def _running_server_without_seed( # Archive session/message identities for the three seeded sessions. # The archive store derives ``session_id`` as ``origin:native_id`` and -# ``message_id`` as ``session_id:message_native_id``; the daemon's reader +# ``message_id`` with a tagged native local component; the daemon's reader # surface returns these identities verbatim. from polylogue.core.identity_law import message_id as _archive_message_id from polylogue.core.identity_law import session_id as _archive_session_id @@ -714,7 +716,14 @@ def _seed_import_explain_archive(workspace: dict[str, Path]) -> tuple[str, str]: message_id, session_id, position, block_type, text, tool_id ) VALUES (?, ?, ?, ?, ?, ?) """, - ("codex-session:route-native:m1", "codex-session:route-native", 0, "tool_use", "pytest", "tool-1"), + ( + _archive_message_id("codex-session:route-native", "m1", position=0), + "codex-session:route-native", + 0, + "tool_use", + "pytest", + "tool-1", + ), ) source_conn.commit() index_conn.commit() @@ -1211,11 +1220,11 @@ def test_query_search_envelope_carries_hit_target_refs(self, workspace_env: dict # for message targets; we assert the load-bearing identity fields. target_ref = hit["match"]["target_ref"] assert target_ref["target_type"] == "message" - assert target_ref["target_id"] == "claude-code-session:c1:m-c1" + assert target_ref["target_id"] == M_C1 assert target_ref["session_id"] == "claude-code-session:c1" - assert target_ref["message_id"] == "claude-code-session:c1:m-c1" - assert target_ref["identity_key"] == "message:claude-code-session:c1:claude-code-session:c1:m-c1" - assert hit["match"]["anchor"] == "message-claude-code-session-c1-m-c1" + assert target_ref["message_id"] == M_C1 + assert target_ref["identity_key"] == f"message:{C1}:{M_C1}" + assert hit["match"]["anchor"] == reader_anchor("message", M_C1) assert hit["match"]["actions"]["copy_text"]["enabled"] is True def test_search_hit_rank_lives_under_match_not_top_level(self, workspace_env: dict[str, Path]) -> None: @@ -1367,11 +1376,8 @@ def test_session_detail_returns_header_and_messages(self, workspace_env: dict[st assert payload["title"].startswith("Claude Code") assert payload["target_ref"]["identity_key"] == "session:claude-code-session:c1" assert payload["anchor"] == "session-claude-code-session-c1" - assert ( - payload["messages"][0]["target_ref"]["identity_key"] - == "message:claude-code-session:c1:claude-code-session:c1:m-c1" - ) - assert payload["messages"][0]["anchor"] == "message-claude-code-session-c1-m-c1" + assert payload["messages"][0]["target_ref"]["identity_key"] == f"message:{C1}:{M_C1}" + assert payload["messages"][0]["anchor"] == reader_anchor("message", M_C1) def test_session_routes_accept_list_emitted_encoded_ids(self, workspace_env: dict[str, Path]) -> None: with _running_server(workspace_env) as (_, base_url): @@ -1408,12 +1414,12 @@ def test_session_messages_envelope_carries_messages_and_total(self, workspace_en assert message["text"] == "Hello reader" assert message["target_ref"] == { "target_type": "message", - "target_id": "claude-code-session:c1:m-c1", + "target_id": M_C1, "session_id": "claude-code-session:c1", - "message_id": "claude-code-session:c1:m-c1", - "identity_key": "message:claude-code-session:c1:claude-code-session:c1:m-c1", + "message_id": M_C1, + "identity_key": f"message:{C1}:{M_C1}", } - assert message["anchor"] == "message-claude-code-session-c1-m-c1" + assert message["anchor"] == reader_anchor("message", M_C1) assert message["actions"]["copy_text"]["enabled"] is True assert message["actions"]["annotate"]["enabled"] is True assert message["actions"]["annotate"]["state"] == "enabled" @@ -1762,11 +1768,11 @@ def test_message_marks_are_target_aware(self, workspace_env: dict[str, Path]) -> payload={ "session_id": "claude-code-session:c1", "target_type": "message", - "message_id": "claude-code-session:c1:m-c1", + "message_id": M_C1, "mark_type": "pin", }, ) - marks = _get_json(base_url, "/api/user/marks?target_type=message&message_id=claude-code-session:c1:m-c1") + marks = _get_json(base_url, f"/api/user/marks?target_type=message&message_id={quote(M_C1, safe='')}") marks_payload = cast(dict[str, object], marks) created_payload = cast(dict[str, object], created) @@ -1774,15 +1780,15 @@ def test_message_marks_are_target_aware(self, workspace_env: dict[str, Path]) -> assert created_payload["status"] == "ok" assert created_payload["operation"] == "mark.add" assert created_payload["target_type"] == "message" - assert created_payload["target_id"] == "claude-code-session:c1:m-c1" - assert created_payload["message_id"] == "claude-code-session:c1:m-c1" + assert created_payload["target_id"] == M_C1 + assert created_payload["message_id"] == M_C1 mark_items = cast(list[dict[str, object]], marks_payload["items"]) assert mark_items == [ { "target_type": "message", - "target_id": "claude-code-session:c1:m-c1", + "target_id": M_C1, "session_id": "claude-code-session:c1", - "message_id": "claude-code-session:c1:m-c1", + "message_id": M_C1, "mark_type": "pin", "created_at": mark_items[0]["created_at"], } @@ -1808,7 +1814,7 @@ def test_annotations_roundtrip_session_and_message_targets(self, workspace_env: "annotation_id": "ann-m1", "session_id": "claude-code-session:c1", "target_type": "message", - "message_id": "claude-code-session:c1:m-c1", + "message_id": M_C1, "note_text": "Important request", }, ) @@ -1837,7 +1843,7 @@ def test_annotations_roundtrip_session_and_message_targets(self, workspace_env: assert msg_note_payload["resource_type"] == "annotation" assert msg_note_payload["resource_id"] == "ann-m1" assert msg_note_payload["target_type"] == "message" - assert msg_note_payload["target_id"] == "claude-code-session:c1:m-c1" + assert msg_note_payload["target_id"] == M_C1 assert fetched_payload["note_text"] == "Important request" assert {item["annotation_id"] for item in items} == {"ann-c1", "ann-m1"} assert delete_status == 200 @@ -2008,7 +2014,7 @@ def test_workspaces_roundtrip_resolved_and_degraded_targets(self, workspace_env: { "target_type": "message", "session_id": "claude-code-session:c1", - "message_id": "claude-code-session:c1:m-c1", + "message_id": M_C1, }, { "target_type": "message", @@ -2021,7 +2027,7 @@ def test_workspaces_roundtrip_resolved_and_degraded_targets(self, workspace_env: "active_target": { "target_type": "message", "session_id": "claude-code-session:c1", - "message_id": "claude-code-session:c1:m-c1", + "message_id": M_C1, }, }, ) diff --git a/tests/unit/sources/test_parsers_drive.py b/tests/unit/sources/test_parsers_drive.py index e14e6e1b9f..89ab2857a8 100644 --- a/tests/unit/sources/test_parsers_drive.py +++ b/tests/unit/sources/test_parsers_drive.py @@ -292,8 +292,10 @@ def test_parse_chunked_prompt_idless_turns_resolve_parent_coordinates_in_archive def test_idless_drive_attachment_owner_changes_hash_and_revision_identity() -> None: + # Same-timestamp turns force the owner anchor to include message content; + # a timestamp-only fallback would make moving the attachment invisible. first: JSONDocument = {"role": "user", "text": "first", "createTime": "2026-01-01T00:00:00Z"} - second: JSONDocument = {"role": "model", "text": "second", "createTime": "2026-01-01T00:00:01Z"} + second: JSONDocument = {"role": "model", "text": "second", "createTime": "2026-01-01T00:00:00Z"} attachment: JSONDocument = {"id": "drive-doc", "name": "note.txt", "mimeType": "text/plain"} first_owner = parse_chunked_prompt( "gemini", From affce94444cc43e8d3cc94cd5a56c3858eea3a93 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sat, 8 Aug 2026 23:50:03 +0200 Subject: [PATCH 02/19] test: construct fixtures with canonical message ids Problem:\nHand-built archive fixtures still used legacy untagged message and block IDs, causing foreign-key failures and masking the generated identity contract.\n\nWhat changed:\n- add shared test-only constructors backed by the identity law\n- repair storage, query, lineage, API, daemon, property, benchmark, and visual fixtures\n- retain malformed branch-point inputs only where the test exercises repair behavior\n\nAlternatives:\n- Replacing literals mechanically would conceal whether the fixture represents a native or positional message identity.\n\nCompatibility:\n- Production identity generation is unchanged by this test-only sweep; fixtures now match the existing generated-column schema.\n --- tests/benchmarks/test_full_session_replace.py | 5 +- .../test_graph_resolve_deferred_tail.py | 5 +- tests/infra/identity.py | 25 ++++++++ .../test_fts_identity_state_machine.py | 16 +++-- .../property/test_write_path_state_machine.py | 5 +- tests/unit/api/test_facade_contracts.py | 23 ++++++-- .../archive/query/test_execution_control.py | 3 +- .../archive/test_with_units_projection.py | 7 ++- tests/unit/cli/test_query_exec_laws.py | 5 +- tests/unit/cli/test_query_expression.py | 59 +++++++++++-------- tests/unit/daemon/test_convergence_stages.py | 3 +- tests/unit/pipeline/test_branching.py | 8 ++- .../test_claude_code_normalization_laws.py | 30 +++++----- .../storage/test_archive_search_contracts.py | 3 +- .../storage/test_archive_tiers_archive.py | 13 ++-- .../storage/test_archive_tiers_assertions.py | 3 +- tests/unit/storage/test_archive_tiers_ddl.py | 17 +++--- .../storage/test_attachment_acquisition.py | 3 +- .../storage/test_attachment_reacquisition.py | 6 +- .../unit/storage/test_fts_identity_ledger.py | 5 +- tests/unit/storage/test_fts_repair_sql.py | 3 +- .../test_incremental_rebuild_equivalence.py | 15 ++--- .../storage/test_lineage_normalization.py | 32 ++++++---- .../unit/storage/test_message_query_reads.py | 51 ++++++++++------ tests/unit/storage/test_pl_fold.py | 3 +- .../test_query_unit_time_expression.py | 3 +- tests/unit/storage/test_schema_safety.py | 3 +- .../test_search_timeless_since_filter.py | 7 ++- .../storage/test_session_insight_refresh.py | 5 +- .../storage/test_spec_driven_hydration.py | 5 +- tests/unit/storage/test_store_ops.py | 9 +-- .../storage/test_unread_wire_batch_v46.py | 13 ++-- tests/visual/conftest.py | 11 ++-- 33 files changed, 260 insertions(+), 144 deletions(-) create mode 100644 tests/infra/identity.py diff --git a/tests/benchmarks/test_full_session_replace.py b/tests/benchmarks/test_full_session_replace.py index 91b682e902..cece92dc67 100644 --- a/tests/benchmarks/test_full_session_replace.py +++ b/tests/benchmarks/test_full_session_replace.py @@ -34,6 +34,7 @@ from polylogue.storage.sqlite.archive_tiers import ARCHIVE_DDL_BY_TIER from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier +from tests.infra.identity import archive_block_id, archive_message_id _INDEX_DDL = ARCHIVE_DDL_BY_TIER[ArchiveTier.INDEX] @@ -88,10 +89,10 @@ def _build_replace_fixture(db_path: Path, *, drop_leading_index: bool) -> tuple[ ) for position in range(3): native_id = f"m{position}" - message_id = f"{session_id}:{native_id}" + message_id = archive_message_id(session_id, native_id, position=position) background_messages.append((session_id, native_id, position, "user", bytes([position]) * 32)) background_blocks.append((message_id, session_id, 0, "text", "hi")) - block_id = f"{message_id}:0" + block_id = archive_block_id(message_id, position=0) background_constructs.append((session_id, message_id, block_id, 0, "chatgpt", "canvas")) conn.executemany( "INSERT INTO messages (session_id, native_id, position, role, content_hash) VALUES (?, ?, ?, ?, ?)", diff --git a/tests/benchmarks/test_graph_resolve_deferred_tail.py b/tests/benchmarks/test_graph_resolve_deferred_tail.py index b8b3cfd595..d0c886601a 100644 --- a/tests/benchmarks/test_graph_resolve_deferred_tail.py +++ b/tests/benchmarks/test_graph_resolve_deferred_tail.py @@ -52,6 +52,7 @@ from polylogue.storage.sqlite.archive_tiers import ARCHIVE_DDL_BY_TIER from polylogue.storage.sqlite.archive_tiers import write as archive_tier_write from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier +from tests.infra.identity import archive_message_id _INDEX_DDL = ARCHIVE_DDL_BY_TIER[ArchiveTier.INDEX] @@ -81,7 +82,7 @@ def _build_deferred_tail_fixture(db_path: Path, *, n_children: int) -> tuple[sql parent_blocks = [] for position in range(_PARENT_MESSAGES): native_id = f"m{position}" - message_id = f"{parent_session_id}:{native_id}" + message_id = archive_message_id(parent_session_id, native_id, position=position) role = "user" if position % 2 == 0 else "assistant" parent_messages.append((parent_session_id, native_id, position, role, b"x" * 32)) parent_blocks.append((message_id, parent_session_id, 0, "text", f"text-{position}")) @@ -107,7 +108,7 @@ def _build_deferred_tail_fixture(db_path: Path, *, n_children: int) -> tuple[sql child_events = [] for position in range(total_child_messages): native_id = f"m{position}" - message_id = f"{child_session_id}:{native_id}" + message_id = archive_message_id(child_session_id, native_id, position=position) role = "user" if position % 2 == 0 else "assistant" # Shared-prefix positions reuse the parent's exact text so the # composed-signature comparison walks the full shared prefix; diff --git a/tests/infra/identity.py b/tests/infra/identity.py new file mode 100644 index 0000000000..f307576b97 --- /dev/null +++ b/tests/infra/identity.py @@ -0,0 +1,25 @@ +"""Canonical archive identity constructors for hand-built test fixtures.""" + +from __future__ import annotations + +from polylogue.core.identity_law import block_id as _block_id +from polylogue.core.identity_law import message_id as _message_id + + +def archive_message_id( + session_id: str, + native_id: str | None, + *, + position: int, + variant_index: int = 0, +) -> str: + """Construct the generated message id used by the archive schema.""" + return _message_id(session_id, native_id, position=position, variant_index=variant_index) + + +def archive_block_id(message_id: str, *, position: int) -> str: + """Construct the generated block id used by the archive schema.""" + return _block_id(message_id, position=position) + + +__all__ = ["archive_block_id", "archive_message_id"] diff --git a/tests/property/test_fts_identity_state_machine.py b/tests/property/test_fts_identity_state_machine.py index 699c20f610..ddd267e314 100644 --- a/tests/property/test_fts_identity_state_machine.py +++ b/tests/property/test_fts_identity_state_machine.py @@ -26,6 +26,7 @@ restore_fts_triggers_sync, ) from polylogue.storage.sqlite.connection import open_connection +from tests.infra.identity import archive_message_id class _Block: @@ -84,14 +85,15 @@ def _insert_block(self, *, text: str | None) -> _Block: self._next_id += 1 message_native_id = f"msg-{self._next_id}" session_id = f"{self._origin}:{session_native_id}" - message_id = f"{session_id}:{message_native_id}" + position = self._next_position(session_native_id) + message_id = archive_message_id(session_id, message_native_id, position=position) content_hash = self._fresh_content_hash() self._conn.execute( """ INSERT INTO messages (session_id, native_id, position, role, message_type, content_hash) VALUES (?, ?, ?, 'user', 'message', ?) """, - (session_id, message_native_id, self._next_position(session_native_id), content_hash), + (session_id, message_native_id, position, content_hash), ) self._conn.execute( """ @@ -170,14 +172,15 @@ def full_session_replace(self) -> None: self._next_id += 1 message_native_id = f"msg-{self._next_id}" session_id = f"{self._origin}:{session_native_id}" - message_id = f"{session_id}:{message_native_id}" + position = self._next_position(session_native_id) + message_id = archive_message_id(session_id, message_native_id, position=position) content_hash = self._fresh_content_hash() self._conn.execute( """ INSERT INTO messages (session_id, native_id, position, role, message_type, content_hash) VALUES (?, ?, ?, 'user', 'message', ?) """, - (session_id, message_native_id, self._next_position(session_native_id), content_hash), + (session_id, message_native_id, position, content_hash), ) self._conn.execute( """ @@ -213,14 +216,15 @@ def rollback_insert(self) -> None: self._next_id += 1 message_native_id = f"rollback-msg-{self._next_id}" session_id = f"{self._origin}:{session_native_id}" - message_id = f"{session_id}:{message_native_id}" + position = self._next_position(session_native_id) + message_id = archive_message_id(session_id, message_native_id, position=position) content_hash = self._fresh_content_hash() self._conn.execute( """ INSERT INTO messages (session_id, native_id, position, role, message_type, content_hash) VALUES (?, ?, ?, 'user', 'message', ?) """, - (session_id, message_native_id, self._next_position(session_native_id), content_hash), + (session_id, message_native_id, position, content_hash), ) self._conn.execute( """ diff --git a/tests/property/test_write_path_state_machine.py b/tests/property/test_write_path_state_machine.py index ae3c15851e..c67a4bbc95 100644 --- a/tests/property/test_write_path_state_machine.py +++ b/tests/property/test_write_path_state_machine.py @@ -31,6 +31,7 @@ write_parsed_session_to_archive, ) from polylogue.storage.sqlite.schema import _ensure_schema +from tests.infra.identity import archive_message_id @dataclass @@ -592,14 +593,14 @@ def test_grandchild_transcript_recomposes_after_intermediate_ancestor_message_de (grandchild_id,), ).fetchone() assert branch_row is not None - assert branch_row[0] == "claude-code-session:cascade-child:child-2" + assert branch_row[0] == archive_message_id("claude-code-session:cascade-child", "child-2", position=2) # Two hops upstream of the grandchild: delete the root parent's # first message, an ancestor edit that precedes every downstream # branch point. conn.execute( "DELETE FROM messages WHERE message_id = ?", - ("claude-code-session:cascade-parent:parent-0",), + (archive_message_id("claude-code-session:cascade-parent", "parent-0", position=0),), ) conn.commit() finally: diff --git a/tests/unit/api/test_facade_contracts.py b/tests/unit/api/test_facade_contracts.py index 73e97ebafa..8f99311b44 100644 --- a/tests/unit/api/test_facade_contracts.py +++ b/tests/unit/api/test_facade_contracts.py @@ -57,6 +57,7 @@ from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier from polylogue.storage.sqlite.archive_tiers.user_write import upsert_assertion from tests.infra.frozen_clock import FrozenClock +from tests.infra.identity import archive_message_id from tests.infra.storage_records import db_setup # --------------------------------------------------------------------------- @@ -470,7 +471,14 @@ def _seed_import_explain_archive(tmp_path: Path, *, source_path: str | None = No message_id, session_id, position, block_type, text, tool_id ) VALUES (?, ?, ?, ?, ?, ?) """, - ("codex-session:native-1:msg-1", "codex-session:native-1", 0, "tool_use", "pytest", "tool-1"), + ( + archive_message_id("codex-session:native-1", "msg-1", position=0), + "codex-session:native-1", + 0, + "tool_use", + "pytest", + "tool-1", + ), ) source_conn.commit() index_conn.commit() @@ -2609,7 +2617,9 @@ async def test_query_units_reports_pipeline_stages(tmp_path: Path) -> None: assert envelope.pipeline["result"] == {"limit": 1, "offset": 1} assert envelope.pipeline["session_scope"] == envelope.pipeline_stages[0]["predicate"] assert envelope.limit == 1 - assert [cast(Any, item).message_id for item in envelope.items] == ["codex-session:unit-pipeline-codex:m2"] + assert [cast(Any, item).message_id for item in envelope.items] == [ + archive_message_id("codex-session:unit-pipeline-codex", "m2", position=1) + ] finally: await archive.close() @@ -2861,7 +2871,7 @@ async def test_resolve_ref_returns_bounded_session_message_block_and_runtime_pay assert session_payload.payload is not None assert session_payload.payload["id"] == session_id - message_id = f"{session_id}:m1" + message_id = archive_message_id(session_id, "m1", position=0) message_payload = await archive.resolve_ref(f"message:{session_id}:{message_id}") assert message_payload.resolved is True assert message_payload.payload_kind == "message" @@ -4188,7 +4198,7 @@ async def test_archive_tiers_api_reads_native_sessions(tmp_path: Path) -> None: assert normal_envelope.total == 1 assert normal_envelope.retrieval_lane == "dialogue" assert [hit.session.id for hit in normal_envelope.hits] == [session_id] - assert normal_envelope.hits[0].match.message_id == f"{session_id}:m1" + assert normal_envelope.hits[0].match.message_id == archive_message_id(session_id, "m1", position=0) assert unit_envelope.mode == "query-unit" assert unit_envelope.unit == "message" assert unit_envelope.total == 1 @@ -4198,9 +4208,10 @@ async def test_archive_tiers_api_reads_native_sessions(tmp_path: Path) -> None: assert normal_neighbors[0].summary.message_count == 1 assert {reason.kind for reason in normal_neighbors[0].reasons} >= {"query_match", "content_similarity"} assert total_messages == 1 - assert [message.id for message in paged_messages] == [f"{session_id}:m1"] + expected_message_id = archive_message_id(session_id, "m1", position=0) + assert [message.id for message in paged_messages] == [expected_message_id] assert list(bulk_messages) == [session_id] - assert [message.id for message in bulk_messages[session_id]] == [f"{session_id}:m1"] + assert [message.id for message in bulk_messages[session_id]] == [expected_message_id] with ArchiveStore.open_existing(archive.config.archive_root, read_only=False) as archive_db: archive_db._conn.execute("INSERT INTO messages_fts(messages_fts) VALUES('delete-all')") diff --git a/tests/unit/archive/query/test_execution_control.py b/tests/unit/archive/query/test_execution_control.py index 3910f5f5fd..3d9fd50f96 100644 --- a/tests/unit/archive/query/test_execution_control.py +++ b/tests/unit/archive/query/test_execution_control.py @@ -31,6 +31,7 @@ execute_archive_read_sync, ) from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore +from tests.infra.identity import archive_message_id pytestmark = pytest.mark.uses_real_clock( "polylogue-z9gh.1 execution-control tests measure real elapsed wall-clock by contract: cancellation/deadline abort SLOs against a genuinely running SQLite statement, event-loop heartbeat gaps during worker-thread offload, and cross-thread admission queue waits. frozen_clock cannot substitute for real thread scheduling and SQLite progress-handler cadence." @@ -709,7 +710,7 @@ def test_exact_session_multi_aggregate_work_is_not_amplified_by_irrelevant_growt for index in range(512): native_id = "target" if index == 0 else f"irrelevant-{index:04d}" session_id = f"codex-session:{native_id}" - message_id = f"{session_id}:m1" + message_id = archive_message_id(session_id, "m1", position=0) tool_id = f"tool-{index:04d}" session_rows.append((native_id, Origin.CODEX_SESSION.value, sha256(session_id.encode()).digest())) message_rows.append( diff --git a/tests/unit/archive/test_with_units_projection.py b/tests/unit/archive/test_with_units_projection.py index c8e3e75b0c..d1992abe80 100644 --- a/tests/unit/archive/test_with_units_projection.py +++ b/tests/unit/archive/test_with_units_projection.py @@ -29,6 +29,7 @@ from polylogue.sources.parsers.base import ParsedContentBlock, ParsedMessage, ParsedSession from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore from polylogue.storage.sqlite.archive_tiers.user_write import upsert_assertion +from tests.infra.identity import archive_message_id # --------------------------------------------------------------------------- # Parser: split + validation @@ -231,8 +232,8 @@ def test_fetch_attached_evidence_units(self, tmp_path: Path) -> None: assert set(attached) == {"message", "action", "file"} assert {row["message_id"] for row in attached["message"][session_id]} == { - "claude-code-session:ext-evidence:m-user", - "claude-code-session:ext-evidence:m-assistant", + archive_message_id("claude-code-session:ext-evidence", "m-user", position=0), + archive_message_id("claude-code-session:ext-evidence", "m-assistant", position=1), } action_payload = attached["action"][session_id][0] output_text = action_payload["output_text"] @@ -268,7 +269,7 @@ def test_fetch_attached_units_applies_payload_field_selection(self, tmp_path: Pa assert attached["message"][session_id] == ( { - "message_id": "claude-code-session:ext-field-select:m-user", + "message_id": archive_message_id("claude-code-session:ext-field-select", "m-user", position=0), "role": "user", }, ) diff --git a/tests/unit/cli/test_query_exec_laws.py b/tests/unit/cli/test_query_exec_laws.py index d83fe2a8c1..3ecbf34e07 100644 --- a/tests/unit/cli/test_query_exec_laws.py +++ b/tests/unit/cli/test_query_exec_laws.py @@ -41,6 +41,7 @@ from polylogue.storage.sqlite.archive_tiers.write import ArchiveSessionEnvelope from polylogue.surfaces.payloads import decode_search_cursor from tests.infra.builders import make_conv, make_msg +from tests.infra.identity import archive_message_id async def _execute_query_params(env: AppEnv, params: dict[str, object]) -> None: @@ -3509,7 +3510,9 @@ def test_output_contract( data = json.loads(result.output) assert data["mode"] == "query-unit", case_id assert data["unit"] == "message", case_id - assert [item["message_id"] for item in data["items"]] == ["chatgpt-export:ext-conv1:m2"], case_id + assert [item["message_id"] for item in data["items"]] == [ + archive_message_id("chatgpt-export:ext-conv1", "m2", position=1) + ], case_id assert data["items"][0]["role"] == "assistant", case_id elif expectation == "plain_list": assert result.output.strip(), case_id diff --git a/tests/unit/cli/test_query_expression.py b/tests/unit/cli/test_query_expression.py index e833b3c0b8..b12f45296c 100644 --- a/tests/unit/cli/test_query_expression.py +++ b/tests/unit/cli/test_query_expression.py @@ -70,6 +70,16 @@ from polylogue.archive.query.spec import SessionQuerySpec from polylogue.core.refs import ObjectRef from polylogue.storage.runtime import MessageRecord +from tests.infra.identity import archive_block_id, archive_message_id + + +def _mid(session_id: str, native_id: str, *, position: int = 0) -> str: + return archive_message_id(session_id, native_id, position=position) + + +def _bid(session_id: str, native_id: str, *, message_position: int = 0, block_position: int = 0) -> str: + return archive_block_id(_mid(session_id, native_id, position=message_position), position=block_position) + # --------------------------------------------------------------------------- # Helpers @@ -1834,7 +1844,7 @@ def test_terminal_message_source_returns_message_rows(self, workspace_env: dict[ assert [(row.session_id, row.message_id, row.role, row.text) for row in rows] == [ ( "chatgpt-export:ext-hit", - "chatgpt-export:ext-hit:m-hit", + _mid("chatgpt-export:ext-hit", "m-hit"), "assistant", "the timeout happened", ) @@ -1859,7 +1869,9 @@ def test_terminal_message_source_filters_by_row_time(self, workspace_env: dict[s with ArchiveStore.open_existing(index_db.parent) as archive: rows = archive.query_messages(source.predicate, limit=100) - assert [(row.message_id, row.text) for row in rows] == [("chatgpt-export:ext-hit:m-new", "new")] + assert [(row.message_id, row.text) for row in rows] == [ + (_mid("chatgpt-export:ext-hit", "m-new", position=1), "new") + ] assert rows[0].occurred_at_ms is not None def test_exists_message_predicate_filters_by_row_time(self, workspace_env: dict[str, Path]) -> None: @@ -2103,7 +2115,7 @@ def test_message_numeric_comparisons_execute_against_archive(self, workspace_env assert len(envelope.items) == 1 row = envelope.items[0] assert isinstance(row, MessageQueryRowPayload) - assert row.message_id == "chatgpt-export:ext-message-numeric:m-hit" + assert row.message_id == _mid("chatgpt-export:ext-message-numeric", "m-hit") def test_session_scoped_message_predicate_executes_against_archive( self, @@ -2169,7 +2181,7 @@ def test_terminal_message_source_accepts_session_scoped_predicate( rows = archive.query_messages(source.predicate, limit=100) assert [(row.session_id, row.message_id) for row in rows] == [ - ("claude-code-session:ext-hit", "claude-code-session:ext-hit:m-hit") + ("claude-code-session:ext-hit", _mid("claude-code-session:ext-hit", "m-hit")) ] def test_session_to_message_pipeline_executes_terminal_rows( @@ -2216,7 +2228,7 @@ def test_session_to_message_pipeline_executes_terminal_rows( assert [(row.session_id, row.message_id, row.text) for row in rows] == [ ( "claude-code-session:ext-hit", - "claude-code-session:ext-hit:m-hit-assistant", + _mid("claude-code-session:ext-hit", "m-hit-assistant", position=1), "the pipeline answer", ) ] @@ -2264,7 +2276,7 @@ def test_session_to_message_pipeline_fts_stage_executes_against_archive( assert [(row.session_id, row.message_id, row.text) for row in rows] == [ ( "chatgpt-export:ext-hit", - "chatgpt-export:ext-hit:m-selected", + _mid("chatgpt-export:ext-hit", "m-selected"), "selected terminal row", ) ] @@ -2309,7 +2321,7 @@ def test_session_to_message_pipeline_exists_stage_executes_against_archive( assert [(row.session_id, row.message_id, row.text) for row in rows] == [ ( "chatgpt-export:ext-hit", - "chatgpt-export:ext-hit:m-selected", + _mid("chatgpt-export:ext-hit", "m-selected"), "selected terminal row", ) ] @@ -2403,7 +2415,7 @@ def test_session_to_message_pipeline_sequence_stage_executes_against_archive( assert [(row.session_id, row.message_id, row.text) for row in rows] == [ ( "claude-code-session:ext-hit", - "claude-code-session:ext-hit:m-selected", + _mid("claude-code-session:ext-hit", "m-selected"), "selected terminal row", ) ] @@ -2452,7 +2464,7 @@ def test_session_to_message_pipeline_limit_offset_executes_terminal_window( from polylogue.surfaces.payloads import MessageQueryRowPayload assert isinstance(row, MessageQueryRowPayload) - assert row.message_id == "claude-code-session:ext-hit:m-2" + assert row.message_id == _mid("claude-code-session:ext-hit", "m-2", position=1) assert row.text == "second" with ArchiveStore.open_existing(index_db.parent) as archive: @@ -2463,7 +2475,7 @@ def test_session_to_message_pipeline_limit_offset_executes_terminal_window( assert len(next_page.items) == 1 next_row = next_page.items[0] assert isinstance(next_row, MessageQueryRowPayload) - assert next_row.message_id == "claude-code-session:ext-hit:m-3" + assert next_row.message_id == _mid("claude-code-session:ext-hit", "m-3", position=2) assert next_row.text == "third" def test_session_to_message_pipeline_lineage_executes_against_archive( @@ -2911,7 +2923,7 @@ def test_terminal_source_pipeline_sort_desc_executes_before_limit( from polylogue.surfaces.payloads import MessageQueryRowPayload assert isinstance(row, MessageQueryRowPayload) - assert row.message_id == "claude-code-session:ext-hit:m-new" + assert row.message_id == _mid("claude-code-session:ext-hit", "m-new", position=1) assert row.text == "new" def test_session_to_message_pipeline_sort_desc_executes_before_limit( @@ -2947,7 +2959,7 @@ def test_session_to_message_pipeline_sort_desc_executes_before_limit( from polylogue.surfaces.payloads import MessageQueryRowPayload assert isinstance(row, MessageQueryRowPayload) - assert row.message_id == "claude-code-session:ext-hit:m-new" + assert row.message_id == _mid("claude-code-session:ext-hit", "m-new", position=1) assert row.text == "new" def test_exists_action_predicate_executes_against_archive(self, workspace_env: dict[str, Path]) -> None: @@ -3038,7 +3050,7 @@ def test_terminal_action_source_returns_action_rows(self, workspace_env: dict[st assert [(row.session_id, row.message_id, row.semantic_type, row.tool_path) for row in rows] == [ ( "claude-code-session:ext-hit", - "claude-code-session:ext-hit:m-hit", + _mid("claude-code-session:ext-hit", "m-hit"), "file_edit", "polylogue/archive/query/expression.py", ) @@ -3210,7 +3222,7 @@ def test_terminal_file_source_returns_distinct_path_rows(self, workspace_env: di assert [(row.session_id, row.path, row.action_count) for row in rows] == [ ("claude-code-session:ext-hit", "polylogue/archive/query/expression.py", 2) ] - assert rows[0].first_tool_use_block_id == "claude-code-session:ext-hit:m-edit-1:0" + assert rows[0].first_tool_use_block_id == _bid("claude-code-session:ext-hit", "m-edit-1") def test_exists_file_source_filters_sessions(self, workspace_env: dict[str, Path]) -> None: from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore @@ -3471,7 +3483,7 @@ def test_query_action_read_accepts_shell_quoted_terminal_action_source( assert result.exit_code == 0, result.output payload = json.loads(result.output) assert payload["session_id"] == "claude-code-session:ext-hit" - assert payload["messages"][0]["id"] == "claude-code-session:ext-hit:m-edit" + assert payload["messages"][0]["id"] == _mid("claude-code-session:ext-hit", "m-edit") def test_terminal_action_source_filters_by_row_time(self, workspace_env: dict[str, Path]) -> None: from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore @@ -3536,7 +3548,7 @@ def test_terminal_action_source_filters_by_row_time(self, workspace_env: dict[st rows = archive.query_actions(source.predicate, limit=100) assert [(row.message_id, row.tool_path) for row in rows] == [ - ("claude-code-session:ext-hit:m-new", "polylogue/archive/new.py") + (_mid("claude-code-session:ext-hit", "m-new"), "polylogue/archive/new.py") ] assert rows[0].occurred_at_ms is not None @@ -3675,7 +3687,7 @@ def test_terminal_block_source_returns_block_rows(self, workspace_env: dict[str, assert [(row.session_id, row.message_id, row.block_type, row.text) for row in rows] == [ ( "chatgpt-export:ext-hit", - "chatgpt-export:ext-hit:m-hit", + _mid("chatgpt-export:ext-hit", "m-hit"), "code", "def query_timeout_guard(): pass", ) @@ -3833,7 +3845,7 @@ def test_terminal_action_source_exposes_followup_class_rows_and_aggregates( assert row.is_error == 1 assert row.exit_code == 1 assert row.followup_class == "silent_proceed" - assert row.followup_message_ref == "message:codex-session:ext-followups:m-silent-followup" + assert row.followup_message_ref == "message:" + _mid("codex-session:ext-followups", "m-silent-followup") aggregate_query = "actions where is_error:true | group by followup_class | count" aggregate_source = parse_unit_source_expression(aggregate_query) @@ -4095,11 +4107,12 @@ def test_terminal_observed_event_tool_finished_reads_blocks_without_materializat assert row.kind == "tool_finished" assert row.delivery_state == "observed" assert row.session_id == "claude-code-session:ext-tool-finished" - assert row.subject_ref == "message:claude-code-session:ext-tool-finished:m-tool" + tool_message_id = _mid("claude-code-session:ext-tool-finished", "m-tool") + assert row.subject_ref == "message:" + tool_message_id assert row.object_refs == ("tool-call:claude-code-session:ext-tool-finished:serena-1",) assert row.evidence_refs == ( - "claude-code-session:ext-tool-finished::claude-code-session:ext-tool-finished:m-tool::0", - "claude-code-session:ext-tool-finished::claude-code-session:ext-tool-finished:m-tool::1", + "claude-code-session:ext-tool-finished::" + tool_message_id + "::0", + "claude-code-session:ext-tool-finished::" + tool_message_id + "::1", ) def test_terminal_observed_event_tool_finished_aggregate_reads_blocks_without_materialization( @@ -5339,8 +5352,8 @@ def query(self, text: str, limit: int = 10) -> list[tuple[str, float]]: assert text == "query compiler" assert limit >= 6 return [ - ("chatgpt-export:ext-hit:m-hit", 0.01), - ("chatgpt-export:ext-miss:m-miss", 0.02), + (_mid("chatgpt-export:ext-hit", "m-hit"), 0.01), + (_mid("chatgpt-export:ext-miss", "m-miss"), 0.02), ] def query_by_session(self, session_id: str, limit: int = 10) -> list[tuple[str, float]]: diff --git a/tests/unit/daemon/test_convergence_stages.py b/tests/unit/daemon/test_convergence_stages.py index a6c70e0522..8fa4efcc39 100644 --- a/tests/unit/daemon/test_convergence_stages.py +++ b/tests/unit/daemon/test_convergence_stages.py @@ -41,6 +41,7 @@ from polylogue.storage.sqlite.archive_tiers.write import write_parsed_session_to_archive from polylogue.storage.sqlite.connection import open_connection from tests.infra.frozen_clock import FrozenClock +from tests.infra.identity import archive_message_id class _SessionIdOnly: @@ -448,7 +449,7 @@ def _seed_minimal_archive(db_path: Path, source_path: Path, *, session_id: str = INSERT INTO blocks(message_id, session_id, position, block_type, text) VALUES (?, ?, 0, 'text', 'archive searchable block') """, - (f"{session_id}:m1", session_id), + (archive_message_id(session_id, "m1", position=0), session_id), ) conn.execute("DELETE FROM messages_fts") conn.commit() diff --git a/tests/unit/pipeline/test_branching.py b/tests/unit/pipeline/test_branching.py index e3e2b08b66..ff24e9afa0 100644 --- a/tests/unit/pipeline/test_branching.py +++ b/tests/unit/pipeline/test_branching.py @@ -23,6 +23,7 @@ from polylogue.storage.sqlite.async_sqlite import SQLiteBackend from polylogue.storage.sqlite.connection import open_connection from tests.infra.archive_scenarios import archive_for_scenario_db +from tests.infra.identity import archive_message_id from tests.infra.live_ingest import ingest_session from tests.infra.storage_records import SessionBuilder, db_setup @@ -249,10 +250,13 @@ async def test_branch_flags_and_message_views_contract(self, workspace_env: Work assert branching is not None bid = ids["branching"] - assert [message.id for message in branching.mainline_messages()] == [f"{bid}:q1", f"{bid}:a1"] + assert [message.id for message in branching.mainline_messages()] == [ + archive_message_id(bid, "q1", position=0), + archive_message_id(bid, "a1", position=1), + ] branches = list(branching.iter_branches()) assert len(branches) == 1 - assert branches[0][0] == f"{bid}:q1" + assert branches[0][0] == archive_message_id(bid, "q1", position=0) assert [message.branch_index for message in branches[0][1]] == [0, 1] assert [message.is_branch for message in branches[0][1]] == [False, True] # is_active_path (not branch_index) is what selected a1: it is the diff --git a/tests/unit/sources/test_claude_code_normalization_laws.py b/tests/unit/sources/test_claude_code_normalization_laws.py index 553e2f6e11..84700eb1d1 100644 --- a/tests/unit/sources/test_claude_code_normalization_laws.py +++ b/tests/unit/sources/test_claude_code_normalization_laws.py @@ -40,25 +40,25 @@ _SUBAGENT_ACOMPACT_FALLBACK_ID = "agent-acompact-task-normalization-proof" _ARCHIVE_SESSION_ID = "claude-code-session:claude-normalization-main" _EXPECTED_ARCHIVE_MESSAGE_IDS = ( - "claude-code-session:claude-normalization-main:main-u1", - "claude-code-session:claude-normalization-main:main-a1", - "claude-code-session:claude-normalization-main:main-bg-start", - "claude-code-session:claude-normalization-main:main-command", - "claude-code-session:claude-normalization-main:main-context", + "claude-code-session:claude-normalization-main:n:main-u1", + "claude-code-session:claude-normalization-main:n:main-a1", + "claude-code-session:claude-normalization-main:n:main-bg-start", + "claude-code-session:claude-normalization-main:n:main-command", + "claude-code-session:claude-normalization-main:n:main-context", # polylogue-slshy: these two wire records carry uuid=None -- the parser # now leaves provider_message_id empty (no positional "msg-N" fallback), # so the generated message_id column falls back to its own # position.variant_index component (COALESCE(native_id, ...)). - "claude-code-session:claude-normalization-main:5.0", - "claude-code-session:claude-normalization-main:main-a2", - "claude-code-session:claude-normalization-main:main-fg-result", - "claude-code-session:claude-normalization-main:8.0", + "claude-code-session:claude-normalization-main:p:5.0", + "claude-code-session:claude-normalization-main:n:main-a2", + "claude-code-session:claude-normalization-main:n:main-fg-result", + "claude-code-session:claude-normalization-main:p:8.0", ) _EXPECTED_MAIN_A1_BLOCK_IDS = ( - "claude-code-session:claude-normalization-main:main-a1:0", - "claude-code-session:claude-normalization-main:main-a1:1", - "claude-code-session:claude-normalization-main:main-a1:2", - "claude-code-session:claude-normalization-main:main-a1:3", + "claude-code-session:claude-normalization-main:n:main-a1:0", + "claude-code-session:claude-normalization-main:n:main-a1:1", + "claude-code-session:claude-normalization-main:n:main-a1:2", + "claude-code-session:claude-normalization-main:n:main-a1:3", ) # Independent normalized-fact oracle. These values are authored alongside the @@ -269,7 +269,9 @@ def test_family_fixture_survives_acquire_parse_store_read_and_action_pairing(tmp assert [message.native_id for message in envelope.messages] == list(_EXPECTED_MAIN_NATIVE_IDS) assert [message.material_origin for message in envelope.messages] == [fact[3] for fact in _EXPECTED_MAIN_MESSAGES] assert [block.block_id for block in envelope.messages[1].blocks] == list(_EXPECTED_MAIN_A1_BLOCK_IDS) - assert envelope.messages[2].blocks[0].block_id == ("claude-code-session:claude-normalization-main:main-bg-start:0") + assert envelope.messages[2].blocks[0].block_id == ( + "claude-code-session:claude-normalization-main:n:main-bg-start:0" + ) assert [block.block_type for block in envelope.messages[1].blocks] == [ "thinking", "text", diff --git a/tests/unit/storage/test_archive_search_contracts.py b/tests/unit/storage/test_archive_search_contracts.py index e7c4655e03..fddb307b8b 100644 --- a/tests/unit/storage/test_archive_search_contracts.py +++ b/tests/unit/storage/test_archive_search_contracts.py @@ -29,6 +29,7 @@ ) from polylogue.storage.sqlite.async_sqlite import SQLiteBackend from polylogue.storage.sqlite.query_store import SQLiteQueryStore +from tests.infra.identity import archive_message_id from tests.infra.live_ingest import ingest_session @@ -349,7 +350,7 @@ async def test_gemini_drive_attachment_id_is_searchable_after_parse_and_prepare( assert hit.session_id == "aistudio-drive:gemini-attachment-identity" assert hit.match_surface == "attachment" assert hit.retrieval_lane == "attachment" - assert hit.message_id == "aistudio-drive:gemini-attachment-identity:msg-doc" + assert hit.message_id == archive_message_id("aistudio-drive:gemini-attachment-identity", "msg-doc", position=0) assert hit.snippet is not None assert expected_snippet in hit.snippet assert 'name="Project Plan"' in hit.snippet diff --git a/tests/unit/storage/test_archive_tiers_archive.py b/tests/unit/storage/test_archive_tiers_archive.py index fed90731c9..3c187ef8db 100644 --- a/tests/unit/storage/test_archive_tiers_archive.py +++ b/tests/unit/storage/test_archive_tiers_archive.py @@ -33,6 +33,7 @@ read_assertion_envelope, ) from polylogue.surfaces.payloads import ActionQueryRowPayload +from tests.infra.identity import archive_message_id from tests.infra.workload_artifacts import build_seeded_archive @@ -58,7 +59,7 @@ def test_active_archive_root_facade_writes_reads_and_searches_archive_db(tmp_pat assert session_id == "codex-session:codex-archive-1" assert envelope.session_id == "codex-session:codex-archive-1" assert len(envelope.messages) == 1 - assert matching_blocks == ["codex-session:codex-archive-1:m1:0"] + assert matching_blocks == [archive_message_id("codex-session:codex-archive-1", "m1", position=0) + ":0"] def test_open_existing_read_timeout_updates_busy_timeout(tmp_path: Path) -> None: @@ -407,7 +408,7 @@ def test_exact_session_action_count_bounds_pairing_before_global_ranking( for index in range(512): native_id = "target" if index == 0 else f"irrelevant-{index:04d}" session_id = f"codex-session:{native_id}" - message_id = f"{session_id}:m1" + message_id = archive_message_id(session_id, "m1", position=0) tool_id = f"tool-{index:04d}" session_rows.append((native_id, Origin.CODEX_SESSION.value, sha256(session_id.encode()).digest())) message_rows.append( @@ -1615,7 +1616,9 @@ def test_archive_tiers_archive_facade_lists_and_searches_session_summaries(tmp_p sampled_summaries = facade.list_summaries(limit=1, offset=1, sample=True) hits = facade.search_summaries("alpha", limit=5) offset_hits = facade.search_summaries("read", limit=1, offset=1) - semantic_hits = facade.semantic_summaries([("codex-session:codex-read-1:m1", 0.2)], limit=5) + semantic_hits = facade.semantic_summaries( + [(archive_message_id("codex-session:codex-read-1", "m1", position=0), 0.2)], limit=5 + ) tagged_hits = facade.search_summaries("alpha", limit=5, tags=("archive",)) excluded_origin_hits = facade.search_summaries("alpha", limit=5, excluded_origins=("chatgpt-export",)) multi_origin_hits = facade.search_summaries("alpha", limit=5, origins=("codex-session", "chatgpt-export")) @@ -1692,7 +1695,7 @@ def test_archive_tiers_archive_facade_lists_and_searches_session_summaries(tmp_p assert hits[0].session_id == first_id assert [hit.rank for hit in offset_hits] == [2] assert semantic_hits[0].session_id == first_id - assert semantic_hits[0].message_id == "codex-session:codex-read-1:m1" + assert semantic_hits[0].message_id == archive_message_id("codex-session:codex-read-1", "m1", position=0) assert tagged_hits[0].session_id == first_id assert excluded_origin_hits[0].session_id == first_id assert multi_origin_hits[0].session_id == first_id @@ -1717,7 +1720,7 @@ def test_archive_tiers_archive_facade_lists_and_searches_session_summaries(tmp_p assert repo_miss_hits == [] assert titled_hits[0].session_id == first_id assert dated_hits[0].session_id == first_id - assert hits[0].block_id == "codex-session:codex-read-1:m1:0" + assert hits[0].block_id == archive_message_id("codex-session:codex-read-1", "m1", position=0) + ":0" assert hits[0].origin == Origin.CODEX_SESSION.value assert "[alpha]" in hits[0].snippet diff --git a/tests/unit/storage/test_archive_tiers_assertions.py b/tests/unit/storage/test_archive_tiers_assertions.py index e593d52a75..8bbc6f36a5 100644 --- a/tests/unit/storage/test_archive_tiers_assertions.py +++ b/tests/unit/storage/test_archive_tiers_assertions.py @@ -52,6 +52,7 @@ upsert_transform_candidate_assertions, ) from polylogue.storage.sqlite.connection_profile import WRITE_CONNECTION_PROFILE, open_connection +from tests.infra.identity import archive_message_id def _connect(path: Path) -> sqlite3.Connection: @@ -107,7 +108,7 @@ def _insert_index_message(conn: sqlite3.Connection, session_id: str, native_id: """, (session_id, native_id, position, Role.ASSISTANT.value, "message", bytes(32)), ) - return f"{session_id}:{native_id}" + return archive_message_id(session_id, native_id, position=position) def _recovery_candidate_session() -> Session: diff --git a/tests/unit/storage/test_archive_tiers_ddl.py b/tests/unit/storage/test_archive_tiers_ddl.py index 0c85171aa9..78cc7086d6 100644 --- a/tests/unit/storage/test_archive_tiers_ddl.py +++ b/tests/unit/storage/test_archive_tiers_ddl.py @@ -12,6 +12,7 @@ initialize_archive_tier, ) from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier +from tests.infra.identity import archive_message_id _HASH = b"x" * 32 @@ -108,9 +109,9 @@ def test_archive_tiers_index_generates_ids_and_actions_view(tmp_path: Path) -> N ) messages = conn.execute("SELECT message_id FROM messages ORDER BY position, variant_index").fetchall() assert [row["message_id"] for row in messages] == [ - "codex-session:native-session:native-message", - "codex-session:native-session:1.0", - "codex-session:native-session:1.1", + archive_message_id("codex-session:native-session", "native-message", position=0), + archive_message_id("codex-session:native-session", None, position=1, variant_index=0), + archive_message_id("codex-session:native-session", None, position=1, variant_index=1), ] conn.execute( @@ -182,9 +183,9 @@ def test_archive_tiers_index_generates_ids_and_actions_view(tmp_path: Path) -> N (session["session_id"],), ).fetchall() assert [row["block_id"] for row in blocks] == [ - "codex-session:native-session:native-message:0", - "codex-session:native-session:native-message:1", - "codex-session:native-session:native-message:2", + archive_message_id("codex-session:native-session", "native-message", position=0) + ":0", + archive_message_id("codex-session:native-session", "native-message", position=0) + ":1", + archive_message_id("codex-session:native-session", "native-message", position=0) + ":2", ] assert blocks[1]["tool_command"] == "pytest -q" assert blocks[1]["tool_path"] == "tests" @@ -210,7 +211,9 @@ def test_archive_tiers_index_generates_ids_and_actions_view(tmp_path: Path) -> N WHERE f.text MATCH 'needle' """ ).fetchone() - assert fts_row["block_id"] == "codex-session:native-session:native-message:0" + assert ( + fts_row["block_id"] == archive_message_id("codex-session:native-session", "native-message", position=0) + ":0" + ) def test_agent_action_and_delegation_views_are_indexed_projections(tmp_path: Path) -> None: diff --git a/tests/unit/storage/test_attachment_acquisition.py b/tests/unit/storage/test_attachment_acquisition.py index 566a2cd6e0..7ea182a78d 100644 --- a/tests/unit/storage/test_attachment_acquisition.py +++ b/tests/unit/storage/test_attachment_acquisition.py @@ -24,6 +24,7 @@ from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_archive_tier from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier from polylogue.storage.sqlite.archive_tiers.write import write_parsed_session_to_archive +from tests.infra.identity import archive_message_id def _connect(path: Path) -> sqlite3.Connection: @@ -222,7 +223,7 @@ def test_claude_extracted_attachment_content_is_acquired(tmp_path: Path, monkeyp ).fetchone() assert ref is not None assert ref[1] == "claude-ai-export:claude-attachment-session" - assert ref[2] == "claude-ai-export:claude-attachment-session:m0" + assert ref[2] == archive_message_id("claude-ai-export:claude-attachment-session", "m0", position=0) @pytest.mark.parametrize("payload", [b"must be reserved first", b""], ids=["nonempty", "empty"]) diff --git a/tests/unit/storage/test_attachment_reacquisition.py b/tests/unit/storage/test_attachment_reacquisition.py index 09384f962c..8274ce7eb8 100644 --- a/tests/unit/storage/test_attachment_reacquisition.py +++ b/tests/unit/storage/test_attachment_reacquisition.py @@ -27,6 +27,7 @@ from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_archive_tier from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier from polylogue.storage.sqlite.archive_tiers.write import write_parsed_session_to_archive +from tests.infra.identity import archive_message_id _CLAUDE_AI_PAYLOAD = { "uuid": "reacq-session-1", @@ -137,6 +138,7 @@ def _write_bare_unfetched_attachment( ) if source_url is not None: dummy_hash = bytes(32) + message_id = archive_message_id("unknown-export:bare-session", "m0", position=0) index_conn.execute( "INSERT INTO sessions (origin, native_id, content_hash) VALUES ('unknown-export', 'bare-session', ?)", (dummy_hash,), @@ -148,8 +150,8 @@ def _write_bare_unfetched_attachment( ) index_conn.execute( "INSERT INTO attachment_refs (attachment_id, session_id, message_id, position, source_url) " - "VALUES (?, 'unknown-export:bare-session', 'unknown-export:bare-session:m0', 0, ?)", - (attachment_id, source_url), + "VALUES (?, 'unknown-export:bare-session', ?, 0, ?)", + (attachment_id, message_id, source_url), ) index_conn.commit() diff --git a/tests/unit/storage/test_fts_identity_ledger.py b/tests/unit/storage/test_fts_identity_ledger.py index 5e5eae881a..be9d73d087 100644 --- a/tests/unit/storage/test_fts_identity_ledger.py +++ b/tests/unit/storage/test_fts_identity_ledger.py @@ -30,6 +30,7 @@ ) from polylogue.storage.fts.sql import FTS_MESSAGES_IDENTITY_RECIPE_ID, message_identity_mismatch_sql from polylogue.storage.sqlite.archive_tiers.ops_write import list_fts_drift_samples, record_fts_drift_sample +from tests.infra.identity import archive_message_id if TYPE_CHECKING: from polylogue.sources.parsers.base import ParsedSession @@ -52,7 +53,7 @@ def _seed_block( """ origin = "unknown-export" session_id = f"{origin}:{native_session_id}" - message_id = f"{session_id}:{native_message_id}" + message_id = archive_message_id(session_id, native_message_id, position=message_position) conn.execute( "INSERT OR IGNORE INTO sessions (native_id, origin, title, content_hash) VALUES (?, ?, ?, ?)", (native_session_id, origin, "Identity ledger test", content_hash), @@ -352,7 +353,7 @@ def test_orphan_identity_row_without_docsize_is_detected(self, test_conn: sqlite native_message_id="msg-identity-orphan", text="will be deleted from messages_fts only", ) - block_id = "unknown-export:conv-identity-orphan:msg-identity-orphan:0" + block_id = archive_message_id("unknown-export:conv-identity-orphan", "msg-identity-orphan", position=0) + ":0" rowid = _block_rowid(test_conn, block_id) assert _identity_mismatch_count(test_conn) == 0 diff --git a/tests/unit/storage/test_fts_repair_sql.py b/tests/unit/storage/test_fts_repair_sql.py index c7522d28de..87e42c9655 100644 --- a/tests/unit/storage/test_fts_repair_sql.py +++ b/tests/unit/storage/test_fts_repair_sql.py @@ -19,12 +19,13 @@ insert_session_rows_sql, trigram_delete_session_rows_sql, ) +from tests.infra.identity import archive_message_id def _seed_text_block(conn: sqlite3.Connection, *, native_session_id: str, native_message_id: str, text: str) -> str: origin = "unknown-export" session_id = f"{origin}:{native_session_id}" - message_id = f"{session_id}:{native_message_id}" + message_id = archive_message_id(session_id, native_message_id, position=0) content_hash = b"x" * 32 conn.execute( "INSERT INTO sessions (native_id, origin, title, content_hash) VALUES (?, ?, ?, ?)", diff --git a/tests/unit/storage/test_incremental_rebuild_equivalence.py b/tests/unit/storage/test_incremental_rebuild_equivalence.py index 08483f25e1..ae94afdda1 100644 --- a/tests/unit/storage/test_incremental_rebuild_equivalence.py +++ b/tests/unit/storage/test_incremental_rebuild_equivalence.py @@ -39,17 +39,18 @@ from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root from polylogue.storage.sqlite.archive_tiers.index import INDEX_SCHEMA_VERSION +from tests.infra.identity import archive_block_id, archive_message_id CHAT_SESSION = "chatgpt-export:branch-canary" PARENT_SESSION = "codex-session:lineage-parent" CHILD_SESSION = "codex-session:lineage-child" -CHAT_USER = f"{CHAT_SESSION}:u1" -CHAT_OLD = f"{CHAT_SESSION}:a-old" -CHAT_NEW = f"{CHAT_SESSION}:a-new" -CHAT_NEW_BLOCK = f"{CHAT_NEW}:0" -PARENT_FIRST = f"{PARENT_SESSION}:lineage-parent-m0" -PARENT_SECOND = f"{PARENT_SESSION}:lineage-parent-m1" -CHILD_FIRST = f"{CHILD_SESSION}:lineage-child-m0" +CHAT_USER = archive_message_id(CHAT_SESSION, "u1", position=0) +CHAT_OLD = archive_message_id(CHAT_SESSION, "a-old", position=1) +CHAT_NEW = archive_message_id(CHAT_SESSION, "a-new", position=1) +CHAT_NEW_BLOCK = archive_block_id(CHAT_NEW, position=0) +PARENT_FIRST = archive_message_id(PARENT_SESSION, "lineage-parent-m0", position=0) +PARENT_SECOND = archive_message_id(PARENT_SESSION, "lineage-parent-m1", position=1) +CHILD_FIRST = archive_message_id(CHILD_SESSION, "lineage-child-m0", position=0) CANONICAL_TOKEN = "quartzneedle" STALE_TOKEN = "staleonlytoken" OVERLAY_TAG = "operator-canary" diff --git a/tests/unit/storage/test_lineage_normalization.py b/tests/unit/storage/test_lineage_normalization.py index d88dd92469..f4c816115a 100644 --- a/tests/unit/storage/test_lineage_normalization.py +++ b/tests/unit/storage/test_lineage_normalization.py @@ -45,6 +45,7 @@ get_messages_with_lineage_completeness, iter_messages, ) +from tests.infra.identity import archive_message_id def _connect(path: Path) -> sqlite3.Connection: @@ -281,14 +282,14 @@ def test_prefix_sharing_child_provider_usage_rollup_counts_only_tail(tmp_path: P ).fetchall() assert [dict(row) for row in events] == [ { - "source_message_id": f"{child_id}:cy", + "source_message_id": archive_message_id(child_id, "cy", position=0), "total_input_tokens": 60, "total_cached_input_tokens": 10, "total_output_tokens": 15, "total_tokens": 75, }, { - "source_message_id": f"{child_id}:cy", + "source_message_id": archive_message_id(child_id, "cy", position=0), "total_input_tokens": 0, "total_cached_input_tokens": 0, "total_output_tokens": 0, @@ -328,7 +329,7 @@ def test_provider_usage_baseline_follows_ancestor_branch_point(tmp_path: Path) - ) ancestor_id = write_parsed_session_to_archive(conn, ancestor) parent_id = "codex-session:parent" - branch_point = f"{ancestor_id}:a1" + branch_point = archive_message_id(ancestor_id, "a1", position=0) baseline = _provider_usage_cumulative_baseline(conn, parent_id, branch_point) @@ -453,7 +454,12 @@ def test_variant_prefix_lineage_converges_across_order_and_parent_replacement( "FROM session_links WHERE src_session_id = ?", (child_id,), ).fetchone() - assert tuple(link) == (parent_id, f"{parent_id}:p1-alt", "prefix-sharing", None) + assert tuple(link) == ( + parent_id, + archive_message_id(parent_id, "p1-alt", position=0), + "prefix-sharing", + None, + ) assert asyncio.run(_read_texts(db, child_id)) == ["root", "primary v1", "sibling v1", "child tail"] replacement = parent.model_copy( @@ -533,7 +539,7 @@ def test_missing_variant_branch_point_keeps_only_owned_child_tail(tmp_path: Path link = conn.execute( "SELECT branch_point_message_id, inheritance, status FROM session_links WHERE src_session_id = ?", (child_id,) ).fetchone() - assert tuple(link) == (f"{parent_id}:p1-alt", "prefix-sharing", None) + assert tuple(link) == (archive_message_id(parent_id, "p1-alt", position=0), "prefix-sharing", None) envelope = read_archive_session_envelope(conn, child_id) assert [message.blocks[0].text for message in envelope.messages] == ["child tail"] assert envelope.lineage_complete is False @@ -566,7 +572,7 @@ def test_reingest_after_dangling_ancestor_does_not_fabricate_a_prefix(tmp_path: ], ) parent_id = write_parsed_session_to_archive(conn, parent) - conn.execute("DELETE FROM messages WHERE message_id = ?", (f"{root_id}:r1",)) + conn.execute("DELETE FROM messages WHERE message_id = ?", (archive_message_id(root_id, "r1", position=0),)) conn.commit() child = ParsedSession( @@ -661,7 +667,7 @@ def test_nested_dangling_ancestor_keeps_only_reachable_tails(tmp_path: Path) -> # session, but its branch-point message disappeared. Do not rewrite a # resolved edge as 'unresolved': the typed degradation belongs on the # composed read result while its known relation remains queryable. - conn.execute("DELETE FROM messages WHERE message_id = ?", (f"{root_id}:r1",)) + conn.execute("DELETE FROM messages WHERE message_id = ?", (archive_message_id(root_id, "r1", position=0),)) conn.commit() envelope = read_archive_session_envelope(conn, child_id) @@ -841,7 +847,7 @@ def test_stale_immediate_parent_branch_point_repairs_to_composed_ancestor(tmp_pa ], ) child_id = write_parsed_session_to_archive(conn, child) - stale_branch_point = f"{parent_id}:a1" + stale_branch_point = archive_message_id(parent_id, "a1", position=0) conn.execute( """ UPDATE session_links @@ -863,7 +869,7 @@ def test_stale_immediate_parent_branch_point_repairs_to_composed_ancestor(tmp_pa "SELECT branch_point_message_id FROM session_links WHERE src_session_id = ?", (child_id,), ).fetchone()[0] - assert branch_point == f"{ancestor_id}:a1" + assert branch_point == archive_message_id(ancestor_id, "a1", position=0) assert [message.blocks[0].text for message in read_archive_session_envelope(conn, child_id).messages] == [ "hello", "hi there", @@ -931,7 +937,7 @@ def test_stale_non_materialized_msg_branch_point_repairs_to_predecessor(tmp_path "SELECT branch_point_message_id FROM session_links WHERE src_session_id = ?", (child_id,), ).fetchone()[0] - assert branch_point == f"{ancestor_id}:msg-10" + assert branch_point == archive_message_id(ancestor_id, "msg-10", position=0) assert [message.blocks[0].text for message in read_archive_session_envelope(conn, child_id).messages] == [ "inherited prompt", "child tail", @@ -1018,7 +1024,7 @@ def test_child_before_parent_reextracts_cleanly_when_foreign_keys_suspended(tmp_ (child_id,), ).fetchone() assert dict(event_ref) == { - "source_message_id": f"{parent_id}:p1", + "source_message_id": archive_message_id(parent_id, "p1", position=0), "source_message_provider_id": "c1", } @@ -1970,7 +1976,7 @@ def test_writer_composes_beyond_recursive_reader_depth(tmp_path: Path) -> None: "SELECT inheritance, branch_point_message_id FROM session_links WHERE src_session_id = ?", (leaf_id,), ).fetchone() - assert tuple(link) == ("prefix-sharing", f"{root_id}:root-0") + assert tuple(link) == ("prefix-sharing", archive_message_id(root_id, "root-0", position=0)) assert conn.execute("SELECT COUNT(*) FROM messages WHERE session_id = ?", (leaf_id,)).fetchone()[0] == 1 # Full replacement/re-ingest exercises writer alignment again rather than @@ -1983,7 +1989,7 @@ def test_writer_composes_beyond_recursive_reader_depth(tmp_path: Path) -> None: "SELECT inheritance, branch_point_message_id FROM session_links WHERE src_session_id = ?", (leaf_id,), ).fetchone() - assert tuple(link) == ("prefix-sharing", f"{root_id}:root-0") + assert tuple(link) == ("prefix-sharing", archive_message_id(root_id, "root-0", position=0)) assert conn.execute("SELECT COUNT(*) FROM messages WHERE session_id = ?", (leaf_id,)).fetchone()[0] == 1 conn.close() diff --git a/tests/unit/storage/test_message_query_reads.py b/tests/unit/storage/test_message_query_reads.py index df41c5ef73..0a1dbaf435 100644 --- a/tests/unit/storage/test_message_query_reads.py +++ b/tests/unit/storage/test_message_query_reads.py @@ -16,6 +16,7 @@ get_messages_paginated, iter_messages, ) +from tests.infra.identity import archive_message_id from tests.infra.storage_records import make_message, make_session, save_session_to_archive @@ -25,12 +26,12 @@ async def test_message_query_reads_cover_type_filters_batches_and_stream_limits( backend = SQLiteBackend(db_path=tmp_path / "index.db") current_session_id = "unknown-export:conv-message-reads" expected_message_ids = [ - f"{current_session_id}:msg-summary", - f"{current_session_id}:msg-summary-2", - f"{current_session_id}:msg-tool", - f"{current_session_id}:msg-user", - f"{current_session_id}:msg-protocol", - f"{current_session_id}:msg-assistant", + archive_message_id(current_session_id, "msg-summary", position=0), + archive_message_id(current_session_id, "msg-summary-2", position=1), + archive_message_id(current_session_id, "msg-tool", position=2), + archive_message_id(current_session_id, "msg-user", position=3), + archive_message_id(current_session_id, "msg-protocol", position=4), + archive_message_id(current_session_id, "msg-assistant", position=5), ] conv = make_session("conv-message-reads", title="Message Reads") messages = [ @@ -107,9 +108,11 @@ async def test_message_query_reads_cover_type_filters_batches_and_stream_limits( message_role=(Role.USER,), ) assert [message.message_id for message in filtered_by_session[current_session_id]] == [ - f"{current_session_id}:msg-user" + archive_message_id(current_session_id, "msg-user", position=3) + ] + assert [message.message_id for message in filtered_messages] == [ + archive_message_id(current_session_id, "msg-user", position=3) ] - assert [message.message_id for message in filtered_messages] == [f"{current_session_id}:msg-user"] assert [message.message_id for message in all_messages] == expected_message_ids traced_sql: list[str] = [] @@ -125,8 +128,12 @@ async def test_message_query_reads_cover_type_filters_batches_and_stream_limits( finally: await conn.set_trace_callback(lambda _statement: None) assert edge_total == 3 - assert [message.message_id for message in first_edge] == [f"{current_session_id}:msg-user"] - assert [message.message_id for message in last_edge] == [f"{current_session_id}:msg-assistant"] + assert [message.message_id for message in first_edge] == [ + archive_message_id(current_session_id, "msg-user", position=3) + ] + assert [message.message_id for message in last_edge] == [ + archive_message_id(current_session_id, "msg-assistant", position=5) + ] assert any("COUNT(*) FROM messages INDEXED BY idx_messages_session_position" in sql for sql in traced_sql) authored_first, authored_last, authored_total = await get_message_edge_windows( @@ -139,8 +146,8 @@ async def test_message_query_reads_cover_type_filters_batches_and_stream_limits( ) assert authored_total == 2 assert [message.message_id for message in authored_first] == [ - f"{current_session_id}:msg-user", - f"{current_session_id}:msg-assistant", + archive_message_id(current_session_id, "msg-user", position=3), + archive_message_id(current_session_id, "msg-assistant", position=5), ] assert authored_last == [] @@ -152,7 +159,9 @@ async def test_message_query_reads_cover_type_filters_batches_and_stream_limits( offset=0, ) assert total == 2 - assert [message.message_id for message in paginated] == [f"{current_session_id}:msg-summary"] + assert [message.message_id for message in paginated] == [ + archive_message_id(current_session_id, "msg-summary", position=0) + ] assert paginated_completeness.complete is True paginated_with_offset, offset_total, _offset_completeness = await get_messages_paginated( @@ -163,7 +172,9 @@ async def test_message_query_reads_cover_type_filters_batches_and_stream_limits( offset=1, ) assert offset_total == 2 - assert [message.message_id for message in paginated_with_offset] == [f"{current_session_id}:msg-summary-2"] + assert [message.message_id for message in paginated_with_offset] == [ + archive_message_id(current_session_id, "msg-summary-2", position=1) + ] tool_messages, tool_total, _tool_completeness = await get_messages_paginated( conn, @@ -173,7 +184,9 @@ async def test_message_query_reads_cover_type_filters_batches_and_stream_limits( offset=0, ) assert tool_total == 1 - assert [message.message_id for message in tool_messages] == [f"{current_session_id}:msg-tool"] + assert [message.message_id for message in tool_messages] == [ + archive_message_id(current_session_id, "msg-tool", position=2) + ] user_messages, user_total, _user_completeness = await get_messages_paginated( conn, @@ -184,9 +197,9 @@ async def test_message_query_reads_cover_type_filters_batches_and_stream_limits( ) assert user_total == 3 assert [message.message_id for message in user_messages] == [ - f"{current_session_id}:msg-user", - f"{current_session_id}:msg-protocol", - f"{current_session_id}:msg-assistant", + archive_message_id(current_session_id, "msg-user", position=3), + archive_message_id(current_session_id, "msg-protocol", position=4), + archive_message_id(current_session_id, "msg-assistant", position=5), ] with pytest.raises(ValueError, match="Unknown message type"): @@ -212,7 +225,7 @@ async def test_message_query_reads_cover_type_filters_batches_and_stream_limits( chunk_size=1, limit=1, ) - ] == [f"{current_session_id}:msg-user"] + ] == [archive_message_id(current_session_id, "msg-user", position=3)] await backend.close() diff --git a/tests/unit/storage/test_pl_fold.py b/tests/unit/storage/test_pl_fold.py index 2b330b2e20..59efe18af4 100644 --- a/tests/unit/storage/test_pl_fold.py +++ b/tests/unit/storage/test_pl_fold.py @@ -25,6 +25,7 @@ from polylogue.storage.search.query_support import escape_fts5_query, normalize_fts5_query from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore from polylogue.storage.sqlite.archive_tiers.index import INDEX_DDL +from tests.infra.identity import archive_message_id # --------------------------------------------------------------------------- # Python fold semantics @@ -139,7 +140,7 @@ def test_all_contentless_fts_surfaces_use_the_same_diacritic_folding_tokenizer() def _seed_text_block(conn: sqlite3.Connection, *, native_session_id: str, native_message_id: str, text: str) -> str: origin = "unknown-export" session_id = f"{origin}:{native_session_id}" - message_id = f"{session_id}:{native_message_id}" + message_id = archive_message_id(session_id, native_message_id, position=0) content_hash = b"x" * 32 conn.execute( "INSERT INTO sessions (native_id, origin, title, content_hash) VALUES (?, ?, ?, ?)", diff --git a/tests/unit/storage/test_query_unit_time_expression.py b/tests/unit/storage/test_query_unit_time_expression.py index d0062fb847..96a87ef9dd 100644 --- a/tests/unit/storage/test_query_unit_time_expression.py +++ b/tests/unit/storage/test_query_unit_time_expression.py @@ -20,6 +20,7 @@ from polylogue.archive.query.expression import parse_unit_source_expression from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore +from tests.infra.identity import archive_message_id _TIMELESS_ORIGIN = "codex-session" @@ -107,7 +108,7 @@ def test_query_files_first_last_seen_ms_is_none_not_epoch_for_timeless_action(tm conn = facade._conn timeless = _insert_timeless_session(conn, native_id="timeless-file") _insert_message(conn, session_id=timeless, position=0) - message_id = f"{timeless}:0.0" + message_id = archive_message_id(timeless, None, position=0) conn.execute( """ INSERT INTO blocks (message_id, session_id, position, block_type, tool_name, tool_id, tool_input) diff --git a/tests/unit/storage/test_schema_safety.py b/tests/unit/storage/test_schema_safety.py index e35efc55e2..15558be43e 100644 --- a/tests/unit/storage/test_schema_safety.py +++ b/tests/unit/storage/test_schema_safety.py @@ -19,6 +19,7 @@ import pytest from polylogue.storage.sqlite.schema import SCHEMA_DDL, SCHEMA_VERSION +from tests.infra.identity import archive_message_id # ============================================================================= # Schema DDL parity: sync and async must use the same DDL (f33ef29) @@ -329,7 +330,7 @@ def test_count_on_regular_table_not_fts(self, test_db: Path) -> None: """, (session_id,), ) - message_id = f"{session_id}:m1" + message_id = archive_message_id(session_id, "m1", position=0) conn.execute( """ INSERT INTO blocks (message_id, session_id, position, block_type, text) diff --git a/tests/unit/storage/test_search_timeless_since_filter.py b/tests/unit/storage/test_search_timeless_since_filter.py index 231e44db4c..a5d4e45618 100644 --- a/tests/unit/storage/test_search_timeless_since_filter.py +++ b/tests/unit/storage/test_search_timeless_since_filter.py @@ -26,6 +26,7 @@ from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore from polylogue.storage.sqlite.queries.attachment_records import search_attachment_identity_evidence_hits from polylogue.storage.sqlite.schema import SCHEMA_DDL +from tests.infra.identity import archive_message_id def _insert_timeless_session_with_text_block( @@ -40,7 +41,7 @@ def _insert_timeless_session_with_text_block( "INSERT INTO messages (session_id, position, role, content_hash) VALUES (?, 0, 'assistant', ?)", (session_id, bytes(32)), ) - message_id = f"{session_id}:0.0" + message_id = archive_message_id(session_id, None, position=0) conn.execute( "INSERT INTO blocks (message_id, session_id, position, block_type, text) VALUES (?, ?, 0, 'text', ?)", (message_id, session_id, text), @@ -100,7 +101,7 @@ def test_ranked_action_search_since_filter_includes_timeless_session(tmp_path: P "INSERT INTO messages (session_id, position, role, content_hash) VALUES (?, 0, 'assistant', ?)", (session_id, bytes(32)), ) - message_id = f"{session_id}:0.0" + message_id = archive_message_id(session_id, None, position=0) conn.execute( "INSERT INTO blocks (message_id, session_id, position, block_type, tool_name, tool_id, text) " "VALUES (?, ?, 0, 'tool_use', 'Bash', 'tool-1', 'run pytest suite')", @@ -128,7 +129,7 @@ def test_ranked_session_search_since_filter_still_excludes_out_of_range_timestam "INSERT INTO messages (session_id, position, role, content_hash) VALUES (?, 0, 'assistant', ?)", (session_id, bytes(32)), ) - message_id = f"{session_id}:0.0" + message_id = archive_message_id(session_id, None, position=0) conn.execute( "INSERT INTO blocks (message_id, session_id, position, block_type, text) VALUES (?, ?, 0, 'text', ?)", (message_id, session_id, "the quick fox jumps"), diff --git a/tests/unit/storage/test_session_insight_refresh.py b/tests/unit/storage/test_session_insight_refresh.py index b6e773f9e7..515a087abd 100644 --- a/tests/unit/storage/test_session_insight_refresh.py +++ b/tests/unit/storage/test_session_insight_refresh.py @@ -31,6 +31,7 @@ from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root from polylogue.storage.sqlite.async_sqlite import SQLiteBackend from polylogue.storage.sqlite.connection import open_connection +from tests.infra.identity import archive_message_id from tests.infra.storage_records import make_message, make_session, store_records @@ -1050,7 +1051,9 @@ def test_session_insight_load_skips_plain_text_blocks(tmp_path: Path) -> None: ) batch = load_sync_batch(conn, [_sid("conv-blocks", "codex-session")]) - assert [str(block.message_id) for block in batch.blocks] == [_sid("conv-blocks", "codex-session") + ":msg-2"] + assert [str(block.message_id) for block in batch.blocks] == [ + archive_message_id(_sid("conv-blocks", "codex-session"), "msg-2", position=1) + ] def test_session_insight_load_includes_compaction_session_events_for_profile_classifiers(tmp_path: Path) -> None: diff --git a/tests/unit/storage/test_spec_driven_hydration.py b/tests/unit/storage/test_spec_driven_hydration.py index e8474e23c8..6c2a9a7782 100644 --- a/tests/unit/storage/test_spec_driven_hydration.py +++ b/tests/unit/storage/test_spec_driven_hydration.py @@ -15,6 +15,7 @@ from polylogue.storage.sqlite.async_sqlite import SQLiteBackend from polylogue.storage.sqlite.queries import message_query_reads from polylogue.storage.sqlite.queries.mappers_archive import _row_to_message +from tests.infra.identity import archive_message_id from tests.infra.live_ingest import ingest_session @@ -70,7 +71,9 @@ async def test_real_archive_write_read_hydrates_spec_fields(tmp_path: Path) -> N finally: await backend.close() - assert str(records[0].message_id) == "claude-code-session:spec-hydration:native-message" + assert str(records[0].message_id) == archive_message_id( + "claude-code-session:spec-hydration", "native-message", position=0 + ) assert str(blocks[str(records[0].message_id)][0].block_id) == f"{records[0].message_id}:0" assert hydrated.stop_reason == "end_turn" assert hydrated.is_active_path is True diff --git a/tests/unit/storage/test_store_ops.py b/tests/unit/storage/test_store_ops.py index 72f8d03e75..f56bafbc97 100644 --- a/tests/unit/storage/test_store_ops.py +++ b/tests/unit/storage/test_store_ops.py @@ -37,6 +37,7 @@ ) from polylogue.storage.sqlite.async_sqlite import SQLiteBackend from polylogue.storage.sqlite.connection import open_connection +from tests.infra.identity import archive_message_id from tests.infra.storage_records import ( _prune_attachment_refs, make_attachment, @@ -840,7 +841,7 @@ def test_prune_attachment_refs_contract(test_conn: sqlite3.Connection) -> None: WHERE ar.session_id = ? AND ar.message_id = ? AND ani.native_id IN ('att-prune-1', 'att-shared') ORDER BY ani.native_id """, - (current_session_id, f"{current_session_id}:msg-prune-1"), + (current_session_id, archive_message_id(current_session_id, "msg-prune-1", position=0)), ).fetchall() keep_refs = {str(row["ref_id"]) for row in keep_rows} assert len(keep_refs) == 2 @@ -897,7 +898,7 @@ def test_upsert_optional_and_attachment_contracts(test_conn: sqlite3.Connection) ("msg-optional",), ).fetchone() assert msg_row is not None - assert msg_row["message_id"] == "unknown-export:conv-optional:msg-optional" + assert msg_row["message_id"] == archive_message_id("unknown-export:conv-optional", "msg-optional", position=0) assert msg_row["role"] == "unknown" assert msg_row["native_id"] == "msg-optional" assert ( @@ -1598,8 +1599,8 @@ async def test_get_eager_multiple_attachments(self, workspace_env: dict[str, Pat for row in rows: by_message.setdefault(str(row["message_id"]), set()).add(str(row["attachment_native_id"])) - assert by_message[f"{session_id}:m1"] == {"att1", "att2"} - assert by_message[f"{session_id}:m2"] == {"att3"} + assert by_message[archive_message_id(session_id, "m1", position=0)] == {"att1", "att2"} + assert by_message[archive_message_id(session_id, "m2", position=1)] == {"att3"} async def test_get_eager_attachment_metadata_not_stored_in_index(self, workspace_env: dict[str, Path]) -> None: """Attachment metadata is not an index-tier escape hatch.""" diff --git a/tests/unit/storage/test_unread_wire_batch_v46.py b/tests/unit/storage/test_unread_wire_batch_v46.py index 16178b9839..c032b31b95 100644 --- a/tests/unit/storage/test_unread_wire_batch_v46.py +++ b/tests/unit/storage/test_unread_wire_batch_v46.py @@ -20,6 +20,7 @@ ) from polylogue.storage.repository import SessionRepository from polylogue.storage.sqlite.async_sqlite import SQLiteBackend +from tests.infra.identity import archive_block_id, archive_message_id from tests.infra.live_ingest import ingest_session @@ -299,8 +300,8 @@ async def test_file_edits_round_trip_keyed_by_tool_use_block(tmp_path: Path) -> assert edit.structured_patch == [{"oldStart": 1, "oldLines": 1, "newStart": 1, "newLines": 2, "lines": ["+x"]}] # Keyed by the TOOL_USE block (message m1, position 0) even though the # evidence was attached to the TOOL_RESULT block reported in message m2. - assert edit.tool_use_block_id == f"{session_id}:m1:0" - assert edit.message_id == f"{session_id}:m2" + assert edit.tool_use_block_id == archive_block_id(archive_message_id(session_id, "m1", position=0), position=0) + assert edit.message_id == archive_message_id(session_id, "m2", position=1) async def test_file_edits_empty_for_session_without_edits(tmp_path: Path) -> None: @@ -456,8 +457,8 @@ async def test_web_content_constructs_round_trip_search_result(tmp_path: Path) - query_construct = by_type["search_query"] assert query_construct.session_id == session_id assert query_construct.query == "polylogue archive" - assert query_construct.message_id == f"{session_id}:m1" - assert query_construct.block_id == f"{session_id}:m1:0" + assert query_construct.message_id == archive_message_id(session_id, "m1", position=0) + assert query_construct.block_id == archive_block_id(query_construct.message_id, position=0) result_construct = by_type["search_result"] assert result_construct.title == "Polylogue" @@ -562,5 +563,7 @@ async def test_session_links_parent_tool_use_block_id_resolves_via_tool_id(tmp_p assert len(links) == 1 link = links[0] - assert link["parent_tool_use_block_id"] == f"{parent_id}:m1:0" + assert link["parent_tool_use_block_id"] == archive_block_id( + archive_message_id(parent_id, "m1", position=0), position=0 + ) assert link["method"] == "parent-tool-use-id" diff --git a/tests/visual/conftest.py b/tests/visual/conftest.py index b5f2a1d196..705ad27fe1 100644 --- a/tests/visual/conftest.py +++ b/tests/visual/conftest.py @@ -18,6 +18,7 @@ import pytest from tests.infra.archive_scenarios import native_session_id_for +from tests.infra.identity import archive_message_id POLYLOGUE_LOCAL_PATH_PREFIXES = ("/home/", "/Users/", "/realm/", "/var/", "/etc/") @@ -136,11 +137,11 @@ def index_db_path(workspace: ReaderWorkspace) -> Path: READER_C1 = native_session_id_for("claude-code", "reader-c1") READER_C2 = native_session_id_for("chatgpt", "reader-c2") READER_C3 = native_session_id_for("claude-ai", "reader-c3") -READER_C1_M1 = f"{READER_C1}:reader-c1-m1" -READER_C1_M2 = f"{READER_C1}:reader-c1-m2" -READER_C1_M3 = f"{READER_C1}:reader-c1-m3" -READER_C3_M1 = f"{READER_C3}:reader-c3-m1" -READER_C3_DIFF = f"{READER_C3}:reader-c3-diff" +READER_C1_M1 = archive_message_id(READER_C1, "reader-c1-m1", position=0) +READER_C1_M2 = archive_message_id(READER_C1, "reader-c1-m2", position=1) +READER_C1_M3 = archive_message_id(READER_C1, "reader-c1-m3", position=2) +READER_C3_M1 = archive_message_id(READER_C3, "reader-c3-m1", position=0) +READER_C3_DIFF = archive_message_id(READER_C3, "reader-c3-diff", position=1) def _attachment_native_id(message_id: str, attachment_id: str) -> str: From 98ec9653dbd8847f25ea922b32d8b2e779676470 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 9 Aug 2026 00:34:31 +0200 Subject: [PATCH 03/19] fix(storage): resolve message owners from indexed identity Problem: parsing canonical message IDs at textual :n: or :p: markers treats opaque provider-native session IDs as delimiters.\n\nWhat changed: list_marks and list_annotations now resolve indexed message ownership through messages.session_id, while retaining the legacy fallback for unresolved or deleted message assertions. Add a real-route regression covering both opaque marker forms.\n\nCompatibility/migration: session targets and non-message targets retain their existing behavior; no schema or production data changes are required. --- .../storage/sqlite/archive_tiers/archive.py | 40 ++++++++++----- .../storage/test_marks_identity_preserving.py | 49 +++++++++++++++++++ 2 files changed, 78 insertions(+), 11 deletions(-) diff --git a/polylogue/storage/sqlite/archive_tiers/archive.py b/polylogue/storage/sqlite/archive_tiers/archive.py index fb15fb971b..fb0dd98bfe 100644 --- a/polylogue/storage/sqlite/archive_tiers/archive.py +++ b/polylogue/storage/sqlite/archive_tiers/archive.py @@ -5496,7 +5496,11 @@ def list_marks( { "target_type": found_target_type, "target_id": found_target_id, - "session_id": _user_mark_session_id(found_target_type, found_target_id), + "session_id": _user_mark_session_id( + found_target_type, + found_target_id, + index_conn=self._conn, + ), "message_id": found_target_id if found_target_type == "message" else "", "mark_type": str(assertion.key or ""), "created_at": str(assertion.created_at_ms), @@ -5669,7 +5673,11 @@ def list_annotations( "annotation_id": found_annotation_id, "target_type": found_target_type, "target_id": found_target_id, - "session_id": _user_mark_session_id(found_target_type, found_target_id), + "session_id": _user_mark_session_id( + found_target_type, + found_target_id, + index_conn=self._conn, + ), "message_id": found_target_id if found_target_type == "message" else "", "note_text": assertion.body_text or "", "created_at": str(assertion.created_at_ms), @@ -9334,18 +9342,28 @@ def _active_assertion_by_kind_key( return None -def _user_mark_session_id(target_type: str, target_id: str) -> str: +def _user_mark_session_id( + target_type: str, + target_id: str, + *, + index_conn: sqlite3.Connection | None = None, +) -> str: if target_type == "session": return target_id if target_type == "message": - # Message identities use tagged local components (``:n:`` or - # ``:p:.``). Splitting at the final colon used to - # work only while native IDs were untagged; with ``session:n:native`` - # it incorrectly reports ``session:n`` as the owning session. - for marker in (":n:", ":p:"): - marker_index = target_id.find(marker) - if marker_index > 0: - return target_id[:marker_index] + # Canonical message IDs are generated from the indexed message row, + # whose session_id column is the authoritative owner. Provider-native + # session IDs are opaque and may contain the ``:n:`` or ``:p:`` tokens, + # so those tokens cannot safely delimit the owning session. + if index_conn is not None: + row = index_conn.execute( + "SELECT session_id FROM messages WHERE message_id = ?", + (target_id,), + ).fetchone() + if row is not None: + return str(row["session_id"]) + # A message assertion can outlive its indexed message. Retain the + # historical delimiter fallback for those legacy/unresolved IDs. session_id, _sep, _message_native_id = target_id.rpartition(":") return session_id return "" diff --git a/tests/unit/storage/test_marks_identity_preserving.py b/tests/unit/storage/test_marks_identity_preserving.py index dc56d32dd4..b03f8290fe 100644 --- a/tests/unit/storage/test_marks_identity_preserving.py +++ b/tests/unit/storage/test_marks_identity_preserving.py @@ -136,6 +136,55 @@ async def test_list_marks_projects_session_vocabulary(workspace_env: dict[str, P assert {(row["target_type"], row["target_id"]) for row in rows} == {("session", session_id)} +@pytest.mark.asyncio +@pytest.mark.parametrize("opaque_marker", [":n:", ":p:"]) +async def test_message_user_state_projects_owner_for_opaque_session_native_ids( + workspace_env: dict[str, Path], + opaque_marker: str, +) -> None: + """Message user state resolves ownership from indexed identity, not delimiters.""" + db_path = db_setup(workspace_env) + builder = ( + SessionBuilder(db_path, f"opaque{opaque_marker}session") + .provider("claude-code") + .add_message( + message_id="message-native", + text="hello", + ) + ) + builder.save() + session_id = builder.native_session_id() + message_id = f"{session_id}:n:message-native" + + async with Polylogue(db_path=db_path, archive_root=workspace_env["archive_root"]) as poly: + assert ( + await poly.add_mark( + session_id, + "pin", + target_type="message", + message_id=message_id, + ) + is True + ) + assert ( + await poly.save_annotation( + "opaque-session-message-note", + session_id, + "important", + target_type="message", + message_id=message_id, + ) + is True + ) + marks = await poly.list_marks(mark_type="pin") + annotations = await poly.list_annotations() + + assert marks[0]["target_id"] == message_id + assert marks[0]["session_id"] == session_id + assert annotations[0]["target_id"] == message_id + assert annotations[0]["session_id"] == session_id + + # --------------------------------------------------------------------------- # The core #1114 acceptance: marks survive hard delete and rebind on reimport # --------------------------------------------------------------------------- From cc7ca8ca625b788325b857ee47bd23759dae04d7 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 9 Aug 2026 00:34:48 +0200 Subject: [PATCH 04/19] test(reindex): record slshy successor coverage Problem: the branch closes polylogue-slshy, but the structured 818fy coverage ledger and campaign graph still report it as in progress with no closure path.\n\nWhat changed: mark slshy closed, link the durable polylogue-xselt child successor in both structured records, and update the validator expectation. Receipts remain empty because no live-production proof exists.\n\nCompatibility/migration: the parser-stamps snapshot remains blocking until the named successor or a future live proof satisfies it. --- docs/plans/reindex-incident-coverage.json | 5 +++-- tests/fixtures/reindex_incident_coverage/campaign_graph.json | 5 +++-- tests/unit/devtools/test_incident_coverage_ledger.py | 1 + 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/plans/reindex-incident-coverage.json b/docs/plans/reindex-incident-coverage.json index c675861092..b78ea7d772 100644 --- a/docs/plans/reindex-incident-coverage.json +++ b/docs/plans/reindex-incident-coverage.json @@ -69,7 +69,8 @@ "polylogue-byte-supersession-live-proof": {"kind": "named-child-bead"}, "polylogue-hook-authority-conflict-proof": {"kind": "named-child-bead"}, "polylogue-excluded-cursor-live-proof": {"kind": "named-child-bead"}, - "polylogue-chatgpt-content-live-proof": {"kind": "named-child-bead"} + "polylogue-chatgpt-content-live-proof": {"kind": "named-child-bead"}, + "polylogue-xselt": {"kind": "named-child-bead"} }, "rows": [ {"bead_id": "polylogue-0qfy", "bead_status": "closed", "incident": {"incident_id": "incident-0qfy", "bead_id": "polylogue-0qfy", "forcing_class": "content-fidelity"}, "route": {"kind": "registry", "entrypoint": "reindex-campaign"}, "schedule": {"phase": "preflight", "order": 1}, "expected_snapshot": {"snapshot_id": "reindex-baseline-2026-08-03", "state": "blocking"}, "registry_checks": ["content-fidelity", "campaign-coverage"], "red_mutation": {"fixture_id": "vintage-reorder", "mutation_id": "mutation-0qfy"}, "receipts": [], "residual_successor": {"bead_id": "polylogue-claude-vintage-live-proof", "kind": "live-proof"}}, @@ -105,7 +106,7 @@ {"bead_id": "polylogue-rrxe4", "bead_status": "open", "incident": {"incident_id": "incident-rrxe4", "bead_id": "polylogue-rrxe4", "forcing_class": "properties"}, "route": {"kind": "campaign", "entrypoint": "reindex-campaign"}, "schedule": {"phase": "preflight", "order": 31}, "expected_snapshot": {"snapshot_id": "derived-model-candidate", "state": "blocking"}, "registry_checks": ["convergence-properties"], "red_mutation": {"fixture_id": "campaign-corpus", "mutation_id": "mutation-rrxe4"}, "receipts": [], "residual_successor": null}, {"bead_id": "polylogue-rrxe4.1", "bead_status": "open", "incident": {"incident_id": "incident-rrxe4-1", "bead_id": "polylogue-rrxe4.1", "forcing_class": "properties"}, "route": {"kind": "campaign", "entrypoint": "reindex-campaign"}, "schedule": {"phase": "preflight", "order": 32}, "expected_snapshot": {"snapshot_id": "derived-model-candidate", "state": "blocking"}, "registry_checks": ["convergence-properties"], "red_mutation": {"fixture_id": "campaign-corpus", "mutation_id": "mutation-rrxe4-1"}, "receipts": [], "residual_successor": null}, {"bead_id": "polylogue-s8s54", "bead_status": "open", "incident": {"incident_id": "incident-s8s54", "bead_id": "polylogue-s8s54", "forcing_class": "origin-repair"}, "route": {"kind": "registry", "entrypoint": "origin-capability"}, "schedule": {"phase": "preflight", "order": 33}, "expected_snapshot": {"snapshot_id": "live-preflight-2026-08-04", "state": "blocking"}, "registry_checks": ["origin-repair"], "red_mutation": {"fixture_id": "origin-matrix", "mutation_id": "mutation-s8s54"}, "receipts": [], "residual_successor": null}, - {"bead_id": "polylogue-slshy", "bead_status": "in_progress", "incident": {"incident_id": "incident-slshy", "bead_id": "polylogue-slshy", "forcing_class": "parser-stamps"}, "route": {"kind": "registry", "entrypoint": "reindex-final-proof"}, "schedule": {"phase": "preflight", "order": 34}, "expected_snapshot": {"snapshot_id": "post-reindex-acceptance", "state": "blocking"}, "registry_checks": ["parser-stamps"], "red_mutation": {"fixture_id": "vintage-reorder", "mutation_id": "mutation-slshy"}, "receipts": [], "residual_successor": null}, + {"bead_id": "polylogue-slshy", "bead_status": "closed", "incident": {"incident_id": "incident-slshy", "bead_id": "polylogue-slshy", "forcing_class": "parser-stamps"}, "route": {"kind": "registry", "entrypoint": "reindex-final-proof"}, "schedule": {"phase": "preflight", "order": 34}, "expected_snapshot": {"snapshot_id": "post-reindex-acceptance", "state": "blocking"}, "registry_checks": ["parser-stamps"], "red_mutation": {"fixture_id": "vintage-reorder", "mutation_id": "mutation-slshy"}, "receipts": [], "residual_successor": {"bead_id": "polylogue-xselt", "kind": "named-child-bead"}}, {"bead_id": "polylogue-sp72", "bead_status": "open", "incident": {"incident_id": "incident-sp72", "bead_id": "polylogue-sp72", "forcing_class": "revision-lineage"}, "route": {"kind": "registry", "entrypoint": "raw-authority-ledger"}, "schedule": {"phase": "preflight", "order": 35}, "expected_snapshot": {"snapshot_id": "live-preflight-2026-08-04", "state": "blocking"}, "registry_checks": ["revision-lineage"], "red_mutation": {"fixture_id": "drive-revision", "mutation_id": "mutation-sp72"}, "receipts": [], "residual_successor": null}, {"bead_id": "polylogue-t0m73", "bead_status": "open", "incident": {"incident_id": "incident-t0m73", "bead_id": "polylogue-t0m73", "forcing_class": "archive-invariants"}, "route": {"kind": "registry", "entrypoint": "archive-verification"}, "schedule": {"phase": "promotion", "order": 36}, "expected_snapshot": {"snapshot_id": "post-reindex-acceptance", "state": "blocking"}, "registry_checks": ["archive-invariants"], "red_mutation": {"fixture_id": "derived-model", "mutation_id": "mutation-t0m73"}, "receipts": [], "residual_successor": null}, {"bead_id": "polylogue-uqwd", "bead_status": "in_progress", "incident": {"incident_id": "incident-uqwd", "bead_id": "polylogue-uqwd", "forcing_class": "lifecycle-anchor"}, "route": {"kind": "registry", "entrypoint": "reindex-campaign"}, "schedule": {"phase": "preflight", "order": 37}, "expected_snapshot": {"snapshot_id": "reindex-baseline-2026-08-03", "state": "blocking"}, "registry_checks": ["lifecycle-anchor"], "red_mutation": {"fixture_id": "lifecycle-anchor-drift", "mutation_id": "mutation-uqwd"}, "receipts": [], "residual_successor": null}, diff --git a/tests/fixtures/reindex_incident_coverage/campaign_graph.json b/tests/fixtures/reindex_incident_coverage/campaign_graph.json index 613ca6efa5..faf49dc804 100644 --- a/tests/fixtures/reindex_incident_coverage/campaign_graph.json +++ b/tests/fixtures/reindex_incident_coverage/campaign_graph.json @@ -36,7 +36,7 @@ {"bead_id": "polylogue-rrxe4", "status": "open", "kind": "verification", "child_bead_ids": []}, {"bead_id": "polylogue-rrxe4.1", "status": "open", "kind": "verification", "child_bead_ids": []}, {"bead_id": "polylogue-s8s54", "status": "open", "kind": "implementation", "child_bead_ids": []}, - {"bead_id": "polylogue-slshy", "status": "in_progress", "kind": "implementation", "child_bead_ids": []}, + {"bead_id": "polylogue-slshy", "status": "closed", "kind": "implementation", "child_bead_ids": ["polylogue-xselt"]}, {"bead_id": "polylogue-sp72", "status": "open", "kind": "implementation", "child_bead_ids": []}, {"bead_id": "polylogue-t0m73", "status": "open", "kind": "verification", "child_bead_ids": []}, {"bead_id": "polylogue-uqwd", "status": "in_progress", "kind": "implementation", "child_bead_ids": []}, @@ -52,6 +52,7 @@ "polylogue-byte-supersession-live-proof", "polylogue-hook-authority-conflict-proof", "polylogue-excluded-cursor-live-proof", - "polylogue-chatgpt-content-live-proof" + "polylogue-chatgpt-content-live-proof", + "polylogue-xselt" ] } diff --git a/tests/unit/devtools/test_incident_coverage_ledger.py b/tests/unit/devtools/test_incident_coverage_ledger.py index 06fb53c05e..455fbf3f26 100644 --- a/tests/unit/devtools/test_incident_coverage_ledger.py +++ b/tests/unit/devtools/test_incident_coverage_ledger.py @@ -43,6 +43,7 @@ def test_real_campaign_graph_resolves_all_current_forcing_dependencies() -> None "polylogue-6753s", "polylogue-foee", "polylogue-ix5r", + "polylogue-slshy", "polylogue-xofj", } assert LEDGER_PATH.is_file() From 9cc22f2cb7faf9727bd9c94d7daa3041e1b3cc56 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 9 Aug 2026 01:10:07 +0200 Subject: [PATCH 05/19] fix: persist message owners for user assertions Problem: message marks and annotations lost exact session ownership when rebuildable index rows disappeared, especially for opaque native IDs containing :n: or :p:. What changed: carry the resolved owner through the async facade, mutation actuators, ArchiveStore, and assertion writers via the existing scope_ref carrier. Reads prefer durable owner metadata, then exact indexed ownership, and bounded legacy rows remain unresolved. Compatibility/migration: no user schema change. Existing message assertions without durable scope remain unresolved after index loss and require no unsafe delimiter migration. --- polylogue/api/archive.py | 24 ++++-- polylogue/operations/mutation_actuators.py | 32 +++++++- .../storage/sqlite/archive_tiers/archive.py | 79 ++++++++++++++----- .../sqlite/archive_tiers/user_write.py | 11 ++- .../storage/test_marks_identity_preserving.py | 30 ++++++- .../storage/test_user_state_target_kinds.py | 2 +- 6 files changed, 145 insertions(+), 33 deletions(-) diff --git a/polylogue/api/archive.py b/polylogue/api/archive.py index 50601bf52b..7cc6986481 100644 --- a/polylogue/api/archive.py +++ b/polylogue/api/archive.py @@ -6847,6 +6847,7 @@ async def add_mark( target_type=str(target["target_type"]), target_id=str(target["target_id"]), mark_type=mark_type, + owner_session_id=str(target["session_id"]) if target.get("session_id") else None, ), capability="archive.add_mark", ) @@ -6883,6 +6884,7 @@ async def remove_mark( target_type=str(target["target_type"]), target_id=str(target["target_id"]), mark_type=mark_type, + owner_session_id=str(target["session_id"]) if target.get("session_id") else None, ), capability="archive.remove_mark", ) @@ -6900,23 +6902,32 @@ async def list_marks( """List marks, optionally filtered by type, target, session, or message.""" resolved_target_type = target_type resolved_target_id = target_id + scope_session_id: str | None = None if message_id is not None: resolved_target_type = TARGET_MESSAGE resolved_target_id = message_id elif session_id is not None and target_id is None: try: - resolved_target_id = await self._resolve_user_state_session_id(session_id) - resolved_target_type = TARGET_SESSION + scope_session_id = await self._resolve_user_state_session_id(session_id) except SessionNotFoundError: - return [] + # A durable user assertion can outlive the rebuildable + # session row. Keep the caller's canonical token so the + # archive read can use its durable message owner scope. + scope_session_id = session_id return await run_archive_read( _active_archive_root(self.config), operation="user_state.marks.list", - arguments={"mark_type": mark_type, "target_type": resolved_target_type, "target_id": resolved_target_id}, + arguments={ + "mark_type": mark_type, + "target_type": resolved_target_type, + "target_id": resolved_target_id, + "session_id": scope_session_id, + }, work=lambda archive: archive.list_marks( mark_type=mark_type, target_type=resolved_target_type, target_id=resolved_target_id, + session_id=scope_session_id, ), projection="marks", stable_order="created_at,target_id", @@ -6960,6 +6971,7 @@ async def save_annotation( target_type=str(target["target_type"]), target_id=str(target["target_id"]), note_text=note_text, + owner_session_id=str(target["session_id"]) if target.get("session_id") else None, ), capability="archive.save_annotation", ) @@ -6994,7 +7006,9 @@ async def list_annotations( try: scope_session_id = await self._resolve_user_state_session_id(session_id) except SessionNotFoundError: - return [] + # See list_marks: the user tier remains authoritative after + # the rebuildable index row is gone. + scope_session_id = session_id return await run_archive_read( _active_archive_root(self.config), operation="user_state.annotations.list", diff --git a/polylogue/operations/mutation_actuators.py b/polylogue/operations/mutation_actuators.py index 3f3d0f64cc..424def4726 100644 --- a/polylogue/operations/mutation_actuators.py +++ b/polylogue/operations/mutation_actuators.py @@ -665,6 +665,7 @@ class MarkArgs: target_type: str target_id: str mark_type: str + owner_session_id: str | None = None @dataclass(frozen=True, slots=True) @@ -688,11 +689,21 @@ def prepare(self, args: MarkArgs) -> MutationPlan: target_refs=(f"{args.target_type}:{args.target_id}",), affected_tiers=("user",), reversible=True, - context={"target_type": args.target_type, "target_id": args.target_id, "mark_type": args.mark_type}, + context={ + "target_type": args.target_type, + "target_id": args.target_id, + "mark_type": args.mark_type, + "owner_session_id": args.owner_session_id, + }, ) def apply(self, plan: MutationPlan, args: MarkArgs) -> MutationReceipt: - added = args.archive.add_mark(args.target_type, args.target_id, args.mark_type) + added = args.archive.add_mark( + args.target_type, + args.target_id, + args.mark_type, + owner_session_id=args.owner_session_id, + ) status: MutationTargetStatus = "applied" if added else "already_satisfied" return MutationReceipt( operation=self.operation, @@ -726,7 +737,12 @@ def prepare(self, args: MarkArgs) -> MutationPlan: target_refs=(f"{args.target_type}:{args.target_id}",), affected_tiers=("user",), reversible=True, - context={"target_type": args.target_type, "target_id": args.target_id, "mark_type": args.mark_type}, + context={ + "target_type": args.target_type, + "target_id": args.target_id, + "mark_type": args.mark_type, + "owner_session_id": args.owner_session_id, + }, ) def apply(self, plan: MutationPlan, args: MarkArgs) -> MutationReceipt: @@ -975,6 +991,7 @@ class AnnotationSaveArgs: target_type: str target_id: str note_text: str + owner_session_id: str | None = None @dataclass(frozen=True, slots=True) @@ -1007,6 +1024,7 @@ def prepare(self, args: AnnotationSaveArgs) -> MutationPlan: "target_type": args.target_type, "target_id": args.target_id, "note_text": args.note_text, + "owner_session_id": args.owner_session_id, }, ) @@ -1015,7 +1033,13 @@ def apply(self, plan: MutationPlan, args: AnnotationSaveArgs) -> MutationReceipt target_type = str(plan.context["target_type"]) target_id = str(plan.context["target_id"]) note_text = str(plan.context["note_text"]) - created = args.archive.save_annotation(annotation_id, target_type, target_id, note_text) + created = args.archive.save_annotation( + annotation_id, + target_type, + target_id, + note_text, + owner_session_id=cast(str | None, plan.context.get("owner_session_id")), + ) return MutationReceipt( operation=self.operation, plan_hash=plan.plan_hash, diff --git a/polylogue/storage/sqlite/archive_tiers/archive.py b/polylogue/storage/sqlite/archive_tiers/archive.py index fb0dd98bfe..a381c8f156 100644 --- a/polylogue/storage/sqlite/archive_tiers/archive.py +++ b/polylogue/storage/sqlite/archive_tiers/archive.py @@ -5441,14 +5441,29 @@ def delete_user_metadata(self, session_id: str, key: str) -> int: finally: user_conn.close() - def add_mark(self, target_type: str, target_id: str, mark_type: str) -> bool: + def add_mark( + self, + target_type: str, + target_id: str, + mark_type: str, + *, + owner_session_id: str | None = None, + ) -> bool: """Add one user mark to archive user.db.""" + if target_type == "message" and not owner_session_id: + raise ValueError("message marks require a resolved owner_session_id") user_conn = self._open_user_write_connection(initialize=True) try: assertion = read_assertion_envelope(user_conn, assertion_id_for_mark(target_type, target_id, mark_type)) exists = assertion is not None and assertion.status != "deleted" with user_conn: - upsert_mark(user_conn, target_type, target_id, mark_type) + upsert_mark( + user_conn, + target_type, + target_id, + mark_type, + owner_session_id=owner_session_id, + ) return not exists finally: user_conn.close() @@ -5474,6 +5489,7 @@ def list_marks( mark_type: str | None = None, target_type: str | None = None, target_id: str | None = None, + session_id: str | None = None, ) -> list[dict[str, str]]: """List user marks from archive user.db.""" if not self.user_db_path.exists(): @@ -5486,21 +5502,25 @@ def list_marks( out: list[dict[str, str]] = [] for assertion in assertions: found_target_type, found_target_id = _split_user_target_ref(assertion.target_ref) + owner_session_id = _user_mark_session_id( + found_target_type, + found_target_id, + durable_scope_ref=assertion.scope_ref, + index_conn=self._conn, + ) if mark_type and assertion.key != mark_type: continue if target_type and found_target_type != target_type: continue if target_id and found_target_id != target_id: continue + if session_id and target_id is None and owner_session_id != session_id: + continue out.append( { "target_type": found_target_type, "target_id": found_target_id, - "session_id": _user_mark_session_id( - found_target_type, - found_target_id, - index_conn=self._conn, - ), + "session_id": owner_session_id, "message_id": found_target_id if found_target_type == "message" else "", "mark_type": str(assertion.key or ""), "created_at": str(assertion.created_at_ms), @@ -5508,8 +5528,18 @@ def list_marks( ) return out - def save_annotation(self, annotation_id: str, target_type: str, target_id: str, note_text: str) -> bool: + def save_annotation( + self, + annotation_id: str, + target_type: str, + target_id: str, + note_text: str, + *, + owner_session_id: str | None = None, + ) -> bool: """Create or update one annotation in archive user.db.""" + if target_type == "message" and not owner_session_id: + raise ValueError("message annotations require a resolved owner_session_id") user_conn = self._open_user_write_connection(initialize=True) try: assertion = read_assertion_envelope(user_conn, assertion_id_for_annotation(annotation_id)) @@ -5520,6 +5550,7 @@ def save_annotation(self, annotation_id: str, target_type: str, target_id: str, target_type, target_id, note_text, + owner_session_id=owner_session_id, annotation_id=annotation_id, ) return not exists @@ -5639,10 +5670,9 @@ def list_annotations( When ``session_id`` is supplied (and no explicit target filter), the result includes both the session-target annotation and every - message-target annotation whose native message id is prefixed by the - session id (``session_id:message_native_id``). This mirrors the read model - contract where annotations on messages belonging to a session were - listed under that session. + message-target annotation whose durable owner scope or exact indexed + message ownership matches that session. Legacy message assertions + without either authority are intentionally excluded. """ if not self.user_db_path.exists(): return [] @@ -5658,9 +5688,14 @@ def list_annotations( if annotation_id and found_annotation_id != annotation_id: continue if session_id and target_id is None: - belongs_to_session = (found_target_type == "session" and found_target_id == session_id) or ( - found_target_type == "message" - and (found_target_id == session_id or found_target_id.startswith(f"{session_id}:")) + belongs_to_session = ( + _user_mark_session_id( + found_target_type, + found_target_id, + durable_scope_ref=assertion.scope_ref, + index_conn=self._conn, + ) + == session_id ) if not belongs_to_session: continue @@ -5676,6 +5711,7 @@ def list_annotations( "session_id": _user_mark_session_id( found_target_type, found_target_id, + durable_scope_ref=assertion.scope_ref, index_conn=self._conn, ), "message_id": found_target_id if found_target_type == "message" else "", @@ -9346,11 +9382,16 @@ def _user_mark_session_id( target_type: str, target_id: str, *, + durable_scope_ref: str | None = None, index_conn: sqlite3.Connection | None = None, ) -> str: if target_type == "session": return target_id if target_type == "message": + if durable_scope_ref is not None and durable_scope_ref.startswith("session:"): + durable_owner = durable_scope_ref[len("session:") :] + if durable_owner: + return durable_owner # Canonical message IDs are generated from the indexed message row, # whose session_id column is the authoritative owner. Provider-native # session IDs are opaque and may contain the ``:n:`` or ``:p:`` tokens, @@ -9362,10 +9403,10 @@ def _user_mark_session_id( ).fetchone() if row is not None: return str(row["session_id"]) - # A message assertion can outlive its indexed message. Retain the - # historical delimiter fallback for those legacy/unresolved IDs. - session_id, _sep, _message_native_id = target_id.rpartition(":") - return session_id + # Legacy message assertions may have neither durable scope metadata + # nor an indexed row. Their opaque ID cannot be decoded exactly, so + # expose no owner rather than manufacturing authority from a token. + return "" return "" diff --git a/polylogue/storage/sqlite/archive_tiers/user_write.py b/polylogue/storage/sqlite/archive_tiers/user_write.py index 95012df57d..ee13754f20 100644 --- a/polylogue/storage/sqlite/archive_tiers/user_write.py +++ b/polylogue/storage/sqlite/archive_tiers/user_write.py @@ -618,17 +618,24 @@ def upsert_mark( target_id: str, mark_type: str, *, + owner_session_id: str | None = None, label: str | None = None, metadata: dict[str, object] | None = None, now_ms: int | None = None, ) -> ArchiveMarkEnvelope: - """Insert-or-update one mark assertion with deterministic ``mark_id``.""" + """Insert-or-update one mark assertion with deterministic ``mark_id``. + + Message targets carry their resolved owning session in the durable + assertion scope. The indexed message row is rebuildable, so callers that + know the owner should pass it before that row can disappear. + """ timestamp = now_ms if now_ms is not None else _now_ms() mark_id = _deterministic_id("mark", target_type, target_id, mark_type) upsert_assertion( conn, assertion_id=assertion_id_for_mark(target_type, target_id, mark_type), target_ref=f"{target_type}:{target_id}", + scope_ref=f"session:{owner_session_id}" if target_type == "message" and owner_session_id else None, kind=AssertionKind.MARK, key=mark_type, value=metadata if metadata else None, @@ -718,6 +725,7 @@ def upsert_annotation( target_id: str, body: str, *, + owner_session_id: str | None = None, annotation_id: str | None = None, now_ms: int | None = None, ) -> ArchiveAnnotationEnvelope: @@ -741,6 +749,7 @@ def upsert_annotation( conn, assertion_id=assertion_id_for_annotation(resolved_id), target_ref=f"{target_type}:{target_id}", + scope_ref=f"session:{owner_session_id}" if target_type == "message" and owner_session_id else None, kind=AssertionKind.ANNOTATION, key=resolved_id, body_text=body, diff --git a/tests/unit/storage/test_marks_identity_preserving.py b/tests/unit/storage/test_marks_identity_preserving.py index b03f8290fe..5535a3d4ae 100644 --- a/tests/unit/storage/test_marks_identity_preserving.py +++ b/tests/unit/storage/test_marks_identity_preserving.py @@ -142,7 +142,7 @@ async def test_message_user_state_projects_owner_for_opaque_session_native_ids( workspace_env: dict[str, Path], opaque_marker: str, ) -> None: - """Message user state resolves ownership from indexed identity, not delimiters.""" + """Message user state keeps exact ownership after index rows disappear.""" db_path = db_setup(workspace_env) builder = ( SessionBuilder(db_path, f"opaque{opaque_marker}session") @@ -179,10 +179,34 @@ async def test_message_user_state_projects_owner_for_opaque_session_native_ids( marks = await poly.list_marks(mark_type="pin") annotations = await poly.list_annotations() + with sqlite3.connect(_user_db_path(workspace_env)) as conn: + durable_scopes = conn.execute( + """ + SELECT kind, scope_ref + FROM assertions + WHERE target_ref = ? + ORDER BY kind + """, + (f"message:{message_id}",), + ).fetchall() + assert durable_scopes == [ + ("annotation", f"session:{session_id}"), + ("mark", f"session:{session_id}"), + ] + + assert await poly.delete_session(session_id) is True + with sqlite3.connect(db_path) as conn: + assert conn.execute("SELECT 1 FROM sessions WHERE session_id = ?", (session_id,)).fetchone() is None + assert conn.execute("SELECT 1 FROM messages WHERE message_id = ?", (message_id,)).fetchone() is None + filtered_marks = await poly.list_marks(session_id=session_id, mark_type="pin") + filtered_annotations = await poly.list_annotations(session_id=session_id) + assert marks[0]["target_id"] == message_id assert marks[0]["session_id"] == session_id assert annotations[0]["target_id"] == message_id assert annotations[0]["session_id"] == session_id + assert filtered_marks == [marks[0]] + assert filtered_annotations == [annotations[0]] # --------------------------------------------------------------------------- @@ -257,7 +281,7 @@ async def test_message_target_marks_survive_reimport(workspace_env: dict[str, Pa ) builder.save() session_id = builder.native_session_id() - message_id = f"{session_id}:msg-id" + message_id = f"{session_id}:n:msg-id" async with Polylogue(db_path=db_path, archive_root=workspace_env["archive_root"]) as poly: assert ( @@ -298,7 +322,7 @@ async def test_message_target_mark_survives_when_message_disappears( ) builder.save() session_id = builder.native_session_id() - message_id = f"{session_id}:msg-id" + message_id = f"{session_id}:n:msg-id" async with Polylogue(db_path=db_path, archive_root=workspace_env["archive_root"]) as poly: assert ( diff --git a/tests/unit/storage/test_user_state_target_kinds.py b/tests/unit/storage/test_user_state_target_kinds.py index 608f8cb19a..33d6cb8764 100644 --- a/tests/unit/storage/test_user_state_target_kinds.py +++ b/tests/unit/storage/test_user_state_target_kinds.py @@ -256,7 +256,7 @@ async def test_recall_pack_resolves_insight_targets_and_degrades_explicitly( builder = SessionBuilder(db_path, "conv-pack").provider("claude-code").add_message(message_id="msg-1", text="Hi") builder.save() session_id = builder.native_session_id() - message_id = f"{session_id}:msg-1" + message_id = f"{session_id}:n:msg-1" _seed_session_profile(db_path, session_id) thread_id = _native_thread_id(db_path, session_id) From 9498a8e084fd45eb72fbf781605b2db1cdd78152 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 9 Aug 2026 02:18:41 +0200 Subject: [PATCH 06/19] fix(identity): harden idless authority and user state Problem: changing idless message identity without invalidating revision authority could reuse stale membership verdicts, while per-row owner lookups and the web shell's session-only mark state left related read paths inconsistent. What changed: advance the revision-membership fingerprint, include structured blocks in idless anchors, batch indexed owner resolution across marks and annotations, filter message marks from session mark state, and strengthen the Drive and authority regressions. Compatibility/migration: this changes no durable data and performs no production reindex. Existing v2 authority receipts are treated as superseded and are eligible for recomputation under v3. Review: addressed Codex findings 3742013546, 3742013549, 3742013551, 3742013552, and 3742013555. The older coverage finding 3741792047 remains satisfied by the tracker and incident-ledger correction. --- polylogue/daemon/web_shell.py | 1 + polylogue/pipeline/ids.py | 10 ++- polylogue/storage/raw_authority.py | 4 +- .../storage/sqlite/archive_tiers/archive.py | 83 +++++++++++++------ .../unit/daemon/test_daemon_http_contracts.py | 8 ++ tests/unit/sources/test_parsers_drive.py | 2 +- .../storage/test_marks_identity_preserving.py | 43 ++++++++++ .../unit/storage/test_raw_authority_ledger.py | 7 +- 8 files changed, 124 insertions(+), 34 deletions(-) diff --git a/polylogue/daemon/web_shell.py b/polylogue/daemon/web_shell.py index 2af5de2f64..5eb8d3d121 100644 --- a/polylogue/daemon/web_shell.py +++ b/polylogue/daemon/web_shell.py @@ -977,6 +977,7 @@ var marks = await fetchJSON(route, {timeoutMs: 5000}); state.marks = {}; (marks.items || []).forEach(function(m) { + if (m.target_type !== 'session') return; setMarkLocal(m.session_id, m.mark_type, true); }); route = '/api/user/annotations'; diff --git a/polylogue/pipeline/ids.py b/polylogue/pipeline/ids.py index 200a5dc23e..ac87e56c68 100644 --- a/polylogue/pipeline/ids.py +++ b/polylogue/pipeline/ids.py @@ -224,8 +224,14 @@ def _is_redundant_text_only_block(message: ParsedMessage) -> bool: def _message_hash_payload(message: ParsedMessage, message_id: str) -> dict[str, JSONValue]: """Build the hash-stable payload for a single message.""" + payload: dict[str, JSONValue] = {"id": message_id} + payload.update(_message_comparison_payload(message)) + return payload + + +def _message_comparison_payload(message: ParsedMessage) -> dict[str, JSONValue]: + """Build the content payload that distinguishes an idless message.""" payload: dict[str, JSONValue] = { - "id": message_id, "role": str(message.role), "text": _normalize_for_hash(message.text), "timestamp": _normalize_for_hash(message.timestamp), @@ -268,7 +274,7 @@ class the attachment identity fix (polylogue-hith/-d8al) removed for a """ if message.provider_message_id: return message.provider_message_id - return f"{_CONTENT_ANCHOR_PREFIX}:{hash_payload({'role': str(message.role), 'timestamp': _normalize_for_hash(message.timestamp), 'text': _normalize_for_hash(message.text)})}" + return f"{_CONTENT_ANCHOR_PREFIX}:{hash_payload(_message_comparison_payload(message))}" def message_identity_hash(*, id: str) -> bytes: diff --git a/polylogue/storage/raw_authority.py b/polylogue/storage/raw_authority.py index 6c7c247d3f..f6ed824f6d 100644 --- a/polylogue/storage/raw_authority.py +++ b/polylogue/storage/raw_authority.py @@ -26,7 +26,7 @@ from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier from polylogue.storage.sqlite.migration_runner import validate_migration_backup_manifest -RAW_AUTHORITY_PARSER_FINGERPRINT = "revision-membership-v2" +RAW_AUTHORITY_PARSER_FINGERPRINT = "revision-membership-v3" #: Fingerprints previously stamped by ``RAW_AUTHORITY_PARSER_FINGERPRINT`` #: whose classification semantics are known to have been superseded by a @@ -41,7 +41,7 @@ #: *quiescence* gate (``uncensused_historical_revision_raw_ids``) accepts any #: known fingerprint (current or superseded) so a bump does not force a full #: archive re-census -- see that function's docstring. -SUPERSEDED_MEMBERSHIP_FINGERPRINTS = frozenset({"revision-membership-v1"}) +SUPERSEDED_MEMBERSHIP_FINGERPRINTS = frozenset({"revision-membership-v1", "revision-membership-v2"}) RAW_AUTHORITY_CENSUS_QUERY_PREFIX = "polylogue://raw-authority-census/" RAW_AUTHORITY_DETAIL_QUERY_PREFIX = "polylogue://raw-authority-detail/" RAW_AUTHORITY_DETAIL_CHUNK_CHARS = 16_384 diff --git a/polylogue/storage/sqlite/archive_tiers/archive.py b/polylogue/storage/sqlite/archive_tiers/archive.py index a381c8f156..57e958a108 100644 --- a/polylogue/storage/sqlite/archive_tiers/archive.py +++ b/polylogue/storage/sqlite/archive_tiers/archive.py @@ -5499,21 +5499,20 @@ def list_marks( assertions = list_assertions_by_kind(user_conn, AssertionKind.MARK) finally: user_conn.close() - out: list[dict[str, str]] = [] + selected: list[tuple[ArchiveAssertionEnvelope, str, str]] = [] for assertion in assertions: found_target_type, found_target_id = _split_user_target_ref(assertion.target_ref) - owner_session_id = _user_mark_session_id( - found_target_type, - found_target_id, - durable_scope_ref=assertion.scope_ref, - index_conn=self._conn, - ) if mark_type and assertion.key != mark_type: continue if target_type and found_target_type != target_type: continue if target_id and found_target_id != target_id: continue + selected.append((assertion, found_target_type, found_target_id)) + owners = _user_state_session_ids((item[0] for item in selected), index_conn=self._conn) + out: list[dict[str, str]] = [] + for assertion, found_target_type, found_target_id in selected: + owner_session_id = owners[assertion.assertion_id] if session_id and target_id is None and owner_session_id != session_id: continue out.append( @@ -5681,39 +5680,29 @@ def list_annotations( assertions = list_assertions_by_kind(user_conn, AssertionKind.ANNOTATION) finally: user_conn.close() - out: list[dict[str, str]] = [] + selected: list[tuple[ArchiveAssertionEnvelope, str, str]] = [] for assertion in assertions: found_annotation_id = str(assertion.key or "") found_target_type, found_target_id = _split_user_target_ref(assertion.target_ref) if annotation_id and found_annotation_id != annotation_id: continue - if session_id and target_id is None: - belongs_to_session = ( - _user_mark_session_id( - found_target_type, - found_target_id, - durable_scope_ref=assertion.scope_ref, - index_conn=self._conn, - ) - == session_id - ) - if not belongs_to_session: - continue if target_type and found_target_type != target_type: continue if target_id and found_target_id != target_id: continue + selected.append((assertion, found_target_type, found_target_id)) + owners = _user_state_session_ids((item[0] for item in selected), index_conn=self._conn) + out: list[dict[str, str]] = [] + for assertion, found_target_type, found_target_id in selected: + owner_session_id = owners[assertion.assertion_id] + if session_id and target_id is None and owner_session_id != session_id: + continue out.append( { - "annotation_id": found_annotation_id, + "annotation_id": str(assertion.key or ""), "target_type": found_target_type, "target_id": found_target_id, - "session_id": _user_mark_session_id( - found_target_type, - found_target_id, - durable_scope_ref=assertion.scope_ref, - index_conn=self._conn, - ), + "session_id": owner_session_id, "message_id": found_target_id if found_target_type == "message" else "", "note_text": assertion.body_text or "", "created_at": str(assertion.created_at_ms), @@ -9363,6 +9352,46 @@ def _split_user_target_ref(target_ref: str) -> tuple[str, str]: return target_type, target_id +def _user_state_session_ids( + assertions: Iterable[ArchiveAssertionEnvelope], + *, + index_conn: sqlite3.Connection | None = None, +) -> dict[str, str]: + """Resolve assertion owners with one bounded indexed-message lookup batch.""" + owners: dict[str, str] = {} + unresolved: list[tuple[str, str]] = [] + for assertion in assertions: + target_type, target_id = _split_user_target_ref(assertion.target_ref) + if target_type == "session": + owners[assertion.assertion_id] = target_id + continue + if target_type != "message": + owners[assertion.assertion_id] = "" + continue + if assertion.scope_ref is not None and assertion.scope_ref.startswith("session:"): + durable_owner = assertion.scope_ref[len("session:") :] + if durable_owner: + owners[assertion.assertion_id] = durable_owner + continue + unresolved.append((assertion.assertion_id, target_id)) + + indexed_owners: dict[str, str] = {} + if index_conn is not None: + for offset in range(0, len(unresolved), 500): + batch = sorted({target_id for _assertion_id, target_id in unresolved[offset : offset + 500]}) + if not batch: + continue + placeholders = ",".join("?" for _ in batch) + rows = index_conn.execute( + f"SELECT message_id, session_id FROM messages WHERE message_id IN ({placeholders})", + batch, + ).fetchall() + indexed_owners.update({str(row[0]): str(row[1]) for row in rows}) + for assertion_id, target_id in unresolved: + owners[assertion_id] = indexed_owners.get(target_id, "") + return owners + + def _id_from_target_ref(target_ref: str, prefix: str) -> str: return target_ref[len(prefix) :] if target_ref.startswith(prefix) else target_ref diff --git a/tests/unit/daemon/test_daemon_http_contracts.py b/tests/unit/daemon/test_daemon_http_contracts.py index 84a14eae43..1bc107e0ee 100644 --- a/tests/unit/daemon/test_daemon_http_contracts.py +++ b/tests/unit/daemon/test_daemon_http_contracts.py @@ -878,6 +878,14 @@ def test_web_shell_usage_cost_preserves_unknown_and_known_subtotal(self) -> None assert "if (value === null || value === undefined || value === '') return 'unknown';" in WEB_SHELL_HTML assert "var n = Number(value || 0);" not in WEB_SHELL_HTML + def test_web_shell_excludes_message_marks_from_session_mark_state(self) -> None: + from polylogue.daemon.web_shell import WEB_SHELL_HTML + + anchor = "(marks.items || []).forEach(function(m) {" + idx = WEB_SHELL_HTML.index(anchor) + following = WEB_SHELL_HTML[idx : idx + 220] + assert "if (m.target_type !== 'session') return;" in following + def test_load_status_success_path_clears_the_status_route_notice(self) -> None: """Dogfood regression (2026-07-08): loadStatus()'s success path set state.routeStates.status to 'ready' but never called renderFacets(), diff --git a/tests/unit/sources/test_parsers_drive.py b/tests/unit/sources/test_parsers_drive.py index 89ab2857a8..c659d75f3b 100644 --- a/tests/unit/sources/test_parsers_drive.py +++ b/tests/unit/sources/test_parsers_drive.py @@ -295,7 +295,7 @@ def test_idless_drive_attachment_owner_changes_hash_and_revision_identity() -> N # Same-timestamp turns force the owner anchor to include message content; # a timestamp-only fallback would make moving the attachment invisible. first: JSONDocument = {"role": "user", "text": "first", "createTime": "2026-01-01T00:00:00Z"} - second: JSONDocument = {"role": "model", "text": "second", "createTime": "2026-01-01T00:00:00Z"} + second: JSONDocument = {"role": "user", "text": "second", "createTime": "2026-01-01T00:00:00Z"} attachment: JSONDocument = {"id": "drive-doc", "name": "note.txt", "mimeType": "text/plain"} first_owner = parse_chunked_prompt( "gemini", diff --git a/tests/unit/storage/test_marks_identity_preserving.py b/tests/unit/storage/test_marks_identity_preserving.py index 5535a3d4ae..1abdab41ad 100644 --- a/tests/unit/storage/test_marks_identity_preserving.py +++ b/tests/unit/storage/test_marks_identity_preserving.py @@ -21,6 +21,7 @@ from polylogue.api import Polylogue from polylogue.core.user_state_targets import identity_key +from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore from tests.infra.storage_records import SessionBuilder, db_setup @@ -306,6 +307,48 @@ async def test_message_target_marks_survive_reimport(workspace_env: dict[str, Pa assert target_id == message_id +@pytest.mark.asyncio +async def test_legacy_message_owner_reads_batch_index_lookup(workspace_env: dict[str, Path]) -> None: + """Legacy message marks resolve through one indexed-message batch.""" + db_path = db_setup(workspace_env) + builder = SessionBuilder(db_path, "batch-owner") + builder.provider("claude-code") + for index in range(1, 4): + builder.add_message(message_id=f"msg-{index}", text=f"message {index}") + builder.save() + session_id = builder.native_session_id() + + async with Polylogue(db_path=db_path, archive_root=workspace_env["archive_root"]) as poly: + for index in range(1, 4): + assert ( + await poly.add_mark( + session_id, + "pin", + target_type="message", + message_id=f"{session_id}:n:msg-{index}", + ) + is True + ) + + with sqlite3.connect(_user_db_path(workspace_env)) as conn: + conn.execute("UPDATE assertions SET scope_ref = NULL WHERE kind = 'mark'") + conn.commit() + + with ArchiveStore.open_existing(workspace_env["archive_root"]) as archive: + statements: list[str] = [] + archive._conn.set_trace_callback(statements.append) + try: + rows = archive.list_marks(mark_type="pin") + finally: + archive._conn.set_trace_callback(None) + + message_lookup_statements = [ + statement for statement in statements if "FROM messages WHERE message_id IN" in statement + ] + assert len(message_lookup_statements) == 1 + assert {row["session_id"] for row in rows} == {session_id} + + @pytest.mark.asyncio async def test_message_target_mark_survives_when_message_disappears( workspace_env: dict[str, Path], diff --git a/tests/unit/storage/test_raw_authority_ledger.py b/tests/unit/storage/test_raw_authority_ledger.py index e2b9aceae5..945c57ab9d 100644 --- a/tests/unit/storage/test_raw_authority_ledger.py +++ b/tests/unit/storage/test_raw_authority_ledger.py @@ -1322,7 +1322,10 @@ def test_ambiguous_verdict_under_current_fingerprint_stays_terminal(tmp_path: Pa assert "ambiguous" in outcome.reason.lower() -def test_ambiguous_verdict_under_superseded_fingerprint_is_replayable(tmp_path: Path) -> None: +@pytest.mark.parametrize("superseded_fingerprint", ["revision-membership-v1", "revision-membership-v2"]) +def test_ambiguous_verdict_under_superseded_fingerprint_is_replayable( + tmp_path: Path, superseded_fingerprint: str +) -> None: """polylogue-9dxn: an 'ambiguous' decision recorded under a fingerprint listed in SUPERSEDED_MEMBERSHIP_FINGERPRINTS is stale -- a corrected classifier deserves a chance to re-derive it, so it must not be @@ -1339,7 +1342,7 @@ def test_ambiguous_verdict_under_superseded_fingerprint_is_replayable(tmp_path: plan as TERMINAL. """ initialize_active_archive_root(tmp_path) - superseded_fingerprint = next(iter(raw_authority_mod.SUPERSEDED_MEMBERSHIP_FINGERPRINTS)) + assert superseded_fingerprint in raw_authority_mod.SUPERSEDED_MEMBERSHIP_FINGERPRINTS _raw_id, outcome = _seed_ambiguous_membership_component( tmp_path, native_id="superseded-ambiguous", parser_fingerprint=superseded_fingerprint ) From 0b22e92e7704757ddb8e5dc35c26d7b52a8c5c6d Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 9 Aug 2026 02:40:38 +0200 Subject: [PATCH 07/19] fix(identity): finish owner read review repairs Problem: the exact-head review identified five fixture expectations using the default message position and found that the old per-row owner helper remained beside the batched read path. The structured slshy disposition also needed an explicit regression so its partial state and named residual scope stay aligned with the campaign graph. What changed: pass canonical positions 1, 1, 2, 2, and 3 in the affected query fixtures, remove the unreferenced single-message owner lookup, exercise one indexed lookup batch for both legacy marks and annotations, and pin slshy's in-progress disposition with both named child beads. Compatibility/migration: no production data or tracker records changed. Legacy unscoped assertions remain intentionally deferred to polylogue-message-owner-scope-backfill. Verification: POLYLOGUE_PYTEST_WORKERS=1 devtools test tests/unit/storage/test_marks_identity_preserving.py tests/unit/devtools/test_incident_coverage_ledger.py tests/unit/cli/test_query_expression.py (456 passed, 1 skipped). POLYLOGUE_PYTEST_WORKERS=1 devtools verify --quick (24 steps passed). --- .../storage/sqlite/archive_tiers/archive.py | 32 ------------------- tests/unit/cli/test_query_expression.py | 12 ++++--- .../devtools/test_incident_coverage_ledger.py | 20 ++++++++++++ .../storage/test_marks_identity_preserving.py | 18 +++++++++-- 4 files changed, 42 insertions(+), 40 deletions(-) diff --git a/polylogue/storage/sqlite/archive_tiers/archive.py b/polylogue/storage/sqlite/archive_tiers/archive.py index 57e958a108..0c1c1a2757 100644 --- a/polylogue/storage/sqlite/archive_tiers/archive.py +++ b/polylogue/storage/sqlite/archive_tiers/archive.py @@ -9407,38 +9407,6 @@ def _active_assertion_by_kind_key( return None -def _user_mark_session_id( - target_type: str, - target_id: str, - *, - durable_scope_ref: str | None = None, - index_conn: sqlite3.Connection | None = None, -) -> str: - if target_type == "session": - return target_id - if target_type == "message": - if durable_scope_ref is not None and durable_scope_ref.startswith("session:"): - durable_owner = durable_scope_ref[len("session:") :] - if durable_owner: - return durable_owner - # Canonical message IDs are generated from the indexed message row, - # whose session_id column is the authoritative owner. Provider-native - # session IDs are opaque and may contain the ``:n:`` or ``:p:`` tokens, - # so those tokens cannot safely delimit the owning session. - if index_conn is not None: - row = index_conn.execute( - "SELECT session_id FROM messages WHERE message_id = ?", - (target_id,), - ).fetchone() - if row is not None: - return str(row["session_id"]) - # Legacy message assertions may have neither durable scope metadata - # nor an indexed row. Their opaque ID cannot be decoded exactly, so - # expose no owner rather than manufacturing authority from a token. - return "" - return "" - - def _learning_correction_from_archive_row(row: sqlite3.Row | tuple[object, ...]) -> LearningCorrection: session_id = str(row[0]) kind = parse_correction_kind(str(row[1])) diff --git a/tests/unit/cli/test_query_expression.py b/tests/unit/cli/test_query_expression.py index b12f45296c..ecd1178bd2 100644 --- a/tests/unit/cli/test_query_expression.py +++ b/tests/unit/cli/test_query_expression.py @@ -2276,7 +2276,7 @@ def test_session_to_message_pipeline_fts_stage_executes_against_archive( assert [(row.session_id, row.message_id, row.text) for row in rows] == [ ( "chatgpt-export:ext-hit", - _mid("chatgpt-export:ext-hit", "m-selected"), + _mid("chatgpt-export:ext-hit", "m-selected", position=1), "selected terminal row", ) ] @@ -2321,7 +2321,7 @@ def test_session_to_message_pipeline_exists_stage_executes_against_archive( assert [(row.session_id, row.message_id, row.text) for row in rows] == [ ( "chatgpt-export:ext-hit", - _mid("chatgpt-export:ext-hit", "m-selected"), + _mid("chatgpt-export:ext-hit", "m-selected", position=1), "selected terminal row", ) ] @@ -2415,7 +2415,7 @@ def test_session_to_message_pipeline_sequence_stage_executes_against_archive( assert [(row.session_id, row.message_id, row.text) for row in rows] == [ ( "claude-code-session:ext-hit", - _mid("claude-code-session:ext-hit", "m-selected"), + _mid("claude-code-session:ext-hit", "m-selected", position=2), "selected terminal row", ) ] @@ -3548,7 +3548,7 @@ def test_terminal_action_source_filters_by_row_time(self, workspace_env: dict[st rows = archive.query_actions(source.predicate, limit=100) assert [(row.message_id, row.tool_path) for row in rows] == [ - (_mid("claude-code-session:ext-hit", "m-new"), "polylogue/archive/new.py") + (_mid("claude-code-session:ext-hit", "m-new", position=2), "polylogue/archive/new.py") ] assert rows[0].occurred_at_ms is not None @@ -3845,7 +3845,9 @@ def test_terminal_action_source_exposes_followup_class_rows_and_aggregates( assert row.is_error == 1 assert row.exit_code == 1 assert row.followup_class == "silent_proceed" - assert row.followup_message_ref == "message:" + _mid("codex-session:ext-followups", "m-silent-followup") + assert row.followup_message_ref == "message:" + _mid( + "codex-session:ext-followups", "m-silent-followup", position=3 + ) aggregate_query = "actions where is_error:true | group by followup_class | count" aggregate_source = parse_unit_source_expression(aggregate_query) diff --git a/tests/unit/devtools/test_incident_coverage_ledger.py b/tests/unit/devtools/test_incident_coverage_ledger.py index 455fbf3f26..98c69a1bda 100644 --- a/tests/unit/devtools/test_incident_coverage_ledger.py +++ b/tests/unit/devtools/test_incident_coverage_ledger.py @@ -50,6 +50,26 @@ def test_real_campaign_graph_resolves_all_current_forcing_dependencies() -> None assert CAMPAIGN_GRAPH_PATH.is_file() +def test_slshy_keeps_partial_disposition_and_named_residual_scope() -> None: + ledger = _ledger() + row = next(row for row in _rows(ledger) if row["bead_id"] == "polylogue-slshy") + graph_entry = next( + entry + for entry in cast(list[dict[str, object]], _graph()["forcing_dependencies"]) + if entry["bead_id"] == "polylogue-slshy" + ) + + assert row["bead_status"] == "in_progress" + assert row["residual_successor"] == { + "bead_id": "polylogue-message-owner-scope-backfill", + "kind": "named-child-bead", + } + assert set(cast(list[str], graph_entry["child_bead_ids"])) == { + "polylogue-xselt", + "polylogue-message-owner-scope-backfill", + } + + def test_deleting_a_forcing_row_is_blocking() -> None: ledger = _ledger() _rows(ledger).pop() diff --git a/tests/unit/storage/test_marks_identity_preserving.py b/tests/unit/storage/test_marks_identity_preserving.py index 1abdab41ad..2305b38587 100644 --- a/tests/unit/storage/test_marks_identity_preserving.py +++ b/tests/unit/storage/test_marks_identity_preserving.py @@ -309,7 +309,7 @@ async def test_message_target_marks_survive_reimport(workspace_env: dict[str, Pa @pytest.mark.asyncio async def test_legacy_message_owner_reads_batch_index_lookup(workspace_env: dict[str, Path]) -> None: - """Legacy message marks resolve through one indexed-message batch.""" + """Legacy message marks and annotations resolve through one batch each.""" db_path = db_setup(workspace_env) builder = SessionBuilder(db_path, "batch-owner") builder.provider("claude-code") @@ -329,9 +329,19 @@ async def test_legacy_message_owner_reads_batch_index_lookup(workspace_env: dict ) is True ) + assert ( + await poly.save_annotation( + f"annotation-{index}", + session_id, + f"note {index}", + target_type="message", + message_id=f"{session_id}:n:msg-{index}", + ) + is True + ) with sqlite3.connect(_user_db_path(workspace_env)) as conn: - conn.execute("UPDATE assertions SET scope_ref = NULL WHERE kind = 'mark'") + conn.execute("UPDATE assertions SET scope_ref = NULL WHERE target_ref LIKE 'message:%'") conn.commit() with ArchiveStore.open_existing(workspace_env["archive_root"]) as archive: @@ -339,14 +349,16 @@ async def test_legacy_message_owner_reads_batch_index_lookup(workspace_env: dict archive._conn.set_trace_callback(statements.append) try: rows = archive.list_marks(mark_type="pin") + annotations = archive.list_annotations() finally: archive._conn.set_trace_callback(None) message_lookup_statements = [ statement for statement in statements if "FROM messages WHERE message_id IN" in statement ] - assert len(message_lookup_statements) == 1 + assert len(message_lookup_statements) == 2 assert {row["session_id"] for row in rows} == {session_id} + assert {row["session_id"] for row in annotations} == {session_id} @pytest.mark.asyncio From 820d3af49ae216d6be39d0983ebae7dae5df51d7 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 9 Aug 2026 03:03:00 +0200 Subject: [PATCH 08/19] ci: revalidate exact PR scope carrier Problem: Circle job 2870 read the PR scope carrier before the PR body was refreshed to the exact reviewed head. What changed: create a signed synchronization commit so the carrier can be re-rendered for the new head before Circle is retriggered. This commit contains no product-code changes. From 90e3b9c56f03b33bba43f4168e5ad411ae841476 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 9 Aug 2026 03:48:36 +0200 Subject: [PATCH 09/19] fix(identity): disambiguate duplicate idless owners Problem: identical idless messages shared one attachment owner anchor, so moving an attachment between duplicate occurrences could leave the session content hash unchanged and make re-ingest skip the ownership change. Claude normalization also needed private evidence keys without publishing synthetic message IDs. What changed: derive duplicate owner anchors from normalized message coordinates only in the attachment hash path, preserve Claude native IDs separately from private reorder-stable evidence keys, and retain transport positions for Claude attachments. Add parser, revision, Drive-equivalent, and archive re-ingest regressions. Compatibility: public provider message IDs remain native or empty. The existing partial owner-scope disposition and named successors remain unchanged. Co-Authored-By: Claude --- polylogue/pipeline/ids.py | 38 ++++- polylogue/sources/parsers/claude/common.py | 130 ++++++++++-------- ...test_message_identity_position_fallback.py | 38 ++++- .../sources/test_claude_web_normalization.py | 77 +++++++++++ .../storage/test_archive_tiers_archive.py | 55 ++++++++ 5 files changed, 276 insertions(+), 62 deletions(-) diff --git a/polylogue/pipeline/ids.py b/polylogue/pipeline/ids.py index ac87e56c68..674423c1a3 100644 --- a/polylogue/pipeline/ids.py +++ b/polylogue/pipeline/ids.py @@ -277,6 +277,38 @@ class the attachment identity fix (polylogue-hith/-d8al) removed for a return f"{_CONTENT_ANCHOR_PREFIX}:{hash_payload(_message_comparison_payload(message))}" +def _message_owner_anchors(messages: list[ParsedMessage], comparison_ids: list[str]) -> dict[int, str]: + """Resolve attachment owners without changing public message identity. + + A content anchor is intentionally shared by duplicate id-less messages. + That is correct for message revision comparison, but it is not enough to + identify which occurrence owns an attachment. When an anchor occurs more + than once, add a private discriminator derived from the parser-normalized + message position. The position is a transport coordinate already used by + ``ParsedAttachment.message_position``; it never enters the public + ``provider_message_id`` or the message comparison payload. + + Messages without a position cannot be addressed by a position-linked + attachment either, so they retain the content anchor in this owner map. + """ + counts = Counter(comparison_ids) + anchors: dict[int, str] = {} + for message, comparison_id in zip(messages, comparison_ids, strict=True): + if message.position is None: + continue + if counts[comparison_id] == 1: + anchors[message.position] = comparison_id + continue + occurrence = hash_payload( + { + "position": message.position, + "variant_index": _normalize_for_hash(message.variant_index), + } + ) + anchors[message.position] = f"{comparison_id}:occurrence:{occurrence}" + return anchors + + def message_identity_hash(*, id: str) -> bytes: """The sole constructor of a message's comparison identity (polylogue-aggz). @@ -525,11 +557,7 @@ def _session_hash_components( _message_hash_payload(message, comparison_id) for message, comparison_id in zip(convo.messages, message_comparison_ids, strict=True) ] - owner_anchor_by_position = { - message.position: comparison_id - for message, comparison_id in zip(convo.messages, message_comparison_ids, strict=True) - if message.position is not None - } + owner_anchor_by_position = _message_owner_anchors(convo.messages, message_comparison_ids) attachments_payload = [ _attachment_hash_payload( attachment, diff --git a/polylogue/sources/parsers/claude/common.py b/polylogue/sources/parsers/claude/common.py index 3f5ba41bbc..87ed6f62ae 100644 --- a/polylogue/sources/parsers/claude/common.py +++ b/polylogue/sources/parsers/claude/common.py @@ -21,6 +21,7 @@ attachment_from_meta, content_blocks_from_segments, human_authored_override, + synthetic_message_id, ) CLAUDE_MISSING_MESSAGE_ID_INGEST_FLAG = "degraded:claude-missing-message-id" @@ -43,7 +44,8 @@ class ClaudeMessageNormalization: @dataclass(frozen=True, slots=True) class _ClaudeMessageEvidence: - provider_message_id: str + evidence_key: str + native_provider_message_id: str raw: Mapping[str, object] original_index: int role: Role @@ -65,11 +67,6 @@ class _ClaudeMessageEvidence: end_turn: bool | None thinking_configuration: dict[str, object] | None - @property - def local_id(self) -> str: - """Return a parser-local key without inventing a provider identity.""" - return self.provider_message_id or f"__idless:{self.original_index}" - @property def has_material(self) -> bool: return bool(self.text or self.blocks or self.attachments) @@ -444,7 +441,7 @@ def _web_tool_evidence_events(evidence: _ClaudeMessageEvidence) -> list[ParsedSe ParsedSessionEvent( event_type="claude_ai_web_tool_evidence", timestamp=start_timestamp if isinstance(start_timestamp, str) else evidence.timestamp, - source_message_provider_id=evidence.provider_message_id, + source_message_provider_id=evidence.native_provider_message_id, payload={"block_index": block_index, **block_evidence}, ) ) @@ -651,7 +648,7 @@ def _sibling_sort_key(evidence: _ClaudeMessageEvidence) -> tuple[int, int, float explicit_variant if explicit_variant is not None else explicit_branch or 0, _timestamp_sort_value(evidence.timestamp), _timestamp_sort_value(evidence.updated_at), - evidence.local_id, + evidence.evidence_key, ) @@ -734,21 +731,21 @@ def _deduplicate_variant_collisions( """ by_position: dict[int, list[_ClaudeMessageEvidence]] = defaultdict(list) for evidence in emitted: - by_position[position_by_id[evidence.local_id]].append(evidence) + by_position[position_by_id[evidence.evidence_key]].append(evidence) for group in by_position.values(): - variants = [variant_index_by_id[evidence.local_id] for evidence in group] + variants = [variant_index_by_id[evidence.evidence_key] for evidence in group] if len(set(variants)) == len(variants): continue ordered = sorted( group, key=lambda evidence: ( - variant_index_by_id[evidence.local_id], + variant_index_by_id[evidence.evidence_key], _timestamp_sort_value(evidence.timestamp), - evidence.local_id, + evidence.evidence_key, ), ) for rank, evidence in enumerate(ordered): - variant_index_by_id[evidence.local_id] = rank + variant_index_by_id[evidence.evidence_key] = rank def _merge_attachment_rows(attachments: list[ParsedAttachment]) -> list[ParsedAttachment]: @@ -763,6 +760,9 @@ def _merge_attachment_rows(attachments: list[ParsedAttachment]) -> list[ParsedAt merged[candidate.provider_attachment_id] = preferred.model_copy( update={ "message_provider_id": preferred.message_provider_id or other.message_provider_id, + "message_position": preferred.message_position + if preferred.message_position is not None + else other.message_position, "name": preferred.name or other.name, "mime_type": preferred.mime_type or other.mime_type, "size_bytes": preferred.size_bytes if preferred.size_bytes is not None else other.size_bytes, @@ -825,7 +825,7 @@ def _active_path_state( leaf_id = None explicit_leaf_ids = [ - evidence.local_id for evidence in evidence_by_id.values() if evidence.explicit_is_active_leaf is True + evidence.evidence_key for evidence in evidence_by_id.values() if evidence.explicit_is_active_leaf is True ] if leaf_id is None and len(explicit_leaf_ids) == 1: leaf_id = explicit_leaf_ids[0] @@ -846,7 +846,7 @@ def _active_path_state( active_children = { evidence.parent_message_provider_id for evidence in evidence_by_id.values() - if evidence.local_id in active_ids and evidence.parent_message_provider_id in active_ids + if evidence.evidence_key in active_ids and evidence.parent_message_provider_id in active_ids } candidates = sorted(active_ids - active_children) if len(candidates) == 1: @@ -866,7 +866,7 @@ def _active_path_state( parent_ids = { evidence.parent_message_provider_id for evidence in evidence_by_id.values() - if evidence.local_id in emitted_ids and evidence.parent_message_provider_id in emitted_ids + if evidence.evidence_key in emitted_ids and evidence.parent_message_provider_id in emitted_ids } terminal_ids = emitted_ids - parent_ids if len(terminal_ids) == 1: @@ -903,12 +903,13 @@ def normalize_chat_messages( """ raw_evidence: list[_ClaudeMessageEvidence] = [] + evidence_key_counts: dict[str, int] = {} ingest_flags: list[str] = [] for index, raw_item in enumerate(chat_messages, start=1): if not isinstance(raw_item, Mapping): continue item = dict(raw_item) - message_id = _first_identity_field( + native_message_id = _first_identity_field( item, "uuid", "id", @@ -916,8 +917,8 @@ def normalize_chat_messages( "messageId", "provider_message_id", ) - if message_id is None: - message_id = "" + if native_message_id is None: + native_message_id = "" ingest_flags.append(CLAUDE_MISSING_MESSAGE_ID_INGEST_FLAG) raw_role = _raw_role(item) @@ -940,17 +941,30 @@ def normalize_chat_messages( raw_created_at = item.get("created_at") or item.get("create_time") or item.get("timestamp") raw_updated_at = item.get("updated_at") or item.get("update_time") or item.get("edited_at") - attachments = _message_attachments(item, message_id) + timestamp = normalize_timestamp(raw_created_at if isinstance(raw_created_at, (int, float, str)) else None) + base_evidence_key = native_message_id or synthetic_message_id( + role=role, + text=text, + timestamp=timestamp, + kind="claude-web-evidence", + ) + occurrence = evidence_key_counts.get(base_evidence_key, 0) + evidence_key_counts[base_evidence_key] = occurrence + 1 + evidence_key = ( + base_evidence_key + if native_message_id or occurrence == 0 + else f"{base_evidence_key}:occurrence:{occurrence}" + ) + attachments = _message_attachments(item, native_message_id) raw_evidence.append( _ClaudeMessageEvidence( - provider_message_id=message_id, + evidence_key=evidence_key, + native_provider_message_id=native_message_id, raw=item, original_index=index, role=role, text=text, - timestamp=normalize_timestamp( - raw_created_at if isinstance(raw_created_at, (int, float, str)) else None - ), + timestamp=timestamp, updated_at=normalize_timestamp( raw_updated_at if isinstance(raw_updated_at, (int, float, str)) else None ), @@ -974,18 +988,18 @@ def normalize_chat_messages( evidence_by_id: dict[str, _ClaudeMessageEvidence] = {} duplicate_ids: set[str] = set() for evidence in raw_evidence: - evidence_id = evidence.local_id + evidence_id = evidence.evidence_key existing = evidence_by_id.get(evidence_id) if existing is None: evidence_by_id[evidence_id] = evidence continue - duplicate_ids.add(evidence.provider_message_id) + duplicate_ids.add(evidence.native_provider_message_id) evidence_by_id[evidence_id] = max((existing, evidence), key=_evidence_richness) if duplicate_ids: ingest_flags.append(CLAUDE_DUPLICATE_MESSAGE_ID_INGEST_FLAG) emitted = [evidence for evidence in evidence_by_id.values() if evidence.has_material] - emitted_ids = {evidence.local_id for evidence in emitted} + emitted_ids = {evidence.evidence_key for evidence in emitted} flat_mode = not any(evidence.parent_message_provider_id for evidence in evidence_by_id.values()) branch_index_by_id: dict[str, int] = {} @@ -996,26 +1010,26 @@ def normalize_chat_messages( evidence.explicit_position if evidence.explicit_position is not None else 2**31, 0 if evidence.timestamp is not None else 1, _timestamp_sort_value(evidence.timestamp), - evidence.local_id if evidence.timestamp is not None else "", + evidence.evidence_key if evidence.timestamp is not None else "", evidence.original_index, ), ) position_by_id = { - evidence.local_id: (evidence.explicit_position if evidence.explicit_position is not None else position) + evidence.evidence_key: (evidence.explicit_position if evidence.explicit_position is not None else position) for position, evidence in enumerate(ordered_flat) } for evidence in emitted: - branch_index_by_id[evidence.local_id] = evidence.explicit_branch_index or 0 + branch_index_by_id[evidence.evidence_key] = evidence.explicit_branch_index or 0 else: depths, cycle_detected = _lineage_depths(evidence_by_id) if cycle_detected: ingest_flags.append(CLAUDE_LINEAGE_CYCLE_INGEST_FLAG) - minimum_emitted_depth = min((depths[evidence.local_id] for evidence in emitted), default=0) + minimum_emitted_depth = min((depths[evidence.evidence_key] for evidence in emitted), default=0) position_by_id = { - evidence.local_id: ( + evidence.evidence_key: ( evidence.explicit_position if evidence.explicit_position is not None - else max(0, depths[evidence.local_id] - minimum_emitted_depth) + else max(0, depths[evidence.evidence_key] - minimum_emitted_depth) ) for evidence in emitted } @@ -1024,24 +1038,24 @@ def normalize_chat_messages( siblings_by_parent[evidence.parent_message_provider_id].append(evidence) for siblings in siblings_by_parent.values(): for rank, evidence in enumerate(sorted(siblings, key=_sibling_sort_key)): - branch_index_by_id[evidence.local_id] = ( + branch_index_by_id[evidence.evidence_key] = ( evidence.explicit_branch_index if evidence.explicit_branch_index is not None else rank ) if flat_mode: variant_index_by_id = { - evidence.local_id: ( + evidence.evidence_key: ( evidence.explicit_variant_index if evidence.explicit_variant_index is not None - else branch_index_by_id.get(evidence.local_id, 0) + else branch_index_by_id.get(evidence.evidence_key, 0) ) for evidence in emitted } else: resolved_variants: dict[str, int] = {} variant_index_by_id = { - evidence.local_id: _resolve_variant_index( - evidence.local_id, + evidence.evidence_key: _resolve_variant_index( + evidence.evidence_key, evidence_by_id, branch_index_by_id, resolved_variants, @@ -1053,10 +1067,10 @@ def normalize_chat_messages( # `_deduplicate_variant_collisions` docstring. _deduplicate_variant_collisions(emitted, position_by_id, variant_index_by_id) order_key_by_id = { - evidence.local_id: ( - position_by_id[evidence.local_id], - variant_index_by_id[evidence.local_id], - evidence.local_id, + evidence.evidence_key: ( + position_by_id[evidence.evidence_key], + variant_index_by_id[evidence.evidence_key], + evidence.evidence_key, ) for evidence in emitted } @@ -1074,17 +1088,17 @@ def _evidence_message_type(evidence: _ClaudeMessageEvidence) -> MessageType: messages = [ ParsedMessage( - provider_message_id=evidence.provider_message_id, + provider_message_id=evidence.native_provider_message_id, role=evidence.role, text=evidence.text, timestamp=evidence.timestamp, blocks=evidence.blocks, parent_message_provider_id=evidence.parent_message_provider_id, - position=position_by_id[evidence.local_id], - branch_index=branch_index_by_id.get(evidence.local_id, 0), - variant_index=variant_index_by_id[evidence.local_id], - is_active_path=path_values.get(evidence.local_id), - is_active_leaf=leaf_values.get(evidence.local_id), + position=position_by_id[evidence.evidence_key], + branch_index=branch_index_by_id.get(evidence.evidence_key, 0), + variant_index=variant_index_by_id[evidence.evidence_key], + is_active_path=path_values.get(evidence.evidence_key), + is_active_leaf=leaf_values.get(evidence.evidence_key), model_name=evidence.model_name, model_effort=evidence.model_effort, duration_ms=evidence.duration_ms, @@ -1106,10 +1120,16 @@ def _evidence_message_type(evidence: _ClaudeMessageEvidence) -> MessageType: ), ), ) - for evidence in sorted(emitted, key=lambda row: order_key_by_id[row.local_id]) + for evidence in sorted(emitted, key=lambda row: order_key_by_id[row.evidence_key]) ] - attachments = _merge_attachment_rows([attachment for evidence in emitted for attachment in evidence.attachments]) + attachments = _merge_attachment_rows( + [ + attachment.model_copy(update={"message_position": position_by_id[evidence.evidence_key]}) + for evidence in emitted + for attachment in evidence.attachments + ] + ) models_used: list[str] = [] for model_name in [session_model, *(message.model_name for message in messages)]: if model_name and model_name not in models_used: @@ -1132,7 +1152,7 @@ def _evidence_message_type(evidence: _ClaudeMessageEvidence) -> MessageType: ) ) - for evidence in sorted(emitted, key=lambda row: order_key_by_id[row.local_id]): + for evidence in sorted(emitted, key=lambda row: order_key_by_id[row.evidence_key]): session_events.extend(_web_tool_evidence_events(evidence)) if evidence.thinking_configuration: payload: dict[str, object] = {"thinking": evidence.thinking_configuration} @@ -1144,7 +1164,7 @@ def _evidence_message_type(evidence: _ClaudeMessageEvidence) -> MessageType: ParsedSessionEvent( event_type="model_configuration", timestamp=evidence.updated_at or evidence.timestamp, - source_message_provider_id=evidence.provider_message_id, + source_message_provider_id=evidence.native_provider_message_id, payload=payload, ) ) @@ -1171,7 +1191,7 @@ def _evidence_message_type(evidence: _ClaudeMessageEvidence) -> MessageType: else "provider_message_update" ), timestamp=evidence.updated_at, - source_message_provider_id=evidence.provider_message_id, + source_message_provider_id=evidence.native_provider_message_id, payload=update_payload, ) ) @@ -1194,7 +1214,9 @@ def _evidence_message_type(evidence: _ClaudeMessageEvidence) -> MessageType: messages=messages, attachments=attachments, active_leaf_message_provider_id=( - evidence_by_id[normalized_active_leaf].provider_message_id if normalized_active_leaf is not None else None + evidence_by_id[normalized_active_leaf].native_provider_message_id + if normalized_active_leaf is not None + else None ), models_used=models_used, session_events=session_events, diff --git a/tests/unit/pipeline/test_message_identity_position_fallback.py b/tests/unit/pipeline/test_message_identity_position_fallback.py index 3a04e51400..ffcea900fd 100644 --- a/tests/unit/pipeline/test_message_identity_position_fallback.py +++ b/tests/unit/pipeline/test_message_identity_position_fallback.py @@ -24,10 +24,10 @@ from polylogue.archive.session_revision_membership import MembershipRevision, classify_membership_revisions from polylogue.core.enums import Provider from polylogue.pipeline.ids import session_revision_projection -from polylogue.sources.parsers.base import ParsedMessage, ParsedSession +from polylogue.sources.parsers.base import ParsedAttachment, ParsedMessage, ParsedSession -def _session(messages: list[ParsedMessage]) -> ParsedSession: +def _session(messages: list[ParsedMessage], attachments: list[ParsedAttachment] | None = None) -> ParsedSession: return ParsedSession( source_name=Provider.CHATGPT, provider_session_id="conv-1", @@ -35,7 +35,7 @@ def _session(messages: list[ParsedMessage]) -> ParsedSession: created_at="2024-01-01T00:00:00Z", updated_at="2024-01-01T00:00:00Z", messages=messages, - attachments=[], + attachments=attachments or [], ) @@ -98,3 +98,35 @@ def test_timestamp_less_idless_duplicates_preserve_unordered_multiplicity() -> N assert classification.accepted_raw_ids == ("one", "two") assert not classification.equivalent_raw_ids assert not classification.ambiguous_raw_ids + + +def test_duplicate_idless_attachment_owner_reassignment_changes_hash() -> None: + """Duplicate content anchors still distinguish position-linked owners. + + The messages remain the same content in both revisions. Only the stable + transport owner coordinate of one attachment moves from occurrence zero + to occurrence one. A content-only owner anchor would make this a false + re-ingest skip. + """ + repeated = [ + _id_less("assistant", "repeat", "2024-01-01T00:00:00Z").model_copy(update={"position": position}) + for position in (0, 1) + ] + attachment = ParsedAttachment( + provider_attachment_id="drive-doc", + message_provider_id="", + message_position=0, + name="note.txt", + mime_type="text/plain", + ) + moved = attachment.model_copy(update={"message_position": 1}) + + first = session_revision_projection(_session(repeated, [attachment])) + second = session_revision_projection(_session(repeated, [moved])) + reordered = session_revision_projection(_session(list(reversed(repeated)), [attachment])) + + assert first.message_contents == second.message_contents + assert first.message_contents == reordered.message_contents + assert first.attachment_identities == reordered.attachment_identities + assert first.attachment_identities != second.attachment_identities + assert first.session_hash != second.session_hash diff --git a/tests/unit/sources/test_claude_web_normalization.py b/tests/unit/sources/test_claude_web_normalization.py index d35d671142..481f614871 100644 --- a/tests/unit/sources/test_claude_web_normalization.py +++ b/tests/unit/sources/test_claude_web_normalization.py @@ -436,6 +436,83 @@ def test_claude_multiple_idless_messages_remain_distinct_and_choose_one_leaf() - ) +def test_claude_same_timestamp_idless_rows_keep_private_association_keys() -> None: + rows = { + "first": { + "sender": "human", + "text": "The first same-timestamp turn.", + "created_at": "2026-07-01T10:00:00Z", + "content": [ + { + "type": "tool_use", + "id": "tool-first", + "name": "lookup", + "input": {"query": "first"}, + "start_timestamp": "2026-07-01T10:00:00.100Z", + "display_content": {"type": "text", "text": "first evidence"}, + } + ], + "files": [{"file_uuid": "file-first", "file_name": "first.txt", "file_type": "text/plain"}], + "is_active_leaf": False, + }, + "leaf": { + "sender": "assistant", + "text": "The active same-timestamp turn.", + "created_at": "2026-07-01T10:00:00Z", + "content": [ + { + "type": "tool_use", + "id": "tool-leaf", + "name": "lookup", + "input": {"query": "leaf"}, + "start_timestamp": "2026-07-01T10:00:00.200Z", + "display_content": {"type": "text", "text": "leaf evidence"}, + } + ], + "files": [{"file_uuid": "file-leaf", "file_name": "leaf.txt", "file_type": "text/plain"}], + "is_active_leaf": True, + }, + } + forward = parse_payload( + Provider.CLAUDE_AI, + {"uuid": "claude-idless-same-timestamp", "chat_messages": [rows["first"], rows["leaf"]]}, + "fallback", + )[0] + reversed_rows = parse_payload( + Provider.CLAUDE_AI, + {"uuid": "claude-idless-same-timestamp", "chat_messages": [rows["leaf"], rows["first"]]}, + "fallback", + )[0] + + assert [message.provider_message_id for message in forward.messages] == ["", ""] + assert [message.provider_message_id for message in reversed_rows.messages] == ["", ""] + assert ( + session_revision_projection(forward).message_contents + == session_revision_projection(reversed_rows).message_contents + ) + + for normalized in (forward, reversed_rows): + messages_by_text = {message.text: message for message in normalized.messages} + assert [message.text for message in normalized.messages if message.is_active_leaf] == [ + "The active same-timestamp turn." + ] + assert normalized.active_leaf_message_provider_id == "" + assert {attachment.message_provider_id for attachment in normalized.attachments} == {""} + assert {attachment.message_position: attachment.name for attachment in normalized.attachments} == { + messages_by_text["The first same-timestamp turn."].position: "first.txt", + messages_by_text["The active same-timestamp turn."].position: "leaf.txt", + } + + tool_events = [ + event for event in normalized.session_events if event.event_type == "claude_ai_web_tool_evidence" + ] + assert [event.source_message_provider_id for event in tool_events] == ["", ""] + assert {json.dumps(event.payload["display_content"], sort_keys=True) for event in tool_events} == { + '{"text": "first evidence", "type": "text"}', + '{"text": "leaf evidence", "type": "text"}', + } + + def test_authenticated_browser_capture_uses_native_payload_and_enriches_attachment() -> None: """Mutations: detector-order theft or attachment replacement loss must fail.""" direct = _parse_real_route(_native_claude_payload()) diff --git a/tests/unit/storage/test_archive_tiers_archive.py b/tests/unit/storage/test_archive_tiers_archive.py index 3c187ef8db..1245ac884c 100644 --- a/tests/unit/storage/test_archive_tiers_archive.py +++ b/tests/unit/storage/test_archive_tiers_archive.py @@ -1199,6 +1199,61 @@ def test_archive_tiers_archive_facade_hash_skips_identical_content_and_refreshes assert stored_raw_id == second.raw_id +def test_archive_tiers_archive_facade_reingests_duplicate_idless_owner_reassignment(tmp_path: Path) -> None: + def session(message_position: int) -> ParsedSession: + return ParsedSession( + source_name=Provider.GEMINI, + provider_session_id="duplicate-idless-owner-reassignment", + title="Duplicate idless owners", + updated_at="2026-04-03T00:00:00Z", + messages=[ + ParsedMessage( + provider_message_id="", + role=Role.USER, + text="same duplicate turn", + timestamp="2026-04-03T00:00:00Z", + position=0, + ), + ParsedMessage( + provider_message_id="", + role=Role.USER, + text="same duplicate turn", + timestamp="2026-04-03T00:00:00Z", + position=1, + ), + ], + attachments=[ + ParsedAttachment( + provider_attachment_id="drive-doc", + message_provider_id="", + message_position=message_position, + name="note.txt", + mime_type="text/plain", + ) + ], + ) + + root = tmp_path / "archive" + with ArchiveStore(root) as facade: + first = facade.write_raw_and_parsed_result( + session(0), + payload=b'{"owner":0}', + source_path="/tmp/duplicate-owner-first.json", + acquired_at_ms=1_767_000_000_000, + ) + reassigned = facade.write_raw_and_parsed_result( + session(1), + payload=b'{"owner":1}', + source_path="/tmp/duplicate-owner-second.json", + acquired_at_ms=1_767_000_000_001, + ) + + assert first.content_changed is True + assert reassigned.content_changed is True + assert reassigned.counts["sessions"] == 1 + assert reassigned.counts["skipped_sessions"] == 0 + + def test_archive_tiers_archive_facade_replaces_same_size_changed_attachment_bytes(tmp_path: Path) -> None: def session(payload: bytes) -> ParsedSession: return ParsedSession( From bada6c7265efbb0584122a3a9a5d27ce016568e2 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 9 Aug 2026 04:51:44 +0200 Subject: [PATCH 10/19] fix(identity): unify private message owner resolution --- polylogue/core/message_owner.py | 42 ++++ polylogue/pipeline/ids.py | 192 +++++++++++++----- polylogue/sources/parsers/base_models.py | 9 + polylogue/sources/parsers/base_support.py | 2 +- .../sources/parsers/claude/code_parser.py | 4 +- polylogue/sources/parsers/claude/common.py | 99 +++++++-- polylogue/sources/parsers/codex.py | 2 +- polylogue/sources/parsers/drive.py | 31 ++- polylogue/sources/parsers/local_agent.py | 2 +- polylogue/storage/attachment_relink.py | 37 ++-- polylogue/storage/raw_authority.py | 6 +- .../storage/sqlite/archive_tiers/write.py | 49 ++--- 12 files changed, 349 insertions(+), 126 deletions(-) create mode 100644 polylogue/core/message_owner.py diff --git a/polylogue/core/message_owner.py b/polylogue/core/message_owner.py new file mode 100644 index 0000000000..c0e7653025 --- /dev/null +++ b/polylogue/core/message_owner.py @@ -0,0 +1,42 @@ +"""Private parser-to-writer coordinates for message ownership.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True, slots=True) +class MessageOwnerCoordinate: + """Private linkage between a parsed attachment and its message. + + ``stable_key`` carries reorder-stable provider evidence when the parser + has it. ``position`` and ``variant_index`` are the complete transport + coordinate used as the fail-closed fallback. The coordinate is excluded + from public parser serialization and is never a provider message id. + """ + + stable_key: str | None = None + position: int | None = None + variant_index: int = 0 + + def __post_init__(self) -> None: + if self.position is not None and self.position < 0: + raise ValueError("message owner position cannot be negative") + if self.variant_index < 0: + raise ValueError("message owner variant_index cannot be negative") + if self.stable_key == "": + raise ValueError("message owner stable_key cannot be empty") + + @property + def physical_key(self) -> tuple[int, int] | None: + """Return the full position/variant coordinate when it is present.""" + if self.position is None: + return None + return self.position, self.variant_index + + +class MessageOwnerAmbiguityError(ValueError): + """Raised when an attachment owner cannot be resolved without guessing.""" + + +__all__ = ["MessageOwnerAmbiguityError", "MessageOwnerCoordinate"] diff --git a/polylogue/pipeline/ids.py b/polylogue/pipeline/ids.py index 674423c1a3..901564f6fb 100644 --- a/polylogue/pipeline/ids.py +++ b/polylogue/pipeline/ids.py @@ -11,6 +11,7 @@ from polylogue.core.enums import BlockType, Origin, Provider from polylogue.core.hashing import hash_bytes, hash_payload from polylogue.core.json import JSONValue +from polylogue.core.message_owner import MessageOwnerAmbiguityError, MessageOwnerCoordinate from polylogue.core.sources import origin_from_provider from polylogue.core.types import ContentHash, MessageId, SessionId @@ -248,9 +249,8 @@ def _message_comparison_payload(message: ParsedMessage) -> dict[str, JSONValue]: _CONTENT_ANCHOR_PREFIX = "__polylogue_msg_content_anchor__" -def _message_comparison_id(message: ParsedMessage) -> str: - """Resolve the id used as both a message's content-payload id and its - comparison identity (``message_identity_hash``'s sole input). +def _message_revision_match_id(message: ParsedMessage) -> str: + """Resolve the stable revision-match identity for one parsed message. Prefers the provider's own id -- stable across reordering even when the export's array ordering is not (polylogue-c429). When a parser could not @@ -261,11 +261,10 @@ class the attachment identity fix (polylogue-hith/-d8al) removed for a array position would get two different fallback "ids" and compare as a conflict instead of the same message (polylogue-gysk3). - The fix: fall back to one content-derived anchor over role, timestamp, - and message text instead of array position. Including text even when a - timestamp exists keeps two same-timestamp id-less messages distinct and - gives attachment ownership a stable non-positional anchor when a Drive - attachment moves between them. + Timestamped id-less messages use only role and timestamp here. Their text + and blocks remain in ``_message_hash_payload`` as mutable content, so an + edit shares one revision axis while still changing the session hash. + Timestamp-less messages need their content to remain distinguishable. Parser normalization maintains the complementary invariant: a missing native id is never replaced with an array-position-derived value before @@ -273,40 +272,143 @@ class the attachment identity fix (polylogue-hith/-d8al) removed for a but they are not persisted as ``provider_message_id``. """ if message.provider_message_id: - return message.provider_message_id - return f"{_CONTENT_ANCHOR_PREFIX}:{hash_payload(_message_comparison_payload(message))}" + return message.provider_message_id.strip() + payload: dict[str, JSONValue] = { + "role": str(message.role), + "timestamp": _normalize_for_hash(message.timestamp), + } + if message.timestamp is None: + payload["text"] = _normalize_for_hash(message.text) + if message.blocks and not _is_redundant_text_only_block(message): + payload["content_blocks"] = [_content_block_payload(b) for b in message.blocks] + return f"{_CONTENT_ANCHOR_PREFIX}:{hash_payload(payload)}" -def _message_owner_anchors(messages: list[ParsedMessage], comparison_ids: list[str]) -> dict[int, str]: - """Resolve attachment owners without changing public message identity. +@dataclass(frozen=True, slots=True) +class MessageOwnerResolution: + """One shared private owner-resolution contract for hash and write paths.""" + + keys: tuple[str, ...] + by_physical_coordinate: Mapping[tuple[int, int], str] + by_stable_key: Mapping[str, str] + ambiguous_keys: frozenset[str] + unique_provider_keys: Mapping[str, str] + ambiguous_provider_ids: frozenset[str] + + +def _message_owner_coordinate(message: ParsedMessage, fallback_position: int) -> MessageOwnerCoordinate: + coordinate = message.owner_coordinate + if coordinate is not None: + return MessageOwnerCoordinate( + stable_key=coordinate.stable_key, + position=coordinate.position if coordinate.position is not None else message.position, + variant_index=coordinate.variant_index, + ) + return MessageOwnerCoordinate( + position=message.position if message.position is not None else fallback_position, + variant_index=message.variant_index or 0, + ) + - A content anchor is intentionally shared by duplicate id-less messages. - That is correct for message revision comparison, but it is not enough to - identify which occurrence owns an attachment. When an anchor occurs more - than once, add a private discriminator derived from the parser-normalized - message position. The position is a transport coordinate already used by - ``ParsedAttachment.message_position``; it never enters the public - ``provider_message_id`` or the message comparison payload. +def message_owner_resolution(messages: list[ParsedMessage]) -> MessageOwnerResolution: + """Resolve stable private owner keys from one parsed message batch. - Messages without a position cannot be addressed by a position-linked - attachment either, so they retain the content anchor in this owner map. + Unique native ids and timestamped id-less role/timestamp anchors are + stable across reordering and edits. Duplicate anchors use a unique + content discriminator when the occurrences differ, then parser-provided + reorder-stable evidence when their content is identical. If neither + distinguishes the occurrences, the duplicate remains typed ambiguity + instead of receiving a position-derived identity. """ - counts = Counter(comparison_ids) - anchors: dict[int, str] = {} - for message, comparison_id in zip(messages, comparison_ids, strict=True): - if message.position is None: - continue - if counts[comparison_id] == 1: - anchors[message.position] = comparison_id - continue - occurrence = hash_payload( - { - "position": message.position, - "variant_index": _normalize_for_hash(message.variant_index), - } + revision_ids = tuple(_message_revision_match_id(message) for message in messages) + revision_counts = Counter(revision_ids) + content_ids = tuple( + f"{_CONTENT_ANCHOR_PREFIX}:{hash_payload(_message_comparison_payload(message))}" for message in messages + ) + content_counts = Counter(content_ids) + coordinates = tuple(_message_owner_coordinate(message, index) for index, message in enumerate(messages)) + + keys: list[str] = [] + for revision_id, content_id, coordinate in zip(revision_ids, content_ids, coordinates, strict=True): + if revision_counts[revision_id] == 1: + key = revision_id + elif content_counts[content_id] == 1: + key = content_id + elif coordinate.stable_key is not None: + key = coordinate.stable_key + else: + key = revision_id + keys.append(key) + + key_counts = Counter(keys) + ambiguous_keys = frozenset(key for key, count in key_counts.items() if count > 1) + by_physical_coordinate = { + coordinate.physical_key: key + for coordinate, key in zip(coordinates, keys, strict=True) + if coordinate.physical_key is not None + } + by_stable_key = { + coordinate.stable_key: key + for coordinate, key in zip(coordinates, keys, strict=True) + if coordinate.stable_key is not None and key not in ambiguous_keys + } + provider_keys: dict[str, str] = {} + provider_counts = Counter( + message.provider_message_id.strip() for message in messages if message.provider_message_id + ) + for message, key in zip(messages, keys, strict=True): + provider_id = message.provider_message_id.strip() + if provider_id and provider_counts[provider_id] == 1: + provider_keys[provider_id] = key + return MessageOwnerResolution( + keys=tuple(keys), + by_physical_coordinate=by_physical_coordinate, + by_stable_key=by_stable_key, + ambiguous_keys=ambiguous_keys, + unique_provider_keys=provider_keys, + ambiguous_provider_ids=frozenset(provider_id for provider_id, count in provider_counts.items() if count > 1), + ) + + +def _attachment_owner_coordinate(attachment: ParsedAttachment) -> MessageOwnerCoordinate: + coordinate = attachment.owner_coordinate + if coordinate is not None: + return MessageOwnerCoordinate( + stable_key=coordinate.stable_key, + position=coordinate.position if coordinate.position is not None else attachment.message_position, + variant_index=coordinate.variant_index, ) - anchors[message.position] = f"{comparison_id}:occurrence:{occurrence}" - return anchors + return MessageOwnerCoordinate( + position=attachment.message_position, + variant_index=attachment.message_variant_index or 0, + ) + + +def attachment_message_owner_key(attachment: ParsedAttachment, resolution: MessageOwnerResolution) -> str | None: + """Resolve one attachment to the same private owner key used by writes.""" + coordinate = _attachment_owner_coordinate(attachment) + if coordinate.stable_key is not None: + if coordinate.stable_key in resolution.ambiguous_keys: + raise MessageOwnerAmbiguityError(f"attachment owner evidence is duplicated: {coordinate.stable_key!r}") + if coordinate.stable_key in resolution.by_stable_key: + return resolution.by_stable_key[coordinate.stable_key] + if coordinate.physical_key is not None: + key = resolution.by_physical_coordinate.get(coordinate.physical_key) + if key is not None: + if key in resolution.ambiguous_keys: + raise MessageOwnerAmbiguityError( + "attachment owner coordinate is indistinguishable from another message: " + f"{coordinate.physical_key!r}" + ) + return key + if attachment.message_provider_id: + provider_id = attachment.message_provider_id.strip() + if provider_id in resolution.ambiguous_provider_ids: + raise MessageOwnerAmbiguityError( + f"attachment provider message id is duplicated without a private coordinate: {provider_id!r}" + ) + return resolution.unique_provider_keys.get(provider_id, provider_id) + return None def message_identity_hash(*, id: str) -> bytes: @@ -367,12 +469,8 @@ def attachment_identity_hash(*, message_id: JSONValue, name: JSONValue, mime_typ def _attachment_hash_payload( attachment: ParsedAttachment, *, message_owner_anchor: str | None = None ) -> dict[str, JSONValue]: - """Build attachment identity without perturbing legacy metadata-only hashes.""" - owner_id = ( - message_owner_anchor - if not attachment.message_provider_id and message_owner_anchor - else attachment.message_provider_id - ) + """Build the full attachment payload using the shared owner coordinate.""" + owner_id = message_owner_anchor or attachment.message_provider_id payload: dict[str, JSONValue] = { "id": _normalize_for_hash(attachment.provider_attachment_id), "message_id": _normalize_for_hash(owner_id), @@ -552,20 +650,16 @@ def _session_hash_components( caller re-deriving its own copy. Byte-identical to computing each payload independently -- pure sharing of an already-pure computation. """ - message_comparison_ids = [_message_comparison_id(msg) for msg in convo.messages] + owner_resolution = message_owner_resolution(convo.messages) + message_comparison_ids = list(owner_resolution.keys) messages_payload = [ _message_hash_payload(message, comparison_id) for message, comparison_id in zip(convo.messages, message_comparison_ids, strict=True) ] - owner_anchor_by_position = _message_owner_anchors(convo.messages, message_comparison_ids) attachments_payload = [ _attachment_hash_payload( attachment, - message_owner_anchor=( - owner_anchor_by_position.get(attachment.message_position) - if attachment.message_position is not None - else None - ), + message_owner_anchor=attachment_message_owner_key(attachment, owner_resolution), ) for attachment in convo.attachments ] diff --git a/polylogue/sources/parsers/base_models.py b/polylogue/sources/parsers/base_models.py index cee186cc9d..df38a8b188 100644 --- a/polylogue/sources/parsers/base_models.py +++ b/polylogue/sources/parsers/base_models.py @@ -10,6 +10,7 @@ from polylogue.archive.message.types import MessageType from polylogue.archive.session.branch_type import BranchType from polylogue.core.enums import BlockType, MaterialOrigin, Provider, SessionKind, TitleSource, WebConstructType +from polylogue.core.message_owner import MessageOwnerCoordinate from polylogue.core.security import sanitize_path as _sanitize_path_helper from polylogue.core.timestamps import parse_timestamp @@ -150,6 +151,9 @@ class ParsedMessage(BaseModel): # only the resulting archive message id. It is never provider identity or # persisted parser data. parent_message_position: int | None = Field(default=None, exclude=True, repr=False) + # Private parser-to-writer evidence. This never becomes provider identity + # or a public message id. + owner_coordinate: MessageOwnerCoordinate | None = Field(default=None, exclude=True, repr=False) position: int | None = None branch_index: int = 0 variant_index: int | None = None @@ -251,6 +255,11 @@ class ParsedAttachment(BaseModel): # Transport-only linkage for id-less source messages. This is parser-local # bookkeeping, never a provider identity or stored attachment column. message_position: int | None = Field(default=None, exclude=True, repr=False) + message_variant_index: int | None = Field(default=None, exclude=True, repr=False) + # Full private owner evidence. ``message_position`` and + # ``message_variant_index`` remain as parser compatibility fields, while + # hash/write/repair paths consume this typed coordinate. + owner_coordinate: MessageOwnerCoordinate | None = Field(default=None, exclude=True, repr=False) name: str | None = None mime_type: str | None = None size_bytes: int | None = None diff --git a/polylogue/sources/parsers/base_support.py b/polylogue/sources/parsers/base_support.py index 73a7e8d7c4..ca870de007 100644 --- a/polylogue/sources/parsers/base_support.py +++ b/polylogue/sources/parsers/base_support.py @@ -440,7 +440,7 @@ def extract_messages_from_list(items: Sequence[object]) -> list[ParsedMessage]: if text: # polylogue-slshy: no positional fallback -- empty id lets - # _message_comparison_id's content-derived anchor fallback run + # _message_revision_match_id's content-derived anchor fallback run # instead of a position-derived string that would change identity # when array order shifts across re-acquisitions. msg_id = str(payload.get("id") or payload.get("uuid") or item.get("uuid") or item.get("id") or "") diff --git a/polylogue/sources/parsers/claude/code_parser.py b/polylogue/sources/parsers/claude/code_parser.py index b12f49e395..5b6548d7b1 100644 --- a/polylogue/sources/parsers/claude/code_parser.py +++ b/polylogue/sources/parsers/claude/code_parser.py @@ -1403,7 +1403,7 @@ def _fold_code_record(acc: _SessionAccumulator, index: int, item: dict[str, obje acc.messages.append( ParsedMessage( # polylogue-slshy: no positional fallback -- empty id lets - # _message_comparison_id's content-anchor fallback run. + # _message_revision_match_id's content-anchor fallback run. provider_message_id=str(item.get("uuid") or ""), role=Role.SYSTEM, text=summary_text, @@ -1622,7 +1622,7 @@ def _fold_code_record(acc: _SessionAccumulator, index: int, item: dict[str, obje # user role avoids false positives from assistant text that quotes a marker. paste_spans = _detect_paste_spans(text) if resolved_role == Role.USER else [] # polylogue-slshy: no positional fallback -- empty id lets - # _message_comparison_id's content-anchor fallback run instead of a + # _message_revision_match_id's content-anchor fallback run instead of a # position-derived string that would change identity when array # order shifts across re-acquisitions. provider_message_id = str(record_uuid or "") diff --git a/polylogue/sources/parsers/claude/common.py b/polylogue/sources/parsers/claude/common.py index 87ed6f62ae..f9354e1fa0 100644 --- a/polylogue/sources/parsers/claude/common.py +++ b/polylogue/sources/parsers/claude/common.py @@ -3,13 +3,15 @@ from __future__ import annotations import json -from collections import defaultdict +from collections import Counter, defaultdict from collections.abc import Mapping from dataclasses import dataclass from polylogue.archive.message.artifacts import classify_block_message_type, classify_material_origin from polylogue.archive.message.roles import Role from polylogue.core.enums import BlockType, MessageType, WebConstructType +from polylogue.core.hashing import hash_payload +from polylogue.core.message_owner import MessageOwnerCoordinate from polylogue.core.timestamps import parse_timestamp from ..base import ( @@ -66,6 +68,7 @@ class _ClaudeMessageEvidence: delivery_status: str | None end_turn: bool | None thinking_configuration: dict[str, object] | None + owner_stable_key: str | None @property def has_material(self) -> bool: @@ -614,6 +617,46 @@ def _message_attachments(item: Mapping[str, object], message_id: str) -> list[Pa return attachments +def _owner_stable_key( + item: Mapping[str, object], + *, + parent_message_provider_id: str | None, + explicit_position: int | None, + explicit_branch_index: int | None, + explicit_variant_index: int | None, + blocks: list[ParsedContentBlock], + attachments: list[ParsedAttachment], +) -> str | None: + """Derive reorder-stable private evidence for duplicate owner anchors.""" + evidence: dict[str, object] = {} + for field, value in ( + ("parent", parent_message_provider_id), + ("position", explicit_position), + ("branch", explicit_branch_index), + ("variant", explicit_variant_index), + ): + if value is not None: + evidence[field] = value + for field in ("message_key", "messageKey", "turn_id", "turnId", "sequence_id", "sequenceId"): + value = _first_identity_field(item, field) + if value is not None: + evidence[field] = value + block_ids = sorted(block.tool_id for block in blocks if block.tool_id) + if block_ids: + evidence["tool_ids"] = block_ids + if explicit_position is None and explicit_branch_index is None and explicit_variant_index is None: + attachment_ids = sorted( + (attachment.provider_attachment_id, attachment.provider_file_id, attachment.provider_drive_id) + for attachment in attachments + if attachment.provider_attachment_id or attachment.provider_file_id or attachment.provider_drive_id + ) + if attachment_ids: + evidence["attachment_ids"] = [list(values) for values in attachment_ids] + if not evidence: + return None + return f"claude-owner-evidence:{hash_payload(evidence)}" + + def _canonical_record(item: Mapping[str, object]) -> str: return json.dumps(dict(item), sort_keys=True, separators=(",", ":"), default=str) @@ -763,6 +806,10 @@ def _merge_attachment_rows(attachments: list[ParsedAttachment]) -> list[ParsedAt "message_position": preferred.message_position if preferred.message_position is not None else other.message_position, + "message_variant_index": preferred.message_variant_index + if preferred.message_variant_index is not None + else other.message_variant_index, + "owner_coordinate": preferred.owner_coordinate or other.owner_coordinate, "name": preferred.name or other.name, "mime_type": preferred.mime_type or other.mime_type, "size_bytes": preferred.size_bytes if preferred.size_bytes is not None else other.size_bytes, @@ -950,12 +997,12 @@ def normalize_chat_messages( ) occurrence = evidence_key_counts.get(base_evidence_key, 0) evidence_key_counts[base_evidence_key] = occurrence + 1 - evidence_key = ( - base_evidence_key - if native_message_id or occurrence == 0 - else f"{base_evidence_key}:occurrence:{occurrence}" - ) + evidence_key = base_evidence_key if occurrence == 0 else f"{base_evidence_key}:occurrence:{occurrence}" attachments = _message_attachments(item, native_message_id) + parent_message_provider_id = _message_parent_id(item) + explicit_position = _first_non_negative_int_field(item, "position") + explicit_branch_index = _first_non_negative_int_field(item, "branch_index", "branchIndex") + explicit_variant_index = _first_non_negative_int_field(item, "variant_index", "variantIndex") raw_evidence.append( _ClaudeMessageEvidence( evidence_key=evidence_key, @@ -970,10 +1017,10 @@ def normalize_chat_messages( ), blocks=content_blocks, attachments=attachments, - parent_message_provider_id=_message_parent_id(item), - explicit_position=_first_non_negative_int_field(item, "position"), - explicit_branch_index=_first_non_negative_int_field(item, "branch_index", "branchIndex"), - explicit_variant_index=_first_non_negative_int_field(item, "variant_index", "variantIndex"), + parent_message_provider_id=parent_message_provider_id, + explicit_position=explicit_position, + explicit_branch_index=explicit_branch_index, + explicit_variant_index=explicit_variant_index, explicit_is_active_path=_first_bool_field(item, "is_active_path", "isActivePath", "active_path"), explicit_is_active_leaf=_first_bool_field(item, "is_active_leaf", "isActiveLeaf", "active_leaf"), model_name=_message_model_name(item) or session_model, @@ -982,18 +1029,29 @@ def normalize_chat_messages( delivery_status=_message_delivery_status(item), end_turn=_message_end_turn(item), thinking_configuration=_thinking_configuration(item), + owner_stable_key=_owner_stable_key( + item, + parent_message_provider_id=parent_message_provider_id, + explicit_position=explicit_position, + explicit_branch_index=explicit_branch_index, + explicit_variant_index=explicit_variant_index, + blocks=content_blocks, + attachments=attachments, + ), ) ) evidence_by_id: dict[str, _ClaudeMessageEvidence] = {} - duplicate_ids: set[str] = set() + native_id_counts = Counter( + evidence.native_provider_message_id for evidence in raw_evidence if evidence.native_provider_message_id + ) + duplicate_ids: set[str] = {native_id for native_id, count in native_id_counts.items() if count > 1} for evidence in raw_evidence: evidence_id = evidence.evidence_key existing = evidence_by_id.get(evidence_id) if existing is None: evidence_by_id[evidence_id] = evidence continue - duplicate_ids.add(evidence.native_provider_message_id) evidence_by_id[evidence_id] = max((existing, evidence), key=_evidence_richness) if duplicate_ids: ingest_flags.append(CLAUDE_DUPLICATE_MESSAGE_ID_INGEST_FLAG) @@ -1094,6 +1152,11 @@ def _evidence_message_type(evidence: _ClaudeMessageEvidence) -> MessageType: timestamp=evidence.timestamp, blocks=evidence.blocks, parent_message_provider_id=evidence.parent_message_provider_id, + owner_coordinate=MessageOwnerCoordinate( + stable_key=evidence.owner_stable_key, + position=position_by_id[evidence.evidence_key], + variant_index=variant_index_by_id[evidence.evidence_key], + ), position=position_by_id[evidence.evidence_key], branch_index=branch_index_by_id.get(evidence.evidence_key, 0), variant_index=variant_index_by_id[evidence.evidence_key], @@ -1125,7 +1188,17 @@ def _evidence_message_type(evidence: _ClaudeMessageEvidence) -> MessageType: attachments = _merge_attachment_rows( [ - attachment.model_copy(update={"message_position": position_by_id[evidence.evidence_key]}) + attachment.model_copy( + update={ + "message_position": position_by_id[evidence.evidence_key], + "message_variant_index": variant_index_by_id[evidence.evidence_key], + "owner_coordinate": MessageOwnerCoordinate( + stable_key=evidence.owner_stable_key, + position=position_by_id[evidence.evidence_key], + variant_index=variant_index_by_id[evidence.evidence_key], + ), + } + ) for evidence in emitted for attachment in evidence.attachments ] diff --git a/polylogue/sources/parsers/codex.py b/polylogue/sources/parsers/codex.py index d236c0ab7d..58c5ca2da7 100644 --- a/polylogue/sources/parsers/codex.py +++ b/polylogue/sources/parsers/codex.py @@ -1607,7 +1607,7 @@ def _code_mode_exec_envelopes(records: Iterable[object]) -> dict[int, _CodexExec raw_tool_id = payload.get("call_id") or payload.get("id") tool_id = str(raw_tool_id) if raw_tool_id else None # polylogue-slshy: no positional fallback -- an empty id lets - # _message_comparison_id's content-anchor (role + timestamp) + # _message_revision_match_id's content-anchor (role + timestamp) # fallback run instead of a position-derived string that would # change identity when array order shifts across re-acquisitions. provider_message_id = str(payload.get("id") or raw_tool_id or "") diff --git a/polylogue/sources/parsers/drive.py b/polylogue/sources/parsers/drive.py index 7bf27f43bc..a8eb790cf2 100644 --- a/polylogue/sources/parsers/drive.py +++ b/polylogue/sources/parsers/drive.py @@ -9,7 +9,9 @@ from polylogue.archive.message.roles import Role from polylogue.archive.message.types import MessageType from polylogue.core.enums import Provider, TitleSource +from polylogue.core.hashing import hash_payload from polylogue.core.json import JSONDocument, json_document +from polylogue.core.message_owner import MessageOwnerCoordinate from polylogue.logging import get_logger from polylogue.sources.providers.gemini import GeminiMessage @@ -378,8 +380,34 @@ def parse_chunked_prompt(provider: Provider | str, payload: JSONDocument, fallba ): session_events.append(usage_event) chunk_attachments = _collect_chunk_attachments(chunk_obj, msg_id) + owner_evidence = sorted( + ( + attachment.provider_attachment_id, + attachment.provider_file_id, + attachment.provider_drive_id, + ) + for attachment in chunk_attachments + if attachment.provider_attachment_id or attachment.provider_file_id or attachment.provider_drive_id + ) + owner_stable_key = ( + "drive-owner-evidence:" + hash_payload({"attachments": [list(values) for values in owner_evidence]}) + if owner_evidence + else None + ) + owner_coordinate = MessageOwnerCoordinate( + stable_key=owner_stable_key, + position=message_position, + variant_index=0, + ) chunk_attachments = [ - attachment.model_copy(update={"message_position": message_position}) for attachment in chunk_attachments + attachment.model_copy( + update={ + "message_position": message_position, + "message_variant_index": 0, + "owner_coordinate": owner_coordinate, + } + ) + for attachment in chunk_attachments ] observed_timestamps.append(message_timestamp) used_typed_model = False @@ -428,6 +456,7 @@ def parse_chunked_prompt(provider: Provider | str, payload: JSONDocument, fallba variant_index=0, is_active_path=True, parent_message_provider_id=(_branch_parent_provider_id(chunk_obj) or branch_child_parents.get(msg_id)), + owner_coordinate=owner_coordinate, input_tokens=usage_fields["input_tokens"], output_tokens=usage_fields["output_tokens"], model_name=model_name, diff --git a/polylogue/sources/parsers/local_agent.py b/polylogue/sources/parsers/local_agent.py index a4809b7dd5..01207efe85 100644 --- a/polylogue/sources/parsers/local_agent.py +++ b/polylogue/sources/parsers/local_agent.py @@ -230,7 +230,7 @@ def _parse_gemini_message(item: object, *, index: int, position: int) -> ParsedM ) return ParsedMessage( # polylogue-slshy: no positional fallback -- empty id lets - # _message_comparison_id's content-anchor fallback run instead. + # _message_revision_match_id's content-anchor fallback run instead. provider_message_id=_string(record.get("id")) or "", role=gemini_role, text=text, diff --git a/polylogue/storage/attachment_relink.py b/polylogue/storage/attachment_relink.py index a98397f74a..d9f09a3797 100644 --- a/polylogue/storage/attachment_relink.py +++ b/polylogue/storage/attachment_relink.py @@ -40,6 +40,7 @@ from pathlib import Path from polylogue.logging import get_logger +from polylogue.pipeline.ids import attachment_message_owner_key, message_owner_resolution from polylogue.pipeline.services.ingest_worker import IngestRecordResult, SessionWritePayload, ingest_record from polylogue.sources.parsers.base import ParsedAttachment, ParsedMessage from polylogue.storage.runtime.raw.records import RawSessionRecord @@ -204,24 +205,16 @@ def _append_materialized_attachment_maps( index_conn: sqlite3.Connection, session_id: str, messages: list[ParsedMessage], -) -> tuple[dict[str, str], dict[int, str]]: +) -> dict[str, str]: resolved = _append_materialized_message_ids(index_conn, session_id, messages) - by_native_message_id: dict[str, str] = {} - by_message_position: dict[int, str] = {} - for message_index, message in enumerate(messages): + resolution = message_owner_resolution(messages) + by_owner_key: dict[str, str] = {} + for message_index, owner_key in enumerate(resolution.keys): message_id = resolved.get(message_index) - if message_id is None: + if message_id is None or owner_key in resolution.ambiguous_keys: continue - provider_message_id = message.provider_message_id - if provider_message_id: - by_native_message_id[str(provider_message_id)] = message_id - normalized = _normalized_message_native_id(message) - if normalized is not None: - by_native_message_id[normalized] = message_id - message_position = message.position - if message_position is not None: - by_message_position[int(message_position)] = message_id - return by_native_message_id, by_message_position + by_owner_key[owner_key] = message_id + return by_owner_key def _iter_raw_session_rows(source_conn: sqlite3.Connection, *, raw_row_limit: int | None) -> list[sqlite3.Row]: @@ -346,19 +339,18 @@ def _match_session_payload( session_id = payload.session_id messages = payload.parsed_session.messages position_offset = _next_message_position(index_conn, session_id) if payload.append_only else 0 - by_native_message_id, by_message_position = _attachment_message_id_maps( + owner_resolution, by_owner_key = _attachment_message_id_maps( session_id, messages, position_offset=position_offset, ) if payload.append_only: - materialized_by_native_id, materialized_by_position = _append_materialized_attachment_maps( + materialized_by_owner_key = _append_materialized_attachment_maps( index_conn, session_id, messages, ) - by_native_message_id.update(materialized_by_native_id) - by_message_position.update(materialized_by_position) + by_owner_key.update(materialized_by_owner_key) # Attachments are session-level (``ParsedSession.attachments``), each # linked to its owning message via ``message_provider_id`` -- mirroring # exactly how ``_write_attachments`` consumes them (write.py:_attachment_message_id_maps). @@ -366,11 +358,8 @@ def _match_session_payload( attachments_by_message: dict[str, list[ParsedAttachment]] = {} for attachment in payload.parsed_session.attachments: attachment_id = _attachment_id(session_id, attachment) - message_id = ( - by_native_message_id.get(attachment.message_provider_id) if attachment.message_provider_id else None - ) - if message_id is None and attachment.message_position is not None: - message_id = by_message_position.get(attachment.message_position) + owner_key = attachment_message_owner_key(attachment, owner_resolution) + message_id = by_owner_key.get(owner_key) if owner_key is not None else None if message_id is None: if attachment_id in pending: ineligible_reasons.setdefault( diff --git a/polylogue/storage/raw_authority.py b/polylogue/storage/raw_authority.py index f6ed824f6d..692f5b5905 100644 --- a/polylogue/storage/raw_authority.py +++ b/polylogue/storage/raw_authority.py @@ -26,7 +26,7 @@ from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier from polylogue.storage.sqlite.migration_runner import validate_migration_backup_manifest -RAW_AUTHORITY_PARSER_FINGERPRINT = "revision-membership-v3" +RAW_AUTHORITY_PARSER_FINGERPRINT = "revision-membership-v4" #: Fingerprints previously stamped by ``RAW_AUTHORITY_PARSER_FINGERPRINT`` #: whose classification semantics are known to have been superseded by a @@ -41,7 +41,9 @@ #: *quiescence* gate (``uncensused_historical_revision_raw_ids``) accepts any #: known fingerprint (current or superseded) so a bump does not force a full #: archive re-census -- see that function's docstring. -SUPERSEDED_MEMBERSHIP_FINGERPRINTS = frozenset({"revision-membership-v1", "revision-membership-v2"}) +SUPERSEDED_MEMBERSHIP_FINGERPRINTS = frozenset( + {"revision-membership-v1", "revision-membership-v2", "revision-membership-v3"} +) RAW_AUTHORITY_CENSUS_QUERY_PREFIX = "polylogue://raw-authority-census/" RAW_AUTHORITY_DETAIL_QUERY_PREFIX = "polylogue://raw-authority-detail/" RAW_AUTHORITY_DETAIL_CHUNK_CHARS = 16_384 diff --git a/polylogue/storage/sqlite/archive_tiers/write.py b/polylogue/storage/sqlite/archive_tiers/write.py index a84449b4da..a0686975de 100644 --- a/polylogue/storage/sqlite/archive_tiers/write.py +++ b/polylogue/storage/sqlite/archive_tiers/write.py @@ -34,6 +34,7 @@ from polylogue.core.sources import origin_from_provider from polylogue.core.timestamps import parse_timestamp from polylogue.logging import get_logger +from polylogue.pipeline.ids import MessageOwnerResolution, attachment_message_owner_key, message_owner_resolution from polylogue.sources.origin_specs import lowering_fingerprint, parser_fingerprint_for_origin from polylogue.sources.parsers.base import ( ParsedAttachment, @@ -3341,20 +3342,20 @@ def _attachment_message_id_maps( *, position_offset: int = 0, duplicate_native_ids: frozenset[str] | None = None, -) -> tuple[dict[str, str], dict[int, str]]: +) -> tuple[MessageOwnerResolution, dict[str, str]]: """Build the authoritative attachment-owner lookup maps. - Native message ids are usable only when unique after the same SQLite - normalization used by the writer. Message positions remain the fallback - for id-less attachments and for attachments whose native id is ambiguous. - Keep this shared with repair/relink paths so they cannot invent a weaker - ownership rule than the production write. + The first result is the shared private owner-resolution contract. The + second maps its resolved owner keys to stored message ids, including the + full ``(position, variant_index)`` coordinate and any reorder-stable + parser evidence. Keep this shared with repair/relink paths so they cannot + invent a weaker ownership rule than the production write. """ duplicates = duplicate_native_ids if duplicate_native_ids is not None else _duplicate_message_native_ids(messages) - by_native_message_id: dict[str, str] = {} - for fallback_position, message in enumerate(messages): - normalized = _normalized_message_native_id(message) - if normalized is None or normalized in duplicates: + resolution = message_owner_resolution(messages) + by_owner_key: dict[str, str] = {} + for fallback_position, (message, owner_key) in enumerate(zip(messages, resolution.keys, strict=True)): + if owner_key in resolution.ambiguous_keys: continue message_id = _message_id( session_id, @@ -3363,21 +3364,8 @@ def _attachment_message_id_maps( position_offset=position_offset, duplicate_native_ids=duplicates, ) - by_native_message_id[normalized] = message_id - if message.provider_message_id != normalized: - by_native_message_id[message.provider_message_id] = message_id - by_message_position = { - message.position: _message_id( - session_id, - message, - fallback_position, - position_offset=position_offset, - duplicate_native_ids=duplicates, - ) - for fallback_position, message in enumerate(messages) - if message.position is not None - } - return by_native_message_id, by_message_position + by_owner_key[owner_key] = message_id + return resolution, by_owner_key def _next_message_position(conn: sqlite3.Connection, session_id: str) -> int: @@ -3401,7 +3389,7 @@ def _write_attachments( preacquired_blobs: dict[int, tuple[bytes | None, int, str]] | None = None, ) -> None: attachments = tuple(attachments) - by_native_message_id, by_message_position = _attachment_message_id_maps( + owner_resolution, by_owner_key = _attachment_message_id_maps( session_id, messages, position_offset=position_offset, @@ -3411,11 +3399,8 @@ def _write_attachments( resolved_message_ids: dict[int, str] = {} attachments_by_message: defaultdict[str, list[ParsedAttachment]] = defaultdict(list) for attachment in attachments: - message_id = ( - by_native_message_id.get(attachment.message_provider_id) if attachment.message_provider_id else None - ) - if message_id is None and attachment.message_position is not None: - message_id = by_message_position.get(attachment.message_position) + owner_key = attachment_message_owner_key(attachment, owner_resolution) + message_id = by_owner_key.get(owner_key) if owner_key is not None else None if message_id is not None: resolved_message_ids[id(attachment)] = message_id attachments_by_message[message_id].append(attachment) @@ -3523,7 +3508,7 @@ def _write_attachments( tuple(sorted(affected_attachment_ids)), ) # polylogue-w06b: a full-replace re-ingest (or a re-ingest whose attachment - # can no longer be matched to a message via `by_native_message_id`, e.g. + # can no longer be matched to a message via the shared owner key, e.g. # the owning message became a duplicate-native-id exclusion or dropped # out of this ingest's message set) drops a previously-written # attachment_refs row for `refresh_attachment_ids` without this From ee14475e82851fbfa72b881b55fe13820c714fcd Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 9 Aug 2026 04:51:48 +0200 Subject: [PATCH 11/19] test(identity): cover message owner roundout --- ...test_message_identity_position_fallback.py | 56 ++--- .../sources/test_claude_web_normalization.py | 238 ++++++++++++++++++ 2 files changed, 264 insertions(+), 30 deletions(-) diff --git a/tests/unit/pipeline/test_message_identity_position_fallback.py b/tests/unit/pipeline/test_message_identity_position_fallback.py index ffcea900fd..0461730ead 100644 --- a/tests/unit/pipeline/test_message_identity_position_fallback.py +++ b/tests/unit/pipeline/test_message_identity_position_fallback.py @@ -1,4 +1,4 @@ -"""message_identity_hash's fallback for id-less messages must not read array position. +"""Revision-match identity and private owner coordinates must not read array position. polylogue-gysk3: the attachment-class bug (a synthetic identity seeded partly by array index, unstable across export vintages that reorder/insert array @@ -6,23 +6,20 @@ excluding the (real-or-synthetic) attachment id from identity entirely and deriving it from stable content fields instead (message_id, name, mime_type). -`message_identity_hash` has no separate field to fall back to -- a message's -provider id (or its absence) IS the sole identity input by construction. Its -one addressable-within-`pipeline/ids.py` instance is the local fallback that -used to fire when a parser could not populate `provider_message_id`: -previously `f"msg-{index}"`, positionally-derived exactly like the old -attachment synthetic id. This file proves the fix: two parses of the same -id-less message in a different array position must resolve to the same -message-content identity when a provider timestamp is available to anchor -on, instead of silently comparing wrong pairs as if their (position-shifted) -fallback ids matched. +The revision-match identity is now separate from mutable message content: +timestamped id-less edits share one axis while their content hash changes. +Private attachment ownership uses the typed coordinate contract and fails +closed for indistinguishable duplicate occurrences. """ from __future__ import annotations +import pytest + from polylogue.archive.message.roles import Role from polylogue.archive.session_revision_membership import MembershipRevision, classify_membership_revisions from polylogue.core.enums import Provider +from polylogue.core.message_owner import MessageOwnerAmbiguityError from polylogue.pipeline.ids import session_revision_projection from polylogue.sources.parsers.base import ParsedAttachment, ParsedMessage, ParsedSession @@ -74,6 +71,20 @@ def test_id_less_messages_with_distinct_timestamps_do_not_collide() -> None: assert len(projection.message_contents) == 2 +def test_timestamped_idless_edit_shares_revision_identity_but_changes_content() -> None: + older = _id_less("assistant", "before", "2024-01-01T00:01:00Z") + edited = _id_less("assistant", "after", "2024-01-01T00:01:00Z") + + old_projection = session_revision_projection(_session([older])) + edited_projection = session_revision_projection(_session([edited])) + + old_identity, old_content, _ = next(iter(old_projection.message_contents)) + edited_identity, edited_content, _ = next(iter(edited_projection.message_contents)) + assert old_identity == edited_identity + assert old_content != edited_content + assert old_projection.session_hash != edited_projection.session_hash + + def test_id_less_and_timestamp_less_message_uses_content_anchor() -> None: bare = _id_less("user", "x", None) keyed = _with_id("m1", "assistant", "y", "2024-01-01T00:00:00Z") @@ -100,14 +111,8 @@ def test_timestamp_less_idless_duplicates_preserve_unordered_multiplicity() -> N assert not classification.ambiguous_raw_ids -def test_duplicate_idless_attachment_owner_reassignment_changes_hash() -> None: - """Duplicate content anchors still distinguish position-linked owners. - - The messages remain the same content in both revisions. Only the stable - transport owner coordinate of one attachment moves from occurrence zero - to occurrence one. A content-only owner anchor would make this a false - re-ingest skip. - """ +def test_indistinguishable_duplicate_idless_attachment_owner_fails_closed() -> None: + """A duplicate with no stable evidence cannot receive guessed ownership.""" repeated = [ _id_less("assistant", "repeat", "2024-01-01T00:00:00Z").model_copy(update={"position": position}) for position in (0, 1) @@ -119,14 +124,5 @@ def test_duplicate_idless_attachment_owner_reassignment_changes_hash() -> None: name="note.txt", mime_type="text/plain", ) - moved = attachment.model_copy(update={"message_position": 1}) - - first = session_revision_projection(_session(repeated, [attachment])) - second = session_revision_projection(_session(repeated, [moved])) - reordered = session_revision_projection(_session(list(reversed(repeated)), [attachment])) - - assert first.message_contents == second.message_contents - assert first.message_contents == reordered.message_contents - assert first.attachment_identities == reordered.attachment_identities - assert first.attachment_identities != second.attachment_identities - assert first.session_hash != second.session_hash + with pytest.raises(MessageOwnerAmbiguityError): + session_revision_projection(_session(repeated, [attachment])) diff --git a/tests/unit/sources/test_claude_web_normalization.py b/tests/unit/sources/test_claude_web_normalization.py index 481f614871..62ee57fb7b 100644 --- a/tests/unit/sources/test_claude_web_normalization.py +++ b/tests/unit/sources/test_claude_web_normalization.py @@ -4,10 +4,14 @@ import copy import json +import sqlite3 from pathlib import Path from typing import Any +import pytest + from polylogue.core.enums import Provider +from polylogue.core.message_owner import MessageOwnerAmbiguityError from polylogue.pipeline.ids import session_revision_projection from polylogue.sources.dispatch import detect_provider, parse_payload from polylogue.sources.parsers.base import ParsedSession @@ -16,6 +20,7 @@ NATIVE_BROWSER_CAPTURE_INGEST_FLAG, ) from polylogue.sources.parsers.claude.common import CLAUDE_LINEAGE_CYCLE_INGEST_FLAG +from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore from polylogue.storage.sqlite.connection import open_connection from tests.infra.pipeline_roundtrip import parse_payload_roundtrip, write_and_hydrate from tests.infra.storage_records import db_setup @@ -513,6 +518,239 @@ def test_claude_same_timestamp_idless_rows_keep_private_association_keys() -> No } +def _duplicate_idless_owner_payload(order: tuple[str, ...]) -> dict[str, Any]: + rows = { + "first": { + "sender": "assistant", + "text": "same duplicate turn", + "created_at": "2026-07-01T10:00:00Z", + "files": [{"file_uuid": "owner-file-first", "file_name": "first.txt", "file_type": "text/plain"}], + }, + "second": { + "sender": "assistant", + "text": "same duplicate turn", + "created_at": "2026-07-01T10:00:00Z", + "files": [{"file_uuid": "owner-file-second", "file_name": "second.txt", "file_type": "text/plain"}], + }, + } + return {"uuid": "claude-idless-owner-evidence", "chat_messages": [copy.deepcopy(rows[key]) for key in order]} + + +def test_claude_duplicate_idless_owner_evidence_survives_parser_reorder() -> None: + """Real parser reorder: provider file evidence, not assigned position, owns each attachment.""" + forward = _parse_real_route(_duplicate_idless_owner_payload(("first", "second"))) + reordered = _parse_real_route(_duplicate_idless_owner_payload(("second", "first"))) + + assert [message.provider_message_id for message in forward.messages] == ["", ""] + assert [message.provider_message_id for message in reordered.messages] == ["", ""] + assert ( + session_revision_projection(forward).message_contents == session_revision_projection(reordered).message_contents + ) + assert ( + session_revision_projection(forward).attachment_identities + == session_revision_projection(reordered).attachment_identities + ) + + def owners(session: ParsedSession) -> dict[str, tuple[str | None, int | None, int | None]]: + return { + attachment.name or "": ( + attachment.owner_coordinate.stable_key if attachment.owner_coordinate else None, + attachment.message_position, + attachment.message_variant_index, + ) + for attachment in session.attachments + } + + assert owners(forward).keys() == owners(reordered).keys() + assert {key: value[0] for key, value in owners(forward).items()} == { + key: value[0] for key, value in owners(reordered).items() + } + assert owners(forward)["first.txt"][1:] != owners(reordered)["first.txt"][1:] + + +def test_claude_indistinguishable_duplicate_idless_owner_is_typed_ambiguity() -> None: + rows = [ + { + "sender": "assistant", + "text": "same duplicate turn", + "created_at": "2026-07-01T10:00:00Z", + "files": [{"file_name": "same.txt", "file_type": "text/plain"}], + }, + { + "sender": "assistant", + "text": "same duplicate turn", + "created_at": "2026-07-01T10:00:00Z", + "files": [{"file_name": "same.txt", "file_type": "text/plain"}], + }, + ] + parsed = _parse_real_route({"uuid": "claude-idless-owner-ambiguous", "chat_messages": rows}) + + with pytest.raises(MessageOwnerAmbiguityError): + session_revision_projection(parsed) + + +def test_claude_timestamped_idless_edit_keeps_axis_and_persists_content(tmp_path: Path) -> None: + def payload(text: str) -> dict[str, Any]: + return { + "uuid": "claude-idless-edit", + "chat_messages": [ + { + "sender": "assistant", + "text": text, + "created_at": "2026-07-01T10:00:00Z", + } + ], + } + + older = _parse_real_route(payload("before")) + edited = _parse_real_route(payload("after")) + old_projection = session_revision_projection(older) + edited_projection = session_revision_projection(edited) + old_identity, old_content, _ = next(iter(old_projection.message_contents)) + edited_identity, edited_content, _ = next(iter(edited_projection.message_contents)) + + assert old_identity == edited_identity + assert old_content != edited_content + assert old_projection.session_hash != edited_projection.session_hash + + root = tmp_path / "archive" + with ArchiveStore(root) as facade: + first = facade.write_raw_and_parsed_result( + older, + payload=json.dumps(payload("before")).encode(), + source_path="/tmp/claude-idless-edit-before.json", + acquired_at_ms=1_775_000_000_000, + ) + second = facade.write_raw_and_parsed_result( + edited, + payload=json.dumps(payload("after")).encode(), + source_path="/tmp/claude-idless-edit-after.json", + acquired_at_ms=1_775_000_000_001, + ) + + conn = sqlite3.connect(f"file:{root / 'index.db'}?mode=ro", uri=True) + try: + stored_text = conn.execute( + """ + SELECT b.search_text + FROM blocks AS b + JOIN messages AS m ON m.message_id = b.message_id + WHERE m.session_id = ? AND b.block_type = 'text' + """, + ("claude-ai-export:claude-idless-edit",), + ).fetchone()[0] + finally: + conn.close() + + assert first.content_changed is True + assert second.content_changed is True + assert second.counts["skipped_sessions"] == 0 + assert stored_text == "after" + + +def _duplicate_native_variant_payload(*, swapped: bool = False) -> dict[str, Any]: + files = (("att-variant-a", "variant-a.txt"), ("att-variant-b", "variant-b.txt")) + if swapped: + files = files[::-1] + return { + "uuid": "claude-duplicate-native-owners", + "chat_messages": [ + { + "uuid": "duplicate-native", + "sender": "assistant", + "text": "duplicate native first", + "created_at": "2026-07-01T10:00:00Z", + "position": 0, + "variant_index": 0, + "files": [{"id": files[0][0], "file_name": files[0][1], "file_type": "text/plain"}], + }, + { + "uuid": "duplicate-native", + "sender": "assistant", + "text": "duplicate native second", + "created_at": "2026-07-01T10:00:00Z", + "position": 0, + "variant_index": 1, + "files": [{"id": files[1][0], "file_name": files[1][1], "file_type": "text/plain"}], + }, + ], + } + + +def test_claude_duplicate_native_attachment_owner_keeps_variant_coordinate( + workspace_env: dict[str, Path], +) -> None: + payload = _duplicate_native_variant_payload() + raw_bytes = json.dumps(payload).encode() + db_path = db_setup(workspace_env) + + with open_connection(db_path) as conn: + parsed = _parse_real_route(payload) + assert [message.provider_message_id for message in parsed.messages] == ["duplicate-native"] * 2 + assert {attachment.message_variant_index for attachment in parsed.attachments} == {0, 1} + roundtrip = parse_payload_roundtrip("claude-ai", raw_bytes, unique_id="claude-duplicate-native-owners") + write_and_hydrate(roundtrip, conn) + rows = [ + tuple(row) + for row in conn.execute( + """ + SELECT n.native_id, m.variant_index + FROM attachment_native_ids AS n + JOIN attachment_refs AS r ON r.ref_id = n.ref_id + JOIN messages AS m ON m.message_id = r.message_id + WHERE m.session_id = ? AND n.id_kind = 'attachment' + ORDER BY n.native_id + """, + ("claude-ai-export:claude-duplicate-native-owners",), + ).fetchall() + ] + + assert rows == [("att-variant-a", 0), ("att-variant-b", 1)] + + +def test_claude_duplicate_native_owner_move_changes_real_ingest_hash_and_owner( + tmp_path: Path, +) -> None: + first_payload = _duplicate_native_variant_payload() + second_payload = _duplicate_native_variant_payload(swapped=True) + root = tmp_path / "archive" + + with ArchiveStore(root) as facade: + first = facade.write_raw_and_parsed_result( + _parse_real_route(first_payload), + payload=json.dumps(first_payload).encode(), + source_path="/tmp/duplicate-native-first.json", + acquired_at_ms=1_775_000_000_000, + ) + second = facade.write_raw_and_parsed_result( + _parse_real_route(second_payload), + payload=json.dumps(second_payload).encode(), + source_path="/tmp/duplicate-native-second.json", + acquired_at_ms=1_775_000_000_001, + ) + + conn = sqlite3.connect(f"file:{root / 'index.db'}?mode=ro", uri=True) + try: + moved = conn.execute( + """ + SELECT n.native_id, m.variant_index + FROM attachment_native_ids AS n + JOIN attachment_refs AS r ON r.ref_id = n.ref_id + JOIN messages AS m ON m.message_id = r.message_id + WHERE m.session_id = ? AND n.id_kind = 'attachment' + ORDER BY n.native_id + """, + ("claude-ai-export:claude-duplicate-native-owners",), + ).fetchall() + finally: + conn.close() + + assert first.content_changed is True + assert second.content_changed is True + assert second.counts["skipped_sessions"] == 0 + assert moved == [("att-variant-a", 1), ("att-variant-b", 0)] + + def test_authenticated_browser_capture_uses_native_payload_and_enriches_attachment() -> None: """Mutations: detector-order theft or attachment replacement loss must fail.""" direct = _parse_real_route(_native_claude_payload()) From 8e934cc0fc7be64921eee1b3bf7614a589df124f Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 9 Aug 2026 07:50:43 +0200 Subject: [PATCH 12/19] fix(identity): close idless owner collision laws Problem: timestamped idless sibling edits could change revision identity when sibling count changed, while duplicate physical owner coordinates and repeated Claude positions could resolve by last-wins behavior. Whitespace-only native IDs also followed a different revision law than persistence.\n\nWhat changed: keep revision axes separate from private owner keys, compare mutable timestamped idless multiplicity by identity, normalize whitespace native IDs, reject duplicate physical owner coordinates, and retain attachment evidence in repeated-position Claude owner keys. Add real parser, projection, and writer regressions.\n\nCompatibility/migration: public native IDs remain unchanged. No durable tier or index schema changes. --- .../archive/session_revision_membership.py | 47 ++++++++-- polylogue/pipeline/ids.py | 31 +++++-- polylogue/sources/parsers/claude/common.py | 15 ++-- ...test_message_identity_position_fallback.py | 88 ++++++++++++++++++- .../sources/test_claude_web_normalization.py | 42 ++++++++- 5 files changed, 203 insertions(+), 20 deletions(-) diff --git a/polylogue/archive/session_revision_membership.py b/polylogue/archive/session_revision_membership.py index 474f04e078..a326848f9b 100644 --- a/polylogue/archive/session_revision_membership.py +++ b/polylogue/archive/session_revision_membership.py @@ -89,24 +89,57 @@ def _message_identities(contents: frozenset[MessageContent]) -> frozenset[bytes] return frozenset(identity for identity, _content, _multiplicity in contents) -def _message_axis_relation(contents_a: frozenset[MessageContent], contents_b: frozenset[MessageContent]) -> _Relation: +def _message_axis_relation( + contents_a: frozenset[MessageContent], + contents_b: frozenset[MessageContent], + *, + mutable_identities: frozenset[bytes] = frozenset(), +) -> _Relation: """Compare message content as an unordered multiset. Reordering remains equivalent, while a repeated id-less message is an additional persisted turn rather than a duplicate that set semantics can - erase. A shared identity carrying different content remains a conflict. + erase. Native shared identities carrying different content remain a + conflict. Timestamped id-less identities are deliberately mutable: their + content hash still triggers archive replacement, while their revision + membership axis must not turn an edit into a false fork merely because a + sibling was added or changed. """ counts_a = {(identity, content): multiplicity for identity, content, multiplicity in contents_a} counts_b = {(identity, content): multiplicity for identity, content, multiplicity in contents_b} + identity_counts_a: dict[bytes, int] = {} + identity_counts_b: dict[bytes, int] = {} + for (identity, _content), multiplicity in counts_a.items(): + identity_counts_a[identity] = identity_counts_a.get(identity, 0) + multiplicity + for (identity, _content), multiplicity in counts_b.items(): + identity_counts_b[identity] = identity_counts_b.get(identity, 0) + multiplicity identities_a = _message_identities(contents_a) identities_b = _message_identities(contents_b) for identity in identities_a & identities_b: content_values_a = {content for candidate_identity, content in counts_a if candidate_identity == identity} content_values_b = {content for candidate_identity, content in counts_b if candidate_identity == identity} + if identity in mutable_identities: + continue if content_values_a != content_values_b: return "conflict" - a_richer = bool(identities_a - identities_b) or any(count > counts_b.get(key, 0) for key, count in counts_a.items()) - b_richer = bool(identities_b - identities_a) or any(count > counts_a.get(key, 0) for key, count in counts_b.items()) + + def _has_extra( + side: dict[tuple[bytes, bytes], int], + other: dict[tuple[bytes, bytes], int], + side_identity_counts: dict[bytes, int], + other_identity_counts: dict[bytes, int], + ) -> bool: + for key, count in side.items(): + identity = key[0] + if identity in mutable_identities: + if side_identity_counts.get(identity, 0) > other_identity_counts.get(identity, 0): + return True + elif count > other.get(key, 0): + return True + return False + + a_richer = bool(identities_a - identities_b) or _has_extra(counts_a, counts_b, identity_counts_a, identity_counts_b) + b_richer = bool(identities_b - identities_a) or _has_extra(counts_b, counts_a, identity_counts_b, identity_counts_a) if a_richer and b_richer: return "conflict" if a_richer: @@ -183,7 +216,11 @@ def _relation(a: SessionRevisionProjection, b: SessionRevisionProjection) -> _Re any single axis already is. """ axes = ( - _message_axis_relation(a.message_contents, b.message_contents), + _message_axis_relation( + a.message_contents, + b.message_contents, + mutable_identities=a.mutable_message_identities | b.mutable_message_identities, + ), _axis_relation(a.attachment_identities, a.attachment_contents, b.attachment_identities, b.attachment_contents), _axis_relation( _identities(a.event_contents), a.event_contents, _identities(b.event_contents), b.event_contents diff --git a/polylogue/pipeline/ids.py b/polylogue/pipeline/ids.py index 901564f6fb..ce836cab11 100644 --- a/polylogue/pipeline/ids.py +++ b/polylogue/pipeline/ids.py @@ -103,6 +103,10 @@ class SessionRevisionProjection: attachment_contents: frozenset[tuple[bytes, bytes]] event_hashes: tuple[bytes, ...] event_contents: frozenset[tuple[bytes, bytes]] + # Timestamped id-less messages intentionally share a revision axis even + # when their mutable content changes. Native-id axes retain strict content + # conflict semantics in ``session_revision_membership``. + mutable_message_identities: frozenset[bytes] = frozenset() def _normalize_nested_for_hash(value: object) -> object: @@ -271,8 +275,9 @@ class the attachment identity fix (polylogue-hith/-d8al) removed for a reaching this function. Parser-local occurrence keys may use position, but they are not persisted as ``provider_message_id``. """ - if message.provider_message_id: - return message.provider_message_id.strip() + native_id = message.provider_message_id.strip() + if native_id: + return native_id payload: dict[str, JSONValue] = { "role": str(message.role), "timestamp": _normalize_for_hash(message.timestamp), @@ -290,6 +295,7 @@ class MessageOwnerResolution: keys: tuple[str, ...] by_physical_coordinate: Mapping[tuple[int, int], str] + ambiguous_physical_coordinates: frozenset[tuple[int, int]] by_stable_key: Mapping[str, str] ambiguous_keys: frozenset[str] unique_provider_keys: Mapping[str, str] @@ -342,10 +348,13 @@ def message_owner_resolution(messages: list[ParsedMessage]) -> MessageOwnerResol key_counts = Counter(keys) ambiguous_keys = frozenset(key for key, count in key_counts.items() if count > 1) + physical_counts = Counter( + coordinate.physical_key for coordinate in coordinates if coordinate.physical_key is not None + ) by_physical_coordinate = { coordinate.physical_key: key for coordinate, key in zip(coordinates, keys, strict=True) - if coordinate.physical_key is not None + if coordinate.physical_key is not None and physical_counts[coordinate.physical_key] == 1 } by_stable_key = { coordinate.stable_key: key @@ -363,6 +372,9 @@ def message_owner_resolution(messages: list[ParsedMessage]) -> MessageOwnerResol return MessageOwnerResolution( keys=tuple(keys), by_physical_coordinate=by_physical_coordinate, + ambiguous_physical_coordinates=frozenset( + coordinate for coordinate, count in physical_counts.items() if count > 1 + ), by_stable_key=by_stable_key, ambiguous_keys=ambiguous_keys, unique_provider_keys=provider_keys, @@ -393,6 +405,8 @@ def attachment_message_owner_key(attachment: ParsedAttachment, resolution: Messa if coordinate.stable_key in resolution.by_stable_key: return resolution.by_stable_key[coordinate.stable_key] if coordinate.physical_key is not None: + if coordinate.physical_key in resolution.ambiguous_physical_coordinates: + raise MessageOwnerAmbiguityError(f"attachment owner coordinate is duplicated: {coordinate.physical_key!r}") key = resolution.by_physical_coordinate.get(coordinate.physical_key) if key is not None: if key in resolution.ambiguous_keys: @@ -651,7 +665,10 @@ def _session_hash_components( payload independently -- pure sharing of an already-pure computation. """ owner_resolution = message_owner_resolution(convo.messages) - message_comparison_ids = list(owner_resolution.keys) + # Private owner keys may use duplicate-occurrence evidence. Revision + # identity must remain the intrinsic role/timestamp axis for timestamped + # id-less messages, independent of the sibling count in this acquisition. + message_comparison_ids = [_message_revision_match_id(message) for message in convo.messages] messages_payload = [ _message_hash_payload(message, comparison_id) for message, comparison_id in zip(convo.messages, message_comparison_ids, strict=True) @@ -748,10 +765,13 @@ def session_revision_projection(convo: ParsedSession) -> SessionRevisionProjecti ) message_content_counts: Counter[tuple[bytes, bytes]] = Counter() message_hashes: list[bytes] = [] - for payload in messages_payload: + mutable_message_identities: set[bytes] = set() + for message, payload in zip(convo.messages, messages_payload, strict=True): message_native_id = payload["id"] assert isinstance(message_native_id, str) # built as str above, never anything else identity = message_identity_hash(id=message_native_id) + if not message.provider_message_id.strip() and message.timestamp is not None: + mutable_message_identities.add(identity) content = bytes.fromhex(hash_payload(payload)) message_content_counts[(identity, content)] += 1 message_hashes.append(content) @@ -809,4 +829,5 @@ def session_revision_projection(convo: ParsedSession) -> SessionRevisionProjecti attachment_contents=frozenset(attachment_contents), event_hashes=tuple(event_hashes), event_contents=frozenset(event_contents), + mutable_message_identities=frozenset(mutable_message_identities), ) diff --git a/polylogue/sources/parsers/claude/common.py b/polylogue/sources/parsers/claude/common.py index f9354e1fa0..0b8db95f15 100644 --- a/polylogue/sources/parsers/claude/common.py +++ b/polylogue/sources/parsers/claude/common.py @@ -644,14 +644,13 @@ def _owner_stable_key( block_ids = sorted(block.tool_id for block in blocks if block.tool_id) if block_ids: evidence["tool_ids"] = block_ids - if explicit_position is None and explicit_branch_index is None and explicit_variant_index is None: - attachment_ids = sorted( - (attachment.provider_attachment_id, attachment.provider_file_id, attachment.provider_drive_id) - for attachment in attachments - if attachment.provider_attachment_id or attachment.provider_file_id or attachment.provider_drive_id - ) - if attachment_ids: - evidence["attachment_ids"] = [list(values) for values in attachment_ids] + attachment_ids = sorted( + (attachment.provider_attachment_id, attachment.provider_file_id, attachment.provider_drive_id) + for attachment in attachments + if attachment.provider_attachment_id or attachment.provider_file_id or attachment.provider_drive_id + ) + if attachment_ids: + evidence["attachment_ids"] = [list(values) for values in attachment_ids] if not evidence: return None return f"claude-owner-evidence:{hash_payload(evidence)}" diff --git a/tests/unit/pipeline/test_message_identity_position_fallback.py b/tests/unit/pipeline/test_message_identity_position_fallback.py index 0461730ead..997a73f17a 100644 --- a/tests/unit/pipeline/test_message_identity_position_fallback.py +++ b/tests/unit/pipeline/test_message_identity_position_fallback.py @@ -17,7 +17,7 @@ import pytest from polylogue.archive.message.roles import Role -from polylogue.archive.session_revision_membership import MembershipRevision, classify_membership_revisions +from polylogue.archive.session_revision_membership import MembershipRevision, _relation, classify_membership_revisions from polylogue.core.enums import Provider from polylogue.core.message_owner import MessageOwnerAmbiguityError from polylogue.pipeline.ids import session_revision_projection @@ -85,6 +85,92 @@ def test_timestamped_idless_edit_shares_revision_identity_but_changes_content() assert old_projection.session_hash != edited_projection.session_hash +def test_timestamped_idless_sibling_edit_is_not_a_membership_conflict() -> None: + """A same-role, same-timestamp sibling set has one mutable revision axis. + + The content hash still changes, so the real writer replaces the stored + payload. Membership must compare the axis cardinality instead of treating + the sibling's changed text as a contradictory native identity. + """ + older = _session( + [ + _id_less("assistant", "first", "2024-01-01T00:01:00Z"), + _id_less("assistant", "second", "2024-01-01T00:01:00Z"), + ] + ) + edited = _session( + [ + _id_less("assistant", "edited first", "2024-01-01T00:01:00Z"), + _id_less("assistant", "second", "2024-01-01T00:01:00Z"), + ] + ) + older_projection = session_revision_projection(older) + edited_projection = session_revision_projection(edited) + + assert older_projection.session_hash != edited_projection.session_hash + assert _relation(older_projection, edited_projection) == "equal" + + +def test_whitespace_native_id_uses_the_same_revision_axis_as_missing_id() -> None: + whitespace = _session( + [ParsedMessage(provider_message_id=" ", role=Role.ASSISTANT, text="same", timestamp="2024-01-01")] + ) + missing = _session([_id_less("assistant", "same", "2024-01-01")]) + + assert ( + session_revision_projection(whitespace).message_contents + == session_revision_projection(missing).message_contents + ) + + +def test_duplicate_physical_owner_coordinate_fails_closed_before_hashing_attachment() -> None: + messages = [ + _id_less("assistant", "first", None).model_copy(update={"position": 3, "variant_index": 0}), + _id_less("assistant", "second", None).model_copy(update={"position": 3, "variant_index": 0}), + ] + attachment = ParsedAttachment( + provider_attachment_id="attachment-1", + message_provider_id="", + message_position=3, + message_variant_index=0, + name="note.txt", + mime_type="text/plain", + ) + + with pytest.raises(MessageOwnerAmbiguityError): + session_revision_projection(_session(messages, [attachment])) + + +def test_duplicate_native_id_at_duplicate_physical_coordinate_fails_closed() -> None: + messages = [ + ParsedMessage( + provider_message_id="duplicate-native", + role=Role.ASSISTANT, + text="first", + position=3, + variant_index=0, + ), + ParsedMessage( + provider_message_id="duplicate-native", + role=Role.ASSISTANT, + text="second", + position=3, + variant_index=0, + ), + ] + attachment = ParsedAttachment( + provider_attachment_id="attachment-duplicate-native", + message_provider_id="duplicate-native", + message_position=3, + message_variant_index=0, + name="note.txt", + mime_type="text/plain", + ) + + with pytest.raises(MessageOwnerAmbiguityError): + session_revision_projection(_session(messages, [attachment])) + + def test_id_less_and_timestamp_less_message_uses_content_anchor() -> None: bare = _id_less("user", "x", None) keyed = _with_id("m1", "assistant", "y", "2024-01-01T00:00:00Z") diff --git a/tests/unit/sources/test_claude_web_normalization.py b/tests/unit/sources/test_claude_web_normalization.py index 62ee57fb7b..3fad9a7d95 100644 --- a/tests/unit/sources/test_claude_web_normalization.py +++ b/tests/unit/sources/test_claude_web_normalization.py @@ -19,7 +19,11 @@ DOM_FALLBACK_INGEST_FLAG, NATIVE_BROWSER_CAPTURE_INGEST_FLAG, ) -from polylogue.sources.parsers.claude.common import CLAUDE_LINEAGE_CYCLE_INGEST_FLAG +from polylogue.sources.parsers.claude.common import ( + CLAUDE_LINEAGE_CYCLE_INGEST_FLAG, + _message_attachments, + _owner_stable_key, +) from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore from polylogue.storage.sqlite.connection import open_connection from tests.infra.pipeline_roundtrip import parse_payload_roundtrip, write_and_hydrate @@ -708,6 +712,42 @@ def test_claude_duplicate_native_attachment_owner_keeps_variant_coordinate( assert rows == [("att-variant-a", 0), ("att-variant-b", 1)] +def test_claude_repeated_position_owner_evidence_keeps_attachment_identity() -> None: + """Attachment evidence remains distinct even before variant repair. + + This directly exercises the private owner-key constructor with repeated + explicit positions and omitted variants. Removing attachment evidence from + ``_owner_stable_key`` makes the two keys collide and the assertion fails. + """ + first = {"position": 4, "files": [{"id": "owner-a", "file_name": "a.txt"}]} + second = {"position": 4, "files": [{"id": "owner-b", "file_name": "b.txt"}]} + + first_attachments = _message_attachments(first, "") + second_attachments = _message_attachments(second, "") + first_key = _owner_stable_key( + first, + parent_message_provider_id=None, + explicit_position=4, + explicit_branch_index=None, + explicit_variant_index=None, + blocks=[], + attachments=first_attachments, + ) + second_key = _owner_stable_key( + second, + parent_message_provider_id=None, + explicit_position=4, + explicit_branch_index=None, + explicit_variant_index=None, + blocks=[], + attachments=second_attachments, + ) + + assert first_key is not None + assert second_key is not None + assert first_key != second_key + + def test_claude_duplicate_native_owner_move_changes_real_ingest_hash_and_owner( tmp_path: Path, ) -> None: From a1cf57001b3d5b52bd352a8ded1c934fe52fc14e Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 9 Aug 2026 07:50:53 +0200 Subject: [PATCH 13/19] fix(identity): hide synthetic parser message ids Problem: purely synthetic parser rows exposed content-derived synthetic-* values as public provider message IDs, making mutable text part of public identity.\n\nWhat changed: Codex reasoning and compaction rows, Grok responses, Antigravity markdown rows, and Claude Design synthetic segments now keep provider_message_id empty. Active-leaf selection preserves native IDs and uses physical position for synthetic rows. Claude Design attachments retain private owner coordinates.\n\nCompatibility/migration: native provider IDs and stored public message IDs remain unchanged. Synthetic rows use the existing positional storage fallback and require no schema migration. --- polylogue/sources/parsers/antigravity.py | 22 +++------- polylogue/sources/parsers/base_support.py | 14 ++++--- polylogue/sources/parsers/claude/ai_parser.py | 42 +++++++------------ polylogue/sources/parsers/codex.py | 19 +++------ polylogue/sources/parsers/grok.py | 12 +----- .../unit/sources/parsers/test_antigravity.py | 11 +++-- tests/unit/sources/parsers/test_grok.py | 10 ++--- .../sources/test_parsers_claude_design.py | 7 ++-- tests/unit/sources/test_parsers_codex.py | 5 ++- 9 files changed, 51 insertions(+), 91 deletions(-) diff --git a/polylogue/sources/parsers/antigravity.py b/polylogue/sources/parsers/antigravity.py index bca1ff58f7..567dcace05 100644 --- a/polylogue/sources/parsers/antigravity.py +++ b/polylogue/sources/parsers/antigravity.py @@ -28,7 +28,6 @@ ParsedSession, human_authored_override, mark_last_occurrence_as_active_leaf, - synthetic_message_id, ) _METADATA_SUFFIX = ".metadata.json" @@ -312,7 +311,9 @@ def parse_markdown_export( created_at=None, updated_at=summary.last_modified_time, messages=messages, - active_leaf_message_provider_id=messages[-1].provider_message_id if messages else None, + active_leaf_message_provider_id=messages[-1].provider_message_id + if messages and messages[-1].provider_message_id + else None, ) @@ -432,16 +433,9 @@ def _messages_from_markdown(markdown: str, cascade_id: str) -> list[ParsedMessag continue heading = section.group("title") role = Role.USER if heading == "User Input" else Role.ASSISTANT - provider_message_id = synthetic_message_id( - namespace=cascade_id, - role=role, - text=text, - timestamp=None, - kind=_message_kind(heading), - ) messages.append( ParsedMessage( - provider_message_id=provider_message_id, + provider_message_id="", role=role, text=text, blocks=[ParsedContentBlock(type=BlockType.TEXT, text=text)], @@ -468,13 +462,7 @@ def _messages_from_markdown(markdown: str, cascade_id: str) -> list[ParsedMessag return [] return [ ParsedMessage( - provider_message_id=synthetic_message_id( - namespace=cascade_id, - role=Role.ASSISTANT, - text=text, - timestamp=None, - kind="export", - ), + provider_message_id="", role=Role.ASSISTANT, text=text, blocks=[ParsedContentBlock(type=BlockType.TEXT, text=text)], diff --git a/polylogue/sources/parsers/base_support.py b/polylogue/sources/parsers/base_support.py index ca870de007..75d59d2563 100644 --- a/polylogue/sources/parsers/base_support.py +++ b/polylogue/sources/parsers/base_support.py @@ -73,12 +73,14 @@ def synthetic_message_id( namespace: str = "", kind: str = "", ) -> str: - """Build a reorder-stable id for a message with no provider id. - - This is reserved for parser-produced rows that are inherently synthetic, - such as an exported summary or a transcript section. Native-id fallback - paths must pass an empty string instead, so ``pipeline.ids`` can use its - role/timestamp/text comparison anchor. + """Build a private reorder-stable evidence key for a message with no provider id. + + This is reserved for parser-local evidence, such as an exported summary or + a transcript section. It must never be assigned to + ``ParsedMessage.provider_message_id``: public parser rows without a native + id use the empty string so storage applies the canonical fallback law. + Native-id fallback paths must pass an empty string instead, so + ``pipeline.ids`` can use its role/timestamp/text comparison anchor. """ seed = "\x1f".join((namespace, str(role), timestamp or "", text or "", kind)) return f"synthetic-{hash_text(seed)[:24]}" diff --git a/polylogue/sources/parsers/claude/ai_parser.py b/polylogue/sources/parsers/claude/ai_parser.py index 114d88a5c3..def5186d96 100644 --- a/polylogue/sources/parsers/claude/ai_parser.py +++ b/polylogue/sources/parsers/claude/ai_parser.py @@ -22,6 +22,7 @@ from polylogue.archive.message.roles import Role from polylogue.archive.message.types import MessageType from polylogue.core.enums import BlockType, MaterialOrigin, Provider, SessionKind, TitleSource +from polylogue.core.message_owner import MessageOwnerCoordinate from polylogue.logging import get_logger from ..base import ( @@ -33,7 +34,6 @@ attachment_from_meta, human_authored_override, mark_last_occurrence_as_active_leaf, - synthetic_message_id, ) from .common import ( _first_identity_field, @@ -270,17 +270,7 @@ def flush() -> None: block.text for block in current_blocks if block.type is BlockType.TEXT and not block.is_error and block.text ] segment_text = "\n".join(text_parts) if text_parts else None - provider_message_id = ( - message_uuid - if message_uuid and not has_interjection - else synthetic_message_id( - namespace=message_uuid, - role=Role.ASSISTANT, - text=segment_text, - timestamp=timestamp_str, - kind="claude-design-assistant-segment", - ) - ) + provider_message_id = message_uuid if message_uuid and not has_interjection else "" input_tokens = 0 if not first_segment_emitted and isinstance(turn_input_tokens, int): input_tokens = turn_input_tokens @@ -356,13 +346,7 @@ def flush() -> None: interjection_timestamp_value = ( str(interjection_timestamp) if isinstance(interjection_timestamp, str) else None ) - interjection_id = str(interjection_message.get("id") or "") or synthetic_message_id( - namespace=message_uuid, - role=interjection_role, - text=interjection_text_value, - timestamp=interjection_timestamp_value, - kind="claude-design-interjection", - ) + interjection_id = str(interjection_message.get("id") or "") messages.append( ParsedMessage( provider_message_id=interjection_id, @@ -432,16 +416,18 @@ def _design_user_message( sender_name = str(author_name) if isinstance(author_name, str) and author_name else None timestamp_str = str(timestamp) if isinstance(timestamp, str) and timestamp else None design_message_text = str(text) if isinstance(text, str) and text else None - message_uuid = str(raw_message.get("uuid") or content_payload.get("id") or "") or synthetic_message_id( - role=Role.USER, - text=design_message_text, - timestamp=timestamp_str, - kind="claude-design-user", - ) + message_uuid = str(raw_message.get("uuid") or content_payload.get("id") or "") for meta in raw_attachments if isinstance(raw_attachments, list) else []: attachment = _design_attachment_from_meta(meta, message_uuid) if attachment is not None: - attachments.append(attachment) + attachments.append( + attachment.model_copy( + update={ + "message_position": position, + "owner_coordinate": MessageOwnerCoordinate(position=position, variant_index=0), + } + ) + ) message = ParsedMessage( provider_message_id=message_uuid, role=Role.USER, @@ -517,7 +503,9 @@ def parse_design(payload: Mapping[str, object], fallback_id: str) -> ParsedSessi "claude-design %s: unrecognized message role %r, dropping message", resolved_session_id, role ) - active_leaf_message_provider_id = messages[-1].provider_message_id if messages else None + active_leaf_message_provider_id = ( + messages[-1].provider_message_id if messages and messages[-1].provider_message_id else None + ) messages = mark_last_occurrence_as_active_leaf(messages) title, title_source, title_ref, title_confidence = _resolve_claude_ai_title( diff --git a/polylogue/sources/parsers/codex.py b/polylogue/sources/parsers/codex.py index 58c5ca2da7..f906d61e93 100644 --- a/polylogue/sources/parsers/codex.py +++ b/polylogue/sources/parsers/codex.py @@ -34,7 +34,6 @@ content_blocks_from_segments, fill_linear_parent_chain, mark_last_occurrence_as_active_leaf, - synthetic_message_id, ) logger = get_logger(__name__) @@ -2030,12 +2029,7 @@ def _codex_reasoning_message( combined_text = "\n\n".join(t for t in (summary_text, content_text) if t) or None timestamp = _iso_or_none(_record_timestamp(record) or timestamp_fallback) return ParsedMessage( - provider_message_id=synthetic_message_id( - role=Role.ASSISTANT, - text=combined_text, - timestamp=timestamp, - kind="codex-reasoning", - ), + provider_message_id="", role=Role.ASSISTANT, text=combined_text, timestamp=timestamp, @@ -2507,12 +2501,7 @@ def _parse_records(records: Iterable[object], fallback_id: str, *, _reiterable: if summary_text: messages.append( ParsedMessage( - provider_message_id=synthetic_message_id( - role=Role.SYSTEM, - text=summary_text, - timestamp=timestamp, - kind="codex-compaction-summary", - ), + provider_message_id="", role=Role.SYSTEM, text=summary_text, timestamp=timestamp, @@ -2927,7 +2916,9 @@ def _parse_records(records: Iterable[object], fallback_id: str, *, _reiterable: commit_val = session_git.get("commit_hash") if isinstance(commit_val, str) and commit_val.strip(): git_commit_hash_typed = commit_val.strip() - active_leaf_message_provider_id = messages[-1].provider_message_id if messages else None + active_leaf_message_provider_id = ( + messages[-1].provider_message_id if messages and messages[-1].provider_message_id else None + ) messages = mark_last_occurrence_as_active_leaf(messages) # bd polylogue-ksgg: Codex rollout messages carry no parent-message # evidence at all (0% parented, 0 variant_index>0 rows) -- a strictly diff --git a/polylogue/sources/parsers/grok.py b/polylogue/sources/parsers/grok.py index 8fd8c18aa9..def4759dea 100644 --- a/polylogue/sources/parsers/grok.py +++ b/polylogue/sources/parsers/grok.py @@ -50,7 +50,6 @@ fill_linear_parent_chain, human_authored_override, mark_last_occurrence_as_active_leaf, - synthetic_message_id, ) _SENDER_ROLE: dict[str, Role] = { @@ -145,16 +144,9 @@ def parse_conversation(payload: Mapping[str, object], fallback_id: str) -> Parse continue grok_role = _role_for_sender(fields.get("sender")) timestamp = _timestamp_text(fields.get("create_time")) - provider_message_id = synthetic_message_id( - namespace=fallback_id, - role=grok_role, - text=text, - timestamp=timestamp, - kind="grok-response", - ) messages.append( ParsedMessage( - provider_message_id=provider_message_id, + provider_message_id="", role=grok_role, text=text, timestamp=timestamp, @@ -174,7 +166,7 @@ def parse_conversation(payload: Mapping[str, object], fallback_id: str) -> Parse ) ) - active_leaf_message_provider_id = messages[-1].provider_message_id if messages else None + active_leaf_message_provider_id = None messages = mark_last_occurrence_as_active_leaf(messages) # bd polylogue-ksgg: Grok exports carry no native conversation/message id # or parent evidence at all (see module docstring) -- a plain ordered diff --git a/tests/unit/sources/parsers/test_antigravity.py b/tests/unit/sources/parsers/test_antigravity.py index 2417af5d26..ada1d645b2 100644 --- a/tests/unit/sources/parsers/test_antigravity.py +++ b/tests/unit/sources/parsers/test_antigravity.py @@ -51,14 +51,13 @@ def test_parse_markdown_export_splits_known_sections() -> None: "The focused checks passed.", ] assert session.messages[0].blocks[0].type == BlockType.TEXT - assert session.messages[0].provider_message_id.startswith("synthetic-") - assert session.messages[1].provider_message_id.startswith("synthetic-") - assert session.messages[0].provider_message_id != session.messages[1].provider_message_id + assert session.messages[0].provider_message_id == "" + assert session.messages[1].provider_message_id == "" assert [message.position for message in session.messages] == [0, 1] assert [message.variant_index for message in session.messages] == [0, 0] assert [message.is_active_path for message in session.messages] == [True, True] assert [message.is_active_leaf for message in session.messages] == [False, True] - assert session.active_leaf_message_provider_id == session.messages[1].provider_message_id + assert session.active_leaf_message_provider_id is None def test_parse_markdown_export_reordering_keeps_synthetic_revision_identity() -> None: @@ -109,11 +108,11 @@ def test_parse_markdown_export_falls_back_to_single_export_message() -> None: session = parse_markdown_export(markdown, summary) assert [message.role for message in session.messages] == [Role.ASSISTANT] - assert session.messages[0].provider_message_id.startswith("synthetic-") + assert session.messages[0].provider_message_id == "" assert session.messages[0].text == "Unstructured transcript body." assert session.messages[0].position == 0 assert session.messages[0].is_active_leaf is True - assert session.active_leaf_message_provider_id == session.messages[0].provider_message_id + assert session.active_leaf_message_provider_id is None assert session.title_source is None diff --git a/tests/unit/sources/parsers/test_grok.py b/tests/unit/sources/parsers/test_grok.py index bfde8a2802..f85426a4c4 100644 --- a/tests/unit/sources/parsers/test_grok.py +++ b/tests/unit/sources/parsers/test_grok.py @@ -96,12 +96,11 @@ def test_parse_conversation_nested_response_shape() -> None: assert session.messages[0].text == "Why is my useEffect running twice?" assert session.messages[0].timestamp == "2024-04-01T19:33:21+00:00" assert session.messages[0].blocks[0].type is BlockType.TEXT - assert session.messages[0].provider_message_id.startswith("synthetic-") - assert session.messages[1].provider_message_id.startswith("synthetic-") - assert session.messages[0].provider_message_id != session.messages[1].provider_message_id + assert session.messages[0].provider_message_id == "" + assert session.messages[1].provider_message_id == "" assert [m.position for m in session.messages] == [0, 1] assert [m.is_active_leaf for m in session.messages] == [False, True] - assert session.active_leaf_message_provider_id == session.messages[1].provider_message_id + assert session.active_leaf_message_provider_id is None assert session.updated_at == session.messages[-1].timestamp @@ -203,7 +202,8 @@ def test_duplicate_grok_response_ids_keep_one_active_leaf() -> None: session = grok.parse_conversation(payload, "grok-duplicate") - assert session.messages[-2].provider_message_id == session.messages[-1].provider_message_id + assert session.messages[-2].provider_message_id == session.messages[-1].provider_message_id == "" + assert session.active_leaf_message_provider_id is None assert sum(message.is_active_leaf is True for message in session.messages) == 1 assert session.messages[-1].is_active_leaf is True diff --git a/tests/unit/sources/test_parsers_claude_design.py b/tests/unit/sources/test_parsers_claude_design.py index 2633bb2384..fe22c82457 100644 --- a/tests/unit/sources/test_parsers_claude_design.py +++ b/tests/unit/sources/test_parsers_claude_design.py @@ -312,11 +312,10 @@ def test_user_interjection_splits_the_assistant_turn_and_preserves_ordering() -> assert session.messages[2].provider_message_id == "interjection-1" # Segments have no independent provider id. Their content-derived ids are # stable across re-export ordering and do not encode the segment position. - assert session.messages[1].provider_message_id.startswith("synthetic-") - assert session.messages[3].provider_message_id.startswith("synthetic-") - assert session.messages[1].provider_message_id == session.messages[3].provider_message_id + assert session.messages[1].provider_message_id == "" + assert session.messages[3].provider_message_id == "" # active leaf is the true last message, not the raw turn's nominal end - assert session.active_leaf_message_provider_id == session.messages[3].provider_message_id + assert session.active_leaf_message_provider_id is None assert session.messages[-1].is_active_leaf is True assert sum(message.is_active_leaf is True for message in session.messages) == 1 diff --git a/tests/unit/sources/test_parsers_codex.py b/tests/unit/sources/test_parsers_codex.py index d449845008..4c09a717a8 100644 --- a/tests/unit/sources/test_parsers_codex.py +++ b/tests/unit/sources/test_parsers_codex.py @@ -676,7 +676,7 @@ def test_reasoning_summary_text_becomes_thinking_block(self) -> None: assert len(message.blocks) == 1 assert message.blocks[0].type == BlockType.THINKING assert message.blocks[0].text == "**Preparing to analyze git changes**" - assert message.provider_message_id.startswith("synthetic-") + assert message.provider_message_id == "" def test_reasoning_reordering_keeps_synthetic_revision_identity(self) -> None: first = { @@ -704,7 +704,8 @@ def test_duplicate_reasoning_ids_keep_one_active_leaf(self) -> None: result = parse([record, record], "codex-duplicate-reasoning") - assert result.messages[0].provider_message_id == result.messages[1].provider_message_id + assert result.messages[0].provider_message_id == result.messages[1].provider_message_id == "" + assert result.active_leaf_message_provider_id is None assert sum(message.is_active_leaf is True for message in result.messages) == 1 assert result.messages[-1].is_active_leaf is True From 66a10337866ff09c5ac7b04d7727471f940e404a Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 9 Aug 2026 07:59:31 +0200 Subject: [PATCH 14/19] fix(identity): narrow synthetic id privacy scope Problem: the initial synthetic-id sweep changed established Grok, Antigravity, and Claude Design public contracts beyond the exact branch-local Codex residual.\n\nWhat changed: retain existing synthetic IDs for those established parser contracts and keep the privacy repair limited to Codex reasoning and compaction rows named by the exact review.\n\nCompatibility/migration: no Grok, Antigravity, or Claude Design public identity changes remain. Codex synthetic rows still use empty provider IDs and the canonical storage fallback. --- polylogue/sources/parsers/antigravity.py | 22 +++++++--- polylogue/sources/parsers/base_support.py | 14 +++---- polylogue/sources/parsers/claude/ai_parser.py | 42 ++++++++++++------- polylogue/sources/parsers/grok.py | 12 +++++- .../unit/sources/parsers/test_antigravity.py | 11 ++--- tests/unit/sources/parsers/test_grok.py | 10 ++--- .../sources/test_parsers_claude_design.py | 7 ++-- 7 files changed, 75 insertions(+), 43 deletions(-) diff --git a/polylogue/sources/parsers/antigravity.py b/polylogue/sources/parsers/antigravity.py index 567dcace05..bca1ff58f7 100644 --- a/polylogue/sources/parsers/antigravity.py +++ b/polylogue/sources/parsers/antigravity.py @@ -28,6 +28,7 @@ ParsedSession, human_authored_override, mark_last_occurrence_as_active_leaf, + synthetic_message_id, ) _METADATA_SUFFIX = ".metadata.json" @@ -311,9 +312,7 @@ def parse_markdown_export( created_at=None, updated_at=summary.last_modified_time, messages=messages, - active_leaf_message_provider_id=messages[-1].provider_message_id - if messages and messages[-1].provider_message_id - else None, + active_leaf_message_provider_id=messages[-1].provider_message_id if messages else None, ) @@ -433,9 +432,16 @@ def _messages_from_markdown(markdown: str, cascade_id: str) -> list[ParsedMessag continue heading = section.group("title") role = Role.USER if heading == "User Input" else Role.ASSISTANT + provider_message_id = synthetic_message_id( + namespace=cascade_id, + role=role, + text=text, + timestamp=None, + kind=_message_kind(heading), + ) messages.append( ParsedMessage( - provider_message_id="", + provider_message_id=provider_message_id, role=role, text=text, blocks=[ParsedContentBlock(type=BlockType.TEXT, text=text)], @@ -462,7 +468,13 @@ def _messages_from_markdown(markdown: str, cascade_id: str) -> list[ParsedMessag return [] return [ ParsedMessage( - provider_message_id="", + provider_message_id=synthetic_message_id( + namespace=cascade_id, + role=Role.ASSISTANT, + text=text, + timestamp=None, + kind="export", + ), role=Role.ASSISTANT, text=text, blocks=[ParsedContentBlock(type=BlockType.TEXT, text=text)], diff --git a/polylogue/sources/parsers/base_support.py b/polylogue/sources/parsers/base_support.py index 75d59d2563..ca870de007 100644 --- a/polylogue/sources/parsers/base_support.py +++ b/polylogue/sources/parsers/base_support.py @@ -73,14 +73,12 @@ def synthetic_message_id( namespace: str = "", kind: str = "", ) -> str: - """Build a private reorder-stable evidence key for a message with no provider id. - - This is reserved for parser-local evidence, such as an exported summary or - a transcript section. It must never be assigned to - ``ParsedMessage.provider_message_id``: public parser rows without a native - id use the empty string so storage applies the canonical fallback law. - Native-id fallback paths must pass an empty string instead, so - ``pipeline.ids`` can use its role/timestamp/text comparison anchor. + """Build a reorder-stable id for a message with no provider id. + + This is reserved for parser-produced rows that are inherently synthetic, + such as an exported summary or a transcript section. Native-id fallback + paths must pass an empty string instead, so ``pipeline.ids`` can use its + role/timestamp/text comparison anchor. """ seed = "\x1f".join((namespace, str(role), timestamp or "", text or "", kind)) return f"synthetic-{hash_text(seed)[:24]}" diff --git a/polylogue/sources/parsers/claude/ai_parser.py b/polylogue/sources/parsers/claude/ai_parser.py index def5186d96..114d88a5c3 100644 --- a/polylogue/sources/parsers/claude/ai_parser.py +++ b/polylogue/sources/parsers/claude/ai_parser.py @@ -22,7 +22,6 @@ from polylogue.archive.message.roles import Role from polylogue.archive.message.types import MessageType from polylogue.core.enums import BlockType, MaterialOrigin, Provider, SessionKind, TitleSource -from polylogue.core.message_owner import MessageOwnerCoordinate from polylogue.logging import get_logger from ..base import ( @@ -34,6 +33,7 @@ attachment_from_meta, human_authored_override, mark_last_occurrence_as_active_leaf, + synthetic_message_id, ) from .common import ( _first_identity_field, @@ -270,7 +270,17 @@ def flush() -> None: block.text for block in current_blocks if block.type is BlockType.TEXT and not block.is_error and block.text ] segment_text = "\n".join(text_parts) if text_parts else None - provider_message_id = message_uuid if message_uuid and not has_interjection else "" + provider_message_id = ( + message_uuid + if message_uuid and not has_interjection + else synthetic_message_id( + namespace=message_uuid, + role=Role.ASSISTANT, + text=segment_text, + timestamp=timestamp_str, + kind="claude-design-assistant-segment", + ) + ) input_tokens = 0 if not first_segment_emitted and isinstance(turn_input_tokens, int): input_tokens = turn_input_tokens @@ -346,7 +356,13 @@ def flush() -> None: interjection_timestamp_value = ( str(interjection_timestamp) if isinstance(interjection_timestamp, str) else None ) - interjection_id = str(interjection_message.get("id") or "") + interjection_id = str(interjection_message.get("id") or "") or synthetic_message_id( + namespace=message_uuid, + role=interjection_role, + text=interjection_text_value, + timestamp=interjection_timestamp_value, + kind="claude-design-interjection", + ) messages.append( ParsedMessage( provider_message_id=interjection_id, @@ -416,18 +432,16 @@ def _design_user_message( sender_name = str(author_name) if isinstance(author_name, str) and author_name else None timestamp_str = str(timestamp) if isinstance(timestamp, str) and timestamp else None design_message_text = str(text) if isinstance(text, str) and text else None - message_uuid = str(raw_message.get("uuid") or content_payload.get("id") or "") + message_uuid = str(raw_message.get("uuid") or content_payload.get("id") or "") or synthetic_message_id( + role=Role.USER, + text=design_message_text, + timestamp=timestamp_str, + kind="claude-design-user", + ) for meta in raw_attachments if isinstance(raw_attachments, list) else []: attachment = _design_attachment_from_meta(meta, message_uuid) if attachment is not None: - attachments.append( - attachment.model_copy( - update={ - "message_position": position, - "owner_coordinate": MessageOwnerCoordinate(position=position, variant_index=0), - } - ) - ) + attachments.append(attachment) message = ParsedMessage( provider_message_id=message_uuid, role=Role.USER, @@ -503,9 +517,7 @@ def parse_design(payload: Mapping[str, object], fallback_id: str) -> ParsedSessi "claude-design %s: unrecognized message role %r, dropping message", resolved_session_id, role ) - active_leaf_message_provider_id = ( - messages[-1].provider_message_id if messages and messages[-1].provider_message_id else None - ) + active_leaf_message_provider_id = messages[-1].provider_message_id if messages else None messages = mark_last_occurrence_as_active_leaf(messages) title, title_source, title_ref, title_confidence = _resolve_claude_ai_title( diff --git a/polylogue/sources/parsers/grok.py b/polylogue/sources/parsers/grok.py index def4759dea..8fd8c18aa9 100644 --- a/polylogue/sources/parsers/grok.py +++ b/polylogue/sources/parsers/grok.py @@ -50,6 +50,7 @@ fill_linear_parent_chain, human_authored_override, mark_last_occurrence_as_active_leaf, + synthetic_message_id, ) _SENDER_ROLE: dict[str, Role] = { @@ -144,9 +145,16 @@ def parse_conversation(payload: Mapping[str, object], fallback_id: str) -> Parse continue grok_role = _role_for_sender(fields.get("sender")) timestamp = _timestamp_text(fields.get("create_time")) + provider_message_id = synthetic_message_id( + namespace=fallback_id, + role=grok_role, + text=text, + timestamp=timestamp, + kind="grok-response", + ) messages.append( ParsedMessage( - provider_message_id="", + provider_message_id=provider_message_id, role=grok_role, text=text, timestamp=timestamp, @@ -166,7 +174,7 @@ def parse_conversation(payload: Mapping[str, object], fallback_id: str) -> Parse ) ) - active_leaf_message_provider_id = None + active_leaf_message_provider_id = messages[-1].provider_message_id if messages else None messages = mark_last_occurrence_as_active_leaf(messages) # bd polylogue-ksgg: Grok exports carry no native conversation/message id # or parent evidence at all (see module docstring) -- a plain ordered diff --git a/tests/unit/sources/parsers/test_antigravity.py b/tests/unit/sources/parsers/test_antigravity.py index ada1d645b2..2417af5d26 100644 --- a/tests/unit/sources/parsers/test_antigravity.py +++ b/tests/unit/sources/parsers/test_antigravity.py @@ -51,13 +51,14 @@ def test_parse_markdown_export_splits_known_sections() -> None: "The focused checks passed.", ] assert session.messages[0].blocks[0].type == BlockType.TEXT - assert session.messages[0].provider_message_id == "" - assert session.messages[1].provider_message_id == "" + assert session.messages[0].provider_message_id.startswith("synthetic-") + assert session.messages[1].provider_message_id.startswith("synthetic-") + assert session.messages[0].provider_message_id != session.messages[1].provider_message_id assert [message.position for message in session.messages] == [0, 1] assert [message.variant_index for message in session.messages] == [0, 0] assert [message.is_active_path for message in session.messages] == [True, True] assert [message.is_active_leaf for message in session.messages] == [False, True] - assert session.active_leaf_message_provider_id is None + assert session.active_leaf_message_provider_id == session.messages[1].provider_message_id def test_parse_markdown_export_reordering_keeps_synthetic_revision_identity() -> None: @@ -108,11 +109,11 @@ def test_parse_markdown_export_falls_back_to_single_export_message() -> None: session = parse_markdown_export(markdown, summary) assert [message.role for message in session.messages] == [Role.ASSISTANT] - assert session.messages[0].provider_message_id == "" + assert session.messages[0].provider_message_id.startswith("synthetic-") assert session.messages[0].text == "Unstructured transcript body." assert session.messages[0].position == 0 assert session.messages[0].is_active_leaf is True - assert session.active_leaf_message_provider_id is None + assert session.active_leaf_message_provider_id == session.messages[0].provider_message_id assert session.title_source is None diff --git a/tests/unit/sources/parsers/test_grok.py b/tests/unit/sources/parsers/test_grok.py index f85426a4c4..bfde8a2802 100644 --- a/tests/unit/sources/parsers/test_grok.py +++ b/tests/unit/sources/parsers/test_grok.py @@ -96,11 +96,12 @@ def test_parse_conversation_nested_response_shape() -> None: assert session.messages[0].text == "Why is my useEffect running twice?" assert session.messages[0].timestamp == "2024-04-01T19:33:21+00:00" assert session.messages[0].blocks[0].type is BlockType.TEXT - assert session.messages[0].provider_message_id == "" - assert session.messages[1].provider_message_id == "" + assert session.messages[0].provider_message_id.startswith("synthetic-") + assert session.messages[1].provider_message_id.startswith("synthetic-") + assert session.messages[0].provider_message_id != session.messages[1].provider_message_id assert [m.position for m in session.messages] == [0, 1] assert [m.is_active_leaf for m in session.messages] == [False, True] - assert session.active_leaf_message_provider_id is None + assert session.active_leaf_message_provider_id == session.messages[1].provider_message_id assert session.updated_at == session.messages[-1].timestamp @@ -202,8 +203,7 @@ def test_duplicate_grok_response_ids_keep_one_active_leaf() -> None: session = grok.parse_conversation(payload, "grok-duplicate") - assert session.messages[-2].provider_message_id == session.messages[-1].provider_message_id == "" - assert session.active_leaf_message_provider_id is None + assert session.messages[-2].provider_message_id == session.messages[-1].provider_message_id assert sum(message.is_active_leaf is True for message in session.messages) == 1 assert session.messages[-1].is_active_leaf is True diff --git a/tests/unit/sources/test_parsers_claude_design.py b/tests/unit/sources/test_parsers_claude_design.py index fe22c82457..2633bb2384 100644 --- a/tests/unit/sources/test_parsers_claude_design.py +++ b/tests/unit/sources/test_parsers_claude_design.py @@ -312,10 +312,11 @@ def test_user_interjection_splits_the_assistant_turn_and_preserves_ordering() -> assert session.messages[2].provider_message_id == "interjection-1" # Segments have no independent provider id. Their content-derived ids are # stable across re-export ordering and do not encode the segment position. - assert session.messages[1].provider_message_id == "" - assert session.messages[3].provider_message_id == "" + assert session.messages[1].provider_message_id.startswith("synthetic-") + assert session.messages[3].provider_message_id.startswith("synthetic-") + assert session.messages[1].provider_message_id == session.messages[3].provider_message_id # active leaf is the true last message, not the raw turn's nominal end - assert session.active_leaf_message_provider_id is None + assert session.active_leaf_message_provider_id == session.messages[3].provider_message_id assert session.messages[-1].is_active_leaf is True assert sum(message.is_active_leaf is True for message in session.messages) == 1 From bf50a2fc4f09a4f1bfe78b493759cbb8f1de1d6d Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 9 Aug 2026 17:29:59 +0200 Subject: [PATCH 15/19] fix(identity): close stable attachment owner ambiguity Problem: private attachment ownership could follow mutable message content, and duplicate stable evidence could silently select the last message at one physical coordinate. What changed: prefer unique stable owner evidence before content discrimination, count stable evidence independently, and require a unique full physical coordinate before resolving duplicated evidence. Add Claude parser, projection, archive write, and re-ingest regressions. Compatibility: native provider IDs, public idless identity, parser-local lineage, and existing provider contracts remain unchanged. --- polylogue/pipeline/ids.py | 25 +++- ...test_message_identity_position_fallback.py | 69 ++++++++++- .../sources/test_claude_web_normalization.py | 112 ++++++++++++++++++ 3 files changed, 199 insertions(+), 7 deletions(-) diff --git a/polylogue/pipeline/ids.py b/polylogue/pipeline/ids.py index ce836cab11..27946efda9 100644 --- a/polylogue/pipeline/ids.py +++ b/polylogue/pipeline/ids.py @@ -297,6 +297,7 @@ class MessageOwnerResolution: by_physical_coordinate: Mapping[tuple[int, int], str] ambiguous_physical_coordinates: frozenset[tuple[int, int]] by_stable_key: Mapping[str, str] + ambiguous_stable_keys: frozenset[str] ambiguous_keys: frozenset[str] unique_provider_keys: Mapping[str, str] ambiguous_provider_ids: frozenset[str] @@ -333,11 +334,14 @@ def message_owner_resolution(messages: list[ParsedMessage]) -> MessageOwnerResol ) content_counts = Counter(content_ids) coordinates = tuple(_message_owner_coordinate(message, index) for index, message in enumerate(messages)) + stable_counts = Counter(coordinate.stable_key for coordinate in coordinates if coordinate.stable_key is not None) keys: list[str] = [] for revision_id, content_id, coordinate in zip(revision_ids, content_ids, coordinates, strict=True): if revision_counts[revision_id] == 1: key = revision_id + elif coordinate.stable_key is not None and stable_counts[coordinate.stable_key] == 1: + key = coordinate.stable_key elif content_counts[content_id] == 1: key = content_id elif coordinate.stable_key is not None: @@ -359,7 +363,11 @@ def message_owner_resolution(messages: list[ParsedMessage]) -> MessageOwnerResol by_stable_key = { coordinate.stable_key: key for coordinate, key in zip(coordinates, keys, strict=True) - if coordinate.stable_key is not None and key not in ambiguous_keys + if ( + coordinate.stable_key is not None + and stable_counts[coordinate.stable_key] == 1 + and key not in ambiguous_keys + ) } provider_keys: dict[str, str] = {} provider_counts = Counter( @@ -376,6 +384,7 @@ def message_owner_resolution(messages: list[ParsedMessage]) -> MessageOwnerResol coordinate for coordinate, count in physical_counts.items() if count > 1 ), by_stable_key=by_stable_key, + ambiguous_stable_keys=frozenset(stable_key for stable_key, count in stable_counts.items() if count > 1), ambiguous_keys=ambiguous_keys, unique_provider_keys=provider_keys, ambiguous_provider_ids=frozenset(provider_id for provider_id, count in provider_counts.items() if count > 1), @@ -399,11 +408,13 @@ def _attachment_owner_coordinate(attachment: ParsedAttachment) -> MessageOwnerCo def attachment_message_owner_key(attachment: ParsedAttachment, resolution: MessageOwnerResolution) -> str | None: """Resolve one attachment to the same private owner key used by writes.""" coordinate = _attachment_owner_coordinate(attachment) - if coordinate.stable_key is not None: - if coordinate.stable_key in resolution.ambiguous_keys: - raise MessageOwnerAmbiguityError(f"attachment owner evidence is duplicated: {coordinate.stable_key!r}") - if coordinate.stable_key in resolution.by_stable_key: - return resolution.by_stable_key[coordinate.stable_key] + if ( + coordinate.stable_key is not None + and coordinate.stable_key not in resolution.ambiguous_stable_keys + and coordinate.stable_key not in resolution.ambiguous_keys + and coordinate.stable_key in resolution.by_stable_key + ): + return resolution.by_stable_key[coordinate.stable_key] if coordinate.physical_key is not None: if coordinate.physical_key in resolution.ambiguous_physical_coordinates: raise MessageOwnerAmbiguityError(f"attachment owner coordinate is duplicated: {coordinate.physical_key!r}") @@ -415,6 +426,8 @@ def attachment_message_owner_key(attachment: ParsedAttachment, resolution: Messa f"{coordinate.physical_key!r}" ) return key + if coordinate.stable_key in resolution.ambiguous_stable_keys: + raise MessageOwnerAmbiguityError(f"attachment owner evidence is duplicated: {coordinate.stable_key!r}") if attachment.message_provider_id: provider_id = attachment.message_provider_id.strip() if provider_id in resolution.ambiguous_provider_ids: diff --git a/tests/unit/pipeline/test_message_identity_position_fallback.py b/tests/unit/pipeline/test_message_identity_position_fallback.py index 997a73f17a..862a0ec2c6 100644 --- a/tests/unit/pipeline/test_message_identity_position_fallback.py +++ b/tests/unit/pipeline/test_message_identity_position_fallback.py @@ -19,7 +19,7 @@ from polylogue.archive.message.roles import Role from polylogue.archive.session_revision_membership import MembershipRevision, _relation, classify_membership_revisions from polylogue.core.enums import Provider -from polylogue.core.message_owner import MessageOwnerAmbiguityError +from polylogue.core.message_owner import MessageOwnerAmbiguityError, MessageOwnerCoordinate from polylogue.pipeline.ids import session_revision_projection from polylogue.sources.parsers.base import ParsedAttachment, ParsedMessage, ParsedSession @@ -141,6 +141,46 @@ def test_duplicate_physical_owner_coordinate_fails_closed_before_hashing_attachm session_revision_projection(_session(messages, [attachment])) +def test_unique_stable_owner_evidence_precedes_mutable_content() -> None: + def session(first_text: str) -> ParsedSession: + messages = [ + _id_less("assistant", first_text, "2024-01-01T00:01:00Z").model_copy( + update={ + "position": 0, + "owner_coordinate": MessageOwnerCoordinate("owner-first", 0, 0), + } + ), + _id_less("assistant", "second", "2024-01-01T00:01:00Z").model_copy( + update={ + "position": 1, + "owner_coordinate": MessageOwnerCoordinate("owner-second", 1, 0), + } + ), + ] + return _session( + messages, + [ + ParsedAttachment( + provider_attachment_id="owner-first-attachment", + message_provider_id="", + message_position=0, + message_variant_index=0, + owner_coordinate=MessageOwnerCoordinate("owner-first", 0, 0), + name="first.txt", + mime_type="text/plain", + ) + ], + ) + + older = session("before") + edited = session("after") + older_projection = session_revision_projection(older) + edited_projection = session_revision_projection(edited) + + assert older_projection.attachment_identities == edited_projection.attachment_identities + assert _relation(older_projection, edited_projection) == "equal" + + def test_duplicate_native_id_at_duplicate_physical_coordinate_fails_closed() -> None: messages = [ ParsedMessage( @@ -212,3 +252,30 @@ def test_indistinguishable_duplicate_idless_attachment_owner_fails_closed() -> N ) with pytest.raises(MessageOwnerAmbiguityError): session_revision_projection(_session(repeated, [attachment])) + + +def test_duplicate_stable_owner_evidence_without_physical_coordinate_fails_closed() -> None: + messages = [ + _id_less("assistant", "first", "2024-01-01T00:01:00Z").model_copy( + update={ + "position": 4, + "owner_coordinate": MessageOwnerCoordinate("same-owner-evidence", 4, 0), + } + ), + _id_less("assistant", "second", "2024-01-01T00:01:00Z").model_copy( + update={ + "position": 4, + "owner_coordinate": MessageOwnerCoordinate("same-owner-evidence", 4, 1), + } + ), + ] + attachment = ParsedAttachment( + provider_attachment_id="shared-owner-attachment", + message_provider_id="", + owner_coordinate=MessageOwnerCoordinate("same-owner-evidence"), + name="note.txt", + mime_type="text/plain", + ) + + with pytest.raises(MessageOwnerAmbiguityError): + session_revision_projection(_session(messages, [attachment])) diff --git a/tests/unit/sources/test_claude_web_normalization.py b/tests/unit/sources/test_claude_web_normalization.py index 3fad9a7d95..a09f1cc67a 100644 --- a/tests/unit/sources/test_claude_web_normalization.py +++ b/tests/unit/sources/test_claude_web_normalization.py @@ -10,6 +10,7 @@ import pytest +from polylogue.archive.session_revision_membership import _relation from polylogue.core.enums import Provider from polylogue.core.message_owner import MessageOwnerAmbiguityError from polylogue.pipeline.ids import session_revision_projection @@ -652,6 +653,117 @@ def payload(text: str) -> dict[str, Any]: assert stored_text == "after" +def test_claude_attachment_owner_stays_stable_across_idless_body_edit(tmp_path: Path) -> None: + def payload(first_text: str) -> dict[str, Any]: + return { + "uuid": "claude-idless-attachment-edit", + "chat_messages": [ + { + "sender": "assistant", + "text": first_text, + "created_at": "2026-07-01T10:00:00Z", + "position": 0, + "files": [{"id": "owner-file-first", "file_name": "first.txt", "file_type": "text/plain"}], + }, + { + "sender": "assistant", + "text": "second", + "created_at": "2026-07-01T10:00:00Z", + "position": 1, + "files": [{"id": "owner-file-second", "file_name": "second.txt", "file_type": "text/plain"}], + }, + ], + } + + older_payload = payload("before") + edited_payload = payload("after") + older = _parse_real_route(older_payload) + edited = _parse_real_route(edited_payload) + older_projection = session_revision_projection(older) + edited_projection = session_revision_projection(edited) + + assert older_projection.attachment_identities == edited_projection.attachment_identities + assert _relation(older_projection, edited_projection) == "equal" + + root = tmp_path / "archive" + with ArchiveStore(root) as facade: + first = facade.write_raw_and_parsed_result( + older, + payload=json.dumps(older_payload).encode(), + source_path="/tmp/claude-idless-attachment-edit-before.json", + acquired_at_ms=1_775_000_000_000, + ) + second = facade.write_raw_and_parsed_result( + edited, + payload=json.dumps(edited_payload).encode(), + source_path="/tmp/claude-idless-attachment-edit-after.json", + acquired_at_ms=1_775_000_000_001, + ) + + assert first.content_changed is True + assert second.content_changed is True + assert second.counts["skipped_sessions"] == 0 + conn = sqlite3.connect(f"file:{root / 'index.db'}?mode=ro", uri=True) + try: + assert ( + conn.execute( + "SELECT COUNT(*) FROM attachment_refs WHERE session_id = ?", + ("claude-ai-export:claude-idless-attachment-edit",), + ).fetchone()[0] + == 2 + ) + finally: + conn.close() + + +def test_claude_duplicate_stable_owner_evidence_uses_full_coordinate( + workspace_env: dict[str, Path], +) -> None: + payload = { + "uuid": "claude-duplicate-stable-owner-evidence", + "chat_messages": [ + { + "sender": "assistant", + "text": "first body", + "created_at": "2026-07-01T10:00:00Z", + "position": 4, + "files": [{"id": "shared-owner-file", "file_name": "first.txt", "file_type": "text/plain"}], + }, + { + "sender": "assistant", + "text": "second body", + "created_at": "2026-07-01T10:00:00Z", + "position": 4, + "files": [{"id": "shared-owner-file", "file_name": "second.txt", "file_type": "text/plain"}], + }, + ], + } + parsed = _parse_real_route(payload) + assert [message.provider_message_id for message in parsed.messages] == ["", ""] + assert [(message.position, message.variant_index) for message in parsed.messages] == [(4, 0), (4, 1)] + assert len(parsed.attachments) == 1 + assert parsed.attachments[0].owner_coordinate is not None + assert parsed.attachments[0].owner_coordinate.physical_key == (4, 0) + + raw_bytes = json.dumps(payload).encode() + db_path = db_setup(workspace_env) + with open_connection(db_path) as conn: + hydrated = write_and_hydrate( + parse_payload_roundtrip("claude-ai", raw_bytes, unique_id="claude-duplicate-stable-owner-evidence"), conn + ) + row = conn.execute( + """ + SELECT m.position, m.variant_index + FROM attachment_refs AS r + JOIN messages AS m ON m.message_id = r.message_id + WHERE r.session_id = ? + """, + (str(hydrated.id),), + ).fetchone() + + assert tuple(row) == (4, 0) + + def _duplicate_native_variant_payload(*, swapped: bool = False) -> dict[str, Any]: files = (("att-variant-a", "variant-a.txt"), ("att-variant-b", "variant-b.txt")) if swapped: From e654d98994066941d970caa28dada5fdf48e7e24 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 9 Aug 2026 21:23:45 +0200 Subject: [PATCH 16/19] fix(identity): preserve attachment and durable owner evidence Problem: idless Claude files could retain an attachment-derived owner key after reassignment, and durable user-state reads could not resolve accepted session aliases once the index row disappeared.\n\nWhat changed: remove attachment identifiers from Claude owner evidence and fail closed when independent owner evidence is absent. Resolve missing-index aliases against canonical active mark and annotation owners with ambiguity checks. Add parser-to-writer, alias, ambiguity, and red-twin regressions.\n\nCompatibility/migration: no schema or production archive migration is performed.\n\nCo-Authored-By: Claude --- polylogue/api/archive.py | 76 ++++++++++- polylogue/sources/parsers/claude/common.py | 13 +- .../sources/test_claude_web_normalization.py | 127 ++++++++++-------- .../storage/test_archive_tiers_archive.py | 3 + .../storage/test_marks_identity_preserving.py | 69 ++++++++++ 5 files changed, 219 insertions(+), 69 deletions(-) diff --git a/polylogue/api/archive.py b/polylogue/api/archive.py index 7cc6986481..36709b7fa7 100644 --- a/polylogue/api/archive.py +++ b/polylogue/api/archive.py @@ -32,7 +32,7 @@ DEFAULT_CONTEXT_IMAGE_MAX_CHARS_PER_MESSAGE, DEFAULT_CONTEXT_IMAGE_MAX_MESSAGES_PER_SESSION, ) -from polylogue.core.enums import AssertionKind, AssertionStatus, MaterialOrigin, Origin, TitleSource +from polylogue.core.enums import AssertionKind, AssertionStatus, MaterialOrigin, Origin, Provider, TitleSource from polylogue.core.errors import PolylogueError from polylogue.core.json import JSONDocument, JSONValue from polylogue.core.refs import ( @@ -44,6 +44,7 @@ parse_delegation_subtree_object_id, parse_public_ref, ) +from polylogue.core.sources import origin_from_provider from polylogue.core.timestamps import parse_archive_datetime from polylogue.core.types import SessionId from polylogue.core.user_state_targets import TARGET_MESSAGE, TARGET_SESSION @@ -296,6 +297,62 @@ class SessionNotFoundError(PolylogueError): http_status_code = 404 +def _resolve_durable_user_state_session_id(archive_root: Path, token: str) -> str | None: + """Resolve a session alias from canonical mark/annotation owners. + + The index is rebuildable, while mark and annotation ownership is durable. + When the index cannot resolve a token, match it against the canonical + session ids already persisted in those user-state rows. Every accepted + alias shape is checked against the complete durable owner set, and an + ambiguous prefix fails closed instead of selecting an arbitrary owner. + """ + if not token: + return None + user_db = archive_root / "user.db" + if not user_db.exists(): + return None + try: + with closing(open_readonly_connection(user_db)) as conn: + rows = conn.execute( + """ + SELECT target_ref, scope_ref + FROM assertions + WHERE kind IN (?, ?) + AND COALESCE(status, 'active') != 'deleted' + """, + (AssertionKind.MARK.value, AssertionKind.ANNOTATION.value), + ).fetchall() + except sqlite3.Error: + return None + + canonical_ids: set[str] = set() + for row in rows: + for value in (row[0], row[1]): + if not isinstance(value, str): + continue + if value.startswith("session:"): + canonical_id = value[len("session:") :] + if canonical_id: + canonical_ids.add(canonical_id) + + matches = {canonical_id for canonical_id in canonical_ids if _durable_session_alias_matches(token, canonical_id)} + if len(matches) > 1: + raise ValueError(f"session id alias {token!r} is ambiguous") + return next(iter(matches), None) + + +def _durable_session_alias_matches(token: str, canonical_id: str) -> bool: + """Apply the same exact/provider/prefix/suffix alias shapes as the index.""" + if token == canonical_id or canonical_id.startswith(token): + return True + if ":" in token: + provider_token, native_id = token.split(":", 1) + provider_origin = origin_from_provider(Provider.from_string(provider_token)).value + return canonical_id == f"{provider_origin}:{native_id}" + _, separator, native_id = canonical_id.partition(":") + return bool(separator and (native_id == token or native_id.startswith(token))) + + def _archive_query_date_ms(field: str, value: str | None) -> int | None: parsed = parse_query_date(field, value) if parsed is None: @@ -6761,10 +6818,19 @@ async def _resolve_user_state_target( } async def _resolve_user_state_session_id(self, session_id: str) -> str: - archive_resolved = await self._archive_resolve_session_id(session_id) - if archive_resolved is None: - raise SessionNotFoundError(session_id) - return archive_resolved + try: + archive_resolved = await self._archive_resolve_session_id(session_id) + except SessionNotFoundError: + archive_resolved = None + if archive_resolved is not None: + return archive_resolved + durable_resolved = _resolve_durable_user_state_session_id( + _active_archive_root(self.config), + session_id, + ) + if durable_resolved is not None: + return durable_resolved + raise SessionNotFoundError(session_id) async def _user_state_message_exists(self, session_id: str, message_id: str) -> bool: return bool(await self._archive_message_exists(session_id, message_id)) diff --git a/polylogue/sources/parsers/claude/common.py b/polylogue/sources/parsers/claude/common.py index 0b8db95f15..983485d899 100644 --- a/polylogue/sources/parsers/claude/common.py +++ b/polylogue/sources/parsers/claude/common.py @@ -644,13 +644,12 @@ def _owner_stable_key( block_ids = sorted(block.tool_id for block in blocks if block.tool_id) if block_ids: evidence["tool_ids"] = block_ids - attachment_ids = sorted( - (attachment.provider_attachment_id, attachment.provider_file_id, attachment.provider_drive_id) - for attachment in attachments - if attachment.provider_attachment_id or attachment.provider_file_id or attachment.provider_drive_id - ) - if attachment_ids: - evidence["attachment_ids"] = [list(values) for values in attachment_ids] + # Attachment identity is deliberately excluded from owner evidence. A + # Claude export can move an idless file between same-role, same-timestamp + # turns while retaining the file id. Including that id makes the file's + # stable key follow the file instead of the owning turn, so the session + # hash can remain unchanged and re-ingest can skip the reassignment. The + # parser-provided position/variant coordinate is the independent fallback. if not evidence: return None return f"claude-owner-evidence:{hash_payload(evidence)}" diff --git a/tests/unit/sources/test_claude_web_normalization.py b/tests/unit/sources/test_claude_web_normalization.py index a09f1cc67a..86a094647d 100644 --- a/tests/unit/sources/test_claude_web_normalization.py +++ b/tests/unit/sources/test_claude_web_normalization.py @@ -22,8 +22,6 @@ ) from polylogue.sources.parsers.claude.common import ( CLAUDE_LINEAGE_CYCLE_INGEST_FLAG, - _message_attachments, - _owner_stable_key, ) from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore from polylogue.storage.sqlite.connection import open_connection @@ -542,35 +540,16 @@ def _duplicate_idless_owner_payload(order: tuple[str, ...]) -> dict[str, Any]: def test_claude_duplicate_idless_owner_evidence_survives_parser_reorder() -> None: - """Real parser reorder: provider file evidence, not assigned position, owns each attachment.""" + """Real parser reorder fails closed without owner-independent evidence.""" forward = _parse_real_route(_duplicate_idless_owner_payload(("first", "second"))) reordered = _parse_real_route(_duplicate_idless_owner_payload(("second", "first"))) assert [message.provider_message_id for message in forward.messages] == ["", ""] assert [message.provider_message_id for message in reordered.messages] == ["", ""] - assert ( - session_revision_projection(forward).message_contents == session_revision_projection(reordered).message_contents - ) - assert ( - session_revision_projection(forward).attachment_identities - == session_revision_projection(reordered).attachment_identities - ) - - def owners(session: ParsedSession) -> dict[str, tuple[str | None, int | None, int | None]]: - return { - attachment.name or "": ( - attachment.owner_coordinate.stable_key if attachment.owner_coordinate else None, - attachment.message_position, - attachment.message_variant_index, - ) - for attachment in session.attachments - } - - assert owners(forward).keys() == owners(reordered).keys() - assert {key: value[0] for key, value in owners(forward).items()} == { - key: value[0] for key, value in owners(reordered).items() - } - assert owners(forward)["first.txt"][1:] != owners(reordered)["first.txt"][1:] + with pytest.raises(MessageOwnerAmbiguityError): + session_revision_projection(forward) + with pytest.raises(MessageOwnerAmbiguityError): + session_revision_projection(reordered) def test_claude_indistinguishable_duplicate_idless_owner_is_typed_ambiguity() -> None: @@ -824,40 +803,74 @@ def test_claude_duplicate_native_attachment_owner_keeps_variant_coordinate( assert rows == [("att-variant-a", 0), ("att-variant-b", 1)] -def test_claude_repeated_position_owner_evidence_keeps_attachment_identity() -> None: - """Attachment evidence remains distinct even before variant repair. +def test_claude_idless_file_reassignment_changes_owner_hash_and_reingests( + tmp_path: Path, +) -> None: + """A moved idless Claude file changes the real archive owner and hash.""" - This directly exercises the private owner-key constructor with repeated - explicit positions and omitted variants. Removing attachment evidence from - ``_owner_stable_key`` makes the two keys collide and the assertion fails. - """ - first = {"position": 4, "files": [{"id": "owner-a", "file_name": "a.txt"}]} - second = {"position": 4, "files": [{"id": "owner-b", "file_name": "b.txt"}]} - - first_attachments = _message_attachments(first, "") - second_attachments = _message_attachments(second, "") - first_key = _owner_stable_key( - first, - parent_message_provider_id=None, - explicit_position=4, - explicit_branch_index=None, - explicit_variant_index=None, - blocks=[], - attachments=first_attachments, - ) - second_key = _owner_stable_key( - second, - parent_message_provider_id=None, - explicit_position=4, - explicit_branch_index=None, - explicit_variant_index=None, - blocks=[], - attachments=second_attachments, + def payload(file_owner: int) -> dict[str, Any]: + rows: list[dict[str, Any]] = [ + { + "sender": "assistant", + "text": "first same-timestamp turn", + "created_at": "2026-07-01T10:00:00Z", + }, + { + "sender": "assistant", + "text": "second same-timestamp turn", + "created_at": "2026-07-01T10:00:00Z", + }, + ] + rows[file_owner]["files"] = [{"id": "reassigned-file", "file_name": "shared.txt", "file_type": "text/plain"}] + return {"uuid": "claude-idless-file-reassignment", "chat_messages": rows} + + first_payload = payload(0) + second_payload = payload(1) + first = _parse_real_route(first_payload) + second = _parse_real_route(second_payload) + first_position = first.attachments[0].message_position + second_position = second.attachments[0].message_position + assert first_position is not None + assert second_position is not None + + assert ( + session_revision_projection(first).attachment_identities + != session_revision_projection(second).attachment_identities ) + assert first_position != second_position + + root = tmp_path / "archive" + with ArchiveStore(root) as facade: + initial = facade.write_raw_and_parsed_result( + first, + payload=json.dumps(first_payload).encode(), + source_path="/tmp/claude-idless-file-reassignment-before.json", + acquired_at_ms=1_775_000_000_000, + ) + reassigned = facade.write_raw_and_parsed_result( + second, + payload=json.dumps(second_payload).encode(), + source_path="/tmp/claude-idless-file-reassignment-after.json", + acquired_at_ms=1_775_000_000_001, + ) - assert first_key is not None - assert second_key is not None - assert first_key != second_key + assert initial.content_changed is True + assert reassigned.content_changed is True + assert reassigned.counts["skipped_sessions"] == 0 + conn = sqlite3.connect(f"file:{root / 'index.db'}?mode=ro", uri=True) + try: + row = conn.execute( + """ + SELECT m.position + FROM attachment_refs AS r + JOIN messages AS m ON m.message_id = r.message_id + WHERE r.session_id = ? + """, + ("claude-ai-export:claude-idless-file-reassignment",), + ).fetchone() + finally: + conn.close() + assert row == (second_position,) def test_claude_duplicate_native_owner_move_changes_real_ingest_hash_and_owner( diff --git a/tests/unit/storage/test_archive_tiers_archive.py b/tests/unit/storage/test_archive_tiers_archive.py index 1245ac884c..5b90ca39f8 100644 --- a/tests/unit/storage/test_archive_tiers_archive.py +++ b/tests/unit/storage/test_archive_tiers_archive.py @@ -11,6 +11,7 @@ from polylogue.archive.message.types import MessageType from polylogue.archive.query.expression import parse_unit_source_expression from polylogue.core.enums import ActionResultState, BlockType, Origin, Provider +from polylogue.core.message_owner import MessageOwnerCoordinate from polylogue.scenarios.workload import ( BudgetVerdict, WorkloadPhaseObservation, @@ -1213,6 +1214,7 @@ def session(message_position: int) -> ParsedSession: text="same duplicate turn", timestamp="2026-04-03T00:00:00Z", position=0, + owner_coordinate=MessageOwnerCoordinate("owner-first", 0, 0), ), ParsedMessage( provider_message_id="", @@ -1220,6 +1222,7 @@ def session(message_position: int) -> ParsedSession: text="same duplicate turn", timestamp="2026-04-03T00:00:00Z", position=1, + owner_coordinate=MessageOwnerCoordinate("owner-second", 1, 0), ), ], attachments=[ diff --git a/tests/unit/storage/test_marks_identity_preserving.py b/tests/unit/storage/test_marks_identity_preserving.py index 2305b38587..fd5f9f1cc9 100644 --- a/tests/unit/storage/test_marks_identity_preserving.py +++ b/tests/unit/storage/test_marks_identity_preserving.py @@ -210,6 +210,75 @@ async def test_message_user_state_projects_owner_for_opaque_session_native_ids( assert filtered_annotations == [annotations[0]] +@pytest.mark.asyncio +@pytest.mark.parametrize("alias_kind", ["native", "provider", "prefix"]) +async def test_message_user_state_resolves_durable_alias_after_index_row_disappears( + workspace_env: dict[str, Path], + alias_kind: str, +) -> None: + """Durable mark/annotation owners resolve accepted aliases without index rows.""" + db_path = db_setup(workspace_env) + builder = ( + SessionBuilder(db_path, "durable-alias-session") + .provider("claude-code") + .add_message(message_id="message-native", text="hello") + ) + builder.save() + session_id = builder.native_session_id() + native_id = session_id.split(":", 1)[1] + origin = session_id.split(":", 1)[0] + alias = { + "native": native_id, + "provider": f"claude-code:{native_id}", + "prefix": f"{origin}:{native_id[:10]}", + }[alias_kind] + message_id = f"{session_id}:n:message-native" + + async with Polylogue(db_path=db_path, archive_root=workspace_env["archive_root"]) as poly: + assert await poly.add_mark(session_id, "pin", target_type="message", message_id=message_id) is True + assert ( + await poly.save_annotation( + "durable-alias-note", + session_id, + "important", + target_type="message", + message_id=message_id, + ) + is True + ) + assert await poly.delete_session(session_id) is True + + marks = await poly.list_marks(session_id=alias, mark_type="pin") + annotations = await poly.list_annotations(session_id=alias) + + assert marks[0]["session_id"] == session_id + assert annotations[0]["session_id"] == session_id + + +@pytest.mark.asyncio +async def test_durable_session_alias_matching_fails_closed_when_ambiguous( + workspace_env: dict[str, Path], +) -> None: + """A durable bare-native prefix never selects one of multiple owners.""" + db_path = db_setup(workspace_env) + builders = [ + SessionBuilder(db_path, native_id) + .provider("claude-code") + .add_message(message_id="message-native", text=native_id) + for native_id in ("ambiguous-one", "ambiguous-two") + ] + for builder in builders: + builder.save() + session_ids = [builder.native_session_id() for builder in builders] + + async with Polylogue(db_path=db_path, archive_root=workspace_env["archive_root"]) as poly: + for session_id in session_ids: + assert await poly.add_mark(session_id, "pin") is True + assert await poly.delete_session(session_id) is True + with pytest.raises(ValueError, match="ambiguous"): + await poly.list_marks(session_id="ext-ambiguous", mark_type="pin") + + # --------------------------------------------------------------------------- # The core #1114 acceptance: marks survive hard delete and rebind on reimport # --------------------------------------------------------------------------- From 3f22a5712994069956eddc2d707500f5c9cd2d24 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 9 Aug 2026 22:16:36 +0200 Subject: [PATCH 17/19] fix(identity): preserve mutable message and attachment ownership --- .../archive/session_revision_membership.py | 10 +++++++ polylogue/pipeline/ids.py | 6 ++-- polylogue/sources/parsers/drive.py | 25 ++++++----------- .../test_session_revision_membership.py | 28 +++++++++++++++++++ 4 files changed, 50 insertions(+), 19 deletions(-) diff --git a/polylogue/archive/session_revision_membership.py b/polylogue/archive/session_revision_membership.py index a326848f9b..4e822041de 100644 --- a/polylogue/archive/session_revision_membership.py +++ b/polylogue/archive/session_revision_membership.py @@ -296,6 +296,16 @@ def _equal_content_representative( if candidate_time.timestamp() > incumbent_time.timestamp() else (incumbent, candidate) ) + if ( + incumbent.observed_at_ms is not None + and candidate.observed_at_ms is not None + and candidate.observed_at_ms != incumbent.observed_at_ms + ): + # Timestamped id-less messages intentionally share a mutable revision + # axis. When their content changes but provider_updated_at is absent + # or unchanged, observation order is the remaining source-backed + # authority; raw_id is only a deterministic last resort. + return (candidate, incumbent) if candidate.observed_at_ms > incumbent.observed_at_ms else (incumbent, candidate) # No distinguishing provenance or provider timestamp -- these two are # already proven identical content, so which raw_id represents them does # not matter for correctness; pick deterministically rather than diff --git a/polylogue/pipeline/ids.py b/polylogue/pipeline/ids.py index 27946efda9..f530183c8c 100644 --- a/polylogue/pipeline/ids.py +++ b/polylogue/pipeline/ids.py @@ -338,10 +338,10 @@ def message_owner_resolution(messages: list[ParsedMessage]) -> MessageOwnerResol keys: list[str] = [] for revision_id, content_id, coordinate in zip(revision_ids, content_ids, coordinates, strict=True): - if revision_counts[revision_id] == 1: - key = revision_id - elif coordinate.stable_key is not None and stable_counts[coordinate.stable_key] == 1: + if coordinate.stable_key is not None and stable_counts[coordinate.stable_key] == 1: key = coordinate.stable_key + elif revision_counts[revision_id] == 1: + key = revision_id elif content_counts[content_id] == 1: key = content_id elif coordinate.stable_key is not None: diff --git a/polylogue/sources/parsers/drive.py b/polylogue/sources/parsers/drive.py index a8eb790cf2..4e8e6ebf82 100644 --- a/polylogue/sources/parsers/drive.py +++ b/polylogue/sources/parsers/drive.py @@ -9,7 +9,6 @@ from polylogue.archive.message.roles import Role from polylogue.archive.message.types import MessageType from polylogue.core.enums import Provider, TitleSource -from polylogue.core.hashing import hash_payload from polylogue.core.json import JSONDocument, json_document from polylogue.core.message_owner import MessageOwnerCoordinate from polylogue.logging import get_logger @@ -380,22 +379,16 @@ def parse_chunked_prompt(provider: Provider | str, payload: JSONDocument, fallba ): session_events.append(usage_event) chunk_attachments = _collect_chunk_attachments(chunk_obj, msg_id) - owner_evidence = sorted( - ( - attachment.provider_attachment_id, - attachment.provider_file_id, - attachment.provider_drive_id, - ) - for attachment in chunk_attachments - if attachment.provider_attachment_id or attachment.provider_file_id or attachment.provider_drive_id - ) - owner_stable_key = ( - "drive-owner-evidence:" + hash_payload({"attachments": [list(values) for values in owner_evidence]}) - if owner_evidence - else None - ) + # Attachment identifiers describe the attachment, not the message + # that owns it. Using them as the message's stable owner key makes a + # metadata-only move between same-timestamp idless chunks look like + # no change at all: the same attachment recreates the same key under + # its new chunk. Keep ownership on the chunk's own provider id or + # physical coordinate instead; ``message_owner_resolution`` then + # supplies the content discriminator for duplicate idless turns and + # fails closed when those turns are genuinely indistinguishable. owner_coordinate = MessageOwnerCoordinate( - stable_key=owner_stable_key, + stable_key=None, position=message_position, variant_index=0, ) diff --git a/tests/unit/archive/test_session_revision_membership.py b/tests/unit/archive/test_session_revision_membership.py index 052ad7b545..74fec6274d 100644 --- a/tests/unit/archive/test_session_revision_membership.py +++ b/tests/unit/archive/test_session_revision_membership.py @@ -1192,3 +1192,31 @@ def test_browser_snapshot_accepts_later_attachment_enrichment_without_provider_u assert result.accepted_raw_ids == ("raw-old", "raw-new") assert result.ambiguous_raw_ids == () + + +def test_observation_order_preserves_newer_mutable_idless_revision_without_provider_timestamp() -> None: + def revision(raw_id: str, text: str, observed_at_ms: int) -> MembershipRevision: + session = ParsedSession( + source_name=Provider.CHATGPT, + provider_session_id="mutable-session", + messages=[ + ParsedMessage( + provider_message_id="", + role=Role.ASSISTANT, + text=text, + timestamp="2026-01-01T00:00:00Z", + ) + ], + ) + return MembershipRevision(raw_id, session_revision_projection(session), observed_at_ms=observed_at_ms) + + result = classify_membership_revisions( + [ + revision("raw-z-old", "before", 1), + revision("raw-a-new", "after", 2), + ] + ) + + assert result.accepted_raw_ids == ("raw-a-new",) + assert result.equivalent_raw_ids == ("raw-z-old",) + assert result.ambiguous_raw_ids == () From 5eddd21f2237112edf8ef6b2d7d7c9425bb79b3f Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 9 Aug 2026 22:21:57 +0200 Subject: [PATCH 18/19] chore(beads): track durable message owner backfill --- .beads/issues.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index ae67b4b40e..43a7fb981c 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -169,6 +169,7 @@ {"_type":"issue","id":"polylogue-tf2.1","title":"Rerun forensics on current archive; price origin_reported providers","description":"Rerun scripts/agent_forensics.py against the current archive (v23+); price origin_reported providers via the vendored LiteLLM catalog (match last path segment); all-provider headline or explicitly-labeled per-provenance figures that cannot be misread; record deltas vs 06-27; verify chart SVGs render. Cache-inclusion must be disambiguated (Codex input INCLUDES cached ~96%; see bd memories). Also blocked on logical-session token attribution — the headline must not be double-counted.","notes":"Correction to close_reason monetary values: stored/provider-priced subset was $239,453.14; catalog API-equivalent was $318,650.88; origin_reported catalog estimate was $79,197.74. The original close_reason text lost dollar-prefixed digits due shell expansion, not measurement drift.","status":"closed","priority":0,"issue_type":"task","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-07-03T04:31:33Z","created_by":"Sinity","updated_at":"2026-07-31T22:35:43Z","started_at":"2026-07-03T09:28:10Z","closed_at":"2026-07-03T09:59:02Z","close_reason":"Completed with blocker caveat preserved: scripts/agent_forensics.py now prices origin_reported rows through the shared vendored LiteLLM pricing catalog while preserving stored provenance; report separates stored/provider-priced cost from catalog API-equivalent estimates and carries logical-session/cache caveats instead of claiming final billing reconciliation. Regenerated current artifact at .agent/demos/agent-forensics against /home/sinity/.local/share/polylogue schema v23: 16,498 physical sessions, 4,142,175 messages, 356.5B tokens, ,453.14 stored/provider-priced subset, ,650.88 catalog API-equivalent, and ,197.74 origin_reported catalog estimate. SVG parse check passed for 9 charts; devtools test tests/unit/scripts/test_agent_forensics.py passed; devtools verify --quick passed run 20260703T095718Z-quick-753466-96559776; devloop-review clean. Remaining final-reconciliation blocker stays open as polylogue-4ts.2.","labels":["area:usage","campaign"],"dependencies":[{"issue_id":"polylogue-tf2.1","depends_on_id":"polylogue-4ts.2","type":"blocks","created_at":"2026-07-03T06:32:45Z","created_by":"Sinity","metadata":"{}"},{"issue_id":"polylogue-tf2.1","depends_on_id":"polylogue-sru.7","type":"blocks","created_at":"2026-07-03T06:31:33Z","created_by":"Sinity","metadata":"{}"},{"issue_id":"polylogue-tf2.1","depends_on_id":"polylogue-tf2","type":"parent-child","created_at":"2026-07-03T06:31:33Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":2,"dependent_count":2,"comment_count":0} {"_type":"issue","id":"polylogue-tf2","title":"Campaign: agent-forensics regeneration + all-provider repricing","description":"Regenerate the agent-forensics packet on the current archive with an honest all-provider headline. The 2026-06-27 report (546.6B tokens, $89,368 API-list equivalent, 216x cache amplification) is the most stranger-legible artifact on any shelf, but its numbers are pre-dedup stale and the headline prices only the priced-provenance subset (Claude Code cost_usd rows); Codex/ChatGPT/Gemini are origin_reported token counts with no dollar value (operator estimate ~$150K all-provider). Sequenced after claim-vs-evidence per operator direction 2026-07-02.","design":"Current slice design: turn the existing agent-forensics/cost headline into a product-backed all-provider repricing artifact. First inspect devtools/scripts and polylogue analyze surfaces for agent_forensics/cost code. Use active archive usage headline (detail=headline) for authoritative physical_session and logical_session_model_high_water token totals. Keep priced-provenance dollars and origin-reported token estimates separate: do not multiply every token by one blended price without a labeled lane. Add or reuse a shared pricing/projection helper so the demo artifact is regenerated from Polylogue product code, not ad hoc SQL. Acceptance for this slice: the generated agent-forensics artifact names archive root/schema, includes physical vs logical token grain, separates priced subset from origin-reported estimate lanes, gives reproduction commands, and has focused tests for any new repricing helper/surface.","acceptance_criteria":"Terminal state: regenerated forensics packet on the current archive with an honest all-provider headline (priced subset AND origin-reported estimate lanes separated), agent_forensics.py folded into polylogue analyze (tf2.2), artifact on the demo shelf with reproduction commands, cold-reader gate passed. Epic closes only when that artifact is recorded.","status":"closed","priority":0,"issue_type":"epic","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-07-03T04:31:32Z","created_by":"Sinity","updated_at":"2026-07-31T22:35:43Z","started_at":"2026-07-03T18:47:23Z","closed_at":"2026-07-03T19:06:44Z","close_reason":"Completed: provider usage headline now exposes product-backed pricing lanes in polylogue analyze usage --detail headline, separating stored/provider-priced cost from catalog API-equivalent estimates for origin_reported rows. Regenerated the current .agent/demos/agent-forensics artifact against /home/sinity/.local/share/polylogue schema v23: physical-session tokens 395,320,980,423; logical high-water tokens 288,741,229,728; stored/provider-priced USD 243,392.189328; catalog API-equivalent USD 337,565.031618; priced lane 13,889 rows / 12,331 sessions / 12,650 matched rows; origin_reported lane 2,308 rows / 2,270 sessions / 2,302 matched rows. Verification: live polylogue --plain analyze usage --detail headline --format json --limit 0 wrote /realm/tmp/polylogue-usage-headline-pricing-current.json; devtools test tests/unit/storage/test_provider_usage_report.py tests/unit/cli/test_diagnostics.py passed 23 tests; devtools verify --quick passed run 20260703T190553Z-quick-2226137-d91d4e8f; devtools workspace demo-shelf --json reported ok. Non-claim preserved: this is not final billing reconciliation and physical/logical token grains stay explicitly separated.","labels":["area:usage","campaign","size:M","spine"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-sru","title":"Campaign: claim-vs-evidence report to finding-grade","description":"Terminal state: an externally publishable finding ('how often do coding agents proceed past failed tool calls, by model/tool') with stated sample frame, calibrated markers, benign/consequential split, seeded stranger-runnable reproduction, and a passed cold-reader gate. Slice closure is NOT campaign closure; this epic stays top-of-frame until its terminal state is recorded.\\n\\nState as of 2026-07-03 after calibrated active-archive regeneration: archive root /home/sinity/.local/share/polylogue, index schema v23, 41,886 structured failures total, 5,000 origin-stratified failures inspected (3,746 claude-code-session, 1,247 codex-session, 7 claude-ai-export), 100 unpaired structured failures. Marker vocabulary was tightened to avoid broad issue/fix/block/gitignored false positives. Immediate next-turn totals: acknowledged=420, silent_proceed=1,205, ambiguous=3,375 (2,624 wordless tool continuations; 751 prose without marker). Lower-bound silent rate is 24.1%; among classified immediate next turns, silent rate is 74.2%. Next-3 sensitivity window, stopping before the next user message, finds 302 acknowledgments that appear only after the next turn; window3 silent lower bound is 37.0%. Calibration: 50 hand-labeled immediate-next-turn rows, acknowledged-marker precision=1.0, recall=0.8421052631578947, invalid rows=0. Artifact: .agent/demos/claim-vs-evidence/claim-vs-evidence.report.json.","notes":"2026-07-03 update: methodology package is now cold-read gated. .agent/demos/claim-vs-evidence contains aggregate live evidence, public-summary.json, PUBLIC_REPRODUCTION.md, COLD_READER_GATE.md, and COLD_READ_RESULT.md. Seeded reproduction is meaningful, not empty: 4 structured failures, 2 acknowledged follow-ups, 2 silent-proceed follow-ups, 0 unpaired. Cold-reader subagent PASS recovered claim/non-claim, sample frame, rates, calibration, caveats, and reproduction commands from the artifact directory only. Remaining campaign child: polylogue-sru.1 productizes action-unit outcome/followup_class capability.","status":"closed","priority":0,"issue_type":"epic","owner":"ezo.dev@gmail.com","created_at":"2026-07-03T04:31:26Z","created_by":"Sinity","updated_at":"2026-07-31T22:35:43Z","closed_at":"2026-07-03T09:28:09Z","close_reason":"Completed: all seven campaign children are closed. The claim-vs-evidence finding now has bounded sample-frame reporting, calibrated marker precision/recall, handler-class and next-3 sensitivity splits, meaningful seeded reproduction, cold-reader PASS, and productized action-unit followup_class/followup_message_ref query capability. Current artifact lives under .agent/demos/claim-vs-evidence and was regenerated against /home/sinity/.local/share/polylogue schema v23.","labels":["area:substrate","campaign"],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-brb07","title":"Backfill durable message assertion owners before reindex","description":"After message identity and user-state owner-scope changes, existing durable message marks and annotations may have target_ref=message:\u003cid\u003e but no session scope_ref. Before any source freeze or index rebuild, inventory those rows, resolve each message owner from the rebuildable index while it exists, and persist canonical session ownership in user.db. Preserve annotation-batch:\u003cid\u003e provenance, reject ambiguous or missing owners into a typed report, and run only through a verified-backup, transactional, idempotent daemon-owned actuator. This is the deferred successor from polylogue-slshy / PR #3898.","acceptance_criteria":"1. Read-only census reports every legacy message mark/annotation with missing session scope, grouped by resolvable, ambiguous, missing, and already-canonical outcomes, with exact counts and row identities. 2. Apply is authorized only after a verified user.db backup and a frozen census digest; it updates only resolvable rows, preserves target_ref and annotation-batch:\u003cid\u003e scope_ref provenance, and stores canonical session ownership in the designated durable owner field. 3. Ambiguous or missing owners remain unchanged and are emitted as typed residuals; no guessed prefix or message-content match is accepted. 4. Re-running the actuator is a no-op with the same digest and receipt, and crash/failure leaves a recoverable transaction state. 5. Focused real user-tier tests cover legacy rows, batch-scoped annotations, ambiguity, backup/rollback, idempotency, and cold reopen; devtools verify --quick passes. 6. A fresh read-only census proves zero resolvable legacy rows remain before source freeze; residuals are linked to named follow-up beads.","status":"open","priority":1,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-08-09T20:20:58Z","created_by":"Sinity","updated_at":"2026-08-09T20:20:58Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-6qjc.1","title":"daemon: persist judgment scheduler receipts and queue health","description":"Complete the additive scheduler receipt and assertion-candidate queue health slice after the judgment actor implementation. Preserve current daemon lifecycle events and expose completed, parked, failed, retryable, and bounded-result state through API, daemon status, CLI, and surfaces.","acceptance_criteria":"1. Scheduler outcomes persist typed receipt rows in ops.db with status, reason, retryability, batch bound, and result counters. 2. Receipt writes are failure-contained and scheduler state remains retryable after transient errors or restart. 3. API, daemon status, CLI, and payload surfaces project parked-pending and scheduler-stalled states from the receipt authority. 4. Real daemon/API/status tests and devtools verify pass on the exact head. 5. No production mutation is performed by the implementation lane; any live deployment remains a named successor.","status":"open","priority":1,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-08-09T14:44:23Z","created_by":"Sinity","updated_at":"2026-08-09T14:44:23Z","labels":["area:orchestration","horizon:mid"],"dependencies":[{"issue_id":"polylogue-6qjc.1","depends_on_id":"polylogue-6qjc","type":"parent-child","created_at":"2026-08-09T14:44:23Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-reindex-preflight-authorization.1","title":"reindex: correct source parse-error and replay preflight predicates","description":"Port the two current-master preflight predicate corrections: count every non-NULL raw_sessions.parse_error, including empty text, as failure evidence; and report replay preflight as fail only when candidate_count exceeds blocked_count. Preserve the existing raw-failure preflight route and current authority gates.","acceptance_criteria":"1. The production preflight source-distribution route counts every non-NULL parse_error, including empty text, and a red twin fails if NULL-only or trimmed-text semantics return. 2. Replay preflight reports fail only when candidate_count \u003e blocked_count, warns for blocked-only work, and preserves existing raw-failure preflight behavior. 3. Focused preflight tests and devtools verify --quick pass on the exact head. 4. No production mutation occurs in this implementation slice. 5. Any remaining preflight authorization scope stays on polylogue-reindex-preflight-authorization.","status":"open","priority":1,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-08-09T14:44:15Z","created_by":"Sinity","updated_at":"2026-08-09T14:44:15Z","labels":["area:maintenance","lane:reindex"],"dependencies":[{"issue_id":"polylogue-reindex-preflight-authorization.1","depends_on_id":"polylogue-reindex-preflight-authorization","type":"parent-child","created_at":"2026-08-09T14:44:15Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-5bxpy","title":"Persist judgment scheduler outcome receipts","description":"The periodic judgment-automation scheduler persisted candidate decisions but did not expose a durable outcome receipt to queue-health or CLI status. A disabled-by-default deployment could append an identical parked event on every tick, while nested failures could emit a detailed receipt and then mask it with a generic outer failure. This leaves operators unable to distinguish a completed empty queue, a deliberately parked capability, and a failed retryable attempt.\n","design":"Persist completed, parked, and failed outcomes in the existing ops-tier daemon event ledger. Correlate each enabled attempt with an operation id, let the innermost production sweep own its detailed receipt, and emit an outer fallback only when no inner route recorded one. Coalesce structurally identical disabled-state receipts for a bounded freshness window. Project the latest typed receipt through assertion-candidate queue health, the public status payload, and the CLI without creating or mutating ops.db during observation.\n","acceptance_criteria":"1. Every enabled bounded scheduler attempt emits exactly one completed, parked, or failed daemon-event receipt with a correlation operation id.\n2. A detailed inner sweep failure is not masked by a generic outer receipt; a coordinator failure before the sweep still emits one retryable fallback receipt.\n3. Repeated identical capability-disabled ticks are coalesced for a bounded freshness window, while a state transition emits a new receipt.\n4. Assertion-candidate queue health and CLI status expose the latest typed receipt state, timestamp, age, and reason through the production read route without initializing an absent ops tier.\n5. Focused scheduler, queue-health, and status tests exercise the production ledger and public projection. Removing receipt emission, coalescing, or inner-receipt ownership makes a focused regression fail.\n6. devtools verify --quick and git diff --check pass on the final head.","status":"closed","priority":1,"issue_type":"task","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-08-08T21:17:17Z","created_by":"Sinity","updated_at":"2026-08-08T21:17:18Z","started_at":"2026-08-08T21:17:17Z","closed_at":"2026-08-08T21:17:18Z","close_reason":"Implemented scheduler outcome receipts, bounded parked-state coalescing, exact operation correlation, inner failure ownership, public queue-health and CLI status projection, with 28 focused tests and all 24 quick gates green in commits 1202fd76a and 351295efa.","labels":["area:assertions","area:daemon","horizon:frontier"],"dependency_count":0,"dependent_count":0,"comment_count":0} From 609ff8085b29e443c36ff5d3f97bda6dc7ba6aac Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 9 Aug 2026 22:23:24 +0200 Subject: [PATCH 19/19] chore(beads): link message owner backfill successor --- .beads/issues.jsonl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 43a7fb981c..60753d2351 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -169,7 +169,7 @@ {"_type":"issue","id":"polylogue-tf2.1","title":"Rerun forensics on current archive; price origin_reported providers","description":"Rerun scripts/agent_forensics.py against the current archive (v23+); price origin_reported providers via the vendored LiteLLM catalog (match last path segment); all-provider headline or explicitly-labeled per-provenance figures that cannot be misread; record deltas vs 06-27; verify chart SVGs render. Cache-inclusion must be disambiguated (Codex input INCLUDES cached ~96%; see bd memories). Also blocked on logical-session token attribution — the headline must not be double-counted.","notes":"Correction to close_reason monetary values: stored/provider-priced subset was $239,453.14; catalog API-equivalent was $318,650.88; origin_reported catalog estimate was $79,197.74. The original close_reason text lost dollar-prefixed digits due shell expansion, not measurement drift.","status":"closed","priority":0,"issue_type":"task","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-07-03T04:31:33Z","created_by":"Sinity","updated_at":"2026-07-31T22:35:43Z","started_at":"2026-07-03T09:28:10Z","closed_at":"2026-07-03T09:59:02Z","close_reason":"Completed with blocker caveat preserved: scripts/agent_forensics.py now prices origin_reported rows through the shared vendored LiteLLM pricing catalog while preserving stored provenance; report separates stored/provider-priced cost from catalog API-equivalent estimates and carries logical-session/cache caveats instead of claiming final billing reconciliation. Regenerated current artifact at .agent/demos/agent-forensics against /home/sinity/.local/share/polylogue schema v23: 16,498 physical sessions, 4,142,175 messages, 356.5B tokens, ,453.14 stored/provider-priced subset, ,650.88 catalog API-equivalent, and ,197.74 origin_reported catalog estimate. SVG parse check passed for 9 charts; devtools test tests/unit/scripts/test_agent_forensics.py passed; devtools verify --quick passed run 20260703T095718Z-quick-753466-96559776; devloop-review clean. Remaining final-reconciliation blocker stays open as polylogue-4ts.2.","labels":["area:usage","campaign"],"dependencies":[{"issue_id":"polylogue-tf2.1","depends_on_id":"polylogue-4ts.2","type":"blocks","created_at":"2026-07-03T06:32:45Z","created_by":"Sinity","metadata":"{}"},{"issue_id":"polylogue-tf2.1","depends_on_id":"polylogue-sru.7","type":"blocks","created_at":"2026-07-03T06:31:33Z","created_by":"Sinity","metadata":"{}"},{"issue_id":"polylogue-tf2.1","depends_on_id":"polylogue-tf2","type":"parent-child","created_at":"2026-07-03T06:31:33Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":2,"dependent_count":2,"comment_count":0} {"_type":"issue","id":"polylogue-tf2","title":"Campaign: agent-forensics regeneration + all-provider repricing","description":"Regenerate the agent-forensics packet on the current archive with an honest all-provider headline. The 2026-06-27 report (546.6B tokens, $89,368 API-list equivalent, 216x cache amplification) is the most stranger-legible artifact on any shelf, but its numbers are pre-dedup stale and the headline prices only the priced-provenance subset (Claude Code cost_usd rows); Codex/ChatGPT/Gemini are origin_reported token counts with no dollar value (operator estimate ~$150K all-provider). Sequenced after claim-vs-evidence per operator direction 2026-07-02.","design":"Current slice design: turn the existing agent-forensics/cost headline into a product-backed all-provider repricing artifact. First inspect devtools/scripts and polylogue analyze surfaces for agent_forensics/cost code. Use active archive usage headline (detail=headline) for authoritative physical_session and logical_session_model_high_water token totals. Keep priced-provenance dollars and origin-reported token estimates separate: do not multiply every token by one blended price without a labeled lane. Add or reuse a shared pricing/projection helper so the demo artifact is regenerated from Polylogue product code, not ad hoc SQL. Acceptance for this slice: the generated agent-forensics artifact names archive root/schema, includes physical vs logical token grain, separates priced subset from origin-reported estimate lanes, gives reproduction commands, and has focused tests for any new repricing helper/surface.","acceptance_criteria":"Terminal state: regenerated forensics packet on the current archive with an honest all-provider headline (priced subset AND origin-reported estimate lanes separated), agent_forensics.py folded into polylogue analyze (tf2.2), artifact on the demo shelf with reproduction commands, cold-reader gate passed. Epic closes only when that artifact is recorded.","status":"closed","priority":0,"issue_type":"epic","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-07-03T04:31:32Z","created_by":"Sinity","updated_at":"2026-07-31T22:35:43Z","started_at":"2026-07-03T18:47:23Z","closed_at":"2026-07-03T19:06:44Z","close_reason":"Completed: provider usage headline now exposes product-backed pricing lanes in polylogue analyze usage --detail headline, separating stored/provider-priced cost from catalog API-equivalent estimates for origin_reported rows. Regenerated the current .agent/demos/agent-forensics artifact against /home/sinity/.local/share/polylogue schema v23: physical-session tokens 395,320,980,423; logical high-water tokens 288,741,229,728; stored/provider-priced USD 243,392.189328; catalog API-equivalent USD 337,565.031618; priced lane 13,889 rows / 12,331 sessions / 12,650 matched rows; origin_reported lane 2,308 rows / 2,270 sessions / 2,302 matched rows. Verification: live polylogue --plain analyze usage --detail headline --format json --limit 0 wrote /realm/tmp/polylogue-usage-headline-pricing-current.json; devtools test tests/unit/storage/test_provider_usage_report.py tests/unit/cli/test_diagnostics.py passed 23 tests; devtools verify --quick passed run 20260703T190553Z-quick-2226137-d91d4e8f; devtools workspace demo-shelf --json reported ok. Non-claim preserved: this is not final billing reconciliation and physical/logical token grains stay explicitly separated.","labels":["area:usage","campaign","size:M","spine"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-sru","title":"Campaign: claim-vs-evidence report to finding-grade","description":"Terminal state: an externally publishable finding ('how often do coding agents proceed past failed tool calls, by model/tool') with stated sample frame, calibrated markers, benign/consequential split, seeded stranger-runnable reproduction, and a passed cold-reader gate. Slice closure is NOT campaign closure; this epic stays top-of-frame until its terminal state is recorded.\\n\\nState as of 2026-07-03 after calibrated active-archive regeneration: archive root /home/sinity/.local/share/polylogue, index schema v23, 41,886 structured failures total, 5,000 origin-stratified failures inspected (3,746 claude-code-session, 1,247 codex-session, 7 claude-ai-export), 100 unpaired structured failures. Marker vocabulary was tightened to avoid broad issue/fix/block/gitignored false positives. Immediate next-turn totals: acknowledged=420, silent_proceed=1,205, ambiguous=3,375 (2,624 wordless tool continuations; 751 prose without marker). Lower-bound silent rate is 24.1%; among classified immediate next turns, silent rate is 74.2%. Next-3 sensitivity window, stopping before the next user message, finds 302 acknowledgments that appear only after the next turn; window3 silent lower bound is 37.0%. Calibration: 50 hand-labeled immediate-next-turn rows, acknowledged-marker precision=1.0, recall=0.8421052631578947, invalid rows=0. Artifact: .agent/demos/claim-vs-evidence/claim-vs-evidence.report.json.","notes":"2026-07-03 update: methodology package is now cold-read gated. .agent/demos/claim-vs-evidence contains aggregate live evidence, public-summary.json, PUBLIC_REPRODUCTION.md, COLD_READER_GATE.md, and COLD_READ_RESULT.md. Seeded reproduction is meaningful, not empty: 4 structured failures, 2 acknowledged follow-ups, 2 silent-proceed follow-ups, 0 unpaired. Cold-reader subagent PASS recovered claim/non-claim, sample frame, rates, calibration, caveats, and reproduction commands from the artifact directory only. Remaining campaign child: polylogue-sru.1 productizes action-unit outcome/followup_class capability.","status":"closed","priority":0,"issue_type":"epic","owner":"ezo.dev@gmail.com","created_at":"2026-07-03T04:31:26Z","created_by":"Sinity","updated_at":"2026-07-31T22:35:43Z","closed_at":"2026-07-03T09:28:09Z","close_reason":"Completed: all seven campaign children are closed. The claim-vs-evidence finding now has bounded sample-frame reporting, calibrated marker precision/recall, handler-class and next-3 sensitivity splits, meaningful seeded reproduction, cold-reader PASS, and productized action-unit followup_class/followup_message_ref query capability. Current artifact lives under .agent/demos/claim-vs-evidence and was regenerated against /home/sinity/.local/share/polylogue schema v23.","labels":["area:substrate","campaign"],"dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"polylogue-brb07","title":"Backfill durable message assertion owners before reindex","description":"After message identity and user-state owner-scope changes, existing durable message marks and annotations may have target_ref=message:\u003cid\u003e but no session scope_ref. Before any source freeze or index rebuild, inventory those rows, resolve each message owner from the rebuildable index while it exists, and persist canonical session ownership in user.db. Preserve annotation-batch:\u003cid\u003e provenance, reject ambiguous or missing owners into a typed report, and run only through a verified-backup, transactional, idempotent daemon-owned actuator. This is the deferred successor from polylogue-slshy / PR #3898.","acceptance_criteria":"1. Read-only census reports every legacy message mark/annotation with missing session scope, grouped by resolvable, ambiguous, missing, and already-canonical outcomes, with exact counts and row identities. 2. Apply is authorized only after a verified user.db backup and a frozen census digest; it updates only resolvable rows, preserves target_ref and annotation-batch:\u003cid\u003e scope_ref provenance, and stores canonical session ownership in the designated durable owner field. 3. Ambiguous or missing owners remain unchanged and are emitted as typed residuals; no guessed prefix or message-content match is accepted. 4. Re-running the actuator is a no-op with the same digest and receipt, and crash/failure leaves a recoverable transaction state. 5. Focused real user-tier tests cover legacy rows, batch-scoped annotations, ambiguity, backup/rollback, idempotency, and cold reopen; devtools verify --quick passes. 6. A fresh read-only census proves zero resolvable legacy rows remain before source freeze; residuals are linked to named follow-up beads.","status":"open","priority":1,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-08-09T20:20:58Z","created_by":"Sinity","updated_at":"2026-08-09T20:20:58Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-brb07","title":"Backfill durable message assertion owners before reindex","description":"After message identity and user-state owner-scope changes, existing durable message marks and annotations may have target_ref=message:\u003cid\u003e but no session scope_ref. Before any source freeze or index rebuild, inventory those rows, resolve each message owner from the rebuildable index while it exists, and persist canonical session ownership in user.db. Preserve annotation-batch:\u003cid\u003e provenance, reject ambiguous or missing owners into a typed report, and run only through a verified-backup, transactional, idempotent daemon-owned actuator. This is the deferred successor from polylogue-slshy / PR #3898.","acceptance_criteria":"1. Read-only census reports every legacy message mark/annotation with missing session scope, grouped by resolvable, ambiguous, missing, and already-canonical outcomes, with exact counts and row identities. 2. Apply is authorized only after a verified user.db backup and a frozen census digest; it updates only resolvable rows, preserves target_ref and annotation-batch:\u003cid\u003e scope_ref provenance, and stores canonical session ownership in the designated durable owner field. 3. Ambiguous or missing owners remain unchanged and are emitted as typed residuals; no guessed prefix or message-content match is accepted. 4. Re-running the actuator is a no-op with the same digest and receipt, and crash/failure leaves a recoverable transaction state. 5. Focused real user-tier tests cover legacy rows, batch-scoped annotations, ambiguity, backup/rollback, idempotency, and cold reopen; devtools verify --quick passes. 6. A fresh read-only census proves zero resolvable legacy rows remain before source freeze; residuals are linked to named follow-up beads.","status":"open","priority":1,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-08-09T20:20:58Z","created_by":"Sinity","updated_at":"2026-08-09T20:20:58Z","dependencies":[{"issue_id":"polylogue-brb07","depends_on_id":"polylogue-slshy","type":"discovered-from","created_at":"2026-08-09T20:23:22Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-6qjc.1","title":"daemon: persist judgment scheduler receipts and queue health","description":"Complete the additive scheduler receipt and assertion-candidate queue health slice after the judgment actor implementation. Preserve current daemon lifecycle events and expose completed, parked, failed, retryable, and bounded-result state through API, daemon status, CLI, and surfaces.","acceptance_criteria":"1. Scheduler outcomes persist typed receipt rows in ops.db with status, reason, retryability, batch bound, and result counters. 2. Receipt writes are failure-contained and scheduler state remains retryable after transient errors or restart. 3. API, daemon status, CLI, and payload surfaces project parked-pending and scheduler-stalled states from the receipt authority. 4. Real daemon/API/status tests and devtools verify pass on the exact head. 5. No production mutation is performed by the implementation lane; any live deployment remains a named successor.","status":"open","priority":1,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-08-09T14:44:23Z","created_by":"Sinity","updated_at":"2026-08-09T14:44:23Z","labels":["area:orchestration","horizon:mid"],"dependencies":[{"issue_id":"polylogue-6qjc.1","depends_on_id":"polylogue-6qjc","type":"parent-child","created_at":"2026-08-09T14:44:23Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-reindex-preflight-authorization.1","title":"reindex: correct source parse-error and replay preflight predicates","description":"Port the two current-master preflight predicate corrections: count every non-NULL raw_sessions.parse_error, including empty text, as failure evidence; and report replay preflight as fail only when candidate_count exceeds blocked_count. Preserve the existing raw-failure preflight route and current authority gates.","acceptance_criteria":"1. The production preflight source-distribution route counts every non-NULL parse_error, including empty text, and a red twin fails if NULL-only or trimmed-text semantics return. 2. Replay preflight reports fail only when candidate_count \u003e blocked_count, warns for blocked-only work, and preserves existing raw-failure preflight behavior. 3. Focused preflight tests and devtools verify --quick pass on the exact head. 4. No production mutation occurs in this implementation slice. 5. Any remaining preflight authorization scope stays on polylogue-reindex-preflight-authorization.","status":"open","priority":1,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-08-09T14:44:15Z","created_by":"Sinity","updated_at":"2026-08-09T14:44:15Z","labels":["area:maintenance","lane:reindex"],"dependencies":[{"issue_id":"polylogue-reindex-preflight-authorization.1","depends_on_id":"polylogue-reindex-preflight-authorization","type":"parent-child","created_at":"2026-08-09T14:44:15Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-5bxpy","title":"Persist judgment scheduler outcome receipts","description":"The periodic judgment-automation scheduler persisted candidate decisions but did not expose a durable outcome receipt to queue-health or CLI status. A disabled-by-default deployment could append an identical parked event on every tick, while nested failures could emit a detailed receipt and then mask it with a generic outer failure. This leaves operators unable to distinguish a completed empty queue, a deliberately parked capability, and a failed retryable attempt.\n","design":"Persist completed, parked, and failed outcomes in the existing ops-tier daemon event ledger. Correlate each enabled attempt with an operation id, let the innermost production sweep own its detailed receipt, and emit an outer fallback only when no inner route recorded one. Coalesce structurally identical disabled-state receipts for a bounded freshness window. Project the latest typed receipt through assertion-candidate queue health, the public status payload, and the CLI without creating or mutating ops.db during observation.\n","acceptance_criteria":"1. Every enabled bounded scheduler attempt emits exactly one completed, parked, or failed daemon-event receipt with a correlation operation id.\n2. A detailed inner sweep failure is not masked by a generic outer receipt; a coordinator failure before the sweep still emits one retryable fallback receipt.\n3. Repeated identical capability-disabled ticks are coalesced for a bounded freshness window, while a state transition emits a new receipt.\n4. Assertion-candidate queue health and CLI status expose the latest typed receipt state, timestamp, age, and reason through the production read route without initializing an absent ops tier.\n5. Focused scheduler, queue-health, and status tests exercise the production ledger and public projection. Removing receipt emission, coalescing, or inner-receipt ownership makes a focused regression fail.\n6. devtools verify --quick and git diff --check pass on the final head.","status":"closed","priority":1,"issue_type":"task","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-08-08T21:17:17Z","created_by":"Sinity","updated_at":"2026-08-08T21:17:18Z","started_at":"2026-08-08T21:17:17Z","closed_at":"2026-08-08T21:17:18Z","close_reason":"Implemented scheduler outcome receipts, bounded parked-state coalescing, exact operation correlation, inner failure ownership, public queue-health and CLI status projection, with 28 focused tests and all 24 quick gates green in commits 1202fd76a and 351295efa.","labels":["area:assertions","area:daemon","horizon:frontier"],"dependency_count":0,"dependent_count":0,"comment_count":0}