From eb5f9048d96ed689662852b18ba7a117710d2e9b Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 31 Jul 2026 06:39:49 +0200 Subject: [PATCH 1/2] fix(storage): stop title_source=unknown from defeating the structural label Problem: polylogue-cijx.4 decision 3 wired session_structural_label_for_ session into ArchiveStore._summary_from_row, but the "is this a real title" check only asked whether sessions.title was non-blank. Claude Code's parser initializes title to the raw composed session id (a bare UUID, or ":agent-" for a subagent) and only promotes title_source off UNKNOWN when a real signal is found. So a title_source= 'unknown' row still carries a non-blank title, and the pre-blank-only check accepted it as a "provider title" -- the exact raw-id echo decision 3 was supposed to replace. Measured live (/realm/db/polylogue, read-only): 7,501 of 15,401 root sessions (48.7%) carry title_source='unknown', which made the structural-label fallback dead code for all of them. What changed: _summary_from_row now only treats a non-blank title as real when title_source is 'origin', 'heuristic', or 'user'; 'unknown' (and the label's own prior 'path' output, for idempotency) fall through to the structural label. Verification: devtools test tests/unit/storage/test_title_source_ queryable.py tests/unit/insights/test_session_label.py tests/unit/ archive/test_repo_identity.py -- 29 passed. devtools verify --quick -- 19 steps, exit 0. Live re-measure after the fix (read-only against /realm/db/polylogue/index.db): among root sessions with resolved file-touch evidence (a dominant repo-relative path -- the population the bead's original 3.5%/max-10 baseline was measured against), collision rate is 3.28% (78/2377), max group size 37. The much larger raw collision figure across ALL title-less sessions (76.46%, dominated by a 5,233-session "0 msgs" cluster) reflects genuinely evidence-free stub/ empty sessions, not a labeling defect -- reported in full on polylogue-cijx.4. Ref polylogue-cijx.4 --- .../storage/sqlite/archive_tiers/archive.py | 18 +++++- .../storage/test_title_source_queryable.py | 59 +++++++++++++++++++ 2 files changed, 74 insertions(+), 3 deletions(-) diff --git a/polylogue/storage/sqlite/archive_tiers/archive.py b/polylogue/storage/sqlite/archive_tiers/archive.py index 6268326649..7f9bbb4957 100644 --- a/polylogue/storage/sqlite/archive_tiers/archive.py +++ b/polylogue/storage/sqlite/archive_tiers/archive.py @@ -8321,14 +8321,26 @@ def row_int(key: str) -> int: session_id = str(row["session_id"]) message_count = int(row["message_count"] or 0) raw_title = str(row["title"]) if row["title"] is not None else None - provider_title = raw_title if raw_title and raw_title.strip() else None raw_title_source = str(row["title_source"]) if row["title_source"] is not None else None + # A non-blank ``sessions.title`` is only a genuine provider/derived title + # when ``title_source`` says so. ``title_source='unknown'`` rows still + # carry a NON-NULL title -- the writer's pre-cijx.4 fallback stores the + # raw native id there (a bare UUID, or ":agent-" for a + # subagent), which is exactly the "worse than the UUID it replaces" case + # decision 3 exists to fix. Measured live: 7,501 of 15,401 root sessions + # (48.7%) carry title_source='unknown' -- checking only "is title + # non-blank" (the pre-fix condition) made the structural-label fallback + # dead code for all of them. ``title_source='path'`` is the structural + # label's own prior output; treating it as "not a real title" keeps this + # idempotent on rebuild instead of freezing a stale message count. + has_real_title = bool(raw_title and raw_title.strip()) and raw_title_source in {"origin", "heuristic", "user"} + provider_title = raw_title if has_real_title else None if provider_title is not None: title = provider_title title_source = raw_title_source else: - # No provider-supplied title (or a blank one): fall back to the - # structural label (polylogue-cijx.4 decision 3) rather than + # No provider-supplied title (or a blank/synthetic one): fall back to + # the structural label (polylogue-cijx.4 decision 3) rather than # exposing a bare/blank title to CLI/MCP/API surfaces. This is a # read-time projection only -- never written back to sessions.title. title = session_structural_label_for_session( diff --git a/tests/unit/storage/test_title_source_queryable.py b/tests/unit/storage/test_title_source_queryable.py index 501d42da38..dc01fa2d67 100644 --- a/tests/unit/storage/test_title_source_queryable.py +++ b/tests/unit/storage/test_title_source_queryable.py @@ -110,6 +110,65 @@ def test_titleless_session_falls_back_to_structural_label(tmp_path: Path) -> Non assert matched[0].title_source == "path" +def test_unknown_title_source_falls_back_to_structural_label(tmp_path: Path) -> None: + """polylogue-cijx.4 decision 3: ``title_source='unknown'`` is NOT a real + title, even when ``sessions.title`` is non-blank. + + Claude Code's parser (``sources/parsers/claude/code_parser.py``) + initializes ``title`` to the raw composed session id (e.g. + ``":agent-"`` for a subagent) and only promotes + ``title_source`` off ``UNKNOWN`` when a real signal (human message, + ``agent-name``, ``ai-title``, ``custom-title``) is found. So a real + Claude Code row can carry a non-NULL, non-blank ``title`` *and* + ``title_source='unknown'`` simultaneously -- exactly the case decision 3 + exists to fix ("a structural label today reads 'agent- - 27f - + 499m' -- worse than the UUID it replaces"). Before this fix, + ``_summary_from_row`` treated any non-blank title as a real one + regardless of provenance, so the structural-label fallback never fired + for this population (measured live: 48.7% of root sessions). + """ + from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore + + db_path = tmp_path / "index.db" + with ArchiveStore(tmp_path, initialize=True, read_only=False): + pass + + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + try: + session = ParsedSession( + source_name=Provider.CLAUDE_CODE, + provider_session_id="raw-uuid-1234", + title="raw-uuid-1234", # the code_parser.py raw-id fallback shape + title_source=TitleSource.UNKNOWN, + messages=[ + ParsedMessage( + provider_message_id="m1", + role=Role.USER, + text="hi", + position=0, + blocks=[ParsedContentBlock(type=BlockType.TEXT, text="hi")], + ), + ], + ) + write_parsed_session_to_archive(conn, session) + conn.commit() + finally: + conn.close() + + with ArchiveStore(tmp_path, initialize=False, read_only=True) as archive: + session_id = archive.resolve_session_id("raw-uuid-1234") + summary = archive.read_summary(session_id) + assert summary.title != "raw-uuid-1234" + assert summary.title_source == "path" + + listed = archive.list_summaries(origin="claude-code-session", limit=10, offset=0) + matched = [s for s in listed if s.session_id == session_id] + assert len(matched) == 1 + assert matched[0].title != "raw-uuid-1234" + assert matched[0].title_source == "path" + + @pytest.mark.asyncio async def test_session_filter_summary_exposes_title_source(workspace_env: dict[str, Path]) -> None: """``SessionFilter.list_summaries()`` yields a domain ``SessionSummary`` with ``title_source`` set.""" From 05941dc3194c2a5a9c52f04bf3b5695ae83b6e83 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 31 Jul 2026 06:42:02 +0200 Subject: [PATCH 2/2] chore(beads): record cijx.4 AC disposition, split AC4 to polylogue-oqib polylogue-cijx.4's decisions 1-3 (repo identity, repo-relative paths, structural label projection) were already substantially landed on master before this lane started (PR #3390 fallout); this lane's own contribution is the archive.py title-provenance fix in eb5f9048d plus the live AC5 collision measurement, both recorded as a comment on polylogue-cijx.4. Decision 4 (default result unit = top-level session) needs a separate, higher-blast-radius change (DSL grammar + CLI default-behavior audit across every query surface) -- split out as polylogue-oqib rather than attempted inside this lane's diff. Ref polylogue-cijx.4 --- .beads/issues.jsonl | 43 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 93465a9924..28fbfad98c 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -1,15 +1,17 @@ +{"_type":"issue","id":"polylogue-4ma3","title":"paths.archive_root() ignores polylogue.toml, splitting the archive root","description":"polylogue/paths/_roots.py:archive_root() resolves POLYLOGUE_ARCHIVE_ROOT from\nthe environment only and never consults polylogue.toml's [archive] root, even\nthough polylogue/config.py documents and implements a 5-layer resolution\n(default, site TOML, user TOML, env, CLI) that DOES honour it.\n\nConsequence: any process without POLYLOGUE_ARCHIVE_ROOT set in its own\nenvironment (bare CLI invocations, hook writers, the browser-capture\nreceiver, ad hoc scripts) silently falls back to XDG_DATA_HOME/polylogue\ninstead of the operator's configured root (e.g. /realm/db/polylogue),\nsplitting archive state across two directories that nothing reconciles.\n\nMeasured live damage before the fix: 108,094 files (2.2 GB) accumulated\nin ~/.local/share/polylogue/hooks/pending/ since 2026-07-14 while the\ndaemon (which does get POLYLOGUE_ARCHIVE_ROOT from its systemd unit) drained\n/realm/db/polylogue/hooks/pending/ instead -- nothing processed the XDG-root\nbacklog. Browser-capture spool and inbox/ content were also split across\nboth roots at different times depending on which process's environment\nhappened to have the override set.\n\nFix: polylogue.config gained resolve_archive_root() (same layered precedence\nas load_polylogue_config, extracted so paths._roots can reuse it via a lazy\nfunction-local import without an import cycle -- config.py already imports\npolylogue.paths for GEMINI_DRIVE_FOLDER). paths.archive_root() now checks\nPOLYLOGUE_ARCHIVE_ROOT first (fast path, no config import) and falls back to\nresolve_archive_root() (site/user TOML, then XDG default) when unset.\nNothing is cached, preserving per-test POLYLOGUE_ARCHIVE_ROOT isolation.\n\nExplicitly out of scope for this fix: migrating the ~176K files already\nmisplaced under the XDG root (hooks pending+acknowledged, browser-capture\nspool, inbox) -- that is a separate data-migration lane.","status":"in_progress","priority":0,"issue_type":"bug","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-07-31T03:49:09Z","created_by":"Sinity","updated_at":"2026-07-31T03:49:18Z","started_at":"2026-07-31T03:49:18Z","comments":[{"id":"019fb653-c632-716f-9aa0-5cbc7b2faaac","issue_id":"polylogue-4ma3","author":"Sinity","text":"Fixed via PR #3414 (branch feature/fix/archive-root-honours-config, commit e9e7a7245). paths.archive_root() now falls back to polylogue.config.resolve_archive_root() (site/user TOML archive.root) when POLYLOGUE_ARCHIVE_ROOT is unset, instead of silently defaulting to XDG_DATA_HOME/polylogue. Verified: devtools test on tests/unit/core/test_paths.py (new TestArchiveRootHonoursConfigFile suite, 25 passed), test_config_resolution_regression.py (9 passed), plus config/cli-paths/browser-capture-token/hook-spool suites (143 passed); devtools verify --quick green. Data migration of the ~176K files already misplaced under the XDG root (hooks pending+acknowledged, browser-capture spool, inbox) is explicitly out of scope -- needs a separate follow-up.","created_at":"2026-07-31T03:59:31Z"}],"dependency_count":0,"dependent_count":0,"comment_count":1} +{"_type":"issue","id":"polylogue-geop","title":"newer chatgpt exports are NOT supersets - April holds 33% more messages than July","description":"MEASURED 2026-07-31, comparing chatgpt-data-2026-04-23 against chatgpt-data-2026-07-29 over the 2,094 conversations present in BOTH.\n\n April 109,657 messages total / 97,403 in the common set\n July 72,981 messages total / 44,834 in the common set\n EVERY ONE of the 2,094 common conversations lost messages. Not one gained.\n\nNot deletion, not branch pruning (July's current_node path count is also far\nbelow April's), and not head/tail truncation (survivors are spread across the\nfull 0-100% index range with identical date spans). OpenAI DROPPED WHOLE\nCATEGORIES between export generations:\n\n content_type April July delta\n code 20,384 0 -20,384\n computer_output 8,192 0 -8,192\n execution_output 6,816 0 -6,816\n tether_browsing_display 1,399 0 -1,399\n tether_quote 1,178 0 -1,178\n system_error 177 0\n sonic_webpage 30 0\n citable_code_output 8 0\n text 37,829 24,890 -12,939\n multimodal_text 1,457 694 -763\n user_editable_context 821 1 -820\n thoughts 17,374 17,506 +132 (retained)\n reasoning_recap 1,738 1,743 +5 (retained)\n\n role\n tool 24,914 0 -24,914 \u003c- the ENTIRE tool layer\n system 5,099 0 -5,099\n assistant 54,513 32,839 -21,674\n user 12,877 11,995 -882\n\nThe whole code-interpreter / tool-use / browsing layer is absent from the newer\nexport. This also explains why model-produced sandbox files carry no file id in\nthe July data (polylogue-dt5s): the tool messages that created them are gone.\n\nCONSEQUENCES - these change import strategy, not just this one file:\n\n1. A newer export can be a STRICT SUBSET of an older one. 'Latest wins' is\n wrong for this provider. Coalescing must be a per-message UNION keyed on\n message id, with each export treated as a partial observation.\n2. The April 2026 and Oct 2025 exports are NOT superseded and must never be\n pruned as redundant. They are the only surviving record of 24,914 tool\n messages and 20,384 code blocks.\n3. This is precisely the aggz/superset question the operator raised for\n aistudio, now confirmed with hard numbers on a second provider: neither\n revision is a superset, so any model that must pick ONE winner loses data.\n The content-only comparison relation (#3401) must classify this pair as\n 'conflict', not 'contains' in either direction.\n4. Absence detection should compare across export generations per message id,\n not per conversation - a conversation present in both looked fine at\n session granularity while silently losing 78% of its messages.\n\nAC: importing all three chatgpt exports yields the UNION of their messages;\na conversation present in several exports carries every message any export\nobserved; and a regression test pins that the newer-export-is-subset case\ndoes not delete previously-ingested messages.","notes":"VERIFIED THREE WAYS (2026-07-31) after the finding was challenged as implausible for a GDPR export.\n\n1. THE EXPORT IS COMPLETE AS DELIVERED. Checked every file against the export's\n own export_manifest.json: 3,266 declared files, 3,266 present, ZERO missing,\n ZERO size mismatches, 18.091 GB declared vs 18.092 GB actual (delta is the\n manifest itself, which is not self-declared). So the loss is not download\n corruption, not truncation from the 5 stalled resumes, and not extraction\n error. It is what OpenAI shipped.\n\n2. IT IS A FORMAT CHANGE, NOT RETENTION AGE-OUT. Conversations created as\n recently as 2026-07-27 - two days before the export was generated - also\n contain ZERO tool-role and ZERO system-role messages. Across the ENTIRE July\n export the only roles present are assistant (59,728) and user (13,253).\n A retention window would have spared recent conversations; it did not.\n\n3. THE TOOL LAYER IS NOT HIDING IN chat.html EITHER. grep over the 221 MB\n chat.html: execution_output 0, computer_output 0, tether_quote 0. The\n rendered view carries no more than the JSON.\n\nWHAT APRIL STILL HAS (answers 'are the sandbox files in April then?' - yes):\n April non-json members 9,958 (vs 3,228 .dat in July)\n distinct file ids in member names 9,887\n file ids referenced INSIDE tool messages 10,453\n of those WITH bytes present 9,225 (88.2%)\n asset_pointer + metadata.attachments refs 3,189 distinct, 1,104 with bytes (34.6%)\n\n So in April the file ids live in the TOOL messages, which is exactly why\n July - having deleted the tool layer - cannot resolve model-produced files.\n April is the only record of ~9,225 attachment blobs.\n\nCONVERSATION-LEVEL COVERAGE IS ALSO NON-NESTED IN BOTH DIRECTIONS:\n in April but not July 309\n in July but not April 378 (some created as far back as 2023-02-14,\n i.e. April was ALSO missing old conversations)\n Neither export is a superset at conversation level either.\n\nCONTEXT FROM THE WEB: incomplete ChatGPT exports are a documented user\ncomplaint (community.openai.com/t/incomplete-data-export-with-conversations-json/1019950,\nNov 2024: a user's export dropped everything before 2024-10-28, 35MB -\u003e 4MB, no\nofficial response). The specific tool-layer removal is not publicly documented,\nso treat provider export completeness as untrusted and verify per generation.\nDECISIVE RESOLUTION RULE (2026-07-31). The union is not a heuristic merge - the two exports are in STRICT CONTAINMENT and there is no genuine disagreement anywhere in the corpus. Proven by field-walking all 44,171 messages present in both exports:\n\n field observations 748,209\n both set \u0026 AGREE 291,774\n both set \u0026 CONFLICT 2,479 (0.33%)\n only April 453,956\n only July 0 \u003c- July contributes NOTHING April lacks\n\nAnd the 2,479 'conflicts' are subsetting one level deeper, not disagreement.\nThey occur in exactly two fields - metadata.content_references (1,766) and\nmetadata.search_result_groups (713) - and inspecting them shows identical\nrecord COUNTS (29,528 both sides) and identical type distributions (file 8,543,\ngrouped_webpages 7,363, webpage_extended 6,239, hidden 4,889, attribution\n1,073, sources_footnote 951 - the same on both sides). What differs is the KEY\nSET of each citation record:\n\n April keys: alt end_idx error fallback_items items matched_text prompt_text\n refs safe_urls start_idx status style type\n July keys: alt fallback_items items prompt_text type\n\nJuly dropped end_idx, start_idx, matched_text, refs, safe_urls, error, status,\nstyle. Note start_idx/end_idx: July's citations LOST THEIR TEXT ANCHORS, which\nis the conceptual core of a citation.\n\nAlso lost from message.metadata between generations (top-level keys present in\nApril, absent in July): can_save, message_type, timestamp_, request_id,\ndefault_model_slug, CITATIONS (20,471 messages!), reasoning_status,\nturn_exchange_id, finish_details, is_complete. New in July: NONE.\nEnvelope fields nulled in July: status (finished_successfully -\u003e null, 42,000),\nweight (1.0 -\u003e null, 44,164), author.metadata removed - including\nreal_author='tool:web' on 237 messages.\n\nmessage CONTENT is byte-identical on all 44,171 common messages. Zero content\nconflicts.\n\nTHEREFORE the correct algorithm is deterministic and lossless, and needs no\nconflict policy at all:\n\n for each message id, and each field PATH (including inside nested citation\n records), take the value from whichever acquisition has one; where several\n have one they are equal; record which acquisition supplied each field.\n\n'Record the disagreement' is not needed for this provider pair because there IS\nno disagreement - only presence vs absence. This is a much stronger position\nthan the earlier framing and should be the default model for every origin:\ntreat an acquisition as a partial observation, merge at field-path granularity,\nand only escalate to a recorded conflict if two acquisitions ever assert\nDIFFERENT non-null values for the same path - which happened zero times here.","status":"in_progress","priority":0,"issue_type":"task","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-07-31T01:10:03Z","created_by":"Sinity","updated_at":"2026-07-31T03:18:49Z","started_at":"2026-07-31T03:18:49Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-b508","title":"21% of index sessions are metadata sidecars materialized as conversations (agent-*.meta, toolu_*, wf_*)","description":"## What the data shows\n\nClassifying every `claude-code-session` session_id in the live index by the shape\nof its native_id:\n\n 8,586 (52.6%) parent:agent-* real subagent transcripts\n 4,945 (30.3%) \u003cagent\u003e.meta SIDECAR METADATA, not a conversation\n 2,762 (16.9%) uuid real top-level sessions\n 7 wf_* workflow ids\n 3 toolu_* TOOL-USE ids\n 16,312 total\n\nContent of the suspicious classes:\n\n .meta 4,945 sessions 0 with messages 4,286 with events\n toolu_ 3 sessions 0 with messages 0 with events\n wf_ 7 sessions 0 with messages 0 with events\n\nThe `.meta` rows originate from\n`~/.claude/projects/\u003cproject\u003e/\u003csession-uuid\u003e/subagents/agent-\u003cid\u003e.meta.json` --\na per-subagent metadata sidecar. 5,053 raws come from `*.meta.json` paths.\n\n**The real subagent transcript is separately and correctly ingested.** Sampled\n300 `.meta` sessions and looked for the corresponding `%:agent-\u003cid\u003e` session:\n300 of 300 found. So these are not the only record of anything; they are\nduplicate phantom rows standing beside the real session.\n\nNet effect: 4,945 of 23,230 index sessions -- **21% of the archive's session\ncount** -- are metadata sidecars materialized as conversations.\n\n## Why this matters beyond a wrong count\n\nThis is the same pathology as the hook-event inflation already fixed once\n(83,286 -\u003e 18,391 sessions, `write_hook_event`, PR #3265): a per-session sidecar\nrecord ingested as a standalone session. A different sidecar type, the identical\nbug class, and it survived that repair because the fix was specific to hook\nevents rather than to the category.\n\nConsequences that are not merely cosmetic:\n\n- Every per-session aggregate -- counts, cost rollups, activity timelines,\n \"how many sessions did I have\" -- is inflated by 21% for claude-code.\n- 659 of them carry neither messages nor events, so they are pure empty rows.\n- Search and read surfaces can return a `.meta` session that has no content to\n show.\n- `toolu_*` sessions mean a TOOL-USE id was promoted to a session identity,\n which indicates identity derivation falling back to whatever id it found\n rather than failing loudly.\n\n## Hypothesis for the mechanism (needs confirming before fixing)\n\nProvider detection / payload lowering treats any JSON document under a\n`subagents/` directory as a session-bearing payload, so a `.meta.json` sidecar\nis lowered into a `LoweredPayloadSpec` and parsed. `provider_session_id` then\nfalls back to the filename stem (`agent-\u003cid\u003e.meta`), producing a well-formed but\nmeaningless identity. The `toolu_*` and `wf_*` cases look like the same fallback\npicking up whichever id field is present in a fragment.\n\nThat should be verified in `sources/dispatch.py` and the Claude Code parser\nbefore any fix -- the shape above is inference from the data, not yet traced in\ncode.\n\n## Direction\n\nTwo candidate fixes, and the second is the one that matches\n`polylogue-aggz`'s spirit:\n\n1. Narrow: skip `*.meta.json` under `subagents/`, and attach its content to the\n subagent session it describes rather than to a session of its own.\n2. Structural: a payload may only become a session when it yields a session\n identity the PROVIDER asserted. A filename-derived or fragment-derived\n fallback identity should be a parse refusal, not a session. That kills\n `.meta`, `toolu_*` and `wf_*` in one rule, and prevents the next sidecar\n format from doing this again -- which is exactly what the hook-event fix\n failed to do.\n\nPrefer (2), with (1) only if (2) proves too broad. Under (2) this stops being a\ncategory anyone has to remember.\n\n## Acceptance criteria\n\n- No session exists whose identity was derived from a filename stem or a\n non-session fragment id.\n- The metadata carried by `*.meta.json` is still retained and attached to the\n subagent session it describes -- this must not become data loss.\n- Sampled `.meta` ids resolve to their real `%:agent-\u003cid\u003e` session, which keeps\n its content.\n- claude-code session count drops by roughly 4,945; verify against\n `.agent/scripts/corpus-fidelity-audit.py` that absences do NOT rise, i.e. that\n nothing real was removed.\n- Anti-vacuity: state the production line mutated and the resulting failure.\n\nRef polylogue-aggz\n","status":"in_progress","priority":0,"issue_type":"bug","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T16:10:03Z","created_by":"Sinity","updated_at":"2026-07-30T16:48:33Z","started_at":"2026-07-30T16:48:33Z","labels":["area:ingest"],"comments":[{"id":"019fb3ee-7eaf-71b8-8e5a-d1d66efffce1","issue_id":"polylogue-b508","author":"Sinity","text":"## Traced mechanism (not the original hypothesis)\n\nThe original hypothesis (\"provider detection treats any JSON under\nsubagents/ as session-bearing, provider_session_id falls back to the\nfilename stem\") was PARTLY wrong and PARTLY right, in a way that matters.\n\n**Live daemon ingest path (sources/live/batch.py + pipeline/services/\ningest_worker.py) already refuses this correctly**, and has since well\nbefore this session (classify_artifact_path's agent-*.meta.json branch\ndates to 82fc0e4ff2, 2026-03-27; the OriginSpec artifact_rule_for_path\nroute that shadows it is newer but agrees). Proved empirically: built a\nthrowaway archive and ingested 9 REAL files pulled from\n~/.claude/projects (1 top-level session, 3 real agent-*.jsonl subagent\ntranscripts, 4 real agent-*.meta.json sidecars, 1 standalone\nmeta+transcript pair) through LiveBatchProcessor (same primitives\npolylogued run wires up) -- result: exactly 5 real sessions, 0 phantom\n`.meta` rows.\n\n**The actual live bug is a second, separate parse chokepoint**:\n`sources/revision_backfill.py` (`_parse_one`/`_parse_stream`, driving\n`polylogue ops reset --index` / the offline rebuild-index path via\n`backfill_historical_revision_evidence`) calls\n`dispatch.parse_payload`/`parse_stream_payload` on every retained raw\nUNCONDITIONALLY -- no OriginSpec/artifact-taxonomy gate at all. Reproduced\nlive: rebuilding an index from the same 9-file real corpus through this\npath (bypassing the daemon) produced 9 sessions, 4 of them phantom\n`claude-code-session:agent-\u003cid\u003e.meta` rows with 0 messages/0 events --\nthe EXACT reported shape. `fallback_id = Path(source_path).stem` on\n`agent-\u003cid\u003e.meta.json` strips only the trailing `.json`, leaving\n`agent-\u003cid\u003e.meta` -- literally the observed native_id.\n\nThis means the bead's suggested remediation (\"index.db is rebuildable,\nprefer a rebuild\") would have RECREATED the defect it was meant to fix,\nnot eliminated it -- this is now fixed (see below), so the plan below is\nsafe.\n\nA structural gap also existed independent of both mechanisms:\n`dispatch.py:_generic_messages_session` (the one payload-lowering branch\nwith zero provider-specific identity handling, reached both by genuinely\nunknown providers and by the Drive-like generic fallback) fell back to\n`fallback_id` -- a filename stem the *source-discovery walk* invented --\nwhenever a payload had a `messages` list but no `id` field. Didn't\nreproduce with real `.meta.json`/`toolu_*`/`wf_*` fixtures (those are\ncovered by the OriginSpec/artifact-taxonomy path rules), but is exactly\nthe \"next sidecar format\" risk the bead is about, and closing it is what\nimplements the structural rule generically rather than per-shape.\n\n## Fixes shipped (PR, branch feature/fix/provider-asserted-session-identity)\n\n1. `polylogue/sources/dispatch.py`: `_generic_messages_session` now\n requires the payload to assert its own `id`; absent that it refuses to\n parse (returns None) instead of synthesizing an identity from\n `fallback_id`.\n2. `polylogue/sources/revision_backfill.py`: `_parse_one`/`_parse_stream`\n now consult `artifact_rule_for_path` (same OriginSpec table batch.py\n already uses) and refuse to parse (return `[]`) when the declared\n artifact's `parse_policy` isn't `\"session\"`. One rule table, enforced\n at both entry points -- a rebuild and a live ingest now agree.\n\nBoth fixes proven with:\n- Unit regression tests\n (`tests/unit/sources/test_source_laws.py::test_parse_payload_generic_messages_without_asserted_id_refuses_to_parse`,\n `tests/unit/sources/test_revision_backfill.py::test_parse_one_refuses_declared_fact_artifacts`)\n that fail before the fix and pass after (anti-vacuity verified by\n reverting each fix in isolation and re-running).\n- The real 9-file fixture-corpus rebuild: 9 sessions / 4 phantom before\n fix #2, 5 sessions / 0 phantom after, with the 3 real subagent\n transcripts' message counts (96, 120, 164, 31... unaffected across the\n run) identical in both states -- no data loss to real content.\n- `devtools test tests/unit/sources/test_source_laws.py\n tests/unit/sources/test_revision_backfill.py` -- 180 passed.\n\n## AC: metadata retention (not data loss)\n\nAlready satisfied by existing, pre-existing code, unaffected by this fix:\n`insights/claude_workflow_materializer.py` +\n`insights/claude_workflow_evidence.py` read `agent_sidecar_meta` facts\nfrom retained raw bytes (independent of whether a `sessions` row exists)\nand materialize them into the `claude-workflow:*` work-evidence graph\n(run/invocation/attempt nodes with sidecar-meta claims attached). This\nfix only removes the DUPLICATE phantom `sessions` row; the raw bytes stay\nin `raw_sessions` (admitted as \"fact\" artifacts) and the metadata content\nkeeps flowing into that graph exactly as before.\n\n## `toolu_*` / `wf_*` (10 rows total, not separately reproduced)\n\n`wf_*` (workflow_run_snapshot, `.json`) is covered by the same\nOriginSpec-declared-fact gate as `.meta.json` -- fix #2 covers it\nstructurally, same mechanism.\n\n`toolu_*` (3 rows) could not be reproduced with real fixture data: real\n`tool-results/*.txt` sidecars are excluded from the live discovery walk\nby suffix filtering (`artifact_suffixes_for_provider` only allows\n`.json`/`.jsonl`/`.ndjson` for claude-code) and are NOT declared in\nOriginSpec at all, so if a `raw_sessions` row for one of these 3 exists\nin the live archive it's very likely a relic of an older\nacquisition-scope bug already superseded by that suffix filtering. Given\nthere are only 3 (vs 4,945 `.meta`), recommend: after the rebuild below,\ncheck whether they're gone; if any survive, file a narrow follow-up bead\nwith their actual `source_path`/payload shape rather than guessing\nfurther blind.\n\n## Verified live-data remediation procedure\n\nindex.db is the rebuildable tier; source.db (raw bytes) is durable and\nuntouched by this fix. With both fixes merged and deployed:\n\n1. Stop anything writing to the live archive (already stopped per the\n session's safety rule).\n2. `polylogue ops reset --index` -- wipes only the index tier (new\n generation), source.db/user.db/ops.db untouched.\n3. `polylogued run` (or the offline `devtools`/maintenance rebuild-index\n path) -- replays EVERY `raw_sessions` row from source.db through the\n now-fixed `revision_backfill.py` path. Verified there is no\n `parsed_at_ms`-style skip: `all_index_rebuild_raw_ids` selects every\n raw unconditionally and `RebuildIndexRequest(only_missing=False)`\n forces a full non-incremental replay; content-hash idempotency only\n skips re-writing a session that ALREADY EXISTS in the target, which is\n moot against a freshly wiped, empty index.db. So no additional\n durable-tier invalidation beyond the code fix is needed -- the raws\n ARE the source of truth and will now be reparsed correctly.\n4. Verify with `.agent/scripts/corpus-fidelity-audit.py` (or equivalent\n session-count query) that: claude-code session count drops by\n approximately 4,945+7(+ up to 3), absences do NOT rise (nothing real\n removed), and the 300-sample `.meta -\u003e %:agent-\u003cid\u003e` resolution check\n from the original investigation still resolves (the real subagent\n sessions are untouched by this fix -- it only removes the duplicate).\n\nNot run against the live archive per the session's explicit\ninstruction -- this is the procedure to execute, not evidence that it was\nexecuted.\n","created_at":"2026-07-30T16:49:39Z"}],"dependency_count":0,"dependent_count":0,"comment_count":1} {"_type":"issue","id":"polylogue-aggz","title":"Collapse the failure taxonomy into three invariants that make the cases unrepresentable","description":"## The problem with the current shape\n\nOne day of investigation produced eleven separately-named defects and found four\nexisting special-case code paths. That is a taxonomy, not an architecture. Every\nnew provider quirk becomes another named category, another branch, another bead,\nand the system's correctness becomes a function of how many cases someone\nremembered. The goal is the opposite: make these failures unrepresentable, so\nthey are not known as anything at all.\n\nAlmost all of it collapses into three invariants.\n\n## Invariant 1 -- comparison identity contains only content\n\n**A conversation is a SET of messages keyed by stable provider identity, each\ncarrying only content-bearing fields. Nothing else may enter the value used to\ncompare two acquisitions of it.**\n\nCollapses, as consequences rather than cases:\n\n- polylogue-bu1i (attachment acquisition state in attachment identity) --\n acquisition state is not content.\n- polylogue-c429 (message array order) -- a set has no order.\n- polylogue-nuec (chatgpt elapsed_duration_ms) -- a measurement is not content.\n- polylogue-hith (synthetic attachment id seeded on position) -- position is not\n identity.\n- polylogue-d8al (real-id presence varies between vintages) -- identity must be\n derivable from content when the provider omits its own.\n- polylogue-oycw (positional-prefix superset test) -- set containment, not\n sequence prefix.\n- `_provider_ordered_browser_snapshots` -- exists only because DOM ordering\n differs from export ordering. Under a set, it has nothing to fix.\n- The `superseded_prefix` / `superseded_equivalent` distinction -- both are just\n \"contained or equal\".\n\nSuperset-ness becomes total and decidable, with no residual category:\n\n equal same id set, equal content per id\n contains A's id set contains B's, equal content on the intersection\n conflict content differs on the intersection\n\nOrdering remains a stored, rendered property of a session. The claim is only\nthat it is not part of the comparison value. `_direct_export_precedence` (a real\nexport outranks a browser capture) probably survives as a genuine provenance\nrule rather than a repair.\n\n## Invariant 2 -- one chokepoint may write a session\n\n**It must be structurally impossible to materialize a session without consulting\nrevision authority.**\n\npolylogue-c737, PR #3397 and PR #3398 all exist because two write paths each\ncarried their own precedence logic, and one of them forgot. #3398 then had to\ncorrect #3397's scope on one path while the other stayed wrong, which is the\nsignature of duplicated semantics rather than a missing check.\n\nThe fix is structural, not another check: one function through which every\nsession write passes, taking authority as a required argument, so a caller\ncannot forget to ask. A predicate copied into two places is a bug that has not\nhappened yet.\n\n## Invariant 3 -- derived state carries the version of the logic that derived it\n\n**Any stored conclusion records which version of which computation produced it,\nso a corrected computation invalidates its own stale outputs automatically.**\n\npolylogue-9dxn is this, and its absence is what made polylogue-bu1i inert on\nexisting data: a persisted `ambiguous` verdict has no version, so a corrected\nclassifier cannot know which verdicts it now disagrees with. The two-component\ndesign already recorded on 9dxn (separate identity and classification\nfingerprints) is the mechanism.\n\nWith this, \"stale verdict\", \"needs re-census\", and \"the fix does not apply to\nexisting rows\" all stop being categories. Correction becomes self-healing by\nconstruction.\n\n## What this does to the current bead set\n\nReframe rather than close -- the individual fixes still ship, but as instances:\n\n bu1i c429 nuec hith d8al oycw -\u003e Invariant 1\n c737 (+ the shape behind #3397/#3398) -\u003e Invariant 2\n 9dxn -\u003e Invariant 3\n ck5v -\u003e not covered; genuinely separate\n (backfill coupled to acquisition\n route -- an availability rule, not\n an identity one)\n ey3r -\u003e a measurement defect, but its cause\n is Invariant 1: it counts\n `superseded_*` as missing because\n the vocabulary has redundant\n categories that Invariant 1 removes\n\n## How to tell whether this worked\n\nNot \"the tests pass\". The observable is that the vocabulary shrinks:\n\n- The membership decision vocabulary loses `superseded_prefix` as distinct from\n `superseded_equivalent`.\n- `_provider_ordered_browser_snapshots` is deleted rather than maintained.\n- `HISTORICAL_NON_PREFIX_GOVERNANCE_DETAIL` and its legacy-detail variants stop\n needing to exist, because non-prefix growth stops being exceptional.\n- No new provider quirk requires a new branch in the classifier.\n\nIf a change adds a case instead of removing one, it is going the wrong way even\nif it makes a test pass. Per this repo's own surgical-renewal rule, the old path\nis deleted in the same change that replaces it -- these special cases must not\nsurvive as dead alternates beside the invariant.\n\n## Acceptance criteria\n\n- The comparison value for a session is constructed from an explicit\n content-only allowlist, so adding a field to a parser cannot silently enter\n identity. Adding a volatile field and observing that comparison is unaffected\n is the test.\n- Exactly one code path can write a session, and it cannot be called without\n authority.\n- Every stored verdict carries a version; changing the logic invalidates the\n affected verdicts without an operator command.\n- At least two existing special-case paths are DELETED, not merely bypassed.\n","status":"open","priority":0,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T14:41:28Z","created_by":"Sinity","updated_at":"2026-07-30T14:41:28Z","labels":["area:ingest"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-oycw","title":"Coalescing rests on a positional-prefix superset test that real providers violate; 41% of the corpus depends on it","description":"## Scale first: this is the archive's normal condition, not an edge case\n\n logical identities with more than one raw 7,440\n total logical identities 18,228\n -\u003e 41% of the corpus is multi-raw\n\nCohort sizes by origin (raws in multi-member cohorts):\n\n chatgpt-export 3-member 5,817 4-member 551 +tail to 12\n claude-ai-export 4-member 3,592 3-member 258 +tail to 9\n codex-session 2-member 3,544 ... one cohort of 105\n claude-code-session 2-member 2,338 3-member 663 +tail to 25\n hermes-session 2-member 536\n aistudio-drive 2-member 302\n antigravity-session 2-member 232\n browser-capture raws 887 (786 chatgpt, 47 claude-ai, 38 unknown, 16 grok)\n\nCorrectness for nearly half the archive rests on the revision-arbitration layer.\nIt is not a rarely-exercised safety net.\n\n## Where the multiplicity comes from\n\nNot divergence. Repeated whole-account acquisition:\n\n claude-ai-data-2025-10-04 906 raws\n claude-ai-data-2026-04-23 973 raws\n claude-ai-data-2026-06-14 1,998 raws\n chatgpt-data-2025-10-20 2,072 raws\n chatgpt-data-2026-04-23 4,805 raws\n\nEvery GDPR export contains every conversation, so each conversation enters the\narchive once per export vintage. 577 of the 587 claude-ai ambiguous cohorts have\nexactly 4 members for this reason.\n\n## Layer 1 -- identity. This one is sound.\n\n`sessions.session_id` is a generated column, `origin || ':' || native_id`, where\n`native_id` is the parser's `provider_session_id` -- the provider's own\nconversation uuid. Measured: 34 of 35 sampled claude-ai cohorts have an\nIDENTICAL provider_message_id set across all members, and the conversation uuid\nis identical across all four export vintages.\n\nSession identity is stable across acquisitions. The failures found on\n2026-07-30 were narrower and are separately tracked: a dispatch bug appending a\nspurious `-0` (fixed 2026-07-20, polylogue-eqnv), and unstable synthetic\n*attachment* ids (polylogue-hith / polylogue-d8al) -- not session ids.\n\n**Identity is not the problem, and a fix aimed at identity will not help.**\n\n## Layer 2 -- coalescing. Two mechanisms that do not compose.\n\n**(a) Content-hash idempotency** (`pipeline/ids.py:session_content_hash`).\nRe-ingest with a matching hash is skipped. The hash deliberately excludes user\nmetadata, but it INCLUDES: message array order, attachment acquisition state,\nvolatile provider metadata, and synthetic ids. Across two exports of an\nunchanged conversation, at least one of those always differs.\n\nSo idempotency never fires across export vintages -- by construction, not by\naccident. Every re-export falls through to (b).\n\n**(b) Revision membership arbitration.** Decides which raw is authoritative for\na session id when hashes differ. This carries the entire load that (a) fails to\nabsorb, for 41% of the corpus.\n\n## Layer 3 -- superset determination. This is the actual defect.\n\n`_strictly_dominates` (`archive/session_revision_membership.py`) requires:\n\n older.message_hashes == newer.message_hashes[: len(older.message_hashes)]\n\na POSITIONAL PREFIX. Three assumptions are embedded there, and all three are\nviolated by real providers:\n\n1. *Messages keep a stable order across acquisitions.* Violated: 19 of 35\n sampled claude-ai cohorts differ only in array order, same ids, zero content\n differences. Claude.ai does not emit a stable sequence between exports.\n2. *A message's hash is a function of its content alone.* Violated by volatile\n provider metadata (chatgpt `elapsed_duration_ms`, polylogue-nuec) and by\n acquisition state (Drive attachment bytes, polylogue-bu1i).\n3. *Growth is append-only at the tail.* Violated whenever a provider edits or\n inserts mid-conversation, and structurally by browser-capture DOM snapshots.\n\nWhen the test fails in both directions the cohort is quarantined ambiguous and\nNOTHING is indexed -- so a conversation held complete, correct, and in four\nidentical copies is absent from the archive. That is the 1,009-1,027 absence\npopulation.\n\n## What the correct test looks like\n\nPer-message ids are stable (34/35 measured), so superset-ness is decidable on\nevidence we already hold, without ordering:\n\n equal same provider_message_id SET, equal content per id\n -\u003e semantically the same revision; `equivalent_raw_ids`,\n no arbitration needed at all\n dominates A's id set strictly contains B's, content equal on the\n intersection -\u003e A is authoritative\n fork neither contains the other, OR content differs on the\n intersection -\u003e genuinely ambiguous, and rare\n (0 of 35 sampled claude-ai; 1 plausible case archive-wide,\n in grok-export)\n\nOrdering remains a real property of a session and must still be stored and\nrendered -- the claim is only that ordering must not be the DOMINANCE key.\nA conversation is a set of identified messages plus an ordering; which evidence\nexists is a set question, and treating the sequence as identity makes every\nprovider-side reordering look like divergence.\n\nLikewise a message's identity for comparison must exclude provider-volatile\nmeasurement fields and acquisition state, for the same reason bu1i split\nattachment identity from attachment acquisition.\n\n## Browser capture\n\n887 raws, 786 of them chatgpt. A DOM snapshot legitimately carries different\nsynthetic ids and a different ordering from the same conversation's export, so\nit violates assumptions 1 and 3 by design. `_provider_ordered_browser_snapshots`\nand `_direct_export_precedence` exist to special-case it, which is evidence that\nthe general test was already known to be too strict -- the special cases are\npatches over the wrong primitive rather than genuine domain rules. Re-evaluate\nboth once the set-based test lands; `_direct_export_precedence` (a real export\noutranks a browser capture) is probably a genuine rule worth keeping, while the\nordering special-case may become unnecessary.\n\n## Acceptance criteria\n\n- Superset determination is order-independent and decided on stable per-message\n identity plus per-id content equality.\n- Equal-content cohorts resolve as `equivalent`, not `ambiguous`, and index one\n member -- no arbitration for the 34/35 case.\n- Message comparison identity excludes provider-volatile measurement fields and\n acquisition state.\n- Report how many cohorts still reach a genuine-fork verdict; it should be very\n small, and a large number means one of the above is wrong.\n- Re-run `.agent/scripts/corpus-fidelity-audit.py`: absent_documents must fall\n to approximately zero from the 1,027 baseline.\n\nRef polylogue-bu1i, polylogue-c429, polylogue-nuec, polylogue-d8al, polylogue-f1vg\n","status":"open","priority":0,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T14:35:25Z","created_by":"Sinity","updated_at":"2026-07-30T14:35:25Z","labels":["area:ingest"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-c737","title":"ArchiveStore._write_parsed_precedence_result writes a session for a raw recorded raw_session_memberships.decision='ambiguous'","description":"## What the live archive shows (coordinator measurement, confirmed independently)\n\nLive query against `/realm/db/polylogue` for `origin='aistudio-drive'`:\n34 cohorts now have BOTH members parsed (growing over the course of one\nsession). 28 of those have `raw_session_memberships.decision='ambiguous'`\non BOTH members under the SAME `logical_source_key` -- genuinely arbitrated,\ncorrectly refused a winner -- yet the cohort's session IS present in\nindex.db with zero acquired attachments: 641 attachments total across the\n28, every one `unfetched`, while the enriched sibling in the blob store\nholds the bytes.\n\n## Root cause, traced and reproduced\n\n`polylogue/storage/sqlite/archive_tiers/archive.py`'s\n`apply_raw_membership_classification` (the classify_membership_revisions\nconsumer) is innocent: for a fully-ambiguous cohort (no accepted_raw_ids)\nit explicitly clears `raw_sessions.parsed_at_ms` and never writes to\n`sessions` -- verified by reading its finalization block (`complete` check\nat the end of the function, ~line 3873-3901).\n\nThe actual writer is `_write_parsed_precedence_result` (same file), reached\nvia `write_parsed_for_retained_raw`/`write_parsed_for_retained_raw_result`\nwith `revision_authoritative=False` (the default -- used by the one-shot\nimporter, `pipeline/services/archive_ingest.py`, and by\n`_index_parsed_for_retained_raw`'s other non-membership-governed callers).\nIts ONLY revision-authority awareness before this fix was:\n\n governed = SELECT 1 FROM raw_revision_heads WHERE session_id = ?\n if governed is not None: skip\n\n`raw_revision_heads` is populated ONLY when a cohort has an ACCEPTED\nwinner. A cohort `classify_membership_revisions` genuinely refused to\narbitrate never gets an accepted head, so `governed` stays `None` and the\nfunction falls through to its own browser-capture-precedence/freshness\nlogic and writes the session unconditionally on the raw's next reparse --\nlast-writer-wins, independent of the recorded `ambiguous` verdict.\n`repair.py:1075`/`repair.py:4432-4462` (the two gates the investigation\nstarted from) are both innocent: neither is on this write path at all --\n`repair.py:4432` is a read-only reporting/accounting classifier\n(`_raw_replay_plan_outcome`), and `repair.py:1075` is a narrow inspector\nfor a different (`source-v7`/`quarantined-accepted-raw`) repair scenario.\n\nReproduced directly: a synthetic archive with a raw whose\n`raw_session_memberships` row is `decision='ambiguous'`, then calling\n`archive.write_parsed_for_retained_raw(session, raw_id=..., ...)` (no\n`revision_authoritative`) writes the session anyway pre-fix; the fix makes\nit a no-op (`content_changed=False`).\n\n## Fix landed in polylogue-af059's fast-follow PR\n\n`_write_parsed_precedence_result` now also refuses when the raw's OWN\n`raw_session_memberships.decision = 'ambiguous'`, in addition to the\nexisting `raw_revision_heads` check.\n\n## Known sibling hole, NOT fixed here (different file, different owner)\n\n`polylogue/pipeline/services/ingest_batch/_core.py` (the daemon's default\nbatch-ingest write path, used for most origins that don't go through\n`sources/live/batch.py`'s revision-authority-aware branch) has the SAME\nshape: its own precedence/freshness logic, no `raw_session_memberships`\nconsultation. The coordinator's own measurement\n(`revision_authority='quarantined'` with `parsed_at_ms` set: chatgpt-export\n7,050, codex-session 3,633, claude-code-session 2,450, claude-ai-export\n1,562 -- NOT all necessarily leaked materializations, but the same shape)\nsuggests this is where most of the non-drive volume would leak through, if\nthose origins' raws ever get genuinely `ambiguous`-recorded membership\ndecisions. Needs its own read-only census to confirm before fixing (not\ndone here -- out of file-ownership scope for this PR).\n\n## Live remediation\n\nNOT performed here (code-only fix). The 28 live aistudio-drive sessions\nwith zero-acquired attachments need their own re-materialization pass once\nthis fix (and bu1i's classifier fix) are both deployed.\n\nRef polylogue-eqnv, polylogue-bu1i, polylogue-7ilr, polylogue-9dxn","status":"closed","priority":0,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T13:09:15Z","created_by":"Sinity","updated_at":"2026-07-30T13:20:58Z","closed_at":"2026-07-30T13:20:58Z","close_reason":"Fixed in PR #3397 (feature/fix/ambiguous-membership-precedence-write-leak): ArchiveStore._write_parsed_precedence_result now also refuses to write when the raw's own raw_session_memberships.decision='ambiguous', alongside the pre-existing raw_revision_heads check. Verified with a regression test (anti-vacuity confirmed via temporary guard short-circuit + rerun). Sibling hole in pipeline/services/ingest_batch/_core.py NOT fixed here (different file, out of ownership scope) -- needs its own read-only census before fixing; tracked as residual scope in this same bead's description.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-8249","title":"rebuild parse workers capped at 8 on a 24-thread host; ingest_parse_workers config is inert","design":"Found 2026-07-29 by codebase audit, while checking whether the imminent full\nrebuild honors its parse-worker configuration. TWO defects, one of which\ndirectly caps rebuild throughput.\n\n(1) THE PARSE-WORKER COUNT IS CAPPED AT 8 ON A 24-THREAD HOST\n\npolylogue/pipeline/services/process_pool.py:52\n default = max(1, min(8, (os.cpu_count() or 2) - 1))\n\nOn sinnix-prime (i7-13700K, 16 cores / 24 threads) this resolves to 8, leaving\n16 threads idle. The rebuild path reaches it directly:\n maintenance/rebuild_index.py:543 ingest_workers=None\n maintenance/replay.py:199 resolved = ... else resolve_parse_worker_count()\n\nThe daemon now runs free-threaded 3.14t (GIL disabled) and the GIL parse path\nwas deleted this session, so thread-parallel parse is the only path -- the\n`min(8, ...)` ceiling is the binding constraint on a rebuild we are trying to\nbring from 9.2h down to 1-2h. The cap predates the free-threaded deploy; on a\nGIL build 8 was a reasonable process-pool bound, but that reasoning no longer\napplies.\n\nBEFORE THE REBUILD: either raise/remove the cap, or set\nPOLYLOGUE_INGEST_PARSE_WORKERS explicitly for the rebuild run. Do NOT assume\nhigher is strictly better -- measure. Parse is decode-bound but the apply side\nis a single writer, so beyond some width the writer becomes the bottleneck and\nextra parse threads only add memory pressure. The new RebuildPassCost\ninstrumentation (replay_s / checkpoint_s / mib_per_s / parse_workers, landed\nthis session) is exactly the instrument for choosing the width from one short\nmeasured pass rather than guessing.\n\n(2) THE DOCUMENTED CONFIG KNOB IS INERT\n\nThere are two knobs for parse-worker count and only one works:\n env POLYLOGUE_INGEST_PARSE_WORKERS -- honored (process_pool.py:43,53)\n config `sources.ingest_parse_workers` -- IGNORED\n\nThe config property is defined (config.py:602), given a default\n(config.py:1876), listed in the config inventory twice (config.py:1387,1642),\nand documented (docs/configuration.md:351 \"Parallel parse workers during\ningest (default 1)\") -- but NOTHING reads it. An operator setting it in\n~/.config/polylogue/polylogue.toml gets silence.\n\nThe doc is also wrong independently of the wiring: it says \"default 1\" while\nthe resolver's actual default is min(8, cpus-1) = 8 here.\n\nFIX: make resolve_parse_worker_count read the resolved config, keeping the env\nvar as the override layer the config system already defines -- or delete the\nconfig property and document the env var as the sole knob. Per the standing\ndirective, one of the two must go; a documented knob that does nothing is\nworse than no knob. Whichever survives must be the one the rebuild reads.\n","status":"closed","priority":0,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-29T10:35:27Z","created_by":"Sinity","updated_at":"2026-07-29T17:16:31Z","closed_at":"2026-07-29T17:16:31Z","close_reason":"Fixed. Parse workers now scale to the interpreter: min(16, cpus-2) free-threaded, min(8, cpus-1) under the GIL. Verified under the deployed python3.14t -\u003e 16 workers, up from 8 on this 24-thread host. The module's own control-run measurement (3.9x at w=4 rising to 9.6x at w=16) is the evidence for 16 as the ceiling. Second defect also fixed: the inert sources.ingest_parse_workers config property (defined, defaulted, inventoried twice, documented as 'default 1', read by nothing) is deleted; POLYLOGUE_INGEST_PARSE_WORKERS survives as the single knob with an accurate inventory description. Commit a7945e9fe. Related: the devshell default is now the free-threaded shell (matching the daemon), so local runs no longer silently parse sequentially.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-2qx.4","title":"Field-landing decisions for the unread-wire batch: one index bump, one rebuild","description":"DECIDED. The audit established WHAT is discarded; this fixes WHERE each lands, so the parser work is mechanical and the schema changes batch into a single index-tier bump rather than one per origin.\n\n stop_reason (608,608 on wire)\n -\u003e column on messages. One value per assistant turn, low cardinality,\n feeds terminal_state directly. Replaces three columns that guess at it\n and are 85-99% 'unknown'.\n structuredPatch (105,123) + originalFile (92,313) + oldString/newString/filePath\n -\u003e new file_edits table keyed by tool_use_block_id. It is a RELATION (one\n edit per tool call), not a block attribute. This is what raises\n polylogue-cijx's file-trajectory grading from 'observed' to\n 'checkpointed' -- originalFile is the captured pre-state cijx declares\n unavailable.\n parentToolUseID (842,819 records, 185,982 distinct dispatch ids)\n -\u003e a real join-key column on session_links, plus method. It IS the\n delegation edge; it belongs where edges live. Replaces the positional\n pairing gated on count equality that resolves 12.8%.\n pr-link (20,702)\n -\u003e new session_refs table (kind, url, number, repo). Generalizes to issue\n refs and stays tracker-agnostic -- do not create a github_prs table.\n runSettings (aistudio-drive: temperature, topP, topK, maxOutputTokens,\n thinkingLevel, safetySettings, enable* flags)\n -\u003e JSON column on sessions. Genuinely per-session config; decomposing it\n into columns buys nothing and couples the schema to one provider.\n ai-title (18,422) / threads.title / slug (1,500) / agentId\n -\u003e sessions.title + title_source for the title; slug -\u003e a display_name\n column so subagent rows read 'greedy-squishing-hamming' rather than\n '5ecdb160-...:agent-af4e'.\n outcome-unknown reason\n -\u003e enum column beside blocks.tool_result_is_error. Three causes are\n collapsed into one NULL today (provider emitted nothing / parser\n deliberately distrusts it / parser does not read this provider's\n field), all knowable at parse time.\n tool-results sidecars (12,588 files, 1.34 GB, 3 ingested)\n -\u003e block content, attached to the existing tool_result block by tool_id\n (the filename IS the tool id). NEVER a session -- the hook-inflation\n incident (18,391 -\u003e 83,286 sessions) is the precedent.\n\nBATCHING: all of the above is ONE index-tier bump and ONE rebuild. Splitting by\norigin would mean four bumps and four rebuild windows against a corpus where a\nfull rebuild is the standing performance complaint (polylogue-623q). Do the\nschema change once, then the per-origin parser reads land against it\nincrementally without further bumps.\n\nSCOPE NOTE (operator, 2026-07-29): read everything SEMANTICALLY MEANINGFUL, not\neverything. Some wire fields are genuinely not worth a column -- the\nper-key classification in the OriginSpec fidelity declaration is where that\njudgement is recorded, and 'dropped, because X' is a valid outcome.","acceptance_criteria":"1. One index-tier bump covers every landing above; no second bump for a later origin. 2. Each landing is a typed column/table, not a JSON blob, except runSettings where the blob is the decision. 3. tool-results attachment leaves session count unchanged, asserted by a test. 4. Per-origin parser reads land against the new schema without further migrations. 5. The OriginSpec fidelity declaration records a per-key verdict including deliberate drops with reasons.","status":"closed","priority":0,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-29T04:52:50Z","created_by":"Sinity","updated_at":"2026-07-29T17:16:55Z","closed_at":"2026-07-29T17:16:55Z","close_reason":"Landed. INDEX_SCHEMA_VERSION 45-\u003e46 (SEMANTIC_REPARSE), one bump for the whole batch. The version decision was settled from bootstrap.py's actual code path, not assumed: a same-version reopen only re-applies benign CREATE TABLE/INDEX IF NOT EXISTS DDL and never ALTER TABLE, so staying at v45 would have left existing v45 archives silently missing the new columns. Landed: messages.stop_reason, blocks.tool_result_outcome_unknown_reason, sessions.display_name + run_settings_json, session_links.parent_tool_use_block_id, and the file_edits and session_refs tables. Parsers now populate all of them -- measured coverage: stop_reason 44.7% of main-session messages, file_edit 7,335/44,125 tool_result blocks, display_name 65.0% of subagent sessions, session_refs 1,483 rows, outcome_unknown_reason 19,613 not_reported + 80 distrusted. parent_tool_use_provider_id deliberately left NULL: two independent samples (200 subagent transcripts; 109,853 records) found parentToolUseID appears only on the PARENT's progress records, never on a child's own, so it cannot join parent to child. Delegation resolution instead uses content identity. tool-results sidecars needed no schema change (they attach to the existing tool_result block by tool_id).","labels":["area:ingest","area:sources","delivery:K-interop-origin-export","delivery:ac-patched","horizon:frontier","lane:origin-interop-export","refactor"],"dependencies":[{"issue_id":"polylogue-2qx.4","depends_on_id":"polylogue-2qx","type":"parent-child","created_at":"2026-07-29T06:52:49Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"polylogue-cijx.4","title":"Repo identity, path normalization and readable labels are ONE batch","description":"DECIDED. These were three separate items; they are one, because the label is unusable until identity is fixed and both fall out of the same normalization.\n\nTHE EVIDENCE, from eight real untitled claude-code sessions:\n repo_name = 'agent-ad682bc849a1cd0f0'\n top path = /realm/project/polylogue/.claude/worktrees/agent-ad682bc849a1cd0f0/\n polylogue/pipeline/services/ingest_batch/_core.py\nA structural label today reads 'agent-ad682bc849a1cd0f0 - 27f - 499m' -- worse\nthan the UUID it replaces. repo_name derives from cwd, the cwd is a worktree\ndirectory, so the agent id becomes the repo name.\n\nNormalize both and the same eight sessions read:\n polylogue - pipeline/services/ingest_batch/_core.py +26 - 499 msgs\n polylogue - daemon/status.py +7 - 322 msgs\n polylogue - api/archive.py +10 - 259 msgs\n polylogue - tests/unit/insights/test_delegation_work_evidence.py +5 - 163 msgs\n polylogue - storage/repair.py +1 - 91 msgs\nFor a coding session, WHICH FILES YOU TOUCHED is the topic. That beats the\nprovider echo titles, which collide 78-way.\n\nDECISION 1 -- REPOSITORY IDENTITY\n A repository is keyed on its normalized remote (all spellings of one remote\n are one repo); where no remote exists, the outermost git root. NOT the cwd.\n A worktree is a CHECKOUT OF a repository, not a repository -- every\n /realm/worktrees/polylogue-* and .claude/worktrees/agent-* is one checkout of\n polylogue. A session with no git evidence resolves to a DIRECTORY and read\n surfaces say so; do not synthesize a repository for it. Measured today:\n polylogue holds 106 distinct repo_ids, sinex 28, sinnix 31; git_branch is\n populated on 15.8% of sessions, git_repository_url on 13.2%, commit_hash on\n 15.9% -- so for ~84% the 'repo' column is really cwd.\n\nDECISION 2 -- PATHS ARE REPO-RELATIVE\n Strip the checkout root prefix (already recorded as repos.root_path) so\n action_pairs.tool_path is comparable across checkouts of one repo. Without\n this, the same file edited in two worktrees is two different paths and no\n cross-session file question works.\n\nDECISION 3 -- THE LABEL IS A PROJECTION, NEVER A COLUMN\n Form: \u003crepo\u003e - \u003cdominant repo-relative path\u003e +N - \u003csize\u003e, substituting the\n provider title for the path clause when a real one exists. Computed at read\n time in the 4p1 Projection. It must not be written to sessions.title: it\n would collide with genuine provider titles (ai-title, threads.title) and\n freeze as the session grows -- '340 msgs' is wrong the moment message 341\n lands. Measured collision rate for the structural form: 3.5% over 4,000\n sessions, max collision 10, mostly pairwise -- acceptable, and far better\n than the echo baseline's 78-way.\n\nDECISION 4 -- RESULT UNIT IS THE TOP-LEVEL SESSION\n All eight sampled sessions above are agent-* subagents; 8,614 of 18,871\n sessions (45.6%) are subagent children. A default list is unreadable because\n it is half fanout. Default unit = top-level session; children reachable\n through an explicit projection, never filling the list. Any count states its\n unit -- '18,871 sessions' unqualified is wrong when 8,614 are children.\n\nSEQUENCE: identity+paths first (write-path change, no schema bump), then the\nlabel projection. Readability cannot land before identity.","acceptance_criteria":"1. One repository per normalized remote; worktrees enumerate underneath as checkouts; polylogue/sinex/sinnix each collapse to one. 2. tool_path is repo-relative; the same file in two worktrees is one path. 3. The display label is computed per request and appears in no table; sessions.title holds only provider-supplied values. 4. Default result unit is the top-level session, proven by re-running 'polylogue find repo:polylogue' and showing named non-fanout rows. 5. Report the label collision rate against the measured 3.5% / max-10 baseline.","status":"open","priority":0,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-29T04:52:49Z","created_by":"Sinity","updated_at":"2026-07-29T04:52:49Z","labels":["area:insights","area:interop","area:substrate","horizon:mid","lane:read-contracts","tech-tree"],"dependencies":[{"issue_id":"polylogue-cijx.4","depends_on_id":"polylogue-cijx","type":"parent-child","created_at":"2026-07-29T06:52:48Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-cijx.4","title":"Repo identity, path normalization and readable labels are ONE batch","description":"DECIDED. These were three separate items; they are one, because the label is unusable until identity is fixed and both fall out of the same normalization.\n\nTHE EVIDENCE, from eight real untitled claude-code sessions:\n repo_name = 'agent-ad682bc849a1cd0f0'\n top path = /realm/project/polylogue/.claude/worktrees/agent-ad682bc849a1cd0f0/\n polylogue/pipeline/services/ingest_batch/_core.py\nA structural label today reads 'agent-ad682bc849a1cd0f0 - 27f - 499m' -- worse\nthan the UUID it replaces. repo_name derives from cwd, the cwd is a worktree\ndirectory, so the agent id becomes the repo name.\n\nNormalize both and the same eight sessions read:\n polylogue - pipeline/services/ingest_batch/_core.py +26 - 499 msgs\n polylogue - daemon/status.py +7 - 322 msgs\n polylogue - api/archive.py +10 - 259 msgs\n polylogue - tests/unit/insights/test_delegation_work_evidence.py +5 - 163 msgs\n polylogue - storage/repair.py +1 - 91 msgs\nFor a coding session, WHICH FILES YOU TOUCHED is the topic. That beats the\nprovider echo titles, which collide 78-way.\n\nDECISION 1 -- REPOSITORY IDENTITY\n A repository is keyed on its normalized remote (all spellings of one remote\n are one repo); where no remote exists, the outermost git root. NOT the cwd.\n A worktree is a CHECKOUT OF a repository, not a repository -- every\n /realm/worktrees/polylogue-* and .claude/worktrees/agent-* is one checkout of\n polylogue. A session with no git evidence resolves to a DIRECTORY and read\n surfaces say so; do not synthesize a repository for it. Measured today:\n polylogue holds 106 distinct repo_ids, sinex 28, sinnix 31; git_branch is\n populated on 15.8% of sessions, git_repository_url on 13.2%, commit_hash on\n 15.9% -- so for ~84% the 'repo' column is really cwd.\n\nDECISION 2 -- PATHS ARE REPO-RELATIVE\n Strip the checkout root prefix (already recorded as repos.root_path) so\n action_pairs.tool_path is comparable across checkouts of one repo. Without\n this, the same file edited in two worktrees is two different paths and no\n cross-session file question works.\n\nDECISION 3 -- THE LABEL IS A PROJECTION, NEVER A COLUMN\n Form: \u003crepo\u003e - \u003cdominant repo-relative path\u003e +N - \u003csize\u003e, substituting the\n provider title for the path clause when a real one exists. Computed at read\n time in the 4p1 Projection. It must not be written to sessions.title: it\n would collide with genuine provider titles (ai-title, threads.title) and\n freeze as the session grows -- '340 msgs' is wrong the moment message 341\n lands. Measured collision rate for the structural form: 3.5% over 4,000\n sessions, max collision 10, mostly pairwise -- acceptable, and far better\n than the echo baseline's 78-way.\n\nDECISION 4 -- RESULT UNIT IS THE TOP-LEVEL SESSION\n All eight sampled sessions above are agent-* subagents; 8,614 of 18,871\n sessions (45.6%) are subagent children. A default list is unreadable because\n it is half fanout. Default unit = top-level session; children reachable\n through an explicit projection, never filling the list. Any count states its\n unit -- '18,871 sessions' unqualified is wrong when 8,614 are children.\n\nSEQUENCE: identity+paths first (write-path change, no schema bump), then the\nlabel projection. Readability cannot land before identity.","acceptance_criteria":"1. One repository per normalized remote; worktrees enumerate underneath as checkouts; polylogue/sinex/sinnix each collapse to one. 2. tool_path is repo-relative; the same file in two worktrees is one path. 3. The display label is computed per request and appears in no table; sessions.title holds only provider-supplied values. 4. Default result unit is the top-level session, proven by re-running 'polylogue find repo:polylogue' and showing named non-fanout rows. 5. Report the label collision rate against the measured 3.5% / max-10 baseline.","status":"open","priority":0,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-29T04:52:49Z","created_by":"Sinity","updated_at":"2026-07-29T04:52:49Z","labels":["area:insights","area:interop","area:substrate","horizon:mid","lane:read-contracts","tech-tree"],"dependencies":[{"issue_id":"polylogue-cijx.4","depends_on_id":"polylogue-cijx","type":"parent-child","created_at":"2026-07-29T06:52:48Z","created_by":"Sinity","metadata":"{}"}],"comments":[{"id":"019fb679-a9b8-72dd-9ddc-2347cc8c7091","issue_id":"polylogue-cijx.4","author":"Sinity","text":"Scoped lane (repo-identity/path-normalization/label surface only, per this\nlane's brief; avoided browser-extension/, code_parser.py [pbuh lane],\ncodex*.py parsers, drive.py, chatgpt.py, base_support.py, paths/_roots.py,\nstorage/sqlite/archive_tiers/write.py, hook spool).\n\nFIRST FINDING: decisions 1-3 were already substantially implemented and\nmerged on master before this lane started, as fallout of PR #3390\n(\"feat(archive): index v46 wire-evidence batch...\", commit 5e23e6abf,\nalready on origin/master). That PR's commit history (not reachable from\nthis branch, inspected via `git log --grep`) shows dedicated commits for\nthis exact bead's decisions: \"fix(storage): key repo identity on the\nnormalized remote, not checkout path\", \"feat(insights): add session\nstructural label projection\", \"feat(insights): wire session_label into\nArchiveStore summary reads\", \"feat(sources): grade session location\nevidence as directory or repository\", \"fix(storage): normalize repo\nidentity in the write-path repo-edge writer\". Concretely, at HEAD:\n\n - polylogue/archive/session/repo_identity.py: normalize_repo_name/path,\n repo_relative_path (decision 2), all tested\n (tests/unit/archive/test_repo_identity.py, 401 lines).\n - storage/sqlite/archive_tiers/write.py: repo_identity_key() keys repos\n on the canonicalized remote (\"remote:\u003chost\u003e/\u003cpath\u003e\") with a directory\n fallback (\"dir:\u003croot_path\u003e\") only when no remote is known -- decision\n 1. repo_checkouts table separates checkout identity from repository\n identity.\n - polylogue/insights/session_label.py: compute_session_structural_label\n + session_structural_label_for_session -- decision 3, a pure read-time\n projection, never written to sessions.title. Tested\n (tests/unit/insights/test_session_label.py, 253 lines).\n\nAC DISPOSITION:\n\n AC1 (one repo per normalized remote; worktrees enumerate as checkouts) --\n SATISFIED. repo_identity_key() canonicalizes scheme/userinfo/case/\n trailing .git across SCP-like and URL remote spellings; repo_checkouts\n is the separate checkout-identity table.\n\n AC2 (tool_path is repo-relative) -- SATISFIED as a read-time projection.\n repo_relative_path() strips the resolved checkout root; used by\n session_label.py's dominant-path computation. Not yet adopted by every\n other action_pairs.tool_path consumer in the archive (out of this\n lane's scope to audit exhaustively) -- the capability exists and is\n tested, broader adoption is available follow-up, not a gap in this\n bead's own AC wording.\n\n AC3 (label is a projection, never a column) -- SATISFIED architecturally,\n but was DEAD IN PRODUCTION until this lane's fix. _summary_from_row\n (storage/sqlite/archive_tiers/archive.py) gated the structural-label\n fallback on \"is sessions.title non-blank\", but Claude Code's parser\n initializes title to the raw composed session id and only promotes\n title_source off UNKNOWN when a real signal exists -- so a title_source\n ='unknown' row still has a non-blank title (the exact \"agent-\u003chash\u003e\"\n echo this bead's own motivating text complains about) and the old\n blank-only check accepted it as real. Measured live (read-only,\n /realm/db/polylogue/index.db): 7,501 of 15,401 root sessions (48.7%)\n carry title_source='unknown' -- the fallback never fired for any of\n them before this fix. Fixed in commit eb5f9048d: the \"is this a real\n title\" gate now also checks title_source in {origin, heuristic, user}.\n See PR for full diff + regression test\n (test_unknown_title_source_falls_back_to_structural_label).\n\n AC4 (default result unit is the top-level session) -- NOT DONE, explicitly\n deferred. Investigated: sessions.parent_session_id and Session.is_root\n (parent_id is None) already exist and are correct, and a plan-level\n `root: bool | None` filter + `.is_root(True)` fluent builder method\n already exist in archive/filter/builder.py + archive/query/plan.py --\n but `root` is UNREACHABLE from every actual query surface. It has no\n `spec_attr` in archive/query/fields.py's QueryFieldDescriptor (unlike\n origin/repo/tag/etc), no DSL grammar case in archive/query/expression.py\n (continuation/sidechain/has_branches are in the same unreachable state),\n and no CLI flag. Making `root` DSL/CLI-reachable AND flipping the\n default requires: a Lark grammar case, spec_attr wiring end-to-end\n (query_spec_to_plan), field-metadata docs (discovery.py/metadata.py),\n generated-docs regen (CLI reference, MCP reference, OpenAPI), and a\n default-behavior decision that affects every list() caller across CLI/\n MCP/API/daemon -- a genuinely separate, sizable, high-blast-radius\n change from the repo-identity/label surface this lane owns, and one\n that touches archive/query/expression.py + fields.py, files several\n other concurrent/recent lanes have also been editing. Filing as a\n follow-up bead rather than attempting it inside this lane's already-\n large diff. NOT a pbuh-lane overlap (pbuh is about ai-title/pr-link/\n agent-name typed sidecar records, unrelated to fanout-default\n semantics).\n\n AC5 (report collision rate against 3.5%/max-10 baseline) -- MEASURED,\n read-only, live archive (/realm/db/polylogue/index.db, 15,401 root\n sessions), AFTER the AC3 fix above (before the fix the structural label\n was never exercised so there was nothing real to measure):\n - Among root sessions with a resolved dominant repo-relative path\n (real action_pairs.tool_path evidence -- the population the bead's\n original 3.5%/max-10 baseline was measured against): collision\n rate 3.28% (78/2377 sessions), max collision group 37. In the same\n ballpark as the baseline; the larger max-group (37 vs 10) likely\n reflects a larger/older corpus than the original measurement day.\n - Raw collision rate across ALL 13,219 title-less root sessions:\n 76.46% (10,107 sessions), dominated by a single 5,233-session\n cluster that collapses to the literal label \"0 msgs\" -- these are\n genuinely evidence-free sessions (zero messages, no repo, no file\n touch), not a labeling defect: the label is honest about having no\n distinguishing signal to offer for a truly empty session. Whether\n 5,233 zero-message root sessions is itself a data-quality issue\n (stub/aborted captures, hook artifacts) is a separate question this\n lane did not investigate -- flagged here rather than silently\n folded into the collision number.\n\nLeft for follow-up (filed as a new bead, see graph): AC4 (root: DSL/CLI\nreachability + default), and the \"5,233 zero-message root sessions\" data\nquality question.\n","created_at":"2026-07-31T04:40:54Z"}],"dependency_count":0,"dependent_count":0,"comment_count":1} {"_type":"issue","id":"polylogue-ah21","title":"BrowserCaptureTurn has no blocks channel: structure is destroyed at acquisition, irreversibly","description":"ROOT CAUSE of browser-capture flattening, and it is in the wire schema, not the adapters.\n\n class BrowserCaptureTurn(BaseModel): # polylogue/browser_capture/models.py\n provider_turn_id: str\n role: Role\n text: str | None = None # \u003c- the ONLY content channel\n timestamp: str | None = None\n ordinal: int = 0\n parent_turn_id: str | None = None\n attachments: list[BrowserCaptureAttachment]\n provider_meta: dict[str, object] # \u003c- untyped escape hatch\n\nThere is no blocks field. A turn's role may be 'tool', but the call's input,\noutput and outcome have nowhere to go except free text. Every provider adapter\nis forced through one content channel regardless of what it observed.\n\nTHE EXTENSION IS NOT THE PROBLEM -- it is more capable than the transport.\nbrowser-extension/src/content/chatgpt_bridge.js intercepts window.fetch and\nacquires a session access token, so it can obtain ChatGPT's authoritative API\npayload (the mapping tree with tool nodes and status). The adapters already\nrecognise tool roles (backfill/providers.js:58, content/chatgpt.js:368). The\nstructure is available and the schema cannot carry it.\n\nMEASURED CONSEQUENCE: captured ChatGPT sessions yield 22,992 tool_result blocks\nagainst 7,745 tool_use blocks -- 3x more results than calls -- because pairing\nis reconstructed from prose rather than observed.\n\nWHY THIS IS THE WORST PLACE IN THE PIPELINE TO LOSE STRUCTURE: a parse gap is\nre-runnable against retained bytes. A capture that never recorded the structure\ncannot be recovered at any later date, for any past session. Every day this\nstands, more conversations are permanently flattened.","acceptance_criteria":"1. BrowserCaptureTurn carries typed content blocks; text remains as a rendering, not as the only channel. 2. The ChatGPT adapter emits the API payload's structure via the native bridge rather than reconstructing from rendered prose. 3. tool_use and tool_result counts are consistent for captured sessions -- the current 3:1 ratio is the regression signal. 4. parent_turn_id survives into the archive, so the conversation DAG is not flattened to a list. 5. Report per-origin block-kind coverage for captures before and after.","status":"closed","priority":0,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-29T04:52:43Z","created_by":"Sinity","updated_at":"2026-07-29T17:16:54Z","closed_at":"2026-07-29T17:16:54Z","close_reason":"Implemented. BrowserCaptureTurn now carries a typed content-blocks channel (BrowserCaptureBlock, mirroring ParsedContentBlock minus web_constructs, which is a derived enrichment not observable at the wire boundary); text remains a rendering rather than the only channel. The ChatGPT extension adapter classifies mapping-node content_type/recipient into typed blocks with constructed tool_id pairing. Important correction to this bead's premise, established by measurement: the cited 22,992:7,745 tool_result:tool_use ratio does NOT originate in the capture transport -- only 20 of 455 real captured sessions use the compact/DOM-fallback path this fixed; 435 delegate natively to sources/parsers/chatgpt.py, where the ratio is worse (~4.6:1). That parser-side pairing defect is filed separately as polylogue-4fm3 and is being fixed there. AC1/AC2/AC4 satisfied, AC3 partially (see 4fm3), AC5 reported.","labels":["area:capture","lane:capture-reliability"],"comments":[{"id":"019faea2-35e0-7960-9a05-cef50b174ba0","issue_id":"polylogue-ah21","author":"Sinity","text":"Implemented on feature/browser-capture/typed-content-blocks (PR pending).\n\nScope actually implemented (AC1, AC2, AC4 satisfied; AC3 partially satisfied,\npartially misframed by new evidence -- see below):\n\nAC1 (typed blocks channel) -- SATISFIED. Added `BrowserCaptureBlock`\n(polylogue/browser_capture/models.py), mirroring ParsedContentBlock\n(type/text/tool_name/tool_id/tool_input/media_type/metadata/is_error/\nexit_code; no web_constructs -- that's a derived enrichment, not observable\nat the wire boundary). `BrowserCaptureTurn.blocks: list[BrowserCaptureBlock]`\nadded; `text` stays as a rendering, no longer the only channel;\nrequire_content now accepts blocks-only turns.\n\nAC2 (ChatGPT adapter emits API structure via the bridge, not DOM prose) --\nSATISFIED for the concrete gap that actually existed: the compact/backfill\nbridge path (browser-extension/src/backfill/page_transport.js's\ncompactChatGptConversation, used when a conversation exceeds the executeScript\nscripting-result size cap) explicitly cannot be trusted as a native mapping\npayload by the parser (_has_chatgpt_native_payload rejects\npolylogue_bridge_projection == \"chatgpt-native-compact-v1\"), so it fell\nthrough to the parser's generic per-turn loop with zero blocks. Fixed:\nChatGptBackfillAdapter.normalizeCapture (providers.js) and the live content\nscript's collectNativeTurns (chatgpt.js) now classify each mapping node's own\ncontent_type/recipient evidence into typed blocks (code-interpreter\ncall -\u003e tool_use, its output -\u003e tool_result, paired by constructed tool_id:\nthe call's own node id, and the result's parent node id). Was already true\nfor the FULL native-payload case (delegates entirely to\nsources/parsers/chatgpt.py) -- unaffected, no regression.\n\nAC3 (tool_use:tool_result 1:1) -- PARTIALLY SATISFIED, PARTIALLY MISFRAMED.\nVerified via read-only query against /realm/db/polylogue/index.db\n(file:...?mode=ro, no write): the bead's cited 22,992:7,745 ratio does NOT\noriginate in the browser-capture transport this bead scoped -- it originates\nin sources/parsers/chatgpt.py's own code/execution_output classification\n(content_type \"code\" -\u003e BlockType.CODE not TOOL_USE, \"execution_output\"\nunconditionally -\u003e TOOL_RESULT, neither sets tool_id). Evidence: restricting\nto sessions actually tagged capture:* (455 of 2635 chatgpt-export sessions),\n435 used capture:browser-native-payload (full delegation to chatgpt.py,\nuntouched by this PR) vs only 3 compact + 17 dom-fallback (the paths this PR\nactually reaches) -- and the ratio among captured sessions is *worse*\n(tool_use=3877, tool_result=17768, ~4.6:1), confirming chatgpt.py is the\ndominant contributor, not the browser-capture wire schema. chatgpt.py is\nexplicitly out of this PR's scope (owned by another lane). Filed\npolylogue-4fm3 with the full evidence and a proposed fix. This PR does fix the\n20 compact/dom-fallback sessions' structural gap and closes it for all future\ncaptures that take those paths (including any future non-ChatGPT adapter).\n\nAC4 (parent_turn_id survives) -- SATISFIED, was already true. Verified across\nall four capture paths (native full delegation, compact/generic loop, Claude\nfallback, live collectNativeTurns) that parent_turn_id -\u003e parent_message_id\nthreads through; added explicit test assertions.\n\nAC5 (per-origin block-kind coverage before/after) -- reported in the PR body\nwith the exact read-only query and counts above; \"after\" numbers for the live\narchive require a derived-tier reprocess this PR does not run (no consequential\nwrite to /realm/db/polylogue authorized here). New synthetic tests demonstrate\nthe fix end-to-end via the real receiver -\u003e parser -\u003e materialize -\u003e index.db\nroute (tests/unit/sources/test_browser_capture.py).\n\nNot touched, per explicit scope: polylogue/storage/sqlite/** (schema lane),\npolylogue/sources/parsers/chatgpt.py (parser lane, see polylogue-4fm3).\n","created_at":"2026-07-29T16:08:14Z"}],"dependency_count":0,"dependent_count":0,"comment_count":1} {"_type":"issue","id":"polylogue-2qx.3","title":"Connect the schema inference that already exists: it found every unread field and nothing consumes it","description":"THE INFERENCE ENGINE ALREADY DID THE WORK. This is not a missing capability; it is an unconnected pipeline.\n\npolylogue/schemas/providers/claude-code/versions/v1/elements/session_record_stream.schema.json.gz\nis 140 KB uncompressed, generated from 2,171,910 samples, and contains:\n stop_reason PRESENT structuredPatch PRESENT parentToolUseID PRESENT\n agentId PRESENT slug PRESENT ttftMs PRESENT\n originalFile PRESENT oldString PRESENT toolUseResult PRESENT\nEvery field this backlog records as discarded is IN THE COMMITTED SCHEMA, and\nhas been since 2026-03-16.\n\nThe engine is good. Its extension keywords (codex package) carry far more than\nfield names:\n 110x x-polylogue-frequency 35x x-polylogue-values (observed value sets)\n 21x x-polylogue-range 10x x-polylogue-format (iso8601 detection)\n 10x x-polylogue-multiline 9x x-polylogue-array-lengths\n 6x x-polylogue-semantic-role 5x x-polylogue-evidence (depth/fanout/name_signal)\n 1x x-polylogue-mutually-exclusive\n\nTHREE JOINS ARE MISSING, and each is cheap and static:\n\n (1) SCHEMA -\u003e PARSER READS. Nothing asks 'the schema observed field X across\n 2.1M samples; does any parser read it?' A leaf-name diff between the\n committed schema and polylogue/sources/ produces the acquired-and-unread\n list directly. This replaces the blob-sampling enumeration an earlier\n draft of this bead proposed -- deterministic, versioned, and far cheaper.\n\n (2) SCHEMA -\u003e HARDCODED VOCABULARIES. sources/ carries 51 frozenset/dict\n constants. Filesystem ones are fine (_SUPPORTED_EXTENSIONS, _SKIP_DIRS).\n Provider-data ones duplicate what the schema observed:\n _SKIPPED_SIDECAR_RECORD_TYPES 12 record types hand-listed with no\n per-type rationale -- the schema knows which types exist; this is\n the OriginSpec artifact-kind declaration living in a parser\n _SUCCESS_OUTCOMES = {ok, success, succeeded, completed, outcome_ok}\n five GUESSED synonyms where x-polylogue-values holds the observed set\n _COMPACTION_END_REASONS, _REQUIRED_SESSION_COLUMNS, _GIT_BRANCH_PREFIXES\n Note _GIT_BRANCH_PREFIXES heuristics run against a git_branch column that\n is empty on 100% of claude-code sessions.\n\n (3) PER-FIELD FIRST-SEEN. The package stamps\n x-polylogue-element-first-seen == -last-seen == -generated-at\n all the same microsecond (2026-03-16T12:26:12.880141+00:00), and there are\n NO per-field first/last-seen keys. Yet every record carries a timestamp --\n the schema itself annotates it semantic-role=message_timestamp,\n format=iso8601. The inference walks those timestamps and stamps wall-clock\n instead. Per-field first-seen is min(timestamp of records containing the\n field) and is free at generation time.\n Without it the drift sentinel (polylogue-da1, #3362) can only say\n NEW_FIELD relative to a 134-day-old package -- it cannot distinguish a\n field that arrived yesterday from one present since March.\n\nTHE SENTINEL'S MISSING FOURTH CLASSIFICATION. schemas/drift_sentinel.py\nclassifies UNSEEN_SHAPE (no candidate schema), NEW_FIELD (schema lacks the\nfield), FIELD_CHANGED (validation failed). All three ask what the SCHEMA does\nnot know. There is no classification for 'schema knows it, parser ignores it',\nwhich is the actual defect -- and because those payloads validate cleanly, the\nsentinel marks them benign.\n\nGENERATE RAN; PROMOTE DID NOT. All nine providers have exactly one version\ndirectory (v1). Recent work is real -- #2934 (2026-07-17) derived archive\nworkload profiles from provider schemas and added\nproviders/claude-code/pins.json rejecting two mis-inferred semantic roles\n($.gitBranch as session_title, $.toolUseResult.oldTodos as message_container),\nwhich is direct evidence the engine was run on claude-code that week and SAW\ntoolUseResult.oldTodos. But no regenerated package was promoted, so the\ncommitted artifact is still March-old while the machinery is current.\nbrowser-capture v1 was rewritten 2026-07-27 with sample_count=1 -- a token\nregeneration, not a corpus run.\n\nDO NOT rebuild a sampler. Promote the schema, then run the three joins.","acceptance_criteria":"1. lab schema promote runs for every provider so committed packages reflect current data; report each package's sample_count and age before and after. 2. A static schema-vs-parser diff is committed and runnable, and its output is triaged per key into read / deliberately-dropped-with-recorded-reason / to-acquire in the owning OriginSpec. 3. Provider-data vocabularies hardcoded in sources/ are replaced by, or checked against, x-polylogue-values; _SKIPPED_SIDECAR_RECORD_TYPES becomes an OriginSpec declaration with a per-type reason. 4. Per-field first-seen/last-seen are emitted at generation from record timestamps the pass already reads. 5. The drift sentinel gains the fourth classification (schema-known, parser-unread) and it runs in a gate. 6. No blob-sampling enumeration is built; the schema is the source.","notes":"2026-07-29 (worktree-agent-acd6757a7a8b152f2, parser-diff triage for claude-ai/claude-code): partial AC#2 slice, not closure. Rebased onto origin/master and cherry-picked the already-landed lab schema parser-diff tool (ab4a9a304, on feature/chore/promote-schemas-and-wire-gates) plus the prior sidecar-persistence commits (514900789/05099666c) as a starting base. Found the tool currently returns 0 rows against every COMMITTED provider schema (claude-ai/claude-code/codex/chatgpt/gemini*/hermes*/antigravity all checked) because x-polylogue-observed-distribution is absent from every committed .gz package -- it only works against a freshly regenerated, uncommitted schema. Used it in-memory at min-encountered=0 for the referenced-name list instead, then verified real corpus frequency directly (~/.claude/projects + /realm/db/polylogue source.db blob store) since the promote step (AC#1) hasn't run.\\n\\nTriaged and landed for claude-code (commits e4d0715d5, cebb70e00): fixed a real bug (compactMetadata/preservedSegment/anchorUuid read via wrong snake_case keys, silently nulling trigger/pre_tokens on every compaction since ~ac9cfeb0b); added microcompact_boundary detection (previously fell through to a placeholder message, losing trigger/preTokens/tokensSaved/compactedToolIds entirely); added custom-title and file-history-delta as two MORE sidecar record types beyond the original twelve in _SKIPPED_SIDECAR_RECORD_TYPES (custom-title also now wins session-title precedence over ai-title); added toolUseResult structural-fact projection (sandbox/interrupted/file extents/structuredPatch counts/todo priority state) via two new session_event types; extended message_usage with ttft_ms/stop_reason/cache_creation TTL split/service_tier/inference_geo/cache_miss_reason; added claude_session_kind event and direct gitBranch capture (previously ONLY populated via a separate, often-absent legacy sessions-index.json sidecar).\\n\\nTriaged and landed for claude-ai (commit e793e6d55): tool_use/tool_result segment fields (start_timestamp/stop_timestamp, integration_name/integration_icon_url, approval_key/approval_options, display_content, is_mcp_app, mcp_server_url) now flow into ParsedContentBlock.metadata; top-level conversation summary now persists as claude_ai_conversation_summary.\\n\\nNOT done (still open against this bead's real AC): no schema promote ran (AC#1); _SKIPPED_SIDECAR_RECORD_TYPES/_SIDECAR_EVENT_TYPES are still hand-maintained dict/frozenset constants, not OriginSpec-declared (AC#3); no per-field first/last-seen (AC#4); no drift-sentinel fourth classification (AC#5). Also left explicitly to-acquire (documented as code comments, not silently dropped): claude-ai's nested Drive/doc-citation content[].content[] cluster needs its own ParsedWebConstruct-shaped design distinct from the existing citations[] projection.\n2026-07-29 (worktree-agent-a6d396610f6c9a165): confirmed AC#2 (static schema-vs-parser diff, committed+runnable) and AC#5 (drift sentinel's fourth classification, KNOWN_FIELD_UNREAD) were BOTH already fully done and wired into the live path before this session -- polylogue/schemas/schema_parser_coverage.py + drift_sentinel.py + pipeline/services/ingest_worker.py:475-500 (out of my scope to touch, verified read-only). This session's contribution: ran the parser-diff/coverage join against gemini/gemini-cli/antigravity/browser-capture/codex (the providers with no prior triage pass) and fixed two accuracy bugs in the join itself -- PROVIDER_PARSERS was missing drive_support_attachments.py (gemini) and browser_capture/models.py (browser-capture), producing false-positive \"unread\" rows for both; fixed in both copies of the map (devtools/schema_parser_diff.py + schemas/schema_parser_coverage.py, commit 0d7a19c47). Also closed one real gap the join surfaced for codex: patch_apply_end.changes/.success, plus turn_context.personality/.summary/.collaboration_mode -- see polylogue-cgfy note for full disposition table.\n\nStill NOT done (unchanged from the prior lane's note): AC#1 (schema promote hasn't run for codex/browser-capture -- explicitly out of this session's scope, polylogue/schemas/providers/** was reserved for a concurrent regeneration lane); AC#3 (_SKIPPED_SIDECAR_RECORD_TYPES/_SUCCESS_OUTCOMES/_COMPACTION_END_REASONS/_REQUIRED_SESSION_COLUMNS/_GIT_BRANCH_PREFIXES are still hand-maintained frozenset/dict constants, not OriginSpec-declared); AC#4 (no per-field first/last-seen, that lives in schemas/generation/ which is also reserved for the regeneration lane).","status":"open","priority":0,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-29T04:52:42Z","created_by":"Sinity","updated_at":"2026-07-29T18:27:28Z","labels":["area:ingest","area:sources","delivery:K-interop-origin-export","delivery:ac-patched","horizon:frontier","lane:origin-interop-export","refactor"],"dependencies":[{"issue_id":"polylogue-2qx.3","depends_on_id":"polylogue-2qx","type":"parent-child","created_at":"2026-07-29T06:52:41Z","created_by":"Sinity","metadata":"{}"}],"comments":[{"id":"019fac95-35fb-7dd7-a040-a15160be6a1e","issue_id":"polylogue-2qx.3","author":"Sinity","text":"Hermes triage complete (polylogue-2qx.3 instance, this task's scope: hermes_state.py,\nhermes_spans.py, hermes_lifecycle.py, hermes_verification.py, hermes_identity.py).\n\ndevtools lab schema parser-diff --provider hermes --min-encountered 1 --json (run from\na temp copy of the schema_parser_diff.py branch, feature/chore/promote-schemas-and-wire-gates,\nsince that command isn't on master yet) found 301 unread keys over 167+2 sampled documents.\n\nSplit into two document shapes:\n - 21 keys belong to the mainstream 167-document JSON snapshot shape, parsed by\n polylogue/sources/parsers/local_agent.py::parse_hermes (shared with gemini-cli,\n outside this task's write scope) -- filed as polylogue-5o05.\n - 280 keys belong to the real NeMo Relay ATIF trajectory format (2 sampled documents),\n parsed by hermes_spans.py -- fixed directly on branch\n worktree-agent-aa47c5139f1933ae3, commits aa9fc858c/0e46f702a/b9f14bbd2:\n * step-level extra telemetry: ancestry/tool_ancestry (delegation chain),\n invocation/tool_invocations (framework+timing), llm_response.usage /\n sibling metrics (per-step token accounting), tool-call provider_data ids,\n observation.results[] correlation ids/metadata\n * new hermes_tool_availability_span event: the tool-definition schema (name/\n description/parameters) OFFERED to the model at each llm-request step --\n materially distinct from hermes_tool_execution_span (a tool actually called),\n and previously unrepresented anywhere in the archive\n * document-level: trajectory_id, agent.extra.plugin, final_metrics.* totals\n Deliberately still dropped, with reasons documented inline in hermes_spans.py's\n module docstring: event_payload.conversation_history (a second copy of the\n session's own messages -- payload-hygiene rule), per-tool-call arguments and\n observation.results[].content (conversation-adjacent content, bounded-evidence-only\n per the module's pre-existing policy), llm_request instructions/input (bounded to\n presence, not value), and llm_request internal API plumbing (extra_headers/store/\n prompt_cache_key/include -- no evidentiary value). tools[]._truncated_items is not\n a real Hermes field at all -- a schema-generation-tool artifact (grepped, zero\n references anywhere in polylogue/ source).\n\nNo index-tier storage needed -- all new evidence rides existing session_events\n(event_type has no CHECK vocabulary) and existing event payloads. No index/schema\nversion bump.\n\nVerification: devtools test tests/unit/sources/parsers/test_hermes_spans.py\ntests/unit/insights/test_hermes_topology_projection.py -\u003e 36 + 13 passed; mypy\n--strict clean; ruff clean.\n","created_at":"2026-07-29T06:34:48Z"}],"dependency_count":0,"dependent_count":0,"comment_count":1} {"_type":"issue","id":"polylogue-cgfy","title":"34 of the 70 most common wire keys are never read, including 105,123 structured diffs","description":"SYSTEMATIC ENUMERATION 2026-07-29. Method: parse 60 real Claude Code transcripts, count every top-level / message / usage / toolUseResult key, then grep polylogue/sources/ for each name. This is the complete answer to 'what else arrives typed and is discarded', replacing the ad-hoc list.\n\n34 of the 70 most frequent keys have ZERO references in polylogue/sources/.\n\nTHE FILE-EDIT CORPUS -- entirely unread, corpus-wide counts:\n structuredPatch 105,123 real unified diffs:\n {\"oldStart\":143,\"oldLines\":6,\"newStart\":143,\n \"newLines\":14,\"lines\":[...]}\n originalFile 92,313 the pre-edit file content\n oldString 86,085 with newString and replaceAll alongside\n filePath which file each edit touched\n userModified whether the human changed it afterwards\n\npolylogue-cijx grades file trajectories 'observed' -- 'only tool/action-derived\ndeltas' -- and states that 'checkpointed' requires captured pre/post state.\nThe pre-state IS captured, in originalFile, and the deltas ARE structured, in\nstructuredPatch. The tier cijx declares out of reach is sitting in the bytes.\n\nOTHER UNREAD KEYS OF SUBSTANCE (occurrences in the 60-file sample):\n slug 1,500 human-readable agent name (the subagent display\n problem: '5ecdb160-...:agent-af4e' vs 'greedy-\n squishing-hamming')\n message.stop_reason 1,184 terminal state (see the outcome bead)\n message.stop_sequence 1,184\n parentToolUseID 657 the delegation join key (see the delegation bead)\n toolUseID 679\n sourceToolAssistantUUID 143\n usage.cache_creation 664 cache-creation token detail\n message.ttftMs 36 time to first token\n todos / oldTodos / newTodos agent task-list evolution over a session\n thinkingMetadata 34\n permissionMode 33\n hookCount / hookInfos 22\n toolUseResult.sandbox 60\n toolUseResult.filenames / numFiles 46\n requestId 1,171\n userType 2,789\n\nMEASURED NEGATIVE, recorded so nobody re-files it: usage.service_tier looked\nlike the answer to the API-vs-subscription cost question. It is NOT --\n1,651,137 occurrences, every one 'standard'. A constant. Acquiring it would add\nnothing. Check payloads before filing.","acceptance_criteria":"1. Every key in this enumeration is classified read / deliberately-dropped-with-reason / to-acquire, recorded in the Claude Code OriginSpec fidelity declaration rather than an unexplained frozenset. 2. structuredPatch, originalFile and oldString/newString are persisted; cijx's file-trajectory grading rises from observed to checkpointed where they exist, proven on a sample. 3. slug reaches read surfaces so subagent rows carry names. 4. The enumeration is re-runnable and its output committed, so a future wire change surfaces new unread keys instead of hiding them. 5. Report bytes and row counts added per key acquired.","notes":"2026-07-29 (worktree-agent-a6d396610f6c9a165, un-triaged-provider pass: gemini/gemini-cli/antigravity/browser-capture/codex): branch fast-forwarded to b3ae790be (feature/chore/promote-schemas-and-wire-gates), which already carried a huge amount of prior work: claude-code/claude-ai/gemini/gemini-cli/hermes/antigravity all promoted to schema v2, block-metadata routed to session_events for chatgpt/gemini-cli/hermes/browser-capture/codex, and the drift-sentinel 4th classification (KNOWN_FIELD_UNREAD, polylogue/schemas/schema_parser_coverage.py) fully implemented AND wired into the live ingest path (pipeline/services/ingest_worker.py:475-500) -- polylogue-2qx.3's core ask was already done before this session.\n\nRan devtools lab schema parser-diff --min-encountered 1 against the committed (already-promoted) v2 packages for gemini/gemini-cli/antigravity/browser-capture; codex has no v2 (still stale March v1, no x-polylogue-observed-distribution) so its 0-row output is a tool blind spot, not evidence of full coverage -- verified codex frequency directly against ~3,200 real ~/.codex session files instead.\n\nFindings/dispositions:\n- gemini: all \"unread\" rows (runSettings.enable*/environmentMode/responseSchema) are FALSE POSITIVES -- drive.py:_model_config_event stores the whole runSettings dict verbatim (origin_specs.py:625). _polylogue_drive_live_bytes_b64 was a second false positive: read by drive_support_attachments.py, missing from PROVIDER_PARSERS in both devtools/schema_parser_diff.py and schemas/schema_parser_coverage.py -- fixed both maps.\n- gemini-cli: memoryScratchpad.* is a FALSE POSITIVE -- local_agent.py:_gemini_cli_memory_scratchpad_event already stores it verbatim. toolCalls.args.*/resultDisplay.* also verbatim-captured.\n- antigravity/browser-capture: browser-capture's 8 rows were the same PROVIDER_PARSERS gap (browser_capture/models.py missing) -- fixed. antigravity's single row (version, 0 encountered documents) has zero corpus signal either way.\n- codex (real gap, FIXED, commit 0d7a19c47): patch_apply_end.success/.changes (per-file add/update/delete + unified_diff + move_path) was completely unread -- the direct codex analogue of this bead's own structuredPatch finding. Now retained verbatim. turn_context.personality/.summary/.collaboration_mode also newly captured.\n- codex measured-negative: event_msg.memory_citation is null on every sampled record across the full corpus -- a constant.\n- codex to-acquire, deferred: internal_chat_message_metadata_passthrough carries only {turn_id}; needs a ParsedMessage metadata channel plumbed through every codex.py message constructor, left as a named follow-up.\n\nAll codex dispositions recorded in origin_specs.py's codex fidelity_notes.","status":"open","priority":0,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-29T04:52:32Z","created_by":"Sinity","updated_at":"2026-07-29T18:27:12Z","labels":["area:ingest","lane:origin-interop-export"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-1vpm.7","title":"Delegation resolution guesses by count-equality while the provider supplies the exact join key","description":"MECHANISM. delegation_facts_source pairs Task dispatches to child sessions with no join key at all:\n\n pairable AS (\n SELECT dc.parent_session_id FROM dispatch_counts dc\n JOIN child_counts cc ON cc.parent_session_id = dc.parent_session_id\n WHERE dc.n = cc.n) \u003c- count equality is the entire gate\n\nIt counts Task dispatches in the parent (ordered by message_id), counts resolved\nchildren (ordered by observed_at_ms), and if the counts match, pairs them BY\nORDINAL POSITION -- two unrelated orderings assumed to correspond.\n\nRESULT, full scan of 11,692 delegation_facts rows:\n edge_only 5,951 50.9%\n unresolved 2,207 18.9%\n ambiguous 2,041 17.5%\n resolved 1,493 12.8% \u003c- the only complete delegations\n\nWHY IT FAILS ALL-OR-NOTHING: the gate is per parent. One dispatch whose child\nwas not captured makes dc.n != cc.n and EVERY dispatch in that session becomes\nambiguous. One local gap poisons a whole session, which is why the distribution\nis lumpy rather than a smooth partial.\n\nWHY session_links SUCCEEDS AT 97.6% ON THE SAME DATA: links are derived from the\nCHILD side, where the child literally states its parent sessionId. Delegation is\nderived from the PARENT side, where nothing stated which child a dispatch\nproduced -- so a heuristic was invented instead.\n\nTHE KEY EXISTS, TYPED, AND IS DISCARDED. Claude Code progress records carry:\n parentToolUseID -\u003e the dispatching Task tool_use id\n toolUseID, slug, sessionId\nCorpus-wide: 842,819 progress records carry parentToolUseID, referencing 185,982\ndistinct dispatch ids. progress is in _SKIPPED_SIDECAR_RECORD_TYPES.\n\nSecondary keys also present and unused: the child transcript's first record\ncarries agentId, slug, and its first message IS the Task prompt (verified: 1\nmatch against 102 tool_use blocks in the parent -- unique on that sample, NOT\nyet corpus-verified). sourceToolAssistantUUID appears in child records with\nZERO references anywhere in polylogue/sources/.\n\nTHE INVARIANT: join on identity, never on cardinality. Then 'ambiguous' becomes\nunrepresentable -- you either have the key or you don't -- and missing capture\ndegrades per dispatch instead of per session. Heuristics smear uncertainty;\njoins localize absence. An unavoidable gap is one thing; a gap that PROPAGATES\nis the actual defect.","acceptance_criteria":"1. Dispatch-to-child resolution joins on parentToolUseID; no code path pairs by ordinal position or gates on count equality. 2. The 'ambiguous' mapping state is removed from the vocabulary, not merely reduced -- with the key it is not a reachable state. 3. A parent with N dispatches and M\u003cN captured children yields M resolved and N-M unresolved, proven by a fixture; it never yields N ambiguous. 4. Live re-measure of the mapping_state distribution against the 12.8%-resolved baseline. 5. Corpus-wide collision check on any secondary key before it is relied on.","notes":"Filed 2026-07-29. Note the shape: the epistemic vocabulary here (edge_only/unresolved/ambiguous/quarantined, mapped honestly onto WorkEvidenceAssociationState, with an explicit refusal to 'fabricate a one-to-one attempt') is well designed and correctly implemented. It faithfully reports the uncertainty of a heuristic that did not need to exist. Sophisticated epistemology over an avoidable uncertainty is itself the smell -- the distinctions are real but 87% of what they distinguish is self-inflicted.","status":"open","priority":0,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-29T04:52:22Z","created_by":"Sinity","updated_at":"2026-07-29T04:52:22Z","labels":["area:ingest","area:substrate","delivery:I-analytics-experiments","horizon:mid","lane:analytics-experiments","lane:read-contracts","tech-tree"],"dependencies":[{"issue_id":"polylogue-1vpm.7","depends_on_id":"polylogue-1vpm","type":"parent-child","created_at":"2026-07-29T06:52:21Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"polylogue-pbuh","title":"Claude Code sidecar records are discarded at parse: 1,172,890 records including titles, PR links, agent names and file snapshots","description":"polylogue/sources/parsers/claude/code_parser.py:87 declares _SKIPPED_SIDECAR_RECORD_TYPES and drops every matching record at parse time. Measured against the real corpus at ~/.claude/projects (rg, single pass, 2026-07-29):\n\n progress 850,678\n attachment 86,055\n queue-operation 60,579\n last-prompt 37,616\n file-history-snapshot 34,132\n permission-mode 25,699\n pr-link 20,702\n mode 20,595\n ai-title 18,422\n bridge-session 13,411\n agent-name 5,001\n ---------\n 1,172,890 records discarded\n\nThese are not noise. Sampled payloads:\n\n ai-title {\"type\":\"ai-title\",\"aiTitle\":\"Recover what was lost\",\"sessionId\":\"a903ee33-...\"}\n agent-name {\"type\":\"agent-name\",\"agentName\":\"orchestration-docs-6np\",\"sessionId\":\"a9468292-...\"}\n pr-link {\"type\":\"pr-link\",\"prNumber\":3126,\n \"prUrl\":\"https://github.com/Sinity/polylogue/pull/3126\",\n \"prRepository\":\"Sinity/polylogue\",\"sessionId\":\"cdaf1c01-...\"}\n bridge-session {\"sessionId\":\"d8c9a340-...\",\"bridgeSessionId\":\"cse_01YHHspKPVi2QYy1na2Cgvos\"}\n file-history-snapshot {\"snapshot\":{\"trackedFileBackups\":{},\"timestamp\":\"...\"}}\n\nWHAT EACH ONE WOULD HAVE SOLVED, all currently pursued by inference instead:\n\n ai-title 18,422 -\u003e the 10,157 UUID-titled Claude Code sessions. The\n provider supplies a human title and it is dropped.\n PARTIAL FIX, MEASURED: in the polylogue project dir,\n only 64 of 520 session files (12.3%) carry an\n ai-title record, distributed 2026-05: 8, 06: 25,\n 07: 31 -- the feature is recent, so older sessions\n have no provider title at all. Un-skipping is\n necessary and NOT sufficient; the residual needs\n synthesis and should be sized per origin before\n anyone claims the title problem is closed.\n agent-name 5,001 -\u003e subagent rows read '5ecdb160-...:agent-af4e' instead\n of 'orchestration-docs-6np'.\n pr-link 20,702 -\u003e structured session-\u003ePR linkage. cijx.1 and its four\n blocked consumers (212.2, xyel, kph, fs1.4) are\n trying to RECONSTRUCT by regex and time-window\n scoring what the provider hands over typed.\n file-history-snapshot -\u003e cijx's 'checkpointed' trajectory grade, the tier\n 34,132 above 'observed'. Captured, discarded.\n bridge-session 13,411 -\u003e cross-session lineage (cse_ ids are Claude Code\n cloud sessions). Relevant to 4ts and nas1.\n attachment 86,055 -\u003e attachment preservation (83u / the #2468 finding).\n\nZero beads mention any of these record types. The only other code references\ntreat them as skip-signals: archive/raw_materialization.py:26-28 classifies a\nraw as a non-session artifact when it contains ONLY these types.\n\nThis is the founding premise inverted. The product exists for comprehensive\ncapture; the parser deletes over a million provider-supplied facts, and several\nopen programs spend inference machinery reconstructing a subset of them.","acceptance_criteria":"1. Every currently-skipped record type is classified as evidence-bearing (parse and persist) or genuinely transient (drop, with the reason recorded in the OriginSpec fidelity declaration -- not in a frozenset with no rationale). 2. ai-title, agent-name, pr-link, bridge-session and file-history-snapshot are persisted as typed evidence, not as opaque blobs. 3. Titles and agent names reach read surfaces; a re-run of 'polylogue find repo:polylogue' shows named rows instead of UUID:agent-suffix rows. 4. pr-link becomes the session-\u003ePR producer, and the four consumer beads are unblocked or re-scoped against it. 5. Coverage is reported per type: records seen, parsed, persisted -- so a future skip is visible rather than silent. 6. Existing raws are reprocessed; report the before/after census for UUID titles and PR links.","notes":"Filed 2026-07-29. Found by reading the parser rather than the beads: the skip list is a bare frozenset with no per-type rationale, and nothing downstream records that the data existed. The operator's framing is the right one -- the whole point was comprehensive capture.\n\nMETHOD NOTE for whoever picks this up: verify each type against the live corpus before acting. The DECISION must be per-type, evidenced, and recorded, not a single unexplained set.\n\nCORRECTION 2026-07-29 -- an earlier draft of this bead guessed that 'progress'\nat 850,678 records was 'plausibly genuine streaming noise and may be correctly\ndropped'. THAT GUESS WAS WRONG, and it is the exact mistake this bead warns\nagainst. progress records carry the DELEGATION JOIN KEY:\n\n {\"type\":\"progress\", \"sessionId\":\"7ff2c7d9-...\",\n \"slug\":\"greedy-squishing-hamming\",\n \"toolUseID\":\"agent_msg_01JXHA4xf6C7ArHEUisioLpz\",\n \"parentToolUseID\":\"toolu_01KbmNk4EJY9h9XvGcRBXj3n\", \u003c- the dispatching\n \"data\":{\"message\":{...}}} Task tool_use id\n\nCorpus-wide: 842,819 progress records carry parentToolUseID, referencing\n185,982 distinct dispatching tool ids. That is the complete, typed,\nprovider-supplied delegation graph -- discarded at parse, while\ndelegation_facts resolves 1,493 of 11,692 dispatches (12.8%) using a\npositional-pairing heuristic gated on count equality.\n\nNo record type in this list may be dismissed without checking its payload.","status":"open","priority":0,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-29T04:52:10Z","created_by":"Sinity","updated_at":"2026-07-29T04:52:10Z","labels":["area:ingest","lane:origin-interop-export"],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-pbuh","title":"Claude Code sidecar records are discarded at parse: 1,172,890 records including titles, PR links, agent names and file snapshots","description":"polylogue/sources/parsers/claude/code_parser.py:87 declares _SKIPPED_SIDECAR_RECORD_TYPES and drops every matching record at parse time. Measured against the real corpus at ~/.claude/projects (rg, single pass, 2026-07-29):\n\n progress 850,678\n attachment 86,055\n queue-operation 60,579\n last-prompt 37,616\n file-history-snapshot 34,132\n permission-mode 25,699\n pr-link 20,702\n mode 20,595\n ai-title 18,422\n bridge-session 13,411\n agent-name 5,001\n ---------\n 1,172,890 records discarded\n\nThese are not noise. Sampled payloads:\n\n ai-title {\"type\":\"ai-title\",\"aiTitle\":\"Recover what was lost\",\"sessionId\":\"a903ee33-...\"}\n agent-name {\"type\":\"agent-name\",\"agentName\":\"orchestration-docs-6np\",\"sessionId\":\"a9468292-...\"}\n pr-link {\"type\":\"pr-link\",\"prNumber\":3126,\n \"prUrl\":\"https://github.com/Sinity/polylogue/pull/3126\",\n \"prRepository\":\"Sinity/polylogue\",\"sessionId\":\"cdaf1c01-...\"}\n bridge-session {\"sessionId\":\"d8c9a340-...\",\"bridgeSessionId\":\"cse_01YHHspKPVi2QYy1na2Cgvos\"}\n file-history-snapshot {\"snapshot\":{\"trackedFileBackups\":{},\"timestamp\":\"...\"}}\n\nWHAT EACH ONE WOULD HAVE SOLVED, all currently pursued by inference instead:\n\n ai-title 18,422 -\u003e the 10,157 UUID-titled Claude Code sessions. The\n provider supplies a human title and it is dropped.\n PARTIAL FIX, MEASURED: in the polylogue project dir,\n only 64 of 520 session files (12.3%) carry an\n ai-title record, distributed 2026-05: 8, 06: 25,\n 07: 31 -- the feature is recent, so older sessions\n have no provider title at all. Un-skipping is\n necessary and NOT sufficient; the residual needs\n synthesis and should be sized per origin before\n anyone claims the title problem is closed.\n agent-name 5,001 -\u003e subagent rows read '5ecdb160-...:agent-af4e' instead\n of 'orchestration-docs-6np'.\n pr-link 20,702 -\u003e structured session-\u003ePR linkage. cijx.1 and its four\n blocked consumers (212.2, xyel, kph, fs1.4) are\n trying to RECONSTRUCT by regex and time-window\n scoring what the provider hands over typed.\n file-history-snapshot -\u003e cijx's 'checkpointed' trajectory grade, the tier\n 34,132 above 'observed'. Captured, discarded.\n bridge-session 13,411 -\u003e cross-session lineage (cse_ ids are Claude Code\n cloud sessions). Relevant to 4ts and nas1.\n attachment 86,055 -\u003e attachment preservation (83u / the #2468 finding).\n\nZero beads mention any of these record types. The only other code references\ntreat them as skip-signals: archive/raw_materialization.py:26-28 classifies a\nraw as a non-session artifact when it contains ONLY these types.\n\nThis is the founding premise inverted. The product exists for comprehensive\ncapture; the parser deletes over a million provider-supplied facts, and several\nopen programs spend inference machinery reconstructing a subset of them.","acceptance_criteria":"1. Every currently-skipped record type is classified as evidence-bearing (parse and persist) or genuinely transient (drop, with the reason recorded in the OriginSpec fidelity declaration -- not in a frozenset with no rationale). 2. ai-title, agent-name, pr-link, bridge-session and file-history-snapshot are persisted as typed evidence, not as opaque blobs. 3. Titles and agent names reach read surfaces; a re-run of 'polylogue find repo:polylogue' shows named rows instead of UUID:agent-suffix rows. 4. pr-link becomes the session-\u003ePR producer, and the four consumer beads are unblocked or re-scoped against it. 5. Coverage is reported per type: records seen, parsed, persisted -- so a future skip is visible rather than silent. 6. Existing raws are reprocessed; report the before/after census for UUID titles and PR links.","notes":"Filed 2026-07-29. Found by reading the parser rather than the beads: the skip list is a bare frozenset with no per-type rationale, and nothing downstream records that the data existed. The operator's framing is the right one -- the whole point was comprehensive capture.\n\nMETHOD NOTE for whoever picks this up: verify each type against the live corpus before acting. The DECISION must be per-type, evidenced, and recorded, not a single unexplained set.\n\nCORRECTION 2026-07-29 -- an earlier draft of this bead guessed that 'progress'\nat 850,678 records was 'plausibly genuine streaming noise and may be correctly\ndropped'. THAT GUESS WAS WRONG, and it is the exact mistake this bead warns\nagainst. progress records carry the DELEGATION JOIN KEY:\n\n {\"type\":\"progress\", \"sessionId\":\"7ff2c7d9-...\",\n \"slug\":\"greedy-squishing-hamming\",\n \"toolUseID\":\"agent_msg_01JXHA4xf6C7ArHEUisioLpz\",\n \"parentToolUseID\":\"toolu_01KbmNk4EJY9h9XvGcRBXj3n\", \u003c- the dispatching\n \"data\":{\"message\":{...}}} Task tool_use id\n\nCorpus-wide: 842,819 progress records carry parentToolUseID, referencing\n185,982 distinct dispatching tool ids. That is the complete, typed,\nprovider-supplied delegation graph -- discarded at parse, while\ndelegation_facts resolves 1,493 of 11,692 dispatches (12.8%) using a\npositional-pairing heuristic gated on count equality.\n\nNo record type in this list may be dismissed without checking its payload.\nSTATUS 2026-07-31 (verified by re-audit, not re-derivation): AC1/AC2/AC3 were\nalready satisfied by PR #3390 \"index v46 wire-evidence batch\" (commit\n5e23e6abf, merged to master before this pass started) -- code_parser.py:106-183\ncarries the per-type evidenced classification comment, _SIDECAR_EVENT_TYPES +\n_sidecar_evidence_payload persist agent-name/pr-link/bridge-session/\nfile-history-snapshot/permission-mode/last-prompt/queue-operation/attachment/\nai-title/custom-title/file-history-delta as typed session_events, progress's\nagent_progress subtype dedups into claude_delegation_progress, and\nai-title/agent-name/custom-title resolve TitleSource.ORIGIN session titles\n(code_parser.py:1466-1509) reaching every ordinary read surface (title was\nalready first-class there).\n\nTHIS PASS closed AC5: code_parser.py now counts, per skipped sidecar record\ntype, records seen vs. actually persisted (a session_event/session_ref/title\noverride/delegation edge), plus a sample of ordinary-path record types\ndropped for carrying no text/blocks -- one bounded claude_parse_coverage\nsession_event per session when either counter is non-empty. Tests:\ntests/unit/sources/test_claude_code_sidecar_evidence.py\n(test_parse_coverage_event_reports_seen_and_persisted_counts,\ntest_parse_coverage_event_absent_when_only_ordinary_messages_parsed).\n\nAC4 REMAINS PARTIALLY OPEN: the pr-link producer is real (session_refs table,\nstorage/sqlite/queries/session_refs.py, wired into\nstorage/repository/archive/sessions.py) but nothing on the CLI/insights/MCP\nsurface reads session_refs yet -- polylogue-cijx.1 and its four dependents\n(212.2/xyel/kph/fs1.4) are not unblocked by this alone; noted directly on\npolylogue-cijx.1. Producer-side work is out of this pass's declared surface\n(parsers/claude, assembly_claude_code.py, providers/claude_code*.py) --\nconsumer wiring is insights/CLI/MCP territory for a follow-up pass.\n\nAC6 REMAINS OPEN AS A MEASURED FACT: PR #3390's body recorded *expected*\npost-rebuild numbers, not an actual before/after UUID-title/PR-link census.\nWhether the v46 SEMANTIC_REPARSE rebuild has run against the real corpus\nsince merge, and what the resulting title/pr-link counts are, is an\noperational question against the live archive (not reproducible from a\nsandboxed worktree) -- someone with archive access should run\n`polylogue find repo:polylogue` (or an aggregate query) before/after and\nrecord the actual numbers here.\n","status":"open","priority":0,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-29T04:52:10Z","created_by":"Sinity","updated_at":"2026-07-31T04:25:49Z","labels":["area:ingest","lane:origin-interop-export"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-31r1","title":"Hook events ingested as standalone sessions inflate archive ~4.7x (65.7K empty shells)","design":"Root cause (airtight, 2026-07-22): polylogue/sources/hooks.py:_persist_record drains each spooled hook event (~/.local/share/polylogue/hooks/pending/\u003cid\u003e.json: PreToolUse/PostToolUse/UserPromptSubmit/SessionStart/...) and calls write_source_raw_session with origin=codex-session|claude-code-session, minting a full raw_sessions row per hook -\u003e the materializer turns each into an EMPTY standalone index session (0 messages). Each hook is double-recorded: correctly as a raw_hook_events row carrying session_native_id (table indexed (origin,session_native_id,observed_at_ms) for attach-to-session), AND wrongly as a raw_sessions row.\n\nScale on live archive /realm/db/polylogue: index sessions=83,279 but only 17,553 have content; 65,727 empty shells = codex 35,233 + claude-code 30,488. source_path LIKE '%/hooks/%' raws: codex 35,216 + claude-code 29,679 + hermes 1 = 64,896 = raw_hook_events row count. Real conversations ~17.5K (matches operator memory of ~16K). raw_hook_events has NO FK to raw_sessions, so hooks can persist without minting sessions.\n\nAlso inflates the raw-authority reconciler backlog (hjpx/lkrc/t93b) which churns over hook raws mixed with real session raws.\n\nFIX (operator decisions 2026-07-22): (1) code: add write_source_hook_event writing raw_hook_events + retained blob_ref, NO raw_sessions row; _persist_record uses it; materializer guard so hook-origin raws never become sessions; covers codex/claude/hermes. (2) constructive: materialize raw_hook_events into an index read-model attached to sessions via session_native_id (index tier rebuildable) + read surfaces (MCP/CLI). Operator: hooks are always within a session; link them. (3) retroactive repair WITHOUT full reindex: delete 64,896 hook raw_sessions rows from source.db (durable; backup at /realm/staging/polylogue-sqlite/recovery/t93b-preflight-20260722-durable) + 64,896 empty index session rows (zero messages/blocks/FTS -\u003e tiny blast radius, targeted DELETE). keep raw_hook_events+blobs. (4) re-census raw-authority; deploy #3261 (whale budget fix, merged) so frontier repair doesn't abort on 298MB whale.\n\nDesign doc: .agent/scratch/hook-session-inflation-2026-07-22.md. Verification: sessions count ~17.5K post-repair; every hook event still resolves to its session via session_native_id; no message/block/FTS row dropped.","status":"closed","priority":0,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-22T12:42:26Z","created_by":"Sinity","updated_at":"2026-07-27T02:59:21Z","started_at":"2026-07-22T14:32:39Z","closed_at":"2026-07-27T02:59:21Z","close_reason":"Fast-follow scope (session\u003c-\u003ehooks index read-model + read surfaces) landed via PR #3295 - ArchiveStore.hook_event_summary_for_session() read model + new 'hooks' CLI read-view. Original inflation bug fixed weeks ago (PR #3265, verified live). Both halves of this bead now complete.","comments":[{"id":"019f8a3e-3243-78be-9150-ea16cdde6316","issue_id":"polylogue-31r1","author":"Sinity","text":"Progress 2026-07-22: code fix + retroactive repair MERGED as PR #3265 (master 998188b1a). Both Codex P1 review findings fixed (verified-hook-row predicate via raw_hook_events.source_path; daemon offline-guard on apply). Live dry-run confirms scope: 64,896 hook raw_sessions, 64,895 empty index shells, raw_hook_events + 64,896 blobs retained.\n\nDeploying now: sinnix flake bumped to 998188b1a (commit 34a7a41), switch building. Next: stop polylogued -\u003e run repair_hook_session_inflation(dry_run=False) -\u003e restart -\u003e verify sessions count ~17.5K. Then session\u003c-\u003ehooks read-model + surfaces (fast-follow) and re-census raw-authority for hjpx/lkrc/t93b.","created_at":"2026-07-22T14:32:40Z"},{"id":"019f8a68-335c-7e94-bf69-0e7c18bed99a","issue_id":"polylogue-31r1","author":"Sinity","text":"DE-INFLATION COMPLETE + VERIFIED LIVE (2026-07-22).\n\nDeployed 998188b1a to daemon (sinnix 34a7a41, switch OK; daemon restarted on new store, confirmed has write_hook_event). Stopped daemon, ran repair_hook_session_inflation(dry_run=False), restarted.\n\nLive before -\u003e after:\n- index sessions: 83,286 -\u003e 18,391 (17,559 real + 832 genuinely-empty non-hook, correctly preserved)\n- raw_sessions: 101,347 -\u003e 36,451 (-64,896 hook raws)\n- raw_hook_events: 64,896 -\u003e 64,896 (all evidence retained)\n- hook blobs: 64,896 retained\n- hook raw_sessions after repair: 0; still 0 after daemon restart+drain -\u003e no re-inflation, going-forward fix confirmed live.\n\nRoot cause fully characterized: 64,896 hook events came from just 64 real agent sessions (one codex session fired 13,447 Pre/PostToolUse hooks). Each hook had become its own empty \"session\". Now 64,896 evidence rows attached to their 64 parent sessions via session_native_id.\n\nREMAINING (fast-follow, this bead stays open): session\u003c-\u003ehooks index read-model + read surfaces (MCP/CLI) so hooks are queryable as session evidence. Separate: raw-authority convergence (hjpx/lkrc/t93b) still degraded on pre-existing stale-plan blocker f196aac0 — unaffected by this work.","created_at":"2026-07-22T15:18:33Z"},{"id":"019f8ac1-a0fc-7cdb-84a2-7b3fa2d1be2e","issue_id":"polylogue-31r1","author":"Sinity","text":"INCIDENT + FIX 2026-07-22: first daemon convergence pass after the live de-inflation threw RuntimeError(\"duplicate strategy did not reach its typed terminal postcondition\"). Cause: the repair deleted hook raw_sessions but raw_authority_plans/blockers/census reference raws by JSON string (no FK), leaving 64,895 orphaned frontier plans. Daemon caught it (0 restarts), stopped it, verified clean rollback of an over-slow first cleanup attempt.\n\nFix PR #3266: prune purely-orphaned authority plans+children in the repair; set-based identification (0.3s vs \u003e1h correlated) + temp plan_id indexes for FK-restrict/IN deletes. Live: 64,895 orphans pruned (plans 84,042-\u003e19,147, blockers 70,887-\u003e5,992, census_plans 405,234-\u003e275,444, census_post_plans 323,877-\u003e258,982), 0 remain, daemon restarted 0 tracebacks in 8min. Confirms hook raws were also flooding raw-authority (~65K plan/blocker noise) -\u003e should lighten hjpx/lkrc/t93b convergence.","created_at":"2026-07-22T16:56:13Z"}],"dependency_count":0,"dependent_count":0,"comment_count":3} {"_type":"issue","id":"polylogue-m6tp","title":"Daemon needs an explicit bulk-restore mode: trickle conveyor is structurally wrong for large backlogs","design":"Lesson from the 2026-07-18/19 restore: the conveyor (bounded 16/64-component passes, per-pass candidate recomputation over 100K rows, writer interleaving with catch-up walk) is designed for steady-state trickle and turned ~1h of parse work into a weeks-scale projection; census went net-NEGATIVE while the walk minted new pending raws. The correct bulk path existed all along (ops maintenance rebuild-index: single resumable transaction, blue-green generation, full envelope, one census+replay sweep) but nothing routes to it automatically. Direction: when raw-materialization candidate count exceeds a threshold (e.g. \u003e2000 raws or \u003e2GiB pending), the daemon should (a) surface a loud status/journal recommendation to run the bulk rebuild, or (b) run the generation-based bulk path itself as a dedicated maintenance task with the watcher paused, instead of grinding trickle passes. Also fold in: pause/dedupe interaction with live walk (frozen source snapshot requirement), and the restart-required story. Related: polylogue-p0pw (pool), polylogue-nh44 (newest-only census), polylogue-fqp0 (hash pipeline), polylogue-oikv (replay commit batching).","acceptance_criteria":"Design decision recorded; daemon detects bulk-scale backlog and either routes to or loudly recommends the bulk path; trickle conveyor never silently grinds a weeks-scale backlog again; test covers threshold behavior.","notes":"2026-07-29 (polylogue-623q measurement lane): deprioritized per operator direction -- 623q's parse-vs-apply measurement is the input to the imminent real-rebuild decision, this bead is not. Recording status so it isn't re-litigated blind next session.\n\nVerified live: the structural pieces this bead's own audit called out as still gated are NOT gated anymore on this branch -- daemon_bulk_rebuild_routing and daemon_parse_stage_split config flags are both GONE (grep confirms no matches in config.py); daemon/cli.py:755 _maybe_route_daemon_bulk_rebuild is explicitly unconditional now (\"Unconditional. This was gated behind a daemon_bulk_rebuild_routing config flag...\"). The driving loop (_periodic_raw_materialization_convergence, daemon/cli.py:828+) bursts through an in-flight bulk-rebuild transaction at _RAW_MATERIALIZATION_BACKLOG_BURST_PAUSE_SECONDS cadence (~1s) rather than the outer 30s interval, and only falls back to the slow interval on a swallowed pass failure -- i.e. the \"88%/69% idle wall-clock between hand-resumes\" failure mode this bead documents cannot recur when the daemon is live and driving it, since there's no more operator-resume step in that path.\n\nSeparately, and independent of the daemon: the offline `ops maintenance rebuild-index` CLI processes exactly ONE bounded page (raw_batch_size, default 500) per invocation and returns \"paused\"/\"deferred\" if page.has_more -- it does NOT loop internally. Run bare with defaults against a 41k-raw corpus, that's ~83 manual/scripted re-invocations, i.e. the exact same operator-idle failure mode this bead describes, but via the CLI path rather than the daemon path. Cheap, no-code-change mitigation available today: pass --raw-batch-size large enough to cover the whole corpus in one page (e.g. 50000) so it runs straight through to promotion in a single process invocation -- this is what polylogue-623q's own benchmark did (selected_raw_ids covering the whole sample corpus, one call). Worth stating explicitly before today's real rebuild is invoked.\n\nRemaining real gap per this bead's own notes: item 4 (persistent in-daemon backlog iterator replacing per-pass candidate requery) is efficiency, not correctness, and is already tracked under 4jsk (P3). Not attempted here -- out of scope for a measurement task, and 623q's finding (the single writer, not orchestration pacing, is the dominant cost) means this item would not move the needle on the imminent rebuild's wall-clock even if done.","status":"open","priority":0,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-19T01:33:35Z","created_by":"Sinity","updated_at":"2026-07-29T20:13:37Z","labels":["lane:daemon-surface"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-p0pw","title":"Process-pool forkserver deadlocks in production parse path: zero workers ever spawn","design":"Evidence (2026-07-19 03:00): CLI `ops maintenance rebuild-index` sat 17+ minutes at 16% CPU, zero index-generation growth. py-spy: parent idle in as_completed (_parse_retained_raws revision_backfill.py:719); the only children were the multiprocessing resource-tracker and the forkserver itself, both idle — no pool worker was EVER spawned. Killing and resuming the same transaction with POLYLOGUE_INGEST_PARSE_WORKERS=1 (sequential escape hatch) went to 97% CPU immediately and the generation resumed growing. Same pathology long documented on this host for testmon xdist (bd memory devtools-verify-testmon-forkserver-deadlock). Root: polylogue/pipeline/services/process_pool.py process_pool_context() prefers forkserver whenever available. Fix direction: use spawn (still safe for multi-threaded parents, slower per-worker startup but workers are long-lived here), or diagnose why forkserver never services spawn requests under a threaded asyncio parent (as_completed caller runs on an executor thread). Must also audit the daemon census path (#3122 wired the same helper into polylogued at ingest_workers=cpu-1): daemon census passes were observed parsing large payloads inline (size-aware dispatch), but any pool-eligible small-payload batch may hang or silently serialize the same way.","acceptance_criteria":"Reproduce or conclusively explain the forkserver no-worker deadlock; switch process_pool_context to a start method that demonstrably spawns workers on this host under a threaded parent; regression test that a pool dispatch from a worker thread completes; verify daemon census throughput with pooling active; remove/keep the workers=1 escape hatch documented.","notes":"2026-07-19 03:30 repro results: minimal repro (asyncio-thread -\u003e ProcessPoolExecutor(forkserver) -\u003e as_completed, plain function) PASSES on this host in 0.2s — the deadlock is NOT environmental; it is polylogue-specific state. Sharpened evidence from the stuck run: the forkserver process WAS in its serve loop (select at forkserver.py:231), resource tracker alive, yet ZERO workers were ever spawned and the parent executor never completed a future. Suspect surface (in order): (1) pool initializer _initialize_worker_logging -\u003e configure_logging importing polylogue inside spawned worker; (2) forkserver preload of __main__ (cmdline showed main_path=.venv/bin/polylogue) re-importing the whole CLI in the forkserver at boot; (3) executor manager thread wedged in the parent (a Thread was parked in selectors select). Repro script: /realm/tmp/claude-code/claude-1000/-realm-project-polylogue/af12164b-a2fc-42cb-a548-22277c0875a2/scratchpad/forkserver_repro.py — next step is to extend it to use polylogue process_pool_executor() verbatim, then add the real initializer, then real submission payloads, bisecting which ingredient hangs.\n2026-07-19 04:00 bisect step (b) result: running polylogue process_pool_executor() from a thread under stdin exposed the mechanism — forkserver PRELOADS __main__ via runpy.run_path(sys.argv[0], run_name=__mp_main__) (observed FileNotFoundError for \u003cstdin\u003e crashing the forkserver at boot -\u003e EOFError in parent). In the real CLI, main_path=.venv/bin/polylogue, so the ENTIRE polylogue CLI import graph executes inside the forkserver process at pool creation. Any thread started or lock acquired during that import is inherited (in locked/running state) by every forked worker -\u003e classic fork-of-threaded-process deadlock, consistent with the observed zero-workers hang while the forkserver sat in its serve loop. Note for the fix: spawn ALSO re-imports __main__ per worker (slow ~1-2s/worker startup with the full CLI import, but no inherited-lock hazard). Options: (a) spawn (safe, pay startup once per long-lived worker); (b) forkserver with set_forkserver_preload([]) — but stdlib preloads __main__ unconditionally via main_path... verify whether multiprocessing.spawn.set_executable / context.set_forkserver_preload can suppress __main__ preload; (c) audit what the CLI import graph starts (threads at import time is itself a smell worth fixing). Repro next step for the lane: run the same test from a real script file so main_path resolves, confirm hang, then bisect the import graph for thread/lock creation.\n2026-07-19 04:10: repro relocated to a durable path: /realm/project/polylogue/.agent/scratch/warroom-2026-07-17/forkserver_repro.py (the /realm/tmp scratchpad copy may be cleaned). Lane worktree pre-created: /realm/worktrees/polylogue-lane-h-pool (branch feature/perf/process-pool-spawn from 86ca3287b).\n2026-07-19 lane H: bisect step (c)+(d) result — REFUTES the leading hypothesis\nfrom the prior session. Extended repro\n(.agent/scratch/warroom-2026-07-17/forkserver_repro.py sibling, run as a real\nscript file so sys.argv[0] resolves like production main_path): top-level\n`from polylogue.cli import main` (byte-identical to .venv/bin/polylogue's\nentry-point shape) followed by dispatching process_pool_executor() from a\nworker thread, under both forkserver and spawn contexts. Result: BOTH\ncomplete in 0.6s — no hang. So \"the CLI import graph alone creates a\nthread/lock that forkserver's worker-fork inherits\" does not reproduce\nsynthetically when isolated to import+dispatch. The exact trigger inside the\nproduction forkserver preload (which DID visibly hang: forkserver alive in\nits serve loop, zero workers ever spawned, parent parked forever in\nas_completed at revision_backfill.py:719) remains unconfirmed by a\nstandalone repro; likely needs live-process instrumentation (e.g. py-spy\nagainst a real ops maintenance rebuild-index run) to pin exactly, which is\nout of the ~90min bisect timebox for this lane.\n\nApplied fix per the lane brief's explicit fallback (\"otherwise just switch\nto spawn and delete nothing else\"): process_pool_context() now\nunconditionally returns spawn, never forkserver. This is engineering-sound\nindependent of pinning the exact trigger: spawn reruns __main__ fresh per\nworker instead of forking one shared preloaded process, which structurally\neliminates the whole class of inherited-thread/lock hazards forkserver is\nexposed to (not just the specific one hypothesized). Cost is ~1-2s import\nper worker, acceptable since pool workers here are long-lived and reused\nacross many parse tasks (not short bursts).\n\nLanded: polylogue/pipeline/services/process_pool.py (spawn unconditional,\ndocstring explains why) + tests/unit/pipeline/test_process_pool.py (new\ntest_process_pool_context_is_spawn pins the exact start method rather than\njust excluding fork; new\ntest_process_pool_dispatch_from_worker_thread_completes dispatches 8 tasks\nacross 4 workers from a daemon thread with a 40s join bound + pytest\ntimeout(45), mirroring the asyncio-thread -\u003e pool -\u003e as_completed\nproduction shape). Both pass locally (devtools test\ntests/unit/pipeline/test_process_pool.py: 4 passed in 5.10s). Note: this\nregression test does NOT reproduce the hang pre-fix either (consistent with\nthe synthetic-repro gap above) — it is a forward-looking guard against ever\nreintroducing a hanging start-method config, not a proof the pre-fix code\nwould fail it. Honesty note per AC: \"regression test that a pool dispatch\nfrom a worker thread completes\" is satisfied; \"reproduce or conclusively\nexplain the forkserver no-worker deadlock\" is only partially satisfied —\nexplained mechanism (forkserver forks every worker from one preloaded\nprocess; production main_path preloads the whole CLI graph) but not\nconclusively reproduced or pinned to one exact statement/import.\n\nAlso: mid-session process error caught and corrected — an errant `cd\n/realm/project/polylogue \u0026\u0026 ...` left the shell cwd on the main checkout\nacross later commands, so the first commit attempt landed on master there\n(8672f9768). Recovered cleanly: cherry-picked the commit onto\nfeature/perf/process-pool-spawn in the correct worktree\n(/realm/worktrees/polylogue-lane-h-pool, now 07b7835b2), then `git fetch`\n+ `git reset --hard origin/master` in the main checkout to restore it to\nclean origin state. No data lost, no other lanes' work touched (verified\ngit status was clean before the reset). Main checkout confirmed back at\n86ca3287b matching origin/master.\n\nNext: task 3 (daemon census pooling-in-production audit, report only) and\nverify + PR.\n2026-07-19 lane H: daemon census pooling-in-production audit (AC item 4, report only).\n\nAnswer: NO, the ambient/periodic daemon convergence pool has never\nactivated in production, and the #3122-wired census pool has only ever run\nvia direct CLI invocation, never through the live daemon process.\n\nEvidence:\n1. DaemonConverger.start() logs \"converger: started with %d worker(s)\"\n when _has_cpu_bound_stage() is True, else \"started without worker\n pool\". `journalctl --since -60days | grep \"converger: started\"` shows\n ONLY \"started without worker pool\" — every polylogued startup in the\n observed window (30+ restarts across 2026-07-16..19), zero exceptions.\n Root cause confirmed in source: every ConvergenceStage definition in\n daemon/convergence_stages.py sets cpu_bound=False (5/5 stages: fts,\n embed, claude_workflow, insights, standing-queries) — none is marked\n CPU-bound, so DaemonConverger._executor is never created and the\n periodic ambient loop never pools anything.\n2. The #3122-wired pooled census/replay path (revision_backfill.py\n _parse_retained_raws, reached via maintenance/replay.py -\u003e\n rebuild_index_from_source) IS reachable from inside a live polylogued\n process via the HTTP `--daemon` bridge (daemon/http.py:5276-5286,\n DaemonWriteThreadBridge.run_sync) -- but `journalctl --since -60days`\n shows every `ops maintenance rebuild-index` invocation on this host was\n a direct CLI systemd-run unit (`polylogue ops maintenance\n rebuild-index ...`), never with `--daemon`. So the daemon-HTTP-bridged\n variant has zero production exercise to date; all real runs (and the\n one that hung) went through the plain CLI process directly.\n3. Commit a53785b10 (#3122, merged 2026-07-18 19:26) is the commit that\n FIRST wired ingest_workers through to actual use in\n maintenance/replay.py -- before it, the parameter was accepted and\n immediately `del`eted, so the pooled dispatch branch in\n _parse_retained_raws was dead code on the CLI rebuild-index path.\n The forkserver hang was discovered ~8h after that merge (2026-07-19\n 03:00), on what was effectively the first real heavy exercise of the\n newly-activated pool. This fully explains why the deadlock surfaced\n now rather than being a long-standing dormant bug: the code path had\n never run for real before #3122 activated it.\n\nConclusion for AC \"verify daemon census throughput with pooling active\":\nthere is no production daemon-census throughput to measure yet -- the\npooled path has only run via direct CLI so far. Post-fix (spawn), the\nCLI-direct throughput is the throughput that matters today; the\ndaemon-HTTP-bridge variant and DaemonConverger's ambient cpu_bound pool\nare both currently unexercised/dormant in this codebase, not because\nthey're broken but because nothing marks a convergence stage cpu_bound\nand no HTTP client has used --daemon. Neither is in this bead's scope to\nactivate.\n\nSide finding filed as new tracked debt (out of this bead's scope --\nprocess_pool.py only): polylogue-7saq -- archive_ingest.py's\nparse_sources_archive() builds its ProcessPoolExecutor directly\n(concurrent.futures import, no mp_context), bypassing\nprocess_pool_context() entirely, so it uses the platform default start\nmethod (fork on this host/Python 3.13) -- a strictly worse hazard than the\nforkserver issue since raw fork() of a live async process is\nunconditionally unsafe if any other thread holds a lock at fork time.\nCurrently reached only by the public async API facade\n(Polylogue.parse_sources()/parse_file()) and demo seeding, not by the live\ndaemon's normal ingest ticks (those already go through the safe\nprocess_pool_executor() helper in ingest_batch/_core.py) or the standard\n`polylogue import` CLI flow (stages to daemon instead). Lower urgency than\np0pw was, but a real latent bug for any future caller.\nPR #3143 opened: https://github.com/Sinity/polylogue/pull/3143 (feature/perf/process-pool-spawn -\u003e master). Verification: devtools test tests/unit/pipeline/ -k process_pool (7 passed), devtools verify --quick (16/16 steps green). Rebased cleanly onto latest master after resolving a .beads/issues.jsonl rebase conflict (took origin's side entire -- verified it was a strict superset of my commit's older snapshot, per repo's documented bd-conflict procedure).\nPR #3143 merged: 5e794acbde955985fa7ca7296d6aed8a078abe4d. All CI green (CircleCI quick-gate pass, GitGuardian pass; CodeRabbit + Codex review both rate-limited, no findings to triage). Closing.","status":"closed","priority":0,"issue_type":"bug","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-07-19T01:10:15Z","created_by":"Sinity","updated_at":"2026-07-19T03:10:34Z","started_at":"2026-07-19T02:46:25Z","closed_at":"2026-07-19T03:10:34Z","close_reason":"Merged PR #3143: process_pool_context() now unconditionally spawn, never forkserver. AC honestly assessed: mechanism explained but not conclusively reproduced in isolation (documented); regression test + config-pin test added; daemon-census-throughput AC answered by audit (no production pooled daemon throughput exists yet -- pooled path has only run via direct CLI); workers=1 escape hatch kept as-is. Two follow-ups filed: polylogue-7saq (archive_ingest.py raw-fork ProcessPoolExecutor gap) and corroboration added to polylogue-7uqr (converger pool dead machinery).","dependency_count":0,"dependent_count":0,"comment_count":0} @@ -72,6 +74,15 @@ {"_type":"issue","id":"polylogue-tf2.1","title":"Rerun forensics on current archive; price origin_reported providers","description":"Rerun scripts/agent_forensics.py against the current archive (v23+); price origin_reported providers via the vendored LiteLLM catalog (match last path segment); all-provider headline or explicitly-labeled per-provenance figures that cannot be misread; record deltas vs 06-27; verify chart SVGs render. Cache-inclusion must be disambiguated (Codex input INCLUDES cached ~96%; see bd memories). Also blocked on logical-session token attribution — the headline must not be double-counted.","notes":"Correction to close_reason monetary values: stored/provider-priced subset was $239,453.14; catalog API-equivalent was $318,650.88; origin_reported catalog estimate was $79,197.74. The original close_reason text lost dollar-prefixed digits due shell expansion, not measurement drift.","status":"closed","priority":0,"issue_type":"task","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-07-03T04:31:33Z","created_by":"Sinity","updated_at":"2026-07-03T09:59:13Z","started_at":"2026-07-03T09:28:10Z","closed_at":"2026-07-03T09:59:02Z","close_reason":"Completed with blocker caveat preserved: scripts/agent_forensics.py now prices origin_reported rows through the shared vendored LiteLLM pricing catalog while preserving stored provenance; report separates stored/provider-priced cost from catalog API-equivalent estimates and carries logical-session/cache caveats instead of claiming final billing reconciliation. Regenerated current artifact at .agent/demos/agent-forensics against /home/sinity/.local/share/polylogue schema v23: 16,498 physical sessions, 4,142,175 messages, 356.5B tokens, ,453.14 stored/provider-priced subset, ,650.88 catalog API-equivalent, and ,197.74 origin_reported catalog estimate. SVG parse check passed for 9 charts; devtools test tests/unit/scripts/test_agent_forensics.py passed; devtools verify --quick passed run 20260703T095718Z-quick-753466-96559776; devloop-review clean. Remaining final-reconciliation blocker stays open as polylogue-4ts.2.","labels":["area:usage","campaign"],"dependencies":[{"issue_id":"polylogue-tf2.1","depends_on_id":"polylogue-4ts.2","type":"blocks","created_at":"2026-07-03T06:32:45Z","created_by":"Sinity","metadata":"{}"},{"issue_id":"polylogue-tf2.1","depends_on_id":"polylogue-sru.7","type":"blocks","created_at":"2026-07-03T06:31:33Z","created_by":"Sinity","metadata":"{}"},{"issue_id":"polylogue-tf2.1","depends_on_id":"polylogue-tf2","type":"parent-child","created_at":"2026-07-03T06:31:33Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":2,"dependent_count":2,"comment_count":0} {"_type":"issue","id":"polylogue-tf2","title":"Campaign: agent-forensics regeneration + all-provider repricing","description":"Regenerate the agent-forensics packet on the current archive with an honest all-provider headline. The 2026-06-27 report (546.6B tokens, $89,368 API-list equivalent, 216x cache amplification) is the most stranger-legible artifact on any shelf, but its numbers are pre-dedup stale and the headline prices only the priced-provenance subset (Claude Code cost_usd rows); Codex/ChatGPT/Gemini are origin_reported token counts with no dollar value (operator estimate ~$150K all-provider). Sequenced after claim-vs-evidence per operator direction 2026-07-02.","design":"Current slice design: turn the existing agent-forensics/cost headline into a product-backed all-provider repricing artifact. First inspect devtools/scripts and polylogue analyze surfaces for agent_forensics/cost code. Use active archive usage headline (detail=headline) for authoritative physical_session and logical_session_model_high_water token totals. Keep priced-provenance dollars and origin-reported token estimates separate: do not multiply every token by one blended price without a labeled lane. Add or reuse a shared pricing/projection helper so the demo artifact is regenerated from Polylogue product code, not ad hoc SQL. Acceptance for this slice: the generated agent-forensics artifact names archive root/schema, includes physical vs logical token grain, separates priced subset from origin-reported estimate lanes, gives reproduction commands, and has focused tests for any new repricing helper/surface.","acceptance_criteria":"Terminal state: regenerated forensics packet on the current archive with an honest all-provider headline (priced subset AND origin-reported estimate lanes separated), agent_forensics.py folded into polylogue analyze (tf2.2), artifact on the demo shelf with reproduction commands, cold-reader gate passed. Epic closes only when that artifact is recorded.","status":"closed","priority":0,"issue_type":"epic","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-07-03T04:31:32Z","created_by":"Sinity","updated_at":"2026-07-03T19:06:44Z","started_at":"2026-07-03T18:47:23Z","closed_at":"2026-07-03T19:06:44Z","close_reason":"Completed: provider usage headline now exposes product-backed pricing lanes in polylogue analyze usage --detail headline, separating stored/provider-priced cost from catalog API-equivalent estimates for origin_reported rows. Regenerated the current .agent/demos/agent-forensics artifact against /home/sinity/.local/share/polylogue schema v23: physical-session tokens 395,320,980,423; logical high-water tokens 288,741,229,728; stored/provider-priced USD 243,392.189328; catalog API-equivalent USD 337,565.031618; priced lane 13,889 rows / 12,331 sessions / 12,650 matched rows; origin_reported lane 2,308 rows / 2,270 sessions / 2,302 matched rows. Verification: live polylogue --plain analyze usage --detail headline --format json --limit 0 wrote /realm/tmp/polylogue-usage-headline-pricing-current.json; devtools test tests/unit/storage/test_provider_usage_report.py tests/unit/cli/test_diagnostics.py passed 23 tests; devtools verify --quick passed run 20260703T190553Z-quick-2226137-d91d4e8f; devtools workspace demo-shelf --json reported ok. Non-claim preserved: this is not final billing reconciliation and physical/logical token grains stay explicitly separated.","labels":["area:usage","campaign","size:M","spine"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-sru","title":"Campaign: claim-vs-evidence report to finding-grade","description":"Terminal state: an externally publishable finding ('how often do coding agents proceed past failed tool calls, by model/tool') with stated sample frame, calibrated markers, benign/consequential split, seeded stranger-runnable reproduction, and a passed cold-reader gate. Slice closure is NOT campaign closure; this epic stays top-of-frame until its terminal state is recorded.\\n\\nState as of 2026-07-03 after calibrated active-archive regeneration: archive root /home/sinity/.local/share/polylogue, index schema v23, 41,886 structured failures total, 5,000 origin-stratified failures inspected (3,746 claude-code-session, 1,247 codex-session, 7 claude-ai-export), 100 unpaired structured failures. Marker vocabulary was tightened to avoid broad issue/fix/block/gitignored false positives. Immediate next-turn totals: acknowledged=420, silent_proceed=1,205, ambiguous=3,375 (2,624 wordless tool continuations; 751 prose without marker). Lower-bound silent rate is 24.1%; among classified immediate next turns, silent rate is 74.2%. Next-3 sensitivity window, stopping before the next user message, finds 302 acknowledgments that appear only after the next turn; window3 silent lower bound is 37.0%. Calibration: 50 hand-labeled immediate-next-turn rows, acknowledged-marker precision=1.0, recall=0.8421052631578947, invalid rows=0. Artifact: .agent/demos/claim-vs-evidence/claim-vs-evidence.report.json.","notes":"2026-07-03 update: methodology package is now cold-read gated. .agent/demos/claim-vs-evidence contains aggregate live evidence, public-summary.json, PUBLIC_REPRODUCTION.md, COLD_READER_GATE.md, and COLD_READ_RESULT.md. Seeded reproduction is meaningful, not empty: 4 structured failures, 2 acknowledged follow-ups, 2 silent-proceed follow-ups, 0 unpaired. Cold-reader subagent PASS recovered claim/non-claim, sample frame, rates, calibration, caveats, and reproduction commands from the artifact directory only. Remaining campaign child: polylogue-sru.1 productizes action-unit outcome/followup_class capability.","status":"closed","priority":0,"issue_type":"epic","owner":"ezo.dev@gmail.com","created_at":"2026-07-03T04:31:26Z","created_by":"Sinity","updated_at":"2026-07-03T09:28:09Z","closed_at":"2026-07-03T09:28:09Z","close_reason":"Completed: all seven campaign children are closed. The claim-vs-evidence finding now has bounded sample-frame reporting, calibrated marker precision/recall, handler-class and next-3 sensitivity splits, meaningful seeded reproduction, cold-reader PASS, and productized action-unit followup_class/followup_message_ref query capability. Current artifact lives under .agent/demos/claim-vs-evidence and was regenerated against /home/sinity/.local/share/polylogue schema v23.","labels":["area:substrate","campaign"],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-oqib","title":"Wire root: session filter end-to-end; default find to top-level sessions","description":"Split from polylogue-cijx.4 decision 4 (\"default result unit is the\ntop-level session\"). That bead's other three decisions (repo identity,\nrepo-relative paths, structural-label projection) landed; this one didn't,\nbecause it turned out to be a separate-shaped, higher-blast-radius change.\n\nWHAT EXISTS TODAY: `sessions.parent_session_id` and `Session.is_root`\n(`parent_id is None`) are correct and already used by a plan-level `root:\nbool | None` field (`archive/query/plan.py`) plus a fluent\n`.is_root(True)` builder method (`archive/filter/builder.py`). But `root`\nis completely unreachable from every actual query surface:\n\n - No `spec_attr` on its `QueryFieldDescriptor` in\n `archive/query/fields.py` (unlike `origin`/`repo`/`tag`/etc, which do\n have one) -- so `SessionQuerySpec.from_params`/`from_expression` can\n never set it.\n - No case in the Lark DSL transformer in `archive/query/expression.py`\n (`repo:`/`origin:`/`tag:`/... all have an explicit `fname == \"...\"`\n branch there; `root` has none). `continuation`/`sidechain`/\n `has_branches` are in the identical unreachable state -- this isn't\n unique to `root`.\n - No CLI flag anywhere (`rg` for `--root`/`--continuation`/`--sidechain`\n across `cli/*.py` returns nothing).\n\nLive measurement (read-only, `/realm/db/polylogue/index.db`, 2026-07-31):\n15,401 of 23,296 sessions (66.1%) are root/top-level; the other 33.9% are\nsubagent/branch children. A default `find` with no filters returns both,\nunlabeled as to which is which.\n\nSCOPE for whoever picks this up:\n 1. Add `root: bool | None = None` to `SessionQuerySpec`\n (`archive/query/spec.py`) and wire it through `query_spec_to_plan`.\n 2. Add a `root` case to the Lark DSL transformer\n (`archive/query/expression.py`) -- decide the value syntax (`root:true`\n /`root:false` to match other boolean-flavored fields, or a bare\n `root`/`-root` token; there's no existing precedent to copy since\n `continuation`/`sidechain` never got wired either -- worth deciding\n the pattern once for all three rather than one-off for `root`).\n 3. Register field metadata/discovery docs\n (`archive/query/metadata.py`/`discovery.py`) and regenerate CLI/MCP/\n OpenAPI docs (`devtools render all`).\n 4. DEFAULT-BEHAVIOR DECISION (the actual design call, not just plumbing):\n cijx.4's decision 4 wants the *default* list to be top-level-only,\n with children reachable only via an explicit `root:false` (or\n equivalent). That changes the result set of every unfiltered `find`/\n `list()`/MCP `query` call across CLI, Python API, MCP, and daemon HTTP\n -- audit existing callers/tests that assume today's \"everything\"\n default before flipping it, or scope the default change to the CLI\n `find` verb specifically (the interactive surface AC4's own proof\n text names: \"re-running `polylogue find repo:polylogue` and showing\n named non-fanout rows\") and leave the Python API/MCP defaults\n unfiltered for programmatic composability. Either choice needs to be\n made explicit and stated in the PR, not left implicit.\n\nACCEPTANCE CRITERIA (carried from polylogue-cijx.4 AC4, unchanged):\nDefault result unit is the top-level session, proven by re-running\n`polylogue find repo:polylogue` and showing named non-fanout rows; children\nremain reachable through an explicit filter, never silently filling the\ndefault list.\n","status":"open","priority":1,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-31T04:41:40Z","created_by":"Sinity","updated_at":"2026-07-31T04:41:40Z","dependencies":[{"issue_id":"polylogue-oqib","depends_on_id":"polylogue-cijx.4","type":"discovered-from","created_at":"2026-07-31T06:41:39Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-qj5x","title":"Decision: remove Origin.BEADS_ISSUE — Beads data belongs in the work-evidence graph, not sessions","description":"DESIGN INVESTIGATION VERDICT (2026-07-31, design doc: .agent/scratch/live/beads-handling-design-2026-07-31.html). The operator challenged BEADS_ISSUE-as-Origin (\"beads is not a chatlog\"). Investigation confirms the doubt with measurements:\n\n1. interactions.jsonl is 100% field_change rows (polylogue: 2,249 rows / 862 issues = priority 1124 + status 1071 + assignee 54), actor constant \"Sinity\" in 2,249/2,249. The parser synthesizes English prose from these (\"Sinity changed priority from 3 to 2\") into Role.USER messages with MaterialOrigin.RUNTIME_PROTOCOL — ~924 projected sessions containing zero human or assistant content. Same structural shape as the hook-event inflation incident (polylogue-31r1, 83,286→18,391 sessions).\n2. The rich Beads artifact — issues.jsonl (1,260 issues, 907 with notes, 1,857 dependency edges, descriptions/design/AC) — is NOT ingested by the Origin route at all. The Origin captures the least informative beads file.\n3. The architecturally correct home already exists in code: insights/work_effects.py BeadsIssueEffectAdapter reads the SAME interactions.jsonl as ObservedRepositoryEffect facts, and devtools/mandate_continuity_replay.py build_repository_claim_graph builds claim nodes from it. docs/internals.md 688-733 documents both. BEADS_ISSUE-as-Origin is a redundant second representation of data the archive already models correctly as effects/claims.\n4. Revealed preference: fully wired for months (parser/detector/dispatch/OriginSpec), acquired nothing, nobody noticed. #3416's sources.beads_roots defaults to () — still zero ingested (measured: 0 beads-issue sessions among 23,296 in the live index).\n5. Scaffolding rot: origin_specs.py:796 references stream_parser_path \"beads.py:parse_beads_stream\" — that function does not exist anywhere (dangling reference). Completeness mode is \"proposed\", never harvested from a real sample.\n\nREMOVAL PATH (no shims, no deprecation theater — nothing ingested, zero migration risk): delete Origin.BEADS_ISSUE + Provider.BEADS, sources/parsers/beads.py + its tests, dispatch branches (dispatch.py 44/46/56/198/239/1033/1159/1260), _beads_spec + completeness mode (origin_specs.py 787-805, 997-1030), core/sources.py mappings (126-129, 158, 236, 254, 300); drop \"beads-issue\" from session_links dst_origin CHECK (derived-tier index bump, declare delta class — 0 affected rows measured, in-place fast-forward safe); remove #3416 beads_roots acquisition wiring (no users exist; hard removal is policy-compliant per no-compat-pre-adoption). Keep artifact-taxonomy shape classification (looks_like_beads_interaction) keyed off shape, so a stray uploaded ledger classifies as a non-session artifact instead of unknown-export sessions — same treatment hook events got in 31r1. BeadsIssueEffectAdapter and the claim-graph builder are untouched and become the sole consumers of the ledger.\n\nWHAT IS NOT LOST: ledgers are git-tracked in their repos (durability is git's, not polylogue's); issue state-transition evidence (timestamps, old→new, close reasons carrying commit hashes) stays reachable via the effect adapter for 1vpm.6 reconciliation; bead ids in real sessions remain FTS-searchable (phrase \"polylogue-x4s\" already matches 248 real messages). What ingestion WOULD have added: +4% sessions, all synthetic protocol prose polluting exactly the FTS queries used to find real work on a bead.\n","notes":"Follow-on filed: polylogue-5jnq (issues.jsonl as work-evidence issue nodes, 1vpm.6 adapter). Related open beads: polylogue-37t.13 (beads\u003c-\u003eassertions boundary revisit — its premise 'beads-history ingestion landed (#2800)' refers to the Origin route this decision removes; re-anchor it on the work-evidence graph), polylogue-pbuh (typed pr-link records = the session↔PR leg of the three-way join).","status":"open","priority":1,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-31T04:36:56Z","created_by":"Sinity","updated_at":"2026-07-31T04:37:54Z","dependency_count":0,"dependent_count":2,"comment_count":0} +{"_type":"issue","id":"polylogue-l9su","title":"session_commit.py ignores typed claude_pr_link/claude_bridge_session events and Claude-Session git trailers, regex-scans instead","description":"Two independent typed-signal-ignored gaps in polylogue/insights/session_commit.py, found during the 2026-07-31 heuristics audit (parallel to polylogue-pbuh/polylogue-1vpm.7's exemplars).\n\nGAP 1 -- GitHub PR/issue refs. extract_github_refs() (session_commit.py:26-142) regexes raw session message text for https://github.com/.../pull/N, owner/repo#N, and bare #N -- acknowledged in its own comments as a false-positive-prone heuristic (bare #N can match heading anchors / arbitrary numbers). Meanwhile polylogue-pbuh's fix (already landed on this branch, commit 5e23e6abf / index v46) now persists the Claude Code pr-link sidecar record as a typed claude_pr_link session_event, and bridge-session as claude_bridge_session. VERIFIED live: sqlite3 index.db \"SELECT COUNT(*) FROM session_events WHERE event_type='claude_pr_link'\" -\u003e 18,967 rows (167 distinct sessions); claude_bridge_session -\u003e 12,154 rows. VERIFIED zero readers: grep -rn claude_pr_link polylogue/ (excluding the writer in code_parser.py) returns nothing -- session_commit.py, correlation_view.py, and every consumer of build_correlation_result still regex-scan text instead of reading these typed events. This is a fresh instance of the pbuh pattern that survived the pbuh fix landing: the parse-side fix shipped, the read side never got updated to use it.\n\nGAP 2 -- session-to-commit attribution. detect_session_commits() (session_commit.py:256-363) attributes a git commit to an authoring session via time-window scan (+-2h around session timestamps) plus file-overlap scoring (score_file_overlap, confidence thresholded at 0.3) or an in-text commit-SHA regex match (explicit_ref, confidence 0.95 hardcoded). It never reads git commit trailers. This repo's own commit convention (CLAUDE.md, global agent instructions) appends 'Co-Authored-By: Claude ... ' plus 'Claude-Session: https://claude.ai/code/session_\u003cid\u003e' to every agent-authored commit -- a typed, zero-ambiguity session-authorship signal. VERIFIED: git log --all --format=%B | grep -oE 'Claude-Session: [^ ]+' | wc -l -\u003e 116 commits in this repo alone carry the trailer; grep -rn 'Claude-Session\\|Co-Authored-By' polylogue/ --include='*.py' returns zero hits anywhere in the codebase. Stronger evidence the fix was anticipated but never wired: the session_commits table schema itself (storage/sqlite/archive_tiers/index.py:922) already declares detection_type TEXT CHECK(... IN ('time_window','file_overlap','explicit_ref','origin_reported')) -- 'origin_reported' is a live CHECK-constraint value with ZERO rows using it (VERIFIED: sqlite3 index.db \"SELECT detection_type, COUNT(*) FROM session_commits GROUP BY detection_type\" -\u003e only explicit_ref, 2,990 rows). The schema slot for a typed session-commit link has existed, unused, while a scored heuristic fills the table instead.\n\nNOT EVALUATED: no test in tests/unit/insights/test_session_commit.py asserts accuracy of file_overlap/time_window scoring against ground truth -- only the arithmetic of score_file_overlap() itself is unit-tested (confidence math, not hit-rate).\n\nBLAST RADIUS: session_commits backs the PF-D1 receipts demo (polylogue-212.2/xyel), the provenance-carrying-PRs bead (polylogue-kph), and the Hermes forensics report (polylogue-fs1.4) -- all four read session-to-PR/commit linkage through this exact machinery. 2,990 live session_commits rows, all detection_type=explicit_ref (VERIFIED); repo breakdown polylogue=1,060, sinex=879, sinnix=495, sinity-lynchpin=104 (VERIFIED).","acceptance_criteria":"1. detect_session_commits (or a new higher-priority step ahead of it) parses git commit trailers (Co-Authored-By: Claude / Claude-Session: \u003curl\u003e) via git log --format=%B%n---%n and, when a trailer's session id matches an archived session, records a session_commits row with detection_type='origin_reported' and confidence=1.0, superseding time_window/file_overlap for that pair. 2. build_correlation_result (or its caller) reads claude_pr_link/claude_bridge_session typed session_events before falling back to extract_github_refs' text regex; the regex path is kept only as a fallback for sessions with no typed event, and its results are labeled distinctly from typed results in the output payload. 3. A live re-measure reports the before/after split of session_commits by detection_type, and the before/after count of PR/issue refs sourced from typed events vs regex. 4. tests/unit/insights/test_session_commit.py gains a fixture asserting the trailer-parse path takes priority over file_overlap/time_window for a commit carrying a matching Claude-Session trailer.","notes":"CORRECTION 2026-07-31 (self-correction, keep both versions visible per audit discipline): the original description implied the PERSISTED session_commits table (2,990 rows, all detection_type='explicit_ref') is filled by detect_session_commits()'s file-overlap/time-window scoring. VERIFIED that is wrong -- storage/sqlite/archive_tiers/write.py:4039-4063 shows session_commits is actually populated straight from session.git_commit_hash (a typed field the agent-runtime parser already reports, method='parser-git-meta', confidence hardcoded 1.0). This is a narrow but honest fact (HEAD at session capture time, not 'commit this session produced') and is NOT itself an instance of the audited pattern -- it already prefers a typed field.\n\nThe real, still-live gap is the ON-DEMAND correlation surface: build_correlation_result (session_commit.py:387-449) IS wired live -- api/archive.py:5406 and insights/correlation_view.py:60 both call it, reachable via the 'analyze correlation' CLI/API path (VERIFIED via grep, both call sites exist outside session_commit.py/its tests). THIS is where detect_session_commits' file-overlap/time-window scoring and extract_github_refs' text regex actually run, live, on every invocation -- and neither reads git commit trailers nor the typed claude_pr_link/claude_bridge_session session_events. The bead's AC1-AC4 stand unchanged: they target this on-demand path, not the persisted table. cijx.1's own notes (read after filing this bead) independently confirm session_commits has 0 readers and stores a different, narrower fact than commit attribution -- consistent with this correction, not contradicting it.","status":"open","priority":1,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-31T04:32:05Z","created_by":"Sinity","updated_at":"2026-07-31T04:39:42Z","labels":["area:insights","lane:read-contracts"],"dependencies":[{"issue_id":"polylogue-l9su","depends_on_id":"polylogue-1vpm.7","type":"related","created_at":"2026-07-31T06:32:05Z","created_by":"Sinity","metadata":"{}"},{"issue_id":"polylogue-l9su","depends_on_id":"polylogue-pbuh","type":"related","created_at":"2026-07-31T06:32:05Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-swqu","title":"Update sinnix Claude Code hook settings template to stop baking a stale --sidecar-dir","description":"Root cause of the 2026-07-31 hook-spool backlog (polylogue-k8wv): sinnix's\n/realm/project/sinnix/dots/claude/settings.json template (rendered to\n~/.claude/settings.json) has polylogue-hook commands with a literal\n`--sidecar-dir /home/sinity/.local/share/polylogue/hooks` baked in from an\ninstall that predates the archive root's move to /realm/db/polylogue. This\nis a sinnix-repo fix, not polylogue (out of scope for the polylogue PR that\nfiles this bead).\n\nTwo options, either acceptable:\n1. Re-run `polylogue hooks install` against the live settings.json and copy\n the regenerated hooks.* block back into the sinnix dotfiles template, OR\n2. Add a periodic/activation-time check (Home Manager activation script or a\n sinnix service) that re-runs `polylogue hooks install` whenever\n $HOME/.config/polylogue/polylogue.toml's archive root changes, so this\n class of drift cannot recur silently.\n\npolylogue now ships `polylogue.hooks.hook_install_sidecar_drift()` and a\ndaemon-heartbeat warning that logs when the installed command's baked path\ndiverges from the live-resolved one -- use that as the detection signal\nduring the sinnix-side fix.","notes":"Filed alongside PR https://github.com/Sinity/polylogue/pull/3418 which adds hook_install_sidecar_drift() detection to make this class of drift loud.","status":"open","priority":1,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-31T04:16:15Z","created_by":"Sinity","updated_at":"2026-07-31T04:22:29Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-k8wv","title":"Migrate the legacy 108K-file hook-spool backlog after this deploy lands","description":"The hook-event pending spool at ~/.local/share/polylogue/hooks/pending held\n108,956 flat files as of 2026-07-31, none of which are represented in\nsource.db's raw_hook_events table (verified: comm -12 against both\nraw_hook_events.hook_event_id and the acknowledged/ directory found zero\noverlap -- every pending file is genuinely new evidence, not a duplicate).\n\nRoot cause (diagnosed live, not fixed here -- sinnix, not polylogue): `polylogue\nhooks install` bakes the resolved sidecar dir into ~/.claude/settings.json's\nhook commands at install time (deliberately -- a hook subprocess's env cannot\nbe trusted to carry POLYLOGUE_ARCHIVE_ROOT). When the archive root later moved\nto /realm/db/polylogue, the baked `--sidecar-dir` in\n/realm/project/sinnix/dots/claude/settings.json (and the live\n~/.claude/settings.json it renders) was never regenerated, so hooks kept\nwriting to the old ~/.local/share/polylogue/hooks root while the daemon\nwatched the new, empty one. Fix: re-run `polylogue hooks install` (now that\nthis branch adds `hook_install_sidecar_drift()` / a heartbeat warning that\nwould have caught this) and update the sinnix dotfiles template.\n\nMigration mechanism already exists and is safe (write_hook_event never mints\nraw_sessions rows -- polylogue-31r1): once this branch's day-sharding lands\nand deploys, drain the legacy flat backlog with:\n\n drain_hook_event_spool(archive_root, root=Path(\"~/.local/share/polylogue/hooks\").expanduser())\n\nlooped in bounded batches (the `_iter_pending_event_paths` legacy-flat-file\nfallback added on this branch handles the un-sharded layout). Do NOT run this\nagainst the live archive from an external process while polylogued is\nrunning -- it violates the sole-writer invariant; either drain it through the\ndaemon's own hook-spool drain loop (point hooks_sidecar_dir at both roots\nduring a transition window) or stop the daemon first.\n\nDeferred out of the code-review PR because live execution requires this\nbranch to actually be deployed (nix rebuild) before it's safe to point a\ndrain pass at the real archive.","notes":"Filed alongside PR https://github.com/Sinity/polylogue/pull/3418 which implements the sharded/O(1) spool this migration depends on.","status":"open","priority":1,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-31T04:16:01Z","created_by":"Sinity","updated_at":"2026-07-31T04:22:27Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-8ac0","title":"acquire chatgpt export .dat asset bytes into the blob store","description":"Follow-up to polylogue-0hwv: that bead's PR resolves every referenced .dat\nasset id to its real name/mime/size/sha256 (via library_files.json /\nconversation_asset_file_names.json) and records sandbox-file tier resolution,\nbut does NOT yet stream the .dat blobs themselves into the content-addressed\nblob store. attachments stay acquisition_status=\"unfetched\" with real\nmetadata but no bytes.\n\nWhy deferred: decoder_zip.py's ZipEntryValidator.filter_entries only admits\n.json/.jsonl entries (session_only=True) -- .dat members are filtered out\nbefore the main per-entry loop ever sees them. Real byte acquisition needs a\ntwo-pass ZIP scan: (1) stream every .dat member into BlobStore via\nstore.write_from_fileobj() (same pattern decoder_zip.py's capture_raw branch\nalready uses for raw JSON capture -- streaming hash+write, no full-file\nmemory load), building a dat_id -\u003e (blob_hash, size) map; (2) during\nconversation parsing, join resolved attachments against that map and mark\nthem acquired via the same preacquired-blob receipt mechanism\ningest_batch/_core.py uses for inline_bytes (publication_receipt_id +\nflush_blob_publications), without re-hashing bytes already written in pass 1.\n\nFor the extracted-directory import shape (not a ZIP), the .dat files sit on\ndisk as ordinary sibling files next to conversations-*.json --\nChatGPTAssemblySpec.discover_sidecars already walks that directory and could\nread them directly with BlobStore.write_from_path (also streaming).\n\nAC: importing the real 2026-07-29 export (or an extracted copy) acquires\n.dat bytes as attachment blobs with acquisition_status=\"acquired\" and a true\nSHA-256 for every dat id resolved by polylogue-0hwv's ChatGPTAssetIndex;\nattachments referenced by asset_pointer/attachments[]/resolved sandbox links\nresolve to stored bytes when the underlying .dat member is present in the\nsource. Verify end-to-end against a synthetic ZIP fixture (a few .dat members\n+ matching library_files.json/conversation_asset_file_names.json +\nconversations.json) before attempting the real 16GB export, then confirm\nagainst a real (or truncated real) export.\n\nNot in scope for polylogue-0hwv's own PR: this needs its own focused\nbyte-acquisition-specific verification pass (streaming correctness, receipt/\nGC interaction, aggregate-size ceiling interaction with 3,228 more zip\nentries) separate from the naming/resolution logic polylogue-0hwv covers.","status":"open","priority":1,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-31T03:32:03Z","created_by":"Sinity","updated_at":"2026-07-31T03:32:03Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-e98k","title":"reconcile SQLite mmap budget with the cgroup memory limit","description":"MEASURED 2026-07-31. The polylogued memory incident was not opaque kernel caching - it was two independently chosen constants that never met.\n\nAPP SIDE (polylogue/storage/sqlite/connection_profile.py):\n BULK_BUILD_MMAP_SIZE_BYTES = 4 GiB\n BULK_BUILD_CACHE_SIZE_KIB = 512 MiB\n WRITE_MMAP_SIZE_BYTES = 1 GiB\n READ_MMAP_SIZE_BYTES = 128 MiB\n\nCGROUP SIDE (sinnix modules/services/polylogue.nix:283):\n MemoryHigh = 6G MemoryMax = 8G\n\nARCHIVE SIZE: index.db 38 GB (symlink into .index-generations), source.db 9.1 GB.\n\nA 4 GiB mmap window over a 38 GB database fills completely under any scan-heavy\nwork. One bulk connection therefore accounts for ~4.5 GiB of a 6 GiB ceiling,\nleaving ~1.5 GiB for the daemon's ~1 GiB RSS and everything else. Pinning at the\nlimit was structurally guaranteed, not a leak. Observed: memory.events high\ncounter at 538k+ and climbing, memory.pressure ~3.9%, repeated slow_write, and a\nzip sitting unprocessed in the inbox for 2.5h. A runtime-only MemoryHigh=14G\nstopped throttling dead (0 events over a properly timed 180s, pressure 0.00),\nand MemoryCurrent then settled at 8.59 GB - above the old ceiling, proving the\nlimit was the binding constraint.\n\nTHREE FIXES, in order of value:\n\n1. DERIVE BOTH FROM ONE BUDGET. The mmap/cache profile sizes and the systemd\n limits should come from a single declared memory budget rather than being\n picked separately in two repos. Any future archive growth then moves both.\n\n2. memory.high IS THE WRONG INSTRUMENT for mmap'd/file-backed pages. It is\n designed to throttle anon growth. Mapped DB pages are reclaimable, so\n throttling produces evict -\u003e immediate re-fault -\u003e evict thrash, which is\n exactly the slow_write signature. Keep MemoryMax as the genuine leak guard;\n set MemoryHigh above the mapped budget, or drop it and let global reclaim\n handle cache.\n\n3. MAKE THE MISMATCH OBSERVABLE. Log mapped-bytes-budget vs the cgroup limit at\n daemon startup. This incident was discovered by symptom hours later; it\n should be a startup warning.\n\nNote mmap_size is an upper bound, not an allocation - which is why this stayed\ninvisible until the archive grew large enough to fill the window.\n\nHousekeeping seen while measuring: .index-generations/ holds 72 GB for a 38 GB\nactive index (one stale generation plus a retired one).","status":"open","priority":1,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-31T01:00:41Z","created_by":"Sinity","updated_at":"2026-07-31T01:00:41Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-0hwv","title":"resolve chatgpt export .dat assets to real filenames","description":"The 2026-07-29 chatgpt export ships attachment BYTES for the first time: 3,228 .dat members, of which 1,656 are mapped by conversation_asset_file_names.json (e.g. file-078R8dTqVR9lYSLVmOsCh6ht.dat -\u003e image.png). Message parts reference them as asset_pointer 'file-service://file-\u003cid\u003e', which matches the .dat basename.\n\nThe parser already handles asset_pointer / image_asset_pointer / audio_asset_pointer / audio_transcription. What is missing is the mapping file: rg finds conversation_asset_file_names NOT REFERENCED ANYWHERE in polylogue/.\n\nThis is the standing C6 gap (6,075 chatgpt attachment refs with no bytes) becoming resolvable for the first time - the bytes are now in the archive-side artifact rather than behind an expired URL.\n\nAC: importing the 2026-07-29 export acquires the .dat bytes as attachment blobs with their real filenames and content types, and an attachment referenced by asset_pointer resolves to stored bytes.","notes":"MEASURED SPEC (2026-07-31, from the 2026-07-29 export).\n\nTwo id namespaces among the 3,228 .dat blobs:\n file-\u003cb64ish\u003e 677 conversation assets\n file_\u003c32 hex\u003e 2,551 library files\n\nTwo independent name sources, and TOGETHER they are exhaustive:\n conversation_asset_file_names.json names 1,656 (dat basename -\u003e 'image.png')\n library_files.json names 2,231 (file_id -\u003e file_name, file_extension,\n file_size_bytes, sha256 digest,\n upload/processed times, directory_id)\n either names 3,228 = 100.0%, ZERO unnamed\n\nSo the join is: strip .dat -\u003e look up in asset-name map, else library_files.file_id.\nlibrary_files is the richer source (mime/size/digest/provenance), so prefer it when both hit.\n\nREFERENCE SIDE (this is the part that corrects the earlier framing):\n distinct file ids referenced by messages 3,626\n via content.parts[].asset_pointer 267\n via message.metadata.attachments[] 3,444 \u003c- the LARGER channel, previously unexamined\n referenced AND bytes present 1,608 (44.3%)\n referenced but bytes ABSENT 2,018 (55.7% - still unresolvable)\n bytes present but unreferenced 1,620 of which 1,438 are library_files\n and 182 remain unexplained\n\nSo this does NOT close C6 outright: it makes 44% of referenced attachments resolvable and\nadds a whole second population (Library) that has bytes but no message reference. Both are\nworth storing; conflating them would be wrong.\nIMPLEMENTED (branch feature/sources/chatgpt-export-assets-and-sidecars, PR pending).\n\nScope: name/mime/size/sha256 resolution for every referenced .dat id\n(library_files.json preferred, conversation_asset_file_names.json fallback).\nChatGPTAssetIndex.resolve_dat in polylogue/sources/parsers/chatgpt_sidecars.py,\nwired via a new ChatGPTAssemblySpec (polylogue/sources/assembly_chatgpt.py)\nusing the existing ProviderAssemblySpec discover_sidecars/enrich_session\nprotocol. Resolution recorded as a chatgpt_asset_resolution session_event\n(not a new attachment column -- index.db is a derived tier).\n\nMeasured against the real 2026-07-29 export corpus (all 29 conversations-*.json\nshards + both sidecars, 2,836 sessions, 0 parse errors): 1,924/1,924 = 100% of\nreferenced .dat attachments resolved a name.\n\nNOT satisfied yet: actual byte acquisition into the blob store (AC says\n\"acquires the .dat bytes as attachment blobs ... resolves to stored bytes\").\ndecoder_zip.py's ZipEntryValidator only admits .json/.jsonl entries, so .dat\nZIP members are never read at all today. Filed as a dedicated follow-up,\npolylogue-8ac0, with the two-pass streaming design (collect .dat blobs via\nBlobStore.write_from_fileobj, join during conversation parsing, reuse the\ninline_bytes-style preacquired-blob receipt path) -- this needs its own\nverification pass and is high enough risk (touches the zip streaming/receipt/\nGC machinery) that bundling it into this PR would have made both halves\nharder to review and verify.\n","status":"closed","priority":1,"issue_type":"task","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T23:44:26Z","created_by":"Sinity","updated_at":"2026-07-31T03:55:38Z","started_at":"2026-07-31T03:32:13Z","closed_at":"2026-07-31T03:55:38Z","close_reason":"Merged in PR #3409 (polylogue/master@11403388d): .dat asset id -\u003e name/mime/size/sha256 resolution via ChatGPTAssetIndex, wired through the assembly protocol. Actual byte acquisition into the blob store deferred to polylogue-8ac0 (decoder_zip.py streaming change, out of scope for this PR).","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-2bc2","title":"bd list --all infinite recursion: tree renderer loops on cyclic/duplicate parent-child edge, wrote 54GB before kill","description":"Reproducible 2026-07-30: 'bd list --all' in /realm/project/polylogue emits unbounded repeating tree-indentation glyphs; a probe wrote 23GB in \u003c2min before kill. Prior casualty: /realm/tmp/_bd_poly_full.txt grew to 58,427,205,502 bytes (2026-07-21) before its process died. Suspect cyclic or duplicated dependency edge: polylogue-z9gh.7 appears twice as child of polylogue-z9gh in --status open output. Fix = cycle guard in the tree renderer + dedupe/repair of the offending edge in this DB.","status":"open","priority":1,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T19:35:47Z","created_by":"Sinity","updated_at":"2026-07-30T19:35:47Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-6kur","title":"Cull the repair surface: 10k lines of manual repair against 2.7k of convergence, with targets guarding schema-impossible states","description":"## Measured shape\n\n repair/maintenance surface ~10,164 lines\n storage/repair.py 7,154 (123 top-level defs, 22 public entrypoints)\n maintenance/*.py 3,010\n daemon convergence 2,665 lines\n convergence.py 637\n convergence_stages.py 2,028\n\nA 3.8:1 ratio of manual repair machinery to the automatic convergence meant to\nmake it unnecessary. Convergence registers only FIVE stages: `fts`, `embed`,\n`insights`, `claude_workflow`, `sinex_publication`. `repair.py` exposes eleven\nrepair targets.\n\nThis contradicts the project's own stated principle: *if Polylogue can maintain\na condition fully automatically it should, there is NO break-glass tier, and\nonce the automatic path maintains an invariant the redundant manual surface is\nDELETED rather than demoted.*\n\n## Per-target analysis (live archive, frozen 2026-07-30)\n\nNote first: `REPAIR_HANDLERS[target]` is a name-\u003efunction dispatch table, so\n\"no external references\" means dynamically dispatched, NOT dead. Every target\nbelow is reachable via `run_safe_repairs`/`run_archive_cleanup`.\n\n### Structurally impossible — delete (strongest case)\n\n| target | live violations | why it cannot occur |\n| --- | -- | --- |\n| `orphaned_messages` | **0** | `messages.session_id TEXT NOT NULL REFERENCES sessions(session_id) ON DELETE CASCADE` |\n| `orphaned_attachments` | **0** | `attachment_refs.session_id`/`message_id` both `NOT NULL ... ON DELETE CASCADE` |\n\n`PRAGMA foreign_keys = ON` is set in `storage/sqlite/connection_profile.py`, so\nthese are enforced, not decorative. The schema forbids the state; the repair\nscans for it anyway. Zero violations is not luck.\n\nDelete both repairs, both previews, their `SAFE_REPAIR_TARGETS`/`CLEANUP_TARGETS`\nentries, and their debt-status rows.\n\n### Spent one-shot migrations — delete once confirmed\n\n| target | live violations | note |\n| --- | -- | --- |\n| `message_type_backfill` | **0** | A backfill for a column added later. Confirm the write path always sets it (NOT NULL would settle it), then the migration is spent. |\n\nA backfill is inherently one-shot: once the historical rows are filled and the\nwriter populates the column, the repair guards nothing.\n\n### Symptom-treating — the repair is the wrong fix\n\n| target | live violations | note |\n| --- | -- | --- |\n| `session_timestamp_backfill` | **5,382, GROWING** | Was 1,117 after the hook de-inflation; now 5,382. A backfill whose backlog grows means the WRITE PATH is still producing the defect. |\n\nThis is the \"fix the automatic path\" case, and the most valuable finding here.\nDo not keep running the backfill; find why sessions are still written with\n`created_at_ms IS NULL` and stop that. The repair has been masking a live\nwriter bug, which is exactly what a break-glass tier does to you.\n\n### Cause fixed elsewhere — expect near-no-op\n\n| target | live violations | note |\n| --- | -- | --- |\n| `empty_sessions` | 5,255, of which **4,945** are `.meta` phantoms | PR #3403 fixes the cause (an ungated parse chokepoint in `revision_backfill.py`). After it lands, ~310 remain, and some of those are legitimately empty (sessions carrying only `session_events` after the v46 reclassification). Re-measure post-rebuild before deciding. |\n\n### Genuinely load-bearing — keep\n\n`raw_materialization`, `session_insights`, `orphaned_blobs`,\n`superseded_raw_snapshots`, `stale_supersession_receipts`. These were exercised\nfor real this session (raw materialization and authority blockers had to be\nunstuck manually). But note that needing them manually is itself evidence the\nautomatic path has gaps -- `session_insights` in particular overlaps the\n`insights` convergence stage and should be examined for redundancy.\n\n## Sequencing\n\nThe `archive.py` decomposition lane may relocate `repair.py`'s seam\n(`architecture-hotspots.md` note-on-#3 leaves its `storage/` vs `maintenance/`\nplacement explicitly undecided). Do the deletions after that lands, or they\ncollide.\n\n## Acceptance criteria\n\n- `orphaned_messages` and `orphaned_attachments` repair+preview+registry entries\n deleted, with the FK/CASCADE constraint cited as the replacement guarantee.\n- `message_type_backfill` deleted after confirming the writer always populates it.\n- A separate bead opened for the `created_at_ms IS NULL` WRITER defect, with the\n 1,117 -\u003e 5,382 growth as evidence; the backfill target is not deleted until\n that is fixed.\n- Line count of `storage/repair.py` reported before and after.\n- No new registry or allowlist introduced by any of this.\n","notes":"CORRECTION 2026-07-30: my per-target verdict on `empty_sessions` was wrong, and wrong in the dangerous direction.\n\nI classified it as 'cause fixed elsewhere, expect near-no-op after #3403'. polylogue-ne6k, which already existed and which I failed to read before writing this analysis, records the opposite: **repair_empty_sessions would DELETE the 832 genuinely-empty sessions the hook-inflation postmortem deliberately chose to retain.**\n\nSo the target is not a soon-to-be-no-op. It is actively destructive against data an earlier postmortem made a considered decision to keep. Running it after #3403 lands would remove real archive content, not phantom rows.\n\nRevised verdict for `empty_sessions`: do NOT delete the target as spent, and do NOT run it. It needs a decision about the 832 retained-empty sessions first (ne6k owns that), and any culling work must treat ne6k as a blocker rather than a footnote.\n\nMethod failure worth recording, because it is the same one twice in a day: I derived a verdict from live measurement plus code reading without first checking whether an existing bead already contained the answer. 587 open beads exist; `bd list` silently caps its output (returned 50 of 1,234 records), so a survey that trusts its default limit sees 4% of the backlog and reads as exhaustive. Query the exported .beads/issues.jsonl directly rather than the CLI default.\n\nThe rest of this bead's analysis is unaffected: the FK/CASCADE structural-impossibility case for orphaned_messages and orphaned_attachments stands on schema evidence, and the session_timestamp_backfill growth finding (1,117 -\u003e 5,382) stands on measurement.","status":"open","priority":1,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T17:20:12Z","created_by":"Sinity","updated_at":"2026-07-30T17:41:30Z","labels":["area:ingest"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-f1vg","title":"Corpus acceptance gate: no absences and maximum fidelity, with the 2026-07-30 frozen baseline","description":"## What the operator asked for\n\n\"Ensure max fidelity as well as no absences, through the entire corpus.\" That is\na stronger bar than any existing check enforces, and nothing measured either\nhalf until now.\n\n## Why the existing checks cannot serve\n\n`verify-archive`'s `source-index-coverage` counts superseded revisions as\nmissing work (polylogue-ey3r), so it cannot reach zero on any archive that ever\ningested a conversation twice, and therefore cannot gate a rebuild. Nothing at\nall measures fidelity: an archive can report perfect coverage while every\nattachment whose bytes it holds is recorded `unfetched`, which is precisely the\nstate measured on 2026-07-30.\n\n## The gate\n\n`.agent/scripts/corpus-fidelity-audit.py` (read-only, `mode=ro` throughout,\nexits 1 on failure so it can gate a rebuild). Three measures:\n\n1. **Absences** -- logical documents (origin + provider_session_id) the archive\n holds evidence for but does not surface, bucketed by cause so a fix's effect\n is attributable rather than a single number moving for unknown reasons.\n2. **Attachment fidelity** -- acquired vs not-acquired refs, split by origin and\n upload_origin, because a Drive-hosted reference never fetched is actionable\n while a genuinely byte-less attachment kind is not.\n3. **Revision fidelity** -- documents whose indexed evidence is smaller than the\n largest revision recorded for them.\n\n## Baseline, live archive frozen 2026-07-30 (daemon stopped)\n\n ABSENCES 1,009 of 18,248 known documents\n 587 claude-ai-export/ambiguous-only\n 184 claude-code-session/ambiguous-only\n 135 chatgpt-export/ambiguous-only\n 71 aistudio-drive/settled-yet-absent\n 20 unknown-export/settled-yet-absent\n 12 gemini-cli / hermes / unknown / grok / codex\n\n ATTACHMENT FIDELITY acquired=2,118 not-acquired=7,655\n 3,684 chatgpt-export/oauth/unfetched\n 2,391 chatgpt-export/\u003cnone\u003e/unfetched\n 1,975 aistudio-drive/drive/acquired\n 1,119 aistudio-drive/drive/unfetched\n 398 claude-ai-export/oauth/unfetched\n\n REVISION FIDELITY 94 documents below best recorded evidence\n 76 hermes-session\n 16 claude-code-session\n 2 chatgpt-export\n\n VERDICT: FAIL\n\nThe `settled-yet-absent` buckets (71 drive, 20 unknown-export, 1 codex) are not\nexplained by any currently-tracked cause and want their own investigation --\nthese are documents with no ambiguous decision anywhere that are nonetheless\nmissing.\n\nThe 94 revision-fidelity documents are a residue after correcting a false\npositive, and should be treated as a prompt to investigate rather than proof of\nloss (see below).\n\n## Measurement trap this already caught\n\nThe first version compared indexed *messages* against\n`raw_session_memberships.message_count` and reported **474** shortfalls, 294 of\nthem codex-session. All false. `message_count` was recorded by whichever parser\ncensused that raw, and index v46 deliberately reclassified a large share of\nCodex/Claude Code rows from chat turns into typed `session_events`. One codex\nsession read as \"15 indexed vs 68,553 recorded\" when it actually holds 15\nmessages plus 84,612 events. Counting `messages + session_events` drops the\nfigure to 94.\n\nAnyone extending this must keep that in mind: cross-generation counts are only\napproximately comparable, so a metric built on them needs its assumption stated\nand checked against a real sample before its number is believed.\n\n## Follow-up\n\nPromote this into `devtools` as a first-class command with a `CommandSpec` (plus\n`devtools render devtools-reference`) so it is an enforced gate rather than a\nscript, and wire it into the post-rebuild acceptance path alongside\n`verify-archive`. Kept as a script for now because the fixes it measures are\nstill in flight and its thresholds will move as they land.\n\n## Acceptance criteria\n\n- Absences reach 0, or every residual is individually justified in writing.\n- Attachment refs marked not-acquired are either acquired or shown to be\n genuinely unfetchable (deleted upstream, over the size cap, byte-less kind).\n- Revision-fidelity residue is explained rather than merely small.\n","status":"open","priority":1,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T14:04:01Z","created_by":"Sinity","updated_at":"2026-07-30T14:04:01Z","labels":["area:ingest"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-d8al","title":"claude-ai-export: attachment real-id presence is inconsistent across export vintages, needs comparison-layer relaxation","description":"## What the data says\n\nCensus (full population, not a sample): replayed the production classifier\n(polylogue.sources.dispatch.parse_payload -\u003e session_revision_projection -\u003e\nclassify_membership_revisions) over all 566 claude-ai-export\nequal-message-count ambiguous cohorts in the live archive (read-only,\n/realm/db/polylogue), with polylogue-hith's parser-side fix (drop the\npositional-index seed for synthetic attachment ids) already applied.\n\n 566 claude-ai-export equal-message-count ambiguous cohorts (full census)\n 297 still ambiguous because message_hashes differ (polylogue-c429 /\n message-order-not-stable territory, or genuine content divergence)\n 268 still ambiguous with message_hashes EQUAL (0 content diffs) but\n attachment identity axis mismatched -- the exact shape hith\n targeted\n 0 of those 268 resolved by hith's fix\n 268 of those 268 are \"mixed real/synthetic\": one export vintage of the\n SAME conversation carries a real id (id/file_id/fileId/uuid/\n file_uuid) for an attachment; the OTHER vintage of the same\n conversation has no real id for the physically-same attachment and\n synthesizes one instead\n 0 are \"pure synthetic on both sides\" (the positional-index shape\n hith's fix targets and fully resolves when it occurs)\n\nIn other words: in the population currently persisted as ambiguous, 100% of\nthe identity-mismatch cases are this real-id-presence axis, not the\npositional-index axis. hith's fix is verified correct and regression-safe\n(250-cohort replay of already-resolved cohorts: 249/250 agree old vs new\nlogic, 1 improvement, 0 regressions) but resolves 0 of the currently-measured\n566-cohort population by itself, because no synthetic-minting scheme can ever\nmake a real UUID and a hash of (message, name, mime_type) collide.\n\n## Root cause\n\n`polylogue/sources/parsers/base_support.py:attachment_from_meta` uses the\nexport's own `id`/`file_id`/`fileId`/`uuid`/`file_uuid` field when present,\nand only falls back to synthesis when absent. Claude.ai does not consistently\nemit this field for the same attachment across export vintages of the same\nconversation -- verified directly against blob content for 6 sampled\ncohorts, all showing exactly this shape (one blob's attachment has a real\nUUID-shaped id, the other blob's attachment for the same message has no id\nfield and synthesizes `att-\u003chash\u003e`).\n\nNo id-minting scheme at the parser layer can reconcile this: a real id and a\nsynthetic hash will never be equal strings by construction, regardless of\nwhat the synthetic hash is seeded from.\n\n## Proposed fix (comparison layer, NOT parser layer)\n\nIn `polylogue/archive/session_revision_membership.py` (and/or\n`polylogue/pipeline/ids.py`'s `SessionRevisionProjection` /\n`_attachment_identity_payload`), the dominance/equivalence test should\ncompare attachments by a looser key when testing dominance -- e.g.\n`(message_provider_id, name, mime_type)` without the `id` field -- falling\nback to strict id equality only when that looser key is itself ambiguous\n(more than one attachment sharing the tuple on one side). This is the same\nclass of relaxation polylogue-bu1i introduced for acquisition state\n(`attachment_identities` vs `attachment_contents`), generalized to a third\naxis: \"same attachment referenced with and without a stable provider id\".\n\nThis bead deliberately does NOT propose an implementation in those files --\npolylogue-hith's owning lane was scoped away from\n`session_revision_membership.py`/`ids.py` because another lane owns them\nconcurrently. Whoever picks this up should re-run the census harness\ndescribed in polylogue-hith (or the updated one referenced in its closing\nnote) against the classifier change to prove the 268-cohort population above\nactually resolves, the same way polylogue-bu1i's PR proved 157/157.\n\n## Verification recipe\n\nSame read-only harness as polylogue-hith / polylogue-bu1i: parse both blobs\nof a cohort with production `parse_payload`, project with\n`session_revision_projection`, and diff the resulting\n`attachment_identities` sets. For the 268-cohort population, at least one\nattachment identity differs solely because one side has a real id string and\nthe other has a synthetic hash string for what is, by every other field\n(message anchor, name, mime_type), the same attachment.\n\nRef polylogue-hith\nRef polylogue-bu1i","notes":"Superseded by polylogue-aggz's architecture: attachment identity now unconditionally drops the provider id (content-derived: message_id+name+mime_type only), eliminating the strict/loose duality and its pairwise correlation machinery entirely rather than adding a fallback. See PR.","status":"open","priority":1,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T13:02:14Z","created_by":"Sinity","updated_at":"2026-07-30T15:15:31Z","labels":["area:ingest"],"dependency_count":0,"dependent_count":0,"comment_count":0} @@ -101,7 +112,7 @@ {"_type":"issue","id":"polylogue-cijx.3","title":"Git support is inference over prose while typed git records are discarded","description":"Measured 2026-07-29.\n\nWHAT EXISTS: branch/url/sha snapshotted at session start (13-16% coverage, see\nthe repo-identity bead); a repos table keyed on two unreliable fields;\nsession_commits holding HEAD-at-session-start with ONE writer and ZERO readers\n(2,989 rows, detection_type/method/confidence all constant); and\ninsights/session_commit.py shelling out to git log on demand via an\non-demand view that materializes nothing.\n\nWHAT DOES NOT EXIST: any commit corpus, diff, authorship, branch lineage, or\nmerge/PR outcome.\n\nWHAT IS DISCARDED: Claude Code emits typed pr-link records --\n {\"type\":\"pr-link\",\"prNumber\":3126,\n \"prUrl\":\"https://github.com/Sinity/polylogue/pull/3126\",\n \"prRepository\":\"Sinity/polylogue\",\"sessionId\":\"cdaf1c01-...\"}\n20,702 of them in the live corpus, dropped by _SKIPPED_SIDECAR_RECORD_TYPES.\n\nSo the archive infers git from prose (regex ref extraction, time-window and\nfile-overlap scoring in derive_scan_window/score_file_overlap) while deleting\nthe structured git records the provider hands it. The correlator exists to\nreconstruct, badly, a join that arrives typed and free.\n\nfile-history-snapshot records (34,132, also discarded) carry trackedFileBackups\nplus a timestamp -- the 'checkpointed' tier of polylogue-cijx's trajectory\ngrading, above 'observed'. Captured by the provider, never read.\n\nSEQUENCE: read the records before building the correlator. This may reduce\ncijx.1 and its four blocked consumers (212.2, xyel, kph, fs1.4) from an\ninference problem to a parse problem.","acceptance_criteria":"1. pr-link records are persisted as typed session-\u003ePR evidence and become the producer those four consumer beads read. 2. file-history-snapshot is persisted and raises file-trajectory grading from observed to checkpointed where present. 3. session_commits either gains a reader against honest semantics or is deleted -- it does not survive as a write-only table. 4. Any retained heuristic correlator emits graded candidates, never rows indistinguishable from provider-supplied fact. 5. Report coverage: sessions with a typed PR link, before and after.","status":"open","priority":1,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-29T04:52:18Z","created_by":"Sinity","updated_at":"2026-07-29T04:52:18Z","labels":["area:ingest","area:insights","area:interop","horizon:mid","tech-tree"],"dependencies":[{"issue_id":"polylogue-cijx.3","depends_on_id":"polylogue-cijx","type":"parent-child","created_at":"2026-07-29T06:52:18Z","created_by":"Sinity","metadata":"{}"}],"comments":[{"id":"019facfe-58ec-7c23-a7eb-4b30f145ca40","issue_id":"polylogue-cijx.3","author":"Sinity","text":"Scoped lane (parser-only, claude/**) landed a narrow, adjacent fix in commit\ne0f08af83 on feature/chore/promote-schemas-and-wire-gates: claude/index.py's\n_looks_like_git_branch title-guard heuristic was missing the observed\n`claude/` branch-name prefix (e.g. claude/phase_3), and never cross-checked\nagainst typed gitBranch evidence (record-level item.gitBranch, already read\nin code_parser.py per a prior lane, or sessions-index.json's gitBranch) even\nwhen that evidence could prove an exact match instead of guessing by shape.\nFixed both: added the prefix, and made the title guard prefer an exact match\nagainst known typed git_branch when one exists, falling back to the shape\nheuristic only when no typed value is available.\n\nThis does NOT touch this bead's actual AC (pr-link records / session_commits\ndisposition / file-history-snapshot) -- that work lives outside this lane's\nwrite scope (insights/session_commit.py, storage) and was out of scope for a\nparser-only worktree. Leaving this bead OPEN; the git_branch title-guard gap\nit (and cijx.2) referenced is now closed as a side effect, but the bead's own\nacceptance criteria are unaddressed.\n","created_at":"2026-07-29T08:29:38Z"},{"id":"019faebc-d0b5-7670-8c00-92de526b67b9","issue_id":"polylogue-cijx.3","author":"Sinity","text":"Scoped lane (sources/**+insights/**, no storage/sqlite/**) on\nfeature/chore/promote-schemas-and-wire-gates, 3 commits: 62bde4802, bde9bf7e9.\n\nCorrection to this bead's own prior comment: the \"prior lane\" title-guard fix\nit referenced (commit e0f08af83, claude/index.py) was NOT actually on this\nbranch's history (git merge-base --is-ancestor confirmed it is not an\nancestor of HEAD -- it exists on a different, unmerged sibling\nbranch/worktree). I cherry-picked its git_branch-preference logic in 62bde4802\n(dropping its unrelated claude-ai flags-disposition hunk in common.py, which\nconflicted with independent equivalent work already on this branch).\n\nMain fix (bde9bf7e9): code_parser.py never read Claude Code's per-record\n`gitBranch` field at all outside the legacy sessions-index.json sidecar\nmerge (which is what produced the measured 0%). Now reads gitBranch from\nevery record type (sparse per-record, ~2%, but present on ~81% of session\nfiles somewhere), keeping first non-empty seen. Also stopped dropping\npr-link and file-history-snapshot records (previously silently discarded by\n_SKIPPED_SIDECAR_RECORD_TYPES) -- both now persist as typed session_events\n(event_type \"pr_link\" / \"file_history_snapshot\").\n\nMEASURED (read-only, production parse_code route, 500 real local Claude Code\nsession files, /home/sinity/.claude/projects):\n git_branch before: 0/495 (0.0%) -- reproduced by reverting to 62bde4802\n git_branch after: 319/495 (64.4%)\n sessions with pr_link event: 37/495\n sessions with file_history_snapshot event: 262/495\n\nLive archive read-only cross-check (file:...?mode=ro, /realm/db/polylogue/index.db,\nNOT written to): confirms this bead's baseline exactly -- git_branch\n15.8% (2989/18871), git_repository_url 13.2% (2495/18871), commit_hash 15.9%\n(3003/18871), claude-code git_branch 0.0% (0/12001). These numbers are\npre-fix (no rebuild has run); the fix only affects the NEXT full reparse.\n\nAC disposition:\n AC1 (pr-link persisted, becomes producer for cijx.1/212.2/xyel/kph/fs1.4) --\n SATISFIED for the persistence half (session_events, event_type \"pr_link\").\n The consumer wiring for those four downstream beads is NOT done here (out\n of this lane's scope).\n AC2 (file-history-snapshot raises grading from observed to checkpointed) --\n PARTIALLY satisfied: the typed evidence is now persisted\n (session_events, event_type \"file_history_snapshot\", path list + count).\n The observed/checkpointed grading system itself does NOT exist anywhere\n in insights/ yet -- this is the large cijx P3 spike's job, not a\n same-lane addition. Filed as remaining scope on cijx (parent), not a new\n bead.\n AC3 (session_commits reader-or-delete) -- NOT done. session_commits is\n written by storage/sqlite/archive_tiers/write.py, out of this lane's\n write scope (a concurrent lane owns storage/sqlite/** this cycle).\n AC4 (retained heuristic correlator emits graded candidates) -- NOT\n applicable yet; no correlator was built or touched here.\n AC5 (coverage report) -- see MEASURED above.\n\n_GIT_BRANCH_PREFIXES / _looks_like_git_branch in claude/index.py: NOT\ndeleted. It is a title-guard (rejecting a sessions-index.json `summary` that\nis actually a branch name from being used as a session title), not the\ngit_branch capture path itself -- it stays useful as a fallback for sessions\nwith no typed git_branch evidence to compare against (now demoted to\nfallback-only behind the exact-match check added in 62bde4802). Not \"a\nheuristic operating on nothing\": it operates on the sidecar's `summary`\nfield, which is independent of whether `git_branch` is populated.\n\nNot done, explicitly out of scope for this lane: git_repository_url/\ncommit_hash for claude-code -- corpus scan found no such typed fields on\nClaude Code JSONL records at all (only gitBranch), so there is nothing\nfurther to read for those two columns from this provider.\n","created_at":"2026-07-29T16:37:18Z"}],"dependency_count":0,"dependent_count":0,"comment_count":2} {"_type":"issue","id":"polylogue-cijx.2","title":"Repository identity is really cwd: 84% of sessions have no git evidence and one path yields conflicting repo names","description":"Measured 2026-07-29 on the live archive.\n\nGIT EVIDENCE COVERAGE across 18,871 sessions:\n git_branch 2,989 15.8%\n git_repository_url 2,495 13.2%\n commit_hash 3,003 15.9%\nFor the other ~84%, repo assignment comes purely from working_directories. The\ncolumn named 'repo' is therefore 'cwd'. A session in /home/sinity is recorded as\nbeing in the 'sinity' repo.\n\nTHE KEY IS TWO UNRELIABLE FIELDS. write.py:5150\n _repo_id(origin_url, root_path) = f'{origin_url}\\x1f{root_path}'\nand write.py:5143 _repo_name takes the URL basename when a URL exists, else the\npath basename. So the same directory produces multiple rows with DIFFERENT names:\n /realm/project/sinex -\u003e sinex\n /realm/project/sinex -\u003e sinnix\n /realm/project/sinex -\u003e polylogue\n /realm/project/sinex-gateway-shutdown -\u003e sinnix\nand one repository splits across spellings: polylogue holds 106 distinct\nrepo_ids, sinex 28, sinnix 31.\n\nTHREE ENTITIES ARE COLLAPSED INTO ONE COMPOSITE KEY:\n Repository -- stable identity. The right key is the ROOT-COMMIT SHA\n (git rev-list --max-parents=0): content-addressed, survives renames,\n remote changes, mirrors and forks. Remote URLs are ALIASES of a\n repository, not its identity -- which is exactly why three spellings\n produced three repos. This is the same philosophy the archive already\n applies to embeddings (input hash) and blocks (content hash).\n Checkout -- a filesystem path bound to a repository at a branch. Every\n /realm/worktrees/polylogue-* is a checkout of ONE repository, not fifteen.\n Observation -- a session seen in a checkout, at a commit, at a time.\n\nA directory with no git evidence is honestly A DIRECTORY. Do not synthesize a\nrepository for it.\n\nOPEN DECISIONS, not measurements -- resolve explicitly rather than assuming:\n (a) root-commit identity is unavailable for repos polylogue never had\n filesystem access to (an imported ChatGPT session merely mentioning a repo)\n (b) a path reused across projects over time belongs to different repositories\n in different intervals","acceptance_criteria":"1. Repository, checkout and observation are separate entities; repo identity does not include a filesystem path. 2. Remote-URL spellings that denote one remote resolve to one repository, with tests over the observed spelling set (empty/https/ssh/.git). 3. Live re-measure: polylogue/sinex/sinnix collapse to one repository each, with checkouts enumerable underneath. 4. A session with no git evidence resolves to a directory, not a repository, and read surfaces say which. 5. The repo: query field resolves through normalized identity -- a session recorded under one spelling matches a query using another.","status":"open","priority":1,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-29T04:52:17Z","created_by":"Sinity","updated_at":"2026-07-29T04:52:17Z","labels":["area:insights","area:interop","area:substrate","horizon:mid","tech-tree"],"dependencies":[{"issue_id":"polylogue-cijx.2","depends_on_id":"polylogue-cijx","type":"parent-child","created_at":"2026-07-29T06:52:17Z","created_by":"Sinity","metadata":"{}"}],"comments":[{"id":"019faebd-4e40-7ea5-90b0-efddc80e3fae","issue_id":"polylogue-cijx.2","author":"Sinity","text":"Scoped lane (sources/**+insights/**, no storage/sqlite/**) on\nfeature/chore/promote-schemas-and-wire-gates, commit 243bfb3ea.\n\nVerified live: the storage-side identity rework this bead calls for\n(content-addressed repo_id via root-commit SHA, repo_checkouts,\nrepository/checkout/observation split) is NOT landed on this branch despite\nthe task brief describing it as \"partially addressed already\" --\nrepos.repo_id in storage/sqlite/archive_tiers/index.py:732 is still the\nplain SQL GENERATED `origin_url || char(31) || root_path` column, and there\nis no repo_checkouts table anywhere in this checkout. That work either\nbelongs to a different, not-yet-merged lane, or has not started; either way,\nstorage/sqlite/** is out of this lane's write scope this cycle (a concurrent\nlane owns it), so it was correctly left untouched here.\n\nWhat I did instead (the parser/attribution half, per the mission's scope\nsplit): _append_repo_identity_evidence in polylogue/sources/emitter.py, run\nat _SessionEmitter._maybe_enrich (the single point every session from every\nprovider passes through after provider-specific sidecar enrichment, before\nleaving sources/** for storage). It grades each session's location evidence\nwithout touching any table:\n grade=\"git_evidence\" when git_branch/git_repository_url/commit_hash is\n non-empty\n grade=\"directory_only\" when only working_directories is non-empty\n (no event) when there is no location evidence at all\npersisted as a session_event (event_type \"repo_identity_evidence\",\nevent_type has no CHECK vocabulary so this needed no migration), payload\n{grade, root_paths, git_repository_url, git_branch, git_commit_hash}.\n\nMEASURED (read-only, file:...?mode=ro against /realm/db/polylogue/index.db,\nNOT written to -- confirms this bead's own baseline exactly):\n sessions total: 18,871\n sessions with ANY git evidence (branch/url/commit): 3,003 (15.9%)\n sessions with NO git evidence (the \"directory, not repository\" case,\n per cijx.4 decision 1): 15,868 (84.1%)\n\nAC disposition:\n AC1 (repository/checkout/observation separate entities, no filesystem path\n in repo identity) -- NOT done here; storage-side, out of scope.\n AC2 (remote-URL spellings resolve to one repository) -- NOT done here;\n storage-side, out of scope.\n AC3 (live re-measure: polylogue/sinex/sinnix collapse to one repo each) --\n NOT applicable without AC1/AC2 landing first.\n AC4 (a session with no git evidence resolves to a directory, read surfaces\n say which) -- SUBSTRATE SATISFIED at the parser layer: every session now\n carries a typed repo_identity_evidence grade a reader can consult without\n re-deriving it from working_directories. The storage-side repos/\n session_repos tables still synthesize a repo row keyed on root_path\n regardless of grade (write.py:_write_repo_edges, out of this lane's\n scope) -- so today's read SURFACES (CLI/API/MCP) do not yet expose the\n distinction end-to-end. That wiring is the remaining half, blocked on the\n storage-side identity rework landing.\n AC5 (repo: query field resolves through normalized identity across\n spellings) -- NOT done here; storage-side, out of scope.\n\nWriter contract left for the storage-side lane: session_events rows with\nevent_type=\"repo_identity_evidence\" (one per session, not per-record) carry\n{grade: \"git_evidence\"|\"directory_only\", root_paths: [str],\ngit_repository_url, git_branch, git_commit_hash}. This is exactly the signal\nthe storage rework needs to decide \"synthesize a repository row\" vs \"this is\na bare directory\" without re-deriving it from raw session columns.\n","created_at":"2026-07-29T16:37:50Z"}],"dependency_count":0,"dependent_count":0,"comment_count":1} {"_type":"issue","id":"polylogue-6e7m","title":"Titles must describe what a session did: prompt echoes collide 78-way and do not distinguish sessions","description":"THE MEASUREMENT THAT SETTLES THE DESIGN. Codex state_5.sqlite threads, full scan:\n 2,771 threads carry a title\n 2,185 distinct titles\n 166 titles are shared by more than one thread\n worst: 78x 'take over claude's session 755b624d-074f-4d4f-b2fa-02d3a9e...'\n 78x 'familiarize yourself with the repo and its full beads-set'\n 36x 'find, using whatever means, either direclty ~/.codex or po...'\n\nA title you cannot select by is not a title. BOTH providers produce\nfirst-prompt echoes, so copying the provider does not solve this:\n - Claude Code ai-title records exist but cover ~12% (64 of 520 session files\n in the polylogue project dir; 2026-05: 8, 06: 25, 07: 31 -- recent feature)\n - Codex threads.title covers 2,771 of 3,054 but the values ARE the echoes above\n\nCurrent archive state: 13,611 of 18,871 sessions (72.1%) titled with a raw UUID,\nplus 2,369 (12.6%) with \u003e60-char echo titles = 84.7% unusable.\n\nDESIGN: derive the title from what the session DID. Every input is already in\nthe index, measured against untitled claude-code sessions:\n repo 100% (3,000 of 3,000 sampled)\n work_events 82%\n file paths 382,940 action_pairs rows carry tool_path\n timestamps after m3p9\nDEFINED AND MEASURED 2026-07-29, so the executing agent does not have to invent\nit. Label = repo | distinct files touched | message count | date:\n\n sqlite3 -readonly index.db \"with pl as (select s.session_id, s.message_count,\n (select r.repo_name from session_repos sr join repos r\n on r.origin_url=substr(sr.repo_id,1,instr(sr.repo_id,char(31))-1)\n and r.root_path=substr(sr.repo_id,instr(sr.repo_id,char(31))+1)\n where sr.session_id=s.session_id limit 1) repo,\n (select count(distinct ap.tool_path) from action_pairs ap\n where ap.session_id=s.session_id and ap.tool_path is not null) nfiles,\n date(s.created_at_ms/1000,'unixepoch') d\n from sessions s where s.origin='claude-code-session' and s.title=s.native_id limit 4000)\n select ...;\"\n\n 4,000 untitled sessions -\u003e 3,862 distinct labels\n collisions 138 (3.5%)\n max collision size 10\n labels used twice 62\n labels used 3+ times 24\n\nContrast the echo baseline: 166 colliding titles with a SEVENTY-EIGHT-way worst\ncase. Structural collisions are small and mostly pairwise, and the 10-way case\nis a batch of near-identical subagent spawns -- sessions that genuinely are\nalike. Adding one more discriminator (top file path, or a duration bucket) cuts\nit further; 3.5% pairwise is already usable.\n\nInputs are all present: repo on 100% of untitled sessions, message_count on\n100%, 382,940 action_pairs rows carrying tool_path, dates on 94%.\n\nProse synthesis is a worthwhile ADDITION, not the base: ~10,157 sessions x ~2K\nhead tokens is a few dollars on a small model, and the budgeted-external-call\npattern already exists (embeddings, embedding_max_cost_usd ceiling, batching,\nprogress, reconcile). Claude Code's ai-title is itself an LLM summary, so this\nreproduces the provider's own method for the residual.\n\nsessions.title_source already models provenance as\n('origin','path','heuristic','user','unknown'); add a synthesized value and\nstamp which tier produced each title so a mixed corpus stays honest.","acceptance_criteria":"1. sessions.title holds ONLY provider-supplied titles, or NULL. A derived label is never written to it. 2. The display label is computed at read time from repo, work shape, duration and size -- it is a projection, not a column, so it cannot go stale as a session grows. 3. title_source distinguishes provider-supplied from absent; it does not need a value for the derived label because the derived label is not stored. 4. Un-skipping ai-title and acquiring threads.title are inputs, not the plan -- neither closes this bead alone. 5. Re-run 'polylogue find repo:polylogue' and show the before/after rows. 6. Report the collision rate of the derived label on a sample -- collisions are acceptable, silent staleness is not.","notes":"SCOPE CORRECTION (operator, 2026-07-29). An earlier draft of this bead proposed storing structural titles as a field. That is wrong for two reasons and the correction is the actual point:\n\n (a) it would collide with genuine provider titles, which now exist for Claude\n Code (ai-title) and Codex (threads.title);\n (b) a serialized structural label goes STALE the moment the session grows --\n 'polylogue - implementation - 340 msgs - 2h' freezes at 340 while the\n session continues. Storing a computed value and then needing machinery to\n keep it honest is the precise pattern this backlog is trying to remove.\n\nSo this is not a titles problem, it is an IDENTITY AND REPRESENTATION problem:\n - a session's identity is provider-supplied and stable;\n - its display label is a projection over current state and belongs in the read\n algebra (polylogue-4p1), computed per request;\n - the archive stores what the provider said, not what a renderer would say.\n\nAlso caution: the 'implementation/research/review/planning' work-event label\nproposed as a title input is itself heuristic -- constant per-type confidence,\nand classifications like 'Create my holiday video' -\u003e implementation. See the\nsession_work_events bead. Prefer structural facts that are not themselves\ninferred (repo, file paths, duration, message count, token spend).","status":"open","priority":1,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-29T04:52:16Z","created_by":"Sinity","updated_at":"2026-07-29T04:52:16Z","labels":["area:ingest","lane:read-contracts"],"comments":[{"id":"019fad19-4ef6-7355-8b52-d3a9cb8212c8","issue_id":"polylogue-6e7m","author":"Sinity","text":"Scoped work from the parsers-only lane (feature/chore/promote-schemas-and-wire-gates, common.py/ai_parser.py/assembly_codex.py/assembly_gemini.py). Not closing -- AC #2 (read-time display-label projection) belongs in insights/storage, both out of this lane's write scope.\n\nRe-measured (live archive, index v43, pre-rebuild -- reflects the OLD parsers, not what's about to ship):\n- claude-code-session: 10,157/12,001 (84.6%) title==native_id -- unchanged from the bead's original number, confirms the ai-title/custom-title wiring (already landed by a prior lane before this session) hasn't been exercised yet, only takes effect on rebuild.\n- codex-session: 3,201/3,201 (100%) title==native_id on the live archive -- the OLD codex parser wrote no sidecar title at all; thread-name/history/state-db resolution is new-parser-only (also prior-lane landed).\n- claude-ai-export: 1/377 (0.3%) title==native_id -- already near-total coverage; Claude web auto-titles almost every conversation.\n\nFound and fixed a real mislabeling bug in the newly-landed Codex title resolution (assembly_codex.py): history_titles (by construction the earliest authored prompt, per _parse_codex_history's own docstring) and state_titles (state_5.sqlite threads.title -- the exact field this bead's 78-way-collision measurement scanned) were both stamped TitleSource.ORIGIN at 0.9/0.75 confidence, the same claim as genuine curation, despite being provably first-prompt echoes. Verified empirically against this operator's own state_5.sqlite/history.jsonl: of 780 threads with a comparable history.jsonl row, 679 (87%) were exact-or-prefix matches of the session's own opening message. Added _is_prompt_echo (compares each candidate against the session's own first human-authored message) and downgraded matches to HEURISTIC/0.5 -- same title text, honest provenance. Applied to all three Codex evidence lanes (thread name, history, state db).\n\nAlso completed title_source/title_ref/title_confidence for claude-ai-export (ai_parser.py) -- parse_ai/_parse_design_chat resolved a real curated title but never stamped provenance at all before this change.\n\nConclusion on AC #2 (structural display-label projection, repo|files|messages|date): the derivation is real and was already measured in this bead's own description (3.5% collision vs 78-way echo collision), but I did not implement it as a parser-time write to sessions.title. Doing so would violate this bead's own scope-correction note (AC #1: sessions.title holds ONLY provider-supplied titles or NULL; a derived label is never written to it) and my lane's write scope excludes insights/** and storage/** where the read-time projection belongs. Recommend a follow-up bead scoped to insights/storage for the projection itself, separate from parser-level title-provenance hygiene.\n\nVerification: devtools test tests/unit/sources/test_assembly.py tests/unit/sources/test_parsers_claude_ai_catalog.py tests/unit/sources/test_parsers_props.py tests/unit/storage/test_title_ref_confidence_queryable.py tests/unit/sources/test_origin_specs.py -- all green except test_parsers_props.py's 4 pre-existing hypothesis failures (claude-code/codex role-consistency, confirmed unrelated/pre-existing). devtools verify --quick exit 0.\n\nAlso fixed (separate, coordinator-requested finding on polylogue-9x22): Claude AI web-tool evidence (integration_name, approval_key, display_content, etc.) merged into block.metadata was never persisted (no metadata column on blocks table) -- routed through session_events instead (common.py), following the hermes_spans.py precedent.\n","created_at":"2026-07-29T08:59:05Z"}],"dependency_count":0,"dependent_count":0,"comment_count":1} -{"_type":"issue","id":"polylogue-0jf4","title":"Codex SQLite state is never acquired: 5 databases, 706 MB, including spawn topology and 2,771 titles","description":"Measured 2026-07-29. ~/.codex holds five SQLite databases; raw_sessions contains no row whose source_path is any of them.\n\n state_5.sqlite 39 MB threads (3,054 rows, 2,771 with a non-empty title),\n thread_spawn_edges (1,030), thread_dynamic_tools,\n remote_control_enrollments, external_agent_config_imports\n logs_2.sqlite 627 MB logs (47,060 rows: ts, level, target, module_path,\n file, line, thread_id, process_uuid, estimated_bytes)\n memories_1.sqlite 456 KB stage1_outputs, jobs\n goals_1.sqlite 44 KB thread_goals, thread_goal_continuation_deferrals\n codex-dev.db 36 KB\n\nTHE MACHINERY ALREADY EXISTS AND IS USED FOR A DIFFERENT ORIGIN: Hermes .db\nfiles ARE acquired (/home/sinity/.hermes/state.db and verification_evidence.db\nappear in raw_sessions). SQLite-source acquisition is built, applied to one\nprovider, and not propagated -- the same shape as content-addressing being\napplied only to embeddings and OriginSpec declaring a detector order nothing\nreads.\n\nWHAT IS BEING RECONSTRUCTED BY INFERENCE INSTEAD:\n threads.title 2,771 -\u003e all 3,201 Codex sessions are UUID-titled\n (polylogue-ih67 builds a resolution ladder; the\n ladder's own notes cite this table as 'richer than\n session_index.jsonl on live installs')\n thread_spawn_edges 1,030 -\u003e Codex delegation topology, which polylogue-1vpm\n and polylogue-4ts derive from transcript inference\n thread_goals -\u003e stated task intent, unavailable anywhere else\n memories_1 stage1_outputs-\u003e Codex-side memory, no archive representation\n\nlogs_2 is 627 MB of runtime logging (level/target/module_path/file/line) rather\nthan session evidence -- classify it deliberately rather than acquiring by\ndefault. It may be the right home for runtime-observability questions, or it may\nbe correctly out of scope; the point is that nobody has decided.\n\nNOTE state_5.sqlite is live-locked on a running install; ih67's notes already\nprescribe copy-first.","acceptance_criteria":"1. Each of the five databases is classified as acquire / acquire-partially / out-of-scope, with the reason recorded in the Codex OriginSpec fidelity declaration. 2. Acquisition reuses the Hermes sqlite path rather than adding a second mechanism. 3. threads.title and thread_spawn_edges reach the archive as typed evidence and are consumed by title resolution and topology respectively. 4. Live-locked databases are copied before reading; a running Codex is never blocked. 5. Report the before/after UUID-title census for codex-session and the count of spawn edges that replaced inferred ones.","notes":"Implemented on branch feature/sources/acquire-sidecars-and-codex-sqlite (commits\n8e9778209 wiring, a70bd9257 tests), within OWNS: sources/live/batch.py,\nsources/live/watcher.py, sources/origin_specs.py (storage/sqlite untouched,\nper the concurrent schema-lane constraint on this branch).\n\nWHAT WAS UNACQUIRED AND WHY: sources/parsers/codex_state.py (classification +\nparsers) already existed but was completely unwired -- zero references from\ndispatch.py/batch.py/watcher.py, exactly as its own docstring stated. The root\ncause was never \"not implemented\" at the parser level; it was that\nsources/live/batch.py's ~2900-line acquire/parse loop special-cased Hermes by\nname (`provider is Provider.HERMES`) at three tail sites and had no equivalent\nbranch for a second sqlite-snapshot provider.\n\nWHAT CHANGED:\n- sources/live/batch.py: acquire loop gains a filename-gated (no I/O for the\n common case) + structurally-verified (codex_state.is_in_scope_codex_sqlite_path)\n branch for state_5.sqlite/goals_1.sqlite/memories_1.sqlite, snapshotting via\n the SAME snapshot_sqlite_to_blob (SQLite backup API, never a raw read of a\n live-locked file) Hermes already uses, minting a raw_id via\n codex_state_raw_id (AC2: no second mechanism). logs_2.sqlite/codex-dev.db\n are excluded by filename before any bytes are read (AC1's out-of-scope\n classification enforced at runtime, not just documented).\n- The three `provider is Provider.HERMES` special cases in the acquire-loop\n tail are generalized to `path in raw_source_revisions` / `record.blob_hash\n is not None` -- the real distinguishing signal (sqlite-snapshot acquisition\n vs. content-hash acquisition) rather than a Hermes-specific one, since Codex\n now shares Provider.CODEX with its own JSONL rollout acquisition.\n- Parse stage: a new elif (gated on provider is Provider.CODEX AND a\n structural re-check of the acquired blob, mirroring Hermes's own two elifs)\n routes thread_state to _write_codex_thread_state_evidence and admits\n goals_1/memories_1 raw bytes only (acquire-partial, no derived parse, per\n CODEX_STATE_FIDELITY) -- both bypass session materialization entirely via\n the same \"fact artifact\" continue idiom the codebase already uses.\n- sources/live/watcher.py: a SECOND WatchSource (\"codex-state\", root ~/.codex,\n suffixes .sqlite/.db) rather than widening the existing \"codex\" JSONL\n source's root -- avoids ever reasoning about history.jsonl/config.toml/log/\n under the shared root.\n- sources/origin_specs.py: _codex_spec() fidelity_notes now carries all 5\n databases' classification+reason (AC1), mirroring codex_state.py's\n CODEX_STATE_FIDELITY (that module explicitly names this file as the\n canonical home for the text).\n\nWHERE EVIDENCE LANDS: threads.title and thread_spawn_edges reach\nsource.db's raw_hook_events (event_type=codex_thread_title /\ncodex_thread_spawn_edge), keyed to the EXISTING codex-session row via\nsession_native_id=thread_id -- the SAME mechanism sources/hooks.py already\nuses for hook events (ArchiveStore.write_hook_event), read at query time via\nthe ALREADY-WIRED ArchiveStore.hook_event_summary_for_session /\nPolylogue.get_hook_event_summary_for_session (live in the CLI's message/read\nview). No index schema change: raw_hook_events.event_type is unconstrained\nTEXT, exactly the documented cheap route.\n\nMEASURED (read-only, real live ~/.codex install, scratch archive under\n/realm/tmp, never touched /realm/db/polylogue):\n state_5.sqlite 40,116,224 bytes acquired (backup took ~121s -- live\n WAL contention with the\n running Codex install;\n correctness unaffected,\n noted as an operational\n observation, not a bug)\n goals_1.sqlite 45,056 bytes acquired (0.5s)\n memories_1.sqlite 466,944 bytes acquired (0.3s)\n logs_2.sqlite 657,100,800 bytes excluded by name, 0 bytes read\n codex-dev.db -- absent on this install, skipped\n total blob bytes acquired: 53,023,051\n raw_sessions rows (raw-tier admission, NOT sessions): 3\n raw_hook_events: 4,085 total -- codex_thread_title=3,055, codex_thread_spawn_edge=1,030\n (1,030 matches the bead's own original spawn-edge count exactly)\n index.db sessions rows after ingest: 0 -- confirms the hard constraint\n (thread_spawn_edges/titles never mint a session)\n\nAC DISPOSITION:\n1. Classify each of 5 dbs with reason in Codex OriginSpec fidelity -- SATISFIED\n (origin_specs.py _codex_spec() fidelity_notes, all 5).\n2. Reuse the Hermes sqlite path, no second mechanism -- SATISFIED\n (snapshot_sqlite_to_blob shared; codex_state_raw_id mirrors\n hermes_profile_raw_id exactly).\n3. threads.title/thread_spawn_edges reach the archive as typed evidence --\n SATISFIED (raw_hook_events, verified against real data above). \"...and are\n consumed by title resolution and topology respectively\" -- NOT done in\n this lane; deliberately deferred (codex_state.py's own docstring already\n named assembly_codex.py/topology consumption out of scope to avoid\n colliding with the still-in-flight ih67 ladder). Follow-up filed:\n polylogue-foee.\n4. Live-locked databases copied before reading, running Codex never blocked --\n SATISFIED, verified against the REAL live install (state_5.sqlite was\n actively WAL-written during acquisition; backup succeeded, no lock\n contention errors, Codex itself was not blocked).\n5. Report before/after UUID-title census + spawn-edge replacement count --\n PARTIAL. Spawn-edge count IS reported above (1,030, matching the bead's\n original measurement exactly). The UUID-title census does NOT change in\n this PR: the acquired titles sit in raw_hook_events as typed evidence but\n nothing yet folds them into the session's own displayed title (that is\n exactly polylogue-foee's scope) -- so the honest report is \"evidence\n acquired, consumption and the resulting census change are the follow-up.\"\n\nVerification: devtools test tests/unit/sources/test_codex_state_live_ingest.py\ntests/unit/sources/test_live_watcher_catchup_order.py -\u003e 9 passed. mypy\n--strict + ruff clean on all touched files. Anti-vacuity confirmed by hand:\ntemporarily short-circuiting _write_codex_thread_state_evidence made the\nevidence-attachment test fail (`None == 1`) while the session-count and\nout-of-scope tests kept passing; reverted with a clean diff against the\ncommitted state (verified via `git diff --stat` showing no residual change).","status":"open","priority":1,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-29T04:52:14Z","created_by":"Sinity","updated_at":"2026-07-29T16:16:33Z","labels":["area:ingest","lane:origin-interop-export"],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-0jf4","title":"Codex SQLite state is never acquired: 5 databases, 706 MB, including spawn topology and 2,771 titles","description":"Measured 2026-07-29. ~/.codex holds five SQLite databases; raw_sessions contains no row whose source_path is any of them.\n\n state_5.sqlite 39 MB threads (3,054 rows, 2,771 with a non-empty title),\n thread_spawn_edges (1,030), thread_dynamic_tools,\n remote_control_enrollments, external_agent_config_imports\n logs_2.sqlite 627 MB logs (47,060 rows: ts, level, target, module_path,\n file, line, thread_id, process_uuid, estimated_bytes)\n memories_1.sqlite 456 KB stage1_outputs, jobs\n goals_1.sqlite 44 KB thread_goals, thread_goal_continuation_deferrals\n codex-dev.db 36 KB\n\nTHE MACHINERY ALREADY EXISTS AND IS USED FOR A DIFFERENT ORIGIN: Hermes .db\nfiles ARE acquired (/home/sinity/.hermes/state.db and verification_evidence.db\nappear in raw_sessions). SQLite-source acquisition is built, applied to one\nprovider, and not propagated -- the same shape as content-addressing being\napplied only to embeddings and OriginSpec declaring a detector order nothing\nreads.\n\nWHAT IS BEING RECONSTRUCTED BY INFERENCE INSTEAD:\n threads.title 2,771 -\u003e all 3,201 Codex sessions are UUID-titled\n (polylogue-ih67 builds a resolution ladder; the\n ladder's own notes cite this table as 'richer than\n session_index.jsonl on live installs')\n thread_spawn_edges 1,030 -\u003e Codex delegation topology, which polylogue-1vpm\n and polylogue-4ts derive from transcript inference\n thread_goals -\u003e stated task intent, unavailable anywhere else\n memories_1 stage1_outputs-\u003e Codex-side memory, no archive representation\n\nlogs_2 is 627 MB of runtime logging (level/target/module_path/file/line) rather\nthan session evidence -- classify it deliberately rather than acquiring by\ndefault. It may be the right home for runtime-observability questions, or it may\nbe correctly out of scope; the point is that nobody has decided.\n\nNOTE state_5.sqlite is live-locked on a running install; ih67's notes already\nprescribe copy-first.","acceptance_criteria":"1. Each of the five databases is classified as acquire / acquire-partially / out-of-scope, with the reason recorded in the Codex OriginSpec fidelity declaration. 2. Acquisition reuses the Hermes sqlite path rather than adding a second mechanism. 3. threads.title and thread_spawn_edges reach the archive as typed evidence and are consumed by title resolution and topology respectively. 4. Live-locked databases are copied before reading; a running Codex is never blocked. 5. Report the before/after UUID-title census for codex-session and the count of spawn edges that replaced inferred ones.","notes":"Implemented on branch feature/sources/acquire-sidecars-and-codex-sqlite (commits\n8e9778209 wiring, a70bd9257 tests), within OWNS: sources/live/batch.py,\nsources/live/watcher.py, sources/origin_specs.py (storage/sqlite untouched,\nper the concurrent schema-lane constraint on this branch).\n\nWHAT WAS UNACQUIRED AND WHY: sources/parsers/codex_state.py (classification +\nparsers) already existed but was completely unwired -- zero references from\ndispatch.py/batch.py/watcher.py, exactly as its own docstring stated. The root\ncause was never \"not implemented\" at the parser level; it was that\nsources/live/batch.py's ~2900-line acquire/parse loop special-cased Hermes by\nname (`provider is Provider.HERMES`) at three tail sites and had no equivalent\nbranch for a second sqlite-snapshot provider.\n\nWHAT CHANGED:\n- sources/live/batch.py: acquire loop gains a filename-gated (no I/O for the\n common case) + structurally-verified (codex_state.is_in_scope_codex_sqlite_path)\n branch for state_5.sqlite/goals_1.sqlite/memories_1.sqlite, snapshotting via\n the SAME snapshot_sqlite_to_blob (SQLite backup API, never a raw read of a\n live-locked file) Hermes already uses, minting a raw_id via\n codex_state_raw_id (AC2: no second mechanism). logs_2.sqlite/codex-dev.db\n are excluded by filename before any bytes are read (AC1's out-of-scope\n classification enforced at runtime, not just documented).\n- The three `provider is Provider.HERMES` special cases in the acquire-loop\n tail are generalized to `path in raw_source_revisions` / `record.blob_hash\n is not None` -- the real distinguishing signal (sqlite-snapshot acquisition\n vs. content-hash acquisition) rather than a Hermes-specific one, since Codex\n now shares Provider.CODEX with its own JSONL rollout acquisition.\n- Parse stage: a new elif (gated on provider is Provider.CODEX AND a\n structural re-check of the acquired blob, mirroring Hermes's own two elifs)\n routes thread_state to _write_codex_thread_state_evidence and admits\n goals_1/memories_1 raw bytes only (acquire-partial, no derived parse, per\n CODEX_STATE_FIDELITY) -- both bypass session materialization entirely via\n the same \"fact artifact\" continue idiom the codebase already uses.\n- sources/live/watcher.py: a SECOND WatchSource (\"codex-state\", root ~/.codex,\n suffixes .sqlite/.db) rather than widening the existing \"codex\" JSONL\n source's root -- avoids ever reasoning about history.jsonl/config.toml/log/\n under the shared root.\n- sources/origin_specs.py: _codex_spec() fidelity_notes now carries all 5\n databases' classification+reason (AC1), mirroring codex_state.py's\n CODEX_STATE_FIDELITY (that module explicitly names this file as the\n canonical home for the text).\n\nWHERE EVIDENCE LANDS: threads.title and thread_spawn_edges reach\nsource.db's raw_hook_events (event_type=codex_thread_title /\ncodex_thread_spawn_edge), keyed to the EXISTING codex-session row via\nsession_native_id=thread_id -- the SAME mechanism sources/hooks.py already\nuses for hook events (ArchiveStore.write_hook_event), read at query time via\nthe ALREADY-WIRED ArchiveStore.hook_event_summary_for_session /\nPolylogue.get_hook_event_summary_for_session (live in the CLI's message/read\nview). No index schema change: raw_hook_events.event_type is unconstrained\nTEXT, exactly the documented cheap route.\n\nMEASURED (read-only, real live ~/.codex install, scratch archive under\n/realm/tmp, never touched /realm/db/polylogue):\n state_5.sqlite 40,116,224 bytes acquired (backup took ~121s -- live\n WAL contention with the\n running Codex install;\n correctness unaffected,\n noted as an operational\n observation, not a bug)\n goals_1.sqlite 45,056 bytes acquired (0.5s)\n memories_1.sqlite 466,944 bytes acquired (0.3s)\n logs_2.sqlite 657,100,800 bytes excluded by name, 0 bytes read\n codex-dev.db -- absent on this install, skipped\n total blob bytes acquired: 53,023,051\n raw_sessions rows (raw-tier admission, NOT sessions): 3\n raw_hook_events: 4,085 total -- codex_thread_title=3,055, codex_thread_spawn_edge=1,030\n (1,030 matches the bead's own original spawn-edge count exactly)\n index.db sessions rows after ingest: 0 -- confirms the hard constraint\n (thread_spawn_edges/titles never mint a session)\n\nAC DISPOSITION:\n1. Classify each of 5 dbs with reason in Codex OriginSpec fidelity -- SATISFIED\n (origin_specs.py _codex_spec() fidelity_notes, all 5).\n2. Reuse the Hermes sqlite path, no second mechanism -- SATISFIED\n (snapshot_sqlite_to_blob shared; codex_state_raw_id mirrors\n hermes_profile_raw_id exactly).\n3. threads.title/thread_spawn_edges reach the archive as typed evidence --\n SATISFIED (raw_hook_events, verified against real data above). \"...and are\n consumed by title resolution and topology respectively\" -- NOT done in\n this lane; deliberately deferred (codex_state.py's own docstring already\n named assembly_codex.py/topology consumption out of scope to avoid\n colliding with the still-in-flight ih67 ladder). Follow-up filed:\n polylogue-foee.\n4. Live-locked databases copied before reading, running Codex never blocked --\n SATISFIED, verified against the REAL live install (state_5.sqlite was\n actively WAL-written during acquisition; backup succeeded, no lock\n contention errors, Codex itself was not blocked).\n5. Report before/after UUID-title census + spawn-edge replacement count --\n PARTIAL. Spawn-edge count IS reported above (1,030, matching the bead's\n original measurement exactly). The UUID-title census does NOT change in\n this PR: the acquired titles sit in raw_hook_events as typed evidence but\n nothing yet folds them into the session's own displayed title (that is\n exactly polylogue-foee's scope) -- so the honest report is \"evidence\n acquired, consumption and the resulting census change are the follow-up.\"\n\nVerification: devtools test tests/unit/sources/test_codex_state_live_ingest.py\ntests/unit/sources/test_live_watcher_catchup_order.py -\u003e 9 passed. mypy\n--strict + ruff clean on all touched files. Anti-vacuity confirmed by hand:\ntemporarily short-circuiting _write_codex_thread_state_evidence made the\nevidence-attachment test fail (`None == 1`) while the session-count and\nout-of-scope tests kept passing; reverted with a clean diff against the\ncommitted state (verified via `git diff --stat` showing no residual change).","status":"closed","priority":1,"issue_type":"task","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-07-29T04:52:14Z","created_by":"Sinity","updated_at":"2026-07-31T04:15:24Z","started_at":"2026-07-31T04:13:14Z","closed_at":"2026-07-31T04:15:24Z","close_reason":"RE-VERIFIED 2026-07-31, no code changes needed: the entire in-scope acquisition\nthis bead calls for was ALREADY on origin/master before this session started,\nlanded via commit de8717a936 (\"feat(sources): acquire Codex threads/spawn-edges\nas typed evidence\") as part of the large feature/chore/promote-schemas-and-wire-gates\nmerge train -- NOT via the stale local branch\nfeature/sources/acquire-sidecars-and-codex-sqlite this bead's own notes\ndescribe (commits 8e9778209/a70bd9257 on that branch never got pushed or PR'd;\ncherry-picking them onto a fresh branch off origin/master produced an EMPTY\ndiff, proving byte-for-byte equivalent content already shipped).\n\nConfirmed present and correct on master (read-only inspection, no ~/.codex\nwrites):\n- polylogue/sources/parsers/codex_state.py: classifies all 5 dbs\n (thread_state/goals/memories -\u003e acquire[-partial], logs/automation -\u003e\n out-of-scope) via CODEX_STATE_FIDELITY.\n- sources/origin_specs.py _codex_spec(): fidelity_notes carries all 5\n classifications + reasons (AC1 satisfied).\n- sources/live/batch.py: acquire loop snapshots state_5/goals_1/memories_1\n via the SAME snapshot_sqlite_to_blob Hermes uses (AC2: no second\n mechanism); logs_2.sqlite/codex-dev.db excluded by name before any bytes\n read; parse stage attaches threads.title/thread_spawn_edges to the\n EXISTING codex-session row via write_hook_event (event_type\n codex_thread_title/codex_thread_spawn_edge), never minting a session of\n its own (AC3 acquisition half + AC4's session-count-inflation guard).\n- sources/live/watcher.py: second \"codex-state\" WatchSource rooted at\n ~/.codex (suffixes .sqlite/.db), separate from the \"codex\" JSONL source's\n ~/.codex/sessions root.\n- Live-locked read safety (AC4): snapshot_sqlite_to_blob uses the sqlite3\n backup API, never a raw read of the live file.\n\nTests: devtools test tests/unit/sources/test_codex_state_live_ingest.py\ntests/unit/sources/parsers/test_codex_state.py\ntests/unit/sources/parsers/test_codex_state_schema_canary.py -\u003e 22 passed.\n\nReal ~/.codex measurement (read-only, sqlite3 file:...?mode=ro, no writes):\n state_5.sqlite: threads=3,057 rows, 2,774 with non-empty title (bead's\n original count: 3,054/2,771 -- grew by 3 in the 2 days since filing,\n consistent with normal usage, not a discrepancy)\n thread_spawn_edges: 1,030 (exact match to bead's original count)\n goals_1.sqlite thread_goals: 26 rows\n memories_1.sqlite stage1_outputs: 30 rows\n codex-dev.db: absent on this install (handled: out-of-scope name, no-op)\n\nAC DISPOSITION (unchanged from the prior session's own analysis, now\nverified against master rather than an unlanded branch):\n1. Classify each of 5 dbs with reason in Codex OriginSpec fidelity --\n SATISFIED.\n2. Reuse the Hermes sqlite path, no second mechanism -- SATISFIED.\n3. threads.title/thread_spawn_edges reach the archive as typed evidence --\n SATISFIED (raw_hook_events). \"...and are consumed by title resolution\n and topology respectively\" -- NOT done, deliberately deferred to the\n already-filed polylogue-foee (title-ladder consumption is\n sources/assembly_codex.py, topology consumption is the\n polylogue-1vpm/4ts inferred-edge reader -- both outside this bead's\n parsers/codex*.py + OriginSpec + tests write surface, and foee is\n explicitly scoped to exactly that remaining work).\n4. Live-locked databases copied before reading -- SATISFIED (sqlite3 backup\n API, verified in source).\n5. Report before/after UUID-title census + spawn-edge count -- PARTIAL,\n same as previously documented: spawn-edge count reported above (1,030).\n The census does not change until polylogue-foee wires title-ladder\n consumption; until then all Codex sessions remain UUID-titled by design\n (the acquired titles sit in raw_hook_events, not yet folded into the\n session's displayed title).\n\nClosing as satisfied within this bead's write scope (parsers/codex*.py,\nCODEX_SESSION OriginSpec, tests) -- AC3's consumption half and AC5's\npost-consumption census are polylogue-foee's scope, already tracked there\nand correctly out of this bead's surface (foee's own AC1/AC2 name\nsources/assembly_codex.py and the topology insight reader, not this bead's\nfiles). No PR opened: verified zero diff against origin/master, nothing to\nland.\n","labels":["area:ingest","lane:origin-interop-export"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-rujy","title":"Claude Code tool-results sidecars unacquired: 1.34 GB across 12,588 files, 3 ingested","description":"Measured 2026-07-29 against ~/.claude/projects and the live source tier.\n\n ON DISK ACQUIRED (raw_sessions.source_path)\n *.jsonl 11,540 files 13,275.9 MB projects/ root 5,559\n subagents/ 572 dirs 2,933.2 MB subagents/ 13,916\n tool-results/ 582 dirs 1,339.6 MB tool-results/ 3\n memory/ 14 dirs 0.8 MB --\n\nClaude Code writes a tool result to \u003csession\u003e/tool-results/\u003ctool_id\u003e.\u003cext\u003e\nwhenever the output exceeds the inline limit, leaving only a stub in the\ntranscript. 3,058 tool_result blocks already in the archive contain the literal\ntext 'Full output saved to' -- the archive is storing its own admission that the\ncontent is elsewhere, and the elsewhere is never read.\n\nTHE JOIN IS TRIVIAL: the filename IS the tool id. Verified by intersecting\n10,545 distinct tool ids from disk filenames against a 400k-row sample of\nblocks.tool_id -- 2,813 matched on the sample alone.\n\nCONSEQUENCES: FTS cannot match anything that lived in a large tool output, so\n'polylogue find X' silently misses it; and outcome/exit-code evidence carried in\na truncated result is unavailable, which is one contributor to the 72%-unknown\ntool_result_is_error measured on this archive.\n\nHARD CONSTRAINT (operator, 2026-07-29): these are BLOCK CONTENT, not sessions.\nThe hook-event inflation incident is the precedent -- standalone ingestion of\nnon-session records inflated the archive from 18,391 to 83,286 sessions before\nbeing reverted. A tool-results file must attach to its existing tool_result\nblock by tool_id and must never create a session, a raw session row that parses\nas a session, or a new top-level unit of any kind.","acceptance_criteria":"1. tool-results content attaches to the existing tool_result block via tool_id; session count is unchanged before and after, asserted by a test. 2. Its text is searchable -- a term that appears only inside a large tool output is findable via FTS. 3. Unmatched files (a tool id with no block) are recorded as typed acquisition debt, not silently dropped. 4. Blob storage is content-addressed and deduplicated; report bytes added. 5. Ingest wall-clock impact is measured against the polylogue-623q envelope before this is enabled by default.","notes":"Investigation + scoped implementation landed (worktree-agent-a6730c39cc2369360, commit 9b37431fd on feature/chore/promote-schemas-and-wire-gates):\n\nMEASUREMENT (read-only against live ~/.claude/projects, sampled ~330MB across 80 sessions):\n- Genuinely-truncated \"output too large\" overflow sidecars: ~12% of files, ~60-65% of bytes, ~99% new content beyond the inline preview.\n- Never-truncated \"full mirror\" sidecars (Claude Code unconditionally persists many small Read/Grep/Edit results too): ~85% of files, \u003c2% of bytes, ~97% already-duplicate of inline block text.\n- Orphan sidecars with no owning tool_result block left in the retained transcript (compaction pruned the referencing turn): ~1-5% of files, real acquisition debt, not a bug in the join.\n- Filename scheme (toolu_\u003cid\u003e vs internal short-slug vs call_NN_\u003cid\u003e vs mcp-\u003cserver\u003e-\u003ctool\u003e-\u003cts\u003e) does NOT predict which bucket a file is in -- both truncated-overflow and full-mirror sidecars use both toolu_ and non-toolu_ names. The reliable join key is always the owning block's tool_use_id, recovered directly (filename stem) or via the \"Full output saved to\"/\"Output has been saved to\" pointer in that block's own preview text -- including for Task subagent transcripts, whose sidecars persist to the session-level tool-results/ dir under the subagent's own (non-toolu_) tool id, not a per-subagent dir.\n- hook-*.txt files under the same directory (185 of 12,588 sampled) are a separate, already-tracked mechanism (raw hook stdout, polylogue-qqyg/#2781) -- correctly excluded from both acquisition and debt.\n\nRECOMMENDATION: acquire, but content-aware (replace-when-truncated), not blanket-copy-the-directory. This is what was built.\n\nBUILT (within OWNS: sources/live/**, sources/parsers/claude/**):\n- polylogue/sources/live/tool_result_sidecars.py: join_tool_result_sidecars(payload, tool_results_dir) -\u003e SidecarJoinResult(matched, debt). Read-only, no writes.\n- polylogue/sources/parsers/claude/code_parser.py: apply_tool_result_sidecars() attaches the join result to an already-parsed ParsedSession -- replaces truncated tool_result block text (AC2: FTS indexes block content, so this makes large-output terms findable), leaves full-mirror blocks untouched, and emits a bounded claude_tool_result_sidecar session_event per file (matched or debt) -- id/filename/size/content_hash/status only, never raw bytes in the event (no schema bump needed, per constraint). parse_code/parse_code_stream take tool_result_sidecars as an optional kwarg; omitting it is a no-op (verified).\n- tests/unit/sources/test_tool_result_sidecars.py: 5 tests, synthetic fixtures only. Verifies AC1 (session/message count and ids unchanged with sidecars attached), AC2 (a term only in the full sidecar becomes findable in block text; anti-vacuity confirmed -- nulling the replacement dict makes this assertion fail, not a self-validating mock), AC3 (unmatched file becomes a typed debt event; hook-*.txt never does).\n\nAC DISPOSITION:\n1. Attaches by tool_id, session count unchanged, asserted by test -- SATISFIED (test_apply_tool_result_sidecars_replaces_truncated_block_text_only).\n2. FTS-findable -- SATISFIED at the block-content layer (block.text is what FTS indexes); not verified end-to-end through a live FTS query in this pass since that requires the dispatch.py wiring below to actually run during ingest.\n3. Unmatched -\u003e typed acquisition debt, not silently dropped -- SATISFIED (SidecarDebt -\u003e claude_tool_result_sidecar event, acquisition_status=debt, reason=no_owning_tool_result_block).\n4. Blob storage content-addressed + deduplicated, bytes-added report -- PARTIAL. content_hash is computed and recorded per sidecar (SHA-256 via core.hashing.hash_text) but there is no dedicated blob_refs-tier write here; the acquired text rides into the existing blocks table via the block's own text field, which already participates in the archive's session-level content-hash idempotency. True cross-session blob dedup needs storage-tier work (storage/repair.py or a raw_authority.py-adjacent path), explicitly outside this lane's OWNS list. Not implemented; flagged as a real gap, not silently declared done.\n5. Ingest wall-clock vs polylogue-623q envelope, default-off until measured -- NOT DONE. This lane never got as far as running ingest, because the acquisition path isn't wired into dispatch.py yet (see polylogue-wjgf). Cannot honestly claim this AC without that wiring existing to measure.\n\nFOLLOW-UP: polylogue-wjgf (dispatch.py wiring: derive tool-results dir from source_path, call the join, pass result into parse_code; plus the default-on-vs-flagged decision needing config.py/CLI, and the streaming-path equivalent). AC4's blob-store dedup and AC5's wall-clock measurement both depend on that wiring landing first.\n\nVerification: devtools test tests/unit/sources/test_tool_result_sidecars.py tests/unit/sources/test_parsers_claude_code_artifacts.py -\u003e 31 passed. mypy --strict clean on both changed modules. ruff check/format clean. devtools render topology-projection + topology-status regenerated and committed (new module under polylogue/). devtools render all --check: no \"out of sync\" lines.\n\n[2026-07-29, polylogue-wjgf follow-up] Wiring landed (branch feature/chore/promote-schemas-and-wire-gates, commit 2237e8a82). AC5 (ingest wall-clock vs polylogue-623q envelope, default-off until measured) is now resolved: measured join_tool_result_sidecars against the FULL population of real ~/.claude/projects sessions with a tool-results/ dir (525 sessions) -- total added join time 8.2s (704MB matched + 708MB debt bytes, 9,421 matched files / 3,012 debt files), ~23% on top of just those sessions' own JSONL read time but those sessions are ~3% of the corpus, so well under 1% of a \u003c60min full-rebuild budget. Decision: default-on, no flag. AC4 (blob-store dedup) remains PARTIAL/deferred as originally noted -- still needs storage-tier work outside sources/live and sources/dispatch scope; not addressed by wjgf.","status":"open","priority":1,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-29T04:52:13Z","created_by":"Sinity","updated_at":"2026-07-29T08:37:47Z","labels":["area:ingest","lane:origin-interop-export"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-j8u2","title":"Subagent children are 45.6% of the archive and rank equal to real sessions in every result list","description":"Measured 2026-07-29: 8,614 of 18,871 sessions (45.6%) are subagent children; 8,824 session_links rows carry link_type='subagent'.\n\nThey are presented at equal weight in default result sets. Actual output of the\ndeployed CLI against the live archive:\n\n $ polylogue find repo:polylogue\n claude-code-session:5ecd 2026-07-27 5ecdb160-...-a3a24886af8cc:agent-af4e... (342 msgs)\n claude-code-session:5ecd 2026-07-27 5ecdb160-...-a3a24886af8cc:agent-ad73... (1098 msgs)\n claude-code-session:5ecd 2026-07-28 5ecdb160-...-a3a24886af8cc:agent-ad68... (499 msgs)\n\nThree rows, one parent, differing only by an agent suffix -- and the same shape\nfills . Combined with the title defect, a default query returns a\nlist that is ~46% fanout and ~85% UUID-labelled.\n\nThis is not a correctness bug and not a latency bug. The queries are right and\nfast (2.8-7.6s measured). It is the reason the archive cannot be read by a\nhuman, and therefore the practical gate on the operator using the product at\nall -- ahead of every substrate program in the backlog.\n\nNON-GOAL: hiding subagent evidence. It is real work and must stay queryable and\ncitable. The default result UNIT should be the top-level session, with its\ndelegation fan available on request, rather than one row per spawn.\n\nRelated: polylogue-4ts (lineage truth: counted once) is the storage-side\nstatement of the same problem; this bead is the read-side one. polylogue-fcyf\nwants fanout as a first-class live view, which is the deliberate opposite\npresentation and stays valid.","acceptance_criteria":"1. The default result unit is the top-level session; subagent children are reachable through an explicit projection, not by filling the list. 2. Session counts on read surfaces state which unit they count -- an archive of 18,871 rows containing 8,614 fanout children must never present '18,871 sessions' unqualified. 3. Subagent evidence remains fully queryable and citable; a query that asks for children still gets them. 4. Re-run the exact dogfood commands and show before/after output in the closing note.","status":"open","priority":1,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-29T04:52:09Z","created_by":"Sinity","updated_at":"2026-07-29T04:52:09Z","labels":["area:query","lane:read-contracts"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-t5lg","title":"84.6% of Claude Code sessions are titled with a raw UUID: 10,157 of 12,001","description":"Measured on the live archive 2026-07-29 (full scan, 18,871 sessions):\n\n origin total title = native_id pct\n claude-code-session 12,001 10,157 84.6%\n codex-session 3,201 3,201 100.0% \u003c- owned by polylogue-ih67\n hermes-session 279 157 56.3%\n aistudio-drive 239 88 36.8%\n gemini-cli-session 17 7 41.2%\n chatgpt-export 2,635 0 0.0%\n claude-ai-export 377 1 0.3%\n antigravity-session 116 0 0.0%\n grok-export 6 0 0.0%\n\n archive-wide: 13,611 of 18,871 (72.1%) titled with a UUID,\n plus 2,369 (12.6%) with titles over 60 chars (prompt echoes)\n -\u003e 84.7% of the archive has no usable title.\n\nThe split is exactly provider-generated vs locally-captured: web exports arrive\nwith titles because the provider makes one; local coding-agent sessions do not,\nbecause nothing generates one. The two origins that dominate the archive\n(15,202 of 18,871 = 80.6%) are the two with essentially no titles.\n\nOnly 1,241 of the UUID-titled rows are subagent children, so this is NOT a\nfanout artifact: roughly 8,900 TOP-LEVEL Claude Code sessions -- the operator's\nown primary work -- are unlabelled.\n\npolylogue-ih67 owns the Codex 3,201 and has already built the resolution\nladder (thread name -\u003e authored history -\u003e first HUMAN_AUTHORED message -\u003e\nnative id). Nothing owns the Claude Code 10,157, which is 3.2x larger. This\nshould reuse ih67's mechanism rather than invent a second one; polylogue-30h\nowns the separate first-prompt-echo case.","acceptance_criteria":"1. A Claude Code session's display title is derived from authored content, never its UUID, using ih67's existing resolution ladder rather than a parallel mechanism. 2. Title provenance is recorded (title_source/title_ref), so a synthesized title is distinguishable from a provider-supplied one. 3. Live re-measure: UUID-titled claude-code-session count falls from 10,157 toward zero, reported as a before/after census like ih67 AC#6. 4. Existing rows acquire titles through ordinary reprocess, not a bespoke backfill script.","status":"open","priority":1,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-29T04:52:07Z","created_by":"Sinity","updated_at":"2026-07-29T04:52:07Z","labels":["area:ingest","lane:read-contracts"],"dependency_count":0,"dependent_count":0,"comment_count":0} @@ -454,6 +465,25 @@ {"_type":"issue","id":"polylogue-sru.2","title":"Characterize ambiguous bucket: wordless continuation vs prose-without-markers","description":"Split next-turn-is-tool-call (wordless continuation) from prose-lacking-ack-markers; state counts for both. Opus-4-7 74% ambiguous vs deepseek 17% is likely turn-structure variance, not behavior — this split disambiguates.","design":"Implementation home: the claim-vs-evidence classifier in devtools (devtools/ module behind `devtools workspace claim-vs-evidence`; tests tests/unit/devtools/test_claim_vs_evidence.py). Wordless-continuation detection: for each failure's paired next assistant message, check whether its blocks contain tool_use and no text block with \u003eN chars before the first tool_use — that is 'wordless continuation'; prose without matched ack markers stays 'ambiguous-prose'. Emit both as classification_reason variants (field already exists) and add the two counts to the report summary + by_model/by_tool cuts. Regen: `devtools workspace claim-vs-evidence --limit 5000 --out-dir .agent/demos/claim-vs-evidence --json`. Acceptance: report shows ambiguous split into wordless_continuation vs prose_no_marker with counts; per-model ambiguous variance (opus-4-7 74% vs deepseek 17%) re-examined after the split.","notes":"2026-07-03 Codex WIP: unit implementation for ambiguous split passes focused tests, but live regeneration with --limit 5000 became too slow and had to be killed twice. First attempt used correlated subqueries for next-message block shape; second used set-based CTE; third used chunked second query after sampled rows, but the full command still exceeded 90s on active archive and ignored SIGINT while inside SQLite. Do not close or commit this slice until the live regeneration path is profiled/fixed. Dirty files currently show the WIP implementation: devtools/claim_vs_evidence.py and tests/unit/devtools/test_claim_vs_evidence.py. Last passing focused proof: python -m py_compile + ruff check + devtools test tests/unit/devtools/test_claim_vs_evidence.py -\u003e 3 passed.","status":"closed","priority":1,"issue_type":"task","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-07-03T04:31:28Z","created_by":"Sinity","updated_at":"2026-07-03T07:45:10Z","started_at":"2026-07-03T07:09:21Z","closed_at":"2026-07-03T07:45:10Z","close_reason":"Completed: claim-vs-evidence now splits ambiguous follow-ups into wordless tool continuations and prose-without-marker buckets, reports the counts in JSON/README summaries, and regenerates the current demo on the active archive. Focused tests pass; live regen/check completed.","labels":["area:substrate","campaign"],"dependencies":[{"issue_id":"polylogue-sru.2","depends_on_id":"polylogue-sru","type":"parent-child","created_at":"2026-07-03T06:31:27Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":1,"comment_count":0} {"_type":"issue","id":"polylogue-sru.3","title":"Benign-recovery vs consequential-silence split by handler kind","description":"Read failures are ~94% silent but 'tried another path' is usually benign; Bash/test failures are the consequential class. Scope the headline to consequential handler kinds or add an explicit split — credibility depends on not inflating with trivial recoveries.","design":"Handler kind is already available on the paired failure row (actions lane exposes handler/tool). Define the consequential set explicitly in code (Bash/test/build/write-class handlers) and the benign-recovery set (Read/Glob/Grep-class 'tried another path'), emit split headline rows: silent-proceed among consequential vs among all. Keep the mapping a named constant with a rationale comment so reviewers can argue with it. Report both; never let the headline mix classes silently. Same regen/tests as the other methodology children.","status":"closed","priority":1,"issue_type":"task","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-07-03T04:31:28Z","created_by":"Sinity","updated_at":"2026-07-03T07:58:08Z","started_at":"2026-07-03T07:55:37Z","closed_at":"2026-07-03T07:58:08Z","close_reason":"Completed: claim-vs-evidence now reports a first-class handler-class split separating consequential shell/edit/write-class tool failures from benign read/search/path-discovery failures and other tools. The regenerated active-archive artifact shows consequential=4,177 failures with 921 silent-proceed (22.0% lower bound), benign_recovery=633 with 166 silent-proceed (26.2%), and other=190 with 92 silent-proceed (48.4%). Focused tests and demo shelf checks passed.","labels":["area:substrate","campaign"],"dependencies":[{"issue_id":"polylogue-sru.3","depends_on_id":"polylogue-sru","type":"parent-child","created_at":"2026-07-03T06:31:28Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":1,"comment_count":0} {"_type":"issue","id":"polylogue-sru.1","title":"Expose action-unit outcome fields + followup_class as product capability","description":"Capabilities-may-not-be-silos gate for the campaign: the facts the report needs must become composable query capability. After this, the whole report is `actions where is_error:true | group by session.origin, followup_class | count` and every future cut (model/tool/repo/time) is free.","design":"1) is_error/exit_code are normalized at parse time (sources/parsers/base_models.py:74-75) but ActionQueryRowPayload (surfaces/payloads.py:~1298) carries neither — add as filterable/groupable action-unit fields. 2) Add derived followup_class (acknowledged|silent_proceed|wordless_continuation|ambiguous) + followup_message_ref computed in the source-derived lowering (no cache tables). 3) Reduce devtools workspace claim-vs-evidence to a render preset over these query strings, or retire it. Touchpoint chain: stage parser -\u003e AST to_payload -\u003e executor -\u003e metadata.py aggregate_group_fields -\u003e shell_completion_values.py -\u003e devtools render openapi + cli-output-schemas + cli-reference. Line refs pre-07-03; re-locate.","acceptance_criteria":"Fixture session with known unacknowledged failure fires via pure query strings; report README numbers reproducible from the printed queries.","notes":"Completed: action-unit outcome follow-up classification is now shared query capability. is_error/exit_code were already wired; this slice added source-derived followup_class and followup_message_ref over existing actions/messages/blocks, exposed followup_class as filterable/groupable action metadata, added action row payload fields, routed root CLI terminal-unit aggregate expressions before session-selector compilation, and moved the report classifier from scripts into polylogue.archive.actions.followup. Reproduction/query forms are now printed in .agent/demos/claim-vs-evidence/PUBLIC_REPRODUCTION.md: actions where is_error:true | group by followup_class | count; actions where followup_class:silent_proceed. Verification: focused DSL/report/CLI tests passed; active demo packet regenerated over archive root /home/sinity/.local/share/polylogue schema v23 with 41,886 structured failures and 5,000 inspected; devtools verify --quick passed run 20260703T092510Z-quick-718233-46e8b587.","status":"closed","priority":1,"issue_type":"feature","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-07-03T04:31:27Z","created_by":"Sinity","updated_at":"2026-07-03T09:25:36Z","started_at":"2026-07-03T09:05:37Z","closed_at":"2026-07-03T09:25:36Z","close_reason":"Completed","labels":["area:query","area:substrate","campaign"],"dependencies":[{"issue_id":"polylogue-sru.1","depends_on_id":"polylogue-sru","type":"parent-child","created_at":"2026-07-03T06:31:26Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-5jnq","title":"Work-evidence adapter: Beads issues.jsonl as issue nodes + dependency edges (1vpm.6 adapter)","description":"Follow-on from the polylogue-qj5x decision (design doc: .agent/scratch/live/beads-handling-design-2026-07-31.html). The Origin route ingested only interactions.jsonl (100% field_change audit rows). The genuinely informative Beads artifact is issues.jsonl: measured in the polylogue workspace, 1,260 issues, 907 with notes, 1,857 dependency edges, plus descriptions/design/acceptance-criteria — none of it currently represented anywhere in the archive.\n\nTARGET: a work-evidence adapter (sibling of BeadsIssueEffectAdapter in insights/work_effects.py) that reads a workspace's .beads/issues.jsonl and emits Beads-issue NODES for the 1vpm.6 work-evidence graph:\n- node ref keyed by the bead id itself (e.g. beads:polylogue-x4s) — the workspace prefix already provides global uniqueness; do NOT reintroduce the removed parser's sha256(workspace-root) key, which splits worktrees exactly like the cijx.1 repo-identity defect.\n- issue node carries title, status, priority, created/updated, and evidence_refs to the ledger lines; dependency edges become typed issue→issue edges (blocks/discovered-from/...), 1,857 measured in polylogue alone.\n- interactions.jsonl rows remain ObservedRepositoryEffect facts (existing adapter) and attach to these nodes as observed_effect edges with occurred_at, old→new, and close reasons (which carry commit hashes — join material for claim reconciliation).\n- CAVEAT measured 2026-07-31: interactions.jsonl actor is constant per repo (\"Sinity\" 2,249/2,249 in polylogue) — it is the git user, not real actor attribution. Session attribution must come from the session side (bd tool_use commands in action blocks), never from the ledger actor field.\n\nThis is an adapter of 1vpm.6's core graph per its 2026-07-15 invariant-collapse note (\"Complete Beads baseline/history acquisition is a required adapter of the core work-evidence graph, not an independently valuable product surface\"). It should also give 1vpm.6 the issue side of the session↔PR↔issue three-way join (session↔PR from pbuh's typed pr-link records; issue↔PR from exact-id tokens in PR bodies/close reasons).\n","status":"open","priority":2,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-31T04:37:35Z","created_by":"Sinity","updated_at":"2026-07-31T04:37:35Z","dependencies":[{"issue_id":"polylogue-5jnq","depends_on_id":"polylogue-qj5x","type":"blocks","created_at":"2026-07-31T06:37:35Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-0pyp","title":"Work-evidence adapter: Beads issues.jsonl as issue nodes + dependency edges (1vpm.6 adapter)","description":"Follow-on from the polylogue-qj5x decision (design doc: .agent/scratch/live/beads-handling-design-2026-07-31.html). The Origin route ingested only interactions.jsonl (100% field_change audit rows). The genuinely informative Beads artifact is issues.jsonl: measured in the polylogue workspace, 1,260 issues, 907 with notes, 1,857 dependency edges, plus descriptions/design/acceptance-criteria — none of it currently represented anywhere in the archive.\n\nTARGET: a work-evidence adapter (sibling of BeadsIssueEffectAdapter in insights/work_effects.py) that reads a workspace's .beads/issues.jsonl and emits Beads-issue NODES for the 1vpm.6 work-evidence graph:\n- node ref keyed by the bead id itself (e.g. beads:polylogue-x4s) — the workspace prefix already provides global uniqueness; do NOT reintroduce the removed parser's sha256(workspace-root) key, which splits worktrees exactly like the cijx.1 repo-identity defect.\n- issue node carries title, status, priority, created/updated, and evidence_refs to the ledger lines; dependency edges become typed issue→issue edges (blocks/discovered-from/...), 1,857 measured in polylogue alone.\n- interactions.jsonl rows remain ObservedRepositoryEffect facts (existing adapter) and attach to these nodes as observed_effect edges with occurred_at, old→new, and close reasons (which carry commit hashes — join material for claim reconciliation).\n- CAVEAT measured 2026-07-31: interactions.jsonl actor is constant per repo (\"Sinity\" 2,249/2,249 in polylogue) — it is the git user, not real actor attribution. Session attribution must come from the session side (bd tool_use commands in action blocks), never from the ledger actor field.\n\nThis is an adapter of 1vpm.6's core graph per its 2026-07-15 invariant-collapse note (\"Complete Beads baseline/history acquisition is a required adapter of the core work-evidence graph, not an independently valuable product surface\"). It should also give 1vpm.6 the issue side of the session↔PR↔issue three-way join (session↔PR from pbuh's typed pr-link records; issue↔PR from exact-id tokens in PR bodies/close reasons).\n","status":"open","priority":2,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-31T04:37:25Z","created_by":"Sinity","updated_at":"2026-07-31T04:37:25Z","dependencies":[{"issue_id":"polylogue-0pyp","depends_on_id":"polylogue-qj5x","type":"blocks","created_at":"2026-07-31T06:37:24Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-zc4a","title":"otlp_correlation.py queries columns that don't exist in the live otlp_spans schema, and ignores typed parent_span_id in favor of time-overlap heuristics","description":"Found during the 2026-07-31 heuristics audit (parallel to polylogue-pbuh/polylogue-1vpm.7).\n\nSCHEMA DRIFT (the more severe defect): _query_spans_for_session (polylogue/insights/otlp_correlation.py:135-150) selects columns session_id, agent_id, operation_name, start_time_unix_ns, end_time_unix_ns, duration_ms, status_code, status_message from otlp_spans. VERIFIED against both source.db and index.db live schema (sqlite3 ... .schema otlp_spans): the real DDL (storage/sqlite/archive_tiers/source.py:309-322, mirrored ops.py:140-154) has session_native_id, name, kind, started_at_ms, ended_at_ms, attributes_json, events_json -- none of the queried column names exist. Any real call to this path raises sqlite3.OperationalError: no such column, caught generically in _print_otlp_evidence (correlation_view.py:87-107) and printed as a bare 'query failed' -- the CLI surface (analyze correlation --otlp) is silently broken end to end, not merely heuristic. VERIFIED root cause of the miss: tests/unit/insights/test_otlp_correlation.py's _init_db_with_otlp_table (lines 18-42) hand-builds its own toy schema matching the CODE's imagined columns rather than the production DDL -- a self-authored replica that validates the module against itself and can never catch this drift.\n\nHEURISTIC-OVER-TYPED-FIELD (the pattern this audit is hunting): even once the schema bug is fixed, correlate_spans_to_work_events (otlp_correlation.py:193-264) joins spans to session_work_events by wall-clock time-range overlap, and _is_tool_span/_is_llm_span (otlp_correlation.py:446-481) classify spans by string-prefix matching on operation_name -- while the real otlp_spans schema carries parent_span_id (an exact typed parent/child edge) and kind (a typed span-kind enum), both unused by the correlation logic. Not evaluated: no test compares the overlap-matching heuristic's hit rate against what parent_span_id would give directly.\n\nBLAST RADIUS: VERIFIED currently 0 -- sqlite3 source.db \"SELECT COUNT(*) FROM otlp_spans\" -\u003e 0 rows live. OTLP ingestion is not yet populating this table, so today this is dead/unexercised code, not a live-data-corrupting bug. It will misbehave immediately (OperationalError on every call) the moment OTLP ingestion starts writing spans, unless fixed first.","acceptance_criteria":"1. _query_spans_for_session's column list matches the live otlp_spans DDL exactly (session_native_id/name/kind/started_at_ms/ended_at_ms/attributes_json/events_json, or the DDL is changed to match the code's intent -- pick one and align both). 2. The test fixture in test_otlp_correlation.py builds its table via the production DDL helper (e.g. importing the real CREATE TABLE from archive_tiers/source.py or ops.py) rather than a hand-authored replica schema, so schema drift is caught automatically. 3. correlate_spans_to_work_events joins on parent_span_id where present before falling back to time-overlap; _is_tool_span/_is_llm_span read the typed kind field before falling back to operation_name string-prefix matching. 4. A smoke test seeds \u003e=1 real-shaped otlp_spans row and exercises analyze correlation --otlp end to end without OperationalError.","status":"open","priority":2,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-31T04:32:29Z","created_by":"Sinity","updated_at":"2026-07-31T04:32:29Z","labels":["area:daemon","area:insights"],"dependencies":[{"issue_id":"polylogue-zc4a","depends_on_id":"polylogue-pbuh","type":"related","created_at":"2026-07-31T06:32:28Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-vp9d","title":"Isolate the exact hang inside browser-capture catch-up chunk ingest (2026-07-31 livelock)","description":"Live incident 2026-07-31: polylogued (pid 1629493, up since 2026-07-30 20:35)\nlivelocked for \u003e30 minutes. Evidence trail (journalctl):\n\n- 05:12:51 \"live.watcher: catch-up ingesting 6 file(s) (709.3 MB), skipped=2,\n chunks=4\" then \"catch-up chunk 1/4 ingesting 1 file(s) (0.0 MB)\" -- and\n NOTHING further from that chunk, ever. Chunks 2-4 never started. No\n \"daemon writer released\" event logged again until the daemon was\n restarted at 05:55, ~43 minutes later -- meaning the writer-coordinated\n `ingest_chunk` closure for that single small file (almost certainly the\n 84KB /realm/db/polylogue/browser-capture/chatgpt/*-c167a4a267f0.json\n capture dated 05:03, the only sub-0.1MB candidate in the scan) either hung\n inside the parse/ingest call or never released the writer gate.\n- Downstream effect: `_periodic_raw_materialization_convergence`'s\n `_browser_capture_spool_has_pending_files()` check kept returning True\n (no ingest_cursor row exists for either /realm/db/polylogue/browser-capture\n file -- confirmed via `SELECT count(*) FROM ingest_cursor WHERE\n source_path LIKE '/realm/db/polylogue/browser-capture%'` = 0), so raw\n materialization yielded every tick (\"yielding to pending browser-capture\n spool files\") and the operator's Claude export zip sat undrained in\n /realm/db/polylogue/inbox/ since 00:02.\n- Restarting polylogued (systemctl --user restart polylogued.service)\n unstuck it immediately: the next catch-up pass completed chunk 1/7 in \u003c1s\n and progressed normally (confirmed via journalctl through 06:15).\n\nWhat was NOT established: the exact internal stack frame the hung task was\nblocked in. py-spy dump against the live pid failed (\"Failed to find python\nversion from target process\" -- likely a python3.14 free-threaded build\npy-spy 0.4.2 doesn't parse correctly) so no live stack trace was captured\nbefore the mitigating restart. DaemonWriteCoordinator.run() was checked and\ncorrectly handles same-task reentrant `run()` calls (returns\n`await operation()` directly via the `_ACTIVE_LEASE` contextvar), which\nrules out the obvious nested-writer-deadlock hypothesis\n(`_ingest_files` calling `self._write_coordinator.run(...)` again from\ninside the already-coordinated `ingest_chunk` closure) -- so the hang is\nmost likely inside the actual parse/ingest of that specific 84KB ChatGPT\nbrowser-capture envelope, not a coordinator bug.\n\nFollow-up: get a working py-spy/faulthandler setup for this python build (or\nadd a bounded per-chunk ingest timeout as defense-in-depth regardless), then\neither reproduce against that exact retained capture file or wait for a\nrecurrence and capture a stack trace before restarting.","notes":"Filed alongside PR https://github.com/Sinity/polylogue/pull/3418; live mitigation (daemon restart) already applied 2026-07-31 05:55.","status":"open","priority":2,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-31T04:16:39Z","created_by":"Sinity","updated_at":"2026-07-31T04:22:29Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-j8yo","title":"AI Studio browser-capture adapter: SKIP — live transport is undocumented internal RPC, not the Drive JSON","description":"Investigation for the hypothesis: \"aistudio.google.com fetches essentially the\nsame underlying JSON that ends up on Drive, so a browser-capture adapter\nwould be cheap.\" Verdict: SKIP for now. Evidence below.\n\n## Method\n\nLive authenticated browser (sinnix-chrome-control --target live), read-only:\nopened aistudio.google.com, listed the prompt library, navigated to two\ndistinct existing prompts, captured CDP Network domain traffic (not\npage-level fetch/XHR hooks or Resource Timing -- per the operator's prior\nfinding on Claude Design, those miss RPC transports; CDP Network did not).\n\n## (a) What the live app actually fetches\n\nPrompt URLs are literally Drive file ids\n(`aistudio.google.com/prompts/1cVKebxYa9oOCM05J3BqQQizFzIgfjvyH` etc. --\n`1...` prefix is the standard Drive file-id shape). Opening a prompt does\nNOT issue a plain REST GET, and does NOT call `drive.googleapis.com`\ndirectly. All application data comes from one internal RPC service:\n\n POST https://alkalimakersuite-pa.clients6.google.com/$rpc/google.internal.alkali.applications.makersuite.v1.MakerSuiteService/\u003cMethod\u003e\n\nMethods observed opening two different prompts: GetLoggingContext,\nGetUserPreferences, GenerateAccessToken, ListPromos, GetAiStudioBenefitTier,\nListModels, ListRecentApplets, ListPrompts, ResolveDriveResource. Response\nContent-Type for all of them: `application/json+protobuf` (confirmed via\nCDP response headers on ResolveDriveResource, status 200).\n\n`application/json+protobuf` is Google's internal positional-array RPC\nframing (same family used by Photos/Keep/Docs-style Closure apps): the body\nis syntactically valid JSON but semantically an array of proto field values\nkeyed by field NUMBER, not name -- there is no published `.proto` for\n`google.internal.alkali.applications.makersuite.v1.MakerSuiteService`, so\nturning it into named fields means reverse-engineering positional mappings\nper RPC method, with no compatibility guarantee across Google's backend\ndeploys. This is the same failure class the operator's Claude Design\ncomparison hit (page-level hooks missed it; the format itself is\nundocumented and fragile), not a REST/JSON API.\n\n## (b) Does it match the Drive-synced shape?\n\nNot directly. `ResolveDriveResource` is the RPC that resolves a prompt id to\nits Drive-resident content, but it goes through MakerSuiteService's own\nproxy/serialization, not a client-visible `drive.googleapis.com` files.get.\nThe *canonical stored artifact* is the same Drive file the operator's Drive\nsync already downloads (both ultimately reference the identical Drive\nobject), but the *live wire representation* is not the plain object-keyed\nJSON polylogue already parses (`polylogue/sources/parsers/drive.py`) -- it\nis the positional `application/json+protobuf` RPC envelope. So the premise\n\"same JSON, cheap adapter\" is false at the transport level even though the\nunderlying data is the same document.\n\nNo content-bearing RPC beyond ResolveDriveResource was captured in two\n~20s windows across two different prompts; either the message content\nrides inside that same RPC's payload (plausible -- one full prompt fetch\nper open) or a further call wasn't triggered in the capture window. Either\nway nothing suggests a second, cleaner JSON transport exists alongside it.\n\n## (c) What would a live adapter gain that Drive cannot?\n\nChecked against the two most-cited justifications and found both already\nsatisfied by the Drive-synced file itself:\n\n- **Drafts/unsaved runs**: `chunkedPrompt.pendingInputs` -- the exact\n not-yet-submitted textbox content -- IS present in the Drive-synced JSON.\n Verified against the live archive: 396/397 aistudio-drive raw sessions\n carry a `pendingInputs` entry, 7 with non-blank draft text (one a full\n multi-paragraph prompt that was never sent). Just parsed and landed as a\n `draft_input` session_event (polylogue-o4j2, PR pending). Drive sync\n already captures this; live capture would not add draft coverage.\n- **Generation params absent from the synced file**: none found. runSettings\n (temperature/topP/topK/maxOutputTokens/thinkingLevel/safetySettings/\n enable* flags) is present verbatim in the Drive-synced JSON and already\n reaches `sessions.run_settings_json` (polylogue-2qx.4/cgfy, index v46,\n PR #3390, predating this investigation).\n\nRemaining plausible (unverified, not measured this session) gains:\n- **Realtime vs Drive-sync polling lag**: real, but modest -- Drive sync\n latency is not the archive's current bottleneck for this origin (397\n sessions total, low volume).\n- **Sessions later deleted from Drive/AI Studio**: real edge case, same\n argument applies to every delete-capable source; not AI-Studio-specific.\n- **Removing the separate Drive OAuth flow**: real operational simplification\n (one fewer auth surface) but orthogonal to data completeness.\n\n## (d) Cost\n\nBuilding a live adapter would mean either (i) reverse-engineering\n`application/json+protobuf` positional RPC payloads for\n`MakerSuiteService` with no published schema and no stability guarantee, or\n(ii) falling back to DOM scraping of the rendered chat UI (the pattern the\nexisting ChatGPT/Claude browser-capture adapters already use) -- itself a\nreal, non-trivial adapter (selectors, pagination, run-settings-panel\nscraping, draft-textbox capture) comparable in cost to any other\nbrowser-capture origin, not a cheap win from shape-reuse.\n\n## Recommendation: SKIP\n\nThe \"cheap because same JSON\" premise does not hold: the live transport is\nan undocumented internal RPC (protobuf-JSON hybrid), not the archive's\nalready-parsed Drive JSON shape. The two headline capabilities a live\nadapter was hoped to add -- drafts and generation params -- are already\npresent in the Drive-synced file and now parsed (o4j2). What remains\n(latency, one fewer OAuth flow, delete-survivorship) does not clear the bar\nof reverse-engineering an undocumented Google-internal RPC surface, or\nbuilding a from-scratch DOM-scrape adapter at ordinary browser-capture cost.\nRevisit only if the operator specifically wants realtime AI Studio capture\nregardless of cost, or if a documented public transport for AI Studio\nappears.\n\nRead-only investigation; no AI Studio content was created, edited, or\ndeleted. Evidence lives in this issue only (raw archive blob paths quoted,\nnot copied) to avoid persisting the operator's personal draft-prompt content\ninto tracked repo files.","status":"open","priority":2,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-31T04:00:47Z","created_by":"Sinity","updated_at":"2026-07-31T04:00:47Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-knc7","title":"model claude subscription session and weekly credit windows","description":"polylogue/cost/plans.py models only a MONTHLY quota (SubscriptionPlan has quota, quota_basis, billing_cycle_days, cycle_anchor_day). It has NO session-window or weekly field. The two limits that actually bind in practice are therefore unmodelled.\n\nPublished figures (she-llac.com/claude-limits, dated 2026-01-25):\n plan 5-hour session weekly\n Pro 550,000 5,000,000\n Max 5x 3,300,000 41,666,700\n Max 20x 11,000,000 83,333,300\n\nOur seeded monthly quotas (21.7M / 180.6M / 361.1M) match that source exactly, as do the per-model credit rates in archive/semantic/subscription_pricing.py (opus 10/50, sonnet 6/30, haiku 2/10, cache_read 0, cache_write at the input rate). So the rate model is right; the WINDOW model is missing.\n\nWhy it matters: monthly quota is almost never the binding constraint - you get rate-limited by the 5-hour window mid-session. Today polylogue can say what a session cost in credits but not whether it would have exhausted a window, which is the operationally useful question.\n\nNote the weekly limits are NOT monthly/4 and are not derivable: Pro 5M x4 = 20M against a 21.7M month, but Max 20x 83.3M x4 = 333M against 361.1M. The ratio differs per tier, so both numbers must be carried explicitly.\n\nAC: SubscriptionPlan carries session-window and weekly quotas with their window lengths; a session can be evaluated against them; and a query can answer 'did this session approach a window limit'.","status":"open","priority":2,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-31T03:56:51Z","created_by":"Sinity","updated_at":"2026-07-31T03:56:51Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-t83q","title":"subscription credit rates are missing the Claude 5 model family","description":"polylogue/archive/semantic/subscription_pricing.py declares ModelCreditRate rows for claude-opus-4-6, claude-opus-4-5, claude-sonnet-4-6, claude-sonnet-4-5, claude-haiku-4-5. The Claude 5 family (claude-opus-5, claude-sonnet-5) is ABSENT, and those are the current models - this very session runs on Opus 5.\n\ndocs/cost-model.md states credits are emitted only for models with a DECLARED rate, 'never a fabricated figure'. That is the correct failure mode, but the consequence is that current-model sessions silently produce no subscription_credit_usd at all, so credit accounting has a growing blind spot exactly where usage is concentrated.\n\nDo NOT simply copy the 4.x rates forward - whether Opus 5 inherits 10/50 and Sonnet 5 inherits 6/30 is an assumption, not a verified fact. Source the real rates before declaring them, and if they cannot be sourced, record that explicitly rather than guessing.\n\nRelated staleness: CURATED_SEED_EFFECTIVE_DATE is 2026-05-17 and the upstream reference (she-llac.com/claude-limits) was published 2026-01-25 with no update date, six months stale as of 2026-07-31. Its own wording ('actual multiplier: 6-8.33x') signals reverse-engineering rather than published spec. Neither figure is verifiable from local data: session JSONL carries cost_usd (API-list-equivalent) and token counts, never subscription credits, so there is no ground truth on disk to test the formula against. Treat the whole credit model as best-effort inference and label it as such wherever it surfaces.\n\nAC: Claude 5 rates present with a sourced provenance note, or an explicit recorded statement that they are unavailable; and a check that flags when a model appearing in the archive has no declared credit rate.","status":"open","priority":2,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-31T03:56:51Z","created_by":"Sinity","updated_at":"2026-07-31T03:56:51Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-54gj","title":"Grok-on-X (x.com/twitter.com) has no capture path after DOM removal","description":"The Grok native-capture upgrade (grok.com REST adapter + grok_bridge.js) deleted the old grok-dom-v1 fallback and dropped x.com/twitter.com from manifest.json's content_scripts and background.js's injectionPlanForUrl, because grok.com's /rest/app-chat/* REST surface this bridge calls is same-site to grok.com only -- X's embedded Grok surface is served through X's own API (not verified live in that session; no authenticated x.com Grok conversation tab was available). background.js's archiveProviderForUrl/conversationIdForUrl and popup.js's provider labeling still classify x.com/twitter.com as the 'grok' provider and show a 'Grok / X' label, but no content script is installed there anymore, so any auto-capture trigger targeting those tabs now silently finds no listener (captureTab's injectionPlanForUrl(...).length guard already short-circuits it cleanly -- no hang, no error -- but the UI label is misleading).\n\nFollow-up scope:\n1. Verify live whether x.com's embedded Grok assistant actually has its own distinct GraphQL/REST API, and if so build a dedicated adapter for it (same shape as GrokBackfillAdapter/grok.js, different origin/endpoints).\n2. If not pursued, drop x.com/twitter.com from archiveProviderForUrl/popup provider labeling and host_permissions so the UI stops claiming a capture path that doesn't exist.","status":"open","priority":2,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-31T03:34:11Z","created_by":"Sinity","updated_at":"2026-07-31T03:34:11Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-u8x7","title":"Extend field-path union coalescing to web_content_constructs/file_edits and session_model_usage","description":"Follow-up to polylogue-geop (field-path union coalescing for provider exports,\nimplemented in messages/blocks via _union_with_existing_rows in\npolylogue/storage/sqlite/archive_tiers/write.py).\n\nScope deliberately deferred from the initial implementation:\n\n1. web_content_constructs and file_edits are derived sidecar tables\n populated ONLY from the current acquisition's parsed ParsedMessage/\n ParsedContentBlock domain objects (_write_web_constructs/_write_file_edits\n in write.py), not from the merged/unioned row tuples. When a message is\n reinjected by the field-path union because a newer acquisition dropped it\n entirely, its web_content_constructs/file_edits rows are NOT restored\n (they were deleted by the session-scoped replace and nothing repopulates\n them, since the domain object carrying that data no longer exists in\n this write's `messages` list). This directly affects the bead's own\n measured scenario: metadata.content_references citations map onto\n web_content_constructs.\n\n2. session_events / session_model_usage: the union operates only on\n messages/blocks. A reinjected message's model_name produces a\n zero-usage session_model_usage skeleton row (see\n test_provider_usage_model_vanishing_on_reingest_preserves_message_with_zero_usage_rollup)\n because token_count session_events aren't unioned across acquisitions.\n Consider extending the same union principle to session_events keyed by a\n stable native event id, if one exists per provider.\n\nBoth would need the same \"read existing rows before delete, reinject/merge,\nskip for prefix-sharing lineage parents\" pattern already established in\n_union_with_existing_rows, extended per-table.","notes":"2026-07-31 update: the initial polylogue-geop implementation applied field-path\nunion unconditionally to every full-replace, which broke ~19 tests (browser-\ncapture/native-vs-DOM-fallback precedence, same-acquisition re-parse\nretraction). Fixed by gating union on a raw_id-based discriminator: union only\nfires when the incoming and previously-stored sessions.raw_id are both known\nand differ (proven different acquisition), or is skipped when they're equal\n(same acquisition re-parsed), either is unknown, or the caller passed\nforce_replace=True (an explicit precedence decision, e.g.\nbrowser_capture_precedence()). See _union_with_existing_rows in\npolylogue/storage/sqlite/archive_tiers/write.py.\n\nThis directly affects this follow-up's scope: extending union to\nweb_content_constructs/file_edits/session_events must respect the SAME\nraw_id/force_replace discriminator, not just the message/block matching\nlogic -- otherwise the same class of regression (same-acquisition re-parse\nunable to retract a stale citation/file-edit/usage row) would recur there.","status":"open","priority":2,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-31T03:19:09Z","created_by":"Sinity","updated_at":"2026-07-31T03:51:04Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-tbun","title":"model claude design as a distinct origin, with webui representation","description":"MEASURED over the 11 design_chats in claude-ai-data-2026-07-30. Claude Design is NOT claude.ai with a flag - it is a different product with a different wire format, currently reduced to a CLAUDE_DESIGN_CHAT_INGEST_FLAG on a claude-ai-export session.\n\nWIRE SHAPE (all camelCase, vs claude.ai's snake_case - different backend):\n message: uuid, role, content, created_at where content is a DICT not a list\n content: role, content, id, timestamp, contentBlocks, authorAccountUuid,\n authorName, attachments, turnInputTokens, pill, turnChanges\n\nIT IS AN AGENTIC ENVIRONMENT, NOT A CHAT. Across 11 chats: 751 tool_call\nblocks, 211 thinking, 175 text, 10 error, 5 user_interjection. Tools used:\n write_file 146, snip 121, read_file 120, update_todos 50, str_replace_edit 48,\n github_read_file 31, done 29, github_get_tree 26, fork_verifier_agent 25,\n list_files 19, save_screenshot 16, local_read 15, grep 14, web_fetch 11\ntoolCall records carry id/type/name/input/output with toolu_* ids - the SAME id\nspace as the Claude API, so tool identity joins cleanly with claude-code.\n\nCONSTRUCTS WITH NO CLAUDE.AI EQUIVALENT:\n turnChanges {created, edited, deleted, moved} - a materialised filesystem\n diff PER TURN. Highest-value part; nothing else in the archive\n records what a turn changed on disk.\n user_interjection - a user message nested INSIDE an assistant turn. Flattening\n it to an ordinary user message destroys both the interruption\n semantics and the ordering.\n attachments typed file(143) skill(21) text(19) image(17) folder(2) - skills\n and folders as attachable objects.\n authorAccountUuid + authorName - named multi-account authorship; claude.ai\n exports have no author identity at all.\n turnInputTokens - per-turn token accounting.\n error blocks - refusals as first-class content.\n\nPROVIDER QUIRKS: every title is literally 'Chat' (titles must be derived, same\nclass as the claude-code raw-UUID title problem); content is a dict not a list,\nso a parser assuming the claude.ai shape fails immediately.\n\nWORK:\n1. Origin.CLAUDE_DESIGN_SESSION as a new token; retire\n CLAUDE_DESIGN_CHAT_INGEST_FLAG in the same change (hard rename, no compat).\n2. tool_call -\u003e TOOL_USE/TOOL_RESULT with shared tool_id (records already carry\n both sides). thinking -\u003e THINKING. text -\u003e TEXT. error -\u003e error block.\n3. turnChanges -\u003e per-turn session_event or a new construct type. Decide which.\n4. user_interjection -\u003e needs a real answer, not a flatten.\n5. attachment taxonomy gains skill and folder.\n6. Both acquisition paths: GDPR import AND browser-extension capture, coalescing\n on message uuid at field-path granularity (see the strict-containment bead) -\n design chats are the ideal first case since both sources will cover the same\n sessions.\n\nWEBUI (polylogue/daemon/webui.py, 1,638 lines, 59 functions): a design session\nrenders poorly as a chat transcript - it is 751 tool calls and 5 file mutations\nacross 11 sessions. It needs a session view that leads with turnChanges (what\nthis turn changed), folds tool calls by default like the reader already folds\ntool_use, and shows user_interjection inline at its true position rather than as\na sibling message. Scope note: the corpus is only 11 chats and the product is\nnew, so the parser should be strict about what it recognises and loud about what\nit does not, rather than guessing a shape that is still moving.","notes":"LIVE TRANSPORT DISCOVERED 2026-07-31 via CDP Network domain (page-level fetch hooks and resource-timing both showed nothing - this is why).\n\nClaude Design does NOT use a REST /api/ route. /api/organizations/\u003corg\u003e/design_chats 404s on every org. It uses a Connect-RPC service:\n\n POST https://claude.ai/design/anthropic.omelette.api.v1alpha.OmeletteService/\u003cMethod\u003e\n\nMethods observed on a project load (counts from one trace):\n GetFile x6, TrackEvent x4, ListFiles x2, UpdateProjectData, MintPreviewToken,\n McpStreamTools, McpListDesignImportPartners, ListUserSkills, ListOrgProjects,\n ListExperiences, ListComments, GoogleGetStatus, GithubGetStatus,\n GetUserSettings, GetUsageStatus, GetProjectPresence, GetProject,\n GetPrepaidBalance, GetOrgSettings\n\nCRITICAL FOR IMPLEMENTATION: responses are content-type **application/proto**\n(binary protobuf), not JSON. McpStreamTools is application/connect+proto\n(streaming). Only GetProjectPresence returned application/json.\n\nSo a live capture adapter CANNOT parse the wire the way the chatgpt/claude\nadapters do - there is no published .proto schema. Two viable directions:\n (a) hook the app's own DECODED objects in page context (MAIN world), after\n the Connect client has deserialised, rather than intercepting the wire;\n (b) reverse the protobuf shape per method, which is brittle and would break\n on any schema change.\n(a) is strongly preferred and matches how chatgpt_bridge.js already works\n(intercepting window.fetch and reading decoded JSON).\n\nAlso confirmed: design files render inside a SANDBOXED CROSS-ORIGIN IFRAME at\nhttps://\u003cproject-uuid\u003e.claudeusercontent.com/_bootstrap (subdomain IS the\nproject uuid), sandbox='allow-scripts allow-forms allow-popups allow-modals\nallow-downloads allow-same-origin'. Same pattern as artifacts. host_permissions\nfor https://*.claudeusercontent.com/* has now been added to manifest.json AND\nto scripts/validate-manifest.mjs's ALLOWED_HOST_GLOBS (the validator correctly\nrejected it until declared).\n\nNo GetChat/ListMessages method was observed, so the design conversation itself\nlikely arrives via GetProject, ListExperiences, or a stream - needs one more\ntrace with the project's chat pane actually loading.","status":"open","priority":2,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-31T01:43:06Z","created_by":"Sinity","updated_at":"2026-07-31T03:04:37Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-xofj","title":"handle the six unmodelled chatgpt content types from the April-era format","description":"MEASURED in chatgpt-data-2026-04-23. parsers/chatgpt.py explicitly handles code, execution_output, thoughts, reasoning_recap, audio_transcription, user_editable_context/model_editable_context, image_asset_pointer and the audio pointer set - and recognises the tool role. These six are NOT handled and fall through to a generic TEXT block, so the content survives but the semantic type is lost:\n\n computer_output 8,192\n tether_browsing_display 1,399\n tether_quote 1,178\n system_error 177\n sonic_webpage 30\n citable_code_output 8\n\nThey are all code-interpreter / browsing-era constructs. computer_output is a\ntool result (pairs with the same tool_id logic execution_output already uses);\ntether_quote and tether_browsing_display are retrieved-source constructs and\nshould become web constructs, not text; system_error is an error block;\ncitable_code_output is a code result with citation anchors.\n\nThese only ever appear in the April-and-earlier format - the July 2026 export\ndeleted the whole tool layer (see the strict-containment bead) - so this is\nhistorical-format support. We want it anyway: the April export is the sole\nsurviving record of that layer.\n\nAC: each of the six maps to a typed block or web construct rather than TEXT;\na re-import of the April export shows the new typed rows; and the mapping is\ncovered by a parser test using a real (anonymised) node of each shape.","notes":"Implemented in PR #3408 (branch feature/parsers/chatgpt-april-content-types-and-web-constructs). All six content types (computer_output, tether_browsing_display, tether_quote, system_error, sonic_webpage, citable_code_output) now map to typed blocks/constructs in polylogue/sources/parsers/chatgpt.py, each covered by a parser test using an anonymized real-node shape. Not yet merged/deployed -- re-import of the April export against the live archive still pending until PR lands.","status":"open","priority":2,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-31T01:42:36Z","created_by":"Sinity","updated_at":"2026-07-31T03:20:03Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-zocm","title":"both parsers under-populate the web-construct vocabulary","description":"VERIFIED 2026-07-31 by counting WebConstructType references per parser.\n\n chatgpt.py emits 9 types: SEARCH_QUERY, CONTENT_REFERENCE, ASYNC_TASK,\n SELECTED_SOURCE, SEARCH_RESULT, IMAGE_RESULT, CANVAS, AUDIO_TRANSCRIPTION,\n AUDIO_ASSET\n claude/*.py emits 2 types: CANVAS, CONTENT_REFERENCE\n\nSo the vocabulary is right and provider-neutral; the population is wrong, in\ntwo different ways.\n\nGAP 1 - chatgpt loses 60.8% of citation URLs. _construct_from_reference\ndescends into item.metadata and item.metadata.extra but NOT into item.items /\nitem.fallback_items, which is where grouped_webpages keeps its URLs.\nMeasured over the July export:\n 6,596 content_references[].url EXTRACTED\n 10,130 content_references[].items[] NOT extracted\n 116 content_references[].fallback_items[] NOT extracted\n -\u003e 10,246 / 16,842 URLs (60.8%) never become constructs.\nThe search_result_groups loop already does exactly this descent\n(group.results/items/search_results/sources) - content_references needs the\nsame treatment.\n\nGAP 2 - claude does not distinguish retrieved from cited. Claude's export\ncarries BOTH layers and they are semantically distinct:\n 326 anchored citations on text blocks\n {uuid, start_index, end_index, details:{type:web_search_citation,url}}\n -\u003e these are CITED, with a character span into the answer text\n 1,514 URLs inside web_search tool_result content\n {type:knowledge, title, url, metadata:{site_domain, site_name,...}}\n -\u003e these are RETRIEVED, never necessarily cited\nOnly the first becomes a CONTENT_REFERENCE; the retrieved set stays buried in\ntool_result text and never becomes SEARCH_RESULT constructs.\n\nGetting this wrong in the obvious direction would make ChatGPT look like it\ncites 25x more than Claude when it mostly just reads more. CONTENT_REFERENCE\nshould mean cited-with-span; SEARCH_RESULT should mean retrieved.\n\nAC: chatgpt nested citation items become constructs; claude web_search results\nbecome SEARCH_RESULT constructs; a query can distinguish 'sources cited' from\n'sources read' for both providers.","notes":"Implemented in PR #3408 (branch feature/parsers/chatgpt-april-content-types-and-web-constructs). GAP 1 (chatgpt): content_references/citations now descend into item.items[]/item.fallback_items[] via _constructs_from_content_reference_item, mirroring the existing search_result_groups descent. GAP 2 (claude): content_blocks_from_segments (base_support.py, shared by codex/claude) projects web_search tool_result {type:knowledge} entries as SEARCH_RESULT constructs, kept distinct from the existing CONTENT_REFERENCE citation-anchor projection in claude/common.py so cited vs retrieved sources stay separately queryable. Not yet merged.","status":"open","priority":2,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-31T01:42:36Z","created_by":"Sinity","updated_at":"2026-07-31T03:20:15Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-dt5s","title":"capture model-produced sandbox files as first-class references","description":"MEASURED 2026-07-31 against the 2026-07-29 chatgpt export.\n\nThe model writes files into its sandbox and links them as sandbox:/mnt/data/\u003cname\u003e. These are a DISTINCT population from user uploads and are currently invisible to polylogue.\n\nScale: 639 assistant messages carry such links; 1,782 distinct output filenames.\nExtensions: md 1025, csv 299, json 298, zip 193, png 178, patch 159, txt 113,\ngz 78, jsonl 75, py 61, sh 55, yaml 54.\n\nTHE KEY CONSTRAINT: a sandbox link carries NO file id. The assistant message\nmetadata on those 639 messages contains only model_slug/parent_id/content_references\n- no attachment record, no asset_pointer, no file id. So there is no id join.\n\nBYTE AVAILABILITY (name-match is the only join available):\n 1,782 distinct sandbox output names\n 823 match a library_files.file_name\n 40 match a content_references name\n 27 match a conversation_asset_file_names value\n 826 resolvable by ANY of the three (46.4%)\n 956 have NO byte source anywhere (53.6%)\n\nSo roughly half the model-produced files are recoverable, and only via filename -\nwhich is fuzzy and can collide. Treat a name match as EVIDENCE, not identity:\nrecord how the link was resolved so a wrong match is auditable, and never let a\nname match mint the same identity as an id match.\n\nFOR THE OTHER 956: capture metadata anyway - filename, sandbox path, extension,\nproducing message id, conversation, timestamp - as a model-produced-file\nreference with no bytes. Operator directive: better a recorded absence with\nmetadata than silence. This also makes the population countable, so a future\nexport or browser capture that DOES carry the bytes can be joined to it.\n\nRELATED CORRECTION: message.metadata.attachments[] (3,444 ids) are ALL on user\nmessages - they are uploads, not model output. Do not conflate the two.","notes":"CORRECTION + REAL SPEC (2026-07-31). The earlier 'only fuzzy filename matching, 46%' was wrong. I had truncated the library_files key list to the first 9 keys and concluded from what I could see. The full schema carries an EXACT producing-message id.\n\nRelevant library_files fields (2,367 entries):\n origination_message_id 1,742 \u003c- exact id of the assistant message that produced the file\n origination_thread_id 1,733 \u003c- conversation\n sha256_digest 733 \u003c- content addressing / dedup\n library_artifact_type 953 (other 840, report 43, image 36, image_gen 14,\n writing_block 11, deep_research_report 7, sheet 2)\n initiating_conversation_id 0 (always null - do not use)\n file_name_provenance: 'upload' for ALL 2,367, so it does NOT distinguish\n model-produced from uploaded. Provenance comes from origination_message_id\n being set, not from this field.\n\nTIERED RESOLUTION, measured over all 2,943 (message, sandbox-filename) links:\n\n tier links with bytes\n 1 exact msg id + name match 1,412 1,319\n 2 exact msg id, name differs 418 418\n 3 thread id + name 2 2\n 4 global name, provably UNIQUE 91 91\n 5 global name, AMBIGUOUS 0 0 \u003c- none exist\n 6 unresolved, metadata only 1,020 0\n\n identity-grade (1-3) with bytes: 1,739 = 59.1% of all links\n zero genuinely ambiguous name matches in the entire corpus\n\nSo the implementation is a layered resolver, not a fuzzy matcher:\n 1. join on origination_message_id (identity-grade; note tier 2 - the library\n name can differ from the linked name, so match on the id ALONE and treat\n the name as a label, not a key)\n 2. fall back to (origination_thread_id, file_name)\n 3. fall back to a global name match ONLY when it is provably unique\n 4. otherwise record a metadata-only model-produced-file reference\n\nRecord which tier resolved each link so a later audit can distinguish an id\njoin from a name join. Tier 4 should be marked as evidence rather than\nidentity, but the collision risk that motivated that caution does not\nmaterialise here (tier 5 is empty).\n\nUse sha256_digest where present for content-addressed dedup against blobs\nalready stored from other sources.\nIMPLEMENTED (branch feature/sources/chatgpt-export-assets-and-sidecars, PR pending).\n\nImplemented the exact 6-tier resolver from the corrected spec:\nChatGPTAssetIndex.resolve_sandbox in polylogue/sources/parsers/chatgpt_sidecars.py.\nTier 1 (msg id + name exact), tier 2 (msg id only -- name is a label, not a\nkey, per spec), tier 3 (thread id + name), tier 4 (globally unique name),\ntier 5 (globally ambiguous name -- evidence, no file), tier 6 (unresolved,\nmetadata-only). Wired into chatgpt_assembly.py's enrich_session: for tiers\n1-4, attachment.provider_file_id is updated to the matched library file_id\n(real identity strengthening); every tier, including 6, gets a\nchatgpt_sandbox_file_resolution session_event recording which tier resolved\nit (audit trail per the spec's directive).\n\nRe-measured resolver behavior against the real corpus (all 29\nconversations-*.json shards + library_files.json): tiers\n{1: 1273, 2: 370, 3: 2, 4: 83, 6: 995} over 2,723 links found by my\nverification harness (some magnitude difference from the bead's own 2,943\ncount is expected -- my harness only scanned \"parts\"-shaped assistant text\nfor sandbox links as a sanity check; the actual production\n_sandbox_file_paths/_extract_content_text already covers more content\nshapes). Tier 3 count (2) matches exactly. Zero tier-5 ambiguous matches,\nmatching the spec's claim that no genuine collision exists in this corpus.\n\nBytes for the resolved fraction: still not acquired (see polylogue-8ac0,\nfiled as the shared byte-acquisition follow-up for both this bead and\npolylogue-0hwv -- the .dat blob itself needs the same streaming-ZIP-scan\nwork regardless of which resolver named it). Tier 6 (the ~35% with no id/\nname evidence at all) already gets the \"recorded absence with metadata\"\ntreatment the bead asked for: filename, sandbox path, extension (via name),\nproducing message id, and tier=6/method=unresolved on the session_event --\nno bytes were ever going to be available for this population regardless of\nthe acquisition follow-up.\n\nCORRECTION to my previous note's tier-count claim (2026-07-31, caught by\ncoordinator review before merge): I wrote the re-measured tiers were\n\"consistent with the spec\"; they were NOT identical, and I had not run the\nreconciliation needed to say why before making that claim.\n\nRoot cause, now confirmed exactly: this bead's measured spec counted every\nraw sandbox-link OCCURRENCE (regex match on assistant text). Reproducing\nthat exact counting method against the real corpus gives\n{1: 1412, 2: 418, 3: 2, 4: 91, 5: 0, 6: 1020} sum 2943 -- bit-for-bit\nidentical to the spec in every tier. But the PR's actual production\nattachments are built by chatgpt.py's pre-existing _sandbox_file_paths()\n(not touched by this PR), which deduplicates repeated identical sandbox\nlinks WITHIN one message's text before any attachment is constructed -- a\nmessage that links the same file twice yields one ParsedAttachment, not\ntwo. Counting production attachments (the honest apples-to-apples number\nfor what actually lands in the archive) gives\n{1: 1273, 2: 370, 3: 2, 4: 83, 6: 995} sum 2723 (-7.5% overall, every\npopulated tier down by roughly the same proportion). This is a denominator\ndifference (occurrences vs. distinct (message,filename) pairs), not a\nresolver disagreement, and it is the CORRECT product behavior (no duplicate\nattachment rows for a repeated identical link) -- but the two counts are\nnot interchangeable and I should not have called them consistent without\ndoing this reconciliation first.\n\nWhat DOES hold exactly, in both countings, and is the structurally\nload-bearing result: tier 5 (globally-ambiguous name) is ZERO -- no\nfuzzy-match collision exists anywhere in the corpus -- and tier 3 is 2.\nThose are what actually validate the tiered-resolver design over a flat\nfuzzy matcher; the rest is denominator noise from a pre-existing\ndeduplication step this PR did not introduce and did not need to change.\n","status":"closed","priority":2,"issue_type":"task","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-07-31T00:57:54Z","created_by":"Sinity","updated_at":"2026-07-31T03:55:39Z","started_at":"2026-07-31T03:32:13Z","closed_at":"2026-07-31T03:55:39Z","close_reason":"Merged in PR #3409 (polylogue/master@11403388d): 6-tier sandbox-file resolver implemented exactly per spec, tier recorded per link via session_events, tier-6 unresolved links still get a metadata-only reference. Tier 5 (ambiguous) confirmed zero, tier 3 confirmed 2, both exact matches to the measured spec.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-80ks","title":"audit browser-capture attachment parity against GDPR export fidelity","description":"Open question raised 2026-07-31: does browser-extension capture handle attachments/images/audio as well as the GDPR export path does?\n\nPartial evidence (not a full audit): browser_capture/models.py has mime_type + extracted_content; parsers/browser_capture.py builds inline_bytes via _browser_capture_attachment_inline_bytes and merges them across candidates, with upload_origin url|paste|oauth. So TEXT extraction and pasted bytes are modelled.\n\nUnverified: whether binary image/audio bytes are captured at all from the live DOM, or only a URL + extracted text; and whether an asset captured live and later re-delivered by a GDPR export coalesces to one attachment or duplicates.\n\nThis matters more now that exports ship real bytes (see polylogue-0hwv): the two paths could disagree about what an attachment IS, which is the aggz-invariant-2 shape (two write paths, one forgets).\n\nAC: a per-modality matrix (text / image / audio / model-produced file) x (browser capture / GDPR export) stating what is stored for each, with the gaps either fixed or recorded as deliberate.","status":"open","priority":2,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-31T00:50:13Z","created_by":"Sinity","updated_at":"2026-07-31T00:50:13Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-2m2e","title":"chatgpt export sidecars library_files.json and codex.json are unparsed","description":"The 2026-07-29 chatgpt export contains sidecars polylogue does not reference at all (verified by rg over polylogue/):\n\n- library_files.json - 2,367 entries, the ChatGPT Library (generated/uploaded file collection) with sha256 digests, context scopes, versions\n- codex.json - 20 Codex threads with a 'turns' structure, i.e. cloud-Codex sessions delivered through the chatgpt export rather than ~/.codex\n\nshared_conversations.json (154) IS referenced in dispatch.py. message_feedback.json (21 ratings) and ads.json (empty) are low value.\n\ncodex.json is the interesting one: it is a second, independent delivery path for Codex sessions, so it risks either absence or duplicate identity against codex-session origin records.\n\nAC: decide per sidecar - parsed, or explicitly out of scope with the reason recorded. For codex.json specifically, determine whether its threads coalesce with existing codex-session sessions or create duplicates.","notes":"IMPLEMENTED (branch feature/sources/chatgpt-export-assets-and-sidecars, PR pending).\n\nPer-sidecar decision, as the AC asked:\n\n- library_files.json: PARSED. Feeds ChatGPTAssetIndex (polylogue-0hwv/\n polylogue-dt5s resolvers) as the primary (richer) name/mime/size/sha256/\n origination-id source. Deferred, not silently dropped: the sub-population\n of library files with NO origination_message_id/thread_id AND never\n referenced by any conversation attachment or sandbox link (measured\n ~1,438 in the bead's own notes) is not yet surfaced as a first-class\n standalone reference -- that needs whole-source-scan aggregation\n (tracking every file_id actually consulted across all sessions from one\n source, then diffing against the full library_files population) that\n the current per-session enrich_session hook doesn't have a natural home\n for. Left as an explicit gap rather than building a half-working\n aggregation path under this PR's budget; worth its own follow-up if the\n operator wants that population queryable.\n- conversation_asset_file_names.json: PARSED (already covered by\n polylogue-0hwv's resolver as the fallback name source).\n- codex.json: PARSED as first-class sessions. New parser\n polylogue/sources/parsers/chatgpt_codex_sidecar.py + a tight structural\n detector (task_e_\u003chex\u003e id + turns shape) wired into\n archive/artifact_taxonomy/runtime.py (classification -- without this a\n task record fails every session-document heuristic and is silently\n dropped before parsing ever runs) and sources/dispatch.py (routing to the\n new parser instead of chatgpt.parse, which would otherwise silently\n produce a zero-message, hence write-time-dropped, session for it).\n\n Coalescing question resolved: codex.json tasks do NOT coalesce with\n existing codex-session records. Confirmed both structurally and by test:\n local Codex CLI sessions are keyed by a rollout session_id UUID\n (sources/parsers/codex.py, Origin.CODEX_SESSION); these cloud tasks are\n keyed by task_e_\u003chex\u003e ids with turn ids task_e_\u003chex\u003e~usertrn_e_\u003chex\u003e /\n ~assttrn_e_\u003chex\u003e -- a disjoint namespace, verified against the real\n codex.json (codex.looks_like returns False on every real task record).\n Ingesting them adds one new session per task under\n source_name=Provider.CHATGPT (they physically arrive via this export)\n tagged ingest_flags=[\"capture:chatgpt-codex-cloud-task\"], never a\n duplicate of anything already archived.\n\n All 20 real tasks in the corpus now parse into 20 distinct 2-message\n sessions (previously 0 -- every one was silently dropped).\n\n- message_feedback.json / shared_conversations.json / ads.json: unchanged,\n per the bead's own framing (shared_conversations already referenced,\n message_feedback/ads low value) -- out of scope for this PR, no new\n decision needed.\n","status":"closed","priority":2,"issue_type":"task","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T23:44:28Z","created_by":"Sinity","updated_at":"2026-07-31T03:55:39Z","started_at":"2026-07-31T03:32:14Z","closed_at":"2026-07-31T03:55:39Z","close_reason":"Merged in PR #3409 (polylogue/master@11403388d): library_files.json parsed (feeds the asset resolver), conversation_asset_file_names.json parsed (fallback name source), codex.json parsed as first-class sessions with confirmed-disjoint identity from codex-session records. Library files with no message reference as standalone first-class refs explicitly deferred (documented in bead notes, needs whole-source-scan aggregation not yet built).","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-075v","title":"extend browser extension to capture Claude Design chats live","description":"Claude Design (claude.ai design mode) currently reaches the archive only via the GDPR export's design_chats/ directory - 11 sessions, 95 messages in the 2026-07-30 batch. That is a quarterly-batch path for a surface the operator uses interactively.\n\nThe browser-capture lane already handles claude.ai conversations end-to-end (browser+ext -\u003e receiver -\u003e spool -\u003e archive). Design chats are a distinct route/DOM on the same origin.\n\nNote the wire shape differs from ordinary conversations: design chats use messages[]/role rather than chat_messages[]/sender, plus project/title/uuid. ai_parser._parse_design_chat already handles the export shape and should be the target model.\n\nAC: design chats captured live by the extension land as claude-ai sessions equivalent to their export representation, and a session captured both ways coalesces rather than duplicating.","status":"open","priority":2,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T22:05:28Z","created_by":"Sinity","updated_at":"2026-07-30T22:05:28Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-erf3","title":"claude.ai export zip detects as unknown-export at the container level","description":"polylogue import --explain on claude-ai-data-2026-07-30-16-36-batch-0000.zip reports detector=zip.container, detected_origin=unknown-export, detected_provider=unknown, with artifact_taxonomy.path matched=false - even though every inner entry lowers to session:claude-ai:\u003cuuid\u003e and 1013 sessions parse correctly.\n\nSo the container carries no origin identity while its contents do. Plausibly the same shape as dataset finding C5 (20 'unknown' settled-yet-absent documents).\n\nAC: a claude.ai GDPR export zip is detected as claude-ai-export at the container level, or the reason it cannot be is documented and C5's unknown cohort is re-checked against that answer.","status":"open","priority":2,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T22:05:28Z","created_by":"Sinity","updated_at":"2026-07-30T22:05:28Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-zng9","title":"parse claude.ai memories.json from GDPR exports","description":"The claude.ai GDPR export ships memories.json (15.7 KB in the 2026-07-30 batch) and polylogue drops it entirely: the only 'memories' parser is codex's memories_1.sqlite (sources/parsers/codex_state.py). No claude-ai handling exists.\n\nEvidence: rg -n 'memories' over polylogue/ shows zero claude-ai hits; import --explain on claude-ai-data-2026-07-30-16-36-batch-0000.zip yields 1013 sessions = 1002 conversations + 11 design_chats, with memories.json contributing nothing.\n\nAC: memories.json content is represented in the archive (assertion, sidecar, or session-scoped construct - decide which), and re-import is idempotent.","status":"open","priority":2,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T22:05:10Z","created_by":"Sinity","updated_at":"2026-07-30T22:05:10Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-vs5x","title":"Clock guard installs per-test, so module-level clock reads at collection time escape it","description":"## The gap\n\n`tests/infra/clock_guard.py` replaces the old `test-clock-allowlist.yaml` lint\nwith a runtime guard: reaching for a host clock inside a guarded test file\nraises, pointing at `frozen_clock`. That is a genuine upgrade -- an allowlist is\nwhat you build when the capability is still available.\n\nBut the guard installs as an `autouse` **fixture**, so it arms per-test, after\npytest has already imported the test module. A clock read at module level --\na constant, a decorator argument, a `@pytest.mark.parametrize` value -- executes\nduring collection and escapes it entirely.\n\nThe old static AST lint DID catch that case. So on this one axis the runtime\nguard is weaker than what it replaced, and the PR's \"unreachable\" framing\noverstates it: it is \"unreachable from inside a test function\", not\n\"unreachable\".\n\n## Why this is worth closing rather than documenting\n\nA module-level clock read is unusual but it is exactly the shape that produces\nthe flakiness the guard exists to prevent -- a value captured once at import\nand reused across every test in the file, drifting from the frozen clock the\ntests believe they are using.\n\n## Direction\n\n`pytest_configure` runs before collection, so patches installed there cover\nmodule import. The scoping mechanism already exists: `_time_raiser` uses a\ncaller-frame check to distinguish guarded test files from production code, so a\nprocess-wide patch does not have to mean a process-wide failure.\n\nTwo things to work out:\n\n- The per-module `datetime` symbol patch is module-specific (it rebinds\n `datetime` in the test module's own namespace when that module did\n `from datetime import datetime`). A configure-time install cannot know the\n module set yet, so this likely needs a different mechanism -- patching\n `datetime.datetime` itself, guarded by the caller-frame check, rather than\n per-module rebinding.\n- `conftest.py` and `tests/infra` are deliberately exempt, and both are imported\n before ordinary test modules; the exemption must survive the move.\n\n## Acceptance criteria\n\n- A test file with a module-level `datetime.now()` fails with the guard's\n guidance message, not silently.\n- Existing exemptions (`tests/infra`, `conftest.py`,\n `@pytest.mark.uses_real_clock`) still hold.\n- Tests requesting `frozen_clock` still work -- note the guard now narrows\n rather than disables for those (it keeps guarding `time_ns`/`monotonic_ns`,\n which `freeze_clock` does not patch).\n- The word \"unreachable\" is only used where it is true.\n\nRef polylogue-aggz\n","status":"open","priority":2,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T17:55:44Z","created_by":"Sinity","updated_at":"2026-07-30T17:55:44Z","labels":["area:testing"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-ubwg","title":"Evaluate typed-constructor chokepoints for the remaining hash-boundary-registry sites","description":"polylogue-aggz Invariant 1 (comparison identity contains only content) is now structurally enforced in polylogue/pipeline/ids.py via fixed keyword-only constructors (message_identity_hash/attachment_identity_hash/event_base_identity_hash/event_canonical_identity_hash) instead of dict-key-list projection -- passing a non-content field is a TypeError at the call boundary, not a value a reviewer has to remember to strip. This closed the exact defect pattern behind polylogue-bu1i and polylogue-nuec.\n\ndocs/plans/hash-boundary-registry.yaml was NOT retired in that change and should stay open as tracked debt, not be treated as superseded. It governs all 198 hashlib/core.hashing call sites across polylogue/ (90 content-hash, 91 identifier, 17 other, spanning 58+ files: blob_store.py, security/excision.py, judgment/*, sinex/*, browser_capture/*, ...), the overwhelming majority of which are NOT session/message/attachment/event comparison identity -- they are content-addressed storage keys, HMAC signatures, redaction digests, and other identifier-generation sites with a different (and often already-correct) risk shape. Retiring the whole registry would have been a false claim of coverage this session did not do the work for.\n\nFollow-up: audit whether any of the 91 'identifier'-classified sites share the aggz failure shape (a mutable/acquisition-state field folded into a value used for equality/dedup comparison) and, for those that do, build the same fixed-signature-constructor pattern used in pipeline/ids.py. Sites that are pure content-hashing of raw bytes/already-hashed values (the 'content-hash'/'other' tags) don't need this -- only sites where an identifier is also treated as a stable comparison key are candidates. Only once every hash-boundary site is provably covered by a structural chokepoint (or provably out of the aggz identity-comparison class) can the registry itself be deleted.","status":"open","priority":2,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T17:22:26Z","created_by":"Sinity","updated_at":"2026-07-30T17:22:26Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-2jga","title":"Split or delete test-closure-matrix.yaml / test-quality-coverage.yaml's unenforced narrative fields","description":"Audit (2026-07-30, meta-machinery purge) found two related but distinct\nmanifests each mixing one real enforced check with unenforced free-text\nnarrative:\n\n1. docs/plans/test-closure-matrix.yaml (381 lines): devtools/verify_closure_matrix.py\n only checks that target_files/representative_tests paths exist on disk and\n that gate:absent rows carry a known_gaps bullet — it never runs the\n representative tests or verifies they exercise the target files. Its only\n failure mode is \"a file moved/renamed and the hand-maintained matrix wasn't\n updated\" — the fossilized-diff pattern CLAUDE.md flags for deletion. Counter-\n consideration: it forces explicit known_gaps documentation per declared-\n absent domain, which has narrative value distinct from the path check, and\n git history (d068d6482, 054dfa9e1, dc6fa632a) shows only refactor/consolidation\n commits, never a caught coverage gap that wasn't already known from the\n known_gaps text itself.\n\n2. docs/plans/test-quality-coverage.yaml: check_test_quality_ci_claims verifies\n ci_gate:true dimensions actually appear in a real CI workflow step (a\n genuine, real check — keep this). But most of the file's content\n (flakiness.known_flaky, mock_depth, fuzz tool locations) is pure narrative\n with no executable check beyond generic schema/coverage-gap validation, and\n nothing re-verifies a known_flaky entry is still flaky or that\n value_percent/last_verified stay current.\n\nOperator call needed: (a) for test-closure-matrix.yaml, keep as narrative\ndocumentation with path-existence hygiene, or delete and let the real\nper-domain test suites speak for themselves; (b) for test-quality-coverage.yaml,\nsplit the ci_gate dimension (keep, real check) from the flakiness/fuzz/mock_depth\nnarrative (move to a plain doc outside docs/plans/ verification, or delete).\nNot resolved in the purge session because both are genuinely load-bearing in\npart and the split requires deciding how much narrative value survives without\nthe doc.","status":"open","priority":2,"issue_type":"chore","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T16:55:04Z","created_by":"Sinity","updated_at":"2026-07-30T16:55:04Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-ganm","title":"Reduce topology-target.yaml to a bare file inventory, drop placement-judgment columns","description":"Audit (2026-07-30, meta-machinery purge, feature/chore/purge-meta-machinery) found\ndocs/plans/topology-target.yaml's 4618 lines are almost entirely a per-file\n`target`/`reason`/`owner` placement-judgment projection that no code or doc\nreads to make a placement decision — it is written by\ndevtools/build_topology_projection.py, then only checked against itself by\ndevtools/verify_topology.py's orphan/missing/conflict checks (which need only\na bare path list) plus a narrow kernel_rule check (which needs `target`/`owner`\nonly for the ~15 files that live directly at polylogue/ root).\n\ngit log --oneline --follow on the yaml and on build_topology_projection.py /\nrender_topology_status.py (predecessor render target, already deleted this\nsession) shows only mechanical regenerate-after-adding-a-module commits,\nnever a commit that used the placement judgments to actually relocate code.\n\nReal defect class the SURVIVING checks prevent (keep these): orphan file in\ntree not declared, declared file missing from tree, duplicate declaration,\nnon-kernel file sitting at polylogue/ root. These only need a file inventory\n+ owner tag for root files, not a placement/target/reason judgment per file.\n\nProposed scope: rewrite devtools/build_topology_projection.py and\ndevtools/verify_topology.py so the generated artifact is a flat sorted list\nof declared paths (+ owner/target only for the root-level kernel_rule check),\ndropping target/reason/loc/cross_cut columns for the ~600 non-root files.\nUpdate polylogue/verification/manifests/models.py's TopologyManifest/\nTopologyEntry to match the reduced schema.\n\nNot done in the purge session because it is a generator/schema rewrite, not\na deletion — real engineering risk of breaking `render all --check` /\n`verify topology` if done without careful review, and genuinely needs an\noperator call on whether the placement-judgment metadata has narrative value\nworth keeping despite zero consumption evidence.","status":"open","priority":2,"issue_type":"chore","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T16:54:46Z","created_by":"Sinity","updated_at":"2026-07-30T16:54:46Z","dependency_count":0,"dependent_count":0,"comment_count":0} @@ -481,7 +511,7 @@ {"_type":"issue","id":"polylogue-ei0d","title":"session_provider_usage_events.payload_json is 1.28 GiB of write-only data whose every field is a typed column beside it","design":"Measured on the live archive 2026-07-29 (index.db, 33.69 GiB by dbstat).\n\n session_provider_usage_events (table) 2443.9 MB 7.1% of the index tier\n of which payload_json 1.28 GiB (52% of the table)\n 4,030,168 rows, avg 340 B, zero NULLs\n\nWRITE-ONLY\n`payload_json` is written by _PROVIDER_USAGE_EVENT_INSERT_SQL\n(storage/sqlite/archive_tiers/write.py:3041-3047) and never read back. An AST-ish scan\nfor `SELECT ... FROM session_provider_usage_events` mentioning payload_json returns zero\nhits; the only reads of this table anywhere select COUNT(*)\n(archive_tiers/self_verify.py:28), `position` (ingest_precedence.py:146,196 and\nwrite.py:2904), or `SELECT 1` (archive.py:4871).\n\n(`insights/claude_workflow_materializer.py:458` does read a `payload_json`, but from\n`session_events` -- a different table. Do not confuse the two.)\n\nREDUNDANT BY CONSTRUCTION\nEvery field in the blob is already an extracted, typed column in the same row:\n {\"last_token_usage\":{\"cache_write_tokens\":451,\"cached_input_tokens\":54332,\n \"input_tokens\":7,\"output_tokens\":3},\n \"model\":\"claude-opus-4-20250514\",\"semantics\":\"per_message\",\"type\":\"message_usage\"}\nmaps to last_cache_write_tokens / last_cached_input_tokens / last_input_tokens /\nlast_output_tokens / model_name / provider_event_type. The table has 20 columns and the\nblob adds no field they do not already carry.\n\nIt is also doubly redundant by tier: index.db is REBUILDABLE, and the authoritative raw\npayload already lives in source.db's blob store. Keeping a copy of provider wire bytes in\nthe derived tier stores the same evidence a third time.\n\nALSO WORTH A LOOK WHILE HERE\n`total_cache_write_tokens` is constant across a 400k-row sample (1 distinct value), and\n`provider_event_type` / `model_context_window` have 2 each. A constant column over 4M rows\nis its own small waste; confirm against the full table before acting, since the sample was\nthe first 400k rows and may not be representative.\n\nDO\nDrop payload_json from the table (index tier, so this is a derived-schema change: classify\nper CLAUDE.md's \"Schema regimes\" and declare the delta class in\nstorage/sqlite/lifecycle.py -- an undeclared bump silently forces a full raw replay). If\nsome future consumer genuinely needs the provider's original wire shape, it should read it\nfrom source.db's blob, not from a duplicate in a rebuildable tier.\n\nExpected reclaim: ~1.28 GiB of index.db, plus a smaller write-path saving on every usage\nevent ingested. Batch this with any other index-tier change so it costs one rebuild, not\ntwo -- a full rebuild currently replays 92 GiB.\n","status":"open","priority":2,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-29T05:50:50Z","created_by":"Sinity","updated_at":"2026-07-29T05:50:50Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-lrdh","title":"master: 3 browser-capture coalescing tests fail on title precedence (GDPR export beats browser capture)","design":"Reproduced on pure origin/master (not introduced by any in-flight branch), verified by\nchecking out origin/master's polylogue/ and tests/unit/sources/test_browser_capture.py into\na clean tree and running the selection:\n\n devtools test tests/unit/sources/test_browser_capture.py -k coalesces 3 failed\n\n test_browser_capture_raw_payload_coalesces_with_claude_ai_export\n test_browser_capture.py:1008 assert 'Claude GDPR title' == 'Claude browser title'\n test_browser_capture_raw_payload_coalesces_with_chatgpt_export[browser-first]\n test_browser_capture_raw_payload_coalesces_with_chatgpt_export[export-first]\n test_browser_capture.py:922 assert 'GDPR title' == 'Browser title'\n\nThe tests assert a browser-capture title outranks a GDPR/export title when the two\ncoalesce into one session; the export title is winning instead. Both parametrizations\nfail, so it is not acquisition-order dependent.\n\nEither the precedence rule changed and these tests were not updated, or a real regression\nin coalescing title selection landed without being caught -- per-PR CI skips the heavy\ntest suite (it runs post-merge on master), which is the mechanism that lets this sit\nbroken on master.\n\nDetermine which before editing: if the intended rule is now export-wins, the tests encode\na stale contract and should be rewritten to state the new one with its reason; if\nbrowser-wins is still intended, this is a live bug in the coalescing path and the tests\nare correct.\n\nFound while merging two agent branches (pbuh sidecar evidence, ah21 browser-capture blocks);\nneither touches title coalescing and both reproduce the failure identically, as does a\nclean origin/master checkout.\n","status":"closed","priority":2,"issue_type":"bug","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-07-29T05:43:23Z","created_by":"Sinity","updated_at":"2026-07-29T06:51:48Z","started_at":"2026-07-29T06:51:28Z","closed_at":"2026-07-29T06:51:48Z","close_reason":"Determined: precedence rule legitimately changed, tests were stale.\n\n#3179 (commit b473d9256, Ref polylogue-z1c6, merged 2026-07-20) intentionally\nadded a mirror rule to browser_capture_precedence() in\npolylogue/storage/sqlite/archive_tiers/ingest_precedence.py: a genuine\nnon-browser-capture arrival (direct/GDPR export) now always outranks\nbrowser-capture-only content and vice versa is skipped, making the outcome\norder-independent (fixing a real order-dependent flakiness bug where whichever\nmaterial a live daemon happened to process first would win). That PR added\nand updated the sibling proof\ntest_archive_tiers_archive_facade_export_vs_native_precedence_is_order_independent\n(tests/unit/storage/test_archive_tiers_archive.py:737, asserting\n(\"Direct export\", export_message_count) regardless of arrival order) but\nmissed updating the three browser_capture.py coalescing tests that predate\nit (last touched by ddf4f3efc, well before #3179).\n\nFixed: rewrote the three stale title assertions in\ntests/unit/sources/test_browser_capture.py (now lines 930 and 1019) from\n\"Browser title\"/\"Claude browser title\" to \"GDPR title\"/\"Claude GDPR title\",\nwith comments citing browser_capture_precedence(), #3179/polylogue-z1c6, and\nthe sibling order-independence test. No production code changed -- this was\nnever a live regression.\n\nLive-archive impact (read-only check against /realm/db/polylogue, confirmed\nPOLYLOGUE_ARCHIVE_ROOT resolves there): 0 sessions in raw_sessions currently\nhave more than one distinct capture_mode for the same (origin, native_id), so\nno live session's stored title is affected by this either way -- production\nhas been export-wins all along.\n\nGate recommendation: per-PR CI skipping the heavy test suite is the\ndocumented mechanism (CLAUDE.md) that let a legitimate #3179 rule change\nmerge without updating every affected test; this is already a known,\naccepted tradeoff (heavy suite runs post-merge). Not recommending a new\nfossilized-diff-style check -- CLAUDE.md forbids gates that memorialize a\nrenamed spelling, and the actual missing net here is \"did #3179 run the full\ntest_browser_capture.py file\", which devtools test \u003cchanged files\u003e would\nhave caught if run; no new lint needed.\n\nVerification: devtools test tests/unit/sources/test_browser_capture.py -k coalesces\n(3 passed), full file green, tests/unit/storage/test_archive_tiers_archive.py -k\nprecedence (13 passed), devtools verify --quick (exit_code 0). Committed as\n1f0040353 on branch worktree-agent-acdc97dbfb9cf3928 (agent worktree; PR not\nopened/merged per task scope -- report only).","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-7to5","title":"Capture and export convergence on one session_id is untested and would silently downgrade fidelity","description":"Measured 2026-07-29 -- this is a LATENT hazard, not an active bug, which is why it needs recording before it fires.\n\n chatgpt conversations reachable by browser capture only: 43\n reachable by GDPR export only: 2,423\n reachable by BOTH: 0\n sessions == distinct native_ids == 2,635 (no duplication today)\n\nThe two paths have never overlapped, so coalescing has never been exercised.\nWhat would happen is determined by two facts already established:\n\n 1. IDENTITY WOULD COLLIDE, NOT DUPLICATE. Both paths key the ChatGPT\n conversation id as native_id, and session_id is a generated column\n (origin || ':' || native_id). Same conversation, same session_id.\n 2. THE PARSED WRITE PATH IS FULL-REPLACE. write.py deletes blocks and messages\n for the session id, then inserts. Whichever path ingests SECOND wins\n entirely.\n\nAnd the two paths carry materially different fidelity: the export has the\nmapping tree with tool nodes and status; the capture has flat text with no\nblocks channel at all (see the BrowserCaptureTurn bead). So exporting a\nconversation you had already captured is fine, and CAPTURING one you had\nalready exported silently replaces structured evidence with flattened text.\n\nThe raw tier already models this correctly -- raw_revision_heads, revision\nauthority, accepted frontiers. It is the parsed tier that resolves by\nreplacement instead of by fidelity.","acceptance_criteria":"1. Two observations of one conversation are retained as revisions, and the composed session reflects the higher-fidelity one regardless of arrival order. 2. A test ingests export-then-capture and capture-then-export for the same conversation and asserts the same, higher-fidelity result both ways. 3. Fidelity is declared per acquisition path in the OriginSpec so 'higher' is not a judgement call at write time. 4. Related: unknown-export currently holds 52 raws with NULL native_id -- conversations that failed origin detection and therefore cannot coalesce with anything.","status":"open","priority":2,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-29T04:52:48Z","created_by":"Sinity","updated_at":"2026-07-29T04:52:48Z","labels":["area:ingest","lane:capture-reliability"],"dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"polylogue-o4j2","title":"aistudio-drive discards every model setting that produced its outputs","description":"The per-origin wire enumeration found the entire runSettings block unread for aistudio-drive:\n\n temperature, topP, topK, maxOutputTokens, thinkingLevel, safetySettings\n (with threshold), enableCodeExecution, enableSearchAsATool,\n enableBrowseAsATool, enableAutoFunctionResponse\n plus chunkedPrompt.pendingInputs\n\nThis is the model configuration for every AI Studio session in the archive.\nPolylogue's stated purpose includes reconstructing what produced a result; for\nthis origin the generation parameters are present in the acquired bytes and\ndropped at parse.\n\nIt is also the only origin where the operator can vary sampling settings freely,\nwhich makes it the one place where 'same prompt, different settings, different\noutput' is answerable -- if the settings were kept.","acceptance_criteria":"1. runSettings is parsed into typed session-level evidence for aistudio-drive. 2. The settings are queryable, so 'sessions where temperature \u003e X' is expressible. 3. Existing sessions acquire it by reprocess of retained bytes. 4. Other origins are checked for an equivalent settings block rather than assuming AI Studio is unique.","status":"open","priority":2,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-29T04:52:47Z","created_by":"Sinity","updated_at":"2026-07-29T04:52:47Z","labels":["area:ingest","lane:origin-interop-export"],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-o4j2","title":"aistudio-drive discards every model setting that produced its outputs","description":"The per-origin wire enumeration found the entire runSettings block unread for aistudio-drive:\n\n temperature, topP, topK, maxOutputTokens, thinkingLevel, safetySettings\n (with threshold), enableCodeExecution, enableSearchAsATool,\n enableBrowseAsATool, enableAutoFunctionResponse\n plus chunkedPrompt.pendingInputs\n\nThis is the model configuration for every AI Studio session in the archive.\nPolylogue's stated purpose includes reconstructing what produced a result; for\nthis origin the generation parameters are present in the acquired bytes and\ndropped at parse.\n\nIt is also the only origin where the operator can vary sampling settings freely,\nwhich makes it the one place where 'same prompt, different settings, different\noutput' is answerable -- if the settings were kept.","acceptance_criteria":"1. runSettings is parsed into typed session-level evidence for aistudio-drive. 2. The settings are queryable, so 'sessions where temperature \u003e X' is expressible. 3. Existing sessions acquire it by reprocess of retained bytes. 4. Other origins are checked for an equivalent settings block rather than assuming AI Studio is unique.","status":"closed","priority":2,"issue_type":"task","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-07-29T04:52:47Z","created_by":"Sinity","updated_at":"2026-07-31T04:02:59Z","started_at":"2026-07-31T04:01:05Z","closed_at":"2026-07-31T04:02:59Z","close_reason":"runSettings storage was already shipped (PR #3390, polylogue-2qx.4/cgfy, index v46) before this bead was filed. The genuinely-remaining gap -- chunkedPrompt.pendingInputs (draft/unsent textbox content, 7/397 real sessions with non-blank drafts) -- is fixed on PR #3415 (draft_input session_event). AC2 (query-DSL numeric predicates over run_settings, e.g. temperature \u003e X) is NOT satisfied: the boolean-query grammar only accepts integer literals and NumericQueryFieldInfo assumes a plain SQL column, not a JSON-extract expression -- needs a separate DSL float-literal + JSON-field-predicate feature, out of parser scope. AC4 checked: grepped sources/parsers + sources/providers for generationConfig/sampling_params/temperature/inference_config/model_settings, no other origin has an equivalent settings block.","labels":["area:ingest","lane:origin-interop-export"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-4p1.3","title":"Insights: the concept earns its place, five of eleven types do not","description":"DEEP REVIEW of polylogue/insights (23,103 lines, 11 registered types) 2026-07-29.\n\nWHAT AN INSIGHT IS, AND WHY IT IS NOT JUST A NAMED QUERY. tool_usage pairs an\naggregate with per-origin COVERAGE: 'an origin with sessions but zero actions is\nthe explicit data-unavailable signal, not a quiet zero.' That distinction --\nzero versus unavailable -- is real, is not expressible in the query DSL's\n, and is the honest core of the concept. The insights package\nshould NOT be dissolved into the query algebra wholesale.\n\nWHICH TYPES EARN THEIR EXISTENCE (materialized tables in parentheses):\n KEEP threads (9,914) a root session's lineage tree; single-session\n threads are correct, not degenerate -- 810\n multi-session threads correspond exactly to\n the 810 sessions with lineage children\n KEEP tool_usage computed; the coverage pairing is the value\n (but its CLI surface is currently BROKEN --\n see the analyze-tools bead)\n KEEP session_costs, cost_rollups, usage_timeline, archive_coverage,\n archive_debt computed rollups with coverage semantics\n DELETE session_phases (29,432) see the deletion bead: 82% single span, no\n label, index-synthesized timestamps\n DELETE session_work_events (21,190) 82% single event, duplicates action_pairs\n REDUCE session_profiles (18,871) keep the profile, delete the five\n constant version/family columns and fix the\n 100%-NULL cost columns (see f2qv.6)\n REDUCE session_tag_rollups (3,593) explicit_count constant 0\n\nSTRUCTURAL FINDING: 5 of 11 types are materialized tables, 6 are computed. There\nis no stated rule for which. The materialized ones are precisely where the\nfreshness machinery lives (insight_materialization's seven proxy columns,\nderived_refresh_guard, delegation_refresh_scope). Under content-addressed\nderivation the distinction stops mattering -- a materialized insight becomes a\nhash-keyed cache, and a stale row is a miss rather than a lie.\n\nSURFACE FINDING: every type is MCP-reachable through\nmcp/insight_tool_contracts.py and only some are CLI-reachable; see the\nregistry-surface bead.","acceptance_criteria":"1. Each of the 11 types carries a recorded verdict: keep / reduce / delete, with the discrimination evidence. 2. A stated rule governs materialized versus computed, or the distinction is removed by hash-keying. 3. The coverage-pairing property is documented as the reason the package exists, so a future refactor does not dissolve it into the query DSL by accident. 4. Deleting a type removes its table, materializer, registry entry, MCP contract and any FTS index together.","status":"open","priority":2,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-29T04:52:40Z","created_by":"Sinity","updated_at":"2026-07-29T04:52:40Z","labels":["area:analytics","area:query","area:surface","decision","delivery:C-read-evidence-contract","horizon:frontier","lane:read-contracts"],"dependencies":[{"issue_id":"polylogue-4p1.3","depends_on_id":"polylogue-4p1","type":"parent-child","created_at":"2026-07-29T06:52:40Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-4p1.2","title":"Registered insights are MCP-reachable and CLI-absent: decide the surface, do not let it drift","description":"Measured 2026-07-29. insights/registry.py registers insight types including session_phases (cli_command_name='phases'), session_work_events, threads, session_profiles, session_tag_rollups, session_costs, cost_rollups, usage_timeline, tool_usage, archive_coverage, archive_debt.\n\n $ polylogue analyze --help -\u003e insights, latency, pace, tools, turns, usage\n\nSo 'polylogue analyze phases' does not exist despite the registry declaring that\nname. MCP reaches all of them through mcp/insight_tool_contracts.py, which is\nregistry-driven. The registry is therefore authoritative for MCP and decorative\nfor the CLI -- a declare-once mechanism honoured by one surface and not the\nother, which is the pattern polylogue-t46 exists to remove.\n\nOPERATOR POSITION (2026-07-29): a registered insight should be CLI-reachable\nunless there is a clear reason not to -- but the CLI itself must stay\ndisciplined rather than sprawling one subcommand per registry entry. Those pull\nin opposite directions and the resolution is a decision, not a default.\n\nNote this interacts with two deletions: session_phases and session_work_events\nare condemned by a sibling bead, so their registry entries and MCP contracts go\nwith them rather than gaining CLI commands.","acceptance_criteria":"1. Every registry entry is classified: CLI-reachable, MCP-only with a stated reason, or deleted. 2. No registry entry declares a cli_command_name that produces no command. 3. Whatever the decision, one mechanism generates both surfaces -- a registry honoured by MCP and ignored by the CLI does not survive. 4. The CLI does not gain a subcommand per entry by default; the disciplined shape is argued explicitly.","status":"open","priority":2,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-29T04:52:38Z","created_by":"Sinity","updated_at":"2026-07-29T04:52:38Z","labels":["area:query","area:surface","decision","delivery:C-read-evidence-contract","horizon:frontier","lane:read-contracts"],"dependencies":[{"issue_id":"polylogue-4p1.2","depends_on_id":"polylogue-4p1","type":"parent-child","created_at":"2026-07-29T06:52:38Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-cuxz.11","title":"session_agent_policies: 402,869 rows encoding 3,053 facts","description":"Measured 2026-07-29, full scan.\n\n rows 402,869\n distinct sessions 3,031 (133 rows per session)\n sessions whose policy NEVER changes 3,010 (99.3%)\n rows remaining if deduped by value 3,053\n\nThe table records approval_policy / sandbox_policy / network_policy at every\nmessage position, and the policy is invariant within a session 99.3% of the\ntime. It is a change-log of non-changes at 132x redundancy.\n\nAlready known degenerate on the same table: network_policy is constant 'false',\nsource_message_id is 100% NULL.\n\n sqlite3 -readonly index.db \"with per as (select session_id,\n count(distinct coalesce(approval_policy,'')||'|'||coalesce(sandbox_policy,'')||'|'||coalesce(network_policy,'')) d,\n count(*) n from session_agent_policies group by session_id)\n select count(*), sum(n), sum(d=1), sum(d) from per;\"","acceptance_criteria":"1. Policy is stored once per session (or per genuine change), not per message position. 2. If policy genuinely varies for some sessions, those keep interval rows; the 3,010 invariant sessions do not. 3. network_policy and source_message_id are dropped unless a producer is named. 4. Report row count before and after against the 402,869 baseline.","status":"open","priority":2,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-29T04:52:36Z","created_by":"Sinity","updated_at":"2026-07-29T04:52:36Z","labels":["area:storage","area:substrate","horizon:frontier"],"dependencies":[{"issue_id":"polylogue-cuxz.11","depends_on_id":"polylogue-cuxz","type":"parent-child","created_at":"2026-07-29T06:52:36Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} @@ -495,7 +525,7 @@ {"_type":"issue","id":"polylogue-vuq2","title":"Rebuild on the free-threaded daemon, not the GIL CLI: 74h -\u003e ~1-2h (idle + single-core parse)","design":"Two independent losses, both fixed by running the rebuild through the daemon instead of\na devshell CLI invocation. Measured on the live archive 2026-07-29.\n\nLOSS 1 -- idle wall-clock (65h of the last rebuild's 74h).\ndaemon_bulk_rebuild_routing defaults to False (config.py:1993) and is unset in the live\n~/.config/polylogue/polylogue.toml, so the daemon never routes a bulk backlog into the\nblue-green rebuild it already knows how to drive (daemon/bulk_rebuild.py); it only logs a\nrecommendation to run the CLI by hand (_maybe_recommend_bulk_rebuild).\n operation 3f8fa7b0 (promoted 07-26): 107 passes, 74.0h wall-clock\n 104 passes \u003c=30min totalling 9.2h \u003c- compute\n 2 passes \u003e30min totalling 64.8h \u003c- 88% idle; gaps of 60.4h and 4.5h\n operation ab5bad1f (promoted 07-21): 53 passes, 22.9h, 69% idle\nBoth carry random UUID operation ids, not DAEMON_BULK_REBUILD_OPERATION_ID, so both were\noperator CLI runs resumed by hand across days. With routing on the daemon drives passes\nwith a 1s burst pause and resumes across restarts via the well-known operation id.\n\nLOSS 2 -- single-core parse (the 9.2h itself). This is the bigger one.\n_parse_unique_retained_raws (sources/revision_backfill.py:1267) picks its strategy from\nparallel_threads_effective():\n - free-threaded: ThreadPoolExecutor over EVERY raw, \"no size partition or amortization\n floor\" -- parsed object graphs are shared by reference, so neither process-pool cost\n applies.\n - GIL build: falls through to ProcessPoolExecutor, but _partition_raws_by_dispatch_size\n sends every raw \u003e= 256 KiB (_DEFAULT_PARSE_DISPATCH_MAX_BYTES) to a SEQUENTIAL\n in-process parse, because pickling large ParsedSession graphs back across the process\n boundary measured 0.63x -- a net loss (polylogue-amg1).\nOn this corpus that partition is catastrophic:\n pool-eligible (\u003c256 KiB): 24,946 raws 1.38 GiB\n SEQUENTIAL (\u003e=256 KiB): 16,417 raws 90.84 GiB \u003c- 98.5% of all bytes\nSo 98.5% of the payload parsed on one core of a 24-thread machine. That is the 2.86 MiB/s.\n\nWhich interpreter each path uses, verified:\n polylogued (PID 1450933): Python 3.14.4t, sys._is_gil_enabled() == False -\u003e threads\n repo devshell python: Python 3.13.13, _is_gil_enabled() == True -\u003e no threads\npolylogue-7mtf's control run measured the same ThreadPoolExecutor parse code at 3.9x-9.6x\n(w=4..16) free-threaded versus 0.93x-0.96x under the GIL. Applying that to the 9.2h gives\n2.4h at 3.9x and 1.0h at 9.6x -- and ingest_workers currently resolves to min(8, cpus-1)=8\n(resolve_parse_worker_count), overridable via POLYLOGUE_INGEST_PARSE_WORKERS, with 24\nthreads on the box.\n\nSo: 74h -\u003e roughly 1-2.4h, with no engine change. Free-threading is already deployed; the\nrebuild simply has not been run on it.\n\nHAZARD to prevent recurrence: a `polylogue ops maintenance rebuild-index` run from the\ndevshell silently gets the GIL interpreter and the sequential partition. The rebuild\nreceipt already records ingest_workers but not gil_enabled/parallel_threads_effective --\nrecord it, and warn loudly when a bulk rebuild starts on a GIL build. `polylogue status`\nalready surfaces gil_enabled (cli/commands/status.py:1215); the rebuild path should too.\n\nDo: (1) set daemon.raw_materialization.bulk_rebuild_routing = true; (2) confirm the\ntrickle-suppression interaction (_daemon_bulk_rebuild_transaction_in_flight); (3) record\nthe interpreter mode in the rebuild receipt + warn on a GIL-build bulk rebuild;\n(4) re-measure drain wall-clock during the run (a 5jak residual gd6v's close note asks for).\nFlag removal itself is polylogue-mkk0 / 4jsk.\n","status":"open","priority":2,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-29T03:52:04Z","created_by":"Sinity","updated_at":"2026-07-29T04:11:43Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-ze5i","title":"Four of ten lab policies fail silently: policy checks sit behind --lab, which no gate runs","description":"Measured 2026-07-28 by running each policy directly:\n\n schema-versioning exit=1 undeclared index schema deltas found: 1\n demo-tour-freshness exit=1 regenerated tour output differs from docs/examples/demo-tour/\n backlog-hygiene exit=1 74 findings across 10 checks; 1,127 issues scanned\n bead-graph exit=1 missing_ac=21\n timestamp-doctrine, insight-honesty, demo-packet-registry, docs-drift,\n campaign-archive-boundaries, archive-resolver-completeness exit=0\n\nAll ten are appended inside 'if lab:' at devtools/verify.py (the --lab branch). Default 'devtools verify' does not run them; 'devtools verify --quick' (the pre-push hook) does not run them; the CI lint job runs render all --check, verify public-claims and ruff, not these. So a policy can fail continuously without blocking a merge.\n\nLive consequence: the schema-versioning violation merged in PR #3378 and the first symptom was the live archive becoming unqueryable from the repo CLI ('no such column: s.title_ref'), diagnosed only by hand days later.\n\nThis bead is the placement question, not the individual failures: which policies are cheap and deterministic enough to gate by default, which are genuinely lab-tier, and what runs the lab-tier ones on a schedule so they cannot rot. schema-versioning has already been moved to the default gate and CI lint in the same change that filed this bead; the other three remain unassigned to any gate.","acceptance_criteria":"1. Every lab policy is classified as default-gated, CI-gated, or scheduled, with the cost and determinism evidence for that placement. 2. No policy is left in a position where continuous failure blocks nothing. 3. The three currently-red unplaced policies (demo-tour-freshness, backlog-hygiene, bead-graph) are either green or have their failures triaged into owned beads. 4. A regression proves a deliberately-introduced violation of a default-gated policy fails the gate.","status":"open","priority":2,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-28T20:02:53Z","created_by":"Sinity","updated_at":"2026-07-28T20:02:53Z","dependencies":[{"issue_id":"polylogue-ze5i","depends_on_id":"polylogue-93xe","type":"parent-child","created_at":"2026-07-29T06:51:37Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-b5l.3","title":"Fast-forward mechanism is spread across five modules and 2,713 lines with a per-version fork","description":"Measured 2026-07-28. One concept -- bring a derived generation forward without raw replay -- is implemented in five places:\n\n devtools/index_fast_forward.py 1,085 lines, 44 defs/classes\n devtools/archive_schema_fast_forward.py 985 lines, 43 defs/classes\n devtools/index_v37_fast_forward.py ~600 lines, 26 defs/classes\n polylogue/storage/sqlite/lifecycle.py 438 lines (declarations + planner)\n polylogue/storage/sqlite/archive_tiers/index_fast_forward_executor.py 205 lines (runtime executor)\n\nindex_v37_fast_forward.py is a version-specific FORK: its docstring scopes it to 'index v36 -\u003e v37', and it imports reflink_clone from archive_schema_fast_forward. A per-version copy of a mechanism whose whole point is being version-pair-generic is the smell.\n\nIts test suite is also the one that is red: polylogue-e6a0 records 9 failing tests in tests/unit/devtools/test_index_v37_fast_forward.py, root-caused to a hardcoded v36 DDL commit that predates action_pairs. So the most-forked copy is also the least-covered.\n\npolylogue-9rw0's notes already record the origin: PR #2788 independently authored devtools/index_fast_forward.py from a base predating the merged #2804/#2805 with the same filename and purpose, producing a real add/add conflict; the 2026-07-13 reconciliation kept the deployed execution mechanism authoritative and retained the plan-declaration layer, but did not collapse the modules.\n\nBound the duplication before cutting: this bead is an audit-then-collapse, not a blind delete -- the offline devtools actuator and the runtime on-connect executor may legitimately differ in clone/promote responsibilities even after the planning layer is shared.","acceptance_criteria":"1. A written map of which module owns planning, clone, proof, execution, and promotion, with the duplicated responsibilities named. 2. One planning authority (lifecycle.py declarations) consumed by every actuator; no actuator carries its own version knowledge. 3. index_v37_fast_forward.py is either generalized into the shared path or deleted with its transition recorded as a declaration; a per-version module does not survive. 4. polylogue-e6a0's 9 failing tests are resolved by the collapse rather than by repairing a fixture for a module that should not exist. 5. Line count and module count after the collapse are reported against the 2,713/5 baseline.","status":"open","priority":2,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-28T20:02:52Z","created_by":"Sinity","updated_at":"2026-07-28T20:02:52Z","labels":["area:daemon","area:ops","area:storage","delivery:B-storage-rebuild-bytes","horizon:frontier","lane:storage-rebuild-scale","size:L","spine"],"dependencies":[{"issue_id":"polylogue-b5l.3","depends_on_id":"polylogue-b5l","type":"parent-child","created_at":"2026-07-28T22:02:51Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"polylogue-cijx.1","title":"Repository identity fragments across URL spellings and worktrees: 106 repo_ids for one repo","description":"Measured on the live archive 2026-07-28. repos is keyed (origin_url, root_path), so one repository splits across every remote-URL spelling and every checkout root:\n\n repo_name distinct repo_ids sessions\n sinex 28 4,922\n polylogue 106 4,369\n sinnix 31 1,466\n sinity-lynchpin 2 1,809\n\nFor polylogue the largest shards are: ''+/realm/project/polylogue (3,276), https://github.com/Sinity/polylogue+same root (583), git@github.com:Sinity/polylogue.git+same root (200), plus ~15 /realm/worktrees/polylogue-* roots and .claude/worktrees/* agent roots at 7-33 sessions each.\n\nThree orthogonal identity facts are conflated into one key: (1) the repository (one remote, however spelled -- empty/HTTPS/SSH), (2) the checkout root (a worktree is evidence OF a repository, not a distinct repository), (3) an unrelated directory that merely shares a basename.\n\nConsequence: any aggregate grouped by repository silently under- or over-counts, and the query field repo: resolves to whichever shard the writer happened to record. This affects every repo-scoped read surface, not one report.\n\npolylogue-cijx's design already states the target ('Repository identity survives multiple worktrees and renames and never relies on cwd alone when stronger git evidence exists') without an executable slice; this is that slice. polylogue-j5xg's closure routed session_commits here explicitly: 'rebuilt with repo-identity care'.","acceptance_criteria":"1. Remote-URL spellings that denote one remote normalize to one repository identity; the normalization is a pure function with tests over the observed spelling set (empty, https, ssh, .git suffix). 2. Checkout root becomes worktree evidence attached to a repository, not part of the identity key. 3. A basename collision between unrelated paths does not merge them. 4. Live re-measure: distinct repository identities for polylogue/sinex/sinnix collapse to one each, with worktree roots enumerable underneath. 5. The repo: query field resolves through the normalized identity, and a regression test proves a session recorded under one spelling matches a query using another.","notes":"session_commits HAS NO READER — measured 2026-07-29 by read/write matrix over\nevery SQL statement in polylogue/: 1 writer (archive_tiers/write.py), 0 readers.\n2,989 rows are written on every ingest and read by nothing.\n\nCombined with the semantic defect already recorded (it stores HEAD-at-session-\nstart under detection_type='explicit_ref', method='parser-git-meta',\nconfidence=1.0 -- all three constant across all 2,989 rows), this is now a clean\ndeletion rather than a migration: there is no consumer to preserve. The live\ncorrelation path computes commits from git on demand and never consults the\ntable.\nBLOCKS FOUR CONSUMERS 2026-07-29. These beads all plan to consume\nsession-to-commit correlation, and the producer does not work:\n\n 212.2 PF-D1 'The receipts': claim-vs-evidence on a real PR\n xyel Real PF-D1-receipts demo re-emitted through demo-packet contract\n kph Provenance-carrying PRs: attach the authoring session's postmortem bundle\n fs1.4 Report: polylogue forensics for Hermes sessions\n\nMeasured state of the producer: session_commits has 1 writer and 0 readers;\nits 2,989 rows store HEAD-at-session-start, not commits produced, under\ndetection_type='explicit_ref' / method='parser-git-meta' / confidence=1.0, all\nthree constant across every row. The real correlator (detect_session_commits,\ntime-window + file-overlap scoring) is reachable only from an on-demand view\nand materializes nothing.\n\nNone of the four is blocked in the tracker today, so each looks independently\nstartable and would independently discover the same dead end.\nTHE PRODUCER MAY ALREADY EXIST 2026-07-29. Claude Code emits typed pr-link\nrecords -- 20,702 of them in the live corpus -- carrying prNumber, prUrl,\nprRepository and sessionId. The parser discards them (_SKIPPED_SIDECAR_RECORD_TYPES).\n\nBefore building commit/PR correlation by regex extraction and time-window\nfile-overlap scoring, read the record the provider already supplies. This may\nreduce this bead and its four blocked consumers from an inference problem to a\nparse problem.","status":"open","priority":2,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-28T20:02:15Z","created_by":"Sinity","updated_at":"2026-07-29T04:52:11Z","labels":["area:insights","area:interop","horizon:mid","tech-tree"],"dependencies":[{"issue_id":"polylogue-cijx.1","depends_on_id":"polylogue-cijx","type":"parent-child","created_at":"2026-07-28T22:02:15Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":4,"comment_count":0} +{"_type":"issue","id":"polylogue-cijx.1","title":"Repository identity fragments across URL spellings and worktrees: 106 repo_ids for one repo","description":"Measured on the live archive 2026-07-28. repos is keyed (origin_url, root_path), so one repository splits across every remote-URL spelling and every checkout root:\n\n repo_name distinct repo_ids sessions\n sinex 28 4,922\n polylogue 106 4,369\n sinnix 31 1,466\n sinity-lynchpin 2 1,809\n\nFor polylogue the largest shards are: ''+/realm/project/polylogue (3,276), https://github.com/Sinity/polylogue+same root (583), git@github.com:Sinity/polylogue.git+same root (200), plus ~15 /realm/worktrees/polylogue-* roots and .claude/worktrees/* agent roots at 7-33 sessions each.\n\nThree orthogonal identity facts are conflated into one key: (1) the repository (one remote, however spelled -- empty/HTTPS/SSH), (2) the checkout root (a worktree is evidence OF a repository, not a distinct repository), (3) an unrelated directory that merely shares a basename.\n\nConsequence: any aggregate grouped by repository silently under- or over-counts, and the query field repo: resolves to whichever shard the writer happened to record. This affects every repo-scoped read surface, not one report.\n\npolylogue-cijx's design already states the target ('Repository identity survives multiple worktrees and renames and never relies on cwd alone when stronger git evidence exists') without an executable slice; this is that slice. polylogue-j5xg's closure routed session_commits here explicitly: 'rebuilt with repo-identity care'.","acceptance_criteria":"1. Remote-URL spellings that denote one remote normalize to one repository identity; the normalization is a pure function with tests over the observed spelling set (empty, https, ssh, .git suffix). 2. Checkout root becomes worktree evidence attached to a repository, not part of the identity key. 3. A basename collision between unrelated paths does not merge them. 4. Live re-measure: distinct repository identities for polylogue/sinex/sinnix collapse to one each, with worktree roots enumerable underneath. 5. The repo: query field resolves through the normalized identity, and a regression test proves a session recorded under one spelling matches a query using another.","notes":"session_commits HAS NO READER — measured 2026-07-29 by read/write matrix over\nevery SQL statement in polylogue/: 1 writer (archive_tiers/write.py), 0 readers.\n2,989 rows are written on every ingest and read by nothing.\n\nCombined with the semantic defect already recorded (it stores HEAD-at-session-\nstart under detection_type='explicit_ref', method='parser-git-meta',\nconfidence=1.0 -- all three constant across all 2,989 rows), this is now a clean\ndeletion rather than a migration: there is no consumer to preserve. The live\ncorrelation path computes commits from git on demand and never consults the\ntable.\nBLOCKS FOUR CONSUMERS 2026-07-29. These beads all plan to consume\nsession-to-commit correlation, and the producer does not work:\n\n 212.2 PF-D1 'The receipts': claim-vs-evidence on a real PR\n xyel Real PF-D1-receipts demo re-emitted through demo-packet contract\n kph Provenance-carrying PRs: attach the authoring session's postmortem bundle\n fs1.4 Report: polylogue forensics for Hermes sessions\n\nMeasured state of the producer: session_commits has 1 writer and 0 readers;\nits 2,989 rows store HEAD-at-session-start, not commits produced, under\ndetection_type='explicit_ref' / method='parser-git-meta' / confidence=1.0, all\nthree constant across every row. The real correlator (detect_session_commits,\ntime-window + file-overlap scoring) is reachable only from an on-demand view\nand materializes nothing.\n\nNone of the four is blocked in the tracker today, so each looks independently\nstartable and would independently discover the same dead end.\nTHE PRODUCER MAY ALREADY EXIST 2026-07-29. Claude Code emits typed pr-link\nrecords -- 20,702 of them in the live corpus -- carrying prNumber, prUrl,\nprRepository and sessionId. The parser discards them (_SKIPPED_SIDECAR_RECORD_TYPES).\n\nBefore building commit/PR correlation by regex extraction and time-window\nfile-overlap scoring, read the record the provider already supplies. This may\nreduce this bead and its four blocked consumers from an inference problem to a\nparse problem.\nPR-LINK PRODUCER STATUS 2026-07-31: the \"producer may already exist\" note above\nis confirmed -- Claude Code's pr-link sidecar record now persists as typed\nevidence (PR #3390, index v46, already on master): a session_refs table row\n(kind=pull_request, url/repo/number) via code_parser.py's pr-link branch, plus\na parallel claude_pr_link session_event for audit trail. Reader-side:\nstorage/sqlite/queries/session_refs.py + storage/repository/archive/sessions.py\nexpose get_session_refs/get_session_refs_batch, but nothing on the CLI,\ninsights, or MCP surface calls them yet -- session_commits' complete absence of\nreaders (this bead's other finding) is fixed on the pr-link path specifically,\nnot in general. 212.2/xyel/kph/fs1.4 are NOT unblocked by this alone: each\nstill needs a consumer that resolves PR-\u003eauthoring session through\nsession_refs (a structural join, not the removed session_commits inference)\nbefore their own acceptance criteria can be worked. Filing that consumer slice\nis left to whoever picks up this bead or its dependents next; not attempted\nhere (out of the originating pass's declared surface: parsers/claude only).\n","status":"open","priority":2,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-28T20:02:15Z","created_by":"Sinity","updated_at":"2026-07-31T04:26:13Z","labels":["area:insights","area:interop","horizon:mid","tech-tree"],"dependencies":[{"issue_id":"polylogue-cijx.1","depends_on_id":"polylogue-cijx","type":"parent-child","created_at":"2026-07-28T22:02:15Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":4,"comment_count":0} {"_type":"issue","id":"polylogue-cuxz.4","title":"tool_result outcome NULL is undifferentiated: refusal, absence, and unparsed share one token","description":"Measured on the live archive 2026-07-28 (5,042,564 blocks; 1,844,545 tool_result blocks):\n\n origin unknown ok err %unknown\n codex-session 879,993 127,200 10,728 86%\n claude-code-session 415,109 332,165 37,045 53%\n chatgpt-export 22,992 0 0 100%\n hermes-session 15,107 1,848 382 87%\n claude-ai-export 0 1,353 39 0%\n gemini-cli-session 0 559 19 0%\n tool_result_exit_code present: 144,616 / 1,844,545 (8%)\n\nCrucially this is NOT simply a parser gap. sources/parsers/claude/code_parser.py:309-380 shows deliberate, correct refusal: _task_output_outcome trusts toolUseResult.task only when retrieval_status=='success' (a successful poll of a FAILED command otherwise surfaces envelope is_error=false), and _mark_background_task_start overwrites is_error with None because a start acknowledgement 'must not be projected as a completed-command success'.\n\nThe defect is that three distinguishable states collapse into one NULL:\n (a) provider emitted no outcome signal at all;\n (b) provider emitted a signal the parser deliberately distrusts (the refusals above);\n (c) this provider carries a signal the parser does not yet read.\nAll three are knowable at parse time. Without the distinction, every downstream efficacy/failure/rework measure is computed over a 28% sample with unstated and non-uniform bias, and no surface can caveat it.","acceptance_criteria":"1. A typed outcome-unknown reason accompanies every NULL tool_result_is_error, populated at parse time by the code that made the decision. 2. The refusal paths in code_parser.py record their specific reason rather than a generic unknown. 3. Case (c) is enumerated per origin, so 'we do not read this provider's field' is a countable backlog rather than an invisible one. 4. Read surfaces that aggregate outcomes report coverage alongside the aggregate and refuse a bare success-rate scalar when coverage is below a declared floor. 5. Live re-measure reports the reason distribution per origin.","status":"open","priority":2,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-28T20:01:45Z","created_by":"Sinity","updated_at":"2026-07-28T20:01:45Z","labels":["area:storage","horizon:frontier"],"dependencies":[{"issue_id":"polylogue-cuxz.4","depends_on_id":"polylogue-cuxz","type":"parent-child","created_at":"2026-07-28T22:01:45Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-4ts.10","title":"session_links.status and .method are NULL on every row: TopologyEdgeStatus is declared but never written","description":"Measured on the live archive 2026-07-28 (index v43, 18,871 sessions):\n\n SELECT count(*), sum(status IS NULL OR status=''), sum(method IS NULL OR method=''),\n sum(resolved_dst_session_id IS NULL) FROM session_links;\n -\u003e 9179 | 9179 | 9179 | 222\n\nEvery one of the 9,179 topology edges has empty status and empty method. TopologyEdgeStatus (unresolved/resolved/repaired/quarantined) is a declared vocabulary with no writer, so a reader cannot distinguish a resolved parent from an asserted-but-absent one except by the weaker proxy resolved_dst_session_id IS NULL (222 rows).\n\nLink-type distribution: subagent 8,824 | continuation 308 | sidechain 31 | branch 16.\n\nConsequences: resume/continuity composition can compose from an unverified parent reference with no typed signal; polylogue-xl25's 'quarantined' BlockAnchorState has no source to read; any lineage-integrity claim rests on a column that is uniformly empty.","acceptance_criteria":"1. Every session_links row written by resolve_session_links_for_session carries a TopologyEdgeStatus value and a method token; no code path writes an empty status. 2. Existing rows acquire status through ordinary derived-tier rebuild, not a bespoke backfill script. 3. A reader can filter edges by status, and composition refuses (or degrades visibly) on a non-resolved parent rather than silently composing. 4. Live re-measure shows zero empty status/method rows and a status distribution consistent with the 222 unresolved-destination rows.","status":"open","priority":2,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-28T20:01:17Z","created_by":"Sinity","updated_at":"2026-07-28T20:01:17Z","labels":["area:lineage","delivery:F-lineage-compaction","horizon:frontier","lane:lineage-compaction"],"dependencies":[{"issue_id":"polylogue-4ts.10","depends_on_id":"polylogue-4ts","type":"parent-child","created_at":"2026-07-28T22:01:17Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":1,"comment_count":0} {"_type":"issue","id":"polylogue-l2cd","title":"Migrate paths/_roots.py duplicate ArchiveLocation resolver call sites","description":"Follow-up from polylogue-ovme.2.1 (PR #3382): devtools/verify_archive_resolver_completeness.py now inventories every call site of polylogue/paths/_roots.py's four ArchiveLocation-duplicating resolvers (active_index_db_path: 43 call sites, resolve_active_index_db_path: 7, sibling_index_db: 16, archive_file_set_root_for_paths: 27 -- 93 total) and prevents growth beyond a recorded baseline, but none of the 93 existing call sites were migrated -- that was explicitly deferred as too large/risky for one session. This bead tracks the actual migration: retire each resolver's call sites in favor of the equivalent ArchiveLocation accessor (active_index_path / configured_tier / active_tier), one resolver/file-batch at a time (largest call-site count last per polylogue-ovme.2.1's own design note), shrinking BASELINE_CALL_SITES in devtools/verify_archive_resolver_completeness.py as each batch lands, verified per-batch with devtools test on the affected directory plus mypy --strict. Full removal ultimately allows deleting the four resolver functions from polylogue/paths/_roots.py entirely.","notes":"Resolver 1/4 (resolve_active_index_db_path, 7 call sites) fully migrated\nand PR opened: https://github.com/Sinity/polylogue/pull/3385\n(branch feature/refactor/migrate-resolve-active-index-db-path).\n\nWhat landed:\n- All 3 production call sites (polylogue/cli/click_app.py,\n polylogue/daemon/health.py, polylogue/daemon/status.py) now call\n polylogue.storage.archive_identity.resolve_active_index_path(archive_root())\n instead of resolve_active_index_db_path(db_anchor=db_path(), index_db=index_db_path()).\n Confirmed every real call site passed archive_root()-derived values for\n BOTH db_anchor and index_db, so the function's \"explicit override\" branch\n was never reachable in production -- only in test mocks.\n- Function deleted from polylogue/paths/_roots.py and polylogue/paths/__init__.py.\n- devtools/verify_archive_resolver_completeness.py: resolve_active_index_db_path\n entry removed from BASELINE_CALL_SITES entirely (0 call sites remain).\n Remaining baseline: active_index_db_path=43, sibling_index_db=16,\n archive_file_set_root_for_paths=27 (unchanged, 86 total).\n- 4 test files' mocks updated from patching db_path/index_db_path to\n patching the wrapper functions (_active_status_db_path / _active_health_db_path)\n directly; 3 tests in test_config.py that tested the retired db_anchor-override\n contract in isolation were deleted (no longer a reachable scenario); one new\n anti-vacuity test added in tests/unit/storage/test_archive_identity.py\n (malformed-pointer rejection via ArchiveLocation.resolve, since the deleted\n resolver's own coverage of that case was gone).\n- Verification: mypy --strict polylogue/ clean (1091 files); ruff check/format\n clean; devtools test across tests/unit/daemon + affected core/cli/storage\n tests: 2105 passed, 4 failed, all 4 confirmed pre-existing via git stash\n (test_paths_root_exports_only_directory_layout_symbols,\n test_medium_tier_inventory_pinned, test_no_token_in_log_or_print_calls,\n test_rebuild_index_handler_forwards_resumable_pass_options_through_writer_bridge);\n devtools render all --check clean after devtools render devtools-reference.\n\nRemaining for next pass (not started this session):\n- sibling_index_db (16 call sites across 14 files: cli/commands/embed.py,\n cli/commands/status.py, daemon/convergence_stages.py,\n daemon/embedding_backlog.py, daemon/embedding_readiness.py,\n daemon/fts_status.py, daemon/metrics.py, daemon/provenance.py,\n daemon/similarity.py, daemon/status.py x3, sources/live/hook_paste_enrichment.py,\n storage/blob_publication.py, storage/embeddings/preflight.py,\n storage/embeddings/status_payload.py). Surveyed but NOT migrated: several\n call sites (e.g. cli/commands/embed.py:_active_archive_index_path) build a\n candidate list combining sibling_index_db(db_path) with an archive_root()\n fallback and defensive existence checks -- migrating these correctly\n requires reading each call site's local fallback/existence semantics\n carefully (not a mechanical rename like resolver 1 was), since\n sibling_index_db's require_exists=True/False parameter is doing real\n work at several sites that ArchiveLocation's active_tier() doesn't\n directly replicate (it doesn't itself check existence). Do this batch\n fully or not at all per the no-partial-migration rule -- do not start\n converting individual call sites without a plan for all 16.\n- archive_file_set_root_for_paths (27 call sites) -- not surveyed this\n session.\n- active_index_db_path (43 call sites, largest) -- not surveyed this\n session, do last per original design note.\nResolver 3/4 (archive_file_set_root_for_paths, 24 production call sites across 18 files) fully migrated and merged: PR #3387 (branch feature/refactor/migrate-archive-file-set-root-for-paths), squash-merged 2026-07-28T23:00:55Z.\n\nWhat landed:\n- polylogue/paths/_roots.py: archive_file_set_root_for_paths deleted; export removed from paths/__init__.py.\n- devtools/verify_archive_resolver_completeness.py: BASELINE_CALL_SITES now only tracks active_index_db_path (43 sites) -- the sole remaining resolver.\n- New polylogue/storage/archive_identity.archive_file_set_root(archive_root, db_path): plain-path helper (not a Config method) replicating the retired resolver conditional (db_path.parent when db_path names index.db, else archive_root). Kept as a duck-typed free function -- not a Config property -- specifically so SimpleNamespace/MagicMock config doubles used across tests/unit/mcp/ keep working without edits.\n- ~15 config.db_path-based call sites (api/archive.py, archive/query/{plan,spec,search_hits}.py, cli/archive_query.py, cli/click_app.py, cli/commands/status.py, cli/read_views/{chronicle,standard}.py, cli/select.py, cli/verb_cardinality.py, mcp/archive_support.py, storage/repair.py, storage/raw_reconciler.py) now call archive_file_set_root() instead of deriving via ArchiveLocation.resolve(config.archive_root).configured_root.\n- Bare-function call sites with no override capability (3 maintenance CLI files, mcp/archive_support.py index-existence check) correctly use ArchiveLocation.resolve(archive_root()).active_index_path/.configured_root instead, since bare db_path()/archive_root() free functions are NOT pointer-aware.\n\nImportant correctness finding from self-review (a dispatched worktree agent did the initial migration; I caught and fixed this before merging): the agent classified most config.db_path sites as ArchiveLocation.resolve(config.archive_root).configured_root, which silently ignores Config's supported split-root override (polylogue-yla8.1 -- an explicit Config(db_path=...) can point at an entirely separate self-contained archive, used by the public Polylogue(archive_root=..., db_path=...) API convenience). This broke 2 tests that pass on master (test_archive_tiers_facade_reads_active_db_override_root, test_archive_tiers_semantic_query_uses_active_root_embeddings_db) and would have broken production callers of that override. Fixed by adding archive_file_set_root() as described above rather than blindly trusting ArchiveLocation for these sites.\n\nVerification: mypy --strict clean (1091 files); ruff check/format clean; devtools test across tests/unit/{api,cli,archive,maintenance,mcp,storage,core,devtools}: 26 failures, all a strict subset of the 27 pre-existing failures already on master (confirmed via direct side-by-side comparison, not assumed) -- zero new regressions; devtools verify --quick exit 0; resolver-completeness check reports 0 unbaselined sites.\n\nRemaining for next pass (not started): active_index_db_path (43 call sites, largest, do last per original design note).\nResolver 4/4 (active_index_db_path, 45 production call sites across 21 files -- 43 originally scoped + 2 more found in devtools/daemon_workload_probe.py and devtools/pipeline_probe/engine.py, plus 2 aliased occurrences in devtools/self_verify.py and devtools/archive_space_report.py) fully migrated and merged: PR #3388 (branch feature/refactor/migrate-active-index-db-path), squash-merged 2026-07-29T00:24:51Z.\n\nWhat landed:\n- polylogue/paths/_roots.py: active_index_db_path() deleted; export removed from paths/__init__.py.\n- All 45 call sites now call polylogue.storage.archive_identity.resolve_active_index_path(archive_root()) directly -- this resolver was NOT itself buggy (it already correctly followed the .index-active-pointer file inline), so migration was pure deduplication onto the canonical implementation, which additionally detects/warns on the \"shadow index\" divergence case the duplicate inline logic missed.\n- devtools/verify_archive_resolver_completeness.py: BASELINE_CALL_SITES now EMPTY -- all four originally-named duplicate resolvers (resolve_active_index_db_path #3385, sibling_index_db #3386, archive_file_set_root_for_paths #3387, active_index_db_path #3388) fully migrated and deleted.\n- Since the completeness lint greps for call sites of four function names that no longer exist anywhere in the codebase, it could structurally never fire again. Per this repo standing policy against completeness-check-theater, I deleted the module (devtools/verify_archive_resolver_completeness.py), its test file, and its command_catalog.py/verify.py wiring, and regenerated docs/devtools.md -- rather than leaving a permanently-inert check in the verify --lab step list.\n\nVerification: mypy --strict clean (polylogue/ 1091 files + devtools/ 173 files); ruff check/format clean; devtools test across tests/unit/{cli,storage,core,coordination,insights,maintenance,devtools,daemon}: 74 failures on the branch vs 75 on master with the identical command -- exact test-name diff confirmed zero new regressions (the 1 discrepancy is a known containment-flake, test_runtime_health_with_readonly_archive_root, reproduced failing on master too); devtools verify --quick exit 0.\n\nFollow-up debt noted but NOT fixed here (out of scope -- this resolver was pure dedup, not a bug fix): several call sites do `resolve_active_index_path(archive_root()).with_name(\"ops.db\")` (daemon/http.py x2, daemon/events.py, daemon/lifecycle.py) or `.parent` (storage/repair.py:repair_session_insights) to derive a DIFFERENT tier/root from the active index path -- this is the same \"index-only external generation\" bug class fixed in the sibling_index_db/archive_file_set_root_for_paths batches, but it PRE-EXISTED this migration unchanged (active_index_db_path and resolve_active_index_path are behaviorally identical, so this is a faithful 1:1 preservation, not a regression). Worth a dedicated follow-up bead if the pointer-external-generation mechanism is ever exercised against these specific daemon paths in production.\n\nALL FOUR RESOLVERS NOW FULLY MIGRATED. Closing this bead.","status":"closed","priority":2,"issue_type":"task","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-07-28T19:34:58Z","created_by":"Sinity","updated_at":"2026-07-29T00:26:11Z","started_at":"2026-07-28T20:11:44Z","closed_at":"2026-07-29T00:26:11Z","close_reason":"All four ArchiveLocation-duplicating resolvers (resolve_active_index_db_path, sibling_index_db, archive_file_set_root_for_paths, active_index_db_path) fully migrated across 4 PRs (#3385, #3386, #3387, #3388), all merged. Completeness lint (devtools/verify_archive_resolver_completeness.py) deleted as permanently-inert once its baseline emptied. Follow-up debt (ops.db/other-tier derivation via .with_name()/.parent on the active index path at ~5 pre-existing daemon call sites) noted in bead notes but not filed as a separate bead -- low-probability edge case (requires an active external index generation), can be filed on discovery if it manifests.","dependency_count":0,"dependent_count":0,"comment_count":0} @@ -815,6 +845,9 @@ {"_type":"issue","id":"polylogue-rii.1","title":"Agent work-event write-leg -\u003e session_events -\u003e materialized read-models","description":"record_work_event/emit_decision write surface routed through the existing idempotent ingest seam (no parallel writer); flows into the run-projection read models. Today agents can only record_correction/blackboard_post/tag — there is no 'I ran this tool / spawned this subagent / decided X' write. GH issue thread (body + comments) is input, not authority; this bead's scope statement wins where they conflict.","design":"Route through the existing idempotent ingest seam (write_raw_and_parsed / the daemon ingest path) — no parallel writer (gh#2459 body is code-grounded here). Surface: MCP tools record_work_event/emit_decision (mutation role) accepting typed events (tool run, subagent spawn, decision, artifact change) with evidence/session refs; land in session_events; run-projection read models pick them up through the normal materializer. MCP registration trap: EXPECTED_TOOL_NAMES + TOOL_CONTRACT + role gating + render openapi/cli-output-schemas regen (see bd memories). Acceptance: an agent posts a work event mid-session; it is queryable via observed-events within one convergence cycle; re-posting is idempotent.","acceptance_criteria":"- MCP tools record_work_event / emit_decision are registered with the mutation role: EXPECTED_TOOL_NAMES + TOOL_CONTRACT updated, role gating enforced, and `devtools render openapi \u0026\u0026 devtools render cli-output-schemas` regenerated with `devtools render all --check` clean.\n- Typed events (tool run, subagent spawn, decision, artifact change) with evidence/session refs route through the existing idempotent ingest seam (write_raw_and_parsed / the daemon ingest path) into session_events — no parallel writer (grep confirms reuse).\n- Behavior test: an agent posts a work event mid-session and it is queryable via observed-events (session_work_events / DSL) within one convergence cycle; re-posting the same event is idempotent (no duplicate row). `devtools test \u003cmcp work-event test\u003e` green.","notes":"[Delivery upgrade 2026-07-07T00:05:00Z] Release=D-agent-context-coordination; lane=agent-coordination; readiness=A-implementation-ready; proof=two-agent separate-worktree proof with before/after coordination envelopes. Original readiness=A-implementation-ready.\n[Prework packet 2026-07-07] Static execution packet (anchors, mechanism, plan, tests, verification): .agent/handoffs/polylogue-gpt-pro-2026-07-07/prework-v2/task_packets/071_polylogue_rii_1.md (depth: bead-localized-from-export; urgency: T2-foundation-before-feature-proof). Generated from master @ 8a975a40 2026-07-06 — verify source anchors before coding; line numbers are snapshot-relative.\nRECONCILED 2026-07-13 with 37t.2 inline protocol: the agent work-event write-leg and the marker channel are ONE channel with two encodings (structured MCP writes; prose markers extracted at enrichment). Unify vocabularies — work-event kinds and marker kinds must share the registry (a ::phase marker IS a work event). Do not build parallel event taxonomies.","status":"open","priority":2,"issue_type":"feature","owner":"ezo.dev@gmail.com","created_at":"2026-07-03T04:31:43Z","created_by":"Sinity","updated_at":"2026-07-13T04:00:08Z","external_ref":"gh-2459","labels":["area:substrate","delivery:D-agent-context-coordination","horizon:frontier","lane:agent-coordination"],"dependencies":[{"issue_id":"polylogue-rii.1","depends_on_id":"polylogue-rii","type":"parent-child","created_at":"2026-07-03T06:31:43Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":1,"comment_count":0} {"_type":"issue","id":"polylogue-fs1.3","title":"Per-source coverage/fidelity declaration for Hermes imports","description":"Every Hermes acquisition tier and schema version needs a machine-readable fidelity declaration that distinguishes what is exact, absent, redacted, degraded, or inferred. The declaration is the guard against a parser test going green while silently dropping forensic history or cost/addressing provenance.","design":"Extend the OriginSpec/fidelity surface with: producer/schema version; installation/profile namespace; acquisition method (sqlite_backup, stable export, JSON fallback, runtime spans); exact retained-blob-to-normalized reproducibility verdict; counts and coverage for active, rewound, compacted, and observed messages; addressing/material-origin semantics; actual/estimated cost with status/source/pricing/billing provenance; lifecycle/relationship coverage; runtime-span coverage and explicit missingness. The snapshot and span lanes may enrich one logical session revision only with per-field provenance; they may not double-count or silently prefer a lower-fidelity tier.","acceptance_criteria":"explain-import on Hermes v16, a later schema, JSON fallback, and a spans-plus-snapshot merge names every capability as exact, absent, redacted, degraded, or inferred; exact-blob reproducibility is stated and verified; the same logical session from two tiers remains one revision with field-level provenance; message-state/addressing and cost-provenance counts reconcile to fixtures; deliberately dropping observed mapping, cost provenance, snapshot proof, or an unpaired span changes the declared fidelity and surfaces a downstream forensics caveat. OriginSpec fixtures and mutation-style negative tests pass.","notes":"[Delivery upgrade 2026-07-07T00:05:00Z] Release=K-interop-origin-export; lane=origin-interop-export; readiness=D-horizon-ready; proof=OriginSpec detector/parser/fixture/fidelity suite and content-hash export/import roundtrip. Original readiness=E-spec-needed.\n2026-07-12 fanout lane finding: blocked as scoped — explain-import cannot inspect SQLite Hermes state DBs and its payload lacks a fidelity-declaration field; both surfaces (import_explain.py + payload schema) must be in scope to implement. Evidence: 37bdfa04c; import_explain.py decodes JSON/JSONL only.","status":"closed","priority":2,"issue_type":"feature","owner":"ezo.dev@gmail.com","created_at":"2026-07-03T04:31:40Z","created_by":"Sinity","updated_at":"2026-07-12T23:15:18Z","closed_at":"2026-07-12T23:15:18Z","close_reason":"PR #2789 merged: Hermes per-source coverage/fidelity declaration shipped (import_explain.py, hermes_state.py, generated CLI-output schema regenerated)","labels":["area:ingest","area:substrate","delivery:K-interop-origin-export","delivery:ac-patched","horizon:frontier","lane:origin-interop-export"],"dependencies":[{"issue_id":"polylogue-fs1.3","depends_on_id":"polylogue-fs1","type":"parent-child","created_at":"2026-07-03T06:31:40Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":3,"comment_count":0} {"_type":"issue","id":"polylogue-tf2.2","title":"Fold agent_forensics.py into polylogue analyze","description":"~70% already materialized (cost_rollups, archive_coverage, total_credit_cost, portfolio, cost_outlook). Real gaps: reasoning-token lane on SessionProfile; usage_timeline archive insight (tokens/cost per month per model) registered in insights/registry.py; optional markdown forensics renderer. Drop the script's hand-rolled _CREDIT_RATES; delete the script. Sequenced AFTER the campaign regen (the campaign uses the script one last time). GH issue thread (body + comments) is input, not authority; this bead's scope statement wins where they conflict.","status":"closed","priority":2,"issue_type":"feature","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-07-03T04:31:34Z","created_by":"Sinity","updated_at":"2026-07-03T11:54:39Z","started_at":"2026-07-03T11:31:18Z","closed_at":"2026-07-03T11:54:39Z","close_reason":"Completed: usage forensics is no longer a standalone script surface. Added registered usage_timeline archive insight with CLI/API/MCP registry coverage, reused the shared subscription-pricing catalog for credit estimates, deleted scripts/agent_forensics.py and its private-helper tests, and rewrote README/docs around polylogue analyze insights coverage/cost-rollups/usage-timeline plus devtools workspace claim-vs-evidence. Verification: focused claim-vs-evidence/insights tests passed, render all --check passed, devtools verify --quick passed, and live active-archive usage-timeline smoke returned valid JSON. Follow-up polylogue-5nn tracks the observed 18s whole-archive aggregation latency for unfiltered month-origin-model usage-timeline.","external_ref":"gh-2480","labels":["area:usage","campaign"],"dependencies":[{"issue_id":"polylogue-tf2.2","depends_on_id":"polylogue-tf2","type":"parent-child","created_at":"2026-07-03T06:31:34Z","created_by":"Sinity","metadata":"{}"},{"issue_id":"polylogue-tf2.2","depends_on_id":"polylogue-tf2.1","type":"blocks","created_at":"2026-07-03T06:31:34Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-iv3v","title":"Verify grok.py export field coverage against a real xAI GDPR export (unverified, no sample corpus available)","description":"Surfaced during the 2026-07-31 heuristics/discard-site audit as a low-confidence, UNVERIFIED lead -- filed as a follow-up investigation, not a confirmed finding, per the audit's evidence discipline.\n\npolylogue/sources/parsers/grok.py (178 lines) extracts only conversation.title, create_time, and per-response sender/message/create_time (grok.py:122-171). It documents itself as reverse-engineered from three third-party sources (a GitHub viewer, a blog post, a userscript) because 'no official xAI schema publication exists' (grok.py:1-38), and asserts the export has 'no native conversation id or attachment/image data.'\n\nThis audit could NOT verify that claim either way: no real Grok GDPR export exists under /realm/data/exports/chatlog, /realm/data/exports, or elsewhere searched (checked at audit time, 2026-07-31). The only grok-adjacent artifact found is a browser-capture DOM dump (/realm/inbox/polylogue-browser-spool-2026-07-10/grok/dom-e4e24461-4b1f7d02f3c4.json), which is a different capture path (live DOM scrape, not the GDPR export grok.py parses) and cannot substitute.\n\nEvery other provider audited this session (Claude Code via polylogue-pbuh/cgfy, ChatGPT, Codex, Hermes) turned out to have MORE typed fields in the real wire format than the parser initially read -- structuredPatch, patch_apply changes, reasoning traces, thread titles. Given that pattern, grok.py's self-reported 'no attachments, no conversation id' claim deserves the same corpus-diff treatment cgfy applied to Claude Code, but doing so requires acquiring one real xAI GDPR export first.","acceptance_criteria":"1. Acquire (or obtain from the operator) one real xAI/Grok GDPR export. 2. Run cgfy's key-enumeration method: list every top-level/response/message key present in the real export, diff against what grok.py currently reads. 3. Classify each unread key as read / deliberately-dropped-with-reason / to-acquire, same as cgfy's disposition table. 4. If grok.py's self-reported field coverage turns out accurate, close as verified-clean; if gaps are found, file follow-up beads per gap with corpus counts.","status":"open","priority":3,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-31T04:33:13Z","created_by":"Sinity","updated_at":"2026-07-31T04:33:13Z","labels":["area:ingest","lane:origin-interop-export"],"dependencies":[{"issue_id":"polylogue-iv3v","depends_on_id":"polylogue-cgfy","type":"related","created_at":"2026-07-31T06:33:13Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-mgf6","title":"Query DSL: float-literal numeric predicates for JSON-extracted fields (run_settings temperature/topP)","description":"Follow-up from polylogue-o4j2 (AC2, deferred).\n\naistudio-drive's runSettings (temperature/topP/topK/maxOutputTokens/\nthinkingLevel/safetySettings/enable* flags) is parsed and stored verbatim as\nsessions.run_settings_json (polylogue-2qx.4/cgfy, index v46). It is not\nexposed to the query DSL, so \"sessions where temperature \u003e 0.5\" is not\nexpressible. Two independent gaps block it:\n\n1. Grammar: the boolean-query numeric-comparison rule only accepts integer\n literals (`COUNT_FIELD COMP_OP INT` in archive/query/expression.py) --\n temperature/topP are floats (0.0-2.0 / 0.0-1.0 range).\n2. SQL builder: NUMERIC_QUERY_FIELD_REGISTRY's NumericQueryFieldInfo.unit_columns\n values are treated as plain column names (`f\"{table_alias}.{column}\"` in\n storage/sqlite/archive_tiers/archive.py, two call sites) -- there is no\n path for a computed/JSON-extract expression like\n `json_extract(run_settings_json, '$.temperature')`.\n\nScope: extend the grammar to accept decimal literals for numeric predicates\n(without breaking existing integer-only fields), and extend the SQL-builder\ncall sites (and NumericQueryFieldInfo, if needed) to support an expression\ncolumn alongside plain columns. Consider starting with the integer-typed\nrun_settings fields (topK, maxOutputTokens) which fit the existing INT-only\ngrammar and only need the SQL-builder JSON-extract half, then float support\n(temperature, topP) as a second phase needing the grammar change too.\n\nNot urgent: run_settings is durably stored and readable via `read --view`\nalready; this is about ergonomic filtering, not data loss.","status":"open","priority":3,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-31T04:03:16Z","created_by":"Sinity","updated_at":"2026-07-31T04:03:16Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-je9t","title":"7 of 95 design-chat messages dropped by _parse_design_chat","description":"Measured _parse_design_chat directly against all 11 design_chats/*.json in claude-ai-data-2026-07-30-16-36-batch-0000.zip: 95 source messages -\u003e 88 parsed. Loss is concentrated in two files (9-\u003e6 and 20-\u003e16); the other nine are lossless.\n\nNot yet diagnosed - candidates are role values the mapper does not recognise, or content shapes _design_content_payload returns {} for.\n\nAC: either all 95 parse, or the dropped shapes are identified and dropping them is shown to be correct (with the reason recorded here).","status":"open","priority":3,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T22:05:41Z","created_by":"Sinity","updated_at":"2026-07-30T22:05:41Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-inoh","title":"Ambiguous-cohort census residue: claude-code-session/hermes-session/grok-export/unknown-export not root-caused","description":"## Context\n\nFollow-up from the cross-origin ambiguous-cohort census (polylogue-c429,\npolylogue-hith, polylogue-nuec; all descend from polylogue-bu1i). That\ninvestigation focused on the three largest equal-message-count ambiguous\npopulations (claude-ai-export 566, chatgpt-export 129, aistudio-drive 151 --\nthe last already proven by polylogue-bu1i). The remaining origins are small\nand were sampled but not root-caused to the same depth; this bead tracks\nthat residue so it doesn't disappear as anonymous debt.\n\n## claude-code-session (6 of 191 equal-message-count ambiguous cohorts)\n\nExplicitly flagged low-priority/different-shape by the parent investigation.\nSampled all 6 with production `parse_payload`/`parse_stream_payload`\n(claude-code-session raws are stream-record JSONL, routed via\n`is_stream_record_provider` + `parse_stream_payload`, unlike the other three\norigins' single-document JSON). 5 of 6 collapsed to a SINGLE distinct\nblob_hash content group when grouped by raw blob bytes -- i.e. under a fresh\nparse, the members recorded 'ambiguous' in `raw_session_memberships` are\nbyte-identical to each other, which should not classify ambiguous at all\nunder current `classify_membership_revisions` logic (a single by-content\ngroup never reaches the dominance-failure branch). This suggests either (a)\nthese decisions are stale relative to the current member set (see\npolylogue-9dxn's general \"persisted ambiguous verdicts never get\nre-derived\" finding -- may be the same root mechanism, not independently\nconfirmed here), or (b) a raw sibling with genuinely different content was\nremoved (GC/retention) since the decision was recorded, or (c) a\nmethodology gap in the reproduction script not caught in this pass. Not\ndisambiguated -- would need dedicated investigation with access to\n`raw_revision_heads`/retention history for these specific\n`logical_source_key`s, which the parent investigation's read-only harness\ndid not attempt.\n\n## hermes-session (3 of 4 equal-message-count ambiguous cohorts)\n\nSampled all 3. 2 have identical message id set/order/attachment keys with\nsingle-message (`n_messages=1`) conversations -- the actual delta wasn't\nisolated (didn't check `session_events`/text content at the level of detail\nused for claude-ai-export/chatgpt-export given the tiny population). 1\nraised a parse-routing error in the census harness (hermes has a\nSQLite-backed raw path -- `looks_like_sqlite_bytes` /\n`hermes_state.parse_state_db` / `hermes_verification.parse_verification_evidence_db`\nin `polylogue/sources/revision_backfill.py:_parse_one` -- that the harness's\ngeneric `parse_payload` call doesn't handle; this is a harness gap, not\nevidence of a real defect).\n\n## grok-export (1 of 1 -- full population)\n\nThe one ambiguous grok-export cohort (`grok:dom:815e0a1c`) is a\nbrowser-capture DOM snapshot with genuinely DIFFERENT message id sets at\nequal count across its two distinct-content revisions -- this looks like a\nreal content divergence (re-captured page state), not a misclassification\nartifact. Tentatively bucket as GENUINELY AMBIGUOUS, not investigated\nfurther given n=1. Note as an aside: one of its four raw rows'\n`source_path` points at\n`/realm/project/polylogue/.cache/dev-loop/feature-docs-accuracy-revamp-*`,\ni.e. a development/test-fixture path, not a personal capture location --\nworth a separate look at whether stale dev-loop fixtures leaked into the\nlive archive, but out of scope here.\n\n## unknown-export (2 of 3 equal-message-count ambiguous cohorts)\n\nSampled 2 of 2. Both collapsed to a single distinct blob_hash content group,\nsame shape as the claude-code-session finding above. Given `unknown-export`\nis itself a fallback/unclassified bucket, not investigated further.\n\n## Acceptance criteria\n\n- Either resolve each sub-population's cause with the same rigor as\n polylogue-c429/hith/nuec (parse both distinct-content sides, run the\n production classifier, characterize the minimal delta), or explicitly\n downgrade/close this bead with the reason each population is too small to\n be worth the investigation cost, stated per-origin.\n- If the claude-code-session/unknown-export \"single distinct content group\"\n pattern is confirmed to be the polylogue-9dxn stale-verdict mechanism\n rather than a new defect, cross-link and close this portion as\n subsumed by 9dxn's fix rather than re-deriving a new root cause.\n\nRef polylogue-bu1i\nRef polylogue-c429\nRef polylogue-hith\nRef polylogue-nuec\nRef polylogue-9dxn\n","status":"open","priority":3,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-30T12:18:20Z","created_by":"Sinity","updated_at":"2026-07-30T12:18:33Z","labels":["area:ingest"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-uyci","title":"Expose sessions.display_name/run_settings_json and session_links.parent_tool_use_block_id on a public surface","description":"Feature-gap sweep finding (2026-07-29, feature/chore/promote-schemas-and-wire-gates\n@ bdeb6d1d2). Three columns are written and readable in SQL but never reach any\ndomain model, so no surface (CLI/MCP/API) can answer for them at all:\n\n1. sessions.display_name -- polylogue/storage/runtime/archive/records.py\n (SessionRecord.display_name) and sessions_reads.py read it, but\n archive/session/domain_models.py::Session/SessionSummary has no\n `display_name` field, so hydrators.py drops it silently when building the\n domain model. Subagent-slug/session display metadata that's stored is\n currently unreachable end-to-end.\n2. sessions.run_settings_json -- same shape: SessionRecord.run_settings is\n read (Drive/Gemini run-settings verbatim JSON, model name etc.) but the\n Session domain model has no field for it either.\n3. session_links.parent_tool_use_block_id -- modeled on\n archive/topology/edge.py::TopologyEdge.parent_tool_use_block_id (the real\n delegation join key, replacing prior best-effort inference), populated by\n storage/sqlite/archive_tiers/write.py, but grep finds zero CLI/MCP/insights\n consumers of TopologyEdge.parent_tool_use_block_id -- the topology surface\n (`read --view` / MCP topology tool) cannot yet answer \"which exact tool_use\n call spawned this subagent session\" even though the join key is stored.\n\nNone of these need a schema change (all already exist on schema v46). Each is\na small, mechanical field-add to a domain model + hydrator + one surface\n(topology reader for #3; Session model + relevant CLI/MCP session payload for\n#1/#2) -- similar shape to the stop_reason fix landed alongside this bead.\nScope each separately since they touch different domain models (Session vs\nTopologyEdge) and different surfaces.","status":"open","priority":3,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-29T18:40:07Z","created_by":"Sinity","updated_at":"2026-07-29T18:40:07Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-zahj","title":"Operator decision: 2 stuck blob-publication reservations (42.5MB) pin unreferenced blobs","description":"Blob-store audit found 2 rows in source.db's blob_publication_reservations that have been 'unresolved' (blob present on disk, not referenced by raw_sessions/blob_refs/index.db attachments) since reservation, with no automatic path to clear them:\n publication_id=f1c44ec2-4250-4f12-87d3-97412cd08144 blob_hash=a8387c87ff8550f69330e30f1ea581e86e3e61fad590b5f8e33bcfe21a53d1a2 size=16021146B reserved_at=2026-07-12 14:03 UTC\n publication_id=0d21f742-779e-49e3-b96f-c8f1abeecc59 blob_hash=bad5d59e51a8b9b509c59e58f21f8afb0e8b7c7fbfe95cc6fc922bfbe7ead83d size=26470839B reserved_at=2026-07-13 10:45 UTC\nBoth blobs total 42.5MB and are on disk at blob/a8/387c... and blob/ba/d5d5.... They are the ONLY 2 truly-orphaned blobs in the entire 69GB/100K-object store (everything else that looked orphaned from source.db alone is still legitimately referenced via index.db's attachments table -- confirmed the store IS correctly content-addressed/deduplicated, no other waste found).\nVerify with: polylogue ops maintenance blob-publications (lists all receipts with referenced/present state), then if the operator confirms these two acquisitions were genuinely superseded/abandoned (not an in-flight publisher), release them with:\npolylogue ops maintenance blob-publications --abandon f1c44ec2-4250-4f12-87d3-97412cd08144 --abandon 0d21f742-779e-49e3-b96f-c8f1abeecc59 --yes\nThis only removes the RESERVATION (the protection), not the blob itself -- the next blob-gc pass would then be free to consider deleting the underlying blob bytes if truly unreferenced. Not doing this myself: blob deletion is explicitly the highest-risk operation in this system (evidence loss unrecoverable) and this decision needs operator judgment on whether the July 2026 acquisitions these reservations protected are safe to release.","status":"open","priority":3,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-29T08:44:55Z","created_by":"Sinity","updated_at":"2026-07-29T08:44:55Z","dependency_count":0,"dependent_count":0,"comment_count":0}