diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 709dfc3386..b7c4889d30 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -909,7 +909,7 @@ {"_type":"issue","id":"polylogue-yp5p","title":"three write-only tables (otlp_telemetry, query_runs, session_commits) and 7 unconsumed config properties","design":"Found 2026-07-29 by codebase audit (same detection that found the orphaned\n`session_agent_policies` reader earlier this session -- that one is now fixed,\nthese three are not).\n\nTables written by production code with NO production reader anywhere -- no\nSELECT/FROM/JOIN outside tests:\n\n otlp_telemetry writer polylogue/daemon/otlp_receiver.py:140\n prod reads 0 | test reads 2\n Receives and durably stores OTLP spans/metrics/logs.\n Nothing ever queries them back out.\n\n query_runs writer polylogue/storage/sqlite/archive_tiers/ops_write.py:503\n prod reads 0 | test reads 4\n Records every query execution. No surface reads it, so\n the query-history it accumulates is unreachable -- note\n `slow_query_notice_seconds` is also an unconsumed config\n property (see below), suggesting a query-observability\n feature that was half-built.\n\n session_commits writer polylogue/storage/sqlite/archive_tiers/write.py:3839\n prod reads 0 | test reads 1\n Git commit attribution per session. Adjacent tables\n (repos, session_repos) ARE read; this one is not.\n\nEach needs a disposition, not a default: wire a reader (the evidence is being\ncollected and is simply unreachable -- this was the right answer for\nsession_agent_policies), or delete the table and its writer (nothing needs the\nevidence, and writing it costs rebuild time and disk on every ingest).\n\n`otlp_telemetry` and `query_runs` are in the disposable ops tier, so deleting\nthem is cheap. `session_commits` is in the rebuildable index tier and its\nsibling tables are live, so it most likely wants a reader.\n\nALSO: 7 config properties with zero production consumers\n(polylogue/config.py) -- each is either an unwired feature or dead:\n active_index_db prod=0 test=0\n hook_sidecar_dir prod=0 test=0\n ingest_parse_workers prod=0 test=0 <- notable: parse worker count,\n relevant to the imminent rebuild; verify the\n rebuild is not silently ignoring it\n log_level prod=0 test=0\n slow_query_notice_seconds prod=0 test=0 <- pairs with query_runs above\n effective_path prod=0 test=2\n layer_paths prod=0 test=2\n\n`ingest_parse_workers` is the one to check FIRST and before the rebuild: if the\nparse-worker count is configurable but unread, the rebuild may not be honoring\nit. (Detection is name-based; confirm each against source before acting --\na property could be reached via getattr or config-inventory reflection.)\n","status":"closed","priority":2,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-29T10:34:19Z","created_by":"Sinity","updated_at":"2026-08-03T22:09:54Z","closed_at":"2026-08-03T22:09:54Z","close_reason":"Merged via PR #3694: dropped write-only query_runs table (ops.db) + record_query_run writer + dead active_index_db config alias. Re-verified against present source before deleting: otlp_telemetry already removed by #3665, session_commits gained a real reader via cijx.3 (kept), 5 of the 7 named config properties were false positives on closer read (real consumers exist), 2 already removed by #3455. devtools test 87+332+28 passed across touched modules; devtools verify --quick exit 0. Found otlp_spans (source.db, durable tier) now fully dead (zero writer since #3665, zero reader) -- filed as follow-up polylogue- since durable-tier table drops need the copy-forward/consent gate, not done here.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-j1vs","title":"three parallel render-destination vocabularies; RenderFormat and alias map are inert","design":"Found 2026-07-29 by codebase audit. THREE parallel vocabularies describe the\nsame concept (where rendered output goes), and none is authoritative:\n\n 1. polylogue/surfaces/projection_spec.py:58 RenderDestination enum\n TERMINAL, STDOUT, BROWSER, CLIPBOARD, FILE (typed, validated)\n 2. polylogue/cli/query_verbs.py:215 _READ_DESTINATIONS\n (\"terminal\",\"stdout\",\"browser\",\"clipboard\",\"file\") (raw strings,\n duplicated literally; this is what click.Choice validates against)\n 3. polylogue/cli/query_contracts.py:23 QueryDeliveryName\n Literal[\"stdout\",\"browser\",\"clipboard\"] (only 3 of the 5)\n\nThe typed enum exists but the actual dispatch sites compare raw strings\n(read_views/base.py:158, read_views/standard.py:97) and the invocation field\nis plain `destination: str` (read_views/base.py:99). So the enum validates\nnothing on the path that matters. polylogue-bvnz (browser silently degrading\nto terminal) is a direct consequence: vocabulary 2 accepts a value that\nvocabulary 3 implements and the dispatch sites do not handle.\n\nSame shape, same file, for timestamp policy:\n RenderTimestampPolicy enum (projection_spec.py:75)\n _READ_TIMESTAMP_POLICIES = (\"renderer-default\",\"include-available\",\"omit\")\n (query_verbs.py:218 -- the same three values re-spelled as strings)\n\nINERT KNOBS in the same module:\n - RenderFormat (projection_spec.py:44) declares 8 members\n (MARKDOWN/JSON/NDJSON/HTML/OBSIDIAN/ORG/YAML/CSV). Nothing anywhere\n dispatches on RenderFormat -- zero `RenderFormat.` references outside the\n defining module. RenderSpec.format is a typed field nobody branches on.\n - RENDER_FORMAT_ALIASES (projection_spec.py:69) maps \"text\"/\"plain\" ->\n PLAINTEXT and has ZERO consumers in production or tests. The aliases it\n promises are not applied anywhere.\n - SelectionSpec (projection_spec.py:83) has zero references in production\n or tests outside its own module.\n\nDO: pick ONE vocabulary -- the enum -- and make click.Choice derive from it\nrather than re-spelling its members as a string tuple, so adding a member\ncannot again produce an accepted-but-unhandled value. Type the invocation\nfield as the enum. Then either wire RenderFormat dispatch or delete the\nmembers that no renderer implements; same for the alias map and SelectionSpec.\nPer the standing directive: where one option dominates, delete the\nalternatives rather than keeping them as inert configuration.\n","status":"open","priority":2,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-29T10:33:49Z","created_by":"Sinity","updated_at":"2026-07-29T10:33:49Z","dependency_count":0,"dependent_count":0,"comment_count":0,"metadata":{"acceptance_contract_v1":{"anti_vacuity":["A controlled mutation that removes the central guard or restores the pre-fix behavior makes the focused regression fail.","The test asserts durable/public behavior, not merely that a helper was called or returned a mocked value."],"bead_id":"polylogue-j1vs","closure":{"disposition":"whole-or-explicit-partial","rule":"Close `polylogue-j1vs` only when the criteria above are evidenced on the final head. Any residual operation, provider/origin, live population, or generalized bug class is transferred to a named successor with a dependency edge before closure.","successor_required_for_partial":true},"confidence":"high","contract_type":"implementation","evidence":["The regression or audit preserves the motivating observation: Found 2026-07-29 by codebase audit. THREE parallel vocabularies describe the same concept (where rendered output goes), and none is authoritative:","Measured evidence remains reconciled with this recorded population: RenderFormat and alias map are inert Found 2026-07-29 by codebase audit. THREE parallel vocabularies describe the sam","Measured evidence remains reconciled with this recorded population: erFormat and alias map are inert Found 2026-07-29 by codebase audit. THREE parallel vocabularies describe the same c"],"generated_at":"2026-08-07T00:00:00Z","outcome":"The production path no longer exhibits the defect or missing capability named “three parallel render-destination vocabularies; RenderFormat and alias map are inert”; the result is observable through the public or operator-facing route.","retained_scope":["SelectionSpec (projection_spec.py:83) has zero references in production"],"risk":"durable-mutation","routes":["Exercise the implementation through these named production surfaces: `polylogue/surfaces/projection_spec.py`, `polylogue/cli/query_verbs.py`, `polylogue/cli/query_contracts.py`, `read_views/base.py`, `destination: str`, `RenderFormat.`."],"safety":["No production mutation is performed by the implementation lane.","Any later apply is dry-run-first, backup-gated, exact-plan-bound, idempotent or resumable, and emits an immutable receipt."],"schema_version":1,"source_digest":"70d8909064f723c5cc8b19ea076beea1de8095971656c903365621cec00af69f","verification":["Add a focused red-before/green-after regression carrying `polylogue-j1vs` or the incident name and executing the owning production route.","Run `devtools verify --quick` on the final head and record the exact head SHA in the closure evidence.","Run `devtools verify` on the final head so the testmon-affected regression set executes; `devtools verify --quick` alone is insufficient.","Run `devtools verify` on the final head so the testmon-affected regression set executes; `devtools verify --quick` alone is insufficient."]}},"acceptance_criteria":"1. Outcome: The production path no longer exhibits the defect or missing capability named “three parallel render-destination vocabularies; RenderFormat and alias map are inert”; the result is observable through the public or operator-facing route.\n2. Existing scope retained: SelectionSpec (projection_spec.py:83) has zero references in production\n3. Production route: Exercise the implementation through these named production surfaces: `polylogue/surfaces/projection_spec.py`, `polylogue/cli/query_verbs.py`, `polylogue/cli/query_contracts.py`, `read_views/base.py`, `destination: str`, `RenderFormat.`.\n4. Evidence: The regression or audit preserves the motivating observation: Found 2026-07-29 by codebase audit. THREE parallel vocabularies describe the same concept (where rendered output goes), and none is authoritative:\n5. Evidence: Measured evidence remains reconciled with this recorded population: RenderFormat and alias map are inert Found 2026-07-29 by codebase audit. THREE parallel vocabularies describe the sam\n6. Evidence: Measured evidence remains reconciled with this recorded population: erFormat and alias map are inert Found 2026-07-29 by codebase audit. THREE parallel vocabularies describe the same c\n7. Verification: Add a focused red-before/green-after regression carrying `polylogue-j1vs` or the incident name and executing the owning production route.\n8. Verification: Run `devtools verify --quick` on the final head and record the exact head SHA in the closure evidence.\n9. Verification: Run `devtools verify` on the final head so the testmon-affected regression set executes; `devtools verify --quick` alone is insufficient.\n10. Verification: Run `devtools verify` on the final head so the testmon-affected regression set executes; `devtools verify --quick` alone is insufficient.\n11. Anti-vacuity: A controlled mutation that removes the central guard or restores the pre-fix behavior makes the focused regression fail.\n12. Anti-vacuity: The test asserts durable/public behavior, not merely that a helper was called or returned a mocked value.\n13. Safety: No production mutation is performed by the implementation lane.\n14. Safety: Any later apply is dry-run-first, backup-gated, exact-plan-bound, idempotent or resumable, and emits an immutable receipt.\n15. Closure disposition: whole-or-explicit-partial\n16. Closure: Close `polylogue-j1vs` only when the criteria above are evidenced on the final head. Any residual operation, provider/origin, live population, or generalized bug class is transferred to a named successor with a dependency edge before closure."} {"_type":"issue","id":"polylogue-c66i","title":"Schema promotion (c53ad94e0) dropped x-polylogue-semantic-role annotations for codex/claude-code","description":"Problem: the 2026-07-29 structural-merge promotion (c53ad94e0, \"fix(schemas):\nmake structural merge monotonic and promote every provider\") regenerated the\ncodex and claude-code baseline provider schemas\n(polylogue/schemas/providers/{codex,claude-code}/versions/v1/elements/session_record_stream.schema.json.gz)\nfrom scratch and did not carry forward the x-polylogue-semantic-role\nannotation overlay (message_role/session_title/message_body/etc).\n\nEvidence:\n- Diffing the pre-promotion schema snapshot (master @ b64a074e5) against the\n current branch's schema for the same file shows the annotation set went\n from 6 entries (codex) / 7 entries (claude-code) including\n \"/properties/payload/properties/role\" (codex, message_role) and\n \"/properties/type\" (claude-code, message_role) down to zero.\n- SyntheticCorpus.generate_batch_for_spec(\"codex\"/\"claude-code\", seed=42) now\n fills the role-discriminator field with an opaque `synthetic-`\n placeholder instead of a real user/assistant value, so every parsed\n message normalizes to Role.UNKNOWN.\n- This is a synthetic-corpus generation defect, not a parser defect: real\n codex/claude-code exports always carry real role values, so production\n parsing is unaffected.\n- Already caught independently by tests/unit/core/test_synthetic_semantic_wiring.py\n (TestBaselineSchemaAnnotations::test_schema_has_expected_semantic_roles[codex],\n [claude-code], and the idempotent-injection tests for\n claude-ai/codex/claude-code) and tests/unit/core/test_synthetic_semantics.py\n (test_generation_plants_independent_wire_facts_before_ingest) -- 6 failures,\n all pre-existing on this branch, unrelated to any parser change today.\n- Also broke tests/unit/sources/test_parsers_props.py\n (test_provider_parser_contract[codex/claude-code],\n TestMessageOrderConsistency::test_messages_have_consistent_roles[codex/claude-code])\n -- worked around at the test-strategy layer in\n tests/infra/strategies/providers.py (repair_role_discriminators) and\n tests/conftest.py (synthetic_source fixture) since polylogue/schemas/** is\n out of scope for that fix.\n\nFix: run `devtools inject-semantic-annotations` (devtools/inject_semantic_annotations.py,\nalready exists as the sanctioned one-shot/re-annotation tool) against the\npromoted codex/claude-code (and audit claude-ai too, since its idempotency\ntest also fails) schemas, verify test_synthetic_semantic_wiring.py goes\ngreen, and consider removing the test-layer workaround in\ntests/infra/strategies/providers.py / tests/conftest.py once the schema\ncarries real annotations again.","status":"closed","priority":2,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-29T09:35:22Z","created_by":"Sinity","updated_at":"2026-07-31T21:25:02Z","closed_at":"2026-07-31T21:25:02Z","close_reason":"Verified SATISFIED (storage triage 2026-07-31): the schema-annotation regression and its own two follow-up fixes were introduced and repaired within one unmerged branch before landing as squash-commit 5e23e6abf (PR #3390, index v46 wire-evidence batch, merged 2026-07-29) -- the regression never reached master in an unfixed state. Decoded live gzipped schemas: x-polylogue-semantic-role present (6 codex / 9 claude-code annotations).","labels":["regression","schemas","test-infra"],"dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"polylogue-7rds","title":"Wire periodic (not just startup) blob-publication reservation reconciliation","description":"Blob-store audit (bytes-per-object, orphan/dedup checks) found the blob-GC store itself correctly deduplicated and orphan-free except 2 stuck blob_publication_reservations rows (42.5MB, reserved 2026-07-12/13) that are blob_present=True, referenced=False -- the 'unresolved' classification that reconcile_blob_publication_reservations_under_exclusion() never auto-clears (by design; only abandon_blob_publication_receipts with --yes can remove it). The real gap: _reconcile_blob_publications() (polylogue/daemon/cli.py:1009) runs exactly ONCE at daemon startup, never periodically -- unlike blob-gc and embedding-orphan-reconcile, which both have periodic_*_check loops. A long-running daemon (weeks of uptime, the common case) means any reservation that becomes safely clearable (referenced or blob-missing) after startup sits until the next restart. Fix: add a periodic_blob_publication_reconcile_check loop (mirror polylogue/daemon/blob_gc_periodic.py's shape, ~900s interval) that calls reconcile_blob_publication_reservations_under_exclusion() on a schedule; it only ever clears rows already proven safe (referenced or blob missing), never touches 'unresolved' rows, so this is a low-risk periodic-maintenance addition, not a policy change.","status":"open","priority":2,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-29T08:44:20Z","created_by":"Sinity","updated_at":"2026-07-29T08:44:20Z","dependency_count":0,"dependent_count":0,"comment_count":0,"metadata":{"acceptance_contract_v1":{"anti_vacuity":["A controlled mutation that removes the central guard or restores the pre-fix behavior makes the focused regression fail.","The test asserts durable/public behavior, not merely that a helper was called or returned a mocked value."],"bead_id":"polylogue-7rds","closure":{"disposition":"whole-or-explicit-partial","rule":"Close `polylogue-7rds` only when the criteria above are evidenced on the final head. Any residual operation, provider/origin, live population, or generalized bug class is transferred to a named successor with a dependency edge before closure.","successor_required_for_partial":true},"confidence":"high","contract_type":"implementation","evidence":["The regression or audit preserves the motivating observation: Blob-store audit (bytes-per-object, orphan/dedup checks) found the blob-GC store itself correctly deduplicated and orphan-free except 2 stuck blob_publication_reservations rows (42.5MB, reserved 2026-07-12/13) that are blob_present=True, referenced=False -- the 'unresolved' classification that reconcile_blob_publication_reservations_under_exclusion() never auto-clears (by design; only abandon_blob_publication_receipts with --yes can remove it). The real gap: _reconcile_blob_publications() (polylogue/daemon/cli.py:1","Measured evidence remains reconciled with this recorded population: orrectly deduplicated and orphan-free except 2 stuck blob_publication_reservations rows (42.5MB, reserved 2026-07-12","Measured evidence remains reconciled with this recorded population: 2 stuck blob_publication_reservations rows (42.5MB, reserved 2026-07-12/13) that are blob_present=True, referenced=False"],"generated_at":"2026-08-07T00:00:00Z","outcome":"The production path no longer exhibits the defect or missing capability named “Wire periodic (not just startup) blob-publication reservation reconciliation”; the result is observable through the public or operator-facing route.","retained_scope":[],"risk":"durable-mutation","routes":["Exercise the implementation through these named production surfaces: `orphan/dedup`, `2026-07-12/13`, `polylogue/daemon/cli.py`, `polylogue/daemon/blob_gc_periodic.py`."],"safety":["No production mutation is performed by the implementation lane.","Any later apply is dry-run-first, backup-gated, exact-plan-bound, idempotent or resumable, and emits an immutable receipt."],"schema_version":1,"source_digest":"17c5901fd5f9fa4dfb7cee5970337b940dcd22c5898cd149c80baf7271989c1c","verification":["Add a focused red-before/green-after regression carrying `polylogue-7rds` or the incident name and executing the owning production route.","Run `devtools verify --quick` on the final head and record the exact head SHA in the closure evidence.","Run `devtools verify` on the final head so the testmon-affected regression set executes; `devtools verify --quick` alone is insufficient.","Run `devtools verify` on the final head so the testmon-affected regression set executes; `devtools verify --quick` alone is insufficient."]}},"acceptance_criteria":"1. Outcome: The production path no longer exhibits the defect or missing capability named “Wire periodic (not just startup) blob-publication reservation reconciliation”; the result is observable through the public or operator-facing route.\n2. Production route: Exercise the implementation through these named production surfaces: `orphan/dedup`, `2026-07-12/13`, `polylogue/daemon/cli.py`, `polylogue/daemon/blob_gc_periodic.py`.\n3. Evidence: The regression or audit preserves the motivating observation: Blob-store audit (bytes-per-object, orphan/dedup checks) found the blob-GC store itself correctly deduplicated and orphan-free except 2 stuck blob_publication_reservations rows (42.5MB, reserved 2026-07-12/13) that are blob_present=True, referenced=False -- the 'unresolved' classification that reconcile_blob_publication_reservations_under_exclusion() never auto-clears (by design; only abandon_blob_publication_receipts with --yes can remove it). The real gap: _reconcile_blob_publications() (polylogue/daemon/cli.py:1\n4. Evidence: Measured evidence remains reconciled with this recorded population: orrectly deduplicated and orphan-free except 2 stuck blob_publication_reservations rows (42.5MB, reserved 2026-07-12\n5. Evidence: Measured evidence remains reconciled with this recorded population: 2 stuck blob_publication_reservations rows (42.5MB, reserved 2026-07-12/13) that are blob_present=True, referenced=False\n6. Verification: Add a focused red-before/green-after regression carrying `polylogue-7rds` or the incident name and executing the owning production route.\n7. Verification: Run `devtools verify --quick` on the final head and record the exact head SHA in the closure evidence.\n8. Verification: Run `devtools verify` on the final head so the testmon-affected regression set executes; `devtools verify --quick` alone is insufficient.\n9. Verification: Run `devtools verify` on the final head so the testmon-affected regression set executes; `devtools verify --quick` alone is insufficient.\n10. Anti-vacuity: A controlled mutation that removes the central guard or restores the pre-fix behavior makes the focused regression fail.\n11. Anti-vacuity: The test asserts durable/public behavior, not merely that a helper was called or returned a mocked value.\n12. Safety: No production mutation is performed by the implementation lane.\n13. Safety: Any later apply is dry-run-first, backup-gated, exact-plan-bound, idempotent or resumable, and emits an immutable receipt.\n14. Closure disposition: whole-or-explicit-partial\n15. Closure: Close `polylogue-7rds` only when the criteria above are evidenced on the final head. Any residual operation, provider/origin, live population, or generalized bug class is transferred to a named successor with a dependency edge before closure."} +{"_type":"issue","id":"polylogue-7rds","title":"Wire periodic (not just startup) blob-publication reservation reconciliation","description":"Blob-store audit (bytes-per-object, orphan/dedup checks) found the blob-GC store itself correctly deduplicated and orphan-free except 2 stuck blob_publication_reservations rows (42.5MB, reserved 2026-07-12/13) that are blob_present=True, referenced=False -- the 'unresolved' classification that reconcile_blob_publication_reservations_under_exclusion() never auto-clears (by design; only abandon_blob_publication_receipts with --yes can remove it). The real gap: _reconcile_blob_publications() (polylogue/daemon/cli.py:1009) runs exactly ONCE at daemon startup, never periodically -- unlike blob-gc and embedding-orphan-reconcile, which both have periodic_*_check loops. A long-running daemon (weeks of uptime, the common case) means any reservation that becomes safely clearable (referenced or blob-missing) after startup sits until the next restart. Fix: add a periodic_blob_publication_reconcile_check loop (mirror polylogue/daemon/blob_gc_periodic.py's shape, ~900s interval) that calls reconcile_blob_publication_reservations_under_exclusion() on a schedule; it only ever clears rows already proven safe (referenced or blob missing), never touches 'unresolved' rows, so this is a low-risk periodic-maintenance addition, not a policy change.","acceptance_criteria":"1. Outcome: The production path no longer exhibits the defect or missing capability named “Wire periodic (not just startup) blob-publication reservation reconciliation”; the result is observable through the public or operator-facing route.\n2. Production route: Exercise the implementation through these named production surfaces: `orphan/dedup`, `2026-07-12/13`, `polylogue/daemon/cli.py`, `polylogue/daemon/blob_gc_periodic.py`.\n3. Evidence: The regression or audit preserves the motivating observation: Blob-store audit (bytes-per-object, orphan/dedup checks) found the blob-GC store itself correctly deduplicated and orphan-free except 2 stuck blob_publication_reservations rows (42.5MB, reserved 2026-07-12/13) that are blob_present=True, referenced=False -- the 'unresolved' classification that reconcile_blob_publication_reservations_under_exclusion() never auto-clears (by design; only abandon_blob_publication_receipts with --yes can remove it). The real gap: _reconcile_blob_publications() (polylogue/daemon/cli.py:1\n4. Evidence: Measured evidence remains reconciled with this recorded population: orrectly deduplicated and orphan-free except 2 stuck blob_publication_reservations rows (42.5MB, reserved 2026-07-12\n5. Evidence: Measured evidence remains reconciled with this recorded population: 2 stuck blob_publication_reservations rows (42.5MB, reserved 2026-07-12/13) that are blob_present=True, referenced=False\n6. Verification: Add a focused red-before/green-after regression carrying `polylogue-7rds` or the incident name and executing the owning production route.\n7. Verification: Run `devtools verify --quick` on the final head and record the exact head SHA in the closure evidence.\n8. Verification: Run `devtools verify` on the final head so the testmon-affected regression set executes; `devtools verify --quick` alone is insufficient.\n9. Verification: Run `devtools verify` on the final head so the testmon-affected regression set executes; `devtools verify --quick` alone is insufficient.\n10. Anti-vacuity: A controlled mutation that removes the central guard or restores the pre-fix behavior makes the focused regression fail.\n11. Anti-vacuity: The test asserts durable/public behavior, not merely that a helper was called or returned a mocked value.\n12. Safety: No production mutation is performed by the implementation lane.\n13. Safety: Any later apply is dry-run-first, backup-gated, exact-plan-bound, idempotent or resumable, and emits an immutable receipt.\n14. Closure disposition: whole-or-explicit-partial\n15. Closure: Close `polylogue-7rds` only when the criteria above are evidenced on the final head. Any residual operation, provider/origin, live population, or generalized bug class is transferred to a named successor with a dependency edge before closure.","notes":"PARTIAL DISPOSITION 2026-08-08: the production behavior is complete. The daemon now reconciles safely terminal blob-publication reservations every 900 seconds through the real write coordinator, while unresolved present-and-unreferenced reservations remain untouched. The focused production-route selection passed 4 tests with 118 deselected at code head 97322f02b0dda41521fce2ae506c7765ef90c73e; removing the periodic route or safe-state filter makes the fixture fail. The final quick gate passed all 24 steps in run 20260808T121906Z-quick-1821210-0d9ec191. No production mutation was performed. The default devtools verify command refused before test selection because current master has no valid testmon seed; the exact-master baseline has 414 failures and 16 errors. That verification-infrastructure residual is transferred to polylogue-93xe through a relates-to edge instead of being waived.","status":"closed","priority":2,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-29T08:44:20Z","created_by":"Sinity","updated_at":"2026-08-08T12:23:30Z","closed_at":"2026-08-08T12:23:30Z","close_reason":"Implementation complete with explicit verification-plane residual transferred to polylogue-93xe: periodic reconciliation clears only safe terminal reservations, preserves unresolved reservations, passes the focused real-route regression and final quick gate, and performs no live mutation.","metadata":{"acceptance_contract_v1":{"risk":"durable-mutation","routes":["Exercise the implementation through these named production surfaces: `orphan/dedup`, `2026-07-12/13`, `polylogue/daemon/cli.py`, `polylogue/daemon/blob_gc_periodic.py`."],"safety":["No production mutation is performed by the implementation lane.","Any later apply is dry-run-first, backup-gated, exact-plan-bound, idempotent or resumable, and emits an immutable receipt."],"bead_id":"polylogue-7rds","closure":{"rule":"Close `polylogue-7rds` only when the criteria above are evidenced on the final head. Any residual operation, provider/origin, live population, or generalized bug class is transferred to a named successor with a dependency edge before closure.","disposition":"whole-or-explicit-partial","successor_required_for_partial":true},"outcome":"The production path no longer exhibits the defect or missing capability named “Wire periodic (not just startup) blob-publication reservation reconciliation”; the result is observable through the public or operator-facing route.","evidence":["The regression or audit preserves the motivating observation: Blob-store audit (bytes-per-object, orphan/dedup checks) found the blob-GC store itself correctly deduplicated and orphan-free except 2 stuck blob_publication_reservations rows (42.5MB, reserved 2026-07-12/13) that are blob_present=True, referenced=False -- the 'unresolved' classification that reconcile_blob_publication_reservations_under_exclusion() never auto-clears (by design; only abandon_blob_publication_receipts with --yes can remove it). The real gap: _reconcile_blob_publications() (polylogue/daemon/cli.py:1","Measured evidence remains reconciled with this recorded population: orrectly deduplicated and orphan-free except 2 stuck blob_publication_reservations rows (42.5MB, reserved 2026-07-12","Measured evidence remains reconciled with this recorded population: 2 stuck blob_publication_reservations rows (42.5MB, reserved 2026-07-12/13) that are blob_present=True, referenced=False"],"confidence":"high","anti_vacuity":["A controlled mutation that removes the central guard or restores the pre-fix behavior makes the focused regression fail.","The test asserts durable/public behavior, not merely that a helper was called or returned a mocked value."],"generated_at":"2026-08-07T00:00:00Z","verification":["Add a focused red-before/green-after regression carrying `polylogue-7rds` or the incident name and executing the owning production route.","Run `devtools verify --quick` on the final head and record the exact head SHA in the closure evidence.","Run `devtools verify` on the final head so the testmon-affected regression set executes; `devtools verify --quick` alone is insufficient.","Run `devtools verify` on the final head so the testmon-affected regression set executes; `devtools verify --quick` alone is insufficient."],"contract_type":"implementation","source_digest":"17c5901fd5f9fa4dfb7cee5970337b940dcd22c5898cd149c80baf7271989c1c","retained_scope":[],"schema_version":1}},"dependencies":[{"issue_id":"polylogue-7rds","depends_on_id":"polylogue-93xe","type":"relates-to","created_at":"2026-08-08T12:23:28Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-wjgf","title":"Wire Claude Code tool-results sidecar join into dispatch.py acquisition path","description":"polylogue-rujy built the join+attach logic (polylogue/sources/live/tool_result_sidecars.py:join_tool_result_sidecars, polylogue/sources/parsers/claude/code_parser.py:apply_tool_result_sidecars) and wired parse_code/parse_code_stream to accept an optional tool_result_sidecars kwarg -- passing nothing preserves current behavior exactly (tested).\n\nRemaining wiring, out of that lane's write scope (polylogue/sources/dispatch.py is not under sources/live/** or sources/parsers/claude/**):\n\n1. polylogue/sources/dispatch.py:1061 currently calls `claude.parse_code(payloads, spec.fallback_id)`. Needs to derive the session's tool-results directory from `spec.source_path` (the sibling `/tool-results/` directory -- for subagent JSONL under `/subagents/agent-*.jsonl`, the sidecar directory is still the SESSION-level `/tool-results/`, not a per-subagent one; verified live), call `join_tool_result_sidecars(payloads, tool_results_dir)`, and pass the result through.\n2. Decide whether this should be default-on immediately or gated behind a config/CLI flag until ingest wall-clock is measured against the polylogue-623q envelope (see AC5 on polylogue-rujy) -- this needs polylogue/config.py and/or CLI wiring, both explicitly out of the rujy lane's OWNS list.\n3. Streaming path (parse_code_stream, used for multi-GiB Claude Code JSONL) needs the equivalent wiring at whatever call site constructs its Iterable[object] payload.\n\nMeasured live: acquiring genuinely-truncated sidecars is worth it (~60-65% of the 1.34GB total is genuinely new content per polylogue-rujy's sampling), so this is a real product win, not speculative -- the remaining work is glue, not design.","status":"closed","priority":2,"issue_type":"task","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-07-29T07:38:25Z","created_by":"Sinity","updated_at":"2026-07-29T08:37:28Z","started_at":"2026-07-29T08:37:08Z","closed_at":"2026-07-29T08:37:28Z","close_reason":"Wired on branch feature/chore/promote-schemas-and-wire-gates (commit 2237e8a82, worktree-agent-a313d11dfce19c361).\n\n1. dispatch.py wiring: the eager Provider.CLAUDE_CODE branch of _parse_lowered_spec (dispatch.py) now derives the tool-results dir from spec.source_path via a new resolve_tool_results_dir() (sources/live/tool_result_sidecars.py) and passes join_tool_result_sidecars(...) into claude.parse_code(..., tool_result_sidecars=...). source_path was previously dropped by _lower_grouped_payload/_claude_code_grouped_record_specs for CLAUDE_CODE -- threaded through both.\n2. Default-on, no flag (measured, not assumed): ran join_tool_result_sidecars against the FULL population of Claude Code sessions with a tool-results/ dir under a real ~/.claude/projects corpus (read-only, scratch measurement script, not committed) -- 525 sessions matched resolve_tool_results_dir. Total added wall time for the join across all 525 dirs: 8.2s (704 MB matched + 708 MB debt bytes read, 9,421 files matched / 3,012 debt). That is ~23% on top of just those 525 sessions' own JSONL read time (35.0s), but those 525 sessions are ~3% of the corpus's ~17K distinct native_ids (polylogue-623q's latest corpus-shape note) -- so the join's share of a full-corpus rebuild is well under 1% of a <60min (3600s) budget. Default-on is correct; a flag nobody flips would be the dark-capability pattern this project avoids.\n3. Streaming path: _claude_code_stream_sessions (dispatch.py) can't materialize the raw payload (that's the whole point of streaming). Added ToolResultIndexAccumulator + observe_tool_result_stream (tool_result_sidecars.py) so each session-group's records are teed through an index-builder as they stream past parse_code_stream; the join runs against the resulting index once the group iterator is exhausted, then apply_tool_result_sidecars (imported directly from code_parser, not edited) attaches it. Both parse_payload and parse_stream_payload now reach the same coverage -- confirmed by a dedicated streaming test and a subagent-source-path resolution test (subagent JSONL correctly resolves to the SESSION-level tool-results/ dir, not a per-subagent one).\n\nNew tests (tests/unit/sources/test_dispatch_payloads.py): 5 new tests covering batch wiring, streaming wiring, subagent-path resolution, and the source_path-absent no-op -- each verified by mutation (temporarily reverting the wiring) to fail without the change, then reverted.\n\nVerification: devtools test tests/unit/sources/test_dispatch_payloads.py tests/unit/sources/test_dispatch_ordering.py tests/unit/sources/test_tool_result_sidecars.py -> 33 passed. mypy --strict clean on all 3 changed/added files. ruff check/format clean. devtools verify hash-boundary-census -> 0 unregistered/stale after updating the registry entry for the hash_text call site that moved into _join_from_index. devtools render all --check -> no \"out of sync\" lines (no new module added, so no topology regen needed).\n\nNo index schema bump: session_events stays bounded (id/filename/size/content_hash/status only), matching the design constraint.","labels":["area:ingest"],"dependencies":[{"issue_id":"polylogue-wjgf","depends_on_id":"polylogue-rujy","type":"relates-to","created_at":"2026-07-31T14:40:08Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-iy3n","title":"Per-tick raw-materialization candidate rescan: cache attempt reverted, needs persistent backlog iterator (phase c)","description":"Two attempts at killing repair_raw_materialization's per-tick full O(backlog)\n_raw_materialization_candidate_ids() rescan have both failed with the same\nobservable staleness shape, via two different mechanisms:\n\n1. PRAGMA data_version memoization (prior lane): reverted because SQLite's\n file-header change counter does not advance under WAL until checkpoint,\n so writes were invisible to the cache. ~29 tests failed on staleness.\n\n2. Explicit generation-counted cache with write-path invalidation hooks\n (this session, polylogue-m6tp fast-follow): reverted because the actual\n writers of raw_sessions/index-tier sessions/raw_membership_census/\n raw_session_memberships live in polylogue/sources/live/* (live ingest),\n polylogue/storage/repository/**, and\n polylogue/storage/sqlite/archive_tiers/archive.py /\n storage/sqlite/queries/{raw_writes,raw_state}.py -- none reachable from\n the repair.py/raw_authority.py/revision_application.py/daemon/** write\n scope this task was granted. Two tests in tests/unit/storage/test_repair.py\n (test_raw_materialization_replays_governed_bundle_after_index_reset,\n test_raw_materialization_reports_uncensused_append_fragments_as_pending_debt)\n proved this concretely: both legitimately mutate raw_sessions/index.db\n directly between two repair_raw_materialization calls (an index reset,\n and an out-of-band census write), and the cache returned stale results\n in both cases. Full writeup: docs/design/convergence-simplification-inventory.md\n item 5.\n\nConclusion: this item cannot be closed by caching a query over the current\nper-tick-stateless design without either (a) invalidation hooks spanning\nseveral other lanes' write scope, or (b) the persistent in-daemon backlog\niterator polylogue-m6tp's design sketch already names as the real fix\n(phase c, bulk-routing). (b) is the only sound path -- it replaces \"cache a\nderived view\" with \"maintain the source of truth incrementally\", which\nsidesteps the invalidation-completeness problem entirely (the iterator is\nupdated by the same code that performs each write, not by a bystander\nguessing which writes matter).\n\nDo not re-attempt with a narrower/smaller cache; the failure is structural\n(a correct source-of-truth cache needs write-scope this task doesn't have),\nnot a tuning problem.","status":"open","priority":2,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-29T07:32:10Z","created_by":"Sinity","updated_at":"2026-07-29T07:32:10Z","dependency_count":0,"dependent_count":0,"comment_count":0,"metadata":{"acceptance_contract_v1":{"anti_vacuity":["A controlled mutation that removes the central guard or restores the pre-fix behavior makes the focused regression fail.","The test asserts durable/public behavior, not merely that a helper was called or returned a mocked value."],"bead_id":"polylogue-iy3n","closure":{"disposition":"whole-or-explicit-partial","rule":"Close `polylogue-iy3n` only when the criteria above are evidenced on the final head. Any residual operation, provider/origin, live population, or generalized bug class is transferred to a named successor with a dependency edge before closure.","successor_required_for_partial":true},"confidence":"high","contract_type":"implementation","evidence":["The regression or audit preserves the motivating observation: Two attempts at killing repair_raw_materialization's per-tick full O(backlog) _raw_materialization_candidate_ids() rescan have both failed with the same observable staleness shape, via two different mechanisms:","Measured evidence remains reconciled with this recorded population: leness shape, via two different mechanisms: 1. PRAGMA data_version memoization (prior lane): reverted because SQLit","Measured evidence remains reconciled with this recorded population: int, so writes were invisible to the cache. ~29 tests failed on staleness. 2. Explicit generation-counted cache with"],"generated_at":"2026-08-07T00:00:00Z","outcome":"The production path no longer exhibits the defect or missing capability named “Per-tick raw-materialization candidate rescan: cache attempt reverted, needs persistent backlog iterator (phase c)”; the result is observable through the public or operator-facing route.","retained_scope":[],"risk":"ordinary","routes":["Exercise the implementation through these named production surfaces: `tests/unit/storage/test_repair.py`, `raw_sessions/index-tier`, `polylogue/storage/sqlite/archive_tiers/archive.py`, `repair.py/raw_authority.py/revision_application`, `raw_sessions/index.db`, `polylogue/storage/repository/**, and`, `polylogue/storage/sqlite/archive_tiers/archive.py /`."],"safety":[],"schema_version":1,"source_digest":"df0973391b289195c499f44d3eee1041c7437d2adeb18d5434c127fac7248600","verification":["Run the focused regression suite: `tests/unit/storage/test_repair.py`.","Run `polylogue/storage/repository/**, and` and record the exit status and material output.","Run `polylogue/storage/sqlite/archive_tiers/archive.py /` and record the exit status and material output.","Run `devtools verify --quick` on the final head and record the exact head SHA in the closure evidence.","Run `devtools verify` on the final head so the testmon-affected regression set executes; `devtools verify --quick` alone is insufficient."]}},"acceptance_criteria":"1. Outcome: The production path no longer exhibits the defect or missing capability named “Per-tick raw-materialization candidate rescan: cache attempt reverted, needs persistent backlog iterator (phase c)”; the result is observable through the public or operator-facing route.\n2. Production route: Exercise the implementation through these named production surfaces: `tests/unit/storage/test_repair.py`, `raw_sessions/index-tier`, `polylogue/storage/sqlite/archive_tiers/archive.py`, `repair.py/raw_authority.py/revision_application`, `raw_sessions/index.db`, `polylogue/storage/repository/**, and`, `polylogue/storage/sqlite/archive_tiers/archive.py /`.\n3. Evidence: The regression or audit preserves the motivating observation: Two attempts at killing repair_raw_materialization's per-tick full O(backlog) _raw_materialization_candidate_ids() rescan have both failed with the same observable staleness shape, via two different mechanisms:\n4. Evidence: Measured evidence remains reconciled with this recorded population: leness shape, via two different mechanisms: 1. PRAGMA data_version memoization (prior lane): reverted because SQLit\n5. Evidence: Measured evidence remains reconciled with this recorded population: int, so writes were invisible to the cache. ~29 tests failed on staleness. 2. Explicit generation-counted cache with\n6. Verification: Run the focused regression suite: `tests/unit/storage/test_repair.py`.\n7. Verification: Run `polylogue/storage/repository/**, and` and record the exit status and material output.\n8. Verification: Run `polylogue/storage/sqlite/archive_tiers/archive.py /` and record the exit status and material output.\n9. Verification: Run `devtools verify --quick` on the final head and record the exact head SHA in the closure evidence.\n10. Verification: Run `devtools verify` on the final head so the testmon-affected regression set executes; `devtools verify --quick` alone is insufficient.\n11. Anti-vacuity: A controlled mutation that removes the central guard or restores the pre-fix behavior makes the focused regression fail.\n12. Anti-vacuity: The test asserts durable/public behavior, not merely that a helper was called or returned a mocked value.\n13. Closure disposition: whole-or-explicit-partial\n14. Closure: Close `polylogue-iy3n` only when the criteria above are evidenced on the final head. Any residual operation, provider/origin, live population, or generalized bug class is transferred to a named successor with a dependency edge before closure."} {"_type":"issue","id":"polylogue-hgsq","title":"Semantic-frontier heads make ~9,300 superseded raws structurally unreleasable","description":"Discovered while implementing the stale-supersession-receipt reissue pass for\npolylogue-ktwa. Live-archive query (2026-07-29, /realm/db/polylogue):\n\nOf the ~11,966 distinct (raw_id, session_id, logical_source_key) groups\ncarrying decision='superseded' receipts, only 2 are genuinely \"stale\"\n(receipt mismatches the current head) after excluding the majority\npopulation. The other ~9,320+ mismatches are all against a\nraw_revision_heads row whose accepted_frontier_kind = 'semantic', not\n'byte'.\n\nactive_raw_retention_authority's eligibility join\n(polylogue/storage/raw_retention.py:148, _active_index_raw_authority)\nunconditionally requires `head.accepted_frontier_kind = 'byte'` -- it never\nadmits a raw for release under a semantic head, by design (semantic\nfrontiers compare parsed-content hashes, not byte offsets, so there is no\nbyte-level proof of subsumption). This means the entire semantic-headed\npopulation (predominantly antigravity multi-file sessions, e.g.\n`antigravity::plan.md` / `task.md` / `report.md` etc.) is structurally\nexcluded from ever being released under the current retention design,\nregardless of how many times it is reissued a fresh receipt.\n\nThe polylogue-ktwa reissue pass (raw_retention.py:\nplan_stale_supersession_reissue / reissue_stale_supersession_receipts)\ncorrectly fails closed on semantic heads (matches the retention join's own\nrequirement) and reports this population as ineligible with reason\n\"current head frontier_kind is not byte\" -- it is not a bug in that pass,\njust evidence that a much larger design question remains open: is a\nsemantic-frontier retention path (an analogous byte-safe proof for\ncontent-hash-based supersession) worth building, or is this population\nsimply permanent evidence by design?\n\nNeeds a product decision before any code: (a) build a semantic-frontier\nretention/reissue path with its own safety proof, or (b) explicitly accept\nthese raws as permanently retained and stop counting them as \"debt\" in\nfuture audits. Either way, do not conflate this with the byte-frontier\nreissue mechanism polylogue-ktwa ships -- they need separate proof\nmechanisms if (a) is chosen.","notes":"VERDICT (2026-07-29, verified against live archive /realm/db/polylogue): frontier_kind='byte'\nis a deliberate, correct, load-bearing safety boundary, not an unexamined narrowing. Semantic\nfrontiers cannot currently authorise release. Do not relax this predicate.\n\nWHAT A SEMANTIC FRONTIER ACTUALLY PROVES. classify_membership_revisions /\n_strictly_dominates (polylogue/archive/session_revision_membership.py:188) proves, at\nmembership-replay time, that an older raw's parsed message_hashes/event_hashes are an exact\nordered PREFIX of the newer accepted session's, and its attachment_hashes a subset -- a real\ncontainment proof, but computed ONCE over transient PARSED projections. The receipt written\nto raw_revision_heads/raw_revision_applications (storage/sqlite/archive_tiers/archive.py:3687)\npersists only accepted_frontier = a SCALAR COUNT (len(message_hashes)+len(event_hashes)+\nlen(attachment_hashes)), never the hash sequences themselves. So the domination proof cannot\nbe re-verified later without re-parsing the raw and re-trusting the classifier code that\nproduced it -- a materially weaker guarantee than a byte frontier, whose claim\n(_validate_byte_head / _validate_active_revision_chain, storage/raw_retention.py:307-345) is\nre-derived independently from source-tier byte offsets/generations alone, every time, with\nno dependency on parser semantics.\n\nEMPIRICAL CONFIRMATION THE GATE IS DOUBLY SAFE, NOT SINGLY. Even setting the domination-proof\nquestion aside: of 10,607 raws superseded under a semantic head (live query), 10,606 carry\nraw_sessions.revision_authority='quarantined' in the source tier -- never byte_proven -- and\n1 is byte_proven for an unrelated reason. So removing the frontier_kind='byte' SQL predicate\nin _active_index_raw_authority (storage/raw_retention.py:182) would change NOTHING\nobservable: every one of these raws would still be rejected downstream by\n_validate_eligible_receipt's unconditional `revision_kind in {'full','append'}` +\n`revision_authority == 'byte_proven'` requirement on the raw itself\n(storage/raw_retention.py, ~line 326). Multi-session raws (the antigravity population) take\nthe deferred membership-census branch in sources/revision_backfill.py:453 and never acquire\nbyte-proven authority via bind_raw_revision -- that's a separate, upstream fact about the\nauthority model, not something this branch's scope (raw_retention.py/repair.py) can fix.\n\nLIVE NUMBERS (2026-07-29, index.db generation gen-1784807190100-34534407):\n raw_revision_heads: 12,301 byte-frontier heads, 6,429 semantic-frontier heads\n superseded application rows joined to CURRENT head (any freshness): byte=1,184,\n semantic=10,862 (matches this bead's original ~9,320+ estimate within live-drift tolerance)\n distinct raw_ids superseded under a semantic head: 10,607, summing ~28.1 GB raw blob_size\n (upper bound on retained \"already-semantically-superseded\" evidence)\n revision_kind/authority of those 10,607 raws: unknown/quarantined=5,785, full/quarantined=4,821,\n full/byte_proven=1 -- i.e. 10,606/10,607 (99.99%) lack the authority state\n _validate_eligible_receipt requires regardless of frontier_kind\n\nDECISION: (b) from this bead's original framing -- accept the semantic-headed population as\npermanently retained evidence under the current authority model. This is NOT \"debt\" to keep\nre-litigating: releasing it safely would require a genuinely new, schema-bearing proof\nmechanism (a durable, source-tier-anchored fingerprint of the domination proof, e.g.\npersisting the accepted message-hash-sequence identity rather than a scalar count, PLUS an\nindependent re-verification step mirroring _validate_byte_head for semantic heads, PLUS\npromoting multi-session raws' raw_sessions.revision_authority off 'quarantined' through some\nanalogous byte_proven-equivalent check) touching raw_authority.py, revision_application.py,\nand sources/revision_backfill.py, and very likely a derived-tier schema bump. All three are\nexplicitly out of this task's write scope (raw_retention.py/repair.py only) and out of\n\"no index schema bump\" scope. If a future operator wants to fund building that mechanism, it\nis a new, separate, ground-up design -- not a relaxation of this predicate.\n\nNO CODE CHANGE TO ELIGIBILITY. Per this task's constraint (\"a wrong answer here destroys\nevidence permanently... do not manufacture a release path\"), no release-eligibility logic\nchanged. Landed only documentation at the two decision points (\n_active_index_raw_authority's SQL predicate and plan_stale_supersession_reissue's docstring,\nstorage/raw_retention.py) recording this verdict + evidence inline, so a future reader does\nnot reopen this as a quick relaxation without rereading this note first. Commit 89afb0850 on\nbranch feature/chore/promote-schemas-and-wire-gates (worktree\nworktree-agent-a796fb2b17960dafa). Verification: devtools test\ntests/unit/storage/test_raw_retention.py -- 63 passed (docstring-only change, no logic\ntouched). devtools verify --quick -- exit 0.","status":"closed","priority":2,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-29T06:45:40Z","created_by":"Sinity","updated_at":"2026-07-29T08:31:04Z","closed_at":"2026-07-29T08:31:04Z","close_reason":"Not a bug: verified frontier_kind='byte' is a correct, doubly-enforced safety boundary. Decision (b) recorded — semantic-headed raws stay permanently retained under the current authority model. See notes for full evidence; a real semantic release path is separate, schema-bearing future work, not tracked as debt here.","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/polylogue/daemon/blob_gc_periodic.py b/polylogue/daemon/blob_gc_periodic.py index 7da818d3c1..f8dbd4063b 100644 --- a/polylogue/daemon/blob_gc_periodic.py +++ b/polylogue/daemon/blob_gc_periodic.py @@ -28,6 +28,8 @@ BLOB_GC_INTERVAL_SECONDS = 900 BLOB_GC_MAX_BATCH = 200 +BLOB_PUBLICATION_RECONCILIATION_INTERVAL_SECONDS = BLOB_GC_INTERVAL_SECONDS +BLOB_PUBLICATION_RECONCILIATION_MAX_BATCH = BLOB_GC_MAX_BATCH async def periodic_blob_gc_check(*, catch_up_complete: asyncio.Event | None = None) -> None: @@ -62,6 +64,39 @@ async def periodic_blob_gc_check(*, catch_up_complete: asyncio.Event | None = No logger.warning("blob gc: periodic reclaim failed", exc_info=True) +async def periodic_blob_publication_reconciliation_check(*, catch_up_complete: asyncio.Event | None = None) -> None: + """Periodically clear only terminal publication reservations. + + The storage reconciler retains unreferenced reservations whose blob is + still present. Those unresolved rows are intentionally left for explicit + abandonment policy. Referenced and blob-missing rows are safe to clear, + but only while the archive-wide publisher exclusion is held. + """ + from polylogue.daemon.cli import _await_catch_up_gate, _reconcile_blob_publications + + await _await_catch_up_gate(catch_up_complete, loop_name="blob publication reconciliation") + after_publication_id: str | None = None + while True: + await asyncio.sleep(BLOB_PUBLICATION_RECONCILIATION_INTERVAL_SECONDS) + try: + outcome = await _reconcile_blob_publications( + actor="maintenance.blob_publication_reconciliation", + max_count=BLOB_PUBLICATION_RECONCILIATION_MAX_BATCH, + after_publication_id=after_publication_id, + ) + if outcome is None or outcome.scanned < BLOB_PUBLICATION_RECONCILIATION_MAX_BATCH: + after_publication_id = None + else: + after_publication_id = outcome.last_scanned_publication_id + except sqlite3.OperationalError as exc: + if is_transient_sqlite_lock(exc): + logger.info("blob publication reconciliation: archive busy; retrying on next tick: %s", exc) + continue + logger.warning("blob publication reconciliation: periodic pass failed", exc_info=True) + except Exception: + logger.warning("blob publication reconciliation: periodic pass failed", exc_info=True) + + def run_blob_gc_once(source_db_path_arg: Path, blob_dir: Path) -> BlobGCResult | None: """Run one bounded daemon blob-GC pass, or ``None`` if the blob store is absent.""" from polylogue.storage.blob_gc import run_blob_gc_report @@ -76,6 +111,9 @@ def run_blob_gc_once(source_db_path_arg: Path, blob_dir: Path) -> BlobGCResult | __all__ = [ "BLOB_GC_INTERVAL_SECONDS", "BLOB_GC_MAX_BATCH", + "BLOB_PUBLICATION_RECONCILIATION_INTERVAL_SECONDS", + "BLOB_PUBLICATION_RECONCILIATION_MAX_BATCH", "periodic_blob_gc_check", + "periodic_blob_publication_reconciliation_check", "run_blob_gc_once", ] diff --git a/polylogue/daemon/cli.py b/polylogue/daemon/cli.py index 04cb667e12..9d665c2a46 100644 --- a/polylogue/daemon/cli.py +++ b/polylogue/daemon/cli.py @@ -76,6 +76,7 @@ from polylogue.daemon.parse_prefetch import DaemonParseStage from polylogue.product.raw_authority import RawMaterializationCounts from polylogue.sources.revision_backfill import RawParsePrefetchCache + from polylogue.storage.blob_publication import BlobPublicationReconciliation logger = get_logger(__name__) _CONVERGENCE_DEBT_RETRY_INTERVAL_SECONDS = 60 @@ -330,6 +331,7 @@ async def _await_catch_up_gate( "fts identity drift recompute", "fts orphan audit", "blob gc check", + "blob publication reconciliation", "secret scan sweep", ) _SCHEMA_BLOCKED_OPTIONAL_DRIVE_CATCHUP_LOOP_NAME = "drive source catch-up" @@ -1190,23 +1192,31 @@ async def _bridge_catch_up_complete( target.set() -async def _reconcile_blob_publications() -> None: - """Classify crash-left publication reservations before source catch-up.""" +async def _reconcile_blob_publications( + *, + actor: str = "startup.blob_publications", + max_count: int | None = None, + after_publication_id: str | None = None, +) -> BlobPublicationReconciliation | None: + """Classify crash-left publication reservations against the active archive.""" from polylogue.paths import archive_root + from polylogue.storage.archive_identity import resolve_active_index_path from polylogue.storage.blob_publication import reconcile_blob_publication_reservations_under_exclusion root = archive_root() if not (root / "source.db").exists(): - return + return None # Reconciliation only clears rows under a live ArchiveWriterExclusion; the # `_under_exclusion` entry point acquires it itself so this startup call # cannot silently regress into a no-op reconciliation (polylogue-qs0a). outcome = await daemon_write_coordinator().run_sync( - "startup.blob_publications", + actor, reconcile_blob_publication_reservations_under_exclusion, root / "source.db", root / "blob", - index_db_path=root / "index.db", + index_db_path=resolve_active_index_path(root), + max_count=max_count, + after_publication_id=after_publication_id, ) if ( outcome.cleared_referenced @@ -1230,6 +1240,7 @@ async def _reconcile_blob_publications() -> None: "blob publications: retained %d receipt(s) for inspection or explicit abandonment", retained, ) + return outcome def _drain_raw_materialization_once( @@ -2450,7 +2461,10 @@ async def run_daemon_services( from polylogue.daemon.antigravity_conversation_acquisition import ( periodic_antigravity_conversation_acquisition_check, ) - from polylogue.daemon.blob_gc_periodic import periodic_blob_gc_check + from polylogue.daemon.blob_gc_periodic import ( + periodic_blob_gc_check, + periodic_blob_publication_reconciliation_check, + ) from polylogue.daemon.convergence import DaemonConverger from polylogue.daemon.convergence_stages import make_default_convergence_stages from polylogue.daemon.embedding_backlog import ( @@ -2499,6 +2513,7 @@ async def run_daemon_services( periodic_fts_identity_drift_recompute(catch_up_complete=catch_up_complete_gate), periodic_fts_orphan_audit(catch_up_complete=catch_up_complete_gate), periodic_blob_gc_check(catch_up_complete=catch_up_complete_gate), + periodic_blob_publication_reconciliation_check(catch_up_complete=catch_up_complete_gate), periodic_secret_scan_sweep(catch_up_complete=catch_up_complete_gate), periodic_antigravity_conversation_acquisition_check(catch_up_complete=catch_up_complete_gate), ] diff --git a/polylogue/storage/blob_publication.py b/polylogue/storage/blob_publication.py index 666c825c4c..966cbc3baf 100644 --- a/polylogue/storage/blob_publication.py +++ b/polylogue/storage/blob_publication.py @@ -52,6 +52,8 @@ class BlobPublicationReconciliation: retained_referenced: int = 0 retained_missing: int = 0 unresolved: int = 0 + scanned: int = 0 + last_scanned_publication_id: str | None = None @dataclass(frozen=True, slots=True) @@ -279,10 +281,14 @@ def inspect_blob_publication_receipts( blob_root: Path, *, index_db_path: Path | None = None, + max_count: int | None = None, + after_publication_id: str | None = None, ) -> tuple[BlobPublicationInspection, ...]: - """Return every receipt with its current path/reference evidence.""" + """Return receipt evidence, optionally bounded by a stable ID cursor.""" from polylogue.storage.archive_identity import ArchiveLocation + if max_count is not None and max_count <= 0: + raise ValueError("max_count must be positive when provided") source_conn = sqlite3.connect(f"file:{source_db_path}?mode=ro", uri=True) index_conn: sqlite3.Connection | None = None try: @@ -295,13 +301,34 @@ def inspect_blob_publication_receipts( store = BlobStore(blob_root) if not _table_exists(source_conn, "blob_publication_reservations"): return () - rows = source_conn.execute( - """ - SELECT publication_id, blob_hash, size_bytes, publisher_id, reserved_at_ms - FROM blob_publication_reservations - ORDER BY reserved_at_ms, publication_id - """ - ).fetchall() + if max_count is None and after_publication_id is None: + rows = source_conn.execute( + """ + SELECT publication_id, blob_hash, size_bytes, publisher_id, reserved_at_ms + FROM blob_publication_reservations + ORDER BY reserved_at_ms, publication_id + """ + ).fetchall() + else: + predicates = "" + parameters: list[object] = [] + if after_publication_id is not None: + predicates = "WHERE publication_id > ?" + parameters.append(after_publication_id) + limit = "" + if max_count is not None: + limit = "LIMIT ?" + parameters.append(max_count) + rows = source_conn.execute( + f""" + SELECT publication_id, blob_hash, size_bytes, publisher_id, reserved_at_ms + FROM blob_publication_reservations + {predicates} + ORDER BY publication_id + {limit} + """, + parameters, + ).fetchall() return tuple( BlobPublicationInspection( publication_id=str(row["publication_id"]), @@ -326,12 +353,16 @@ def reconcile_blob_publication_reservations( *, index_db_path: Path | None = None, writer_exclusion: ArchiveWriterExclusion | None = None, + max_count: int | None = None, + after_publication_id: str | None = None, ) -> BlobPublicationReconciliation: """Classify receipts; clear safe rows only with archive-wide exclusion.""" inspections = inspect_blob_publication_receipts( source_db_path, blob_root, index_db_path=index_db_path, + max_count=max_count, + after_publication_id=after_publication_id, ) may_clear = ( writer_exclusion is not None @@ -379,6 +410,8 @@ def reconcile_blob_publication_reservations( retained_referenced=retained_referenced, retained_missing=retained_missing, unresolved=unresolved, + scanned=len(inspections), + last_scanned_publication_id=inspections[-1].publication_id if inspections else None, ) @@ -387,6 +420,8 @@ def reconcile_blob_publication_reservations_under_exclusion( blob_root: Path, *, index_db_path: Path | None = None, + max_count: int | None = None, + after_publication_id: str | None = None, ) -> BlobPublicationReconciliation: """Reconcile receipts while holding archive-wide publisher exclusion. @@ -403,6 +438,8 @@ def reconcile_blob_publication_reservations_under_exclusion( blob_root, index_db_path=index_db_path, writer_exclusion=exclusion, + max_count=max_count, + after_publication_id=after_publication_id, ) diff --git a/tests/unit/daemon/test_blob_gc_periodic.py b/tests/unit/daemon/test_blob_gc_periodic.py index 6a8dc8491d..35291119ea 100644 --- a/tests/unit/daemon/test_blob_gc_periodic.py +++ b/tests/unit/daemon/test_blob_gc_periodic.py @@ -14,12 +14,16 @@ from __future__ import annotations import asyncio +import contextlib import os import sqlite3 from pathlib import Path +import pytest + +from polylogue.daemon import blob_gc_periodic from polylogue.daemon.blob_gc_periodic import run_blob_gc_once -from polylogue.daemon.write_coordinator import DaemonWriteCoordinator +from polylogue.daemon.write_coordinator import DaemonWriteCoordinator, DaemonWriteEvent from polylogue.storage.blob_store import BlobStore @@ -120,3 +124,210 @@ async def run() -> object: assert result.deleted_count == 1 # type: ignore[attr-defined] assert coordinator.snapshot().active_actor is None assert not blob_store.blob_path(blob_hash).exists() + + +def _make_publication_reconciliation_fixture(tmp_path: Path) -> tuple[Path, str]: + from polylogue.core.enums import Origin + from polylogue.storage.blob_publication import ArchiveBlobPublisher + from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root + from polylogue.storage.sqlite.archive_tiers.source_write import write_source_raw_session_blob_ref + + archive_root = tmp_path / "archive" + initialize_active_archive_root(archive_root) + source_db = archive_root / "source.db" + store = BlobStore(archive_root / "blob") + publisher = ArchiveBlobPublisher(source_db, store.root) + missing_hash, _ = publisher.write_from_bytes(b"periodic-missing-terminal") + referenced_hash, referenced_size = publisher.write_from_bytes(b"periodic-referenced-terminal") + unresolved_hash, _ = publisher.write_from_bytes(b"periodic-unresolved") + publisher.flush() + store.blob_path(missing_hash).unlink() + with sqlite3.connect(source_db) as conn: + write_source_raw_session_blob_ref( + conn, + origin=Origin.CHATGPT_EXPORT, + source_path="periodic-referenced.json", + source_index=0, + blob_hash=bytes.fromhex(referenced_hash), + blob_size=referenced_size, + acquired_at_ms=1, + raw_id="periodic-referenced-raw", + ) + return archive_root, unresolved_hash + + +def test_periodic_publication_reconciliation_repeats_safe_cleanup_and_retains_unresolved( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """The scheduled production route clears terminal rows on every tick only.""" + archive_root, unresolved_hash = _make_publication_reconciliation_fixture(tmp_path) + source_db = archive_root / "source.db" + coordinator_events: list[str] = [] + second_tick = asyncio.Event() + + def observe(event: DaemonWriteEvent) -> None: + if event.phase == "acquired": + coordinator_events.append(event.actor) + if len(coordinator_events) == 2: + second_tick.set() + + coordinator = DaemonWriteCoordinator(observer=observe) + monkeypatch.setattr(blob_gc_periodic, "BLOB_PUBLICATION_RECONCILIATION_INTERVAL_SECONDS", 0) + monkeypatch.setattr("polylogue.paths.archive_root", lambda: archive_root) + monkeypatch.setattr("polylogue.paths.source_db_path", lambda: source_db) + monkeypatch.setattr("polylogue.daemon.cli.daemon_write_coordinator", lambda: coordinator) + + async def exercise() -> None: + task = asyncio.create_task(blob_gc_periodic.periodic_blob_publication_reconciliation_check()) + try: + await asyncio.wait_for(second_tick.wait(), timeout=2.0) + finally: + if not task.done(): + task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await task + assert await coordinator.shutdown(timeout=1.0) + + asyncio.run(exercise()) + + assert coordinator_events == [ + "maintenance.blob_publication_reconciliation", + "maintenance.blob_publication_reconciliation", + ] + assert coordinator.snapshot().active_actor is None + with sqlite3.connect(source_db) as conn: + remaining = conn.execute( + "SELECT blob_hash FROM blob_publication_reservations ORDER BY publication_id" + ).fetchall() + assert [bytes(row[0]).hex() for row in remaining] == [unresolved_hash] + + +def test_periodic_publication_reconciliation_pages_past_unresolved_rows( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """A bounded pass advances past retained rows instead of starving later cleanup.""" + from polylogue.core.enums import Origin + from polylogue.storage.blob_publication import ArchiveBlobPublisher + from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root + from polylogue.storage.sqlite.archive_tiers.source_write import write_source_raw_session_blob_ref + + archive_root = tmp_path / "archive" + initialize_active_archive_root(archive_root) + source_db = archive_root / "source.db" + store = BlobStore(archive_root / "blob") + publisher = ArchiveBlobPublisher(source_db, store.root) + unresolved_a, _ = publisher.write_from_bytes(b"bounded-unresolved-a") + unresolved_b, _ = publisher.write_from_bytes(b"bounded-unresolved-b") + missing_hash, _ = publisher.write_from_bytes(b"bounded-missing") + referenced_hash, referenced_size = publisher.write_from_bytes(b"bounded-referenced") + receipts = publisher.flush() + deterministic_ids = { + unresolved_a: "publication-a", + unresolved_b: "publication-b", + missing_hash: "publication-c", + referenced_hash: "publication-d", + } + store.blob_path(missing_hash).unlink() + with sqlite3.connect(source_db) as conn: + for receipt in receipts: + conn.execute( + "UPDATE blob_publication_reservations SET publication_id = ? WHERE publication_id = ?", + (deterministic_ids[receipt.blob_hash], receipt.publication_id), + ) + write_source_raw_session_blob_ref( + conn, + origin=Origin.CHATGPT_EXPORT, + source_path="bounded-referenced.json", + source_index=0, + blob_hash=bytes.fromhex(referenced_hash), + blob_size=referenced_size, + acquired_at_ms=1, + raw_id="bounded-referenced-raw", + ) + + acquired = asyncio.Event() + coordinator_events: list[str] = [] + + def observe(event: DaemonWriteEvent) -> None: + if event.phase == "acquired": + coordinator_events.append(event.actor) + if len(coordinator_events) == 2: + acquired.set() + + coordinator = DaemonWriteCoordinator(observer=observe) + monkeypatch.setattr(blob_gc_periodic, "BLOB_PUBLICATION_RECONCILIATION_INTERVAL_SECONDS", 0) + monkeypatch.setattr(blob_gc_periodic, "BLOB_PUBLICATION_RECONCILIATION_MAX_BATCH", 2) + monkeypatch.setattr("polylogue.paths.archive_root", lambda: archive_root) + monkeypatch.setattr("polylogue.daemon.cli.daemon_write_coordinator", lambda: coordinator) + + async def exercise() -> None: + task = asyncio.create_task(blob_gc_periodic.periodic_blob_publication_reconciliation_check()) + try: + await asyncio.wait_for(acquired.wait(), timeout=2.0) + finally: + task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await task + assert await coordinator.shutdown(timeout=1.0) + + asyncio.run(exercise()) + + assert coordinator_events == [ + "maintenance.blob_publication_reconciliation", + "maintenance.blob_publication_reconciliation", + ] + with sqlite3.connect(source_db) as conn: + remaining = conn.execute( + "SELECT publication_id FROM blob_publication_reservations ORDER BY publication_id" + ).fetchall() + assert remaining == [("publication-a",), ("publication-b",)] + + +def test_blob_publication_reconciliation_reads_attachment_refs_from_active_index( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """The daemon resolves the promoted generation before classifying receipts.""" + from polylogue.daemon.cli import _reconcile_blob_publications + from polylogue.storage.blob_publication import ArchiveBlobPublisher + from polylogue.storage.sqlite.archive_tiers.bootstrap import ( + initialize_active_archive_root, + initialize_archive_database, + ) + from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier + + archive_root = tmp_path / "archive" + initialize_active_archive_root(archive_root) + source_db = archive_root / "source.db" + store = BlobStore(archive_root / "blob") + publisher = ArchiveBlobPublisher(source_db, store.root) + blob_hash, size = publisher.write_from_bytes(b"active-generation-attachment") + publisher.flush() + + active_index = archive_root / ".index-generations" / "gen-active" / "index.db" + active_index.parent.mkdir(parents=True) + initialize_archive_database(active_index, ArchiveTier.INDEX) + with sqlite3.connect(active_index) as conn: + conn.execute( + """ + INSERT INTO attachments(attachment_id, blob_hash, byte_count, acquisition_status, ref_count) + VALUES ('active-attachment', ?, ?, 'acquired', 1) + """, + (bytes.fromhex(blob_hash), size), + ) + conn.commit() + (archive_root / ".index-active-pointer").write_text(str(active_index.resolve()), encoding="utf-8") + + coordinator = DaemonWriteCoordinator() + monkeypatch.setattr("polylogue.paths.archive_root", lambda: archive_root) + monkeypatch.setattr("polylogue.daemon.cli.daemon_write_coordinator", lambda: coordinator) + + outcome = asyncio.run(_reconcile_blob_publications(actor="maintenance.blob_publication_reconciliation")) + + assert outcome is not None + assert outcome.cleared_referenced == 1 + assert outcome.scanned == 1 + with sqlite3.connect(source_db) as conn: + assert conn.execute("SELECT COUNT(*) FROM blob_publication_reservations").fetchone()[0] == 0 diff --git a/tests/unit/daemon/test_daemon_cli.py b/tests/unit/daemon/test_daemon_cli.py index 6b6f6c63e1..af236c5bcc 100644 --- a/tests/unit/daemon/test_daemon_cli.py +++ b/tests/unit/daemon/test_daemon_cli.py @@ -3688,6 +3688,12 @@ def fake_emit_daemon_event(kind: str, **kwargs: object) -> None: stack.enter_context(patch.object(daemon_cli, "_configure_fts_automerge", fake_configure_fts_automerge)) stack.enter_context(patch.object(daemon_cli, "_periodic_wal_checkpoint", lambda: fake_loop("wal"))) stack.enter_context(patch.object(daemon_cli, "_periodic_fts_merge", lambda: fake_loop("fts-merge"))) + stack.enter_context( + patch( + "polylogue.daemon.blob_gc_periodic.periodic_blob_publication_reconciliation_check", + lambda **_kwargs: fake_loop("blob-publication-reconciliation"), + ) + ) stack.enter_context( patch.object( daemon_cli, @@ -3744,6 +3750,7 @@ def fake_emit_daemon_event(kind: str, **kwargs: object) -> None: assert events.index("fts") < events.index("watcher") assert events.index("fts") < events.index("lineage") < events.index("watcher") assert events.index("lineage") < events.index("blob-publications") < events.index("watcher") + assert "blob-publication-reconciliation" in events # Drive catch-up is background work: it must never gate the watcher or # the local convergence loops on serial network I/O. assert "drive-once" not in events @@ -3990,6 +3997,10 @@ async def exercise() -> None: patch.object(daemon_cli, "_run_drive_source_catchup_safely", no_drive_changes), patch.object(daemon_cli, "_periodic_wal_checkpoint", wait_forever), patch.object(daemon_cli, "_periodic_fts_merge", wait_forever), + patch( + "polylogue.daemon.blob_gc_periodic.periodic_blob_publication_reconciliation_check", + lambda **_kwargs: wait_forever(), + ), patch.object(daemon_cli, "_periodic_heartbeat", wait_forever), patch.object(daemon_cli, "_periodic_drive_source_catchup", wait_forever), patch.object(daemon_cli, "_periodic_health_check", wait_forever),