From 62232e591ad497decc90f7ad99fbc72d14d4280c Mon Sep 17 00:00:00 2001 From: Sinity Date: Wed, 22 Jul 2026 20:07:38 +0200 Subject: [PATCH 1/2] feat(maintenance): raw-authority census reset + orphaned index-seed prune MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Convergence recovery for the hook de-inflation aftermath (polylogue-31r1). Deleting hook raw_sessions left residue in two derived read models that blocked daemon convergence; both are cleaned here with tested, offline-guarded, dry-run-capable functions. - reset_raw_authority_census: empties the census PLANNING ledger (censuses/plans/blockers/census_plans/census_post_plans; parser_census kept). The ledger chains each census to its predecessor and carries plans forward; a stale-plan blocker had marked siblings CARRIED_FORWARD and later raw deletions dropped them from the recomputed frontier, so the finalize postflight (persistent ⊄ post_ids) threw every pass and no census could finalize to a clean baseline. Emptying it lets the daemon build census #1 fresh (no predecessor, no carried-forward poison). Accepted state (raw_sessions.revision_authority + index raw_revision_heads/applications) is separate and preserved. - prune_orphaned_index_revision_seeds: deletes index raw_revision_heads / raw_revision_applications whose raw is gone from source, i.e. the broken predecessor chains the daemon's raw-frontier integrity check reports as violated and cannot converge past. Live effect: census unblocked (no postflight error; sequence advances again), the pre-existing stale-plan blocker cleared, and the daemon's broken-seed count fell 67,105 -> 2,210 (the residual all reference PRESENT raws = genuine pre-existing t93b chain-repair work, untouched here). Accepted materialization (36,451 raws, 18,391 sessions, 4.77M messages) preserved throughout. Ref polylogue-31r1, polylogue-t93b. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01QUsH3Rhq6oAZpYPWcsZqnZ --- .beads/issues.jsonl | 4 +- docs/plans/topology-target.yaml | 6 +- docs/topology-status.md | 6 +- polylogue/maintenance/raw_authority_reset.py | 152 ++++++++++++++++++ .../maintenance/test_raw_authority_reset.py | 115 +++++++++++++ 5 files changed, 277 insertions(+), 6 deletions(-) create mode 100644 polylogue/maintenance/raw_authority_reset.py create mode 100644 tests/unit/maintenance/test_raw_authority_reset.py diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index dfde7d916e..e95027579c 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -1,4 +1,4 @@ -{"_type":"issue","id":"polylogue-31r1","title":"Hook events ingested as standalone sessions inflate archive ~4.7x (65.7K empty shells)","design":"Root cause (airtight, 2026-07-22): polylogue/sources/hooks.py:_persist_record drains each spooled hook event (~/.local/share/polylogue/hooks/pending/\u003cid\u003e.json: PreToolUse/PostToolUse/UserPromptSubmit/SessionStart/...) and calls write_source_raw_session with origin=codex-session|claude-code-session, minting a full raw_sessions row per hook -\u003e the materializer turns each into an EMPTY standalone index session (0 messages). Each hook is double-recorded: correctly as a raw_hook_events row carrying session_native_id (table indexed (origin,session_native_id,observed_at_ms) for attach-to-session), AND wrongly as a raw_sessions row.\n\nScale on live archive /realm/db/polylogue: index sessions=83,279 but only 17,553 have content; 65,727 empty shells = codex 35,233 + claude-code 30,488. source_path LIKE '%/hooks/%' raws: codex 35,216 + claude-code 29,679 + hermes 1 = 64,896 = raw_hook_events row count. Real conversations ~17.5K (matches operator memory of ~16K). raw_hook_events has NO FK to raw_sessions, so hooks can persist without minting sessions.\n\nAlso inflates the raw-authority reconciler backlog (hjpx/lkrc/t93b) which churns over hook raws mixed with real session raws.\n\nFIX (operator decisions 2026-07-22): (1) code: add write_source_hook_event writing raw_hook_events + retained blob_ref, NO raw_sessions row; _persist_record uses it; materializer guard so hook-origin raws never become sessions; covers codex/claude/hermes. (2) constructive: materialize raw_hook_events into an index read-model attached to sessions via session_native_id (index tier rebuildable) + read surfaces (MCP/CLI). Operator: hooks are always within a session; link them. (3) retroactive repair WITHOUT full reindex: delete 64,896 hook raw_sessions rows from source.db (durable; backup at /realm/staging/polylogue-sqlite/recovery/t93b-preflight-20260722-durable) + 64,896 empty index session rows (zero messages/blocks/FTS -\u003e tiny blast radius, targeted DELETE). keep raw_hook_events+blobs. (4) re-census raw-authority; deploy #3261 (whale budget fix, merged) so frontier repair doesn't abort on 298MB whale.\n\nDesign doc: .agent/scratch/hook-session-inflation-2026-07-22.md. Verification: sessions count ~17.5K post-repair; every hook event still resolves to its session via session_native_id; no message/block/FTS row dropped.","status":"in_progress","priority":0,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-22T12:42:26Z","created_by":"Sinity","updated_at":"2026-07-22T14:32:39Z","started_at":"2026-07-22T14:32:39Z","comments":[{"id":"019f8a3e-3243-78be-9150-ea16cdde6316","issue_id":"polylogue-31r1","author":"Sinity","text":"Progress 2026-07-22: code fix + retroactive repair MERGED as PR #3265 (master 998188b1a). Both Codex P1 review findings fixed (verified-hook-row predicate via raw_hook_events.source_path; daemon offline-guard on apply). Live dry-run confirms scope: 64,896 hook raw_sessions, 64,895 empty index shells, raw_hook_events + 64,896 blobs retained.\n\nDeploying now: sinnix flake bumped to 998188b1a (commit 34a7a41), switch building. Next: stop polylogued -\u003e run repair_hook_session_inflation(dry_run=False) -\u003e restart -\u003e verify sessions count ~17.5K. Then session\u003c-\u003ehooks read-model + surfaces (fast-follow) and re-census raw-authority for hjpx/lkrc/t93b.","created_at":"2026-07-22T14:32:40Z"},{"id":"019f8a68-335c-7e94-bf69-0e7c18bed99a","issue_id":"polylogue-31r1","author":"Sinity","text":"DE-INFLATION COMPLETE + VERIFIED LIVE (2026-07-22).\n\nDeployed 998188b1a to daemon (sinnix 34a7a41, switch OK; daemon restarted on new store, confirmed has write_hook_event). Stopped daemon, ran repair_hook_session_inflation(dry_run=False), restarted.\n\nLive before -\u003e after:\n- index sessions: 83,286 -\u003e 18,391 (17,559 real + 832 genuinely-empty non-hook, correctly preserved)\n- raw_sessions: 101,347 -\u003e 36,451 (-64,896 hook raws)\n- raw_hook_events: 64,896 -\u003e 64,896 (all evidence retained)\n- hook blobs: 64,896 retained\n- hook raw_sessions after repair: 0; still 0 after daemon restart+drain -\u003e no re-inflation, going-forward fix confirmed live.\n\nRoot cause fully characterized: 64,896 hook events came from just 64 real agent sessions (one codex session fired 13,447 Pre/PostToolUse hooks). Each hook had become its own empty \"session\". Now 64,896 evidence rows attached to their 64 parent sessions via session_native_id.\n\nREMAINING (fast-follow, this bead stays open): session\u003c-\u003ehooks index read-model + read surfaces (MCP/CLI) so hooks are queryable as session evidence. Separate: raw-authority convergence (hjpx/lkrc/t93b) still degraded on pre-existing stale-plan blocker f196aac0 — unaffected by this work.","created_at":"2026-07-22T15:18:33Z"}],"dependency_count":0,"dependent_count":0,"comment_count":2} +{"_type":"issue","id":"polylogue-31r1","title":"Hook events ingested as standalone sessions inflate archive ~4.7x (65.7K empty shells)","design":"Root cause (airtight, 2026-07-22): polylogue/sources/hooks.py:_persist_record drains each spooled hook event (~/.local/share/polylogue/hooks/pending/\u003cid\u003e.json: PreToolUse/PostToolUse/UserPromptSubmit/SessionStart/...) and calls write_source_raw_session with origin=codex-session|claude-code-session, minting a full raw_sessions row per hook -\u003e the materializer turns each into an EMPTY standalone index session (0 messages). Each hook is double-recorded: correctly as a raw_hook_events row carrying session_native_id (table indexed (origin,session_native_id,observed_at_ms) for attach-to-session), AND wrongly as a raw_sessions row.\n\nScale on live archive /realm/db/polylogue: index sessions=83,279 but only 17,553 have content; 65,727 empty shells = codex 35,233 + claude-code 30,488. source_path LIKE '%/hooks/%' raws: codex 35,216 + claude-code 29,679 + hermes 1 = 64,896 = raw_hook_events row count. Real conversations ~17.5K (matches operator memory of ~16K). raw_hook_events has NO FK to raw_sessions, so hooks can persist without minting sessions.\n\nAlso inflates the raw-authority reconciler backlog (hjpx/lkrc/t93b) which churns over hook raws mixed with real session raws.\n\nFIX (operator decisions 2026-07-22): (1) code: add write_source_hook_event writing raw_hook_events + retained blob_ref, NO raw_sessions row; _persist_record uses it; materializer guard so hook-origin raws never become sessions; covers codex/claude/hermes. (2) constructive: materialize raw_hook_events into an index read-model attached to sessions via session_native_id (index tier rebuildable) + read surfaces (MCP/CLI). Operator: hooks are always within a session; link them. (3) retroactive repair WITHOUT full reindex: delete 64,896 hook raw_sessions rows from source.db (durable; backup at /realm/staging/polylogue-sqlite/recovery/t93b-preflight-20260722-durable) + 64,896 empty index session rows (zero messages/blocks/FTS -\u003e tiny blast radius, targeted DELETE). keep raw_hook_events+blobs. (4) re-census raw-authority; deploy #3261 (whale budget fix, merged) so frontier repair doesn't abort on 298MB whale.\n\nDesign doc: .agent/scratch/hook-session-inflation-2026-07-22.md. Verification: sessions count ~17.5K post-repair; every hook event still resolves to its session via session_native_id; no message/block/FTS row dropped.","status":"in_progress","priority":0,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-22T12:42:26Z","created_by":"Sinity","updated_at":"2026-07-22T14:32:39Z","started_at":"2026-07-22T14:32:39Z","comments":[{"id":"019f8a3e-3243-78be-9150-ea16cdde6316","issue_id":"polylogue-31r1","author":"Sinity","text":"Progress 2026-07-22: code fix + retroactive repair MERGED as PR #3265 (master 998188b1a). Both Codex P1 review findings fixed (verified-hook-row predicate via raw_hook_events.source_path; daemon offline-guard on apply). Live dry-run confirms scope: 64,896 hook raw_sessions, 64,895 empty index shells, raw_hook_events + 64,896 blobs retained.\n\nDeploying now: sinnix flake bumped to 998188b1a (commit 34a7a41), switch building. Next: stop polylogued -\u003e run repair_hook_session_inflation(dry_run=False) -\u003e restart -\u003e verify sessions count ~17.5K. Then session\u003c-\u003ehooks read-model + surfaces (fast-follow) and re-census raw-authority for hjpx/lkrc/t93b.","created_at":"2026-07-22T14:32:40Z"},{"id":"019f8a68-335c-7e94-bf69-0e7c18bed99a","issue_id":"polylogue-31r1","author":"Sinity","text":"DE-INFLATION COMPLETE + VERIFIED LIVE (2026-07-22).\n\nDeployed 998188b1a to daemon (sinnix 34a7a41, switch OK; daemon restarted on new store, confirmed has write_hook_event). Stopped daemon, ran repair_hook_session_inflation(dry_run=False), restarted.\n\nLive before -\u003e after:\n- index sessions: 83,286 -\u003e 18,391 (17,559 real + 832 genuinely-empty non-hook, correctly preserved)\n- raw_sessions: 101,347 -\u003e 36,451 (-64,896 hook raws)\n- raw_hook_events: 64,896 -\u003e 64,896 (all evidence retained)\n- hook blobs: 64,896 retained\n- hook raw_sessions after repair: 0; still 0 after daemon restart+drain -\u003e no re-inflation, going-forward fix confirmed live.\n\nRoot cause fully characterized: 64,896 hook events came from just 64 real agent sessions (one codex session fired 13,447 Pre/PostToolUse hooks). Each hook had become its own empty \"session\". Now 64,896 evidence rows attached to their 64 parent sessions via session_native_id.\n\nREMAINING (fast-follow, this bead stays open): session\u003c-\u003ehooks index read-model + read surfaces (MCP/CLI) so hooks are queryable as session evidence. Separate: raw-authority convergence (hjpx/lkrc/t93b) still degraded on pre-existing stale-plan blocker f196aac0 — unaffected by this work.","created_at":"2026-07-22T15:18:33Z"},{"id":"019f8ac1-a0fc-7cdb-84a2-7b3fa2d1be2e","issue_id":"polylogue-31r1","author":"Sinity","text":"INCIDENT + FIX 2026-07-22: first daemon convergence pass after the live de-inflation threw RuntimeError(\"duplicate strategy did not reach its typed terminal postcondition\"). Cause: the repair deleted hook raw_sessions but raw_authority_plans/blockers/census reference raws by JSON string (no FK), leaving 64,895 orphaned frontier plans. Daemon caught it (0 restarts), stopped it, verified clean rollback of an over-slow first cleanup attempt.\n\nFix PR #3266: prune purely-orphaned authority plans+children in the repair; set-based identification (0.3s vs \u003e1h correlated) + temp plan_id indexes for FK-restrict/IN deletes. Live: 64,895 orphans pruned (plans 84,042-\u003e19,147, blockers 70,887-\u003e5,992, census_plans 405,234-\u003e275,444, census_post_plans 323,877-\u003e258,982), 0 remain, daemon restarted 0 tracebacks in 8min. Confirms hook raws were also flooding raw-authority (~65K plan/blocker noise) -\u003e should lighten hjpx/lkrc/t93b convergence.","created_at":"2026-07-22T16:56:13Z"}],"dependency_count":0,"dependent_count":0,"comment_count":3} {"_type":"issue","id":"polylogue-p0pw","title":"Process-pool forkserver deadlocks in production parse path: zero workers ever spawn","design":"Evidence (2026-07-19 03:00): CLI `ops maintenance rebuild-index` sat 17+ minutes at 16% CPU, zero index-generation growth. py-spy: parent idle in as_completed (_parse_retained_raws revision_backfill.py:719); the only children were the multiprocessing resource-tracker and the forkserver itself, both idle — no pool worker was EVER spawned. Killing and resuming the same transaction with POLYLOGUE_INGEST_PARSE_WORKERS=1 (sequential escape hatch) went to 97% CPU immediately and the generation resumed growing. Same pathology long documented on this host for testmon xdist (bd memory devtools-verify-testmon-forkserver-deadlock). Root: polylogue/pipeline/services/process_pool.py process_pool_context() prefers forkserver whenever available. Fix direction: use spawn (still safe for multi-threaded parents, slower per-worker startup but workers are long-lived here), or diagnose why forkserver never services spawn requests under a threaded asyncio parent (as_completed caller runs on an executor thread). Must also audit the daemon census path (#3122 wired the same helper into polylogued at ingest_workers=cpu-1): daemon census passes were observed parsing large payloads inline (size-aware dispatch), but any pool-eligible small-payload batch may hang or silently serialize the same way.","acceptance_criteria":"Reproduce or conclusively explain the forkserver no-worker deadlock; switch process_pool_context to a start method that demonstrably spawns workers on this host under a threaded parent; regression test that a pool dispatch from a worker thread completes; verify daemon census throughput with pooling active; remove/keep the workers=1 escape hatch documented.","notes":"2026-07-19 03:30 repro results: minimal repro (asyncio-thread -\u003e ProcessPoolExecutor(forkserver) -\u003e as_completed, plain function) PASSES on this host in 0.2s — the deadlock is NOT environmental; it is polylogue-specific state. Sharpened evidence from the stuck run: the forkserver process WAS in its serve loop (select at forkserver.py:231), resource tracker alive, yet ZERO workers were ever spawned and the parent executor never completed a future. Suspect surface (in order): (1) pool initializer _initialize_worker_logging -\u003e configure_logging importing polylogue inside spawned worker; (2) forkserver preload of __main__ (cmdline showed main_path=.venv/bin/polylogue) re-importing the whole CLI in the forkserver at boot; (3) executor manager thread wedged in the parent (a Thread was parked in selectors select). Repro script: /realm/tmp/claude-code/claude-1000/-realm-project-polylogue/af12164b-a2fc-42cb-a548-22277c0875a2/scratchpad/forkserver_repro.py — next step is to extend it to use polylogue process_pool_executor() verbatim, then add the real initializer, then real submission payloads, bisecting which ingredient hangs.\n2026-07-19 04:00 bisect step (b) result: running polylogue process_pool_executor() from a thread under stdin exposed the mechanism — forkserver PRELOADS __main__ via runpy.run_path(sys.argv[0], run_name=__mp_main__) (observed FileNotFoundError for \u003cstdin\u003e crashing the forkserver at boot -\u003e EOFError in parent). In the real CLI, main_path=.venv/bin/polylogue, so the ENTIRE polylogue CLI import graph executes inside the forkserver process at pool creation. Any thread started or lock acquired during that import is inherited (in locked/running state) by every forked worker -\u003e classic fork-of-threaded-process deadlock, consistent with the observed zero-workers hang while the forkserver sat in its serve loop. Note for the fix: spawn ALSO re-imports __main__ per worker (slow ~1-2s/worker startup with the full CLI import, but no inherited-lock hazard). Options: (a) spawn (safe, pay startup once per long-lived worker); (b) forkserver with set_forkserver_preload([]) — but stdlib preloads __main__ unconditionally via main_path... verify whether multiprocessing.spawn.set_executable / context.set_forkserver_preload can suppress __main__ preload; (c) audit what the CLI import graph starts (threads at import time is itself a smell worth fixing). Repro next step for the lane: run the same test from a real script file so main_path resolves, confirm hang, then bisect the import graph for thread/lock creation.\n2026-07-19 04:10: repro relocated to a durable path: /realm/project/polylogue/.agent/scratch/warroom-2026-07-17/forkserver_repro.py (the /realm/tmp scratchpad copy may be cleaned). Lane worktree pre-created: /realm/worktrees/polylogue-lane-h-pool (branch feature/perf/process-pool-spawn from 86ca3287b).\n2026-07-19 lane H: bisect step (c)+(d) result — REFUTES the leading hypothesis\nfrom the prior session. Extended repro\n(.agent/scratch/warroom-2026-07-17/forkserver_repro.py sibling, run as a real\nscript file so sys.argv[0] resolves like production main_path): top-level\n`from polylogue.cli import main` (byte-identical to .venv/bin/polylogue's\nentry-point shape) followed by dispatching process_pool_executor() from a\nworker thread, under both forkserver and spawn contexts. Result: BOTH\ncomplete in 0.6s — no hang. So \"the CLI import graph alone creates a\nthread/lock that forkserver's worker-fork inherits\" does not reproduce\nsynthetically when isolated to import+dispatch. The exact trigger inside the\nproduction forkserver preload (which DID visibly hang: forkserver alive in\nits serve loop, zero workers ever spawned, parent parked forever in\nas_completed at revision_backfill.py:719) remains unconfirmed by a\nstandalone repro; likely needs live-process instrumentation (e.g. py-spy\nagainst a real ops maintenance rebuild-index run) to pin exactly, which is\nout of the ~90min bisect timebox for this lane.\n\nApplied fix per the lane brief's explicit fallback (\"otherwise just switch\nto spawn and delete nothing else\"): process_pool_context() now\nunconditionally returns spawn, never forkserver. This is engineering-sound\nindependent of pinning the exact trigger: spawn reruns __main__ fresh per\nworker instead of forking one shared preloaded process, which structurally\neliminates the whole class of inherited-thread/lock hazards forkserver is\nexposed to (not just the specific one hypothesized). Cost is ~1-2s import\nper worker, acceptable since pool workers here are long-lived and reused\nacross many parse tasks (not short bursts).\n\nLanded: polylogue/pipeline/services/process_pool.py (spawn unconditional,\ndocstring explains why) + tests/unit/pipeline/test_process_pool.py (new\ntest_process_pool_context_is_spawn pins the exact start method rather than\njust excluding fork; new\ntest_process_pool_dispatch_from_worker_thread_completes dispatches 8 tasks\nacross 4 workers from a daemon thread with a 40s join bound + pytest\ntimeout(45), mirroring the asyncio-thread -\u003e pool -\u003e as_completed\nproduction shape). Both pass locally (devtools test\ntests/unit/pipeline/test_process_pool.py: 4 passed in 5.10s). Note: this\nregression test does NOT reproduce the hang pre-fix either (consistent with\nthe synthetic-repro gap above) — it is a forward-looking guard against ever\nreintroducing a hanging start-method config, not a proof the pre-fix code\nwould fail it. Honesty note per AC: \"regression test that a pool dispatch\nfrom a worker thread completes\" is satisfied; \"reproduce or conclusively\nexplain the forkserver no-worker deadlock\" is only partially satisfied —\nexplained mechanism (forkserver forks every worker from one preloaded\nprocess; production main_path preloads the whole CLI graph) but not\nconclusively reproduced or pinned to one exact statement/import.\n\nAlso: mid-session process error caught and corrected — an errant `cd\n/realm/project/polylogue \u0026\u0026 ...` left the shell cwd on the main checkout\nacross later commands, so the first commit attempt landed on master there\n(8672f9768). Recovered cleanly: cherry-picked the commit onto\nfeature/perf/process-pool-spawn in the correct worktree\n(/realm/worktrees/polylogue-lane-h-pool, now 07b7835b2), then `git fetch`\n+ `git reset --hard origin/master` in the main checkout to restore it to\nclean origin state. No data lost, no other lanes' work touched (verified\ngit status was clean before the reset). Main checkout confirmed back at\n86ca3287b matching origin/master.\n\nNext: task 3 (daemon census pooling-in-production audit, report only) and\nverify + PR.\n2026-07-19 lane H: daemon census pooling-in-production audit (AC item 4, report only).\n\nAnswer: NO, the ambient/periodic daemon convergence pool has never\nactivated in production, and the #3122-wired census pool has only ever run\nvia direct CLI invocation, never through the live daemon process.\n\nEvidence:\n1. DaemonConverger.start() logs \"converger: started with %d worker(s)\"\n when _has_cpu_bound_stage() is True, else \"started without worker\n pool\". `journalctl --since -60days | grep \"converger: started\"` shows\n ONLY \"started without worker pool\" — every polylogued startup in the\n observed window (30+ restarts across 2026-07-16..19), zero exceptions.\n Root cause confirmed in source: every ConvergenceStage definition in\n daemon/convergence_stages.py sets cpu_bound=False (5/5 stages: fts,\n embed, claude_workflow, insights, standing-queries) — none is marked\n CPU-bound, so DaemonConverger._executor is never created and the\n periodic ambient loop never pools anything.\n2. The #3122-wired pooled census/replay path (revision_backfill.py\n _parse_retained_raws, reached via maintenance/replay.py -\u003e\n rebuild_index_from_source) IS reachable from inside a live polylogued\n process via the HTTP `--daemon` bridge (daemon/http.py:5276-5286,\n DaemonWriteThreadBridge.run_sync) -- but `journalctl --since -60days`\n shows every `ops maintenance rebuild-index` invocation on this host was\n a direct CLI systemd-run unit (`polylogue ops maintenance\n rebuild-index ...`), never with `--daemon`. So the daemon-HTTP-bridged\n variant has zero production exercise to date; all real runs (and the\n one that hung) went through the plain CLI process directly.\n3. Commit a53785b10 (#3122, merged 2026-07-18 19:26) is the commit that\n FIRST wired ingest_workers through to actual use in\n maintenance/replay.py -- before it, the parameter was accepted and\n immediately `del`eted, so the pooled dispatch branch in\n _parse_retained_raws was dead code on the CLI rebuild-index path.\n The forkserver hang was discovered ~8h after that merge (2026-07-19\n 03:00), on what was effectively the first real heavy exercise of the\n newly-activated pool. This fully explains why the deadlock surfaced\n now rather than being a long-standing dormant bug: the code path had\n never run for real before #3122 activated it.\n\nConclusion for AC \"verify daemon census throughput with pooling active\":\nthere is no production daemon-census throughput to measure yet -- the\npooled path has only run via direct CLI so far. Post-fix (spawn), the\nCLI-direct throughput is the throughput that matters today; the\ndaemon-HTTP-bridge variant and DaemonConverger's ambient cpu_bound pool\nare both currently unexercised/dormant in this codebase, not because\nthey're broken but because nothing marks a convergence stage cpu_bound\nand no HTTP client has used --daemon. Neither is in this bead's scope to\nactivate.\n\nSide finding filed as new tracked debt (out of this bead's scope --\nprocess_pool.py only): polylogue-7saq -- archive_ingest.py's\nparse_sources_archive() builds its ProcessPoolExecutor directly\n(concurrent.futures import, no mp_context), bypassing\nprocess_pool_context() entirely, so it uses the platform default start\nmethod (fork on this host/Python 3.13) -- a strictly worse hazard than the\nforkserver issue since raw fork() of a live async process is\nunconditionally unsafe if any other thread holds a lock at fork time.\nCurrently reached only by the public async API facade\n(Polylogue.parse_sources()/parse_file()) and demo seeding, not by the live\ndaemon's normal ingest ticks (those already go through the safe\nprocess_pool_executor() helper in ingest_batch/_core.py) or the standard\n`polylogue import` CLI flow (stages to daemon instead). Lower urgency than\np0pw was, but a real latent bug for any future caller.\nPR #3143 opened: https://github.com/Sinity/polylogue/pull/3143 (feature/perf/process-pool-spawn -\u003e master). Verification: devtools test tests/unit/pipeline/ -k process_pool (7 passed), devtools verify --quick (16/16 steps green). Rebased cleanly onto latest master after resolving a .beads/issues.jsonl rebase conflict (took origin's side entire -- verified it was a strict superset of my commit's older snapshot, per repo's documented bd-conflict procedure).\nPR #3143 merged: 5e794acbde955985fa7ca7296d6aed8a078abe4d. All CI green (CircleCI quick-gate pass, GitGuardian pass; CodeRabbit + Codex review both rate-limited, no findings to triage). Closing.","status":"closed","priority":0,"issue_type":"bug","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-07-19T01:10:15Z","created_by":"Sinity","updated_at":"2026-07-19T03:10:34Z","started_at":"2026-07-19T02:46:25Z","closed_at":"2026-07-19T03:10:34Z","close_reason":"Merged PR #3143: process_pool_context() now unconditionally spawn, never forkserver. AC honestly assessed: mechanism explained but not conclusively reproduced in isolation (documented); regression test + config-pin test added; daemon-census-throughput AC answered by audit (no production pooled daemon throughput exists yet -- pooled path has only run via direct CLI); workers=1 escape hatch kept as-is. Two follow-ups filed: polylogue-7saq (archive_ingest.py raw-fork ProcessPoolExecutor gap) and corroboration added to polylogue-7uqr (converger pool dead machinery).","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-5jak","title":"Daemon convergence starves bulk drains: backlog-aware conveyor + ungated startup","description":"Perf investigation 2026-07-18 (.agent/scratch/warroom-2026-07-17/perf-investigation-2026-07-18.md findings 1,2,3,6): (1) raw materialization conveyor = raw_artifact_limit=1 every 30s (daemon/cli.py:70-72) -\u003e 2 rows/min -\u003e a 73k backlog takes ~25 DAYS; this is why the poisoned index persisted under a healthy daemon. Each tick also pays fixed overhead (blob-ref restore scan, recover_interrupted_frontier, FTS close) amortized over ONE row. (2) startup Drive catch-up is awaited BEFORE all periodic loops and the LiveWatcher (cli.py:1465) — measured 4 serial network fetches/min blocking ALL convergence for hours per restart. (3) catch_up_complete gates raw materialization/insights/embeddings on full watcher catch-up though materializing durable local raws needs no such precondition.","design":"Backlog-aware conveyor: query pending materialization count; debt\u003ethreshold -\u003e per-tick limit 200-500 (still bounded, still single-writer), decay to 1 at quiescence; move per-tick frontier/blob-ref recovery to event-driven (post-crash) or backed-off schedule. Startup: launch Drive catch-up as a background task (periodic variant already exists cli.py:381); remove catch_up_complete gate from raw-materialization loop entirely (source.db is local authority). READ docs/retro/2026-05-24-1498-cascade.md before touching convergence stages.","acceptance_criteria":"Synthetic archive with 10k unmaterialized raws + daemon start: index converges in minutes not days (measured, receipt); daemon restart with a large Drive corpus begins local materialization within 60s (not after Drive completes); steady-state tick cost unchanged at quiescence; existing convergence-stage tests green.","notes":"[2026-07-18 Fable] Daemon-side P0 fix MERGED: PR #3102 (squash 9d01a41d4). Landed: backlog-aware burst draining (16-row passes back-to-back while remaining_candidates\u003e0 and progress made, 1s writer yield between passes, no-progress ends burst), conveyor ungated from watcher catch-up, interrupted-frontier recovery first-pass-only, startup Drive catch-up moved to immediate background pass of the periodic loop (awaited startup pass deleted). RESIDUAL in-scope findings from perf-investigation-2026-07-18.md: (5) DaemonConverger max_workers=2 — measure post-deploy before changing; (7) Drive attachment fetches strictly serial — googleapiclient/httplib2 are NOT thread-safe (documented in sources/drive/__init__.py iter_drive_raw_data docstring), so bounded concurrency needs per-thread service/http objects via the gateway, not a naive ThreadPool over one client. Finding 8 (cursor-claims-vs-index) is polylogue-emx2.\n[2026-07-18 late, Fable] Two more drain fixes merged+deploying (PR #3125): (a) catch-up planning opened source.db+index.db per cursor-less file (~40k connection opens ≈ 10 min silent 98%-CPU startup per restart, py-spy-confirmed) — now one read-only pair per planning pass, deliberately pass-bounded so blue-green index swaps are never read through a stale inode; (b) census-paused conveyor passes counted as no-progress and ended the backlog burst — 16 census components per 30s tick ≈ half a day for the live 22k-raw census backlog; census attempts now count as burst progress. Earlier tonight #3123 (spool-first catch-up ordering) unparked the conveyor. CURRENT drain shape after all fixes: watcher succeeds on fresh/changed files; the historical quarantined-cohort population drains via conveyor census→replay bursts; residual watcher failure classes are now \"CAS rejected an older accepted frontier\" (gemini jsons) and \"captured JSONL payload ends before a complete record boundary\" (live-appending claude-code files) — both bounded, not mass-refusals.\n[2026-07-18 23:35 Fable — post-deploy measurement] Final build (through #3125) live at 23:25. Startup-to-scan now ~2.5 min (was ~10+ min; residual is the 20k-file scan+plan itself). Conveyor census measured at ~1 pass/3min during early catch-up: the mid-burst spool check breaks the burst while the browser-capture chunks are still ingesting (transient, by design), and per-pass fixed cost (candidate discovery + component ordering over 23k raws) is the next amortization target if overnight throughput proves insufficient — same shape as the original finding #1 one level up: consider a larger census_component_limit for census-mode passes (discovery cost amortized over 16 seeds today). NOTE: census count grows while the watcher acquires (23,018→23,147 in 4 min) — do not read the census counter as a pure drain during catch-up. Morning decision point: if census+replay projected completion is unacceptable, lane D 9p8x parallel rebuild is the sanctioned fallback.\n2026-07-19 03:15: OPERATOR ESCALATION confirmed structurally: restore was absurdly slow vs the historical 1-2h full import. Root causes found tonight: (1) polylogue-p0pw — process_pool forkserver deadlock: the CLI rebuild-index ran 17min with ZERO parse workers ever spawned (parent idle in as_completed, forkserver idle at select); resumed with POLYLOGUE_INGEST_PARSE_WORKERS=1 -\u003e 97% CPU immediately. Same helper wired into daemon census (#3122) — daemon-path impact unaudited. (2) polylogue-nh44 — census parses all revisions: 97.4GB stored blobs vs 52.2GB newest-only (45GB superseded snapshots; one file = 800 revisions/6.2GB). (3) Daemon conveyor orchestration (bounded passes, per-component calls, 50/50 writer share with walk) turned ~1h of parse into a weeks-scale projection; census went net-NEGATIVE once the walk minted new pending raws faster than census cleared them. Fallback executed: daemon stopped, blue-green rebuild-index transaction 7e245ea7 running sequentially. amg1 (#3136) landed but per lane D own note mostly benefits this CLI path, not the daemon loop.\n2026-07-19 04:00: fourth structural finding — polylogue-l3tk: fresh generations run unanalyzed, planner chose global block_type index for refresh_action_pairs = O(N^2) replay writes (72% of replay CPU). Live ANALYZE on the running generation: \u003e20x sustained replay speedup (2.9 -\u003e 60 sessions/min). Rebuild now pacing toward hours, not days. Perf-wave lane prompts staged (H/I/J/K in lanes-perf-wave-2026-07-19.md); demo cold-pass green; outreach draft skeleton staged.\n2026-07-19 09:20 (coordinator): deploy switch auto-restarted polylogued (systemd activation) while the offline rebuild transaction 7e245ea7 was mid-flight; the daemon wrote +252 raw rows in ~5 min before I stopped it, drifting source_revision_snapshot (8a15ebf2 -\u003e 32a6ec93) which would have staled the operation and discarded ~5h of generation work (4,766 sessions / 2.03M messages). DECISION: hand-patched the transaction json source_snapshot forward to the current value and resumed. Safety rationale: drift verified append-only (raw count 101,095 -\u003e 101,347, no deletions possible on this path); pagination is (acquired_at_ms, raw_id)-ordered with a cursor, appended rows sort strictly later so processed pages are unaffected and new rows are simply included later; replay is idempotent and cohort expansion reads current source at replay time, so no older-looks-newest hazard. The snapshot guard is deliberately conservative (full-freeze) — a future bead may want an explicit append-tolerant mode instead of operator json surgery. OPERATIONAL RULE until promote: no sinnix switch (it restarts polylogued and re-drifts source); daemon stays stopped.\n2026-07-19 coordinator: remaining scope maps onto the m6tp program — conveyor starvation root-fix = bulk routing (polylogue-gd6v); parse-out-of-writer-holds shipped behind a flag (PR #3168). 5jak stays P0 as the umbrella symptom bead until gd6v lands and the 73k-backlog scenario is re-measured through normal daemon convergence. The append-tolerant snapshot mode this bead noted is now a designed requirement of gd6v.","status":"closed","priority":0,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-07-18T14:35:19Z","created_by":"Sinity","updated_at":"2026-07-20T05:53:11Z","closed_at":"2026-07-20T05:53:11Z","close_reason":"Investigation 2026-07-20 (receipts in matrix): all concrete findings and ACs satisfied by PR #3102 (merged 2026-07-18) — burst-until-drained raw-materialization loop (16/64 bounded passes, 1s writer yield, 30s interval only at quiescence; pinned by 3 daemon_cli tests), Drive catch-up backgrounded not awaited, raw materialization deliberately ungated on watcher catch-up (insights/embeddings gating is intentional, depends on parsed content). Design-detail deltas are legitimate substitutions (burst pacing instead of dynamic limit scaling; recovery scan first-pass-only). The remaining backlog-window efficiency gap during bulk-scale routing is owned by polylogue-gd6v (suppression lane in flight) — keeping 5jak open would duplicate that tracking. Post-flag-flip re-measure of the 73k-backlog scenario belongs on gd6v archive-scale receipt.","labels":["area:daemon"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-k8kj","title":"Live archive: interrupted index rebuild serves stale data + query_unit_frame_state missing breaks query transactions","description":"Discovered 2026-07-18 during Lane C MCP six-tool cutover live-proof pass (read-only probes against POLYLOGUE_ARCHIVE_ROOT=/home/sinity/.local/share/polylogue under sinnix-scope background). Two distinct, serious findings on the live daemon-served archive, NOT caused by and NOT fixable within the MCP six-tool cutover branch:\n\n1. STALE DEFAULT INDEX PATH: ordinary Config/RuntimeServices db_path resolution (archive_root / \"index.db\") reads a stale regular file at /home/sinity/.local/share/polylogue/index.db containing only 4 sessions. The real active index (18796 sessions, 4,900,824 messages, user_version=39) lives at /realm/db/polylogue/index.db, a symlink into the generation-based blue-green rebuild directory /realm/db/polylogue/.index-generations/gen-1784204285162-6260ad8b/, reachable only via the .index-active-pointer indirection file. Ordinary config resolution never follows .index-active-pointer. /home/sinity/.local/share/polylogue/.index-rebuild.lock is held by pid=449975, which is no longer a running process -- an interrupted rebuild that updated .index-active-pointer but never promoted/symlinked the new generation into the conventional archive_root/index.db path (or removed the stale file). Any fresh process resolving the conventional path -- including a newly spawned claude/codex MCP client, unless something else in that startup path already knows to follow the generation pointer -- silently gets a near-empty 4-session view instead of the real archive. Unclear whether the live daemon (daemon.pid=2844903, confirmed alive) is also affected, or resolved correctly at its own startup before/after the interrupted rebuild -- needs investigation without blindly restarting it.\n\n2. query_unit_frame_state TABLE MISSING ON THE ACTIVE GENERATION: even when pointed directly at the correct active generation (/realm/db/polylogue/index.db), any query touching the QueryTransaction continuation/epoch-tracking mechanism (archive_snapshot_epoch() in polylogue/archive/query/transaction.py) fails with QueryArchiveEpochUnreadableError (\"could not establish archive frame for query continuation\") because of sqlite3.OperationalError: no such table: query_unit_frame_state -- required in BOTH index.db and user.db (attached as user_tier), maintained by triggers. This table is part of the z9gh.9.1 epoch-tracking machinery landed recently on master; the live archive generation predates it or the rebuild that would add it never completed. Practical impact: the new six-tool query() MCP tool -- the single most important of the six read transactions -- is completely non-functional against the live archive right now for any messages/actions/blocks/etc terminal query. status(scope=archive/operation), explain, and context all work fine (different code paths, do not touch query_unit_frame_state).\n\nRecommended fix path (derived-tier schema mismatch, per project doctrine in CLAUDE.md \"Schema regimes\" section): `polylogue ops reset --index \u0026\u0026 polylogued run` to rebuild index.db from source with the current schema, INCLUDING query_unit_frame_state. This is a live 4.9M-message reindex against the daily-use archive -- requires explicit operator authorization before running (Destructive Operations policy), a verified backup per the derived-tier rebuild plan, and should NOT be run blind by an agent. Also verify/fix whatever is supposed to promote/symlink a completed rebuild generation into archive_root/index.db so future rebuilds do not leave the conventional path stale again -- investigate why the promotion step did not run when pid 449975 died.","notes":"Implementation trail (agent session, worktree-agent-a7b08f75dffd86e2f):\n\nScope understood: 4 deliverables per war-room lane assignment -- dead-pid\nrebuild lock reclaim, stale conventional index path vs .index-active-pointer,\nquery_unit_frame_state missing-table crash, durable per-pass rebuild receipts.\nCode+tests against fixtures only; live archive untouched throughout.\n\nPR: https://github.com/Sinity/polylogue/pull/3150 (branch\nfeature/fix/rebuild-path-robustness-k8kj, 4 commits: 5bcefe69e, c79f4d48a,\nb95bdf9d9, ec9aa56d4)\n\nWhat changed:\n1. storage/index_generation.py: RebuildLease/ActiveWriterLease now check the\n recorded lock-file pid's liveness on BlockingIOError and reclaim (fresh\n inode swapped in via os.replace) when the holder pid is dead, logging a\n warning. Live holders still refuse exactly as before.\n2. config.py + storage/archive_identity.py: new resolve_active_index_path()\n follows .index-active-pointer (pure fn of archive_root, no env/cwd reads)\n using the existing ArchiveLocation/shadow_index machinery. Wired into\n Config.__init__'s default db_path and resolve_runtime_config()'s\n ResolvedArchivePaths -- the two chokepoints most bare\n Config(archive_root=..., sources=[]) callers (MCP server, daemon status,\n several CLI entrypoints) actually go through. A stale conventional file\n diverging from the pointer now logs loudly; the pointer target is still\n what gets served (heal + report, not silent staleness).\n3. archive/query/transaction.py: archive_snapshot_epoch() recognizes the\n specific \"no such table: query_unit_frame_state\" OperationalError and\n raises the same QueryArchiveEpochUnreadableError type/code with an\n actionable rebuild-guidance message instead of the generic one. No surface\n wiring changes needed (daemon/http.py, mcp/server_cutover.py already\n forward exc.code + str(exc)).\n4. maintenance/rebuild_index.py + IndexGenerationStore.save_pass_receipt():\n every pass receipt (paused/deferred early return AND terminal replayed\n return) is now durably persisted as \u003coperation_id\u003e.receipts/pass-NNNNNN.json\n alongside the transaction record (tmp+os.replace+fsync), independent of\n the CLI's stdout JSON.\n\nDecisions recorded (per assignment ask):\n- Finding 1 fix direction: made db_path resolution FOLLOW the pointer,\n rather than forcing promote()/recovery to always keep the conventional\n path physically a symlink. Rationale: the pointer-following logic already\n existed in 3 places (ArchiveLocation.resolve, paths/_roots.py's\n resolve_active_index_db_path + active_index_db_path) but was never wired\n into Config/resolve_runtime_config, the chokepoint most callers actually\n use -- that's the real gap. Noted residual: 3-way duplication of\n \"follow .index-active-pointer\" logic across archive_identity.py and\n paths/_roots.py is worth a follow-up consolidation; not attempted here\n (out of scope, touches call sites this fix didn't need to change).\n- Finding 2 fix direction: confirmed via storage/sqlite/schema_bootstrap.py\n + schema.py that decide_schema_bootstrap() already rejects any on-disk\n user_version outside {0, SCHEMA_VERSION} -- a genuine version mismatch is\n already caught before this code path runs. The live gap is specifically a\n generation whose recorded version MATCHES yet is missing a structural\n piece (query_unit_frame_state) -- not catchable by that version gate, and\n not something to patch with a runtime auto-upgrade (derived tiers have no\n in-place upgrade chain per project doctrine). Converted the crash into a\n clear, actionable rejection instead. Did NOT touch\n storage/sqlite/runtime_indexes.py (owns ensuring runtime-created\n indexes/tables on open) -- explicitly out of scope, actively owned by\n polylogue-crd8.\n\nVerification: devtools test on all 4 touched test files (95+16+14+14 =\n139 tests passed) + devtools verify --quick exit 0. Anti-vacuity: every new\nregression test run against pre-fix code (via git stash / reconstructed\ndiff) and confirmed to fail with the exact expected symptom before the fix,\npass after.\n\nIncident during this session: git stash operations from a concurrent agent\n(coordinator, working PR feature/perf/pool-dispatch-floor in a DIFFERENT\nworktree /realm/worktrees/polylogue-conveyor-perf) collided with mine on the\nshared refs/stash ref (stash is shared across all worktrees of one repo,\neven though working directories are separate) -- one of my `git stash pop`\ncalls applied their uncommitted polylogue/sources/revision_backfill.py diff\ninto my working tree and dropped it from the shared stash list. Recovered:\nmy own lost uncommitted diff was salvaged from an unreachable stash merge\ncommit found via `git fsck --unreachable` (commit 80700042); the\ncoordinator's stranded diff was exported to\n/realm/tmp/revision_backfill-rescued-for-coordinator.patch and removed from\nmy tree via `git checkout --`. No data was permanently lost. Lesson for\nfuture sessions: avoid `git stash` in shared-checkout/concurrent-worktree\nsetups for anti-vacuity checks -- use `git show HEAD:\u003cpath\u003e` to reconstruct\npre-fix file content instead, since it never touches the shared stash ref.\n\nResidual scope (bead stays open): the live archive's actual stale-index\nstate at /home/sinity/.local/share/polylogue and /realm/db/polylogue is\nNOT touched or resolved by this PR (fixtures only, per assignment\nconstraint) -- needs explicit operator authorization for any live\nops reset/rebuild. The paths/_roots.py vs archive_identity.py\npointer-resolution duplication noted above is an open follow-up.","status":"closed","priority":0,"issue_type":"bug","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-07-18T13:24:07Z","created_by":"Sinity","updated_at":"2026-07-21T20:35:35Z","started_at":"2026-07-19T07:12:52Z","closed_at":"2026-07-21T20:35:35Z","close_reason":"Residual live-state scope verified resolved by read-only closure census 2026-07-21 (.agent/reports/yla8-closure-census-2026-07-21.md): on the promoted v42→v43 generation gen-1784486727919, query_unit_frame_state exists and is populated (epoch=90334192), the archive symlink resolves to the promoted generation correctly, and the stale conventional-index serving path is gone. Fixture-level fixes merged earlier; live state now matches. Census performed no mutation.","dependency_count":0,"dependent_count":0,"comment_count":0} @@ -60,7 +60,7 @@ {"_type":"issue","id":"polylogue-tf2","title":"Campaign: agent-forensics regeneration + all-provider repricing","description":"Regenerate the agent-forensics packet on the current archive with an honest all-provider headline. The 2026-06-27 report (546.6B tokens, $89,368 API-list equivalent, 216x cache amplification) is the most stranger-legible artifact on any shelf, but its numbers are pre-dedup stale and the headline prices only the priced-provenance subset (Claude Code cost_usd rows); Codex/ChatGPT/Gemini are origin_reported token counts with no dollar value (operator estimate ~$150K all-provider). Sequenced after claim-vs-evidence per operator direction 2026-07-02.","design":"Current slice design: turn the existing agent-forensics/cost headline into a product-backed all-provider repricing artifact. First inspect devtools/scripts and polylogue analyze surfaces for agent_forensics/cost code. Use active archive usage headline (detail=headline) for authoritative physical_session and logical_session_model_high_water token totals. Keep priced-provenance dollars and origin-reported token estimates separate: do not multiply every token by one blended price without a labeled lane. Add or reuse a shared pricing/projection helper so the demo artifact is regenerated from Polylogue product code, not ad hoc SQL. Acceptance for this slice: the generated agent-forensics artifact names archive root/schema, includes physical vs logical token grain, separates priced subset from origin-reported estimate lanes, gives reproduction commands, and has focused tests for any new repricing helper/surface.","acceptance_criteria":"Terminal state: regenerated forensics packet on the current archive with an honest all-provider headline (priced subset AND origin-reported estimate lanes separated), agent_forensics.py folded into polylogue analyze (tf2.2), artifact on the demo shelf with reproduction commands, cold-reader gate passed. Epic closes only when that artifact is recorded.","status":"closed","priority":0,"issue_type":"epic","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-07-03T04:31:32Z","created_by":"Sinity","updated_at":"2026-07-03T19:06:44Z","started_at":"2026-07-03T18:47:23Z","closed_at":"2026-07-03T19:06:44Z","close_reason":"Completed: provider usage headline now exposes product-backed pricing lanes in polylogue analyze usage --detail headline, separating stored/provider-priced cost from catalog API-equivalent estimates for origin_reported rows. Regenerated the current .agent/demos/agent-forensics artifact against /home/sinity/.local/share/polylogue schema v23: physical-session tokens 395,320,980,423; logical high-water tokens 288,741,229,728; stored/provider-priced USD 243,392.189328; catalog API-equivalent USD 337,565.031618; priced lane 13,889 rows / 12,331 sessions / 12,650 matched rows; origin_reported lane 2,308 rows / 2,270 sessions / 2,302 matched rows. Verification: live polylogue --plain analyze usage --detail headline --format json --limit 0 wrote /realm/tmp/polylogue-usage-headline-pricing-current.json; devtools test tests/unit/storage/test_provider_usage_report.py tests/unit/cli/test_diagnostics.py passed 23 tests; devtools verify --quick passed run 20260703T190553Z-quick-2226137-d91d4e8f; devtools workspace demo-shelf --json reported ok. Non-claim preserved: this is not final billing reconciliation and physical/logical token grains stay explicitly separated.","labels":["area:usage","campaign","size:M","spine"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-sru","title":"Campaign: claim-vs-evidence report to finding-grade","description":"Terminal state: an externally publishable finding ('how often do coding agents proceed past failed tool calls, by model/tool') with stated sample frame, calibrated markers, benign/consequential split, seeded stranger-runnable reproduction, and a passed cold-reader gate. Slice closure is NOT campaign closure; this epic stays top-of-frame until its terminal state is recorded.\\n\\nState as of 2026-07-03 after calibrated active-archive regeneration: archive root /home/sinity/.local/share/polylogue, index schema v23, 41,886 structured failures total, 5,000 origin-stratified failures inspected (3,746 claude-code-session, 1,247 codex-session, 7 claude-ai-export), 100 unpaired structured failures. Marker vocabulary was tightened to avoid broad issue/fix/block/gitignored false positives. Immediate next-turn totals: acknowledged=420, silent_proceed=1,205, ambiguous=3,375 (2,624 wordless tool continuations; 751 prose without marker). Lower-bound silent rate is 24.1%; among classified immediate next turns, silent rate is 74.2%. Next-3 sensitivity window, stopping before the next user message, finds 302 acknowledgments that appear only after the next turn; window3 silent lower bound is 37.0%. Calibration: 50 hand-labeled immediate-next-turn rows, acknowledged-marker precision=1.0, recall=0.8421052631578947, invalid rows=0. Artifact: .agent/demos/claim-vs-evidence/claim-vs-evidence.report.json.","notes":"2026-07-03 update: methodology package is now cold-read gated. .agent/demos/claim-vs-evidence contains aggregate live evidence, public-summary.json, PUBLIC_REPRODUCTION.md, COLD_READER_GATE.md, and COLD_READ_RESULT.md. Seeded reproduction is meaningful, not empty: 4 structured failures, 2 acknowledged follow-ups, 2 silent-proceed follow-ups, 0 unpaired. Cold-reader subagent PASS recovered claim/non-claim, sample frame, rates, calibration, caveats, and reproduction commands from the artifact directory only. Remaining campaign child: polylogue-sru.1 productizes action-unit outcome/followup_class capability.","status":"closed","priority":0,"issue_type":"epic","owner":"ezo.dev@gmail.com","created_at":"2026-07-03T04:31:26Z","created_by":"Sinity","updated_at":"2026-07-03T09:28:09Z","closed_at":"2026-07-03T09:28:09Z","close_reason":"Completed: all seven campaign children are closed. The claim-vs-evidence finding now has bounded sample-frame reporting, calibrated marker precision/recall, handler-class and next-3 sensitivity splits, meaningful seeded reproduction, cold-reader PASS, and productized action-unit followup_class/followup_message_ref query capability. Current artifact lives under .agent/demos/claim-vs-evidence and was regenerated against /home/sinity/.local/share/polylogue schema v23.","labels":["area:substrate","campaign"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-m3p9","title":"sessions.created_at_ms NULL for 79% of archive: since:/recency silently exclude 65,946 sessions","description":"Found 2026-07-22 while fact-checking README examples: SELECT count(*), sum(sort_key_ms IS NULL) FROM sessions on the promoted v43 archive = 83,198 total, 65,946 NULL (79%). sort_key_ms = COALESCE(updated_at_ms, created_at_ms), both plain columns the writer only sets when the provider payload carries session-level timestamps. Result: find since:… matched exactly 17,252 (= the non-NULL population) — date filters, --by year/month histograms, and recency ordering silently exclude four-fifths of the archive, including most claude-code subagent sessions and hermes/observer material, even though their MESSAGES carry timestamps.","design":"Derive session timestamps from message evidence at write/materialize time: created_at_ms = min(message timestamp), updated_at_ms = max(message timestamp) when the provider gives none at session level (messages table already stores per-message timestamps for these origins). Classify: additive-derived (index tier) — either benign in-place backfill on same-version open (benign-DDL/backfill registry) or fold into next semantic bump; the insight/profile layer may already compute first/last message times (session_profiles) — prefer deriving the sessions columns from the same source rather than a second scan. Verify since:/analyze --by coverage jumps from 17,252 to ~all sessions with any timestamped message; regression test: session whose payload lacks session-level timestamps but has dated messages gets non-NULL sort_key_ms.","acceptance_criteria":"since:/until:/recency and --by year/month cover every session that has at least one timestamped message; NULL sort_key remains only for genuinely undatable sessions (count them in the receipt); regression test for the derive-from-messages path; live archive backfilled with receipt.","status":"in_progress","priority":1,"issue_type":"bug","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-07-21T22:59:34Z","created_by":"Sinity","updated_at":"2026-07-21T23:57:57Z","started_at":"2026-07-21T23:57:57Z","labels":["area:query","area:storage"],"dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"polylogue-t93b","title":"Daemon must converge whale raw components: census permanently refuses \u003e64MiB, witness 6.33GB codex source unrecoverable automatically","description":"Operator ruling 2026-07-21: unacceptable that components exceeding _RAW_MATERIALIZATION_DAEMON_BLOB_LIMIT_BYTES (64MiB, daemon/cli.py:89) are resource-blocked FOREVER by the daemon census — the live witness codex:019f49d8 (788 raws, 6.33GB, 20495 messages at peak) plus 3 claude-code sources have zero index presence on the promoted v43 archive solely because every daemon pass logs \"resource-blocked ... exceed replay limit 67108864\" and moves on. Automagic-invariants doctrine: if the daemon owns raw-\u003eindex convergence it must converge whales too; a permanent manual/offline requirement is a policy bug. The refusal exists to bound writer-hold transaction length and parse memory — both concerns now have productized answers: streaming parsers for the dominant origins (codex parse_codex_stream, claude-code streaming JSONL; _raw_materialization_stream_safe at storage/repair.py:3972) and bounded commit batches (raw_authority_commit_batch_size config, PR #3248).","design":"Escalation tier, not a blanket limit raise: (1) keep the 64MiB fast-path limit for ordinary census passes; (2) when a component is resource-blocked AND the backlog is otherwise quiescent, schedule a dedicated whale pass for that single component: parse via the streaming path (require every member stream-safe, else remain typed-blocked with a distinct reason), bounded parse memory via the existing RawParsePrefetchCache inflight budget, replay with commit-batched transactions (raw_authority_commit_batch_size) so the writer hold stays bounded; (3) the resource-blocked durable fingerprint machinery (revision_backfill.py _resource_blocked_parser_fingerprint) already persists typed state — the whale pass consumes it; (4) emit daemon events for whale-pass start/receipt. Key anchors: daemon/cli.py:89 + _periodic_raw_materialization_convergence (:770) + _drain_raw_materialization_once (:958); storage/repair.py repair_raw_materialization (:5702), resource-blocked catch sites (:5833, :6245); revision_backfill.py:491 raise site. Verify against a synthetic multi-raw whale fixture exceeding the limit; the witness component on the live archive is the acceptance witness.","acceptance_criteria":"A component whose total raw bytes exceed the daemon limit but whose members are stream-safe converges to a resolved head through the DAEMON (no offline pass), with writer-hold time bounded (commit batches) and memory bounded (streaming parse + inflight budget); non-stream-safe oversized components get a distinct typed blocked reason; regression test with synthetic whale fixture; live witness codex:019f49d8 resolves after deploy; daemon event receipts recorded.","notes":"2026-07-22: implementation merged as PR #3256 (quiescence-gated single-component escalation pass, 8GiB default envelope via raw_authority_whale_payload_bytes, stream-safe-only, commit-batched, daemon events, default-on with daemon_whale_raw_materialization off-switch; coordinator review on the PR). Deployed to sinnix via flake bump 354be99 + switch. REMAINING for close: live witness codex:019f49d8 resolves to a head via the daemon whale pass — blocked until the operator resolves the durable stale-plan blocker (raw-authority-blocker:5406c7c3…, script staged) since ALL materialization passes fail-closed behind it.","status":"open","priority":1,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-21T21:17:41Z","created_by":"Sinity","updated_at":"2026-07-21T22:05:22Z","labels":["area:daemon","area:perf","area:storage"],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"polylogue-t93b","title":"Daemon must converge whale raw components: census permanently refuses \u003e64MiB, witness 6.33GB codex source unrecoverable automatically","description":"Operator ruling 2026-07-21: unacceptable that components exceeding _RAW_MATERIALIZATION_DAEMON_BLOB_LIMIT_BYTES (64MiB, daemon/cli.py:89) are resource-blocked FOREVER by the daemon census — the live witness codex:019f49d8 (788 raws, 6.33GB, 20495 messages at peak) plus 3 claude-code sources have zero index presence on the promoted v43 archive solely because every daemon pass logs \"resource-blocked ... exceed replay limit 67108864\" and moves on. Automagic-invariants doctrine: if the daemon owns raw-\u003eindex convergence it must converge whales too; a permanent manual/offline requirement is a policy bug. The refusal exists to bound writer-hold transaction length and parse memory — both concerns now have productized answers: streaming parsers for the dominant origins (codex parse_codex_stream, claude-code streaming JSONL; _raw_materialization_stream_safe at storage/repair.py:3972) and bounded commit batches (raw_authority_commit_batch_size config, PR #3248).","design":"Escalation tier, not a blanket limit raise: (1) keep the 64MiB fast-path limit for ordinary census passes; (2) when a component is resource-blocked AND the backlog is otherwise quiescent, schedule a dedicated whale pass for that single component: parse via the streaming path (require every member stream-safe, else remain typed-blocked with a distinct reason), bounded parse memory via the existing RawParsePrefetchCache inflight budget, replay with commit-batched transactions (raw_authority_commit_batch_size) so the writer hold stays bounded; (3) the resource-blocked durable fingerprint machinery (revision_backfill.py _resource_blocked_parser_fingerprint) already persists typed state — the whale pass consumes it; (4) emit daemon events for whale-pass start/receipt. Key anchors: daemon/cli.py:89 + _periodic_raw_materialization_convergence (:770) + _drain_raw_materialization_once (:958); storage/repair.py repair_raw_materialization (:5702), resource-blocked catch sites (:5833, :6245); revision_backfill.py:491 raise site. Verify against a synthetic multi-raw whale fixture exceeding the limit; the witness component on the live archive is the acceptance witness.","acceptance_criteria":"A component whose total raw bytes exceed the daemon limit but whose members are stream-safe converges to a resolved head through the DAEMON (no offline pass), with writer-hold time bounded (commit batches) and memory bounded (streaming parse + inflight budget); non-stream-safe oversized components get a distinct typed blocked reason; regression test with synthetic whale fixture; live witness codex:019f49d8 resolves after deploy; daemon event receipts recorded.","notes":"2026-07-22: implementation merged as PR #3256 (quiescence-gated single-component escalation pass, 8GiB default envelope via raw_authority_whale_payload_bytes, stream-safe-only, commit-batched, daemon events, default-on with daemon_whale_raw_materialization off-switch; coordinator review on the PR). Deployed to sinnix via flake bump 354be99 + switch. REMAINING for close: live witness codex:019f49d8 resolves to a head via the daemon whale pass — blocked until the operator resolves the durable stale-plan blocker (raw-authority-blocker:5406c7c3…, script staged) since ALL materialization passes fail-closed behind it.","status":"open","priority":1,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-21T21:17:41Z","created_by":"Sinity","updated_at":"2026-07-21T22:05:22Z","labels":["area:daemon","area:perf","area:storage"],"comments":[{"id":"019f8acc-2842-7288-a38b-5e51f6bbfd97","issue_id":"polylogue-t93b","author":"Sinity","text":"2026-07-22 census-state note (from hook de-inflation, polylogue-31r1): the live archive's raw-authority census is internally inconsistent and must be reconciled/rebuilt as part of this convergence work. Cause: hook de-inflation deleted 64,896 hook raw_sessions; those hook raws had ~64,895 frontier plans + blockers + census_plans/post_plans (hook noise flooding the authority machinery). A surgical orphan-plan deletion (PR #3266, now closed) removed the dangling plans but broke the carried-forward/retryable postflight invariant (raw_authority.py:1315). Daemon now defers census passes to convergence_debt (736+) instead of the prior stale-plan-blocker degradation; it survives (0 crashes), archive data correct. Recommended resolution: full raw-authority census rebuild over the current hook-free raw set (no prior-census carried-forward comparison), preserving accepted heads/revision_authority (byte_proven 15,465 / quarantined 20,986). No dedicated census-reset mechanism exists yet.","created_at":"2026-07-22T17:07:43Z"}],"dependency_count":0,"dependent_count":0,"comment_count":1} {"_type":"issue","id":"polylogue-meoz","title":"ArchiveStore.delete_sessions detonates per-row derived-refresh triggers: 91-session delete ran 3h with 375GB reads and zero commit","description":"Live incident 2026-07-21 (yqeo retirement): ArchiveStore.delete_sessions on 91 hermes sessions sat 3h in one transaction: 375GB read (11 full scans of the 34GB index.db), 2MB written, WAL empty — killed and rolled back. py-spy: stuck in the per-session DELETE FROM sessions loop (archive.py:6725). Root cause: blocks_action_pairs_ad fires PER DELETED BLOCK ROW and each firing (a) deletes+rebuilds the whole session action_pairs with two window-function scans and (b) re-derives delegation_facts from delegation_facts_source. The production bulk write path suppresses this machinery via derived_refresh_guard rows (session-write, fts-bulk-session-write) but delete_sessions — the PRODUCT deletion API used by the CLI delete verb and SessionDeleteActuator — never sets them. Same pathology family as polylogue-crd8 (whale prefix-tail rewrite FTS/trigram detonation).","design":"Fix in delete_sessions itself (and any sibling bulk mutation entrypoints): wrap the delete in the derived_refresh_guard rows, do one-pass FTS maintenance explicitly (blocks_command_trigram delete commands with old text before block rows go away; contentless messages_fts DELETE by rowid), let indexed FK cascades remove the tree, clear guards, commit. Working reference implementation: /realm/tmp/worktrees/yqeo-v42/yqeo_retire_stale_v2.py (operator-run 2026-07-21). Regression test: seeded session with tool_use blocks, delete via product API, assert FTS docsize parity and action_pairs cleanup without trigger-driven rebuild (e.g. count trigger firings via guard-sensitive canary or measure statement count). Also audit epoch triggers (query_unit_frame_*_delete) cost under bulk cascade.","acceptance_criteria":"delete_sessions (and executor SessionDeleteActuator route) deletes a many-block session in seconds not hours; FTS/trigram stay coherent (docsize==indexable parity) after delete; regression test proves per-row action_pairs/delegation rebuild machinery does not fire during bulk delete; crd8 relation noted.","status":"in_progress","priority":1,"issue_type":"bug","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-07-21T19:15:22Z","created_by":"Sinity","updated_at":"2026-07-21T23:57:55Z","started_at":"2026-07-21T23:57:55Z","labels":["area:perf","area:storage"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-zoc3","title":"ingest_record decode rejects binary provider payloads the rebuild parse path handles","description":"Found 2026-07-21 during the polylogue-yqeo targeted Hermes reprocess: parse_from_raw → process_ingest_batch → ingest_record fails all 3 hermes verification raws (source_path ~/.hermes/verification_evidence.db, SQLite database bytes) with \"decode: str is not valid UTF-8: surrogates not allowed: line 1 column 1\" — the worker decode step assumes text/JSON payloads before provider dispatch. The REBUILD path (revision_backfill._parse_retained_raw → sources/dispatch.parse_payload) parses these same raws fine (the v42 walk materialized verification sessions from them), so the two parse routes disagree on binary-payload providers. Consequence: targeted reprocess cannot re-materialize hermes verification sessions under the composed verification:\u003craw_id\u003e@profile-\u003ckey\u003e scheme (#3227); 4 stale old-pattern verification:2026* sessions remain in the index with no composed successors (retained deliberately — deleting them would lose read coverage).\n\nFix: route ingest_record payload decoding through the same provider-dispatch-aware envelope the rebuild path uses (binary-capable: detect_provider on bytes before any text decode), or teach build_raw_payload_envelope the binary lane. Add a contract test: any raw parseable by revision_backfill._parse_retained_raw must be parseable by ingest_record (parse-route parity for a representative binary fixture — the hermes verification fixture family exists under tests/fixtures/hermes/).\n\nAfter the fix: reprocess the 3 verification raws (coordinator, live archive), retire the 4 stale verification:2026* ids, and update the yqeo receipt.","status":"closed","priority":1,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-21T15:08:01Z","created_by":"Sinity","updated_at":"2026-07-21T16:34:33Z","closed_at":"2026-07-21T16:34:33Z","close_reason":"Fixed in PR #3247 (merged): build_raw_payload_envelope now probes BOTH Hermes SQLite artifacts (state.db + verification_evidence.db) via the parsers own looks_like_*/marker_payload helpers BEFORE any text decode — ingest_record and the rebuild route now agree on binary payloads; marker classification extracted+shared so the decoded marker session classification is not shadowed by the .db path-only sidecar rule; profile_root threaded in backfill for identical composed ids on both routes. Parity contract test (243 lines) incl. exact live-error reproduction. Lane was 522-killed twice post-push; coordinator verified helpers + re-ran 51 tests on the branch and opened/merged the PR. Unblocks the yqeo verification-raw reprocess.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-sv5q","title":"Bundle replay fail-closed contract unmasked: unconvertible-head refusal relied on UNIQUE abort","description":"Found 2026-07-21 immediately after merging #3236+#3239: tests/unit/sources/test_live_batch_support.py::test_bundle_replay_respects_unconvertible_single_session_head[bundle_texts2-False-False] and [bundle_texts3-False-True] fail on master — the succeeds=False parametrizations pin that an OLDER bundle that cannot convert the newer accepted single-session head must FAIL (fail-closed, head unchanged), but the ingest now reports success (result.failed == []). Working hypothesis: these were among the pre-existing failures whose real cause was the UNIQUE(block_id) IntegrityError abort; #3239 fixed the abort (INSERT OR REPLACE), unmasking that the replay path completes where the contract says refuse — i.e., the fail-closed refusal may have been an ACCIDENTAL crash, not an explicit check. Must determine whether older content actually perturbs the accepted head (correctness bug → make the refusal explicit in the production path) or the head survives and only the failed-list bookkeeping changed (→ deliberately update the test contract, still asserting head_after == head_before + message_count invariants). Lane dispatched on branch fix/sources/bundle-head-fail-closed; diagnosis pending.","status":"closed","priority":1,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-07-21T13:14:37Z","created_by":"Sinity","updated_at":"2026-07-21T13:41:30Z","closed_at":"2026-07-21T13:41:30Z","close_reason":"Fixed in PR #3240 (merged): root cause was NOT the UNIQUE-abort unmasking hypothesis — lane bisected to #3211 removing the #2718 byte-governance refusal in apply_raw_membership_classification on a false premise (_apply_membership_sessions injects the un-converted accepted head into the cohort). Confirmed real fail-closed violation: older bundle superset content silently moved the head (message_count 2-\u003e3, accepted_raw_id changed). Restored as a narrower guard (only when replay changes the accepted raw AND live predecessor_source_revision append evidence chains off the existing head), preserving #3211 drift resumption. Promoted v42 archive UNAFFECTED (resume26 loaded code 20:39, #3211 merged 21:12). Both bundle-head tests green; anti-vacuity via guard revert.","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/docs/plans/topology-target.yaml b/docs/plans/topology-target.yaml index 86520a38e6..bcb0e4240d 100644 --- a/docs/plans/topology-target.yaml +++ b/docs/plans/topology-target.yaml @@ -2107,7 +2107,7 @@ files: target: polylogue/maintenance/failure_routing.py owner: stable - path: polylogue/maintenance/hook_deinflation.py - loc: 130 + loc: 157 target: polylogue/maintenance/hook_deinflation.py owner: stable - path: polylogue/maintenance/invalidation.py @@ -2130,6 +2130,10 @@ files: loc: 505 target: polylogue/maintenance/preview.py owner: stable + - path: polylogue/maintenance/raw_authority_reset.py + loc: 152 + target: polylogue/maintenance/raw_authority_reset.py + owner: stable - path: polylogue/maintenance/rebuild_index.py loc: 507 target: polylogue/maintenance/rebuild_index.py diff --git a/docs/topology-status.md b/docs/topology-status.md index 8e594c004e..b18397169e 100644 --- a/docs/topology-status.md +++ b/docs/topology-status.md @@ -28,12 +28,12 @@ Generated by `devtools render topology-status`. Reads `docs/plans/topology-targe ### Summary -- **Stable** (no move scoped): 904 +- **Stable** (no move scoped): 905 - **Kernel** (polylogue/ root): 8 - **Primitives** (storage-root): 19 - **TBD** (cell needs explicit assignment): 9 -- **Total declared**: 1075 -- **Realized polylogue/**/*.py**: 1075 files declared +- **Total declared**: 1076 +- **Realized polylogue/**/*.py**: 1076 files declared ### TBD cells (require explicit routing) diff --git a/polylogue/maintenance/raw_authority_reset.py b/polylogue/maintenance/raw_authority_reset.py new file mode 100644 index 0000000000..e799bbdd4d --- /dev/null +++ b/polylogue/maintenance/raw_authority_reset.py @@ -0,0 +1,152 @@ +"""Reset the raw-authority census planning ledger so the daemon rebuilds it. + +The raw-authority census tables (``raw_authority_censuses`` and its +``plans``/``blockers``/``census_plans``/``census_post_plans`` children) are +DERIVED convergence bookkeeping: each census chains to its predecessor +(``sequence_no + 1``) and carries unresolved plans forward. The ACCEPTED +materialization state — ``raw_sessions.revision_authority`` and the index's +``raw_revision_heads`` / ``raw_revision_applications`` — lives OUTSIDE these +tables and is untouched here. + +When the ledger accumulates inconsistent carried-forward state (e.g. a stale-plan +blocker marks sibling plans ``CARRIED_FORWARD``, and later raw deletions drop +them out of the recomputed frontier so the finalize postflight +``persistent ⊄ post_ids`` throws — live incident 2026-07-22 after hook +de-inflation), no new census can finalize to become a clean baseline, and the +daemon defers every pass. Emptying the ledger removes the poisoned predecessor: +the next daemon pass builds census #1 fresh over the current raw set +(``predecessor = None``), with no carried-forward history and no stale blocker. + +``raw_authority_parser_census`` is intentionally KEPT — it holds resource-blocked +parser fingerprints keyed to raws (FK-cascaded from ``raw_sessions``), not +census-cycle bookkeeping, and the whale pass consumes it. +""" + +from __future__ import annotations + +import sqlite3 +from dataclasses import dataclass +from pathlib import Path + +from polylogue.config import Config +from polylogue.maintenance.offline_guard import offline_maintenance_block_reason +from polylogue.paths import render_root +from polylogue.storage.archive_identity import ArchiveLocation + +# Children before parents: blockers/census_plans/census_post_plans reference +# raw_authority_plans via NO ACTION (RESTRICT) FKs; census_plans/post_plans also +# CASCADE from raw_authority_censuses. Deleting in this order with foreign_keys +# ON leaves no dangling reference. +_LEDGER_TABLES_CHILD_FIRST = ( + "raw_authority_blockers", + "raw_authority_census_plans", + "raw_authority_census_post_plans", + "raw_authority_plans", + "raw_authority_censuses", +) + + +@dataclass(frozen=True, slots=True) +class RawAuthorityResetReport: + """Row counts removed from the census ledger (dry-run or applied).""" + + censuses: int + plans: int + blockers: int + census_plans: int + census_post_plans: int + applied: bool + + +def _counts(conn: sqlite3.Connection) -> dict[str, int]: + return { + table: int(conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0]) for table in _LEDGER_TABLES_CHILD_FIRST + } + + +def reset_raw_authority_census(archive_root: Path, *, dry_run: bool = True) -> RawAuthorityResetReport: + """Empty the census planning ledger. ``dry_run`` reports counts only.""" + source_db = archive_root / "source.db" + if not source_db.is_file(): + raise FileNotFoundError(source_db) + + with sqlite3.connect(source_db) as conn: + conn.execute("PRAGMA foreign_keys = ON") + before = _counts(conn) + + if not dry_run: + active_config = Config( + archive_root=archive_root, + render_root=render_root(), + sources=[], + db_path=ArchiveLocation.resolve(archive_root).active_index_path, + ) + if reason := offline_maintenance_block_reason(active_config, active=True, dry_run=False): + raise RuntimeError(reason) + for table in _LEDGER_TABLES_CHILD_FIRST: + conn.execute(f"DELETE FROM {table}") + + return RawAuthorityResetReport( + censuses=before["raw_authority_censuses"], + plans=before["raw_authority_plans"], + blockers=before["raw_authority_blockers"], + census_plans=before["raw_authority_census_plans"], + census_post_plans=before["raw_authority_census_post_plans"], + applied=not dry_run, + ) + + +@dataclass(frozen=True, slots=True) +class IndexSeedPruneReport: + """Index revision-authority read-model rows removed (dry-run or applied).""" + + revision_heads: int + revision_applications: int + applied: bool + + +def prune_orphaned_index_revision_seeds(archive_root: Path, *, dry_run: bool = True) -> IndexSeedPruneReport: + """Delete index raw-frontier seeds whose raw is gone from the source tier. + + ``raw_revision_heads`` / ``raw_revision_applications`` are the index's + (rebuildable) revision-authority read model. After a source raw is deleted + (hook de-inflation), the seeds referencing it become broken predecessor + chains — the daemon's raw-frontier integrity check reports them as violated + and cannot converge past them. Deleting the seeds whose ``accepted_raw_id`` / + ``raw_id`` no longer exists in ``source.raw_sessions`` restores a clean + frontier; seeds for present raws are untouched. + """ + index_db = archive_root / "index.db" + source_db = archive_root / "source.db" + if not index_db.exists() or not source_db.is_file(): + raise FileNotFoundError(index_db if not index_db.exists() else source_db) + + with sqlite3.connect(index_db) as conn: + conn.execute("ATTACH DATABASE ? AS src", (str(source_db),)) + heads = int( + conn.execute( + "SELECT COUNT(*) FROM raw_revision_heads WHERE accepted_raw_id NOT IN (SELECT raw_id FROM src.raw_sessions)" + ).fetchone()[0] + ) + apps = int( + conn.execute( + "SELECT COUNT(*) FROM raw_revision_applications WHERE raw_id NOT IN (SELECT raw_id FROM src.raw_sessions)" + ).fetchone()[0] + ) + if not dry_run: + active_config = Config( + archive_root=archive_root, + render_root=render_root(), + sources=[], + db_path=ArchiveLocation.resolve(archive_root).active_index_path, + ) + if reason := offline_maintenance_block_reason(active_config, active=True, dry_run=False): + raise RuntimeError(reason) + conn.execute( + "DELETE FROM raw_revision_heads WHERE accepted_raw_id NOT IN (SELECT raw_id FROM src.raw_sessions)" + ) + conn.execute( + "DELETE FROM raw_revision_applications WHERE raw_id NOT IN (SELECT raw_id FROM src.raw_sessions)" + ) + + return IndexSeedPruneReport(revision_heads=heads, revision_applications=apps, applied=not dry_run) diff --git a/tests/unit/maintenance/test_raw_authority_reset.py b/tests/unit/maintenance/test_raw_authority_reset.py new file mode 100644 index 0000000000..30b749a765 --- /dev/null +++ b/tests/unit/maintenance/test_raw_authority_reset.py @@ -0,0 +1,115 @@ +"""Raw-authority census-ledger reset (convergence recovery).""" + +from __future__ import annotations + +import hashlib +import sqlite3 +from pathlib import Path + +from polylogue.maintenance.raw_authority_reset import ( + prune_orphaned_index_revision_seeds, + reset_raw_authority_census, +) +from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root + + +def _seed_ledger(source_db: Path) -> None: + with sqlite3.connect(source_db) as conn: + conn.execute("PRAGMA foreign_keys = OFF") # seeding only + conn.execute( + "INSERT INTO raw_authority_censuses (census_id, sequence_no, scope_json, residual_json, " + "parser_fingerprint, mode, lifecycle_status, quiescent, inventory_digest, residual_digest, " + "plan_count, executable_plan_count, residual_plan_count, created_at_ms) " + "VALUES ('c1',1,'{}','{}','fp','apply','planned',1,?,?,1,1,0,1)", + (hashlib.sha256(b"inv").hexdigest(), hashlib.sha256(b"res").hexdigest()), + ) + digest = hashlib.sha256(b"plan-1").hexdigest() + conn.execute( + "INSERT INTO raw_authority_plans (plan_id, input_digest, input_raw_ids_json, logical_keys_json, " + "authority_witness_json, source_preconditions_json, index_preconditions_json, created_at_ms) " + "VALUES ('plan-1',?,'[\"r1\"]','[]','{}','{}','{}',1)", + (digest,), + ) + conn.execute( + "INSERT INTO raw_authority_blockers (blocker_id, plan_id, census_id, reason, expected_json, " + "observed_json, created_at_ms) VALUES ('blk-1','plan-1','c1','r','{}','{}',1)" + ) + conn.execute( + "INSERT INTO raw_authority_census_plans (census_id, plan_id, ordinal, selected, outcome_status, " + "reason, next_action, recorded_at_ms) VALUES ('c1','plan-1',0,1,'carried_forward','r','n',1)" + ) + conn.execute( + "INSERT INTO raw_authority_census_post_plans (census_id, plan_id, ordinal) VALUES ('c1','plan-1',0)" + ) + + +def test_reset_empties_ledger_but_preserves_accepted_state(tmp_path: Path) -> None: + initialize_active_archive_root(tmp_path) + source_db = tmp_path / "source.db" + _seed_ledger(source_db) + # Accepted materialization state that MUST survive a census-ledger reset. + with sqlite3.connect(source_db) as conn: + conn.execute( + "INSERT INTO raw_sessions (raw_id, origin, source_path, source_index, blob_hash, blob_size, " + "acquired_at_ms, revision_authority) VALUES ('r-keep','codex-session','/p',0,?,10,1,'byte_proven')", + (b"\x01" * 32,), + ) + + dry = reset_raw_authority_census(tmp_path, dry_run=True) + assert dry.applied is False + assert (dry.censuses, dry.plans, dry.blockers, dry.census_plans, dry.census_post_plans) == (1, 1, 1, 1, 1) + with sqlite3.connect(source_db) as conn: + assert conn.execute("SELECT COUNT(*) FROM raw_authority_censuses").fetchone()[0] == 1 # dry: nothing deleted + + report = reset_raw_authority_census(tmp_path, dry_run=False) + assert report.applied is True + + with sqlite3.connect(source_db) as conn: + for table in ( + "raw_authority_censuses", + "raw_authority_plans", + "raw_authority_blockers", + "raw_authority_census_plans", + "raw_authority_census_post_plans", + ): + assert conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] == 0, table + # Accepted state preserved. + row = conn.execute("SELECT revision_authority FROM raw_sessions WHERE raw_id='r-keep'").fetchone() + assert row == ("byte_proven",) + + +def test_prune_orphaned_index_revision_seeds(tmp_path: Path) -> None: + initialize_active_archive_root(tmp_path) + source_db = tmp_path / "source.db" + index_db = tmp_path / "index.db" + # One present raw in source; the seeds referencing 'r-gone' are orphaned. + with sqlite3.connect(source_db) as conn: + conn.execute( + "INSERT INTO raw_sessions (raw_id, origin, source_path, source_index, blob_hash, blob_size, " + "acquired_at_ms) VALUES ('r-present','codex-session','/p',0,?,10,1)", + (b"\x02" * 32,), + ) + with sqlite3.connect(index_db) as conn: + for raw_id in ("r-present", "r-gone"): + conn.execute( + "INSERT INTO raw_revision_heads (logical_source_key, session_id, accepted_raw_id, " + "accepted_source_revision, accepted_content_hash, accepted_frontier_kind, accepted_frontier, " + "acquisition_generation, decided_at_ms) VALUES (?,?,?,'sr',?,'byte',1,0,1)", + (f"k-{raw_id}", f"s-{raw_id}", raw_id, b"\x03" * 32), + ) + conn.execute( + "INSERT INTO raw_revision_applications (decision_id, raw_id, session_id, logical_source_key, " + "source_revision, acquisition_generation, decision, detail, decided_at_ms) " + "VALUES (?,?,?,?,'sr',0,'selected_baseline','d',1)", + (f"d-{raw_id}", raw_id, f"s-{raw_id}", f"k-{raw_id}"), + ) + + dry = prune_orphaned_index_revision_seeds(tmp_path, dry_run=True) + assert (dry.revision_heads, dry.revision_applications, dry.applied) == (1, 1, False) + + report = prune_orphaned_index_revision_seeds(tmp_path, dry_run=False) + assert report.applied is True and report.revision_heads == 1 and report.revision_applications == 1 + + with sqlite3.connect(index_db) as conn: + assert {r[0] for r in conn.execute("SELECT accepted_raw_id FROM raw_revision_heads")} == {"r-present"} + assert {r[0] for r in conn.execute("SELECT raw_id FROM raw_revision_applications")} == {"r-present"} From 18be08e52de03f26af8830925baaf7b5848ebb92 Mon Sep 17 00:00:00 2001 From: Sinity Date: Wed, 22 Jul 2026 21:02:23 +0200 Subject: [PATCH 2/2] fix(maintenance): guard raw-authority census reset durability Move census-reset and index-seed mutation semantics into the storage layer. Require a verified source-tier backup manifest before deleting durable census history, and resolve the active index generation through its pointer before pruning rebuildable seeds. Ref polylogue-31r1, polylogue-t93b. Co-Authored-By: Codex --- polylogue/maintenance/raw_authority_reset.py | 111 ++++++------------ polylogue/storage/raw_authority.py | 109 +++++++++++++++++ .../maintenance/test_raw_authority_reset.py | 38 +++++- 3 files changed, 178 insertions(+), 80 deletions(-) diff --git a/polylogue/maintenance/raw_authority_reset.py b/polylogue/maintenance/raw_authority_reset.py index e799bbdd4d..424235e3b5 100644 --- a/polylogue/maintenance/raw_authority_reset.py +++ b/polylogue/maintenance/raw_authority_reset.py @@ -24,26 +24,14 @@ from __future__ import annotations -import sqlite3 from dataclasses import dataclass from pathlib import Path from polylogue.config import Config from polylogue.maintenance.offline_guard import offline_maintenance_block_reason from polylogue.paths import render_root -from polylogue.storage.archive_identity import ArchiveLocation - -# Children before parents: blockers/census_plans/census_post_plans reference -# raw_authority_plans via NO ACTION (RESTRICT) FKs; census_plans/post_plans also -# CASCADE from raw_authority_censuses. Deleting in this order with foreign_keys -# ON leaves no dangling reference. -_LEDGER_TABLES_CHILD_FIRST = ( - "raw_authority_blockers", - "raw_authority_census_plans", - "raw_authority_census_post_plans", - "raw_authority_plans", - "raw_authority_censuses", -) +from polylogue.storage.raw_authority import prune_orphaned_index_revision_seeds as _prune_orphaned_index_revision_seeds +from polylogue.storage.raw_authority import reset_raw_authority_census_ledger @dataclass(frozen=True, slots=True) @@ -58,40 +46,33 @@ class RawAuthorityResetReport: applied: bool -def _counts(conn: sqlite3.Connection) -> dict[str, int]: - return { - table: int(conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0]) for table in _LEDGER_TABLES_CHILD_FIRST - } +def _offline_config(archive_root: Path) -> Config: + return Config(archive_root=archive_root, render_root=render_root(), sources=[]) -def reset_raw_authority_census(archive_root: Path, *, dry_run: bool = True) -> RawAuthorityResetReport: +def reset_raw_authority_census( + archive_root: Path, + *, + backup_manifest: Path | None = None, + dry_run: bool = True, +) -> RawAuthorityResetReport: """Empty the census planning ledger. ``dry_run`` reports counts only.""" - source_db = archive_root / "source.db" - if not source_db.is_file(): - raise FileNotFoundError(source_db) - - with sqlite3.connect(source_db) as conn: - conn.execute("PRAGMA foreign_keys = ON") - before = _counts(conn) - - if not dry_run: - active_config = Config( - archive_root=archive_root, - render_root=render_root(), - sources=[], - db_path=ArchiveLocation.resolve(archive_root).active_index_path, - ) - if reason := offline_maintenance_block_reason(active_config, active=True, dry_run=False): - raise RuntimeError(reason) - for table in _LEDGER_TABLES_CHILD_FIRST: - conn.execute(f"DELETE FROM {table}") + if not dry_run and ( + reason := offline_maintenance_block_reason(_offline_config(archive_root), active=True, dry_run=False) + ): + raise RuntimeError(reason) + before = reset_raw_authority_census_ledger( + archive_root, + backup_manifest=backup_manifest, + dry_run=dry_run, + ) return RawAuthorityResetReport( - censuses=before["raw_authority_censuses"], - plans=before["raw_authority_plans"], - blockers=before["raw_authority_blockers"], - census_plans=before["raw_authority_census_plans"], - census_post_plans=before["raw_authority_census_post_plans"], + censuses=before.censuses, + plans=before.plans, + blockers=before.blockers, + census_plans=before.census_plans, + census_post_plans=before.census_post_plans, applied=not dry_run, ) @@ -116,37 +97,13 @@ def prune_orphaned_index_revision_seeds(archive_root: Path, *, dry_run: bool = T ``raw_id`` no longer exists in ``source.raw_sessions`` restores a clean frontier; seeds for present raws are untouched. """ - index_db = archive_root / "index.db" - source_db = archive_root / "source.db" - if not index_db.exists() or not source_db.is_file(): - raise FileNotFoundError(index_db if not index_db.exists() else source_db) - - with sqlite3.connect(index_db) as conn: - conn.execute("ATTACH DATABASE ? AS src", (str(source_db),)) - heads = int( - conn.execute( - "SELECT COUNT(*) FROM raw_revision_heads WHERE accepted_raw_id NOT IN (SELECT raw_id FROM src.raw_sessions)" - ).fetchone()[0] - ) - apps = int( - conn.execute( - "SELECT COUNT(*) FROM raw_revision_applications WHERE raw_id NOT IN (SELECT raw_id FROM src.raw_sessions)" - ).fetchone()[0] - ) - if not dry_run: - active_config = Config( - archive_root=archive_root, - render_root=render_root(), - sources=[], - db_path=ArchiveLocation.resolve(archive_root).active_index_path, - ) - if reason := offline_maintenance_block_reason(active_config, active=True, dry_run=False): - raise RuntimeError(reason) - conn.execute( - "DELETE FROM raw_revision_heads WHERE accepted_raw_id NOT IN (SELECT raw_id FROM src.raw_sessions)" - ) - conn.execute( - "DELETE FROM raw_revision_applications WHERE raw_id NOT IN (SELECT raw_id FROM src.raw_sessions)" - ) - - return IndexSeedPruneReport(revision_heads=heads, revision_applications=apps, applied=not dry_run) + if not dry_run and ( + reason := offline_maintenance_block_reason(_offline_config(archive_root), active=True, dry_run=False) + ): + raise RuntimeError(reason) + counts = _prune_orphaned_index_revision_seeds(archive_root, dry_run=dry_run) + return IndexSeedPruneReport( + revision_heads=counts.revision_heads, + revision_applications=counts.revision_applications, + applied=not dry_run, + ) diff --git a/polylogue/storage/raw_authority.py b/polylogue/storage/raw_authority.py index 6d9749dfe6..a08532f6d5 100644 --- a/polylogue/storage/raw_authority.py +++ b/polylogue/storage/raw_authority.py @@ -20,6 +20,9 @@ from polylogue.core.json import JSONDocument, json_document from polylogue.logging import get_logger +from polylogue.storage.archive_identity import ArchiveLocation +from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier +from polylogue.storage.sqlite.migration_runner import validate_migration_backup_manifest RAW_AUTHORITY_PARSER_FINGERPRINT = "revision-membership-v1" RAW_AUTHORITY_CENSUS_QUERY_PREFIX = "polylogue://raw-authority-census/" @@ -27,6 +30,33 @@ RAW_AUTHORITY_DETAIL_CHUNK_CHARS = 16_384 logger = get_logger(__name__) +_RESET_LEDGER_TABLES_CHILD_FIRST = ( + "raw_authority_blockers", + "raw_authority_census_plans", + "raw_authority_census_post_plans", + "raw_authority_plans", + "raw_authority_censuses", +) + + +@dataclass(frozen=True, slots=True) +class RawAuthorityCensusResetCounts: + """Counts for a reset of derived raw-authority census bookkeeping.""" + + censuses: int + plans: int + blockers: int + census_plans: int + census_post_plans: int + + +@dataclass(frozen=True, slots=True) +class OrphanedIndexRevisionSeedCounts: + """Counts for rebuildable revision-seed rows absent from source authority.""" + + revision_heads: int + revision_applications: int + class RawReplayPlanStatus(StrEnum): EXECUTED = "executed" @@ -1847,12 +1877,89 @@ def reject_invalid_raw_replay_application( return outcome +def reset_raw_authority_census_ledger( + archive_root: Path, + *, + backup_manifest: Path | None, + dry_run: bool, +) -> RawAuthorityCensusResetCounts: + """Reset derived census bookkeeping after authenticating a source backup.""" + source_db = archive_root / "source.db" + if not source_db.is_file(): + raise FileNotFoundError(source_db) + with closing(sqlite3.connect(source_db)) as conn: + conn.execute("PRAGMA foreign_keys = ON") + counts = { + "raw_authority_censuses": int(conn.execute("SELECT COUNT(*) FROM raw_authority_censuses").fetchone()[0]), + "raw_authority_plans": int(conn.execute("SELECT COUNT(*) FROM raw_authority_plans").fetchone()[0]), + "raw_authority_blockers": int(conn.execute("SELECT COUNT(*) FROM raw_authority_blockers").fetchone()[0]), + "raw_authority_census_plans": int( + conn.execute("SELECT COUNT(*) FROM raw_authority_census_plans").fetchone()[0] + ), + "raw_authority_census_post_plans": int( + conn.execute("SELECT COUNT(*) FROM raw_authority_census_post_plans").fetchone()[0] + ), + } + if not dry_run: + if backup_manifest is None: + raise ValueError("raw-authority census reset requires a verified source backup manifest") + validate_migration_backup_manifest(backup_manifest, ArchiveTier.SOURCE, connection=conn) + for table in _RESET_LEDGER_TABLES_CHILD_FIRST: + conn.execute(f"DELETE FROM {table}") + conn.commit() + return RawAuthorityCensusResetCounts( + censuses=counts["raw_authority_censuses"], + plans=counts["raw_authority_plans"], + blockers=counts["raw_authority_blockers"], + census_plans=counts["raw_authority_census_plans"], + census_post_plans=counts["raw_authority_census_post_plans"], + ) + + +def prune_orphaned_index_revision_seeds( + archive_root: Path, + *, + dry_run: bool, +) -> OrphanedIndexRevisionSeedCounts: + """Prune rebuildable revision seeds that no longer have source authority.""" + source_db = archive_root / "source.db" + index_db = ArchiveLocation.resolve(archive_root).active_index_path + if not source_db.is_file() or not index_db.is_file(): + raise FileNotFoundError(source_db if not source_db.is_file() else index_db) + with closing(sqlite3.connect(index_db)) as conn: + conn.execute("ATTACH DATABASE ? AS src", (str(source_db),)) + heads = int( + conn.execute( + "SELECT COUNT(*) FROM raw_revision_heads WHERE accepted_raw_id NOT IN (SELECT raw_id FROM src.raw_sessions)" + ).fetchone()[0] + ) + applications = int( + conn.execute( + "SELECT COUNT(*) FROM raw_revision_applications WHERE raw_id NOT IN (SELECT raw_id FROM src.raw_sessions)" + ).fetchone()[0] + ) + if not dry_run: + conn.execute( + "DELETE FROM raw_revision_heads WHERE accepted_raw_id NOT IN (SELECT raw_id FROM src.raw_sessions)" + ) + conn.execute( + "DELETE FROM raw_revision_applications WHERE raw_id NOT IN (SELECT raw_id FROM src.raw_sessions)" + ) + conn.commit() + return OrphanedIndexRevisionSeedCounts( + revision_heads=heads, + revision_applications=applications, + ) + + __all__ = [ "RAW_AUTHORITY_CENSUS_QUERY_PREFIX", "RAW_AUTHORITY_DETAIL_CHUNK_CHARS", "RAW_AUTHORITY_DETAIL_QUERY_PREFIX", "RAW_AUTHORITY_PARSER_FINGERPRINT", "RawAuthorityCensusReceipt", + "RawAuthorityCensusResetCounts", + "OrphanedIndexRevisionSeedCounts", "RawReplayPlan", "RawReplayPlanOutcome", "RawReplayPlanStatus", @@ -1872,6 +1979,8 @@ def reject_invalid_raw_replay_application( "read_raw_authority_detail", "record_raw_authority_census", "record_raw_replay_outcome", + "reset_raw_authority_census_ledger", + "prune_orphaned_index_revision_seeds", "reject_invalid_raw_replay_application", "reject_stale_raw_replay_plan", "resolve_raw_authority_blocker", diff --git a/tests/unit/maintenance/test_raw_authority_reset.py b/tests/unit/maintenance/test_raw_authority_reset.py index 30b749a765..35e6e4529d 100644 --- a/tests/unit/maintenance/test_raw_authority_reset.py +++ b/tests/unit/maintenance/test_raw_authority_reset.py @@ -3,14 +3,18 @@ from __future__ import annotations import hashlib +import shutil import sqlite3 from pathlib import Path +import pytest + from polylogue.maintenance.raw_authority_reset import ( prune_orphaned_index_revision_seeds, reset_raw_authority_census, ) from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root +from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier def _seed_ledger(source_db: Path) -> None: @@ -43,7 +47,7 @@ def _seed_ledger(source_db: Path) -> None: ) -def test_reset_empties_ledger_but_preserves_accepted_state(tmp_path: Path) -> None: +def test_reset_empties_ledger_but_preserves_accepted_state(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: initialize_active_archive_root(tmp_path) source_db = tmp_path / "source.db" _seed_ledger(source_db) @@ -61,8 +65,18 @@ def test_reset_empties_ledger_but_preserves_accepted_state(tmp_path: Path) -> No with sqlite3.connect(source_db) as conn: assert conn.execute("SELECT COUNT(*) FROM raw_authority_censuses").fetchone()[0] == 1 # dry: nothing deleted - report = reset_raw_authority_census(tmp_path, dry_run=False) + validated: list[tuple[Path, object]] = [] + + def validate(manifest: Path, tier: object, *, connection: sqlite3.Connection) -> Path: + validated.append((manifest, tier)) + assert connection.execute("SELECT 1").fetchone() == (1,) + return manifest.with_name("verification-receipt.json") + + monkeypatch.setattr("polylogue.storage.raw_authority.validate_migration_backup_manifest", validate) + manifest = tmp_path / "verified-backup" / "manifest.json" + report = reset_raw_authority_census(tmp_path, backup_manifest=manifest, dry_run=False) assert report.applied is True + assert validated == [(manifest, ArchiveTier.SOURCE)] with sqlite3.connect(source_db) as conn: for table in ( @@ -78,6 +92,17 @@ def test_reset_empties_ledger_but_preserves_accepted_state(tmp_path: Path) -> No assert row == ("byte_proven",) +def test_reset_refuses_to_delete_without_verified_backup_manifest(tmp_path: Path) -> None: + initialize_active_archive_root(tmp_path) + _seed_ledger(tmp_path / "source.db") + + with pytest.raises(ValueError, match="verified source backup manifest"): + reset_raw_authority_census(tmp_path, dry_run=False) + + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT COUNT(*) FROM raw_authority_censuses").fetchone() == (1,) + + def test_prune_orphaned_index_revision_seeds(tmp_path: Path) -> None: initialize_active_archive_root(tmp_path) source_db = tmp_path / "source.db" @@ -104,12 +129,19 @@ def test_prune_orphaned_index_revision_seeds(tmp_path: Path) -> None: (f"d-{raw_id}", raw_id, f"s-{raw_id}", f"k-{raw_id}"), ) + active_index = tmp_path / "active-generation" / "index.db" + active_index.parent.mkdir() + shutil.copy2(index_db, active_index) + (tmp_path / ".index-active-pointer").write_text(str(active_index), encoding="utf-8") + dry = prune_orphaned_index_revision_seeds(tmp_path, dry_run=True) assert (dry.revision_heads, dry.revision_applications, dry.applied) == (1, 1, False) report = prune_orphaned_index_revision_seeds(tmp_path, dry_run=False) assert report.applied is True and report.revision_heads == 1 and report.revision_applications == 1 - with sqlite3.connect(index_db) as conn: + with sqlite3.connect(active_index) as conn: assert {r[0] for r in conn.execute("SELECT accepted_raw_id FROM raw_revision_heads")} == {"r-present"} assert {r[0] for r in conn.execute("SELECT raw_id FROM raw_revision_applications")} == {"r-present"} + with sqlite3.connect(index_db) as conn: + assert {r[0] for r in conn.execute("SELECT accepted_raw_id FROM raw_revision_heads")} == {"r-present", "r-gone"}