From f4c2598ce22ed48414b10d12654a0493081e272b Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 2 Aug 2026 13:15:30 +0200 Subject: [PATCH] feat(sources): admit ~/.claude/todos plan snapshots with a session read model Problem: Claude Code writes several artifacts beyond the session JSONL transcript that the archive never captured. ~/.claude/todos/*.json holds the agent's live TODO/plan state per session (task lists with status), overwritten wholesale on every TodoWrite call and pruned by the harness on its own schedule -- unread state is eventually lost, not merely delayed (polylogue-t0p). Solution: - New todo_snapshot OriginArtifactRule (polylogue/sources/origin_specs.py) admits ~/.claude/todos/[-agent-].json as a fact-tier Claude Code artifact, following the same admission shape as the existing workflow_run_snapshot/agent_sidecar_meta rules, with an explicit fidelity note (preserved: content/status/priority/id and list order; lost: intermediate transitions between two watcher-observed snapshots, no in-file timestamp). - polylogue/sources/parsers/claude/todos.py parses the bare JSON array, recovering session_id/agent_id from the filename. - ~/.claude/todos is a new configured/watched source root (polylogue/config.py, polylogue/paths/_roots.py, polylogue/sources/live/watcher.py), aliased to the claude-code provider (polylogue/core/provider_identity.py) so the existing _admit_non_session_origin_artifacts admission path and OriginSpec classification pick it up with no bespoke wiring. - polylogue/insights/claude_todo_projection.py is a storage-free, session-linked read model (mirrors insights/run_projection.py's own precedent) over every retained todo_snapshot raw revision: per-session plan state, latest completion rate, and per-item status-transition history across observed snapshots. - plan_completion_rate registered in insights/measurement/registered_metrics.py's DEFAULT_METRIC_REGISTRY (rxdo.9.1 identity layer), the same bounded slice session_cost_usd already uses -- reachable via MCP get(ref="metric:plan_completion_rate"). polylogue-9l5.7's full statistics/composition registry remains unstarted; this does not invent a second identity mechanism. Deferred (see PR body for file-history follow-up scope and what this session learned about its on-disk shape): ~/.claude/file-history/ content acquisition, and the lower-priority history.jsonl/debug/mcp-logs/ide/jobs artifacts the bead also names. Verification: - devtools test tests/unit/sources/test_origin_specs.py tests/unit/sources/test_parsers_claude_todos.py tests/unit/insights/test_claude_todo_projection.py tests/unit/core/test_config.py tests/unit/sources/test_live_watcher.py -> 219 passed (1 pre-existing failure, test_live_batch_processor_records_durable_attempt, reproduces identically on master with this diff stashed -- unrelated to this change). - mypy polylogue -> Success: no issues found in 1113 source files. - ruff check / ruff format --check -> clean. - devtools render all --check -> OK (topology projection regenerated for the two new modules). - devtools lab policy classifier-fingerprints reports one pre-existing drift (ai_parser.py:looks_like_ai) reproduced identically with this diff stashed -- not introduced by this change. Ref polylogue-t0p Co-Authored-By: Claude --- docs/plans/topology-target.yaml | 50 +++--- polylogue/archive/artifact_taxonomy/models.py | 1 + polylogue/config.py | 3 + polylogue/core/provider_identity.py | 1 + polylogue/insights/claude_todo_projection.py | 139 ++++++++++++++++ .../measurement/registered_metrics.py | 37 ++++- polylogue/paths/__init__.py | 2 + polylogue/paths/_roots.py | 12 ++ polylogue/sources/live/watcher.py | 12 ++ polylogue/sources/origin_specs.py | 20 +++ polylogue/sources/parsers/claude/__init__.py | 2 + polylogue/sources/parsers/claude/todos.py | 151 ++++++++++++++++++ tests/unit/core/test_config.py | 23 +++ .../insights/test_claude_todo_projection.py | 134 ++++++++++++++++ tests/unit/sources/test_live_watcher.py | 13 ++ tests/unit/sources/test_origin_specs.py | 1 + .../unit/sources/test_parsers_claude_todos.py | 121 ++++++++++++++ 17 files changed, 696 insertions(+), 26 deletions(-) create mode 100644 polylogue/insights/claude_todo_projection.py create mode 100644 polylogue/sources/parsers/claude/todos.py create mode 100644 tests/unit/insights/test_claude_todo_projection.py create mode 100644 tests/unit/sources/test_parsers_claude_todos.py diff --git a/docs/plans/topology-target.yaml b/docs/plans/topology-target.yaml index ba19f5c2a7..ff817e415b 100644 --- a/docs/plans/topology-target.yaml +++ b/docs/plans/topology-target.yaml @@ -70,7 +70,7 @@ files: owner: stable cross_cut: { api: async } - path: polylogue/api/archive.py - loc: 7264 + loc: 7343 target: polylogue/api/archive.py owner: stable cross_cut: { api: async } @@ -174,7 +174,7 @@ files: owner: archive-artifact-taxonomy reason: archive-domain semantics - path: polylogue/archive/artifact_taxonomy/models.py - loc: 40 + loc: 41 target: polylogue/archive/artifact_taxonomy/models.py owner: archive-artifact-taxonomy reason: archive-domain semantics @@ -1309,7 +1309,7 @@ files: target: polylogue/cli/verb_names.py owner: stable - path: polylogue/config.py - loc: 2897 + loc: 2900 target: polylogue/config.py owner: kernel reason: kernel root rule @@ -1444,7 +1444,7 @@ files: owner: core-primitive reason: core primitive - path: polylogue/core/provider_identity.py - loc: 179 + loc: 180 target: polylogue/core/provider_identity.py owner: core-primitive reason: core primitive @@ -1454,7 +1454,7 @@ files: owner: core-primitive reason: core primitive - path: polylogue/core/refs.py - loc: 470 + loc: 521 target: polylogue/core/refs.py owner: core-primitive reason: core primitive @@ -1922,9 +1922,13 @@ files: target: polylogue/insights/authored_payloads.py owner: stable - path: polylogue/insights/capture_coverage.py - loc: 414 + loc: 413 target: polylogue/insights/capture_coverage.py owner: stable + - path: polylogue/insights/claude_todo_projection.py + loc: 139 + target: polylogue/insights/claude_todo_projection.py + owner: stable - path: polylogue/insights/claude_workflow_evidence.py loc: 778 target: polylogue/insights/claude_workflow_evidence.py @@ -2054,7 +2058,7 @@ files: target: polylogue/insights/measurement/ratio.py owner: stable - path: polylogue/insights/measurement/registered_metrics.py - loc: 57 + loc: 84 target: polylogue/insights/measurement/registered_metrics.py owner: stable - path: polylogue/insights/measurement/registration.py @@ -2427,11 +2431,11 @@ files: target: polylogue/operations/work_effect_reconciliation.py owner: stable - path: polylogue/paths/__init__.py - loc: 76 + loc: 78 target: polylogue/paths/__init__.py owner: stable - path: polylogue/paths/_roots.py - loc: 246 + loc: 258 target: polylogue/paths/_roots.py owner: stable - path: polylogue/paths/sanitize.py @@ -3127,7 +3131,7 @@ files: target: polylogue/schemas/tooling_registry.py owner: stable - path: polylogue/schemas/type_narrowing.py - loc: 71 + loc: 84 target: polylogue/schemas/type_narrowing.py owner: stable - path: polylogue/schemas/validation/__init__.py @@ -3392,11 +3396,11 @@ files: target: polylogue/sources/live/tool_result_sidecars.py owner: stable - path: polylogue/sources/live/watcher.py - loc: 1499 + loc: 1511 target: polylogue/sources/live/watcher.py owner: stable - path: polylogue/sources/origin_specs.py - loc: 1390 + loc: 1410 target: polylogue/sources/origin_specs.py owner: stable - path: polylogue/sources/parsers/antigravity.py @@ -3437,7 +3441,7 @@ files: target: polylogue/sources/parsers/chatgpt_sidecars.py owner: stable - path: polylogue/sources/parsers/claude/__init__.py - loc: 78 + loc: 80 target: polylogue/sources/parsers/claude/__init__.py owner: stable - path: polylogue/sources/parsers/claude/ai_parser.py @@ -3468,6 +3472,10 @@ files: loc: 230 target: polylogue/sources/parsers/claude/orchestration.py owner: stable + - path: polylogue/sources/parsers/claude/todos.py + loc: 151 + target: polylogue/sources/parsers/claude/todos.py + owner: stable - path: polylogue/sources/parsers/codex.py loc: 2651 target: polylogue/sources/parsers/codex.py @@ -4149,7 +4157,7 @@ files: target: polylogue/storage/sqlite/archive_tiers/__init__.py owner: stable - path: polylogue/storage/sqlite/archive_tiers/archive.py - loc: 11433 + loc: 11598 target: polylogue/storage/sqlite/archive_tiers/archive.py owner: stable - path: polylogue/storage/sqlite/archive_tiers/archive_init.py @@ -4189,7 +4197,7 @@ files: target: polylogue/storage/sqlite/archive_tiers/embeddings.py owner: stable - path: polylogue/storage/sqlite/archive_tiers/index.py - loc: 2184 + loc: 2197 target: polylogue/storage/sqlite/archive_tiers/index.py owner: stable - path: polylogue/storage/sqlite/archive_tiers/index_convergence.py @@ -4233,11 +4241,11 @@ files: target: polylogue/storage/sqlite/archive_tiers/session_annotations_write.py owner: stable - path: polylogue/storage/sqlite/archive_tiers/source.py - loc: 456 + loc: 475 target: polylogue/storage/sqlite/archive_tiers/source.py owner: stable - path: polylogue/storage/sqlite/archive_tiers/source_write.py - loc: 1185 + loc: 1286 target: polylogue/storage/sqlite/archive_tiers/source_write.py owner: stable - path: polylogue/storage/sqlite/archive_tiers/types.py @@ -4297,7 +4305,7 @@ files: target: polylogue/storage/sqlite/finding_provenance.py owner: stable - path: polylogue/storage/sqlite/lifecycle.py - loc: 819 + loc: 832 target: polylogue/storage/sqlite/lifecycle.py owner: stable - path: polylogue/storage/sqlite/maintenance.py @@ -4414,7 +4422,7 @@ files: target: polylogue/storage/sqlite/queries/raw.py owner: stable - path: polylogue/storage/sqlite/queries/raw_reads.py - loc: 341 + loc: 372 target: polylogue/storage/sqlite/queries/raw_reads.py owner: stable cross_cut: { layer: read } @@ -4423,7 +4431,7 @@ files: target: polylogue/storage/sqlite/queries/raw_state.py owner: stable - path: polylogue/storage/sqlite/queries/raw_writes.py - loc: 107 + loc: 123 target: polylogue/storage/sqlite/queries/raw_writes.py owner: stable cross_cut: { layer: write } @@ -4601,7 +4609,7 @@ files: target: polylogue/surfaces/chronicle.py owner: stable - path: polylogue/surfaces/payloads.py - loc: 3919 + loc: 4022 target: polylogue/surfaces/payloads.py owner: stable - path: polylogue/surfaces/projection_spec.py diff --git a/polylogue/archive/artifact_taxonomy/models.py b/polylogue/archive/artifact_taxonomy/models.py index 9bb1752408..201b5e58ff 100644 --- a/polylogue/archive/artifact_taxonomy/models.py +++ b/polylogue/archive/artifact_taxonomy/models.py @@ -17,6 +17,7 @@ class ArtifactKind(StrEnum): AGENT_TRANSCRIPT = "agent_transcript" ADOPT_MANIFEST = "adopt_manifest" COORDINATOR_SESSION_STREAM = "coordinator_session_stream" + TODO_SNAPSHOT = "todo_snapshot" SESSION_INDEX = "session_index" BRIDGE_POINTER = "bridge_pointer" METADATA_DOCUMENT = "metadata_document" diff --git a/polylogue/config.py b/polylogue/config.py index d1621443ef..02443c6da5 100644 --- a/polylogue/config.py +++ b/polylogue/config.py @@ -2109,6 +2109,7 @@ class ResolvedSourcePaths: """Immutable local source discovery roots for one runtime.""" claude_code: Path + claude_code_todos: Path codex: Path gemini_cli: Path hermes: Path @@ -2256,6 +2257,7 @@ def resolve_runtime_config( ) source_paths = ResolvedSourcePaths( claude_code=bootstrap.home / ".claude" / "projects", + claude_code_todos=bootstrap.home / ".claude" / "todos", codex=bootstrap.home / ".codex" / "sessions", gemini_cli=bootstrap.home / ".gemini" / "tmp", hermes=_resolved_runtime_path( @@ -2272,6 +2274,7 @@ def resolve_runtime_config( ) local_candidates = ( ("claude-code", source_paths.claude_code), + ("claude-code-todos", source_paths.claude_code_todos), ("codex", source_paths.codex), ("gemini-cli", source_paths.gemini_cli), ("hermes", source_paths.hermes), diff --git a/polylogue/core/provider_identity.py b/polylogue/core/provider_identity.py index 7210d6d354..5587b85769 100644 --- a/polylogue/core/provider_identity.py +++ b/polylogue/core/provider_identity.py @@ -96,6 +96,7 @@ _RUNTIME_PROVIDER_ALIASES: Final[dict[str, str]] = { "claude": "claude-ai", "anthropic": "claude-ai", + "claude-code-todos": "claude-code", "openai": "chatgpt", "google": "gemini", "google-gemini": "gemini", diff --git a/polylogue/insights/claude_todo_projection.py b/polylogue/insights/claude_todo_projection.py new file mode 100644 index 0000000000..e834c54af1 --- /dev/null +++ b/polylogue/insights/claude_todo_projection.py @@ -0,0 +1,139 @@ +"""Session-linked read model over admitted ``todo_snapshot`` artifacts (polylogue-t0p). + +Every ``~/.claude/todos/*.json`` file Claude Code writes is admitted as raw, +provenance-carrying evidence via the ``todo_snapshot`` ``OriginArtifactRule`` +(``polylogue.sources.origin_specs``) exactly like Workflow's own fact +artifacts -- ``source.db`` remains the sole authority and every raw revision +the watcher ever observed is retained (``raw_sessions``); ``raw_artifacts`` +records the current classification per source path. + +This module is the read side: it loads every currently-classified +``todo_snapshot`` row, parses its payload, and groups the result by session so +a consumer gets one :class:`ClaudeTodoPlanState` per session -- the agent's +CURRENT plan plus, when the watcher captured more than one snapshot for the +same session, a per-item status-transition history. Deliberately +storage-free (mirrors ``insights/run_projection.py``'s own justification): +this is a pure read-time projection over already-admitted raw evidence, not a +new durable table, because there is no measured query pressure yet that would +justify one. +""" + +from __future__ import annotations + +import sqlite3 +from dataclasses import dataclass +from pathlib import Path + +from polylogue.core.enums import Origin, Provider +from polylogue.core.refs import ObjectRef +from polylogue.sources.origin_specs import artifact_rule_for_path +from polylogue.sources.parsers.claude.todos import ClaudeTodoSnapshot, parse_claude_todo_artifact +from polylogue.storage.blob_store import BlobStore + + +@dataclass(frozen=True, slots=True) +class ClaudeTodoSnapshotEvidence: + """One observed snapshot plus its acquisition time and evidence pointer.""" + + snapshot: ClaudeTodoSnapshot + acquired_at_ms: int + evidence_ref: ObjectRef + + +@dataclass(frozen=True, slots=True) +class ClaudeTodoPlanState: + """One session's plan history: every observed snapshot, oldest first.""" + + session_id: str + agent_id: str | None + snapshots: tuple[ClaudeTodoSnapshotEvidence, ...] + + @property + def latest(self) -> ClaudeTodoSnapshotEvidence: + return self.snapshots[-1] + + @property + def plan_completion_rate(self) -> float | None: + """The construct ``metric:plan_completion_rate`` reports (see ``registered_metrics.py``).""" + return self.latest.snapshot.completion_rate + + def status_transitions(self) -> dict[str, tuple[str, ...]]: + """Per-item-id status sequence across every observed snapshot, oldest first. + + Only tracks items with a stable ``id`` (Claude Code always assigns + one; a malformed entry without one is excluded -- there is no + identity to track a transition against). A single-snapshot session + still returns one-element sequences: "no transition observed yet" is + a real, distinct fact from "this item flipped status". + """ + history: dict[str, list[str]] = {} + for evidence in self.snapshots: + for item in evidence.snapshot.items: + if item.item_id is None: + continue + history.setdefault(item.item_id, []).append(item.status) + return {item_id: tuple(statuses) for item_id, statuses in history.items()} + + +def load_claude_todo_plan_states(archive_root: Path) -> tuple[ClaudeTodoPlanState, ...]: + """Materialize every session's plan state from EVERY retained ``todo_snapshot`` raw revision. + + Reads ``raw_sessions`` directly rather than the current-revision-only + ``raw_artifacts`` pointer table: Claude Code overwrites the same path on + every ``TodoWrite`` call, but the archive's mutable-file admission path + retains every observed revision as its own ``raw_sessions`` row (see + ``docs/internals.md``'s revision-retention model). Reading only the + current pointer would collapse a session's whole plan history down to + its latest snapshot, defeating the point of a status-transition read + model. + """ + source_db = archive_root / "source.db" + if not source_db.exists(): + return () + + with sqlite3.connect(source_db) as conn: + conn.row_factory = sqlite3.Row + rows = conn.execute( + """ + SELECT raw_id, source_path, lower(hex(blob_hash)) AS blob_hash, acquired_at_ms + FROM raw_sessions + WHERE origin = ? + ORDER BY acquired_at_ms ASC, rowid ASC + """, + (Origin.CLAUDE_CODE_SESSION.value,), + ).fetchall() + + if not rows: + return () + + blob_store = BlobStore(archive_root / "blob") + by_session: dict[tuple[str, str | None], list[ClaudeTodoSnapshotEvidence]] = {} + for row in rows: + source_path = str(row["source_path"]) + rule = artifact_rule_for_path(Provider.CLAUDE_CODE, source_path) + if rule is None or rule.kind != "todo_snapshot": + continue + payload = blob_store.read_all(str(row["blob_hash"])) + snapshot = parse_claude_todo_artifact(source_path, payload) + if snapshot is None or snapshot.session_id is None: + continue + key = (snapshot.session_id, snapshot.agent_id) + by_session.setdefault(key, []).append( + ClaudeTodoSnapshotEvidence( + snapshot=snapshot, + acquired_at_ms=int(row["acquired_at_ms"]), + evidence_ref=ObjectRef(kind="artifact", object_id=f"raw:{row['raw_id']}"), + ) + ) + + return tuple( + ClaudeTodoPlanState(session_id=session_id, agent_id=agent_id, snapshots=tuple(evidences)) + for (session_id, agent_id), evidences in sorted(by_session.items()) + ) + + +__all__ = [ + "ClaudeTodoPlanState", + "ClaudeTodoSnapshotEvidence", + "load_claude_todo_plan_states", +] diff --git a/polylogue/insights/measurement/registered_metrics.py b/polylogue/insights/measurement/registered_metrics.py index fe6b5db527..def4256966 100644 --- a/polylogue/insights/measurement/registered_metrics.py +++ b/polylogue/insights/measurement/registered_metrics.py @@ -9,10 +9,12 @@ scope here (see ``polylogue-rxdo.9`` epic-expansion guard). This module is the bounded, honest slice available without it: a real, -process-wide registry populated with one concrete metric definition for an -existing, already-computed construct (session cost, ``cost/pricing.py``/ -``cost/outlook.py``), reachable through the MCP ``get`` tool -(``get(ref="metric:session_cost_usd")``, see ``polylogue/mcp/ +process-wide registry populated with concrete metric definitions for +existing, already-computed constructs -- session cost (``cost/pricing.py``/ +``cost/outlook.py``) and, since polylogue-t0p, plan-completion rate +(``insights/claude_todo_projection.py``) -- reachable through the MCP ``get`` +tool (``get(ref="metric:session_cost_usd")`` / +``get(ref="metric:plan_completion_rate")``, see ``polylogue/mcp/ server_cutover.py``). This proves the identity/registry machinery resolves through a real production surface -- it does NOT execute the metric (no composition/aggregation engine exists yet; that is 9l5.7's job) or attach a @@ -45,6 +47,30 @@ output_schema="usd:float", ) +#: Plan-vs-outcome measure (polylogue-t0p): fraction of a session's Claude +#: Code plan (``~/.claude/todos/*.json``, ``insights/claude_todo_projection.py``) +#: marked ``completed`` at its LATEST observed snapshot. ``structural``, not +#: ``heuristic``: the status string is a provider-reported field Claude Code +#: itself writes, never inferred from transcript prose. ``census`` enumeration +#: (not a sample) -- every admitted snapshot for a session is used, not a +#: subset -- so no sampling interval attaches per the registry's own +#: census-vs-sample doctrine (9l5.7 bead notes, ``MeasureSpec`` design). +#: Denominator hazard: a session with an empty plan (``item_count == 0``) has +#: ``completion_rate is None`` (``ClaudeTodoSnapshot.completion_rate``), not +#: zero -- "nothing planned" must never render as "nothing done". +PLAN_COMPLETION_RATE_METRIC = MetricDefinition( + construct="fraction of a session's latest Claude Code TODO plan marked completed", + unit="ratio", + unit_source="claude_todo_plan_states", + aggregation="mean", + grain="logical", + required_enumeration="census", + measurement_authority=("structural",), + provenance_mixing="single-authority", + null_policy="exclude", + output_schema="ratio:float|null", +) + #: Process-wide default registry. A module-level singleton is the correct #: shape for an in-process content-addressed identity registry (mirrors #: ``polylogue.insights.registry.INSIGHT_REGISTRY``) -- registration is @@ -52,6 +78,7 @@ #: -registers or drifts. DEFAULT_METRIC_REGISTRY = MetricRegistry() DEFAULT_METRIC_REGISTRY.register(SESSION_COST_USD_METRIC, name="session_cost_usd") +DEFAULT_METRIC_REGISTRY.register(PLAN_COMPLETION_RATE_METRIC, name="plan_completion_rate") -__all__ = ["DEFAULT_METRIC_REGISTRY", "SESSION_COST_USD_METRIC"] +__all__ = ["DEFAULT_METRIC_REGISTRY", "PLAN_COMPLETION_RATE_METRIC", "SESSION_COST_USD_METRIC"] diff --git a/polylogue/paths/__init__.py b/polylogue/paths/__init__.py index d1c4591fb9..e1223d9f73 100644 --- a/polylogue/paths/__init__.py +++ b/polylogue/paths/__init__.py @@ -21,6 +21,7 @@ cache_home, cache_root, claude_code_path, + claude_code_todos_path, codex_path, config_home, config_root, @@ -54,6 +55,7 @@ "cache_home", "cache_root", "claude_code_path", + "claude_code_todos_path", "codex_path", "config_home", "config_root", diff --git a/polylogue/paths/_roots.py b/polylogue/paths/_roots.py index 2b526625d7..f53e967e76 100644 --- a/polylogue/paths/_roots.py +++ b/polylogue/paths/_roots.py @@ -195,6 +195,18 @@ def claude_code_path() -> Path: return Path.home() / ".claude" / "projects" +def claude_code_todos_path() -> Path: + """Claude Code live plan-snapshot directory (polylogue-t0p). + + Sibling of ``claude_code_path()``, not nested under it: Claude Code + writes one ``[-agent-].json`` file per session + (rewritten wholesale on every ``TodoWrite`` call, never appended), and + prunes old entries on its own schedule -- unread state here is + eventually lost, not merely delayed. + """ + return Path.home() / ".claude" / "todos" + + def codex_path() -> Path: """Codex sessions directory.""" return Path.home() / ".codex" / "sessions" diff --git a/polylogue/sources/live/watcher.py b/polylogue/sources/live/watcher.py index b21879e109..3dbfc3a29f 100644 --- a/polylogue/sources/live/watcher.py +++ b/polylogue/sources/live/watcher.py @@ -1389,6 +1389,7 @@ def default_sources(*, hermes_root: Path | None = None, beads_roots: tuple[Path, archive_root, browser_capture_spool_root, claude_code_path, + claude_code_todos_path, codex_path, gemini_cli_path, hermes_sessions_path, @@ -1414,6 +1415,17 @@ def default_sources(*, hermes_root: Path | None = None, beads_roots: tuple[Path, root=claude_code_path(), suffixes=artifact_suffixes_for_provider(Provider.CLAUDE_CODE, defaults=(".jsonl",)), ), + # polylogue-t0p: Claude Code's live plan-snapshot directory + # (~/.claude/todos/) is a sibling of claude_code_path(), not nested + # under it -- a second, narrower WatchSource rooted there, same + # precedent as "codex-state" below, so the main claude-code root + # doesn't have to widen its own suffix/path assumptions to reach a + # completely different directory tree. + WatchSource( + name="claude-code-todos", + root=claude_code_todos_path(), + suffixes=(".json",), + ), WatchSource(name="codex", root=codex_path()), # polylogue-0jf4: Codex also keeps live SQLite state (thread titles, # spawn topology, goals, memories) as siblings of the sessions/ diff --git a/polylogue/sources/origin_specs.py b/polylogue/sources/origin_specs.py index 3b0f4ad2e6..8acc7ed5b3 100644 --- a/polylogue/sources/origin_specs.py +++ b/polylogue/sources/origin_specs.py @@ -546,6 +546,26 @@ def _claude_code_spec() -> OriginSpec: ), path_suffixes=(".jsonl", ".ndjson"), ), + OriginArtifactRule( + kind="todo_snapshot", + path_pattern=r"(?:^|/)todos/[^/]+\.json$", + parse_policy="fact", + parser_path="polylogue/sources/parsers/claude/todos.py:parse_claude_todo_artifact", + coverage_role="plan_snapshot", + fidelity_note=( + "~/.claude/todos/[-agent-].json is Claude Code's live plan " + "state (polylogue-t0p): a session's own TodoWrite invocations overwrite the same " + "path wholesale, never append. Preserved: every task's content/status/priority/id " + "and the agent's own list ORDER (a real priority signal, not alphabetical/insertion " + "order) for every snapshot the watcher actually observed on disk. Lost: Claude Code " + "prunes this directory on its own schedule and each write is a full overwrite, so " + "intermediate transitions between two watcher-observed snapshots (e.g. pending -> " + "in_progress -> completed within one polling gap) are unrecoverable once overwritten " + "or pruned -- there is no in-file timestamp; observation time is acquisition-time, " + "not plan-mutation time." + ), + path_suffixes=(".json",), + ), ), assembly_spec_path="polylogue/sources/assembly_claude_code.py:ClaudeCodeAssemblySpec", display_description="Claude Code local sessions (lab: Anthropic)", diff --git a/polylogue/sources/parsers/claude/__init__.py b/polylogue/sources/parsers/claude/__init__.py index 3f481e9052..bb3c774c3a 100644 --- a/polylogue/sources/parsers/claude/__init__.py +++ b/polylogue/sources/parsers/claude/__init__.py @@ -28,6 +28,7 @@ parse_sessions_index, ) from .orchestration import parse_claude_orchestration_artifact +from .todos import parse_claude_todo_artifact def looks_like_ai(payload: object) -> bool: @@ -74,5 +75,6 @@ def parse_memories(payload: Mapping[str, object], fallback_id: str) -> ParsedSes "reconcile_code_session_chunks", "parse_sessions_index", "parse_claude_orchestration_artifact", + "parse_claude_todo_artifact", "parse_stream", ] diff --git a/polylogue/sources/parsers/claude/todos.py b/polylogue/sources/parsers/claude/todos.py new file mode 100644 index 0000000000..8b1e23c00b --- /dev/null +++ b/polylogue/sources/parsers/claude/todos.py @@ -0,0 +1,151 @@ +"""Claude Code ``~/.claude/todos/*.json`` plan-snapshot parsing (polylogue-t0p). + +Claude Code persists the agent's current TODO/plan list to +``~/.claude/todos/[-agent-].json`` -- a directory sibling +to ``~/.claude/projects/`` (the session transcript root), not nested under it +-- and overwrites the same path wholesale on every ``TodoWrite`` tool call +(never appended). Each file is a bare JSON array of task objects +(``content``/``status``/``priority``/``id``), unlike every other Claude Code +sidecar this package parses (those are JSONL streams or single JSON *objects* +under ``workflows/``/``jobs/``). See the ``todo_snapshot`` ``OriginArtifactRule`` +in ``polylogue.sources.origin_specs`` for the admission/fidelity declaration +this module implements. +""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass +from pathlib import Path + +#: Claude Code names todo files by the owning session's UUID, optionally +#: suffixed with the delegated subagent's own UUID (``agent-*.jsonl`` +#: transcripts follow the identical ``-agent-`` convention elsewhere in +#: this package, e.g. ``orchestration.py:_agent_id_from_path``). +_FILENAME_RE = re.compile( + r"^(?P[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})" + r"(?:-agent-(?P[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}))?$" +) + + +def session_and_agent_id_from_filename(source_path: str | Path) -> tuple[str | None, str | None]: + """Recover ``(session_id, agent_id)`` from a todo snapshot's filename stem. + + Returns ``(None, None)`` when the stem doesn't match Claude Code's own + naming convention (e.g. a hand-placed test fixture with a non-UUID name); + callers should treat that as "cannot link this snapshot to a session", + not fabricate an identity. + """ + stem = Path(source_path).stem + match = _FILENAME_RE.match(stem) + if not match: + return None, None + return match.group("session_id"), match.group("agent_id") + + +@dataclass(frozen=True, slots=True) +class ClaudeTodoItem: + """One task in a Claude Code plan snapshot, in the agent's own list order.""" + + position: int + content: str + status: str + item_id: str | None + priority: str | None + + +@dataclass(frozen=True, slots=True) +class ClaudeTodoSnapshot: + """One parsed ``todos/*.json`` plan snapshot.""" + + source_path: str + session_id: str | None + agent_id: str | None + items: tuple[ClaudeTodoItem, ...] + parse_error: str | None = None + + @property + def item_count(self) -> int: + return len(self.items) + + @property + def completed_count(self) -> int: + return sum(1 for item in self.items if item.status == "completed") + + @property + def completion_rate(self) -> float | None: + """Fraction of items with ``status == "completed"``, or ``None`` for an empty plan. + + ``None`` (not ``0.0``) for zero items: an empty plan has no + denominator to divide by, and reporting a bare ``0.0`` would read as + "nothing done" rather than "nothing planned" -- the honest-analytics + null-policy convention this codebase applies elsewhere (see + ``insights/measurement/metric.py``'s ``NullPolicy``). + """ + if not self.items: + return None + return self.completed_count / len(self.items) + + +def parse_claude_todo_artifact(source_path: str, payload: bytes | str | object) -> ClaudeTodoSnapshot | None: + """Parse one ``todo_snapshot`` artifact's raw payload. + + Returns ``None`` only when ``source_path`` doesn't even look like a + todos-directory file (defensive: callers should already have matched the + ``OriginArtifactRule`` before reaching here). A malformed or + unexpectedly-shaped payload still returns a ``ClaudeTodoSnapshot`` with + ``parse_error`` set and ``items=()`` -- the session linkage recovered from + the filename remains available even when the content itself didn't parse. + """ + session_id, agent_id = session_and_agent_id_from_filename(source_path) + try: + loaded = _decode(payload) + except (ValueError, UnicodeDecodeError) as exc: + return ClaudeTodoSnapshot(source_path, session_id, agent_id, (), parse_error=f"{type(exc).__name__}: {exc}") + + if not isinstance(loaded, list): + return ClaudeTodoSnapshot( + source_path, + session_id, + agent_id, + (), + parse_error=f"expected a JSON array, got {type(loaded).__name__}", + ) + + items: list[ClaudeTodoItem] = [] + for position, entry in enumerate(loaded): + if not isinstance(entry, dict): + continue + content = entry.get("content") + status = entry.get("status") + if not isinstance(content, str) or not isinstance(status, str): + continue + item_id = entry.get("id") + priority = entry.get("priority") + items.append( + ClaudeTodoItem( + position=position, + content=content, + status=status, + item_id=item_id if isinstance(item_id, str) else None, + priority=priority if isinstance(priority, str) else None, + ) + ) + return ClaudeTodoSnapshot(source_path, session_id, agent_id, tuple(items)) + + +def _decode(payload: bytes | str | object) -> object: + if isinstance(payload, bytes): + payload = payload.decode("utf-8") + if isinstance(payload, str): + return json.loads(payload) + return payload + + +__all__ = [ + "ClaudeTodoItem", + "ClaudeTodoSnapshot", + "parse_claude_todo_artifact", + "session_and_agent_id_from_filename", +] diff --git a/tests/unit/core/test_config.py b/tests/unit/core/test_config.py index 450797142c..b5b8054866 100644 --- a/tests/unit/core/test_config.py +++ b/tests/unit/core/test_config.py @@ -336,6 +336,29 @@ def test_get_sources_skips_drive_source_without_cache_or_credentials( sources = get_sources(resolve_runtime_config()) assert [source.name for source in sources] == [] + def test_claude_code_todos_source_discovered_when_directory_exists( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ) -> None: + """polylogue-t0p: ~/.claude/todos/ is a distinct discovered source, not folded into claude-code.""" + home = tmp_path / "home" + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setenv("XDG_DATA_HOME", str(tmp_path / "data")) + monkeypatch.setenv("XDG_STATE_HOME", str(tmp_path / "state")) + monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path / "config")) + todos_dir = home / ".claude" / "todos" + todos_dir.mkdir(parents=True) + + from polylogue.config import get_sources, resolve_runtime_config + + runtime = resolve_runtime_config() + assert runtime.source_paths.claude_code_todos == todos_dir + + names = {source.name for source in get_sources(runtime)} + assert "claude-code-todos" in names + assert "claude-code" not in names # sibling ~/.claude/projects/ was never created + def test_get_sources_includes_drive_source_when_credentials_exist( self, monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/insights/test_claude_todo_projection.py b/tests/unit/insights/test_claude_todo_projection.py new file mode 100644 index 0000000000..fb61e256ad --- /dev/null +++ b/tests/unit/insights/test_claude_todo_projection.py @@ -0,0 +1,134 @@ +"""End-to-end ``todo_snapshot`` admission + session-linked read model (polylogue-t0p). + +Production dependencies exercised: configured source walk + OriginSpec path +classification (``_admit_non_session_origin_artifacts``), raw source/blob +persistence, and the ``claude_todo_projection`` read model built on top. + +Anti-vacuity mutation: removing the ``todo_snapshot`` ``OriginArtifactRule`` +(or its ``.json`` suffix) makes ``parse_sources_archive`` admit zero raw +rows for the todos fixture below, so ``load_claude_todo_plan_states`` would +return an empty tuple instead of two sessions. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from polylogue.config import Source +from polylogue.core.enums import Provider +from polylogue.insights.claude_todo_projection import load_claude_todo_plan_states +from polylogue.insights.measurement.registered_metrics import ( + DEFAULT_METRIC_REGISTRY, + PLAN_COMPLETION_RATE_METRIC, +) +from polylogue.pipeline.services.archive_ingest import parse_sources_archive + +_SESSION_A = "005aeeae-bd21-4a4c-b928-e9ee5f6b8394" +_SESSION_B = "0092150d-6b81-43f9-85e2-ceeb2d1c8773" +_AGENT_B = "00afa7b1-f546-425e-95dc-d1a7093b09d2" + + +def _write_todos_fixture(root: Path) -> Path: + todos = root / "todos" + todos.mkdir(parents=True) + (todos / f"{_SESSION_A}.json").write_text( + json.dumps( + [ + {"content": "bootstrap the repo", "status": "completed", "priority": "high", "id": "a1"}, + {"content": "write docs", "status": "in_progress", "priority": "medium", "id": "a2"}, + ] + ), + encoding="utf-8", + ) + (todos / f"{_SESSION_B}-agent-{_AGENT_B}.json").write_text( + json.dumps( + [ + {"content": "subagent task", "status": "pending", "priority": "low", "id": "b1"}, + ] + ), + encoding="utf-8", + ) + return todos + + +@pytest.mark.asyncio +async def test_todos_admitted_and_materialized_into_session_linked_plan_states( + workspace_env: dict[str, Path], +) -> None: + archive_root = workspace_env["archive_root"] + todos_root = _write_todos_fixture(workspace_env["data_root"] / ".claude") + + result = await parse_sources_archive( + archive_root, + [Source(name="claude-code-todos", path=todos_root)], + ) + assert result.parse_failures == 0 + + plan_states = load_claude_todo_plan_states(archive_root) + by_session = {state.session_id: state for state in plan_states} + + assert set(by_session) == {_SESSION_A, _SESSION_B} + + plan_a = by_session[_SESSION_A] + assert plan_a.agent_id is None + assert len(plan_a.snapshots) == 1 + assert plan_a.latest.snapshot.item_count == 2 + assert plan_a.plan_completion_rate == pytest.approx(0.5) + assert plan_a.status_transitions() == {"a1": ("completed",), "a2": ("in_progress",)} + + plan_b = by_session[_SESSION_B] + assert plan_b.agent_id == _AGENT_B + assert plan_b.plan_completion_rate == pytest.approx(0.0) + + +@pytest.mark.asyncio +async def test_second_observed_snapshot_extends_status_transition_history( + workspace_env: dict[str, Path], +) -> None: + """A later snapshot for the SAME session is a distinct raw revision, not an overwrite in place.""" + archive_root = workspace_env["archive_root"] + todos_root = _write_todos_fixture(workspace_env["data_root"] / ".claude") + + await parse_sources_archive(archive_root, [Source(name="claude-code-todos", path=todos_root)]) + + # Claude Code overwrites the same path on the next TodoWrite call; the + # watcher observes this as a new file revision with an advanced status. + (todos_root / f"{_SESSION_A}.json").write_text( + json.dumps( + [ + {"content": "bootstrap the repo", "status": "completed", "priority": "high", "id": "a1"}, + {"content": "write docs", "status": "completed", "priority": "medium", "id": "a2"}, + ] + ), + encoding="utf-8", + ) + await parse_sources_archive(archive_root, [Source(name="claude-code-todos", path=todos_root)]) + + plan_states = load_claude_todo_plan_states(archive_root) + plan_a = next(state for state in plan_states if state.session_id == _SESSION_A) + + assert len(plan_a.snapshots) == 2 + assert plan_a.status_transitions()["a2"] == ("in_progress", "completed") + assert plan_a.plan_completion_rate == pytest.approx(1.0) + + +def test_plan_completion_rate_metric_is_registered() -> None: + """Plan-vs-outcome measure registration (polylogue-t0p AC), rxdo.9.1 identity layer. + + Mirrors the already-shipped ``session_cost_usd`` slice: proves the metric + identity/registry machinery resolves this construct through the same + process-wide registry, reachable via MCP ``get(ref="metric:plan_completion_rate")``. + """ + resolved = DEFAULT_METRIC_REGISTRY.resolve("plan_completion_rate") + assert resolved is not None + assert resolved.ref == PLAN_COMPLETION_RATE_METRIC.ref + assert resolved.measurement_authority == ("structural",) + assert resolved.null_policy == "exclude" + + +def test_todos_source_name_resolves_to_claude_code_provider() -> None: + """Admission gating (archive_ingest._admit_non_session_origin_artifacts) keys off this.""" + assert Provider.from_string("claude-code-todos") is Provider.CLAUDE_CODE diff --git a/tests/unit/sources/test_live_watcher.py b/tests/unit/sources/test_live_watcher.py index 9f3ff0a245..923763bee9 100644 --- a/tests/unit/sources/test_live_watcher.py +++ b/tests/unit/sources/test_live_watcher.py @@ -2813,6 +2813,19 @@ def test_claude_default_source_projects_originspec_suffixes() -> None: assert claude.accepts(claude.root / "project" / "session.jsonl") +def test_claude_todos_default_source_watches_its_own_sibling_root() -> None: + """polylogue-t0p: ~/.claude/todos/ is watched separately from ~/.claude/projects/.""" + from polylogue.sources.live.watcher import default_sources + + todos = next(source for source in default_sources() if source.name == "claude-code-todos") + claude = next(source for source in default_sources() if source.name == "claude-code") + + assert todos.root != claude.root + assert todos.root.name == "todos" + assert todos.suffixes == (".json",) + assert todos.accepts(todos.root / "005aeeae-bd21-4a4c-b928-e9ee5f6b8394.json") + + def test_browser_capture_spool_is_default_json_source( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, diff --git a/tests/unit/sources/test_origin_specs.py b/tests/unit/sources/test_origin_specs.py index 037c1c0de3..7055b10af7 100644 --- a/tests/unit/sources/test_origin_specs.py +++ b/tests/unit/sources/test_origin_specs.py @@ -46,6 +46,7 @@ def test_origin_specs_cover_the_public_enum_and_admission_lifecycles() -> None: "agent_sidecar_meta", "adopt_manifest", "coordinator_session_stream", + "todo_snapshot", } assert artifact_suffixes_for_provider(Provider.CLAUDE_CODE) == (".json", ".jsonl", ".ndjson") assert claude.detector_tightness == 60 diff --git a/tests/unit/sources/test_parsers_claude_todos.py b/tests/unit/sources/test_parsers_claude_todos.py new file mode 100644 index 0000000000..10622f4e27 --- /dev/null +++ b/tests/unit/sources/test_parsers_claude_todos.py @@ -0,0 +1,121 @@ +"""Tests for ``~/.claude/todos/*.json`` plan-snapshot parsing (polylogue-t0p).""" + +from __future__ import annotations + +import json + +from polylogue.core.enums import Provider +from polylogue.sources.origin_specs import artifact_rule_for_path +from polylogue.sources.parsers.claude.todos import ( + parse_claude_todo_artifact, + session_and_agent_id_from_filename, +) + +_SESSION_ID = "005aeeae-bd21-4a4c-b928-e9ee5f6b8394" +_AGENT_ID = "0092150d-6b81-43f9-85e2-ceeb2d1c8773" + + +def test_session_id_recovered_from_plain_filename() -> None: + session_id, agent_id = session_and_agent_id_from_filename(f"/home/x/.claude/todos/{_SESSION_ID}.json") + assert session_id == _SESSION_ID + assert agent_id is None + + +def test_session_and_agent_id_recovered_from_subagent_filename() -> None: + session_id, agent_id = session_and_agent_id_from_filename( + f"/home/x/.claude/todos/{_SESSION_ID}-agent-{_AGENT_ID}.json" + ) + assert session_id == _SESSION_ID + assert agent_id == _AGENT_ID + + +def test_non_uuid_filename_returns_no_identity() -> None: + assert session_and_agent_id_from_filename("/home/x/.claude/todos/not-a-uuid.json") == (None, None) + + +def test_parse_claude_todo_artifact_preserves_order_and_fields() -> None: + source_path = f"/home/x/.claude/todos/{_SESSION_ID}.json" + payload = json.dumps( + [ + {"content": "Write bootstrap.sh", "status": "completed", "priority": "high", "id": "t1"}, + {"content": "Improve README", "status": "in_progress", "priority": "medium", "id": "t2"}, + {"content": "Run adversarial tests", "status": "pending", "priority": "high", "id": "t3"}, + ] + ).encode("utf-8") + + snapshot = parse_claude_todo_artifact(source_path, payload) + + assert snapshot is not None + assert snapshot.session_id == _SESSION_ID + assert snapshot.agent_id is None + assert snapshot.parse_error is None + assert [item.content for item in snapshot.items] == [ + "Write bootstrap.sh", + "Improve README", + "Run adversarial tests", + ] + assert [item.position for item in snapshot.items] == [0, 1, 2] + assert snapshot.item_count == 3 + assert snapshot.completed_count == 1 + assert snapshot.completion_rate == 1 / 3 + + +def test_parse_claude_todo_artifact_empty_plan_has_no_completion_rate() -> None: + source_path = f"/home/x/.claude/todos/{_SESSION_ID}.json" + snapshot = parse_claude_todo_artifact(source_path, b"[]") + + assert snapshot is not None + assert snapshot.items == () + assert snapshot.item_count == 0 + assert snapshot.completion_rate is None + + +def test_parse_claude_todo_artifact_skips_malformed_entries() -> None: + source_path = f"/home/x/.claude/todos/{_SESSION_ID}.json" + payload = json.dumps( + [ + {"content": "valid", "status": "pending", "id": "t1"}, + {"content": "missing status"}, + "not-a-dict", + {"status": "completed"}, + ] + ).encode("utf-8") + + snapshot = parse_claude_todo_artifact(source_path, payload) + + assert snapshot is not None + assert [item.content for item in snapshot.items] == ["valid"] + + +def test_parse_claude_todo_artifact_reports_error_for_non_array_json() -> None: + source_path = f"/home/x/.claude/todos/{_SESSION_ID}.json" + snapshot = parse_claude_todo_artifact(source_path, b'{"not": "an array"}') + + assert snapshot is not None + assert snapshot.items == () + assert snapshot.parse_error is not None + assert snapshot.session_id == _SESSION_ID + + +def test_parse_claude_todo_artifact_reports_error_for_invalid_json() -> None: + source_path = f"/home/x/.claude/todos/{_SESSION_ID}.json" + snapshot = parse_claude_todo_artifact(source_path, b"{not json") + + assert snapshot is not None + assert snapshot.items == () + assert snapshot.parse_error is not None + + +def test_origin_spec_admits_todos_directory_artifact_as_fact_tier() -> None: + """Production dependency: OriginSpec must classify a real todos path as a fact artifact. + + Anti-vacuity mutation: removing the ``todo_snapshot`` OriginArtifactRule + (or narrowing its ``path_pattern`` to exclude ``todos/*.json``) makes this + assertion fail -- ``artifact_rule_for_path`` would return ``None``. + """ + rule = artifact_rule_for_path(Provider.CLAUDE_CODE, f"/home/x/.claude/todos/{_SESSION_ID}.json") + + assert rule is not None + assert rule.kind == "todo_snapshot" + assert rule.parse_policy == "fact" + assert rule.parser_path == "polylogue/sources/parsers/claude/todos.py:parse_claude_todo_artifact"