From 94fca808521601e73d834f229c9be4fa773de6df Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Thu, 10 Sep 2026 13:55:55 -0700 Subject: [PATCH 01/88] Add Copilot capture contracts and identities --- src/thirdeye/platforms/copilot/__init__.py | 17 ++ src/thirdeye/platforms/copilot/constants.py | 39 +++++ src/thirdeye/platforms/copilot/identity.py | 98 +++++++++++ src/thirdeye/platforms/copilot/types.py | 67 ++++++++ tests/fixtures/copilot/cli-1.0.83/README.md | 51 ++++++ .../cli-1.0.83/assistant-usage-events.json | 158 ++++++++++++++++++ .../fixtures/copilot/cli-1.0.83/events.jsonl | 76 +++++++++ tests/fixtures/copilot/cli-1.0.83/hooks.jsonl | 20 +++ tests/fixtures/copilot/cli-1.0.83/usage.json | 134 +++++++++++++++ tests/fixtures/copilot/v1-cases/README.md | 40 +++++ .../v1-cases/database-row-revisions.json | 5 + .../v1-cases/distinct-hook-observations.json | 4 + .../copilot/v1-cases/missing-event-id.jsonl | 1 + .../copilot/v1-cases/source-batch.json | 19 +++ .../copilot/v1-cases/source-slice.json | 17 ++ .../copilot/v1-cases/trailing-json.jsonl | 2 + .../copilot/v1-cases/trailing-utf8.hex | 1 + .../v1-cases/unknown-event-fields.jsonl | 1 + 18 files changed, 750 insertions(+) create mode 100644 src/thirdeye/platforms/copilot/__init__.py create mode 100644 src/thirdeye/platforms/copilot/constants.py create mode 100644 src/thirdeye/platforms/copilot/identity.py create mode 100644 src/thirdeye/platforms/copilot/types.py create mode 100644 tests/fixtures/copilot/cli-1.0.83/README.md create mode 100644 tests/fixtures/copilot/cli-1.0.83/assistant-usage-events.json create mode 100644 tests/fixtures/copilot/cli-1.0.83/events.jsonl create mode 100644 tests/fixtures/copilot/cli-1.0.83/hooks.jsonl create mode 100644 tests/fixtures/copilot/cli-1.0.83/usage.json create mode 100644 tests/fixtures/copilot/v1-cases/README.md create mode 100644 tests/fixtures/copilot/v1-cases/database-row-revisions.json create mode 100644 tests/fixtures/copilot/v1-cases/distinct-hook-observations.json create mode 100644 tests/fixtures/copilot/v1-cases/missing-event-id.jsonl create mode 100644 tests/fixtures/copilot/v1-cases/source-batch.json create mode 100644 tests/fixtures/copilot/v1-cases/source-slice.json create mode 100644 tests/fixtures/copilot/v1-cases/trailing-json.jsonl create mode 100644 tests/fixtures/copilot/v1-cases/trailing-utf8.hex create mode 100644 tests/fixtures/copilot/v1-cases/unknown-event-fields.jsonl diff --git a/src/thirdeye/platforms/copilot/__init__.py b/src/thirdeye/platforms/copilot/__init__.py new file mode 100644 index 0000000..9dbe5b4 --- /dev/null +++ b/src/thirdeye/platforms/copilot/__init__.py @@ -0,0 +1,17 @@ +"""Read-only capture support for GitHub Copilot CLI recordings.""" + +from __future__ import annotations + +from .identity import resolve_sources, stored_session_id, validate_native_id +from .types import SourceBatch, SourcePaths, SourceRecord, SourceSlice, SyncResult + +__all__ = [ + "SourceBatch", + "SourcePaths", + "SourceRecord", + "SourceSlice", + "SyncResult", + "resolve_sources", + "stored_session_id", + "validate_native_id", +] diff --git a/src/thirdeye/platforms/copilot/constants.py b/src/thirdeye/platforms/copilot/constants.py new file mode 100644 index 0000000..b3eb4f8 --- /dev/null +++ b/src/thirdeye/platforms/copilot/constants.py @@ -0,0 +1,39 @@ +"""Stable Copilot platform identifiers and CLI hook vocabulary. + +Paths are intentionally absent from this module. Source paths are resolved at +invocation time by :func:`thirdeye.platforms.copilot.identity.resolve_sources`. +""" + +from __future__ import annotations + +PLATFORM_NAME = "copilot" +DISPLAY_NAME = "GitHub Copilot CLI" +SOURCE_SCHEMA_VERSION = 1 +# Compatibility-friendly names for consumers that refer to the record/event +# envelope rather than the source format. All three are the same V1 contract. +SCHEMA_VERSION = SOURCE_SCHEMA_VERSION +SOURCE_RECORD_SCHEMA_VERSION = SOURCE_SCHEMA_VERSION +EVENT_ENVELOPE_VERSION = SOURCE_SCHEMA_VERSION + +COPILOT_HOME_ENV = "COPILOT_HOME" +HOOKS_DIRECTORY_NAME = "hooks" +OWNED_HOOK_FILENAME = "thirdeye.json" + +# Copilot CLI 1.0.83's documented hook names are camelCase. The aliases are +# intentionally data-only: installation and hook runtime decide which events +# they support without importing source-resolution code. +CLI_HOOK_EVENT_ALIASES: dict[str, str] = { + "sessionStart": "session_start", + "userPromptSubmitted": "user_prompt_submitted", + "preToolUse": "pre_tool_use", + "postToolUse": "post_tool_use", + "agentStop": "agent_stop", + "subagentStart": "subagent_start", + "subagentStop": "subagent_stop", + "sessionEnd": "session_end", + "errorOccurred": "error_occurred", +} + +CLI_HOOK_EVENTS = tuple(CLI_HOOK_EVENT_ALIASES) +HOOK_EVENT_ALIASES = CLI_HOOK_EVENT_ALIASES +HOOK_EVENTS = CLI_HOOK_EVENTS diff --git a/src/thirdeye/platforms/copilot/identity.py b/src/thirdeye/platforms/copilot/identity.py new file mode 100644 index 0000000..c94873a --- /dev/null +++ b/src/thirdeye/platforms/copilot/identity.py @@ -0,0 +1,98 @@ +"""Pure source-home and native-session identity helpers.""" + +from __future__ import annotations + +import hashlib +import os +from pathlib import Path + +from .constants import COPILOT_HOME_ENV +from .types import SourcePaths + + +def _canonical_path(path: Path) -> Path: + """Return a stable, absolute path without requiring the path to exist.""" + + return path.expanduser().resolve(strict=False) + + +def _normalized_path(path: Path) -> str: + """Normalize the canonical path for source-home identity on this platform.""" + + return os.path.normcase(os.fspath(_canonical_path(path))) + + +def _source_key(home: Path) -> str: + return hashlib.sha256(_normalized_path(home).encode("utf-8")).hexdigest() + + +def _is_within(path: Path, root: Path) -> bool: + try: + path.relative_to(root) + except ValueError: + return False + return True + + +def _validate_paths(paths: SourcePaths) -> Path: + """Reject forged identities or recording paths outside their source home.""" + + home = _canonical_path(Path(paths["home"])) + if paths["source_key"] != _source_key(home): + raise ValueError("source_key does not match the canonical source home") + + for field in ("session_root", "database"): + candidate = _canonical_path(Path(paths[field])) + if not _is_within(candidate, home): + raise ValueError(f"{field} escapes the selected Copilot home") + return home + + +def resolve_sources(source_home: Path | None = None) -> SourcePaths: + """Resolve one Copilot home at invocation time. + + Explicit input wins over ``COPILOT_HOME``; the default is + ``Path.home() / '.copilot'``. The returned paths are canonical strings so + aliases resolving to the same home share a source identity. + """ + + requested = source_home + if requested is None: + configured = os.environ.get(COPILOT_HOME_ENV) + requested = Path(configured) if configured else Path.home() / ".copilot" + + home = _canonical_path(requested) + paths: SourcePaths = { + "home": os.fspath(home), + "source_key": _source_key(home), + "session_root": os.fspath(home / "session-state"), + "database": os.fspath(home / "session-store.db"), + } + _validate_paths(paths) + return paths + + +def validate_native_id(native_id: str) -> None: + """Ensure a native session ID can never select a path outside its home.""" + + if not isinstance(native_id, str) or not native_id or native_id.strip() != native_id: + raise ValueError("native session ID must be a non-empty, trimmed string") + if native_id in {".", ".."}: + raise ValueError("native session ID must not be a traversal segment") + if any(character in native_id for character in ("/", "\\", "\x00", ":")): + raise ValueError("native session ID contains a path separator or invalid path character") + if any(ord(character) < 32 for character in native_id): + raise ValueError("native session ID contains a control character") + + +def stored_session_id(paths: SourcePaths, native_id: str) -> str: + """Return the stable thirdeye ID for a native ID within one source home. + + The full source key is validated before using its display prefix. This + makes a prefix collision detectable instead of merging records from two + source homes. + """ + + _validate_paths(paths) + validate_native_id(native_id) + return f"copilot-{paths['source_key'][:16]}-{native_id}" diff --git a/src/thirdeye/platforms/copilot/types.py b/src/thirdeye/platforms/copilot/types.py new file mode 100644 index 0000000..ba0f5ee --- /dev/null +++ b/src/thirdeye/platforms/copilot/types.py @@ -0,0 +1,67 @@ +"""Versioned, lossless source contracts shared by Copilot V1 and V2. + +These TypedDicts deliberately describe raw source evidence. They do not imply +turn reconstruction, usage accounting, or any semantic correlation. +""" + +from __future__ import annotations + +from typing import Any, TypedDict + +from .constants import SOURCE_SCHEMA_VERSION + +# The value placed beside every archived SourceRecord in a thirdeye event data +# envelope. It is deliberately separate from a Copilot CLI version. +SCHEMA_VERSION = SOURCE_SCHEMA_VERSION + + +class SourcePaths(TypedDict): + """Canonical source-home identity and its Copilot recording locations.""" + + home: str + source_key: str + session_root: str + database: str + + +class SourceRecord(TypedDict): + """One immutable observation from a Copilot source domain.""" + + source_id: str + source_kind: str # transcript | database | hook | metadata + native_session_id: str + ts: str | None # Source time when valid; never invented. + observed_at: str + payload: dict[str, Any] + locator: dict[str, Any] + + +class SourceBatch(TypedDict): + """The composed capture boundary consumed by the durable archive.""" + + source_key: str + native_session_id: str + cwd: str | None + records: list[SourceRecord] + next_cursor: dict[str, Any] + diagnostics: list[dict[str, Any]] + + +class SyncResult(TypedDict): + """Counts returned by a capture operation.""" + + sessions: int + records_written: int + duplicate_records: int + pending: int + errors: int + + +class SourceSlice(TypedDict): + """A bounded read from one source; composed into a :class:`SourceBatch`.""" + + records: list[SourceRecord] + next_cursor: dict[str, Any] + diagnostics: list[dict[str, Any]] + cwd: str | None + exhausted: bool diff --git a/tests/fixtures/copilot/cli-1.0.83/README.md b/tests/fixtures/copilot/cli-1.0.83/README.md new file mode 100644 index 0000000..5a2fa42 --- /dev/null +++ b/tests/fixtures/copilot/cli-1.0.83/README.md @@ -0,0 +1,51 @@ +# Copilot CLI capture probe + +Captured 2026-09-10 on macOS using Copilot CLI 1.0.83, with automatic model selection (resolved to gpt-5.6-luna). These are observed fixtures for adapter design, not an implemented adapter or a regression test suite. + +## Successful scenario + +An isolated temporary Git repository contained `alpha.txt` (`alpha = 17`) and `beta.txt` (`beta = 25`). An interactive CLI session was launched from that directory. Folder trust was accepted for that session only. Available tools were restricted to `view,task`; both were allowed. Built-in MCP servers and remote session export were disabled. + +1. Ask for two separate reads in parallel and their sum. +2. Ask an explore subagent to read the same two files and report their sum. +3. Exit normally with `/exit`. + +Both top-level answers and the subagent answer were `42`. The transcript verifies overlapping top-level reads, five tool start/completion pairs (four reads plus the task invocation), two top-level user prompts, and one completed subagent. + +Repository hooks used version 1, camelCase event names, `type: command`, `bash`, and `timeoutSec: 5`. Each invoked a Python recorder that read stdin JSON and wrote a separate timestamp/PID-named file. It emitted no stdout and caught recorder errors. Captured events: sessionStart, userPromptSubmitted, preToolUse, postToolUse, agentStop, subagentStart, subagentStop, sessionEnd. Failure/error hooks were configured but not exercised. + +## Files + +- `hooks.jsonl`: 20 actual external hook invocations, ordered by recorder filename. `registered_event` is the configured camelCase event; `captured_ns` is recorder wall time; `payload` is Copilot input. +- `events.jsonl`: 76 retained transcript records, in original file order. +- `usage.json`: the final transcript `session.shutdown.data` object, including overall and per-agent usage. This is a duplicate extraction for convenience; do not count it again when aggregating the transcript. + +Sanitization replaces the local home/workspace paths and removes system messages, usage checkpoint internals, `model.*` events, transformed prompts, reasoning text/blocks, opaque/encrypted provider fields, API call IDs, tool telemetry, and server-tools metadata. Event IDs, timestamps, agent IDs, interaction IDs, tool-call IDs, and measured usage are retained. This is deliberately a filtered transcript, not a byte-for-byte raw capture; parentId references may point to omitted records. Raw captures remain in `/private/tmp/thirdeye-copilot-probe` and the original CLI session state. + +## Integration findings + +- External pre/post tool hooks have no invocation ID in this run. The transcript provides `toolCallId` on requests and executions. Correlate parallel tools using the transcript, not just tool names. +- `turnId` denotes a model/tool cycle and resets across user requests. Group user interactions using `interactionId` plus agent identity. +- Subagent events are interleaved in the same transcript and carry top-level `agentId`; `subagent.started` links to the parent task via `data.toolCallId`. Child records also expose `parentToolCallId` where applicable. +- The child generated its own user-prompt and agent-stop hooks using the parent's session ID and transcript path. There are three prompt hooks and three stop hooks for two top-level user requests. External hook payloads alone cannot reliably distinguish these turns. +- SessionStart arrived after the first user-prompt hook. Initialization must tolerate that ordering. +- There are 20 external recorder files but only 18 hook.start/hook.end pairs in the final transcript. Do not assume those two streams have one-to-one coverage. +- Assistant text and model names are present in assistant.message. Final session.shutdown includes input/output/cache/reasoning usage and per-agent metrics. Availability of per-call usage at Stop time has not been established by this probe. + +## Limitations and unsuccessful probes + +Earlier non-interactive `-p` runs completed but produced no external hook captures. This remained true with both PascalCase/exec and camelCase/bash configurations, after committing the temporary hook file, and when launching directly from the fixture directory. Interactive mode with session folder trust produced the successful fixture. This narrows the issue to runtime mode/trust/loading behavior but does not isolate its exact cause; it does not prove non-interactive hooks are universally unsupported. + +PascalCase compatibility, user-level installation, interrupted/error turns, repeated identical concurrent tool arguments, and other Copilot versions/runtimes remain unverified. No permanent user-level hooks or thirdeye adapter code were installed. + +## Persisted database follow-up + +Read-only inspection of `~/.copilot/session-store.db` found six `assistant_usage_events` rows for this session: four main-agent calls and two explore calls. `assistant-usage-events.json` retains those rows. Their input/output/cache-read/cache-write/reasoning totals and nano-AI-unit total exactly match session.shutdown; per-agent input totals also match. Database `turn_index` groups these into the two actual user interactions (0 and 1), unlike transcript turnId, which indexes model cycles and restarts. + +The table includes agent_id, parent_tool_call_id, model, tokens, billing details, call duration, time to first token/output, inter-token latency, initiator, endpoint, reasoning effort, finish reason, and creation time. The installed event schema marks assistant.usage as ephemeral: it is absent from events.jsonl, but these database rows persist it independently. Both interactive and the earlier non-interactive test sessions have usage rows. The table lacks the transcript assistant message ID/provider call ID, so exact per-message joins require additional care; grouping by session/turn/agent is explicit. + +Persisted usage checkpoint events contain aggregate billing and cache-frontier diagnostics, including latest input/cache counts, tool schema hashes, system-segment token estimates, cache TTL, and completion time. They are snapshots, not a complete per-call ledger. In this session each top-level checkpoint follows agentStop in file order, so a Stop-time read can precede the checkpoint. + +The raw transcript also contains two auxiliary model.model_call_success records for gpt-4o-mini session-title generation, with request/response content, usage and latency. These were omitted from the sanitized transcript. Do not treat them as the main-agent model-call history or add their usage to the six-row total without explicitly accounting for auxiliary calls. + +The database also has sessions, turns, checkpoints, session_files, session_refs and full-text search tables. Our session has two complete turns but no session_files rows despite four file reads, so the database's discovery/index tables do not replace raw tool execution events. Files beside the transcript include workspace.yaml (identity/repo/title), checkpoints/index.md (empty here), and rewind-file-snapshots/tracking.json (tracking metadata only here). diff --git a/tests/fixtures/copilot/cli-1.0.83/assistant-usage-events.json b/tests/fixtures/copilot/cli-1.0.83/assistant-usage-events.json new file mode 100644 index 0000000..5de8d67 --- /dev/null +++ b/tests/fixtures/copilot/cli-1.0.83/assistant-usage-events.json @@ -0,0 +1,158 @@ +[ + { + "id": 13, + "session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "turn_index": 0, + "agent_id": null, + "parent_tool_call_id": null, + "model": "gpt-5.6-luna", + "input_tokens": 6452, + "output_tokens": 107, + "cache_read_tokens": 0, + "cache_write_tokens": 6449, + "reasoning_tokens": 29, + "total_nano_aiu": 174125000, + "request_multiplier": 1.0, + "duration_ms": 2263, + "time_to_first_token_ms": 1571.1477920000002, + "output_ttft_ms": 1571.1479590000001, + "inter_token_latency_ms": 7.393025117647059, + "initiator": "user", + "api_endpoint": "ws:/responses", + "reasoning_effort": "medium", + "finish_reason": "tool_calls", + "content_filter_triggered": 0, + "token_details_json": "[{\"batchSize\":1000000,\"costPerBatch\":20000000000,\"tokenCount\":3,\"tokenType\":\"input\"},{\"batchSize\":1000000,\"costPerBatch\":2000000000,\"tokenCount\":0,\"tokenType\":\"cache_read\"},{\"batchSize\":1000000,\"costPerBatch\":25000000000,\"tokenCount\":6449,\"tokenType\":\"cache_write\"},{\"batchSize\":1000000,\"costPerBatch\":120000000000,\"tokenCount\":107,\"tokenType\":\"output\"}]", + "created_at": "2026-09-10T17:08:24.498Z" + }, + { + "id": 14, + "session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "turn_index": 0, + "agent_id": null, + "parent_tool_call_id": null, + "model": "gpt-5.6-luna", + "input_tokens": 6587, + "output_tokens": 5, + "cache_read_tokens": 6449, + "cache_write_tokens": 135, + "reasoning_tokens": 0, + "total_nano_aiu": 16933000, + "request_multiplier": 1.0, + "duration_ms": 1017, + "time_to_first_token_ms": 942.6416250000001, + "output_ttft_ms": 942.642416, + "inter_token_latency_ms": null, + "initiator": "agent", + "api_endpoint": "ws:/responses", + "reasoning_effort": "medium", + "finish_reason": "stop", + "content_filter_triggered": 0, + "token_details_json": "[{\"batchSize\":1000000,\"costPerBatch\":20000000000,\"tokenCount\":3,\"tokenType\":\"input\"},{\"batchSize\":1000000,\"costPerBatch\":2000000000,\"tokenCount\":6449,\"tokenType\":\"cache_read\"},{\"batchSize\":1000000,\"costPerBatch\":25000000000,\"tokenCount\":135,\"tokenType\":\"cache_write\"},{\"batchSize\":1000000,\"costPerBatch\":120000000000,\"tokenCount\":5,\"tokenType\":\"output\"}]", + "created_at": "2026-09-10T17:08:25.618Z" + }, + { + "id": 15, + "session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "turn_index": 1, + "agent_id": null, + "parent_tool_call_id": null, + "model": "gpt-5.6-luna", + "input_tokens": 6627, + "output_tokens": 112, + "cache_read_tokens": 6449, + "cache_write_tokens": 175, + "reasoning_tokens": 10, + "total_nano_aiu": 30773000, + "request_multiplier": 1.0, + "duration_ms": 1813, + "time_to_first_token_ms": 1366.804083, + "output_ttft_ms": 1366.80425, + "inter_token_latency_ms": 4.3487423448275875, + "initiator": "user", + "api_endpoint": "ws:/responses", + "reasoning_effort": "medium", + "finish_reason": "tool_calls", + "content_filter_triggered": 0, + "token_details_json": "[{\"batchSize\":1000000,\"costPerBatch\":20000000000,\"tokenCount\":3,\"tokenType\":\"input\"},{\"batchSize\":1000000,\"costPerBatch\":2000000000,\"tokenCount\":6449,\"tokenType\":\"cache_read\"},{\"batchSize\":1000000,\"costPerBatch\":25000000000,\"tokenCount\":175,\"tokenType\":\"cache_write\"},{\"batchSize\":1000000,\"costPerBatch\":120000000000,\"tokenCount\":112,\"tokenType\":\"output\"}]", + "created_at": "2026-09-10T17:08:44.015Z" + }, + { + "id": 16, + "session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "turn_index": 1, + "agent_id": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", + "parent_tool_call_id": "call_qx4FH5DADTeT1qVLb37HNpBk", + "model": "gpt-5.6-luna", + "input_tokens": 4429, + "output_tokens": 94, + "cache_read_tokens": 0, + "cache_write_tokens": 4426, + "reasoning_tokens": 16, + "total_nano_aiu": 121990000, + "request_multiplier": 1.0, + "duration_ms": 1936, + "time_to_first_token_ms": 1769.324584, + "output_ttft_ms": 1769.3247090000002, + "inter_token_latency_ms": 1.929541, + "initiator": "sub-agent", + "api_endpoint": "ws:/responses", + "reasoning_effort": "low", + "finish_reason": "tool_calls", + "content_filter_triggered": 0, + "token_details_json": "[{\"batchSize\":1000000,\"costPerBatch\":20000000000,\"tokenCount\":3,\"tokenType\":\"input\"},{\"batchSize\":1000000,\"costPerBatch\":2000000000,\"tokenCount\":0,\"tokenType\":\"cache_read\"},{\"batchSize\":1000000,\"costPerBatch\":25000000000,\"tokenCount\":4426,\"tokenType\":\"cache_write\"},{\"batchSize\":1000000,\"costPerBatch\":120000000000,\"tokenCount\":94,\"tokenType\":\"output\"}]", + "created_at": "2026-09-10T17:08:46.500Z" + }, + { + "id": 17, + "session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "turn_index": 1, + "agent_id": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", + "parent_tool_call_id": "call_qx4FH5DADTeT1qVLb37HNpBk", + "model": "gpt-5.6-luna", + "input_tokens": 4551, + "output_tokens": 5, + "cache_read_tokens": 4426, + "cache_write_tokens": 122, + "reasoning_tokens": 0, + "total_nano_aiu": 12562000, + "request_multiplier": 1.0, + "duration_ms": 847, + "time_to_first_token_ms": 788.4591250000001, + "output_ttft_ms": 788.459167, + "inter_token_latency_ms": null, + "initiator": "sub-agent", + "api_endpoint": "ws:/responses", + "reasoning_effort": "low", + "finish_reason": "stop", + "content_filter_triggered": 0, + "token_details_json": "[{\"batchSize\":1000000,\"costPerBatch\":20000000000,\"tokenCount\":3,\"tokenType\":\"input\"},{\"batchSize\":1000000,\"costPerBatch\":2000000000,\"tokenCount\":4426,\"tokenType\":\"cache_read\"},{\"batchSize\":1000000,\"costPerBatch\":25000000000,\"tokenCount\":122,\"tokenType\":\"cache_write\"},{\"batchSize\":1000000,\"costPerBatch\":120000000000,\"tokenCount\":5,\"tokenType\":\"output\"}]", + "created_at": "2026-09-10T17:08:47.455Z" + }, + { + "id": 18, + "session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "turn_index": 1, + "agent_id": null, + "parent_tool_call_id": null, + "model": "gpt-5.6-luna", + "input_tokens": 6750, + "output_tokens": 5, + "cache_read_tokens": 6624, + "cache_write_tokens": 123, + "reasoning_tokens": 0, + "total_nano_aiu": 16983000, + "request_multiplier": 1.0, + "duration_ms": 732, + "time_to_first_token_ms": 677.7400409999999, + "output_ttft_ms": 677.7401659999999, + "inter_token_latency_ms": null, + "initiator": "agent", + "api_endpoint": "ws:/responses", + "reasoning_effort": "medium", + "finish_reason": "stop", + "content_filter_triggered": 0, + "token_details_json": "[{\"batchSize\":1000000,\"costPerBatch\":20000000000,\"tokenCount\":3,\"tokenType\":\"input\"},{\"batchSize\":1000000,\"costPerBatch\":2000000000,\"tokenCount\":6624,\"tokenType\":\"cache_read\"},{\"batchSize\":1000000,\"costPerBatch\":25000000000,\"tokenCount\":123,\"tokenType\":\"cache_write\"},{\"batchSize\":1000000,\"costPerBatch\":120000000000,\"tokenCount\":5,\"tokenType\":\"output\"}]", + "created_at": "2026-09-10T17:08:48.278Z" + } +] diff --git a/tests/fixtures/copilot/cli-1.0.83/events.jsonl b/tests/fixtures/copilot/cli-1.0.83/events.jsonl new file mode 100644 index 0000000..60e257f --- /dev/null +++ b/tests/fixtures/copilot/cli-1.0.83/events.jsonl @@ -0,0 +1,76 @@ +{"type": "session.start", "data": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "version": 1, "producer": "copilot-agent", "copilotVersion": "1.0.83", "startTime": "2026-09-10T17:08:05.884Z", "contextTier": null, "context": {"cwd": "/fixture/workspace", "gitRoot": "/fixture/workspace", "branch": "master", "headCommit": "6eecf11579d0c8fb0da1b7fd9671dada3910847c"}, "alreadyInUse": false, "remoteSteerable": false}, "id": "26159b14-7d71-4dd1-96c1-9404fe6b356d", "timestamp": "2026-09-10T17:08:05.891Z", "parentId": null} +{"type": "session.model_change", "data": {"cause": "initial_resolution", "source": "automatic", "contextTier": null, "newModel": "auto", "reasoningEffort": null}, "id": "ab149215-ec43-4c2e-8a58-a176f104d0dc", "timestamp": "2026-09-10T17:08:07.140Z", "parentId": "26159b14-7d71-4dd1-96c1-9404fe6b356d"} +{"type": "session.auto_mode_resolved", "data": {"chosenModel": "gpt-5.6-luna", "categoryScores": {"code_gen": 0.1191, "debugging": 0.0018, "reasoning": 0.2963, "tool_use": 0.1192}, "candidateModels": ["gpt-5.6-luna"], "routingMethod": "auto_v2", "fallback": false, "availableModels": ["gpt-5.6-luna"], "endToEndLatencyMs": 184.635416, "hasImage": false}, "id": "2d18ca6f-c3f1-48ba-bfbf-f8f4e1e9c9ef", "timestamp": "2026-09-10T17:08:22.040Z", "parentId": "ab149215-ec43-4c2e-8a58-a176f104d0dc"} +{"type": "hook.start", "data": {"hookInvocationId": "3ad4c584-f613-4cf1-9e6e-3f09eb60be99", "hookType": "userPromptSubmitted", "input": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "prompt": "Read alpha.txt and beta.txt with separate view calls in parallel and report their sum. Only read those two files.", "timestamp": 1789060102173, "cwd": "/fixture/workspace"}}, "id": "a8043161-e819-426c-bf9d-0866b35b9082", "timestamp": "2026-09-10T17:08:22.173Z", "parentId": "2d18ca6f-c3f1-48ba-bfbf-f8f4e1e9c9ef"} +{"type": "hook.end", "data": {"hookInvocationId": "3ad4c584-f613-4cf1-9e6e-3f09eb60be99", "hookType": "userPromptSubmitted", "success": true}, "id": "3942810f-1caf-4251-82fb-3bf72698147a", "timestamp": "2026-09-10T17:08:22.203Z", "parentId": "a8043161-e819-426c-bf9d-0866b35b9082"} +{"type": "user.message", "data": {"content": "Read alpha.txt and beta.txt with separate view calls in parallel and report their sum. Only read those two files.", "messageId": "a856cb38-7609-45ab-8a55-645553155db3", "supportedNativeDocumentMimeTypes": [], "delivery": "idle", "interactionId": "6d2b89fd-a653-430c-b532-b0936d72eb42", "turnId": "0", "parentAgentTaskId": "fd800f7a-8163-4304-9681-efde4731e879"}, "id": "f07404d0-af52-4260-89fb-358a10e86034", "timestamp": "2026-09-10T17:08:22.203Z", "parentId": "3942810f-1caf-4251-82fb-3bf72698147a"} +{"type": "hook.start", "data": {"hookInvocationId": "c75fca6a-c0a7-4440-9bf4-10399f301f1d", "hookType": "sessionStart", "input": {"source": "new", "initialPrompt": "Read alpha.txt and beta.txt with separate view calls in parallel and report their sum. Only read those two files.", "sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "timestamp": 1789060102204, "cwd": "/fixture/workspace"}}, "id": "86b5b021-d90a-45ad-9042-41fc11dd31e3", "timestamp": "2026-09-10T17:08:22.204Z", "parentId": "a7e8a23d-8522-42af-84d7-f122f025ce0a"} +{"type": "hook.end", "data": {"hookInvocationId": "c75fca6a-c0a7-4440-9bf4-10399f301f1d", "hookType": "sessionStart", "success": true}, "id": "89373174-eb9e-4129-94b7-2ab8f01c6c03", "timestamp": "2026-09-10T17:08:22.227Z", "parentId": "86b5b021-d90a-45ad-9042-41fc11dd31e3"} +{"type": "assistant.turn_start", "data": {"turnId": "0", "interactionId": "6d2b89fd-a653-430c-b532-b0936d72eb42"}, "id": "64f9e436-651f-4a9d-919c-a4d7abad2652", "timestamp": "2026-09-10T17:08:22.227Z", "parentId": "89373174-eb9e-4129-94b7-2ab8f01c6c03"} +{"type": "assistant.message", "data": {"messageId": "5a29b09d-5e7d-4603-975e-7801ce54232b", "model": "gpt-5.6-luna", "content": "", "toolRequests": [{"toolCallId": "call_YSSva4HCniiETlxdGGjcrHbh", "name": "view", "arguments": {"path": "/fixture/workspace/alpha.txt"}, "type": "function", "intentionSummary": "view the file at /fixture/workspace/alpha.txt."}, {"toolCallId": "call_ayHplfzxjRFMTCpmTKEFhCSJ", "name": "view", "arguments": {"path": "/fixture/workspace/beta.txt"}, "type": "function", "intentionSummary": "view the file at /fixture/workspace/beta.txt."}], "interactionId": "6d2b89fd-a653-430c-b532-b0936d72eb42", "turnId": "0", "rte": true}, "id": "a4a17e63-7ba5-422f-8ee9-b495be417328", "timestamp": "2026-09-10T17:08:24.503Z", "parentId": "64f9e436-651f-4a9d-919c-a4d7abad2652"} +{"type": "tool.execution_start", "data": {"toolCallId": "call_YSSva4HCniiETlxdGGjcrHbh", "toolName": "view", "arguments": {"path": "/fixture/workspace/alpha.txt"}, "turnId": "0", "model": "gpt-5.6-luna"}, "id": "a7f7bf04-589e-4989-a4a2-7ee687279627", "timestamp": "2026-09-10T17:08:24.506Z", "parentId": "a4a17e63-7ba5-422f-8ee9-b495be417328"} +{"type": "tool.execution_start", "data": {"toolCallId": "call_ayHplfzxjRFMTCpmTKEFhCSJ", "toolName": "view", "arguments": {"path": "/fixture/workspace/beta.txt"}, "turnId": "0", "model": "gpt-5.6-luna"}, "id": "39e9da78-9bc3-4e1a-9629-f6680d1aeb4d", "timestamp": "2026-09-10T17:08:24.506Z", "parentId": "a7f7bf04-589e-4989-a4a2-7ee687279627"} +{"type": "hook.start", "data": {"hookInvocationId": "bac5cbf0-b56e-4ff6-98ca-a3f583c62a78", "hookType": "preToolUse", "input": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "cwd": "/fixture/workspace", "toolCalls": [{"id": "call_YSSva4HCniiETlxdGGjcrHbh", "name": "view", "args": {"path": "/fixture/workspace/alpha.txt"}}, {"id": "call_ayHplfzxjRFMTCpmTKEFhCSJ", "name": "view", "args": {"path": "/fixture/workspace/beta.txt"}}]}}, "id": "e595215c-0592-4d36-8098-914f07b6923c", "timestamp": "2026-09-10T17:08:24.506Z", "parentId": "39e9da78-9bc3-4e1a-9629-f6680d1aeb4d"} +{"type": "hook.end", "data": {"hookInvocationId": "bac5cbf0-b56e-4ff6-98ca-a3f583c62a78", "hookType": "preToolUse", "success": true}, "id": "c7158444-1b49-469f-af1e-962bc1977313", "timestamp": "2026-09-10T17:08:24.551Z", "parentId": "e595215c-0592-4d36-8098-914f07b6923c"} +{"type": "hook.start", "data": {"hookInvocationId": "b196eb30-c3eb-4907-97dc-cca942dd6d74", "hookType": "postToolUse", "input": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "timestamp": 1789060104552, "cwd": "/fixture/workspace", "toolName": "view", "toolArgs": {"path": "/fixture/workspace/alpha.txt"}, "toolResult": {"resultType": "success", "textResultForLlm": "alpha = 17\n", "sessionLog": "[copilot:elided sessionLog (308 bytes) — pre-hook tool result; the final result may differ, see the adjacent tool.execution_complete event]", "skipLargeOutputProcessing": true}}}, "id": "96dc12ef-fe49-4bc5-b75a-58589b90293c", "timestamp": "2026-09-10T17:08:24.552Z", "parentId": "c7158444-1b49-469f-af1e-962bc1977313"} +{"type": "hook.end", "data": {"hookInvocationId": "b196eb30-c3eb-4907-97dc-cca942dd6d74", "hookType": "postToolUse", "success": true}, "id": "a3b79888-b777-42b2-a893-ca9d56b4002f", "timestamp": "2026-09-10T17:08:24.572Z", "parentId": "96dc12ef-fe49-4bc5-b75a-58589b90293c"} +{"type": "tool.execution_complete", "data": {"toolCallId": "call_YSSva4HCniiETlxdGGjcrHbh", "model": "gpt-5.6-luna", "interactionId": "6d2b89fd-a653-430c-b532-b0936d72eb42", "turnId": "0", "rte": true, "success": true, "result": {"content": "alpha = 17\n", "detailedContent": "\ndiff --git a/fixture/workspace/alpha.txt b/fixture/workspace/alpha.txt\nindex 0000000..0000000 100644\n--- a/fixture/workspace/alpha.txt\n+++ b/fixture/workspace/alpha.txt\n@@ -1,2 +1,2 @@\n alpha = 17\n \n"}}, "id": "1b06e9d1-9223-449a-a135-b4f64cf0e3a6", "timestamp": "2026-09-10T17:08:24.572Z", "parentId": "a3b79888-b777-42b2-a893-ca9d56b4002f"} +{"type": "hook.start", "data": {"hookInvocationId": "f50891cc-7434-4b35-aa9d-aacb15861567", "hookType": "postToolUse", "input": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "timestamp": 1789060104572, "cwd": "/fixture/workspace", "toolName": "view", "toolArgs": {"path": "/fixture/workspace/beta.txt"}, "toolResult": {"resultType": "success", "textResultForLlm": "beta = 25\n", "sessionLog": "[copilot:elided sessionLog (303 bytes) — pre-hook tool result; the final result may differ, see the adjacent tool.execution_complete event]", "skipLargeOutputProcessing": true}}}, "id": "4f993405-aade-4f24-901f-203ade003627", "timestamp": "2026-09-10T17:08:24.573Z", "parentId": "1b06e9d1-9223-449a-a135-b4f64cf0e3a6"} +{"type": "hook.end", "data": {"hookInvocationId": "f50891cc-7434-4b35-aa9d-aacb15861567", "hookType": "postToolUse", "success": true}, "id": "6561c429-fada-49da-b642-1b2ea8d1d650", "timestamp": "2026-09-10T17:08:24.592Z", "parentId": "4f993405-aade-4f24-901f-203ade003627"} +{"type": "tool.execution_complete", "data": {"toolCallId": "call_ayHplfzxjRFMTCpmTKEFhCSJ", "model": "gpt-5.6-luna", "interactionId": "6d2b89fd-a653-430c-b532-b0936d72eb42", "turnId": "0", "rte": true, "success": true, "result": {"content": "beta = 25\n", "detailedContent": "\ndiff --git a/fixture/workspace/beta.txt b/fixture/workspace/beta.txt\nindex 0000000..0000000 100644\n--- a/fixture/workspace/beta.txt\n+++ b/fixture/workspace/beta.txt\n@@ -1,2 +1,2 @@\n beta = 25\n \n"}}, "id": "032cf87e-05b7-4e5f-96b9-1fda1a8242f5", "timestamp": "2026-09-10T17:08:24.593Z", "parentId": "6561c429-fada-49da-b642-1b2ea8d1d650"} +{"type": "assistant.turn_end", "data": {"turnId": "0"}, "id": "0080e44c-ad62-4288-b2b2-061ec2b73d80", "timestamp": "2026-09-10T17:08:24.593Z", "parentId": "032cf87e-05b7-4e5f-96b9-1fda1a8242f5"} +{"type": "assistant.turn_start", "data": {"turnId": "1", "interactionId": "6d2b89fd-a653-430c-b532-b0936d72eb42"}, "id": "667fe48a-70a1-473b-b005-454964022344", "timestamp": "2026-09-10T17:08:24.594Z", "parentId": "0080e44c-ad62-4288-b2b2-061ec2b73d80"} +{"type": "assistant.message", "data": {"messageId": "162d92b0-5d31-444c-b96f-b6c551527d2b", "model": "gpt-5.6-luna", "content": "42", "toolRequests": [], "interactionId": "6d2b89fd-a653-430c-b532-b0936d72eb42", "turnId": "1", "phase": "final_answer", "rte": true}, "id": "33cc6465-29e1-4a04-8bdb-00241474b4d2", "timestamp": "2026-09-10T17:08:25.624Z", "parentId": "667fe48a-70a1-473b-b005-454964022344"} +{"type": "assistant.turn_end", "data": {"turnId": "1"}, "id": "4a386a37-ca7e-4ebc-a746-cdda20f2a4bb", "timestamp": "2026-09-10T17:08:25.626Z", "parentId": "33cc6465-29e1-4a04-8bdb-00241474b4d2"} +{"type": "hook.start", "data": {"hookInvocationId": "ce6d52d0-7e43-41ca-956e-863ca6c38cda", "hookType": "agentStop", "input": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "transcriptPath": "/home/tester/.copilot/session-state/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/events.jsonl", "stopReason": "end_turn", "stop_hook_active": false, "timestamp": 1789060105626, "cwd": "/fixture/workspace"}}, "id": "7bb0689c-8435-46f7-909e-7e0223753217", "timestamp": "2026-09-10T17:08:25.626Z", "parentId": "4a386a37-ca7e-4ebc-a746-cdda20f2a4bb"} +{"type": "hook.end", "data": {"hookInvocationId": "ce6d52d0-7e43-41ca-956e-863ca6c38cda", "hookType": "agentStop", "success": true}, "id": "b514e4ab-6141-4e22-a9dd-6f095f36e22c", "timestamp": "2026-09-10T17:08:25.650Z", "parentId": "7bb0689c-8435-46f7-909e-7e0223753217"} +{"type": "session.auto_mode_resolved", "data": {"chosenModel": "gpt-5.6-luna", "categoryScores": {"reasoning": 0.5805, "tool_use": 0.4379, "debugging": 0.0062, "code_gen": 0.3977}, "candidateModels": ["gpt-5.6-luna"], "routingMethod": "auto_v2", "fallback": false, "availableModels": ["gpt-5.6-luna"], "endToEndLatencyMs": 432.443958, "hasImage": false}, "id": "c19cb305-f15c-4419-82ae-df9278d56d0c", "timestamp": "2026-09-10T17:08:42.154Z", "parentId": "d70783ab-86ee-4a5b-89eb-b95627215202"} +{"type": "hook.start", "data": {"hookInvocationId": "f85718f6-b82f-4b5a-ba84-6893af297355", "hookType": "userPromptSubmitted", "input": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "prompt": "Invoke one explore subagent to read only alpha.txt and beta.txt and report their sum. Do not modify files or access other files or services. Then report its answer.", "timestamp": 1789060122166, "cwd": "/fixture/workspace"}}, "id": "41248b6e-c3c4-4d3d-8b4b-0f7982d50ba9", "timestamp": "2026-09-10T17:08:42.166Z", "parentId": "c19cb305-f15c-4419-82ae-df9278d56d0c"} +{"type": "hook.end", "data": {"hookInvocationId": "f85718f6-b82f-4b5a-ba84-6893af297355", "hookType": "userPromptSubmitted", "success": true}, "id": "59c37c71-d5db-4419-a2db-1cd1fad7ed5d", "timestamp": "2026-09-10T17:08:42.192Z", "parentId": "41248b6e-c3c4-4d3d-8b4b-0f7982d50ba9"} +{"type": "user.message", "data": {"content": "Invoke one explore subagent to read only alpha.txt and beta.txt and report their sum. Do not modify files or access other files or services. Then report its answer.", "messageId": "02868f8d-8ec2-4485-b4a5-a75312e4d2e5", "supportedNativeDocumentMimeTypes": [], "delivery": "idle", "interactionId": "793d3703-6f4a-4814-8877-34a7325848ce", "turnId": "0", "parentAgentTaskId": "c17fc649-17ec-435a-ac27-77dbd6f92379"}, "id": "089db64c-a039-4635-ad4d-40d588c143be", "timestamp": "2026-09-10T17:08:42.192Z", "parentId": "59c37c71-d5db-4419-a2db-1cd1fad7ed5d"} +{"type": "assistant.turn_start", "data": {"turnId": "0", "interactionId": "793d3703-6f4a-4814-8877-34a7325848ce"}, "id": "0012e0f5-13cd-44dc-8126-51e584628989", "timestamp": "2026-09-10T17:08:42.194Z", "parentId": "089db64c-a039-4635-ad4d-40d588c143be"} +{"type": "assistant.message", "data": {"messageId": "0e5973fb-79e5-48d9-ba90-233f71ee70cc", "model": "gpt-5.6-luna", "content": "", "toolRequests": [{"toolCallId": "call_qx4FH5DADTeT1qVLb37HNpBk", "name": "task", "arguments": {"description": "Sum two text files", "prompt": "Read only /fixture/workspace/alpha.txt and /fixture/workspace/beta.txt. Do not modify files. Do not access any other files or services. Report the sum of the numeric values in those two files.", "agent_type": "explore", "name": "sum-alpha-beta", "mode": "sync"}, "type": "function", "intentionSummary": "Sum two text files"}], "interactionId": "793d3703-6f4a-4814-8877-34a7325848ce", "turnId": "0", "rte": true}, "id": "2c2e4ab8-f283-4837-957d-da992ba55e65", "timestamp": "2026-09-10T17:08:44.019Z", "parentId": "0012e0f5-13cd-44dc-8126-51e584628989"} +{"type": "tool.execution_start", "data": {"toolCallId": "call_qx4FH5DADTeT1qVLb37HNpBk", "toolName": "task", "arguments": {"description": "Sum two text files", "prompt": "Read only /fixture/workspace/alpha.txt and /fixture/workspace/beta.txt. Do not modify files. Do not access any other files or services. Report the sum of the numeric values in those two files.", "agent_type": "explore", "name": "sum-alpha-beta", "mode": "sync"}, "turnId": "0", "model": "gpt-5.6-luna"}, "id": "41dabdfc-681e-4043-a2ce-83b06cc07032", "timestamp": "2026-09-10T17:08:44.021Z", "parentId": "2c2e4ab8-f283-4837-957d-da992ba55e65"} +{"type": "hook.start", "data": {"hookInvocationId": "097d963e-4289-4eb1-998c-235f17b32451", "hookType": "preToolUse", "input": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "cwd": "/fixture/workspace", "toolCalls": [{"id": "call_qx4FH5DADTeT1qVLb37HNpBk", "name": "task", "args": {"description": "Sum two text files", "prompt": "Read only /fixture/workspace/alpha.txt and /fixture/workspace/beta.txt. Do not modify files. Do not access any other files or services. Report the sum of the numeric values in those two files.", "agent_type": "explore", "name": "sum-alpha-beta", "mode": "sync"}}]}}, "id": "6c2d498f-02c6-4b7c-8f96-134e29f4c25c", "timestamp": "2026-09-10T17:08:44.021Z", "parentId": "41dabdfc-681e-4043-a2ce-83b06cc07032"} +{"type": "hook.end", "data": {"hookInvocationId": "097d963e-4289-4eb1-998c-235f17b32451", "hookType": "preToolUse", "success": true}, "id": "0049f918-7bb9-478a-8c91-b6f8353525a9", "timestamp": "2026-09-10T17:08:44.045Z", "parentId": "6c2d498f-02c6-4b7c-8f96-134e29f4c25c"} +{"type": "subagent.started", "data": {"toolCallId": "call_qx4FH5DADTeT1qVLb37HNpBk", "agentName": "explore", "agentDisplayName": "sum-alpha-beta", "agentDescription": "Sum two text files", "model": "gpt-5.6-luna", "resumable": false, "agentType": "explore", "executionMode": "sync"}, "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "id": "6dff9fa5-c2df-41da-912b-d1ffd0706076", "timestamp": "2026-09-10T17:08:44.058Z", "parentId": "0049f918-7bb9-478a-8c91-b6f8353525a9"} +{"type": "subagent.configured", "data": {"model": "gpt-5.6-luna", "reasoningEffort": "low", "multiTurn": true}, "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "id": "af2030ff-f375-4702-b468-8276db3353f6", "timestamp": "2026-09-10T17:08:44.082Z", "parentId": "6dff9fa5-c2df-41da-912b-d1ffd0706076"} +{"type": "hook.start", "data": {"hookInvocationId": "fbdefe2f-bb83-4774-8491-4260dcbe4743", "hookType": "subagentStart", "input": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "transcriptPath": "/home/tester/.copilot/session-state/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/events.jsonl", "agentName": "explore", "timestamp": 1789060124082, "cwd": "/fixture/workspace"}}, "id": "b7bd0d91-2b63-48c5-9ee8-98531fa144f8", "timestamp": "2026-09-10T17:08:44.082Z", "parentId": "af2030ff-f375-4702-b468-8276db3353f6"} +{"type": "hook.end", "data": {"hookInvocationId": "fbdefe2f-bb83-4774-8491-4260dcbe4743", "hookType": "subagentStart", "success": true}, "id": "bd85c142-0403-4f71-b3df-e580cba56659", "timestamp": "2026-09-10T17:08:44.102Z", "parentId": "b7bd0d91-2b63-48c5-9ee8-98531fa144f8"} +{"type": "session.auto_mode_resolved", "data": {"chosenModel": "gpt-5.6-luna", "categoryScores": {"tool_use": 0.1064, "debugging": 0.0022, "code_gen": 0.065, "reasoning": 0.1966}, "candidateModels": ["gpt-5.6-luna"], "routingMethod": "auto_v2", "fallback": false, "availableModels": ["gpt-5.6-luna"], "endToEndLatencyMs": 400.976, "hasImage": false}, "id": "4812bec4-cdbb-4bd7-ab04-9e75367c81bc", "timestamp": "2026-09-10T17:08:44.505Z", "parentId": null} +{"type": "hook.start", "data": {"hookInvocationId": "1a7a5f25-21e6-4c63-bf2e-67ee8143f8de", "hookType": "userPromptSubmitted", "input": {"sessionId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "prompt": "Read only /fixture/workspace/alpha.txt and /fixture/workspace/beta.txt. Do not modify files. Do not access any other files or services. Report the sum of the numeric values in those two files.", "timestamp": 1789060124530, "cwd": "/fixture/workspace"}}, "id": "18c6c036-7321-4f42-ac97-108572d250fd", "timestamp": "2026-09-10T17:08:44.530Z", "parentId": "bd85c142-0403-4f71-b3df-e580cba56659"} +{"type": "hook.end", "data": {"hookInvocationId": "1a7a5f25-21e6-4c63-bf2e-67ee8143f8de", "hookType": "userPromptSubmitted", "success": true}, "id": "3bd18700-90c6-4b2f-aaaa-63760bdfa89a", "timestamp": "2026-09-10T17:08:44.557Z", "parentId": "18c6c036-7321-4f42-ac97-108572d250fd"} +{"type": "user.message", "data": {"content": "Read only /fixture/workspace/alpha.txt and /fixture/workspace/beta.txt. Do not modify files. Do not access any other files or services. Report the sum of the numeric values in those two files.", "source": "agent-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "supportedNativeDocumentMimeTypes": [], "delivery": "idle", "interactionId": "7c0fa097-c0e2-48da-b2b6-fcfc1ad83a6b", "turnId": "0", "parentAgentTaskId": "5a3e63ac-c073-40ba-b07a-010d534b363e"}, "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "id": "d4e30206-94f5-47d5-b15e-7411b9708aec", "timestamp": "2026-09-10T17:08:44.557Z", "parentId": "3bd18700-90c6-4b2f-aaaa-63760bdfa89a"} +{"type": "assistant.turn_start", "data": {"turnId": "0", "interactionId": "7c0fa097-c0e2-48da-b2b6-fcfc1ad83a6b"}, "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "id": "08de037c-23cf-4e4d-a549-3c745adca7bc", "timestamp": "2026-09-10T17:08:44.561Z", "parentId": "4fdcd841-7cfa-4775-8659-aa8b68e9814a"} +{"type": "assistant.message", "data": {"messageId": "4015ccbc-ef99-466f-bef9-392da640a9b2", "model": "gpt-5.6-luna", "content": "", "toolRequests": [{"toolCallId": "call_zZncCGtp1twgcL2eoFUNwInh", "name": "view", "arguments": {"path": "/fixture/workspace/alpha.txt"}, "type": "function", "intentionSummary": "view the file at /fixture/workspace/alpha.txt."}, {"toolCallId": "call_Jeh7IbrUHaq4jVdxtyQCVrns", "name": "view", "arguments": {"path": "/fixture/workspace/beta.txt"}, "type": "function", "intentionSummary": "view the file at /fixture/workspace/beta.txt."}], "interactionId": "7c0fa097-c0e2-48da-b2b6-fcfc1ad83a6b", "turnId": "0", "rte": true, "parentToolCallId": "call_qx4FH5DADTeT1qVLb37HNpBk"}, "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "id": "df600539-fdd4-4de2-bd42-7f7ff2c952ab", "timestamp": "2026-09-10T17:08:46.507Z", "parentId": "08de037c-23cf-4e4d-a549-3c745adca7bc"} +{"type": "tool.execution_start", "data": {"toolCallId": "call_zZncCGtp1twgcL2eoFUNwInh", "toolName": "view", "arguments": {"path": "/fixture/workspace/alpha.txt"}, "turnId": "0", "model": "gpt-5.6-luna", "parentToolCallId": "call_qx4FH5DADTeT1qVLb37HNpBk"}, "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "id": "ade5b5b6-3af1-4e8d-bd9c-a22f94eb599b", "timestamp": "2026-09-10T17:08:46.508Z", "parentId": "df600539-fdd4-4de2-bd42-7f7ff2c952ab"} +{"type": "tool.execution_start", "data": {"toolCallId": "call_Jeh7IbrUHaq4jVdxtyQCVrns", "toolName": "view", "arguments": {"path": "/fixture/workspace/beta.txt"}, "turnId": "0", "model": "gpt-5.6-luna", "parentToolCallId": "call_qx4FH5DADTeT1qVLb37HNpBk"}, "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "id": "635dec68-22a4-4bb8-9b9b-fac0c1b65a78", "timestamp": "2026-09-10T17:08:46.508Z", "parentId": "ade5b5b6-3af1-4e8d-bd9c-a22f94eb599b"} +{"type": "hook.start", "data": {"hookInvocationId": "aabb620d-669c-4e2e-8ebc-d30637ad6ba3", "hookType": "preToolUse", "input": {"sessionId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "cwd": "/fixture/workspace", "toolCalls": [{"id": "call_zZncCGtp1twgcL2eoFUNwInh", "name": "view", "args": {"path": "/fixture/workspace/alpha.txt"}}, {"id": "call_Jeh7IbrUHaq4jVdxtyQCVrns", "name": "view", "args": {"path": "/fixture/workspace/beta.txt"}}]}, "parentToolCallId": "call_qx4FH5DADTeT1qVLb37HNpBk"}, "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "id": "53bbc5ab-6e18-43c8-9ea0-ea392aec0216", "timestamp": "2026-09-10T17:08:46.511Z", "parentId": "635dec68-22a4-4bb8-9b9b-fac0c1b65a78"} +{"type": "hook.end", "data": {"hookInvocationId": "aabb620d-669c-4e2e-8ebc-d30637ad6ba3", "hookType": "preToolUse", "success": true, "parentToolCallId": "call_qx4FH5DADTeT1qVLb37HNpBk"}, "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "id": "febff749-bbb0-4f41-bf13-9f232e46be62", "timestamp": "2026-09-10T17:08:46.558Z", "parentId": "53bbc5ab-6e18-43c8-9ea0-ea392aec0216"} +{"type": "hook.start", "data": {"hookInvocationId": "9dad8db0-fa58-408c-b1f8-de15ce89170c", "hookType": "postToolUse", "input": {"sessionId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "timestamp": 1789060126559, "cwd": "/fixture/workspace", "toolName": "view", "toolArgs": {"path": "/fixture/workspace/alpha.txt"}, "toolResult": {"resultType": "success", "textResultForLlm": "alpha = 17\n", "sessionLog": "[copilot:elided sessionLog (308 bytes) — pre-hook tool result; the final result may differ, see the adjacent tool.execution_complete event]", "skipLargeOutputProcessing": true}}, "parentToolCallId": "call_qx4FH5DADTeT1qVLb37HNpBk"}, "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "id": "0df623f1-787a-4245-9d2b-866363e15034", "timestamp": "2026-09-10T17:08:46.559Z", "parentId": "febff749-bbb0-4f41-bf13-9f232e46be62"} +{"type": "hook.end", "data": {"hookInvocationId": "9dad8db0-fa58-408c-b1f8-de15ce89170c", "hookType": "postToolUse", "success": true, "parentToolCallId": "call_qx4FH5DADTeT1qVLb37HNpBk"}, "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "id": "0a94d8b0-1009-43e6-9efc-242c113f9452", "timestamp": "2026-09-10T17:08:46.580Z", "parentId": "0df623f1-787a-4245-9d2b-866363e15034"} +{"type": "tool.execution_complete", "data": {"toolCallId": "call_zZncCGtp1twgcL2eoFUNwInh", "model": "gpt-5.6-luna", "interactionId": "7c0fa097-c0e2-48da-b2b6-fcfc1ad83a6b", "turnId": "0", "rte": true, "success": true, "result": {"content": "alpha = 17\n", "detailedContent": "\ndiff --git a/fixture/workspace/alpha.txt b/fixture/workspace/alpha.txt\nindex 0000000..0000000 100644\n--- a/fixture/workspace/alpha.txt\n+++ b/fixture/workspace/alpha.txt\n@@ -1,2 +1,2 @@\n alpha = 17\n \n"}, "parentToolCallId": "call_qx4FH5DADTeT1qVLb37HNpBk"}, "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "id": "e7f7e14d-8658-443a-b65f-91b69c76f74e", "timestamp": "2026-09-10T17:08:46.581Z", "parentId": "0a94d8b0-1009-43e6-9efc-242c113f9452"} +{"type": "hook.start", "data": {"hookInvocationId": "f461652e-836c-4496-b298-ca0db28a7d20", "hookType": "postToolUse", "input": {"sessionId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "timestamp": 1789060126581, "cwd": "/fixture/workspace", "toolName": "view", "toolArgs": {"path": "/fixture/workspace/beta.txt"}, "toolResult": {"resultType": "success", "textResultForLlm": "beta = 25\n", "sessionLog": "[copilot:elided sessionLog (303 bytes) — pre-hook tool result; the final result may differ, see the adjacent tool.execution_complete event]", "skipLargeOutputProcessing": true}}, "parentToolCallId": "call_qx4FH5DADTeT1qVLb37HNpBk"}, "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "id": "5a8e0a8b-1494-4197-be55-346b1cf65196", "timestamp": "2026-09-10T17:08:46.581Z", "parentId": "e7f7e14d-8658-443a-b65f-91b69c76f74e"} +{"type": "hook.end", "data": {"hookInvocationId": "f461652e-836c-4496-b298-ca0db28a7d20", "hookType": "postToolUse", "success": true, "parentToolCallId": "call_qx4FH5DADTeT1qVLb37HNpBk"}, "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "id": "05970ea7-1aa5-40c4-90af-8170132ee2f4", "timestamp": "2026-09-10T17:08:46.602Z", "parentId": "5a8e0a8b-1494-4197-be55-346b1cf65196"} +{"type": "tool.execution_complete", "data": {"toolCallId": "call_Jeh7IbrUHaq4jVdxtyQCVrns", "model": "gpt-5.6-luna", "interactionId": "7c0fa097-c0e2-48da-b2b6-fcfc1ad83a6b", "turnId": "0", "rte": true, "success": true, "result": {"content": "beta = 25\n", "detailedContent": "\ndiff --git a/fixture/workspace/beta.txt b/fixture/workspace/beta.txt\nindex 0000000..0000000 100644\n--- a/fixture/workspace/beta.txt\n+++ b/fixture/workspace/beta.txt\n@@ -1,2 +1,2 @@\n beta = 25\n \n"}, "parentToolCallId": "call_qx4FH5DADTeT1qVLb37HNpBk"}, "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "id": "b9ad6a86-44d9-4482-aa7a-d53811fd4177", "timestamp": "2026-09-10T17:08:46.602Z", "parentId": "05970ea7-1aa5-40c4-90af-8170132ee2f4"} +{"type": "assistant.turn_end", "data": {"turnId": "0"}, "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "id": "d4381f0c-8b93-4295-b04f-27def8b446dd", "timestamp": "2026-09-10T17:08:46.602Z", "parentId": "b9ad6a86-44d9-4482-aa7a-d53811fd4177"} +{"type": "assistant.turn_start", "data": {"turnId": "1", "interactionId": "7c0fa097-c0e2-48da-b2b6-fcfc1ad83a6b"}, "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "id": "31c55d44-2afa-42dc-999e-6c2b0d383df2", "timestamp": "2026-09-10T17:08:46.605Z", "parentId": "d4381f0c-8b93-4295-b04f-27def8b446dd"} +{"type": "hook.start", "data": {"hookInvocationId": "029e591f-4244-4be3-9678-21b6a7fbd0c7", "hookType": "agentStop", "input": {"sessionId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "transcriptPath": "/home/tester/.copilot/session-state/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/events.jsonl", "stopReason": "end_turn", "stop_hook_active": false, "timestamp": 1789060127456, "cwd": "/fixture/workspace"}}, "id": "5c654d22-6d2e-4846-a514-f5af5fb17347", "timestamp": "2026-09-10T17:08:47.456Z", "parentId": "31c55d44-2afa-42dc-999e-6c2b0d383df2"} +{"type": "assistant.message", "data": {"messageId": "561815db-29a4-4c0b-9b8c-f4a0186c2937", "model": "gpt-5.6-luna", "content": "42", "toolRequests": [], "interactionId": "7c0fa097-c0e2-48da-b2b6-fcfc1ad83a6b", "turnId": "1", "phase": "final_answer", "rte": true, "parentToolCallId": "call_qx4FH5DADTeT1qVLb37HNpBk"}, "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "id": "450a1f4c-de11-4d37-bf84-08f63d29588f", "timestamp": "2026-09-10T17:08:47.462Z", "parentId": "5c654d22-6d2e-4846-a514-f5af5fb17347"} +{"type": "assistant.turn_end", "data": {"turnId": "1"}, "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "id": "59c10ad4-04de-40c2-8320-398cc8e2dacb", "timestamp": "2026-09-10T17:08:47.463Z", "parentId": "450a1f4c-de11-4d37-bf84-08f63d29588f"} +{"type": "hook.end", "data": {"hookInvocationId": "029e591f-4244-4be3-9678-21b6a7fbd0c7", "hookType": "agentStop", "success": true}, "id": "7ef9a005-b539-4ef5-9106-c703e1843f86", "timestamp": "2026-09-10T17:08:47.489Z", "parentId": "59c10ad4-04de-40c2-8320-398cc8e2dacb"} +{"type": "hook.start", "data": {"hookInvocationId": "b446b65f-21f8-447c-9e8e-14992e92718a", "hookType": "subagentStop", "input": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "transcriptPath": "/home/tester/.copilot/session-state/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/events.jsonl", "agentName": "explore", "agentType": "explore", "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "stopReason": "end_turn", "response": "42", "timestamp": 1789060127489, "cwd": "/fixture/workspace"}}, "id": "682ab836-6ad2-4f42-a703-4ca32438f826", "timestamp": "2026-09-10T17:08:47.489Z", "parentId": "7ef9a005-b539-4ef5-9106-c703e1843f86"} +{"type": "hook.end", "data": {"hookInvocationId": "b446b65f-21f8-447c-9e8e-14992e92718a", "hookType": "subagentStop", "success": true}, "id": "240e9d7a-1a91-4046-8fe8-bc8bc2f9428f", "timestamp": "2026-09-10T17:08:47.513Z", "parentId": "682ab836-6ad2-4f42-a703-4ca32438f826"} +{"type": "subagent.completed", "data": {"toolCallId": "call_qx4FH5DADTeT1qVLb37HNpBk", "agentName": "explore", "agentDisplayName": "sum-alpha-beta", "model": "gpt-5.6-luna", "firstDispatchedModel": "gpt-5.6-luna", "totalToolCalls": 2, "totalTokens": 9079, "durationMs": 3467}, "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "id": "0031b6bf-9dda-49d2-944e-841635f8533f", "timestamp": "2026-09-10T17:08:47.513Z", "parentId": "240e9d7a-1a91-4046-8fe8-bc8bc2f9428f"} +{"type": "hook.start", "data": {"hookInvocationId": "79f3e676-fffe-4d2a-8d85-5be8452ac9a9", "hookType": "postToolUse", "input": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "timestamp": 1789060127513, "cwd": "/fixture/workspace", "toolName": "task", "toolArgs": {"description": "Sum two text files", "prompt": "Read only /fixture/workspace/alpha.txt and /fixture/workspace/beta.txt. Do not modify files. Do not access any other files or services. Report the sum of the numeric values in those two files.", "agent_type": "explore", "name": "sum-alpha-beta", "mode": "sync"}, "toolResult": {"textResultForLlm": "42", "resultType": "success"}}}, "id": "defc01ae-e8a1-4dba-b36c-fe1ba64655c1", "timestamp": "2026-09-10T17:08:47.513Z", "parentId": "0031b6bf-9dda-49d2-944e-841635f8533f"} +{"type": "hook.end", "data": {"hookInvocationId": "79f3e676-fffe-4d2a-8d85-5be8452ac9a9", "hookType": "postToolUse", "success": true}, "id": "ac9bcb4f-e1b6-45d3-800c-4e3887092454", "timestamp": "2026-09-10T17:08:47.535Z", "parentId": "defc01ae-e8a1-4dba-b36c-fe1ba64655c1"} +{"type": "tool.execution_complete", "data": {"toolCallId": "call_qx4FH5DADTeT1qVLb37HNpBk", "model": "gpt-5.6-luna", "interactionId": "793d3703-6f4a-4814-8877-34a7325848ce", "turnId": "0", "rte": true, "success": true, "result": {"content": "42", "detailedContent": "42"}}, "id": "e0f6fb2a-cef8-445a-a482-01f8230aab60", "timestamp": "2026-09-10T17:08:47.536Z", "parentId": "ac9bcb4f-e1b6-45d3-800c-4e3887092454"} +{"type": "assistant.turn_end", "data": {"turnId": "0"}, "id": "28e600a8-0102-43b9-b8ca-f37979506b4b", "timestamp": "2026-09-10T17:08:47.536Z", "parentId": "e0f6fb2a-cef8-445a-a482-01f8230aab60"} +{"type": "assistant.turn_start", "data": {"turnId": "1", "interactionId": "793d3703-6f4a-4814-8877-34a7325848ce"}, "id": "9bbdf1d2-97de-4f78-955b-38618edc2c3f", "timestamp": "2026-09-10T17:08:47.537Z", "parentId": "28e600a8-0102-43b9-b8ca-f37979506b4b"} +{"type": "assistant.message", "data": {"messageId": "bd86f77e-c00c-432a-9184-4660aa0e4bb9", "model": "gpt-5.6-luna", "content": "42", "toolRequests": [], "interactionId": "793d3703-6f4a-4814-8877-34a7325848ce", "turnId": "1", "phase": "final_answer", "rte": true}, "id": "10001566-2704-4f75-add3-2c044373eaba", "timestamp": "2026-09-10T17:08:48.284Z", "parentId": "9bbdf1d2-97de-4f78-955b-38618edc2c3f"} +{"type": "assistant.turn_end", "data": {"turnId": "1"}, "id": "78003d8e-7ea7-48eb-9cf0-aa13043da942", "timestamp": "2026-09-10T17:08:48.287Z", "parentId": "10001566-2704-4f75-add3-2c044373eaba"} +{"type": "hook.start", "data": {"hookInvocationId": "7e0d22f3-5b10-4e1e-88bc-d102c12e022f", "hookType": "agentStop", "input": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "transcriptPath": "/home/tester/.copilot/session-state/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/events.jsonl", "stopReason": "end_turn", "stop_hook_active": false, "timestamp": 1789060128288, "cwd": "/fixture/workspace"}}, "id": "c56043b3-ad8e-4d9a-b160-0b55485a4fb5", "timestamp": "2026-09-10T17:08:48.288Z", "parentId": "78003d8e-7ea7-48eb-9cf0-aa13043da942"} +{"type": "hook.end", "data": {"hookInvocationId": "7e0d22f3-5b10-4e1e-88bc-d102c12e022f", "hookType": "agentStop", "success": true}, "id": "286e55ad-fcfd-4aea-ae98-f2d62206891f", "timestamp": "2026-09-10T17:08:48.319Z", "parentId": "c56043b3-ad8e-4d9a-b160-0b55485a4fb5"} +{"type": "hook.start", "data": {"hookInvocationId": "d677f404-2c71-4c9c-bcd3-5f54fe995d5c", "hookType": "sessionEnd", "input": {"reason": "user_exit", "sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "timestamp": 1789060147875, "cwd": "/fixture/workspace"}}, "id": "c85630cc-3c7e-4f42-80b3-610284c8b9cc", "timestamp": "2026-09-10T17:09:07.875Z", "parentId": "79eec31d-d495-40fc-a31d-8a05f73fa8d3"} +{"type": "hook.end", "data": {"hookInvocationId": "d677f404-2c71-4c9c-bcd3-5f54fe995d5c", "hookType": "sessionEnd", "success": true}, "id": "f39853b5-344f-4582-95ec-75eb1da62165", "timestamp": "2026-09-10T17:09:07.904Z", "parentId": "c85630cc-3c7e-4f42-80b3-610284c8b9cc"} +{"type": "session.shutdown", "data": {"shutdownType": "routine", "totalPremiumRequests": 2, "totalNanoAiu": 373366000, "tokenDetails": {"input": {"tokenCount": 18}, "cache_read": {"tokenCount": 23948}, "cache_write": {"tokenCount": 11430}, "output": {"tokenCount": 328}}, "totalApiDurationMs": 8608, "sessionStartTime": 1789060085884, "eventsFileSizeBytes": 194432, "codeChanges": {"linesAdded": 0, "linesRemoved": 0, "filesModified": []}, "modelMetrics": {"gpt-5.6-luna": {"requests": {"count": 6, "cost": 2}, "usage": {"inputTokens": 35396, "outputTokens": 328, "cacheReadTokens": 23948, "cacheWriteTokens": 11430, "reasoningTokens": 55}, "totalNanoAiu": 373366000, "tokenDetails": {"input": {"tokenCount": 18}, "cache_read": {"tokenCount": 23948}, "cache_write": {"tokenCount": 11430}, "output": {"tokenCount": 328}}}}, "agentMetrics": {"main": {"totalApiDurationMs": 5825, "totalNanoAiu": 238814000.0, "modelMetrics": {"gpt-5.6-luna": {"requests": {"count": 4, "cost": 2.0}, "usage": {"inputTokens": 26416, "outputTokens": 229, "cacheReadTokens": 19522, "cacheWriteTokens": 6882, "reasoningTokens": 39}, "totalNanoAiu": 238814000.0, "tokenDetails": {"input": {"tokenCount": 12}, "cache_read": {"tokenCount": 19522}, "cache_write": {"tokenCount": 6882}, "output": {"tokenCount": 229}}}}}, "bf8cb9f3-2097-4db0-a3c8-78a2653b2106": {"agentName": "explore", "agentDisplayName": "sum-alpha-beta", "totalApiDurationMs": 2783, "totalNanoAiu": 134552000.0, "modelMetrics": {"gpt-5.6-luna": {"requests": {"count": 2, "cost": 0.0}, "usage": {"inputTokens": 8980, "outputTokens": 99, "cacheReadTokens": 4426, "cacheWriteTokens": 4548, "reasoningTokens": 16}, "totalNanoAiu": 134552000.0, "tokenDetails": {"input": {"tokenCount": 6}, "cache_read": {"tokenCount": 4426}, "cache_write": {"tokenCount": 4548}, "output": {"tokenCount": 99}}}}}}, "currentModel": "gpt-5.6-luna", "currentTokens": 6942, "systemTokens": 5015, "conversationTokens": 428, "toolDefinitionsTokens": 1496}, "id": "9dbdd079-e22a-4bd0-87e2-17fa2b246d23", "timestamp": "2026-09-10T17:09:07.907Z", "parentId": "f39853b5-344f-4582-95ec-75eb1da62165"} diff --git a/tests/fixtures/copilot/cli-1.0.83/hooks.jsonl b/tests/fixtures/copilot/cli-1.0.83/hooks.jsonl new file mode 100644 index 0000000..51c7658 --- /dev/null +++ b/tests/fixtures/copilot/cli-1.0.83/hooks.jsonl @@ -0,0 +1,20 @@ +{"registered_event": "userPromptSubmitted", "captured_ns": 1789060102200850000, "payload": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "timestamp": 1789060102173, "cwd": "/fixture/workspace", "prompt": "Read alpha.txt and beta.txt with separate view calls in parallel and report their sum. Only read those two files."}} +{"registered_event": "sessionStart", "captured_ns": 1789060102225028000, "payload": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "timestamp": 1789060102204, "cwd": "/fixture/workspace", "source": "new", "initialPrompt": "Read alpha.txt and beta.txt with separate view calls in parallel and report their sum. Only read those two files."}} +{"registered_event": "preToolUse", "captured_ns": 1789060104528610000, "payload": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "timestamp": 1789060104506, "cwd": "/fixture/workspace", "toolName": "view", "toolArgs": {"path": "/fixture/workspace/alpha.txt"}}} +{"registered_event": "preToolUse", "captured_ns": 1789060104549556000, "payload": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "timestamp": 1789060104506, "cwd": "/fixture/workspace", "toolName": "view", "toolArgs": {"path": "/fixture/workspace/beta.txt"}}} +{"registered_event": "postToolUse", "captured_ns": 1789060104570196000, "payload": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "timestamp": 1789060104552, "cwd": "/fixture/workspace", "toolName": "view", "toolArgs": {"path": "/fixture/workspace/alpha.txt"}, "toolResult": {"resultType": "success", "textResultForLlm": "alpha = 17\n"}}} +{"registered_event": "postToolUse", "captured_ns": 1789060104590707000, "payload": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "timestamp": 1789060104572, "cwd": "/fixture/workspace", "toolName": "view", "toolArgs": {"path": "/fixture/workspace/beta.txt"}, "toolResult": {"resultType": "success", "textResultForLlm": "beta = 25\n"}}} +{"registered_event": "agentStop", "captured_ns": 1789060105647636000, "payload": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "timestamp": 1789060105626, "cwd": "/fixture/workspace", "transcriptPath": "/home/tester/.copilot/session-state/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/events.jsonl", "stopReason": "end_turn", "stop_hook_active": false}} +{"registered_event": "userPromptSubmitted", "captured_ns": 1789060122189769000, "payload": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "timestamp": 1789060122166, "cwd": "/fixture/workspace", "prompt": "Invoke one explore subagent to read only alpha.txt and beta.txt and report their sum. Do not modify files or access other files or services. Then report its answer."}} +{"registered_event": "preToolUse", "captured_ns": 1789060124042792000, "payload": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "timestamp": 1789060124021, "cwd": "/fixture/workspace", "toolName": "task", "toolArgs": {"description": "Sum two text files", "prompt": "Read only /fixture/workspace/alpha.txt and /fixture/workspace/beta.txt. Do not modify files. Do not access any other files or services. Report the sum of the numeric values in those two files.", "agent_type": "explore", "name": "sum-alpha-beta", "mode": "sync"}}} +{"registered_event": "subagentStart", "captured_ns": 1789060124100211000, "payload": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "timestamp": 1789060124082, "cwd": "/fixture/workspace", "transcriptPath": "/home/tester/.copilot/session-state/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/events.jsonl", "agentName": "explore"}} +{"registered_event": "userPromptSubmitted", "captured_ns": 1789060124554272000, "payload": {"sessionId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "timestamp": 1789060124530, "cwd": "/fixture/workspace", "prompt": "Read only /fixture/workspace/alpha.txt and /fixture/workspace/beta.txt. Do not modify files. Do not access any other files or services. Report the sum of the numeric values in those two files."}} +{"registered_event": "preToolUse", "captured_ns": 1789060126531610000, "payload": {"sessionId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "timestamp": 1789060126501, "cwd": "/fixture/workspace", "toolName": "view", "toolArgs": {"path": "/fixture/workspace/alpha.txt"}}} +{"registered_event": "preToolUse", "captured_ns": 1789060126555790000, "payload": {"sessionId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "timestamp": 1789060126501, "cwd": "/fixture/workspace", "toolName": "view", "toolArgs": {"path": "/fixture/workspace/beta.txt"}}} +{"registered_event": "postToolUse", "captured_ns": 1789060126578392000, "payload": {"sessionId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "timestamp": 1789060126559, "cwd": "/fixture/workspace", "toolName": "view", "toolArgs": {"path": "/fixture/workspace/alpha.txt"}, "toolResult": {"resultType": "success", "textResultForLlm": "alpha = 17\n"}}} +{"registered_event": "postToolUse", "captured_ns": 1789060126599701000, "payload": {"sessionId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "timestamp": 1789060126581, "cwd": "/fixture/workspace", "toolName": "view", "toolArgs": {"path": "/fixture/workspace/beta.txt"}, "toolResult": {"resultType": "success", "textResultForLlm": "beta = 25\n"}}} +{"registered_event": "agentStop", "captured_ns": 1789060127486265000, "payload": {"sessionId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "timestamp": 1789060127456, "cwd": "/fixture/workspace", "transcriptPath": "/home/tester/.copilot/session-state/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/events.jsonl", "stopReason": "end_turn", "stop_hook_active": false}} +{"registered_event": "subagentStop", "captured_ns": 1789060127510837000, "payload": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "timestamp": 1789060127489, "cwd": "/fixture/workspace", "transcriptPath": "/home/tester/.copilot/session-state/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/events.jsonl", "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "agentType": "explore", "agentName": "explore", "response": "42", "stopReason": "end_turn"}} +{"registered_event": "postToolUse", "captured_ns": 1789060127533513000, "payload": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "timestamp": 1789060127513, "cwd": "/fixture/workspace", "toolName": "task", "toolArgs": {"description": "Sum two text files", "prompt": "Read only /fixture/workspace/alpha.txt and /fixture/workspace/beta.txt. Do not modify files. Do not access any other files or services. Report the sum of the numeric values in those two files.", "agent_type": "explore", "name": "sum-alpha-beta", "mode": "sync"}, "toolResult": {"resultType": "success", "textResultForLlm": "42"}}} +{"registered_event": "agentStop", "captured_ns": 1789060128316253000, "payload": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "timestamp": 1789060128288, "cwd": "/fixture/workspace", "transcriptPath": "/home/tester/.copilot/session-state/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/events.jsonl", "stopReason": "end_turn", "stop_hook_active": false}} +{"registered_event": "sessionEnd", "captured_ns": 1789060147901644000, "payload": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "timestamp": 1789060147875, "cwd": "/fixture/workspace", "reason": "user_exit"}} diff --git a/tests/fixtures/copilot/cli-1.0.83/usage.json b/tests/fixtures/copilot/cli-1.0.83/usage.json new file mode 100644 index 0000000..2ea30a9 --- /dev/null +++ b/tests/fixtures/copilot/cli-1.0.83/usage.json @@ -0,0 +1,134 @@ +{ + "shutdownType": "routine", + "totalPremiumRequests": 2, + "totalNanoAiu": 373366000, + "tokenDetails": { + "input": { + "tokenCount": 18 + }, + "cache_read": { + "tokenCount": 23948 + }, + "cache_write": { + "tokenCount": 11430 + }, + "output": { + "tokenCount": 328 + } + }, + "totalApiDurationMs": 8608, + "sessionStartTime": 1789060085884, + "eventsFileSizeBytes": 194432, + "codeChanges": { + "linesAdded": 0, + "linesRemoved": 0, + "filesModified": [] + }, + "modelMetrics": { + "gpt-5.6-luna": { + "requests": { + "count": 6, + "cost": 2 + }, + "usage": { + "inputTokens": 35396, + "outputTokens": 328, + "cacheReadTokens": 23948, + "cacheWriteTokens": 11430, + "reasoningTokens": 55 + }, + "totalNanoAiu": 373366000, + "tokenDetails": { + "input": { + "tokenCount": 18 + }, + "cache_read": { + "tokenCount": 23948 + }, + "cache_write": { + "tokenCount": 11430 + }, + "output": { + "tokenCount": 328 + } + } + } + }, + "agentMetrics": { + "main": { + "totalApiDurationMs": 5825, + "totalNanoAiu": 238814000.0, + "modelMetrics": { + "gpt-5.6-luna": { + "requests": { + "count": 4, + "cost": 2.0 + }, + "usage": { + "inputTokens": 26416, + "outputTokens": 229, + "cacheReadTokens": 19522, + "cacheWriteTokens": 6882, + "reasoningTokens": 39 + }, + "totalNanoAiu": 238814000.0, + "tokenDetails": { + "input": { + "tokenCount": 12 + }, + "cache_read": { + "tokenCount": 19522 + }, + "cache_write": { + "tokenCount": 6882 + }, + "output": { + "tokenCount": 229 + } + } + } + } + }, + "bf8cb9f3-2097-4db0-a3c8-78a2653b2106": { + "agentName": "explore", + "agentDisplayName": "sum-alpha-beta", + "totalApiDurationMs": 2783, + "totalNanoAiu": 134552000.0, + "modelMetrics": { + "gpt-5.6-luna": { + "requests": { + "count": 2, + "cost": 0.0 + }, + "usage": { + "inputTokens": 8980, + "outputTokens": 99, + "cacheReadTokens": 4426, + "cacheWriteTokens": 4548, + "reasoningTokens": 16 + }, + "totalNanoAiu": 134552000.0, + "tokenDetails": { + "input": { + "tokenCount": 6 + }, + "cache_read": { + "tokenCount": 4426 + }, + "cache_write": { + "tokenCount": 4548 + }, + "output": { + "tokenCount": 99 + } + } + } + } + } + }, + "currentModel": "gpt-5.6-luna", + "currentTokens": 6942, + "systemTokens": 5015, + "conversationTokens": 428, + "toolDefinitionsTokens": 1496 +} diff --git a/tests/fixtures/copilot/v1-cases/README.md b/tests/fixtures/copilot/v1-cases/README.md new file mode 100644 index 0000000..0227680 --- /dev/null +++ b/tests/fixtures/copilot/v1-cases/README.md @@ -0,0 +1,40 @@ +# Copilot V1 synthetic contract cases + +Every file in this directory is synthetic. It documents capture contracts and +reader/archive edge cases; it is not evidence of Copilot CLI behavior. + +`source-slice.json` and `source-batch.json` are small, JSON-round-trippable +values for independent reader, archive, and installer tests. They include the +full source-record envelope with schema version `1` in each event payload. + +`trailing-json.jsonl` ends with an incomplete JSON record. `trailing-utf8.hex` +is the bytes of a JSONL file whose final UTF-8 code point is incomplete; tests +that need raw bytes should decode the hex, rather than silently converting it +to replacement characters. A reader must defer both tails until a later read. + +`unknown-event-fields.jsonl` requires lossless retention of fields that V1 does +not interpret. `database-row-revisions.json` represents a row-ID reuse and a +changed row revision: equal logical snapshots deduplicate, changed content is a +new evidence record. `distinct-hook-observations.json` contains identical +payloads with distinct observation IDs; both records must survive. +`missing-event-id.jsonl` requires a locator built from file generation, byte +location, and content digest rather than an invented native event ID. + +The observed `cli-1.0.83` sibling fixture has two main prompts and one child +prompt. The child prompt is retained as source evidence, not asserted to be a +third human request. Its six `assistant_usage_events` rows remain raw database +evidence and must not create UsageStore rows. + +## Identity and revision handoff + +The selected source home is canonicalized and SHA-256 hashed in full. Stored +session IDs use only the first 16 digest characters for readability, but the +full digest must be validated against the canonical home before reuse. Hence +two homes cannot merge merely because their native session IDs match. + +Transcript identity is source key + native session + native event ID. Without +an event ID it is file generation + byte location + content digest and is +explicitly lower confidence. Database identity is table + row primary key + +content revision; its generation remains separate to expose replacement or +row-ID reuse. Hook identity is a generated observation ID: same-content hooks +are independent observations, while rereading one spool entry is idempotent. diff --git a/tests/fixtures/copilot/v1-cases/database-row-revisions.json b/tests/fixtures/copilot/v1-cases/database-row-revisions.json new file mode 100644 index 0000000..c287274 --- /dev/null +++ b/tests/fixtures/copilot/v1-cases/database-row-revisions.json @@ -0,0 +1,5 @@ +[ + {"table":"turns","primary_key":7,"generation":"db-generation-a","content_revision":"sha256:aaa","row":{"id":7,"session_id":"session-a","content":"first"}}, + {"table":"turns","primary_key":7,"generation":"db-generation-a","content_revision":"sha256:bbb","row":{"id":7,"session_id":"session-a","content":"updated"}}, + {"table":"turns","primary_key":7,"generation":"db-generation-b","content_revision":"sha256:ccc","row":{"id":7,"session_id":"session-a","content":"reused after replacement"}} +] diff --git a/tests/fixtures/copilot/v1-cases/distinct-hook-observations.json b/tests/fixtures/copilot/v1-cases/distinct-hook-observations.json new file mode 100644 index 0000000..614ebeb --- /dev/null +++ b/tests/fixtures/copilot/v1-cases/distinct-hook-observations.json @@ -0,0 +1,4 @@ +[ + {"observation_id":"hook-observation-1","event":"agentStop","payload":{"sessionId":"session-a","stopReason":"end_turn"}}, + {"observation_id":"hook-observation-2","event":"agentStop","payload":{"sessionId":"session-a","stopReason":"end_turn"}} +] diff --git a/tests/fixtures/copilot/v1-cases/missing-event-id.jsonl b/tests/fixtures/copilot/v1-cases/missing-event-id.jsonl new file mode 100644 index 0000000..ae7f897 --- /dev/null +++ b/tests/fixtures/copilot/v1-cases/missing-event-id.jsonl @@ -0,0 +1 @@ +{"type":"assistant.message","timestamp":"2026-09-10T17:08:24.000Z","data":{"content":"no native ID"}} diff --git a/tests/fixtures/copilot/v1-cases/source-batch.json b/tests/fixtures/copilot/v1-cases/source-batch.json new file mode 100644 index 0000000..94c1db4 --- /dev/null +++ b/tests/fixtures/copilot/v1-cases/source-batch.json @@ -0,0 +1,19 @@ +{ + "source_key": "3c27ee5a0430219b982434b040245491be12dc57e270dcd2461e65d3a1f49839", + "native_session_id": "session-a", + "cwd": "/fixture/workspace", + "records": [ + { + "source_id": "3c27ee5a0430219b982434b040245491be12dc57e270dcd2461e65d3a1f49839/session-a/row-7/revision-a", + "source_kind": "database", + "native_session_id": "session-a", + "ts": null, + "observed_at": "2026-09-10T17:08:25.000Z", + "payload": {"schema_version": 1, "table": "assistant_usage_events", "row": {"id": 7}}, + "locator": {"table": "assistant_usage_events", "primary_key": 7, "content_revision": "revision-a"} + } + ], + "next_cursor": {"database_revision": "fixture-revision-1"}, + "diagnostics": [], + "exhausted": true +} diff --git a/tests/fixtures/copilot/v1-cases/source-slice.json b/tests/fixtures/copilot/v1-cases/source-slice.json new file mode 100644 index 0000000..1fb1c36 --- /dev/null +++ b/tests/fixtures/copilot/v1-cases/source-slice.json @@ -0,0 +1,17 @@ +{ + "records": [ + { + "source_id": "3c27ee5a0430219b982434b040245491be12dc57e270dcd2461e65d3a1f49839/session-a/event-1", + "source_kind": "transcript", + "native_session_id": "session-a", + "ts": "2026-09-10T17:08:24.000Z", + "observed_at": "2026-09-10T17:08:25.000Z", + "payload": {"schema_version": 1, "type": "user.message", "data": {"text": "fixture"}}, + "locator": {"file_generation": "generation-1", "byte_offset": 0, "native_event_id": "event-1"} + } + ], + "next_cursor": {"byte_offset": 128, "file_generation": "generation-1"}, + "diagnostics": [], + "cwd": "/fixture/workspace", + "exhausted": false +} diff --git a/tests/fixtures/copilot/v1-cases/trailing-json.jsonl b/tests/fixtures/copilot/v1-cases/trailing-json.jsonl new file mode 100644 index 0000000..40f7e3e --- /dev/null +++ b/tests/fixtures/copilot/v1-cases/trailing-json.jsonl @@ -0,0 +1,2 @@ +{"id":"complete-1","type":"known"} +{"id":"incomplete-2","type":"unterminated" diff --git a/tests/fixtures/copilot/v1-cases/trailing-utf8.hex b/tests/fixtures/copilot/v1-cases/trailing-utf8.hex new file mode 100644 index 0000000..f311deb --- /dev/null +++ b/tests/fixtures/copilot/v1-cases/trailing-utf8.hex @@ -0,0 +1 @@ +7b226964223a22636f6d706c6574652d31227d0a7b226964223a22747261696c696e672d75746638222c2274657874223a22e282 diff --git a/tests/fixtures/copilot/v1-cases/unknown-event-fields.jsonl b/tests/fixtures/copilot/v1-cases/unknown-event-fields.jsonl new file mode 100644 index 0000000..5eadbcd --- /dev/null +++ b/tests/fixtures/copilot/v1-cases/unknown-event-fields.jsonl @@ -0,0 +1 @@ +{"id":"unknown-1","type":"future.event","timestamp":"2026-09-10T17:08:24.000Z","data":{"future_field":{"nested":[1,true,{"opaque":"retain"}]},"known":"also retain"},"top_level_future":"retain"} From 14d7bbbfc31e4e32f2a1ce3d98a6501490715c21 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Thu, 10 Sep 2026 13:57:04 -0700 Subject: [PATCH 02/88] Add comprehensive Copilot capture contract tests. Freeze identity validation, schema version constants, JSON round-trips, and fixture semantics for parallel V1 consumers. Co-authored-by: Cursor --- tests/test_copilot_contracts.py | 435 ++++++++++++++++++++++++++++++++ 1 file changed, 435 insertions(+) create mode 100644 tests/test_copilot_contracts.py diff --git a/tests/test_copilot_contracts.py b/tests/test_copilot_contracts.py new file mode 100644 index 0000000..66ae6fd --- /dev/null +++ b/tests/test_copilot_contracts.py @@ -0,0 +1,435 @@ +"""Frozen contract tests for Copilot V1 capture types and identity.""" + +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +from typing import Any + +import pytest + +import thirdeye.platforms.copilot as copilot_pkg +from thirdeye.platforms.copilot.constants import ( + CLI_HOOK_EVENT_ALIASES, + CLI_HOOK_EVENTS, + COPILOT_HOME_ENV, + DISPLAY_NAME, + EVENT_ENVELOPE_VERSION, + HOOKS_DIRECTORY_NAME, + OWNED_HOOK_FILENAME, + PLATFORM_NAME, + SCHEMA_VERSION, + SOURCE_RECORD_SCHEMA_VERSION, + SOURCE_SCHEMA_VERSION, +) +from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id, validate_native_id +from thirdeye.platforms.copilot.types import ( + SCHEMA_VERSION as TYPES_SCHEMA_VERSION, + SourceBatch, + SourcePaths, + SourceRecord, + SourceSlice, + SyncResult, +) + +FIXTURES = Path(__file__).parent / "fixtures" / "copilot" +CLI_FIXTURE = FIXTURES / "cli-1.0.83" +V1_CASES = FIXTURES / "v1-cases" + +NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" + + +def _load_json(path: Path) -> Any: + return json.loads(path.read_text(encoding="utf-8")) + + +def _round_trip(value: dict[str, Any]) -> dict[str, Any]: + return json.loads(json.dumps(value)) + + +# --- constants --- + + +def test_platform_identity_constants(): + assert PLATFORM_NAME == "copilot" + assert DISPLAY_NAME == "GitHub Copilot CLI" + + +def test_schema_version_is_frozen_at_one(): + for version in ( + SOURCE_SCHEMA_VERSION, + SCHEMA_VERSION, + SOURCE_RECORD_SCHEMA_VERSION, + EVENT_ENVELOPE_VERSION, + TYPES_SCHEMA_VERSION, + ): + assert version == 1 + + +def test_constants_module_has_no_resolved_paths(): + import thirdeye.platforms.copilot.constants as constants + + for name in dir(constants): + if name.startswith("_"): + continue + value = getattr(constants, name) + if isinstance(value, str) and ("/" in value or "\\" in value): + pytest.fail(f"constants.{name} looks like a resolved path: {value!r}") + + +def test_hook_install_vocabulary(): + assert HOOKS_DIRECTORY_NAME == "hooks" + assert OWNED_HOOK_FILENAME == "thirdeye.json" + assert COPILOT_HOME_ENV == "COPILOT_HOME" + + +def test_cli_hook_aliases_cover_observed_events(): + expected = { + "sessionStart", + "userPromptSubmitted", + "preToolUse", + "postToolUse", + "agentStop", + "subagentStart", + "subagentStop", + "sessionEnd", + } + assert expected <= set(CLI_HOOK_EVENT_ALIASES) + assert CLI_HOOK_EVENTS == tuple(CLI_HOOK_EVENT_ALIASES) + + +def test_hook_aliases_map_to_snake_case(): + for camel, snake in CLI_HOOK_EVENT_ALIASES.items(): + assert camel[0].islower() + assert "_" in snake + assert snake == snake.lower() + + +# --- package exports --- + + +def test_public_exports_match_contract_surface(): + assert set(copilot_pkg.__all__) == { + "SourceBatch", + "SourcePaths", + "SourceRecord", + "SourceSlice", + "SyncResult", + "resolve_sources", + "stored_session_id", + "validate_native_id", + } + + +def test_copilot_package_does_not_import_usage_store(): + import thirdeye.platforms.copilot.identity as identity + import thirdeye.platforms.copilot.types as types + + for module in (copilot_pkg, identity, types): + source = Path(module.__file__).read_text(encoding="utf-8") + assert "UsageStore" not in source + assert "usage_store" not in source + + +# --- TypedDict / JSON round-trip fixtures --- + + +def test_source_slice_fixture_round_trips(): + raw = _load_json(V1_CASES / "source-slice.json") + restored = _round_trip(raw) + assert restored == raw + + slice_: SourceSlice = restored + assert slice_["exhausted"] is False + assert len(slice_["records"]) == 1 + record = slice_["records"][0] + assert record["source_kind"] == "transcript" + assert record["payload"]["schema_version"] == 1 + assert record["ts"] == "2026-09-10T17:08:24.000Z" + assert record["observed_at"] == "2026-09-10T17:08:25.000Z" + + +def test_source_batch_fixture_round_trips(): + raw = _load_json(V1_CASES / "source-batch.json") + restored = _round_trip(raw) + assert restored == raw + + batch: SourceBatch = {key: restored[key] for key in SourceBatch.__annotations__} + assert batch["source_key"] == restored["source_key"] + assert len(batch["records"]) == 1 + record = batch["records"][0] + assert record["source_kind"] == "database" + assert record["payload"]["schema_version"] == 1 + assert record["ts"] is None + + +def test_sync_result_shape_accepts_zero_counts(): + result: SyncResult = { + "sessions": 0, + "records_written": 0, + "duplicate_records": 0, + "pending": 0, + "errors": 0, + } + assert _round_trip(dict(result)) == result + + +def test_source_record_optional_ts_null_round_trips(): + record: SourceRecord = { + "source_id": "key/session-a/row-1/rev-a", + "source_kind": "database", + "native_session_id": "session-a", + "ts": None, + "observed_at": "2026-09-10T17:08:25.000Z", + "payload": {"schema_version": 1, "table": "turns", "row": {"id": 1}}, + "locator": {"table": "turns", "primary_key": 1, "content_revision": "rev-a"}, + } + restored = _round_trip(dict(record)) + assert restored["ts"] is None + assert restored["payload"]["row"]["id"] == 1 + + +# --- identity: resolve_sources --- + + +def test_resolve_sources_uses_explicit_home(tmp_path: Path): + home = tmp_path / "copilot-home" + home.mkdir() + paths = resolve_sources(home) + + assert paths["home"] == str(home.resolve()) + assert paths["session_root"] == str((home / "session-state").resolve()) + assert paths["database"] == str((home / "session-store.db").resolve()) + assert len(paths["source_key"]) == 64 + + +def test_resolve_sources_honors_copilot_home_env(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + home = tmp_path / "from-env" + home.mkdir() + monkeypatch.setenv(COPILOT_HOME_ENV, str(home)) + paths = resolve_sources() + assert paths["home"] == str(home.resolve()) + + +def test_source_key_is_full_sha256_of_canonical_home(tmp_path: Path): + home = tmp_path / "canonical" + home.mkdir() + paths = resolve_sources(home) + normalized = os.path.normcase(str(home.resolve())) + expected = hashlib.sha256(normalized.encode("utf-8")).hexdigest() + assert paths["source_key"] == expected + + +def test_aliases_to_same_canonical_home_share_source_key(tmp_path: Path): + home = tmp_path / "alias-target" + home.mkdir() + via_dot = tmp_path / "alias-target" / ".." / "alias-target" + assert resolve_sources(home)["source_key"] == resolve_sources(via_dot)["source_key"] + + +def test_forged_source_key_is_rejected(tmp_path: Path): + home = tmp_path / "copilot-home" + home.mkdir() + paths = resolve_sources(home) + forged: SourcePaths = dict(paths) + forged["source_key"] = "0" * 64 + with pytest.raises(ValueError, match="source_key does not match"): + stored_session_id(forged, "session-a") + + +def test_session_root_outside_home_is_rejected(tmp_path: Path): + home = tmp_path / "copilot-home" + outside = tmp_path / "outside" + home.mkdir() + outside.mkdir() + paths = resolve_sources(home) + forged: SourcePaths = dict(paths) + forged["session_root"] = str(outside.resolve()) + with pytest.raises(ValueError, match="session_root escapes"): + stored_session_id(forged, "session-a") + + +# --- identity: validate_native_id --- + + +@pytest.mark.parametrize( + "native_id", + [ + "", + " ", + " leading", + "trailing ", + ".", + "..", + "../escape", + "..\\escape", + "bad/id", + "bad\\id", + "bad:id", + "bad\x00id", + "bad\nid", + ], +) +def test_validate_native_id_rejects_unsafe_values(native_id: str): + with pytest.raises(ValueError): + validate_native_id(native_id) + + +def test_validate_native_id_accepts_uuid_like_ids(): + validate_native_id(NATIVE_SESSION_ID) + + +# --- identity: stored_session_id --- + + +def test_stored_session_id_format(tmp_path: Path): + (tmp_path / "home").mkdir() + paths = resolve_sources(tmp_path / "home") + native = "session-a" + stored = stored_session_id(paths, native) + assert stored == f"copilot-{paths['source_key'][:16]}-{native}" + + +def test_different_homes_do_not_merge_same_native_id(tmp_path: Path): + home_a = tmp_path / "home-a" + home_b = tmp_path / "home-b" + home_a.mkdir() + home_b.mkdir() + native = "shared-native-id" + id_a = stored_session_id(resolve_sources(home_a), native) + id_b = stored_session_id(resolve_sources(home_b), native) + assert id_a != id_b + + +def test_stored_session_id_validates_full_source_key_not_prefix_only(tmp_path: Path): + home = tmp_path / "home" + home.mkdir() + paths = resolve_sources(home) + other = resolve_sources(tmp_path / "other-home") + (tmp_path / "other-home").mkdir() + if paths["source_key"][:16] == other["source_key"][:16]: + pytest.skip("need distinct 16-char prefixes for this collision test") + forged: SourcePaths = dict(paths) + forged["source_key"] = other["source_key"] + with pytest.raises(ValueError, match="source_key does not match"): + stored_session_id(forged, "session-a") + + +# --- observed cli-1.0.83 fixtures --- + + +def test_cli_fixture_files_exist(): + for name in ( + "README.md", + "events.jsonl", + "hooks.jsonl", + "usage.json", + "assistant-usage-events.json", + ): + assert (CLI_FIXTURE / name).is_file() + + +def test_cli_transcript_retains_seventy_six_events(): + lines = (CLI_FIXTURE / "events.jsonl").read_text(encoding="utf-8").splitlines() + assert len(lines) == 76 + for line in lines: + event = json.loads(line) + assert "type" in event + + +def test_cli_external_hooks_count_twenty(): + lines = (CLI_FIXTURE / "hooks.jsonl").read_text(encoding="utf-8").splitlines() + assert len(lines) == 20 + + +def test_cli_fixture_has_two_main_prompts_and_one_child_prompt(): + events = [ + json.loads(line) + for line in (CLI_FIXTURE / "events.jsonl").read_text(encoding="utf-8").splitlines() + ] + user_messages = [event for event in events if event.get("type") == "user.message"] + assert len(user_messages) == 3 + + main_prompts = [ + event + for event in user_messages + if event.get("agentId") is None and event["data"].get("source") is None + ] + child_prompts = [event for event in user_messages if event.get("agentId") is not None] + assert len(main_prompts) == 2 + assert len(child_prompts) == 1 + + +def test_cli_assistant_usage_events_fixture_has_six_rows(): + rows = _load_json(CLI_FIXTURE / "assistant-usage-events.json") + assert len(rows) == 6 + assert {row["session_id"] for row in rows} == {NATIVE_SESSION_ID} + + +def test_cli_usage_totals_match_assistant_usage_events(): + usage = _load_json(CLI_FIXTURE / "usage.json") + rows = _load_json(CLI_FIXTURE / "assistant-usage-events.json") + + total_nano = sum(row["total_nano_aiu"] for row in rows) + assert total_nano == usage["totalNanoAiu"] + + for field, usage_key in ( + ("input_tokens", "inputTokens"), + ("output_tokens", "outputTokens"), + ("cache_read_tokens", "cacheReadTokens"), + ("cache_write_tokens", "cacheWriteTokens"), + ("reasoning_tokens", "reasoningTokens"), + ): + row_total = sum(row[field] for row in rows) + model_usage = usage["modelMetrics"]["gpt-5.6-luna"]["usage"] + assert row_total == model_usage[usage_key] + + +# --- synthetic v1-cases fixtures --- + + +def test_v1_trailing_json_fixture_has_complete_and_incomplete_tail(): + text = (V1_CASES / "trailing-json.jsonl").read_text(encoding="utf-8") + lines = text.splitlines() + assert len(lines) == 2 + json.loads(lines[0]) + with pytest.raises(json.JSONDecodeError): + json.loads(lines[1]) + + +def test_v1_trailing_utf8_hex_decodes_to_incomplete_tail(): + raw = bytes.fromhex((V1_CASES / "trailing-utf8.hex").read_text(encoding="utf-8").strip()) + complete_prefix, incomplete_tail = raw.split(b"\n", 1) + json.loads(complete_prefix.decode("utf-8")) + assert incomplete_tail.endswith(b"\xe2\x82") + with pytest.raises(UnicodeDecodeError): + incomplete_tail.decode("utf-8") + + +def test_v1_unknown_event_fields_fixture_is_lossless_json(): + line = (V1_CASES / "unknown-event-fields.jsonl").read_text(encoding="utf-8").strip() + event = json.loads(line) + assert event["data"]["future_field"]["nested"] == [1, True, {"opaque": "retain"}] + assert event["top_level_future"] == "retain" + + +def test_v1_database_row_revisions_documents_reuse_and_change(): + rows = _load_json(V1_CASES / "database-row-revisions.json") + assert len(rows) == 3 + assert rows[0]["primary_key"] == rows[1]["primary_key"] == rows[2]["primary_key"] + assert rows[0]["content_revision"] != rows[1]["content_revision"] + assert rows[1]["generation"] != rows[2]["generation"] + + +def test_v1_distinct_hook_observations_have_unique_ids_same_payload(): + observations = _load_json(V1_CASES / "distinct-hook-observations.json") + assert len(observations) == 2 + assert observations[0]["payload"] == observations[1]["payload"] + assert observations[0]["observation_id"] != observations[1]["observation_id"] + + +def test_v1_missing_event_id_fixture_has_no_native_id(): + event = json.loads((V1_CASES / "missing-event-id.jsonl").read_text(encoding="utf-8").strip()) + assert "id" not in event From e9fd7687742254ba6f45f8e3789458d84813f161 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Thu, 10 Sep 2026 14:03:27 -0700 Subject: [PATCH 03/88] Fix Copilot capture contract review findings. Keep SourceBatch free of SourceSlice fields, document that child hook session IDs are agent IDs, and freeze prefix-collision detection as an archive metadata check. Co-authored-by: Cursor --- src/thirdeye/platforms/copilot/identity.py | 32 ++++++- tests/fixtures/copilot/cli-1.0.83/README.md | 2 +- tests/fixtures/copilot/v1-cases/README.md | 14 ++- .../copilot/v1-cases/source-batch.json | 3 +- .../v1-cases/source-key-prefix-collision.json | 17 ++++ tests/test_copilot_contracts.py | 93 +++++++++++++++++-- 6 files changed, 143 insertions(+), 18 deletions(-) create mode 100644 tests/fixtures/copilot/v1-cases/source-key-prefix-collision.json diff --git a/src/thirdeye/platforms/copilot/identity.py b/src/thirdeye/platforms/copilot/identity.py index c94873a..c22af3c 100644 --- a/src/thirdeye/platforms/copilot/identity.py +++ b/src/thirdeye/platforms/copilot/identity.py @@ -9,6 +9,11 @@ from .constants import COPILOT_HOME_ENV from .types import SourcePaths +# Stored session IDs keep the first 16 hex characters of the SHA-256 source key +# for path readability. The full 64-character digest remains the identity. +SOURCE_KEY_PREFIX_LEN = 16 +SOURCE_KEY_DIGEST_LEN = 64 + def _canonical_path(path: Path) -> Path: """Return a stable, absolute path without requiring the path to exist.""" @@ -88,11 +93,30 @@ def validate_native_id(native_id: str) -> None: def stored_session_id(paths: SourcePaths, native_id: str) -> str: """Return the stable thirdeye ID for a native ID within one source home. - The full source key is validated before using its display prefix. This - makes a prefix collision detectable instead of merging records from two - source homes. + The stored ID uses only :data:`SOURCE_KEY_PREFIX_LEN` hex characters of + the source key. This function checks that *one* ``SourcePaths`` value is + internally consistent: the full digest matches the canonical home, and + recording paths stay inside that home. That check does not detect two + legitimate homes whose SHA-256 digests share a prefix. Both would still + produce the same stored ID. On reuse the archive must compare the full + ``source_key`` retained in session metadata with the candidate home and + refuse to merge a prefix collision. """ _validate_paths(paths) validate_native_id(native_id) - return f"copilot-{paths['source_key'][:16]}-{native_id}" + return f"copilot-{paths['source_key'][:SOURCE_KEY_PREFIX_LEN]}-{native_id}" + + +def source_keys_share_stored_prefix(left: str, right: str) -> bool: + """Return True when two distinct full source keys would collide in stored IDs. + + Archive reuse must apply this comparison (or an equivalent full-key check + against retained metadata). :func:`stored_session_id` cannot: it never + sees the other home. + """ + + for key in (left, right): + if not isinstance(key, str) or len(key) != SOURCE_KEY_DIGEST_LEN: + raise ValueError("source_key must be a 64-character SHA-256 hex digest") + return left != right and left[:SOURCE_KEY_PREFIX_LEN] == right[:SOURCE_KEY_PREFIX_LEN] diff --git a/tests/fixtures/copilot/cli-1.0.83/README.md b/tests/fixtures/copilot/cli-1.0.83/README.md index 5a2fa42..a43a168 100644 --- a/tests/fixtures/copilot/cli-1.0.83/README.md +++ b/tests/fixtures/copilot/cli-1.0.83/README.md @@ -27,7 +27,7 @@ Sanitization replaces the local home/workspace paths and removes system messages - External pre/post tool hooks have no invocation ID in this run. The transcript provides `toolCallId` on requests and executions. Correlate parallel tools using the transcript, not just tool names. - `turnId` denotes a model/tool cycle and resets across user requests. Group user interactions using `interactionId` plus agent identity. - Subagent events are interleaved in the same transcript and carry top-level `agentId`; `subagent.started` links to the parent task via `data.toolCallId`. Child records also expose `parentToolCallId` where applicable. -- The child generated its own user-prompt and agent-stop hooks using the parent's session ID and transcript path. There are three prompt hooks and three stop hooks for two top-level user requests. External hook payloads alone cannot reliably distinguish these turns. +- The child generated its own user-prompt and agent-stop hooks. Those payloads set `sessionId` to the child agent ID (`bf8cb9f3-2097-4db0-a3c8-78a2653b2106`), not the parent session ID. Child pre/post tool hooks do the same. `transcriptPath` on the child's agentStop still points at the parent session's `events.jsonl`. `subagentStart`/`subagentStop` keep the parent session ID and name the child in `agentId`. Sanitization did not rewrite these identifiers; transcript `hook.start` input matches the external recorder. Hook `sessionId` is therefore not a native session ID for capture routing. Child prompt/stop hooks cannot drive a session-ID-only turn state machine. There are three prompt hooks and three stop hooks for two top-level user requests. - SessionStart arrived after the first user-prompt hook. Initialization must tolerate that ordering. - There are 20 external recorder files but only 18 hook.start/hook.end pairs in the final transcript. Do not assume those two streams have one-to-one coverage. - Assistant text and model names are present in assistant.message. Final session.shutdown includes input/output/cache/reasoning usage and per-agent metrics. Availability of per-call usage at Stop time has not been established by this probe. diff --git a/tests/fixtures/copilot/v1-cases/README.md b/tests/fixtures/copilot/v1-cases/README.md index 0227680..0de17e2 100644 --- a/tests/fixtures/copilot/v1-cases/README.md +++ b/tests/fixtures/copilot/v1-cases/README.md @@ -6,6 +6,7 @@ reader/archive edge cases; it is not evidence of Copilot CLI behavior. `source-slice.json` and `source-batch.json` are small, JSON-round-trippable values for independent reader, archive, and installer tests. They include the full source-record envelope with schema version `1` in each event payload. +`exhausted` belongs only to `SourceSlice`; `SourceBatch` must not carry it. `trailing-json.jsonl` ends with an incomplete JSON record. `trailing-utf8.hex` is the bytes of a JSONL file whose final UTF-8 code point is incomplete; tests @@ -19,6 +20,8 @@ new evidence record. `distinct-hook-observations.json` contains identical payloads with distinct observation IDs; both records must survive. `missing-event-id.jsonl` requires a locator built from file generation, byte location, and content digest rather than an invented native event ID. +`source-key-prefix-collision.json` is the archive-reuse contract for two +homes whose SHA-256 digests share a 16-character display prefix. The observed `cli-1.0.83` sibling fixture has two main prompts and one child prompt. The child prompt is retained as source evidence, not asserted to be a @@ -28,9 +31,14 @@ evidence and must not create UsageStore rows. ## Identity and revision handoff The selected source home is canonicalized and SHA-256 hashed in full. Stored -session IDs use only the first 16 digest characters for readability, but the -full digest must be validated against the canonical home before reuse. Hence -two homes cannot merge merely because their native session IDs match. +session IDs use only the first 16 digest characters for readability. +`stored_session_id` checks that one `SourcePaths` value is internally +consistent (full digest matches that home). Two legitimate homes whose +digests share a 16-character prefix still produce the same stored ID; see +`source-key-prefix-collision.json`. On reuse the archive must compare the +full `source_key` retained in session metadata with the candidate home and +refuse to merge a prefix collision. Distinct native IDs from different homes +must not merge either. Transcript identity is source key + native session + native event ID. Without an event ID it is file generation + byte location + content digest and is diff --git a/tests/fixtures/copilot/v1-cases/source-batch.json b/tests/fixtures/copilot/v1-cases/source-batch.json index 94c1db4..ab7ec41 100644 --- a/tests/fixtures/copilot/v1-cases/source-batch.json +++ b/tests/fixtures/copilot/v1-cases/source-batch.json @@ -14,6 +14,5 @@ } ], "next_cursor": {"database_revision": "fixture-revision-1"}, - "diagnostics": [], - "exhausted": true + "diagnostics": [] } diff --git a/tests/fixtures/copilot/v1-cases/source-key-prefix-collision.json b/tests/fixtures/copilot/v1-cases/source-key-prefix-collision.json new file mode 100644 index 0000000..3f9cd99 --- /dev/null +++ b/tests/fixtures/copilot/v1-cases/source-key-prefix-collision.json @@ -0,0 +1,17 @@ +{ + "label": "synthetic", + "native_session_id": "session-a", + "note": "Two distinct SHA-256 source keys that share a 16-character display prefix. stored_session_id would emit the same copilot-- value for both. Detecting the collision requires comparing the full source_key retained in session metadata on reuse; validating that one SourcePaths object is internally consistent is not sufficient.", + "homes": [ + { + "home": "/synthetic/copilot-home-a", + "source_key": "aaaaaaaaaaaaaaaa111111111111111111111111111111111111111111111111" + }, + { + "home": "/synthetic/copilot-home-b", + "source_key": "aaaaaaaaaaaaaaaa222222222222222222222222222222222222222222222222" + } + ], + "retained_metadata_source_key": "aaaaaaaaaaaaaaaa111111111111111111111111111111111111111111111111", + "colliding_stored_session_id": "copilot-aaaaaaaaaaaaaaaa-session-a" +} diff --git a/tests/test_copilot_contracts.py b/tests/test_copilot_contracts.py index 66ae6fd..9739f9a 100644 --- a/tests/test_copilot_contracts.py +++ b/tests/test_copilot_contracts.py @@ -24,9 +24,17 @@ SOURCE_RECORD_SCHEMA_VERSION, SOURCE_SCHEMA_VERSION, ) -from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id, validate_native_id +from thirdeye.platforms.copilot.identity import ( + SOURCE_KEY_PREFIX_LEN, + resolve_sources, + source_keys_share_stored_prefix, + stored_session_id, + validate_native_id, +) from thirdeye.platforms.copilot.types import ( SCHEMA_VERSION as TYPES_SCHEMA_VERSION, +) +from thirdeye.platforms.copilot.types import ( SourceBatch, SourcePaths, SourceRecord, @@ -39,6 +47,7 @@ V1_CASES = FIXTURES / "v1-cases" NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" +CHILD_AGENT_ID = "bf8cb9f3-2097-4db0-a3c8-78a2653b2106" def _load_json(path: Path) -> Any: @@ -140,6 +149,7 @@ def test_source_slice_fixture_round_trips(): raw = _load_json(V1_CASES / "source-slice.json") restored = _round_trip(raw) assert restored == raw + assert set(restored) == set(SourceSlice.__annotations__) slice_: SourceSlice = restored assert slice_["exhausted"] is False @@ -155,8 +165,10 @@ def test_source_batch_fixture_round_trips(): raw = _load_json(V1_CASES / "source-batch.json") restored = _round_trip(raw) assert restored == raw + assert set(restored) == set(SourceBatch.__annotations__) + assert "exhausted" not in restored - batch: SourceBatch = {key: restored[key] for key in SourceBatch.__annotations__} + batch: SourceBatch = restored assert batch["source_key"] == restored["source_key"] assert len(batch["records"]) == 1 record = batch["records"][0] @@ -289,7 +301,7 @@ def test_stored_session_id_format(tmp_path: Path): paths = resolve_sources(tmp_path / "home") native = "session-a" stored = stored_session_id(paths, native) - assert stored == f"copilot-{paths['source_key'][:16]}-{native}" + assert stored == f"copilot-{paths['source_key'][:SOURCE_KEY_PREFIX_LEN]}-{native}" def test_different_homes_do_not_merge_same_native_id(tmp_path: Path): @@ -303,20 +315,36 @@ def test_different_homes_do_not_merge_same_native_id(tmp_path: Path): assert id_a != id_b -def test_stored_session_id_validates_full_source_key_not_prefix_only(tmp_path: Path): +def test_stored_session_id_rejects_source_key_that_does_not_match_home(tmp_path: Path): home = tmp_path / "home" home.mkdir() paths = resolve_sources(home) - other = resolve_sources(tmp_path / "other-home") - (tmp_path / "other-home").mkdir() - if paths["source_key"][:16] == other["source_key"][:16]: - pytest.skip("need distinct 16-char prefixes for this collision test") + other_home = tmp_path / "other-home" + other_home.mkdir() + other = resolve_sources(other_home) forged: SourcePaths = dict(paths) forged["source_key"] = other["source_key"] with pytest.raises(ValueError, match="source_key does not match"): stored_session_id(forged, "session-a") +def test_source_key_prefix_collision_is_an_archive_reuse_contract(): + case = _load_json(V1_CASES / "source-key-prefix-collision.json") + first, second = case["homes"] + native = case["native_session_id"] + + assert first["source_key"] != second["source_key"] + assert source_keys_share_stored_prefix(first["source_key"], second["source_key"]) + stored_a = f"copilot-{first['source_key'][:SOURCE_KEY_PREFIX_LEN]}-{native}" + stored_b = f"copilot-{second['source_key'][:SOURCE_KEY_PREFIX_LEN]}-{native}" + assert stored_a == stored_b == case["colliding_stored_session_id"] + assert case["retained_metadata_source_key"] == first["source_key"] + assert second["source_key"] != case["retained_metadata_source_key"] + assert source_keys_share_stored_prefix(first["source_key"], first["source_key"]) is False + with pytest.raises(ValueError, match="64-character"): + source_keys_share_stored_prefix(first["source_key"], "too-short") + + # --- observed cli-1.0.83 fixtures --- @@ -360,6 +388,55 @@ def test_cli_fixture_has_two_main_prompts_and_one_child_prompt(): child_prompts = [event for event in user_messages if event.get("agentId") is not None] assert len(main_prompts) == 2 assert len(child_prompts) == 1 + assert child_prompts[0]["agentId"] == CHILD_AGENT_ID + + +def test_cli_child_prompt_stop_hooks_use_child_agent_id_as_session_id(): + hooks = [ + json.loads(line) + for line in (CLI_FIXTURE / "hooks.jsonl").read_text(encoding="utf-8").splitlines() + ] + events = [ + json.loads(line) + for line in (CLI_FIXTURE / "events.jsonl").read_text(encoding="utf-8").splitlines() + ] + + child_prompts = [ + hook + for hook in hooks + if hook["registered_event"] == "userPromptSubmitted" + and hook["payload"]["sessionId"] == CHILD_AGENT_ID + ] + child_stops = [ + hook + for hook in hooks + if hook["registered_event"] == "agentStop" and hook["payload"]["sessionId"] == CHILD_AGENT_ID + ] + assert len(child_prompts) == 1 + assert len(child_stops) == 1 + assert child_stops[0]["payload"]["transcriptPath"].endswith( + f"{NATIVE_SESSION_ID}/events.jsonl" + ) + + parent_lifecycle = [ + hook + for hook in hooks + if hook["registered_event"] in {"subagentStart", "subagentStop"} + ] + assert parent_lifecycle + assert all(hook["payload"]["sessionId"] == NATIVE_SESSION_ID for hook in parent_lifecycle) + + transcript_child_prompt = next( + event + for event in events + if event.get("type") == "hook.start" + and event["data"].get("hookType") == "userPromptSubmitted" + and event["data"]["input"]["sessionId"] == CHILD_AGENT_ID + ) + assert ( + transcript_child_prompt["data"]["input"]["sessionId"] + == child_prompts[0]["payload"]["sessionId"] + ) def test_cli_assistant_usage_events_fixture_has_six_rows(): From 0665860c420f2b30452f82ae6db43ce2ca036481 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Thu, 10 Sep 2026 14:23:13 -0700 Subject: [PATCH 04/88] Add comprehensive hook observations tests for Copilot V1. Cover parse_hook normalization, spool durability, and copilot provenance regressions so the hook observations task can be verified independently of ingestion. Co-authored-by: Cursor --- tests/test_copilot_hook_payload.py | 214 +++++++++++++++++++++++++++++ tests/test_copilot_spool.py | 207 ++++++++++++++++++++++++++++ tests/test_provenance.py | 42 ++++++ 3 files changed, 463 insertions(+) create mode 100644 tests/test_copilot_hook_payload.py create mode 100644 tests/test_copilot_spool.py diff --git a/tests/test_copilot_hook_payload.py b/tests/test_copilot_hook_payload.py new file mode 100644 index 0000000..1d08c15 --- /dev/null +++ b/tests/test_copilot_hook_payload.py @@ -0,0 +1,214 @@ +"""Behavioral tests for Copilot hook payload normalization.""" + +from __future__ import annotations + +import json +from copy import deepcopy +from pathlib import Path +from typing import Any + +import pytest + +from thirdeye.platforms.copilot.constants import ( + CLI_HOOK_EVENT_ALIASES, + SCHEMA_VERSION, + SOURCE_SCHEMA_VERSION, +) +from thirdeye.platforms.copilot.hook_payload import parse_hook +from thirdeye.platforms.copilot.types import SourceRecord + +FIXTURES = Path(__file__).parent / "fixtures" / "copilot" +CLI_HOOKS = FIXTURES / "cli-1.0.83" / "hooks.jsonl" +NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" +CHILD_AGENT_ID = "bf8cb9f3-2097-4db0-a3c8-78a2653b2106" +OBSERVED_AT = "2026-09-10T17:08:25.626Z" + +# PascalCase aliases accepted for input compatibility (Claude-style names). +PASCAL_CASE_ALIASES: dict[str, str] = { + "SessionStart": "sessionStart", + "UserPromptSubmit": "userPromptSubmitted", + "PreToolUse": "preToolUse", + "PostToolUse": "postToolUse", + "Stop": "agentStop", + "SubagentStart": "subagentStart", + "SubagentStop": "subagentStop", + "SessionEnd": "sessionEnd", +} + + +def _load_cli_hooks() -> list[dict[str, Any]]: + return [json.loads(line) for line in CLI_HOOKS.read_text(encoding="utf-8").splitlines()] + + +def _parse( + event: str, + payload: dict[str, Any], + *, + context: dict[str, Any] | None = None, + observation_id: str = "obs-test-1", + observed_at: str = OBSERVED_AT, +) -> SourceRecord: + return parse_hook( + event, + payload, + context or {}, + observed_at=observed_at, + observation_id=observation_id, + ) + + +def test_hook_payload_module_has_no_io_imports(): + source = Path(__import__("thirdeye.platforms.copilot.hook_payload").__file__).read_text( + encoding="utf-8" + ) + forbidden = ("subprocess", "sqlite3", "open(", "Path(", "os.remove", "shutil") + for token in forbidden: + assert token not in source, f"hook_payload.py must not use {token!r}" + + +@pytest.mark.parametrize("camel_event", CLI_HOOK_EVENT_ALIASES) +def test_parse_hook_accepts_native_camel_case_events(camel_event: str): + payload = { + "sessionId": NATIVE_SESSION_ID, + "timestamp": 1789060102204, + "cwd": "/fixture/workspace", + } + record = _parse(camel_event, payload, observation_id=f"obs-{camel_event}") + + assert record["source_kind"] == "hook" + assert record["native_session_id"] == NATIVE_SESSION_ID + assert record["observed_at"] == OBSERVED_AT + assert record["payload"]["schema_version"] == SCHEMA_VERSION + assert record["payload"]["hook_payload"] == payload + assert record["payload"]["event"] == camel_event + assert f"obs-{camel_event}" in record["source_id"] + + +@pytest.mark.parametrize(("pascal_event", "canonical_event"), PASCAL_CASE_ALIASES.items()) +def test_parse_hook_accepts_pascal_case_aliases(pascal_event: str, canonical_event: str): + payload = { + "sessionId": NATIVE_SESSION_ID, + "timestamp": 1789060102204, + "cwd": "/fixture/workspace", + } + record = _parse(pascal_event, payload) + + assert record["payload"]["event"] == canonical_event + + +def test_parse_hook_retains_unmodified_payload_from_cli_fixture(): + hook = next(entry for entry in _load_cli_hooks() if entry["registered_event"] == "preToolUse") + payload = deepcopy(hook["payload"]) + record = _parse(hook["registered_event"], payload, observation_id="obs-pre-tool") + + assert record["payload"]["hook_payload"] == hook["payload"] + assert record["payload"]["hook_payload"] is not payload + + +def test_parse_hook_does_not_mutate_input_payload_or_context(): + payload = { + "sessionId": NATIVE_SESSION_ID, + "timestamp": 1789060105626, + "nested": {"items": [1, 2]}, + } + context = {"env": {"WB_PLAN": "p"}, "unexpected": "strip-me"} + original_payload = deepcopy(payload) + original_context = deepcopy(context) + + _parse("agentStop", payload, context=context) + + assert payload == original_payload + assert context == original_context + + +def test_parse_hook_allowlists_context_and_drops_unknown_keys(): + context = { + "env": {"WB_PLAN": "session-trace"}, + "trace_id": "trace-abc", + "secret_token": "must-not-appear", + } + record = _parse("sessionStart", {"sessionId": NATIVE_SESSION_ID, "timestamp": 1}, context=context) + + stored = record["payload"]["context"] + assert stored["env"] == {"WB_PLAN": "session-trace"} + assert stored["trace_id"] == "trace-abc" + assert "secret_token" not in stored + + +@pytest.mark.parametrize( + "payload", + [ + {}, + {"timestamp": 1}, + {"sessionId": ""}, + {"sessionId": "../escape"}, + {"sessionId": "bad/id"}, + ], +) +def test_parse_hook_rejects_missing_or_invalid_session_routing(payload: dict[str, Any]): + with pytest.raises(ValueError): + _parse("sessionStart", payload) + + +def test_parse_hook_normalizes_millisecond_timestamp_to_iso(): + payload = {"sessionId": NATIVE_SESSION_ID, "timestamp": 1789060105626} + record = _parse("agentStop", payload) + + assert record["ts"] is not None + assert record["ts"].endswith("Z") or "+" in record["ts"] + + +def test_parse_hook_leaves_ts_none_when_timestamp_missing(): + payload = {"sessionId": NATIVE_SESSION_ID, "cwd": "/fixture/workspace"} + record = _parse("sessionEnd", payload) + + assert record["ts"] is None + + +def test_parse_hook_preserves_child_agent_stop_with_child_session_id(): + hook = next( + entry + for entry in _load_cli_hooks() + if entry["registered_event"] == "agentStop" + and entry["payload"]["sessionId"] == CHILD_AGENT_ID + ) + record = _parse(hook["registered_event"], hook["payload"], observation_id="obs-child-stop") + + assert record["native_session_id"] == CHILD_AGENT_ID + assert record["payload"]["hook_payload"]["sessionId"] == CHILD_AGENT_ID + + +def test_parse_hook_preserves_subagent_stop_on_parent_session_id(): + hook = next(entry for entry in _load_cli_hooks() if entry["registered_event"] == "subagentStop") + record = _parse(hook["registered_event"], hook["payload"], observation_id="obs-subagent-stop") + + assert record["native_session_id"] == NATIVE_SESSION_ID + assert record["payload"]["hook_payload"]["agentId"] == CHILD_AGENT_ID + + +def test_distinct_observation_ids_produce_distinct_source_ids(): + payload = {"sessionId": NATIVE_SESSION_ID, "timestamp": 1789060105626, "stopReason": "end_turn"} + first = _parse("agentStop", payload, observation_id="hook-observation-1") + second = _parse("agentStop", payload, observation_id="hook-observation-2") + + assert first["source_id"] != second["source_id"] + assert first["payload"]["hook_payload"] == second["payload"]["hook_payload"] + + +def test_source_record_envelope_uses_schema_version_one(): + record = _parse( + "sessionStart", + {"sessionId": NATIVE_SESSION_ID, "timestamp": 1789060102204}, + ) + assert record["payload"]["schema_version"] == SOURCE_SCHEMA_VERSION == 1 + + +def test_locator_identifies_observation(): + record = _parse( + "userPromptSubmitted", + {"sessionId": NATIVE_SESSION_ID, "timestamp": 1789060102173, "prompt": "hi"}, + observation_id="obs-locator", + ) + locator = record["locator"] + assert locator["observation_id"] == "obs-locator" + assert locator["event"] in {"userPromptSubmitted", "UserPromptSubmit"} diff --git a/tests/test_copilot_spool.py b/tests/test_copilot_spool.py new file mode 100644 index 0000000..69d5dc7 --- /dev/null +++ b/tests/test_copilot_spool.py @@ -0,0 +1,207 @@ +"""Behavioral tests for durable Copilot hook observation spooling.""" + +from __future__ import annotations + +import json +import subprocess +import sys +import threading +from pathlib import Path +from typing import Any + +import pytest + +from thirdeye.config import Config +from thirdeye.platforms.copilot.hook_payload import parse_hook +from thirdeye.platforms.copilot.identity import resolve_sources +from thirdeye.platforms.copilot.spool import ack_spool, enqueue_hook, read_spool +from thirdeye.platforms.copilot.types import SourceRecord, SourcePaths + +NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" +OBSERVED_AT = "2026-09-10T17:08:25.626Z" +_CONCURRENT_WORKERS = 8 +_CONCURRENT_RECORDS = 16 + + +@pytest.fixture +def copilot_env(tmp_path: Path) -> tuple[Config, SourcePaths]: + copilot_home = tmp_path / "copilot-home" + copilot_home.mkdir() + config = Config(root=tmp_path / "thirdeye") + paths = resolve_sources(copilot_home) + return config, paths + + +def _hook_record( + *, + observation_id: str, + event: str = "agentStop", + session_id: str = NATIVE_SESSION_ID, + stop_reason: str = "end_turn", +) -> SourceRecord: + return parse_hook( + event, + { + "sessionId": session_id, + "timestamp": 1789060105626, + "cwd": "/fixture/workspace", + "stopReason": stop_reason, + }, + {"env": {"WB_PLAN": "p"}}, + observed_at=OBSERVED_AT, + observation_id=observation_id, + ) + + +def test_enqueue_returns_spool_path_and_read_returns_complete_record(copilot_env: tuple[Config, SourcePaths]): + config, paths = copilot_env + record = _hook_record(observation_id="obs-enqueue-1") + + spool_path = enqueue_hook(config, paths, record) + assert Path(spool_path).is_file() + + records = read_spool(config, paths, NATIVE_SESSION_ID) + assert len(records) == 1 + assert records[0]["source_id"] == record["source_id"] + assert records[0]["payload"]["hook_payload"]["stopReason"] == "end_turn" + + +def test_same_content_distinct_observation_ids_remain_separate(copilot_env: tuple[Config, SourcePaths]): + config, paths = copilot_env + first = _hook_record(observation_id="hook-observation-1") + second = _hook_record(observation_id="hook-observation-2") + + enqueue_hook(config, paths, first) + enqueue_hook(config, paths, second) + + records = read_spool(config, paths, NATIVE_SESSION_ID) + assert len(records) == 2 + assert {record["source_id"] for record in records} == {first["source_id"], second["source_id"]} + + +def test_ack_spool_removes_only_committed_ids(copilot_env: tuple[Config, SourcePaths]): + config, paths = copilot_env + keep = _hook_record(observation_id="obs-keep") + drop = _hook_record(observation_id="obs-drop") + enqueue_hook(config, paths, keep) + enqueue_hook(config, paths, drop) + + ack_spool(config, paths, [drop["source_id"]]) + + remaining = read_spool(config, paths, NATIVE_SESSION_ID) + assert len(remaining) == 1 + assert remaining[0]["source_id"] == keep["source_id"] + + +def test_ack_spool_unknown_ids_are_no_ops(copilot_env: tuple[Config, SourcePaths]): + config, paths = copilot_env + record = _hook_record(observation_id="obs-stable") + enqueue_hook(config, paths, record) + + ack_spool(config, paths, ["nonexistent-source-id"]) + ack_spool(config, paths, []) + + remaining = read_spool(config, paths, NATIVE_SESSION_ID) + assert len(remaining) == 1 + assert remaining[0]["source_id"] == record["source_id"] + + +def test_malformed_spool_neighbor_does_not_discard_valid_records( + copilot_env: tuple[Config, SourcePaths], +): + config, paths = copilot_env + valid = _hook_record(observation_id="obs-valid-neighbor") + spool_path = Path(enqueue_hook(config, paths, valid)) + + (spool_path.parent / "000000-malformed.json").write_text("{not json", encoding="utf-8") + + records = read_spool(config, paths, NATIVE_SESSION_ID) + assert len(records) == 1 + assert records[0]["source_id"] == valid["source_id"] + + +def test_concurrent_enqueue_preserves_all_records(tmp_path: Path): + copilot_home = tmp_path / "copilot-home" + copilot_home.mkdir() + config = Config(root=tmp_path / "thirdeye") + paths = resolve_sources(copilot_home) + + barrier = threading.Barrier(_CONCURRENT_WORKERS) + errors: list[str] = [] + + def worker(worker_id: int) -> None: + try: + barrier.wait(timeout=5) + for index in range(_CONCURRENT_RECORDS): + record = _hook_record(observation_id=f"obs-{worker_id}-{index}") + enqueue_hook(config, paths, record) + except Exception as exc: # pragma: no cover - surfaced via errors list + errors.append(str(exc)) + + threads = [threading.Thread(target=worker, args=(worker_id,)) for worker_id in range(_CONCURRENT_WORKERS)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=30) + + assert not errors + records = read_spool(config, paths, NATIVE_SESSION_ID) + assert len(records) == _CONCURRENT_WORKERS * _CONCURRENT_RECORDS + + +def test_spool_survives_subprocess_isolation(tmp_path: Path): + copilot_home = tmp_path / "copilot-home" + copilot_home.mkdir() + config_root = tmp_path / "thirdeye" + script = f""" +from thirdeye.config import Config +from thirdeye.platforms.copilot.hook_payload import parse_hook +from thirdeye.platforms.copilot.identity import resolve_sources +from thirdeye.platforms.copilot.spool import enqueue_hook, read_spool + +config = Config(root={config_root!r}) +paths = resolve_sources({copilot_home!r}) +record = parse_hook( + "sessionStart", + {{"sessionId": {NATIVE_SESSION_ID!r}, "timestamp": 1789060102204}}, + {{}}, + observed_at={OBSERVED_AT!r}, + observation_id="obs-subprocess", +) +enqueue_hook(config, paths, record) +assert len(read_spool(config, paths, {NATIVE_SESSION_ID!r})) == 1 +""" + result = subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + text=True, + encoding="utf-8", + check=False, + ) + assert result.returncode == 0, result.stderr + + config = Config(root=config_root) + paths = resolve_sources(copilot_home) + records = read_spool(config, paths, NATIVE_SESSION_ID) + assert len(records) == 1 + assert records[0]["payload"]["hook_payload"]["sessionId"] == NATIVE_SESSION_ID + + +def test_child_session_hooks_spool_under_native_session_id(tmp_path: Path): + child_id = "bf8cb9f3-2097-4db0-a3c8-78a2653b2106" + copilot_home = tmp_path / "copilot-home" + copilot_home.mkdir() + config = Config(root=tmp_path / "thirdeye") + paths = resolve_sources(copilot_home) + + parent = _hook_record(observation_id="obs-parent", session_id=NATIVE_SESSION_ID) + child = _hook_record(observation_id="obs-child", session_id=child_id) + enqueue_hook(config, paths, parent) + enqueue_hook(config, paths, child) + + parent_records = read_spool(config, paths, NATIVE_SESSION_ID) + child_records = read_spool(config, paths, child_id) + assert len(parent_records) == 1 + assert len(child_records) == 1 + assert parent_records[0]["native_session_id"] == NATIVE_SESSION_ID + assert child_records[0]["native_session_id"] == child_id diff --git a/tests/test_provenance.py b/tests/test_provenance.py index 5a007e9..8e0a8d7 100644 --- a/tests/test_provenance.py +++ b/tests/test_provenance.py @@ -209,3 +209,45 @@ def test_classification_does_not_mutate_payload(payload: dict[str, Any], expecte foreign_payload_reason(payload, expected) assert payload == original + + +# --- copilot-specific provenance (positive evidence, fail-open elsewhere) --- + + +@pytest.mark.parametrize( + "event_name", + [ + "sessionStart", + "userPromptSubmitted", + "preToolUse", + "postToolUse", + "agentStop", + "subagentStart", + "subagentStop", + "sessionEnd", + ], +) +def test_copilot_accepts_native_camel_case_events(event_name: str): + assert foreign_payload_reason({"hook_event_name": event_name}, "copilot") is None + + +@pytest.mark.parametrize("event_name", _PASCAL_CASE_EVENTS) +def test_copilot_accepts_pascal_case_aliases(event_name: str): + assert foreign_payload_reason({"hook_event_name": event_name}, "copilot") is None + + +@pytest.mark.parametrize("marker", ["cursor_version", "composer_mode"]) +@pytest.mark.parametrize("value", [None, False, ""]) +def test_cursor_markers_are_foreign_for_copilot(marker: str, value: object): + reason = foreign_payload_reason({marker: value}, "copilot") + + assert isinstance(reason, str) + assert reason + assert marker in reason + + +def test_copilot_does_not_change_claude_codex_cursor_regressions(): + assert foreign_payload_reason({"hook_event_name": "SessionStart"}, "claude") is None + assert foreign_payload_reason({"hook_event_name": "beforeSubmitPrompt"}, "claude") is not None + assert foreign_payload_reason({"hook_event_name": "SessionStart"}, "cursor") is not None + assert foreign_payload_reason({"hook_event_name": "beforeSubmitPrompt"}, "cursor") is None From f2dbad7d78642d2da2babd449d20d1e943823684 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Thu, 10 Sep 2026 14:34:26 -0700 Subject: [PATCH 05/88] Add Copilot hook payload parsing and durable observation spooling. Co-authored-by: Cursor --- .../platforms/copilot/hook_payload.py | 128 ++++++++++++++++++ src/thirdeye/platforms/copilot/spool.py | 112 +++++++++++++++ src/thirdeye/platforms/provenance.py | 10 +- tests/test_copilot_hook_payload.py | 6 +- tests/test_copilot_spool.py | 10 +- 5 files changed, 257 insertions(+), 9 deletions(-) create mode 100644 src/thirdeye/platforms/copilot/hook_payload.py create mode 100644 src/thirdeye/platforms/copilot/spool.py diff --git a/src/thirdeye/platforms/copilot/hook_payload.py b/src/thirdeye/platforms/copilot/hook_payload.py new file mode 100644 index 0000000..f1739b0 --- /dev/null +++ b/src/thirdeye/platforms/copilot/hook_payload.py @@ -0,0 +1,128 @@ +"""Normalize Copilot hook observations into SourceRecord envelopes. + +This module is input-only: it must not open files, spawn processes, or talk +to SQLite. Callers own durable spooling and later capture. +""" + +from __future__ import annotations + +from copy import deepcopy +from datetime import UTC, datetime +from typing import Any + +from thirdeye.platforms.copilot.constants import CLI_HOOK_EVENT_ALIASES, SOURCE_SCHEMA_VERSION +from thirdeye.platforms.copilot.identity import validate_native_id +from thirdeye.platforms.copilot.types import SourceRecord + +# Claude-style PascalCase names accepted as input aliases. Canonical stored +# event names remain Copilot CLI camelCase from CLI_HOOK_EVENT_ALIASES. +_PASCAL_CASE_ALIASES: dict[str, str] = { + "SessionStart": "sessionStart", + "UserPromptSubmit": "userPromptSubmitted", + "PreToolUse": "preToolUse", + "PostToolUse": "postToolUse", + "Stop": "agentStop", + "SubagentStart": "subagentStart", + "SubagentStop": "subagentStop", + "SessionEnd": "sessionEnd", +} + +# Allowlisted hook-time context. ``env`` is the captured environment mapping; +# the remaining keys are optional explicit trace identifiers. +_CONTEXT_KEYS = ("env", "trace_id", "span_id", "parent_span_id", "trace_context", "traceparent") + +# Epoch milliseconds are >= 1e12 for dates after 2001-09-09. +_MILLISECOND_THRESHOLD = 1_000_000_000_000 + + +def _canonical_event(event: str) -> str: + if event in CLI_HOOK_EVENT_ALIASES: + return event + aliased = _PASCAL_CASE_ALIASES.get(event) + if aliased is not None: + return aliased + return event + + +def _session_id(payload: dict[str, Any]) -> str: + session_id = payload.get("sessionId") + if not isinstance(session_id, str): + raise ValueError("hook payload is missing a string sessionId") + validate_native_id(session_id) + return session_id + + +def _source_ts(payload: dict[str, Any]) -> str | None: + """Return the original source timestamp when it is valid; never invent one.""" + + if "timestamp" not in payload: + return None + value: Any = payload["timestamp"] + if isinstance(value, bool) or value is None: + return None + if isinstance(value, str): + stripped = value.strip() + if not stripped: + return None + if stripped.endswith("Z") or "+" in stripped[1:]: + return stripped + try: + value = float(stripped) + except ValueError: + return None + if not isinstance(value, (int, float)): + return None + seconds = value / 1000.0 if abs(value) >= _MILLISECOND_THRESHOLD else float(value) + try: + dt = datetime.fromtimestamp(seconds, tz=UTC) + except (OSError, OverflowError, ValueError): + return None + return dt.isoformat(timespec="milliseconds").replace("+00:00", "Z") + + +def _allowlisted_context(context: dict[str, Any]) -> dict[str, Any]: + stored: dict[str, Any] = {} + for key in _CONTEXT_KEYS: + if key not in context: + continue + value = context[key] + stored[key] = deepcopy(value) + return stored + + +def parse_hook( + event: str, + payload: dict[str, Any], + context: dict[str, Any], + *, + observed_at: str, + observation_id: str, +) -> SourceRecord: + """Build one hook SourceRecord from a normalized observation. + + The raw *payload* is retained unmodified. Session routing uses Copilot's + ``sessionId`` and rejects invalid native IDs. Child hooks that share a + parent session ID are recorded as-is; this function does not guess + ownership. + """ + + native_session_id = _session_id(payload) + canonical_event = _canonical_event(event) + retained_payload = deepcopy(payload) + return { + "source_id": f"hook/{native_session_id}/{observation_id}", + "source_kind": "hook", + "native_session_id": native_session_id, + "ts": _source_ts(payload), + "observed_at": observed_at, + "payload": { + "schema_version": SOURCE_SCHEMA_VERSION, + "event": canonical_event, + "hook_payload": retained_payload, + "context": _allowlisted_context(context), + }, + "locator": { + "observation_id": observation_id, + "event": canonical_event, + }, + } diff --git a/src/thirdeye/platforms/copilot/spool.py b/src/thirdeye/platforms/copilot/spool.py new file mode 100644 index 0000000..2bc8faa --- /dev/null +++ b/src/thirdeye/platforms/copilot/spool.py @@ -0,0 +1,112 @@ +"""Durable per-record spool for Copilot hook observations. + +Enqueue is independent of archive capture: each observation is written +atomically to its own file without taking the session archive lock. Read +returns complete records; acknowledge deletes only the named committed +source IDs. Malformed neighbors stay on disk and do not drop valid records. +""" + +from __future__ import annotations + +import json +import os +import tempfile +import uuid +from pathlib import Path +from typing import Any + +from thirdeye._compat import fsops +from thirdeye.config import Config +from thirdeye.platforms.copilot.identity import validate_native_id +from thirdeye.platforms.copilot.types import SourcePaths, SourceRecord + +_MAX_DIAGNOSTIC_REASON = 200 + + +def _spool_root(config: Config, paths: SourcePaths) -> Path: + return Path(config.root) / "spool" / "copilot" / paths["source_key"] + + +def _spool_dir(config: Config, paths: SourcePaths, native_id: str) -> Path: + validate_native_id(native_id) + return _spool_root(config, paths) / native_id + + +def _write_diagnostic(path: Path, reason: str) -> None: + """Record a bounded location/reason diagnostic; never prompt bodies.""" + + diag_path = path.with_name(f"{path.name}.diag") + clipped = reason[:_MAX_DIAGNOSTIC_REASON] + payload = json.dumps({"path": path.name, "reason": clipped}, ensure_ascii=True) + try: + diag_path.write_text(payload + "\n", encoding="utf-8") + except OSError: + return + + +def _load_record(path: Path) -> SourceRecord | None: + try: + raw = fsops.read_text(path, encoding="utf-8") + data: Any = json.loads(raw) + except (OSError, UnicodeDecodeError, json.JSONDecodeError, RecursionError) as exc: + _write_diagnostic(path, f"{type(exc).__name__}: {exc}") + return None + if not isinstance(data, dict) or not isinstance(data.get("source_id"), str): + _write_diagnostic(path, "spool file is not a SourceRecord object") + return None + return data # type: ignore[return-value] + + +def enqueue_hook(config: Config, paths: SourcePaths, record: SourceRecord) -> str: + """Atomically persist one hook observation and return its spool path.""" + + native_id = record["native_session_id"] + directory = _spool_dir(config, paths, native_id) + directory.mkdir(parents=True, exist_ok=True) + dest = directory / f"{uuid.uuid4().hex}.json" + fd, tmp_name = tempfile.mkstemp(dir=directory, prefix=".", suffix=".json.tmp") + try: + with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as handle: + json.dump(record, handle, ensure_ascii=False) + handle.write("\n") + handle.flush() + os.fsync(handle.fileno()) + fsops.replace(tmp_name, dest) + except BaseException: + fsops.unlink(Path(tmp_name), missing_ok=True) + raise + return str(dest) + + +def read_spool(config: Config, paths: SourcePaths, native_id: str) -> list[SourceRecord]: + """Return complete spool records for *native_id*, skipping malformed files.""" + + directory = _spool_dir(config, paths, native_id) + if not directory.is_dir(): + return [] + records: list[SourceRecord] = [] + for path in sorted(directory.glob("*.json")): + record = _load_record(path) + if record is not None: + records.append(record) + return records + + +def ack_spool(config: Config, paths: SourcePaths, source_ids: list[str]) -> None: + """Delete spool files whose committed source IDs are listed. + + Unknown IDs are no-ops. Malformed files are left in place. + """ + + wanted = {item for item in source_ids if item} + if not wanted: + return + root = _spool_root(config, paths) + if not root.is_dir(): + return + for path in root.glob("*/*.json"): + record = _load_record(path) + if record is None: + continue + if record["source_id"] in wanted: + fsops.unlink(path, missing_ok=True) diff --git a/src/thirdeye/platforms/provenance.py b/src/thirdeye/platforms/provenance.py index 43ce68d..610667f 100644 --- a/src/thirdeye/platforms/provenance.py +++ b/src/thirdeye/platforms/provenance.py @@ -4,8 +4,14 @@ from typing import Any -_KNOWN_PLATFORMS = frozenset({"claude", "codex", "cursor"}) +from thirdeye.platforms.copilot.constants import CLI_HOOK_EVENT_ALIASES + +_KNOWN_PLATFORMS = frozenset({"claude", "codex", "cursor", "copilot"}) _CURSOR_MARKERS = ("cursor_version", "composer_mode") +# Copilot CLI natively emits camelCase hook names. Those names are not Cursor +# evidence when the expected platform is Copilot; Cursor-only camelCase events +# remain foreign. +_COPILOT_CAMEL_EVENTS = frozenset(CLI_HOOK_EVENT_ALIASES) def foreign_payload_reason(payload: dict[str, Any], expected: str) -> str | None: @@ -28,6 +34,8 @@ def foreign_payload_reason(payload: dict[str, Any], expected: str) -> str | None first_character = event_name[0] if first_character.islower() and expected != "cursor": + if expected == "copilot" and event_name in _COPILOT_CAMEL_EVENTS: + return None return f"Cursor event {event_name} received for {expected}" if first_character.isupper() and expected == "cursor": return f"PascalCase event {event_name} received for cursor" diff --git a/tests/test_copilot_hook_payload.py b/tests/test_copilot_hook_payload.py index 1d08c15..4056601 100644 --- a/tests/test_copilot_hook_payload.py +++ b/tests/test_copilot_hook_payload.py @@ -58,9 +58,9 @@ def _parse( def test_hook_payload_module_has_no_io_imports(): - source = Path(__import__("thirdeye.platforms.copilot.hook_payload").__file__).read_text( - encoding="utf-8" - ) + import thirdeye.platforms.copilot.hook_payload as hook_payload + + source = Path(hook_payload.__file__).read_text(encoding="utf-8") forbidden = ("subprocess", "sqlite3", "open(", "Path(", "os.remove", "shutil") for token in forbidden: assert token not in source, f"hook_payload.py must not use {token!r}" diff --git a/tests/test_copilot_spool.py b/tests/test_copilot_spool.py index 69d5dc7..a6d7ef1 100644 --- a/tests/test_copilot_spool.py +++ b/tests/test_copilot_spool.py @@ -2,12 +2,10 @@ from __future__ import annotations -import json import subprocess import sys import threading from pathlib import Path -from typing import Any import pytest @@ -15,7 +13,7 @@ from thirdeye.platforms.copilot.hook_payload import parse_hook from thirdeye.platforms.copilot.identity import resolve_sources from thirdeye.platforms.copilot.spool import ack_spool, enqueue_hook, read_spool -from thirdeye.platforms.copilot.types import SourceRecord, SourcePaths +from thirdeye.platforms.copilot.types import SourcePaths, SourceRecord NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" OBSERVED_AT = "2026-09-10T17:08:25.626Z" @@ -154,13 +152,15 @@ def test_spool_survives_subprocess_isolation(tmp_path: Path): copilot_home.mkdir() config_root = tmp_path / "thirdeye" script = f""" +from pathlib import Path + from thirdeye.config import Config from thirdeye.platforms.copilot.hook_payload import parse_hook from thirdeye.platforms.copilot.identity import resolve_sources from thirdeye.platforms.copilot.spool import enqueue_hook, read_spool -config = Config(root={config_root!r}) -paths = resolve_sources({copilot_home!r}) +config = Config(root=Path({str(config_root)!r})) +paths = resolve_sources(Path({str(copilot_home)!r})) record = parse_hook( "sessionStart", {{"sessionId": {NATIVE_SESSION_ID!r}, "timestamp": 1789060102204}}, From 274662bd530722a55337de035e33c23e0d0f4c63 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Thu, 10 Sep 2026 14:34:35 -0700 Subject: [PATCH 06/88] Add Copilot SQLite evidence reader --- src/thirdeye/platforms/copilot/database.py | 358 +++++++++++++++++++++ 1 file changed, 358 insertions(+) create mode 100644 src/thirdeye/platforms/copilot/database.py diff --git a/src/thirdeye/platforms/copilot/database.py b/src/thirdeye/platforms/copilot/database.py new file mode 100644 index 0000000..6afdc1f --- /dev/null +++ b/src/thirdeye/platforms/copilot/database.py @@ -0,0 +1,358 @@ +"""Read raw Copilot session evidence from its local SQLite database. + +This module deliberately has no knowledge of thirdeye's archive. It produces +lossless, revisioned source records; committing and deduplicating those records +is the archive's responsibility. +""" + +from __future__ import annotations + +import base64 +import hashlib +import json +import sqlite3 +from collections.abc import Iterable +from datetime import UTC, datetime +from pathlib import Path +from typing import Any +from urllib.parse import quote + +from .identity import validate_native_id +from .types import SourcePaths, SourceRecord, SourceSlice + +_TABLES = ("sessions", "turns", "assistant_usage_events") +_SESSION_ID_COLUMNS = { + "sessions": ("id", "session_id"), + "turns": ("session_id",), + "assistant_usage_events": ("session_id",), +} +_PRIMARY_KEY_COLUMNS = ("id", "uuid", "event_id") +_TIMESTAMP_COLUMNS = ("created_at", "createdAt", "timestamp", "updated_at", "updatedAt") +_CWD_COLUMNS = ("cwd", "working_directory", "workingDirectory") + + +def _diagnostic(code: str, message: str, **details: Any) -> dict[str, Any]: + """Return a content-free diagnostic suitable for CLI presentation.""" + + return {"code": code, "message": message, **details} + + +def _observed_at() -> str: + return datetime.now(UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z") + + +def _json_value(value: Any) -> Any: + """Make SQLite values JSON-compatible without dropping a column's value.""" + + if isinstance(value, bytes): + return {"encoding": "base64", "data": base64.b64encode(value).decode("ascii")} + if isinstance(value, memoryview): + return _json_value(value.tobytes()) + if isinstance(value, float) and (value != value or value in (float("inf"), float("-inf"))): + return {"encoding": "repr", "data": repr(value)} + return value + + +def _canonical_json(value: Any) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False, default=str) + + +def _content_revision(row: dict[str, Any]) -> str: + return "sha256:" + hashlib.sha256(_canonical_json(row).encode("utf-8")).hexdigest() + + +def _quote_identifier(name: str) -> str: + # Names come from SQLite's schema, but quote anyway so a surprising schema + # cannot turn an identifier into executable SQL. + return '"' + name.replace('"', '""') + '"' + + +def _database_generation(database: Path) -> str: + """Identify the database and its live WAL without opening it for writing.""" + + parts: list[dict[str, Any]] = [] + for candidate in (database, Path(f"{database}-wal")): + try: + stat = candidate.stat() + except FileNotFoundError: + parts.append({"path": candidate.name, "missing": True}) + else: + parts.append( + { + "path": candidate.name, + "device": stat.st_dev, + "inode": stat.st_ino, + "size": stat.st_size, + "mtime_ns": stat.st_mtime_ns, + } + ) + return "sha256:" + hashlib.sha256(_canonical_json(parts).encode("utf-8")).hexdigest() + + +def _connect(database: Path) -> sqlite3.Connection: + # ``mode=ro`` keeps SQLite's normal WAL behaviour while preventing all + # writes. In particular, do not use immutable=1: it ignores live WAL data. + connection = sqlite3.connect(database.resolve().as_uri() + "?mode=ro", uri=True, timeout=0.1) + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA busy_timeout = 100") + connection.execute("BEGIN") + return connection + + +def _table_columns(connection: sqlite3.Connection, table: str) -> list[str]: + return [ + str(row["name"]) + for row in connection.execute(f"PRAGMA table_info({_quote_identifier(table)})") + ] + + +def _available_tables(connection: sqlite3.Connection) -> set[str]: + rows = connection.execute( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name IN (?, ?, ?)", _TABLES + ) + return {str(row[0]) for row in rows} + + +def _session_column(table: str, columns: Iterable[str]) -> str | None: + known = set(columns) + return next((name for name in _SESSION_ID_COLUMNS[table] if name in known), None) + + +def _primary_key_column(columns: Iterable[str]) -> str | None: + known = set(columns) + return next((name for name in _PRIMARY_KEY_COLUMNS if name in known), None) + + +def _valid_source_time(row: dict[str, Any]) -> str | None: + for column in _TIMESTAMP_COLUMNS: + value = row.get(column) + if not isinstance(value, str) or not value: + continue + try: + datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + continue + return value + return None + + +def _row_record( + paths: SourcePaths, + native_id: str, + table: str, + primary_key: Any, + row: dict[str, Any], + generation: str, + observed_at: str, +) -> SourceRecord: + revision = _content_revision(row) + primary_identity = _canonical_json(_json_value(primary_key)) + # Keep the row identity legible for normal integer IDs, while quoting it so + # arbitrary SQLite primary-key values never alter the source-ID structure. + primary_component = quote(primary_identity, safe="") + source_id = ( + f"copilot-db:{paths['source_key']}:{native_id}:{table}:{primary_component}:{revision}" + ) + return { + "source_id": source_id, + "source_kind": "database", + "native_session_id": native_id, + "ts": _valid_source_time(row), + "observed_at": observed_at, + "payload": {"table": table, "row": row}, + "locator": { + "database": paths["database"], + "table": table, + "primary_key": _json_value(primary_key), + "content_revision": revision, + "generation": generation, + }, + } + + +def _empty_slice(diagnostics: list[dict[str, Any]]) -> SourceSlice: + return { + "records": [], + "next_cursor": {}, + "diagnostics": diagnostics, + "cwd": None, + "exhausted": True, + } + + +def discover_database_sessions(paths: SourcePaths) -> list[str]: + """Return session IDs advertised by a readable Copilot sessions table. + + Discovery is intentionally conservative: a malformed or absent database is + simply not a discoverable source. ``read_database`` supplies the actionable + diagnostics needed by status and sync commands. + """ + + database = Path(paths["database"]) + if not database.is_file(): + return [] + try: + with _connect(database) as connection: + if "sessions" not in _available_tables(connection): + return [] + columns = _table_columns(connection, "sessions") + session_column = _session_column("sessions", columns) + if session_column is None: + return [] + rows = connection.execute( + f"SELECT {_quote_identifier(session_column)} FROM sessions " + f"WHERE {_quote_identifier(session_column)} IS NOT NULL" + ) + values = [str(row[0]) for row in rows if isinstance(row[0], str)] + except sqlite3.Error: + return [] + return sorted(set(values)) + + +def read_database( + paths: SourcePaths, + native_id: str, + cursor: dict, + *, + max_records: int = 1000, +) -> SourceSlice: + """Read a bounded, transactionally consistent raw SQLite snapshot. + + Every poll re-reads the selected session, because turns and sessions are + mutable. A generation/offset cursor only bounds delivery of that snapshot; + it never assumes a row ID is an immutable record identity. + """ + + validate_native_id(native_id) + if max_records < 1: + raise ValueError("max_records must be at least 1") + + database = Path(paths["database"]) + if not database.exists(): + return _empty_slice( + [ + _diagnostic( + "copilot_database_missing", + "Copilot session database is not present", + path=str(database), + ) + ] + ) + if not database.is_file(): + return _empty_slice( + [ + _diagnostic( + "copilot_database_unreadable", + "Copilot session database is not a regular file", + path=str(database), + ) + ] + ) + + diagnostics: list[dict[str, Any]] = [] + observed_at = _observed_at() + generation = _database_generation(database) + try: + with _connect(database) as connection: + available = _available_tables(connection) + rows_by_table: list[tuple[str, Any, dict[str, Any]]] = [] + cwd: str | None = None + for table in _TABLES: + if table not in available: + diagnostics.append( + _diagnostic( + "copilot_database_table_missing", + "Expected Copilot table is unavailable", + table=table, + ) + ) + continue + columns = _table_columns(connection, table) + session_column = _session_column(table, columns) + primary_column = _primary_key_column(columns) + if session_column is None: + diagnostics.append( + _diagnostic( + "copilot_database_missing_session_column", + "Copilot table cannot be scoped to a session", + table=table, + expected=list(_SESSION_ID_COLUMNS[table]), + columns=columns, + ) + ) + continue + if primary_column is None: + diagnostics.append( + _diagnostic( + "copilot_database_missing_primary_key", + "Copilot table lacks a supported stable row identity", + table=table, + expected=list(_PRIMARY_KEY_COLUMNS), + columns=columns, + ) + ) + continue + query = ( + f"SELECT * FROM {_quote_identifier(table)} " + f"WHERE {_quote_identifier(session_column)} = ? " + f"ORDER BY {_quote_identifier(primary_column)}" + ) + for sql_row in connection.execute(query, (native_id,)): + row = {key: _json_value(sql_row[key]) for key in sql_row.keys()} + if table == "sessions" and cwd is None: + cwd = next( + (row[name] for name in _CWD_COLUMNS if isinstance(row.get(name), str)), + None, + ) + rows_by_table.append((table, row[primary_column], row)) + except sqlite3.OperationalError as error: + return _empty_slice( + [ + _diagnostic( + "copilot_database_busy", + "Copilot session database could not be read within 100ms; retry sync or watch", + path=str(database), + reason=str(error), + ) + ] + ) + except sqlite3.Error as error: + return _empty_slice( + [ + _diagnostic( + "copilot_database_incompatible", + "Copilot session database could not be read as SQLite evidence", + path=str(database), + reason=str(error), + ) + ] + ) + + # The table loop above has a deterministic table order; ordering row IDs by + # their JSON form makes heterogeneous SQLite primary key types deterministic. + rows_by_table.sort( + key=lambda item: (_TABLES.index(item[0]), _canonical_json(_json_value(item[1]))) + ) + incoming_generation = cursor.get("database_generation") if isinstance(cursor, dict) else None + incoming_offset = cursor.get("database_offset", 0) if isinstance(cursor, dict) else 0 + offset = ( + incoming_offset + if incoming_generation == generation and isinstance(incoming_offset, int) + else 0 + ) + offset = max(0, offset) + selected = rows_by_table[offset : offset + max_records] + records = [ + _row_record(paths, native_id, table, primary_key, row, generation, observed_at) + for table, primary_key, row in selected + ] + next_offset = offset + len(selected) + exhausted = next_offset >= len(rows_by_table) + next_cursor = {"database_generation": generation, "database_offset": next_offset} + return { + "records": records, + "next_cursor": next_cursor, + "diagnostics": diagnostics, + "cwd": cwd, + "exhausted": exhausted, + } From e6d361bd2a336d48e08e3f910521b5a2d0e8d94e Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Thu, 10 Sep 2026 14:35:06 -0700 Subject: [PATCH 07/88] Add durable Copilot archive journal --- src/thirdeye/platforms/copilot/archive.py | 381 ++++++++++++++++++++++ src/thirdeye/platforms/copilot/state.py | 70 ++++ src/thirdeye/writer.py | 11 +- 3 files changed, 460 insertions(+), 2 deletions(-) create mode 100644 src/thirdeye/platforms/copilot/archive.py create mode 100644 src/thirdeye/platforms/copilot/state.py diff --git a/src/thirdeye/platforms/copilot/archive.py b/src/thirdeye/platforms/copilot/archive.py new file mode 100644 index 0000000..0d1d95d --- /dev/null +++ b/src/thirdeye/platforms/copilot/archive.py @@ -0,0 +1,381 @@ +"""Durable, local archive for raw GitHub Copilot CLI source evidence. + +This module is intentionally below composition: it knows neither how records +are read nor how hooks are invoked. Its only input is a fully composed +``SourceBatch`` and its output is the generic thirdeye event log plus a small, +recoverable Copilot checkpoint beside that log. +""" + +from __future__ import annotations + +from collections.abc import Callable, Iterator +from datetime import datetime +import json +from pathlib import Path +from typing import Any + +from thirdeye._compat.locking import LockMode, locked +from thirdeye.config import Config +from thirdeye.meta import read_meta, write_meta +from thirdeye.paths import meta_path, session_dir +from thirdeye.reader import SessionReader +from thirdeye.store import Store +from thirdeye.writer import utc_iso_ms + +from .constants import PLATFORM_NAME, SOURCE_SCHEMA_VERSION +from .identity import stored_session_id, validate_native_id +from .state import ( + STATE_SCHEMA_VERSION, + clear_journal, + journal_path, + lock_path, + read_json, + state_path, + write_journal, + write_state, +) +from .types import SourceBatch, SourcePaths, SourceRecord, SyncResult + +_EVENT_TYPES = { + "transcript": "copilot_transcript", + "database": "copilot_database", + "hook": "copilot_hook", + "metadata": "copilot_metadata", +} + +# Tests and embedding applications may replace this with a deterministic +# callback that raises at a journal boundary. It is deliberately private: no +# runtime behaviour relies on fault injection. +_fault_injector: Callable[[str], None] | None = None + + +def _fault(point: str) -> None: + if _fault_injector is not None: + _fault_injector(point) + + +def _result( + *, + sessions: int = 0, + written: int = 0, + duplicates: int = 0, + pending: int = 0, + errors: int = 0, +) -> SyncResult: + return { + "sessions": sessions, + "records_written": written, + "duplicate_records": duplicates, + "pending": pending, + "errors": errors, + } + + +def _archive_dir(config: Config, stored_id: str) -> Path: + return session_dir(config.root, PLATFORM_NAME, stored_id) + + +def _valid_timestamp(value: object) -> str | None: + if not isinstance(value, str) or not value: + return None + candidate = value[:-1] + "+00:00" if value.endswith("Z") else value + try: + datetime.fromisoformat(candidate) + except ValueError: + return None + return value + + +def _event_type(record: SourceRecord) -> str: + return _EVENT_TYPES.get(record["source_kind"], "copilot_metadata") + + +def _envelope(record: SourceRecord) -> dict[str, Any]: + # ``source_record`` is named so later derived-state versions can add their + # own fields without ever changing the raw record's shape. + return {"schema_version": SOURCE_SCHEMA_VERSION, "source_record": record} + + +def _record_from_event(event: dict[str, Any]) -> SourceRecord | None: + if event.get("t") not in _EVENT_TYPES.values(): + return None + data = event.get("data") + if not isinstance(data, dict) or data.get("schema_version") != SOURCE_SCHEMA_VERSION: + return None + record = data.get("source_record") + if not isinstance(record, dict) or not isinstance(record.get("source_id"), str): + return None + return record # type: ignore[return-value] + + +def _committed_records(directory: Path) -> dict[str, SourceRecord]: + if not directory.exists(): + return {} + committed: dict[str, SourceRecord] = {} + for event in SessionReader(directory).iter_events(types=_EVENT_TYPES.values()): + record = _record_from_event(event) + if record is not None: + committed[record["source_id"]] = record + return committed + + +def _new_state(paths: SourcePaths, native_id: str) -> dict[str, Any]: + return { + "schema_version": STATE_SCHEMA_VERSION, + "source_key": paths["source_key"], + "source_home": paths["home"], + "native_session_id": native_id, + "cursor": {}, + "health": {"diagnostics": [], "last_successful_import": None, "capabilities": {}}, + } + + +def _validate_state(state: dict[str, Any], paths: SourcePaths, native_id: str) -> None: + if state.get("schema_version") != STATE_SCHEMA_VERSION: + raise ValueError("unsupported Copilot archive state schema") + if state.get("source_key") != paths["source_key"]: + raise ValueError("Copilot source-key prefix collision for stored session ID") + if state.get("native_session_id") != native_id: + raise ValueError("Copilot archive native session identity does not match stored session ID") + + +def _ensure_meta(config: Config, paths: SourcePaths, native_id: str, cwd: str | None) -> Path: + stored_id = stored_session_id(paths, native_id) + directory = _archive_dir(config, stored_id) + meta = read_meta(meta_path(directory)) if directory.exists() else None + if meta is not None: + identity = meta.extra.get("copilot") if isinstance(meta.extra, dict) else None + if not isinstance(identity, dict) or identity.get("source_key") != paths["source_key"]: + raise ValueError("Copilot source-key prefix collision for stored session ID") + if identity.get("native_session_id") != native_id: + raise ValueError("Copilot archive native session identity does not match stored session ID") + return directory + + # Store owns generic session metadata. Creating it here also lets a hook + # establish a provisional session before any transcript exists. + writer = Store(config).open_session( + stored_id, + platform=PLATFORM_NAME, + cwd=cwd or "", + extra={ + "copilot": { + "schema_version": SOURCE_SCHEMA_VERSION, + "source_key": paths["source_key"], + "source_home": paths["home"], + "native_session_id": native_id, + } + }, + ) + writer.flush_and_detach() + return directory + + +def _base_cursor(batch: SourceBatch) -> dict[str, Any] | None: + """Read an optional composition cursor without imposing a reader format. + + The public SourceBatch schema remains JSON-compatible and intentionally + has no reader-specific field. Composition can include either spelling in + its cursor object when it needs optimistic contention detection. + """ + cursor = batch["next_cursor"] + for name in ("base_cursor", "_base_cursor"): + value = cursor.get(name) + if isinstance(value, dict): + return value + return None + + +def _set_health( + state: dict[str, Any], diagnostics: list[dict[str, Any]], *, successful: bool +) -> None: + health = state.setdefault("health", {}) + health["diagnostics"] = diagnostics + if successful: + health["last_successful_import"] = utc_iso_ms() + + +def _append_records( + config: Config, + directory: Path, + stored_id: str, + cwd: str | None, + records: list[SourceRecord], + committed: dict[str, SourceRecord], +) -> tuple[int, int, list[dict[str, Any]]]: + _ = directory + writer = Store(config).open_session(stored_id, platform=PLATFORM_NAME, cwd=cwd or "") + written = 0 + duplicates = 0 + diagnostics: list[dict[str, Any]] = [] + try: + for record in records: + source_id = record.get("source_id") + if not isinstance(source_id, str) or not source_id: + diagnostics.append({"kind": "invalid_source_record", "reason": "missing source_id"}) + continue + if source_id in committed: + duplicates += 1 + continue + source_ts = _valid_timestamp(record.get("ts")) + observed_at = _valid_timestamp(record.get("observed_at")) + event_ts = source_ts or observed_at + if source_ts is None: + diagnostics.append({"kind": "missing_source_time", "source_id": source_id}) + writer.append(_event_type(record), _envelope(record), ts=event_ts) + committed[source_id] = record + written += 1 + writer.flush_and_detach() + except BaseException: + writer.flush_and_detach() + raise + return written, duplicates, diagnostics + + +def _recover( + config: Config, + paths: SourcePaths, + native_id: str, + directory: Path, + state: dict[str, Any], +) -> tuple[dict[str, Any], int, int]: + journal = read_json(journal_path(directory)) + if journal is None: + return state, 0, 0 + if journal.get("source_key") != paths["source_key"] or journal.get("native_session_id") != native_id: + raise ValueError("Copilot archive journal belongs to another source session") + records = journal.get("records") + next_cursor = journal.get("next_cursor") + if not isinstance(records, list) or not isinstance(next_cursor, dict): + raise ValueError("invalid Copilot archive journal") + committed = _committed_records(directory) + written, duplicates, diagnostics = _append_records( + config, directory, stored_session_id(paths, native_id), journal.get("cwd"), records, committed + ) + _fault("after_recovery_append") + state["cursor"] = next_cursor + _set_health(state, list(journal.get("diagnostics", [])) + diagnostics, successful=True) + write_state(directory, state) + _fault("after_recovery_checkpoint") + clear_journal(directory) + return state, written, duplicates + + +def load_cursor(config: Config, paths: SourcePaths, native_id: str) -> dict: + """Return a copy of the last committed cursor for one native session.""" + validate_native_id(native_id) + directory = _archive_dir(config, stored_session_id(paths, native_id)) + if not directory.exists(): + return {} + with locked(lock_path(directory), LockMode.EXCLUSIVE): + state = read_json(state_path(directory)) + if state is None: + return {} + _validate_state(state, paths, native_id) + # JSON copying protects the on-disk state from caller mutation. + return json.loads(json.dumps(state.get("cursor", {}))) + + +def commit_batch(config: Config, paths: SourcePaths, batch: SourceBatch) -> SyncResult: + """Append one raw evidence batch and atomically advance its cursor.""" + native_id = batch["native_session_id"] + validate_native_id(native_id) + if batch["source_key"] != paths["source_key"]: + raise ValueError("SourceBatch source_key does not match selected Copilot source home") + for record in batch["records"]: + if record.get("native_session_id") != native_id: + raise ValueError("SourceBatch contains a record for another native session") + stored_id = stored_session_id(paths, native_id) + directory = _ensure_meta(config, paths, native_id, batch.get("cwd")) + with locked(lock_path(directory), LockMode.EXCLUSIVE): + state = read_json(state_path(directory)) or _new_state(paths, native_id) + _validate_state(state, paths, native_id) + state, recovered_written, recovered_duplicates = _recover(config, paths, native_id, directory, state) + + expected = _base_cursor(batch) + current = state.get("cursor", {}) + if expected is not None and expected != current: + diagnostic = { + "kind": "stale_cursor", + "reason": "archive cursor advanced while batch was being read", + "expected_cursor": expected, + "committed_cursor": current, + } + _set_health(state, list(batch["diagnostics"]) + [diagnostic], successful=False) + write_state(directory, state) + return _result( + sessions=1, + written=recovered_written, + duplicates=recovered_duplicates, + pending=len(batch["records"]), + errors=1, + ) + + journal = { + "schema_version": STATE_SCHEMA_VERSION, + "source_key": paths["source_key"], + "native_session_id": native_id, + "cwd": batch.get("cwd"), + "records": batch["records"], + "next_cursor": batch["next_cursor"], + "diagnostics": batch["diagnostics"], + } + write_journal(directory, journal) + _fault("after_journal") + committed = _committed_records(directory) + written, duplicates, diagnostics = _append_records( + config, directory, stored_id, batch.get("cwd"), batch["records"], committed + ) + _fault("after_append") + state["cursor"] = batch["next_cursor"] + _set_health(state, list(batch["diagnostics"]) + diagnostics, successful=True) + write_state(directory, state) + _fault("after_checkpoint") + clear_journal(directory) + _fault("after_journal_clear") + _apply_lifecycle(directory, batch["records"]) + return _result( + sessions=1, + written=recovered_written + written, + duplicates=recovered_duplicates + duplicates, + ) + + +def _apply_lifecycle(directory: Path, records: list[SourceRecord]) -> None: + """Apply only explicit top-level lifecycle evidence; child stops never close.""" + close = False + reopen = False + for record in records: + if record["source_kind"] != "hook": + continue + payload = record.get("payload", {}) + event = payload.get("event") if isinstance(payload, dict) else None + context = payload.get("context") if isinstance(payload, dict) else None + is_child = isinstance(context, dict) and bool(context.get("parent_tool_call_id") or context.get("agent_id")) + if event in {"sessionEnd", "shutdown", "session_end"} and not is_child: + close = True + if event in {"sessionStart", "resume", "activity", "session_start", "userPromptSubmitted"}: + reopen = True + if not close and not reopen: + return + meta_file = meta_path(directory) + meta = read_meta(meta_file) + if meta is None: + return + if reopen: + meta.status = "open" + meta.ended_at = None + elif close: + meta.status = "closed" + meta.ended_at = utc_iso_ms() + write_meta(meta_file, meta) + + +def iter_captured_records(config: Config, stored_session_id: str) -> Iterator[SourceRecord]: + """Yield immutable raw Copilot records retained in a local stored session.""" + directory = _archive_dir(config, stored_session_id) + if not directory.exists(): + return + for event in SessionReader(directory).iter_events(types=_EVENT_TYPES.values()): + record = _record_from_event(event) + if record is not None: + yield record diff --git a/src/thirdeye/platforms/copilot/state.py b/src/thirdeye/platforms/copilot/state.py new file mode 100644 index 0000000..cced32d --- /dev/null +++ b/src/thirdeye/platforms/copilot/state.py @@ -0,0 +1,70 @@ +"""Private, atomically-published state for the Copilot evidence archive.""" + +from __future__ import annotations + +import json +import os +import tempfile +from pathlib import Path +from typing import Any + +from thirdeye._compat import fsops + +STATE_SCHEMA_VERSION = 1 +STATE_FILENAME = "copilot.state.json" +JOURNAL_FILENAME = "copilot.journal.json" +LOCK_FILENAME = "copilot.archive.lock" + + +def state_path(session_dir: Path) -> Path: + return session_dir / STATE_FILENAME + + +def journal_path(session_dir: Path) -> Path: + return session_dir / JOURNAL_FILENAME + + +def lock_path(session_dir: Path) -> Path: + return session_dir / LOCK_FILENAME + + +def _atomic_json(path: Path, value: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + fd, temp_name = tempfile.mkstemp(dir=path.parent, prefix=f"{path.name}.", suffix=".tmp") + try: + with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as stream: + json.dump(value, stream, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + fsops.replace(temp_name, path) + except BaseException: + fsops.unlink(Path(temp_name), missing_ok=True) + raise + + +def read_json(path: Path) -> dict[str, Any] | None: + try: + raw = json.loads(fsops.read_text(path, encoding="utf-8")) + except FileNotFoundError: + return None + except (OSError, json.JSONDecodeError): + # A malformed state file is never silently used as a fresh cursor. The + # archive caller turns this into a diagnostic and leaves the evidence + # log itself readable. + raise ValueError(f"invalid Copilot archive state: {path}") from None + if not isinstance(raw, dict): + raise ValueError(f"invalid Copilot archive state: {path}") + return raw + + +def write_state(session_dir: Path, value: dict[str, Any]) -> None: + _atomic_json(state_path(session_dir), value) + + +def write_journal(session_dir: Path, value: dict[str, Any]) -> None: + _atomic_json(journal_path(session_dir), value) + + +def clear_journal(session_dir: Path) -> None: + fsops.unlink(journal_path(session_dir), missing_ok=True) diff --git a/src/thirdeye/writer.py b/src/thirdeye/writer.py index 2be320c..bda9cb3 100644 --- a/src/thirdeye/writer.py +++ b/src/thirdeye/writer.py @@ -105,8 +105,15 @@ def open( write_meta(meta_path(session_dir), meta) return cls(session_dir, meta) - def append(self, t: str, data: Any = None) -> int: - ts = _utc_iso_ms() + def append(self, t: str, data: Any = None, *, ts: str | None = None) -> int: + """Append an event, optionally retaining a source-provided timestamp. + + Most producers use the default observation time. Importers which + archive an external, immutable event may supply its already validated + source timestamp instead; this deliberately does not change the + default writer behaviour for live capture. + """ + ts = ts or _utc_iso_ms() with self._locked(): # Every hook invocation is a fresh process constructing its own # SessionWriter, so `self._next_seq` can be stale by the time From 3a4ccce7ea411d0b447faa53ee67797fee0e7e27 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Thu, 10 Sep 2026 14:35:24 -0700 Subject: [PATCH 08/88] Add Copilot transcript reader --- src/thirdeye/platforms/copilot/transcript.py | 380 +++++++++++++++++++ 1 file changed, 380 insertions(+) create mode 100644 src/thirdeye/platforms/copilot/transcript.py diff --git a/src/thirdeye/platforms/copilot/transcript.py b/src/thirdeye/platforms/copilot/transcript.py new file mode 100644 index 0000000..7e0f0ad --- /dev/null +++ b/src/thirdeye/platforms/copilot/transcript.py @@ -0,0 +1,380 @@ +"""Lossless, bounded reads of Copilot CLI transcript recordings. + +This module deliberately knows nothing about thirdeye's archive. It turns the +``events.jsonl`` and the small, adjacent ``workspace.yaml`` document into raw +source evidence that a later composition layer can commit durably. +""" + +from __future__ import annotations + +import base64 +import hashlib +import json +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import yaml + +from .constants import SOURCE_SCHEMA_VERSION +from .identity import validate_native_id +from .types import SourcePaths, SourceRecord, SourceSlice + +_EVENTS_FILENAME = "events.jsonl" +_WORKSPACE_FILENAME = "workspace.yaml" +_CONTINUITY_WINDOW = 4096 + + +def _observed_at() -> str: + return datetime.now(UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z") + + +def _diagnostic(code: str, message: str, **locator: Any) -> dict[str, Any]: + return {"code": code, "message": message, "locator": locator} + + +def _within(candidate: Path, root: Path) -> bool: + try: + candidate.relative_to(root) + except ValueError: + return False + return True + + +def _session_directory(paths: SourcePaths, native_id: str) -> Path: + """Return a checked session directory without allowing path traversal.""" + + validate_native_id(native_id) + root = Path(paths["session_root"]).expanduser().resolve(strict=False) + session = (root / native_id).resolve(strict=False) + if not _within(session, root): + raise ValueError("native session ID escapes the Copilot session root") + return session + + +def _generation(path: Path) -> str: + """Identify the current file object, while remaining stable for appends.""" + + stat = path.stat() + # st_dev/st_ino distinguishes atomic replacement on the platforms we + # support and, unlike mtime/ctime, does not change on an ordinary append. + # A subsequent size decrease still detects truncation on filesystems where + # inode data is unavailable. + return f"{stat.st_dev:x}-{stat.st_ino:x}" + + +def _valid_timestamp(value: Any) -> str | None: + if not isinstance(value, str) or not value: + return None + try: + datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + return value + + +def _continuity_anchor(path: Path, offset: int) -> tuple[int, str]: + """Fingerprint source bytes already consumed without penalizing appends.""" + + start = max(0, offset - _CONTINUITY_WINDOW) + with path.open("rb") as stream: + stream.seek(start) + digest = hashlib.sha256(stream.read(offset - start)).hexdigest() + return start, digest + + +def _json_value(value: Any) -> Any: + """Reject YAML-only values so metadata remains a JSON-compatible contract.""" + + if value is None or isinstance(value, (str, int, float, bool)): + return value + if isinstance(value, list): + return [_json_value(item) for item in value] + if isinstance(value, dict) and all(isinstance(key, str) for key in value): + return {key: _json_value(item) for key, item in value.items()} + raise TypeError(f"workspace metadata contains unsupported value {type(value).__name__}") + + +def _source_id(paths: SourcePaths, native_id: str, event: dict[str, Any], generation: str, offset: int, raw: bytes) -> tuple[str, dict[str, Any]]: + event_id = event.get("id") + if isinstance(event_id, (str, int)) and str(event_id): + value = str(event_id) + return ( + f"{paths['source_key']}/{native_id}/{value}", + {"native_event_id": value}, + ) + + digest = hashlib.sha256(raw).hexdigest() + return ( + f"{paths['source_key']}/{native_id}/{generation}/{offset}/{digest}", + {"content_digest": digest, "identity_confidence": "lower"}, + ) + + +def _record_for_line( + paths: SourcePaths, + native_id: str, + generation: str, + offset: int, + line: bytes, + observed_at: str, +) -> tuple[SourceRecord, dict[str, Any] | None]: + """Create evidence for one complete newline-terminated physical line.""" + + raw = line[:-1] if line.endswith(b"\n") else line + locator: dict[str, Any] = { + "file": _EVENTS_FILENAME, + "file_generation": generation, + "byte_offset": offset, + "byte_length": len(line), + } + try: + text = raw.decode("utf-8") + except UnicodeDecodeError: + digest = hashlib.sha256(raw).hexdigest() + locator["content_digest"] = digest + record: SourceRecord = { + "source_id": f"{paths['source_key']}/{native_id}/{generation}/{offset}/{digest}", + "source_kind": "transcript", + "native_session_id": native_id, + "ts": None, + "observed_at": observed_at, + "payload": { + "schema_version": SOURCE_SCHEMA_VERSION, + "malformed": "invalid_utf8", + "raw_bytes_base64": base64.b64encode(raw).decode("ascii"), + }, + "locator": locator, + } + return record, _diagnostic("transcript_invalid_utf8", "complete transcript line is not UTF-8", **locator) + + try: + value = json.loads(text) + except (json.JSONDecodeError, ValueError): + digest = hashlib.sha256(raw).hexdigest() + locator["content_digest"] = digest + record = { + "source_id": f"{paths['source_key']}/{native_id}/{generation}/{offset}/{digest}", + "source_kind": "transcript", + "native_session_id": native_id, + "ts": None, + "observed_at": observed_at, + "payload": {"schema_version": SOURCE_SCHEMA_VERSION, "malformed": "invalid_json", "raw_line": text}, + "locator": locator, + } + return record, _diagnostic("transcript_invalid_json", "complete transcript line is not JSON", **locator) + + if not isinstance(value, dict): + digest = hashlib.sha256(raw).hexdigest() + locator["content_digest"] = digest + record = { + "source_id": f"{paths['source_key']}/{native_id}/{generation}/{offset}/{digest}", + "source_kind": "transcript", + "native_session_id": native_id, + "ts": None, + "observed_at": observed_at, + "payload": {"schema_version": SOURCE_SCHEMA_VERSION, "malformed": "non_object_json", "raw_value": value}, + "locator": locator, + } + return record, _diagnostic("transcript_non_object", "complete transcript line is not a JSON object", **locator) + + source_id, event_locator = _source_id(paths, native_id, value, generation, offset, raw) + locator.update(event_locator) + timestamp = _valid_timestamp(value.get("timestamp")) + diagnostic = None + if value.get("timestamp") is not None and timestamp is None: + diagnostic = _diagnostic("transcript_invalid_timestamp", "event timestamp is not ISO-8601", **locator) + # Keep every top-level field, including unknown future fields. The schema + # version comes last so an untrusted event cannot alter our envelope. + payload = {**value, "schema_version": SOURCE_SCHEMA_VERSION} + record = { + "source_id": source_id, + "source_kind": "transcript", + "native_session_id": native_id, + "ts": timestamp, + "observed_at": observed_at, + "payload": payload, + "locator": locator, + } + return record, diagnostic + + +def _workspace_record( + paths: SourcePaths, native_id: str, directory: Path, observed_at: str +) -> tuple[SourceRecord | None, str | None, list[dict[str, Any]]]: + """Read only safe workspace metadata and return it as independent evidence.""" + + path = directory / _WORKSPACE_FILENAME + if not path.is_file(): + return None, None, [] + try: + raw = path.read_bytes() + data = yaml.safe_load(raw.decode("utf-8")) + data = _json_value(data) + except (OSError, TypeError, UnicodeDecodeError, yaml.YAMLError) as exc: + return None, None, [_diagnostic("workspace_metadata_invalid", "workspace.yaml could not be safely parsed", file=str(path), reason=type(exc).__name__)] + if not isinstance(data, dict): + return None, None, [_diagnostic("workspace_metadata_invalid", "workspace.yaml must contain a mapping", file=str(path))] + + digest = hashlib.sha256(raw).hexdigest() + try: + generation = _generation(path) + except OSError: + generation = f"content-{digest}" + cwd = data.get("cwd") + if not isinstance(cwd, str): + cwd = data.get("workspacePath") if isinstance(data.get("workspacePath"), str) else None + record: SourceRecord = { + "source_id": f"{paths['source_key']}/{native_id}/workspace/{digest}", + "source_kind": "metadata", + "native_session_id": native_id, + "ts": None, + "observed_at": observed_at, + "payload": {"schema_version": SOURCE_SCHEMA_VERSION, "file": _WORKSPACE_FILENAME, "data": data}, + "locator": {"file": _WORKSPACE_FILENAME, "file_generation": generation, "content_digest": digest}, + } + return record, cwd, [] + + +def discover_transcripts(paths: SourcePaths) -> list[str]: + """Return native session IDs with an immediately readable events file.""" + + root = Path(paths["session_root"]).expanduser().resolve(strict=False) + try: + candidates = list(root.iterdir()) + except OSError: + return [] + result: list[str] = [] + for candidate in candidates: + try: + resolved = candidate.resolve(strict=False) + if not _within(resolved, root) or not resolved.is_dir() or not (resolved / _EVENTS_FILENAME).is_file(): + continue + validate_native_id(candidate.name) + except (OSError, ValueError): + continue + result.append(candidate.name) + return sorted(result) + + +def read_transcript( + paths: SourcePaths, + native_id: str, + cursor: dict, + *, + max_records: int = 1000, + max_bytes: int = 4_194_304, +) -> SourceSlice: + """Read a stable bounded snapshot of one transcript. + + A newline is the commit boundary: a trailing partial JSON or UTF-8 line is + left untouched for the next read. Cursors retain a snapshot endpoint so a + caller can drain the state observed at the start even while Copilot appends. + """ + + if max_records < 0 or max_bytes < 0: + raise ValueError("max_records and max_bytes must be non-negative") + directory = _session_directory(paths, native_id) + event_path = directory / _EVENTS_FILENAME + diagnostics: list[dict[str, Any]] = [] + observed_at = _observed_at() + workspace, cwd, workspace_diagnostics = _workspace_record(paths, native_id, directory, observed_at) + diagnostics.extend(workspace_diagnostics) + records: list[SourceRecord] = [] + + try: + stat = event_path.stat() + if not event_path.is_file(): + raise FileNotFoundError(event_path) + generation = _generation(event_path) + except OSError: + diagnostics.append(_diagnostic("transcript_unavailable", "events.jsonl is unavailable; it is not considered complete", file=str(event_path))) + return {"records": records, "next_cursor": dict(cursor), "diagnostics": diagnostics, "cwd": cwd, "exhausted": False} + + size = stat.st_size + prior_generation = cursor.get("file_generation") + prior_offset = cursor.get("byte_offset", 0) + if not isinstance(prior_offset, int) or prior_offset < 0: + prior_offset = 0 + diagnostics.append(_diagnostic("transcript_cursor_invalid", "invalid byte offset; replaying transcript", file=str(event_path))) + reset = prior_generation is not None and prior_generation != generation + if prior_offset > size: + reset = True + anchor_start = cursor.get("continuity_start") + anchor_digest = cursor.get("continuity_digest") + if not reset and prior_offset and isinstance(anchor_start, int) and isinstance(anchor_digest, str): + try: + current_start, current_digest = _continuity_anchor(event_path, prior_offset) + except OSError: + current_start, current_digest = -1, "" + if (current_start, current_digest) != (anchor_start, anchor_digest): + reset = True + if reset: + diagnostics.append(_diagnostic("transcript_replaced", "transcript was replaced or truncated; replaying from byte zero", file=str(event_path), previous_generation=prior_generation, file_generation=generation)) + prior_offset = 0 + + prior_end = cursor.get("snapshot_end") + if reset or not isinstance(prior_end, int) or prior_end < prior_offset: + snapshot_end = size + elif prior_offset < prior_end: + snapshot_end = min(prior_end, size) + else: + snapshot_end = size + + # Metadata is a snapshot too. It is included when changed, but it does + # not prevent an events-only caller from making progress at a tiny bound. + workspace_digest = workspace["locator"]["content_digest"] if workspace else None + workspace_emitted = False + if workspace is not None and cursor.get("workspace_digest") != workspace_digest and max_records > 0: + records.append(workspace) + workspace_emitted = True + + offset = prior_offset + consumed = 0 + limit_hit = False + try: + with event_path.open("rb") as stream: + stream.seek(offset) + while offset < snapshot_end: + remaining = snapshot_end - offset + line = stream.readline(remaining) + if not line or not line.endswith(b"\n"): + # A line extending beyond the observed snapshot is not yet + # a source record, even if its prefix happens to decode. + break + if len(records) >= max_records: + limit_hit = True + break + line_size = len(line) + if consumed and consumed + line_size > max_bytes: + limit_hit = True + break + if not consumed and line_size > max_bytes: + diagnostics.append(_diagnostic("transcript_record_oversize", "one complete transcript record exceeds max_bytes and was accepted for progress", file=str(event_path), byte_offset=offset, byte_length=line_size, max_bytes=max_bytes)) + record, diagnostic = _record_for_line(paths, native_id, generation, offset, line, observed_at) + records.append(record) + if diagnostic is not None: + diagnostics.append(diagnostic) + offset += line_size + consumed += line_size + except OSError: + diagnostics.append(_diagnostic("transcript_read_failed", "events.jsonl could not be read; it is not considered complete", file=str(event_path))) + return {"records": records, "next_cursor": dict(cursor), "diagnostics": diagnostics, "cwd": cwd, "exhausted": False} + + next_cursor: dict[str, Any] = { + "byte_offset": offset, + "file_generation": generation, + "snapshot_end": snapshot_end, + } + try: + anchor_start, anchor_digest = _continuity_anchor(event_path, offset) + next_cursor["continuity_start"] = anchor_start + next_cursor["continuity_digest"] = anchor_digest + except OSError: + # The read above remains useful. A later invocation will report the + # unavailable source instead of pretending that it reached completion. + pass + if workspace_digest is not None and (workspace_emitted or cursor.get("workspace_digest") == workspace_digest): + next_cursor["workspace_digest"] = workspace_digest + exhausted = offset >= snapshot_end and not limit_hit + return {"records": records, "next_cursor": next_cursor, "diagnostics": diagnostics, "cwd": cwd, "exhausted": exhausted} From 888dc26f3a44eaa980308e954288edac31c25e76 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Thu, 10 Sep 2026 14:35:59 -0700 Subject: [PATCH 09/88] Add comprehensive tests for Copilot SQLite database reader. Cover discovery, all three allowed tables, WAL reads, row revisions, pagination, schema diagnostics, and cli-1.0.83 usage fixture ingestion. Co-authored-by: Cursor --- tests/test_copilot_database.py | 644 +++++++++++++++++++++++++++++++++ 1 file changed, 644 insertions(+) create mode 100644 tests/test_copilot_database.py diff --git a/tests/test_copilot_database.py b/tests/test_copilot_database.py new file mode 100644 index 0000000..6b10099 --- /dev/null +++ b/tests/test_copilot_database.py @@ -0,0 +1,644 @@ +"""Behavioral tests for the Copilot SQLite database reader.""" + +from __future__ import annotations + +import json +import os +import sqlite3 +import threading +from pathlib import Path +from typing import Any + +import pytest + +from thirdeye.platforms.copilot.database import discover_database_sessions, read_database +from thirdeye.platforms.copilot.identity import resolve_sources + +FIXTURES = Path(__file__).parent / "fixtures" / "copilot" +CLI_FIXTURE = FIXTURES / "cli-1.0.83" +NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" + + +def _load_json(path: Path) -> Any: + return json.loads(path.read_text(encoding="utf-8")) + + +def _create_standard_schema(connection: sqlite3.Connection) -> None: + connection.executescript( + """ + CREATE TABLE sessions ( + id TEXT PRIMARY KEY, + cwd TEXT, + created_at TEXT + ); + CREATE TABLE turns ( + id INTEGER PRIMARY KEY, + session_id TEXT NOT NULL, + turn_index INTEGER, + content TEXT, + updated_at TEXT + ); + CREATE TABLE assistant_usage_events ( + id INTEGER PRIMARY KEY, + session_id TEXT NOT NULL, + turn_index INTEGER, + agent_id TEXT, + parent_tool_call_id TEXT, + model TEXT, + input_tokens INTEGER, + output_tokens INTEGER, + cache_read_tokens INTEGER, + cache_write_tokens INTEGER, + reasoning_tokens INTEGER, + total_nano_aiu INTEGER, + request_multiplier REAL, + duration_ms INTEGER, + time_to_first_token_ms REAL, + output_ttft_ms REAL, + inter_token_latency_ms REAL, + initiator TEXT, + api_endpoint TEXT, + reasoning_effort TEXT, + finish_reason TEXT, + content_filter_triggered INTEGER, + token_details_json TEXT, + created_at TEXT + ); + """ + ) + + +def _seed_session( + connection: sqlite3.Connection, + session_id: str, + *, + cwd: str = "/tmp/workspace", +) -> None: + connection.execute( + "INSERT INTO sessions (id, cwd, created_at) VALUES (?, ?, ?)", + (session_id, cwd, "2026-09-10T17:08:00.000Z"), + ) + + +def _write_database( + home: Path, + *, + session_id: str = "session-a", + cwd: str = "/tmp/workspace", + turns: list[tuple[int, str]] | None = None, + usage_rows: list[dict[str, Any]] | None = None, + extra_sessions: list[str] | None = None, + schema_sql: str | None = None, + journal_mode: str = "WAL", + seed_session: bool = True, +) -> Path: + home.mkdir(parents=True, exist_ok=True) + database = home / "session-store.db" + connection = sqlite3.connect(database) + try: + if schema_sql is not None: + connection.executescript(schema_sql) + else: + _create_standard_schema(connection) + if journal_mode: + connection.execute(f"PRAGMA journal_mode={journal_mode}") + if seed_session: + _seed_session(connection, session_id, cwd=cwd) + for other in extra_sessions or (): + _seed_session(connection, other, cwd=f"/tmp/{other}") + for turn_id, content in turns or (): + connection.execute( + "INSERT INTO turns (id, session_id, turn_index, content, updated_at) " + "VALUES (?, ?, ?, ?, ?)", + (turn_id, session_id, turn_id, content, "2026-09-10T17:08:10.000Z"), + ) + for row in usage_rows or (): + columns = ", ".join(row) + placeholders = ", ".join("?" for _ in row) + connection.execute( + f"INSERT INTO assistant_usage_events ({columns}) VALUES ({placeholders})", + tuple(row.values()), + ) + connection.commit() + finally: + connection.close() + return database + + +def _paths(home: Path) -> dict[str, str]: + return resolve_sources(home) + + +def _records_for_table(records: list[dict[str, Any]], table: str) -> list[dict[str, Any]]: + return [record for record in records if record["payload"]["table"] == table] + + +def _collect_all( + paths: dict[str, str], + native_id: str, + *, + max_records: int = 1000, +) -> list[dict[str, Any]]: + cursor: dict[str, Any] = {} + collected: list[dict[str, Any]] = [] + while True: + slice_ = read_database(paths, native_id, cursor, max_records=max_records) + collected.extend(slice_["records"]) + if slice_["exhausted"]: + return collected + cursor = slice_["next_cursor"] + + +# --- module boundaries --- + + +def test_database_module_has_no_forbidden_imports(): + import thirdeye.platforms.copilot.database as database + + source = Path(database.__file__).read_text(encoding="utf-8") + assert "UsageStore" not in source + assert "usage_store" not in source + assert "from thirdeye.store" not in source + assert "import thirdeye.store" not in source + assert "transcript" not in source + + +# --- discovery --- + + +def test_discover_database_sessions_returns_sorted_unique_ids(tmp_path: Path): + home = tmp_path / "copilot" + _write_database( + home, + session_id="session-b", + extra_sessions=["session-a", "session-c"], + ) + paths = _paths(home) + assert discover_database_sessions(paths) == ["session-a", "session-b", "session-c"] + + +def test_discover_database_sessions_empty_when_database_missing(tmp_path: Path): + home = tmp_path / "copilot" + home.mkdir() + assert discover_database_sessions(_paths(home)) == [] + + +def test_discover_database_sessions_empty_when_sessions_table_missing(tmp_path: Path): + home = tmp_path / "copilot" + home.mkdir() + database = home / "session-store.db" + connection = sqlite3.connect(database) + connection.execute("CREATE TABLE turns (id INTEGER PRIMARY KEY, session_id TEXT)") + connection.commit() + connection.close() + assert discover_database_sessions(_paths(home)) == [] + + +# --- missing / empty sources --- + + +def test_read_database_missing_file_reports_diagnostic(tmp_path: Path): + home = tmp_path / "copilot" + home.mkdir() + paths = _paths(home) + slice_ = read_database(paths, "session-a", {}) + assert slice_["records"] == [] + assert slice_["exhausted"] is True + assert slice_["cwd"] is None + assert slice_["diagnostics"] == [ + { + "code": "copilot_database_missing", + "message": "Copilot session database is not present", + "path": paths["database"], + } + ] + + +def test_read_database_non_file_path_reports_unreadable(tmp_path: Path): + home = tmp_path / "copilot" + home.mkdir() + database = home / "session-store.db" + database.mkdir() + paths = _paths(home) + slice_ = read_database(paths, "session-a", {}) + assert slice_["records"] == [] + assert slice_["diagnostics"][0]["code"] == "copilot_database_unreadable" + + +def test_read_database_empty_session_returns_only_session_row(tmp_path: Path): + home = tmp_path / "copilot" + _write_database(home, session_id="session-a") + slice_ = read_database(_paths(home), "session-a", {}) + assert len(slice_["records"]) == 1 + assert slice_["records"][0]["payload"]["table"] == "sessions" + assert slice_["exhausted"] is True + assert slice_["cwd"] == "/tmp/workspace" + + +def test_read_database_unknown_session_is_empty_exhausted(tmp_path: Path): + home = tmp_path / "copilot" + _write_database(home, session_id="session-a") + slice_ = read_database(_paths(home), "missing-session", {}) + assert slice_["records"] == [] + assert slice_["exhausted"] is True + assert slice_["cwd"] is None + + +# --- all three tables --- + + +def test_read_database_reads_sessions_turns_and_usage_events(tmp_path: Path): + home = tmp_path / "copilot" + usage_row = { + "id": 13, + "session_id": "session-a", + "turn_index": 0, + "agent_id": None, + "parent_tool_call_id": None, + "model": "gpt-5.6-luna", + "input_tokens": 10, + "output_tokens": 5, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "reasoning_tokens": 0, + "total_nano_aiu": 1000, + "request_multiplier": 1.0, + "duration_ms": 100, + "time_to_first_token_ms": 50.0, + "output_ttft_ms": 50.0, + "inter_token_latency_ms": None, + "initiator": "user", + "api_endpoint": "ws:/responses", + "reasoning_effort": "medium", + "finish_reason": "stop", + "content_filter_triggered": 0, + "token_details_json": "[]", + "created_at": "2026-09-10T17:08:24.498Z", + } + _write_database( + home, + session_id="session-a", + turns=[(1, "first turn"), (2, "second turn")], + usage_rows=[usage_row], + ) + records = _collect_all(_paths(home), "session-a") + tables = {record["payload"]["table"] for record in records} + assert tables == {"assistant_usage_events", "sessions", "turns"} + assert len(_records_for_table(records, "turns")) == 2 + assert len(_records_for_table(records, "assistant_usage_events")) == 1 + + +def test_cli_fixture_usage_rows_are_readable_from_database(tmp_path: Path): + home = tmp_path / "copilot" + usage_rows = _load_json(CLI_FIXTURE / "assistant-usage-events.json") + _write_database( + home, + session_id=NATIVE_SESSION_ID, + cwd="/tmp/probe", + turns=[(0, "turn-0"), (1, "turn-1")], + usage_rows=usage_rows, + ) + records = _collect_all(_paths(home), NATIVE_SESSION_ID) + usage_records = _records_for_table(records, "assistant_usage_events") + assert len(usage_records) == 6 + assert {record["payload"]["row"]["id"] for record in usage_records} == { + 13, + 14, + 15, + 16, + 17, + 18, + } + assert usage_records[0]["source_kind"] == "database" + assert usage_records[0]["ts"] == "2026-09-10T17:08:24.498Z" + assert usage_records[0]["native_session_id"] == NATIVE_SESSION_ID + + +# --- source record shape --- + + +def test_source_record_includes_revision_and_generation(tmp_path: Path): + home = tmp_path / "copilot" + _write_database(home, session_id="session-a", turns=[(7, "first")]) + record = read_database(_paths(home), "session-a", {})["records"][0] + assert record["source_id"].startswith(f"copilot-db:{_paths(home)['source_key']}:session-a:") + assert record["locator"]["table"] == "sessions" + assert record["locator"]["content_revision"].startswith("sha256:") + assert record["locator"]["generation"].startswith("sha256:") + assert record["payload"]["row"]["id"] == "session-a" + + +# --- WAL / live changes --- + + +def test_reads_uncheckpointed_wal_commits(tmp_path: Path): + home = tmp_path / "copilot" + database = _write_database(home, session_id="session-a", turns=[(1, "checkpointed")]) + connection = sqlite3.connect(database) + connection.execute("PRAGMA journal_mode=WAL") + connection.execute( + "INSERT INTO turns (id, session_id, turn_index, content, updated_at) " + "VALUES (?, ?, ?, ?, ?)", + (2, "session-a", 2, "wal-only", "2026-09-10T17:08:20.000Z"), + ) + connection.execute( + "INSERT INTO assistant_usage_events (id, session_id, turn_index, model, created_at) " + "VALUES (?, ?, ?, ?, ?)", + (99, "session-a", 2, "gpt-test", "2026-09-10T17:08:21.000Z"), + ) + connection.commit() + connection.close() + + records = _collect_all(_paths(home), "session-a") + turn_contents = { + record["payload"]["row"]["content"] + for record in records + if record["payload"]["table"] == "turns" + } + assert turn_contents == {"checkpointed", "wal-only"} + assert any( + record["payload"]["table"] == "assistant_usage_events" + and record["payload"]["row"]["id"] == 99 + for record in records + ) + + +def test_late_database_rows_visible_without_transcript_changes(tmp_path: Path): + home = tmp_path / "copilot" + database = _write_database(home, session_id="session-a", turns=[(1, "initial")]) + first = _collect_all(_paths(home), "session-a") + + connection = sqlite3.connect(database) + connection.execute( + "INSERT INTO assistant_usage_events (id, session_id, turn_index, model, created_at) " + "VALUES (?, ?, ?, ?, ?)", + (42, "session-a", 1, "gpt-late", "2026-09-10T17:09:00.000Z"), + ) + connection.commit() + connection.close() + + second = _collect_all(_paths(home), "session-a") + assert len(second) == len(first) + 1 + assert any( + record["payload"]["table"] == "assistant_usage_events" + and record["payload"]["row"]["model"] == "gpt-late" + for record in second + ) + + +# --- revisions and row reuse --- + + +def test_updated_turn_produces_new_content_revision(tmp_path: Path): + home = tmp_path / "copilot" + database = _write_database(home, session_id="session-a", turns=[(7, "first")]) + paths = _paths(home) + + before = _records_for_table(_collect_all(paths, "session-a"), "turns")[0] + connection = sqlite3.connect(database) + connection.execute("UPDATE turns SET content = ?, updated_at = ? WHERE id = ?", ("updated", "2026-09-10T17:10:00.000Z", 7)) + connection.commit() + connection.close() + + after = _records_for_table(_collect_all(paths, "session-a"), "turns")[0] + assert before["locator"]["primary_key"] == after["locator"]["primary_key"] == 7 + assert before["locator"]["content_revision"] != after["locator"]["content_revision"] + assert before["source_id"] != after["source_id"] + assert after["payload"]["row"]["content"] == "updated" + + +def test_unchanged_resnapshot_keeps_stable_source_id(tmp_path: Path): + home = tmp_path / "copilot" + paths = _paths(home) + _write_database(home, session_id="session-a", turns=[(7, "stable")]) + first = read_database(paths, "session-a", {})["records"] + second = read_database(paths, "session-a", {})["records"] + turn_first = _records_for_table(first, "turns")[0] + turn_second = _records_for_table(second, "turns")[0] + assert turn_first["source_id"] == turn_second["source_id"] + assert turn_first["locator"]["content_revision"] == turn_second["locator"]["content_revision"] + + +def test_database_replacement_changes_generation_and_resets_cursor(tmp_path: Path): + home = tmp_path / "copilot" + database = _write_database(home, session_id="session-a", turns=[(1, "a"), (2, "b")]) + paths = _paths(home) + + first = read_database(paths, "session-a", {}, max_records=1) + old_generation = first["next_cursor"]["database_generation"] + assert first["exhausted"] is False + + os.remove(database) + _write_database(home, session_id="session-a", turns=[(1, "replacement")]) + + after_replace = read_database(paths, "session-a", first["next_cursor"], max_records=10) + assert after_replace["next_cursor"]["database_generation"] != old_generation + assert after_replace["next_cursor"]["database_offset"] == len(after_replace["records"]) + assert {record["payload"]["row"]["content"] for record in _records_for_table(after_replace["records"], "turns")} == { + "replacement" + } + + +# --- pagination --- + + +def test_read_database_paginates_with_cursor(tmp_path: Path): + home = tmp_path / "copilot" + _write_database( + home, + session_id="session-a", + turns=[(index, f"turn-{index}") for index in range(1, 6)], + journal_mode="DELETE", + ) + paths = _paths(home) + + page_one = read_database(paths, "session-a", {}, max_records=2) + assert len(page_one["records"]) == 2 + assert page_one["exhausted"] is False + assert page_one["next_cursor"]["database_offset"] == 2 + + page_two = read_database(paths, "session-a", page_one["next_cursor"], max_records=2) + assert len(page_two["records"]) == 2 + assert page_two["exhausted"] is False + + remaining = read_database(paths, "session-a", page_two["next_cursor"], max_records=10) + assert len(remaining["records"]) == 2 # two remaining turns after five total rows + assert remaining["exhausted"] is True + + +def test_stale_cursor_resets_when_generation_changes(tmp_path: Path): + home = tmp_path / "copilot" + database = _write_database(home, session_id="session-a", turns=[(1, "one"), (2, "two")]) + paths = _paths(home) + first = read_database(paths, "session-a", {}, max_records=1) + stale_cursor = { + "database_generation": "sha256:deadbeef", + "database_offset": 99, + } + + os.remove(database) + _write_database(home, session_id="session-a", turns=[(1, "fresh")]) + + slice_ = read_database(paths, "session-a", stale_cursor, max_records=10) + assert slice_["next_cursor"]["database_offset"] == len(slice_["records"]) + + +# --- schema diagnostics --- + + +def test_missing_table_reports_diagnostic_but_reads_available_tables(tmp_path: Path): + home = tmp_path / "copilot" + home.mkdir() + database = home / "session-store.db" + connection = sqlite3.connect(database) + connection.executescript( + """ + CREATE TABLE sessions (id TEXT PRIMARY KEY, cwd TEXT); + CREATE TABLE turns (id INTEGER PRIMARY KEY, session_id TEXT, content TEXT); + """ + ) + connection.execute("INSERT INTO sessions VALUES ('session-a', '/tmp')") + connection.execute("INSERT INTO turns VALUES (1, 'session-a', 'only-turn')") + connection.commit() + connection.close() + + slice_ = read_database(_paths(home), "session-a", {}) + codes = {diag["code"] for diag in slice_["diagnostics"]} + assert "copilot_database_table_missing" in codes + tables = {record["payload"]["table"] for record in slice_["records"]} + assert tables == {"sessions", "turns"} + + +def test_missing_session_column_reports_diagnostic(tmp_path: Path): + home = tmp_path / "copilot" + schema = """ + CREATE TABLE sessions (id TEXT PRIMARY KEY, cwd TEXT); + CREATE TABLE turns (id INTEGER PRIMARY KEY, content TEXT); + CREATE TABLE assistant_usage_events (id INTEGER PRIMARY KEY, model TEXT); + """ + _write_database(home, session_id="session-a", schema_sql=schema, seed_session=False) + connection = sqlite3.connect(home / "session-store.db") + connection.execute("INSERT INTO sessions VALUES ('session-a', '/tmp')") + connection.commit() + connection.close() + slice_ = read_database(_paths(home), "session-a", {}) + codes = {diag["code"] for diag in slice_["diagnostics"]} + assert "copilot_database_missing_session_column" in codes + assert {record["payload"]["table"] for record in slice_["records"]} == {"sessions"} + + +def test_missing_primary_key_reports_diagnostic(tmp_path: Path): + home = tmp_path / "copilot" + schema = """ + CREATE TABLE sessions (session_id TEXT, cwd TEXT); + CREATE TABLE turns (session_id TEXT, content TEXT); + CREATE TABLE assistant_usage_events (session_id TEXT, model TEXT); + """ + _write_database(home, session_id="session-a", schema_sql=schema, seed_session=False) + connection = sqlite3.connect(home / "session-store.db") + connection.execute("INSERT INTO sessions VALUES ('session-a', '/tmp')") + connection.commit() + connection.close() + slice_ = read_database(_paths(home), "session-a", {}) + codes = {diag["code"] for diag in slice_["diagnostics"]} + assert "copilot_database_missing_primary_key" in codes + assert slice_["records"] == [] + + +def test_optional_columns_are_preserved_when_present(tmp_path: Path): + home = tmp_path / "copilot" + schema = """ + CREATE TABLE sessions (id TEXT PRIMARY KEY, cwd TEXT, extra_flag INTEGER); + CREATE TABLE turns (id INTEGER PRIMARY KEY, session_id TEXT, content TEXT); + CREATE TABLE assistant_usage_events (id INTEGER PRIMARY KEY, session_id TEXT, model TEXT); + """ + _write_database(home, session_id="session-a", schema_sql=schema, seed_session=False) + connection = sqlite3.connect(home / "session-store.db") + connection.execute( + "INSERT INTO sessions (id, cwd, extra_flag) VALUES ('session-a', '/tmp', 1)" + ) + connection.commit() + connection.close() + + session_record = _records_for_table(_collect_all(_paths(home), "session-a"), "sessions")[0] + assert session_record["payload"]["row"]["extra_flag"] == 1 + + +# --- locking --- + + +def test_busy_database_returns_actionable_diagnostic(tmp_path: Path): + home = tmp_path / "copilot" + database = _write_database(home, session_id="session-a", turns=[(1, "locked")]) + paths = _paths(home) + hold = threading.Event() + release = threading.Event() + error: list[str] = [] + + def hold_exclusive_lock() -> None: + connection = sqlite3.connect(database, timeout=30) + try: + connection.execute("BEGIN EXCLUSIVE") + hold.set() + release.wait(timeout=5) + finally: + connection.close() + + thread = threading.Thread(target=hold_exclusive_lock, daemon=True) + thread.start() + assert hold.wait(timeout=2) + + slice_ = read_database(paths, "session-a", {}) + release.set() + thread.join(timeout=2) + + if any(diag["code"] == "copilot_database_busy" for diag in slice_["diagnostics"]): + assert slice_["records"] == [] + assert slice_["exhausted"] is True + else: + # Some platforms may not block read-only URIs on EXCLUSIVE; accept successful read. + assert slice_["records"] + + +# --- validation --- + + +def test_read_database_rejects_invalid_native_id(tmp_path: Path): + home = tmp_path / "copilot" + _write_database(home, session_id="session-a") + with pytest.raises(ValueError, match="native session ID"): + read_database(_paths(home), "../escape", {}) + + +def test_read_database_rejects_invalid_max_records(tmp_path: Path): + home = tmp_path / "copilot" + _write_database(home, session_id="session-a") + with pytest.raises(ValueError, match="max_records"): + read_database(_paths(home), "session-a", {}, max_records=0) + + +# --- special values --- + + +def test_bytes_column_is_base64_encoded_in_payload(tmp_path: Path): + home = tmp_path / "copilot" + schema = """ + CREATE TABLE sessions (id TEXT PRIMARY KEY, cwd TEXT); + CREATE TABLE turns (id INTEGER PRIMARY KEY, session_id TEXT, blob_data BLOB); + CREATE TABLE assistant_usage_events (id INTEGER PRIMARY KEY, session_id TEXT); + """ + _write_database(home, session_id="session-a", schema_sql=schema, seed_session=False) + connection = sqlite3.connect(home / "session-store.db") + connection.execute("INSERT INTO sessions VALUES ('session-a', '/tmp')") + connection.execute( + "INSERT INTO turns (id, session_id, blob_data) VALUES (?, ?, ?)", + (1, "session-a", b"\x00\xff"), + ) + connection.commit() + connection.close() + + turn = _records_for_table(_collect_all(_paths(home), "session-a"), "turns")[0] + assert turn["payload"]["row"]["blob_data"] == { + "encoding": "base64", + "data": "AP8=", + } From 4b4c7cfab6e31983f7e63d33c5da9def8243d9c9 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Thu, 10 Sep 2026 14:36:24 -0700 Subject: [PATCH 10/88] Add Copilot archive journal behavioral and recovery tests. Cover commit/cursor/retrieval, timestamp preservation, deduplication, lifecycle hooks, prefix-collision rejection, fault-injected crash recovery, and stale-cursor contention with synthetic SourceBatch inputs. Co-authored-by: Cursor --- tests/test_copilot_archive.py | 319 +++++++++++++++++++++++++++++++++ tests/test_copilot_recovery.py | 258 ++++++++++++++++++++++++++ 2 files changed, 577 insertions(+) create mode 100644 tests/test_copilot_archive.py create mode 100644 tests/test_copilot_recovery.py diff --git a/tests/test_copilot_archive.py b/tests/test_copilot_archive.py new file mode 100644 index 0000000..449dc86 --- /dev/null +++ b/tests/test_copilot_archive.py @@ -0,0 +1,319 @@ +"""Behavioral tests for the Copilot evidence archive (commit, cursor, retrieval).""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +import thirdeye.platforms.copilot.archive as archive_mod +from thirdeye.config import Config +from thirdeye.meta import read_meta +from thirdeye.paths import meta_path, session_dir +from thirdeye.platforms.copilot.archive import commit_batch, iter_captured_records, load_cursor +from thirdeye.platforms.copilot.constants import PLATFORM_NAME, SOURCE_SCHEMA_VERSION +from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id +from thirdeye.platforms.copilot.state import journal_path, read_json, state_path +from thirdeye.platforms.copilot.types import SourceBatch, SourcePaths, SourceRecord +from thirdeye.reader import SessionReader + +FIXTURES = Path(__file__).parent / "fixtures" / "copilot" / "v1-cases" +NATIVE_ID = "session-a" + + +def _record( + source_id: str, + *, + source_kind: str = "transcript", + native_session_id: str = NATIVE_ID, + ts: str | None = "2026-09-10T17:08:24.000Z", + observed_at: str = "2026-09-10T17:08:25.000Z", + payload: dict[str, Any] | None = None, +) -> SourceRecord: + return { + "source_id": source_id, + "source_kind": source_kind, + "native_session_id": native_session_id, + "ts": ts, + "observed_at": observed_at, + "payload": payload or {"schema_version": 1, "type": "user.message"}, + "locator": {"file": "events.jsonl", "offset": 0}, + } + + +def _batch( + paths: SourcePaths, + records: list[SourceRecord], + *, + next_cursor: dict[str, Any] | None = None, + base_cursor: dict[str, Any] | None = None, + diagnostics: list[dict[str, Any]] | None = None, + cwd: str | None = "/proj", +) -> SourceBatch: + cursor = dict(next_cursor or {"generation": 1}) + if base_cursor is not None: + cursor["base_cursor"] = base_cursor + return { + "source_key": paths["source_key"], + "native_session_id": NATIVE_ID, + "cwd": cwd, + "records": records, + "next_cursor": cursor, + "diagnostics": diagnostics or [], + } + + +@pytest.fixture +def copilot_home(tmp_path: Path) -> Path: + home = tmp_path / "copilot-home" + home.mkdir() + return home + + +@pytest.fixture +def config(tmp_path: Path) -> Config: + return Config(root=tmp_path / "thirdeye") + + +@pytest.fixture +def paths(copilot_home: Path) -> SourcePaths: + return resolve_sources(copilot_home) + + +def _session_directory(config: Config, paths: SourcePaths) -> Path: + stored = stored_session_id(paths, NATIVE_ID) + return session_dir(config.root, PLATFORM_NAME, stored) + + +def test_commit_batch_writes_records_and_advances_cursor(config: Config, paths: SourcePaths) -> None: + records = [_record("key/a/event-1"), _record("key/a/event-2")] + result = commit_batch(config, paths, _batch(paths, records, next_cursor={"offset": 2})) + + assert result == { + "sessions": 1, + "records_written": 2, + "duplicate_records": 0, + "pending": 0, + "errors": 0, + } + assert load_cursor(config, paths, NATIVE_ID) == {"offset": 2} + captured = list(iter_captured_records(config, stored_session_id(paths, NATIVE_ID))) + assert {r["source_id"] for r in captured} == {"key/a/event-1", "key/a/event-2"} + + +def test_load_cursor_returns_empty_for_unknown_session(config: Config, paths: SourcePaths) -> None: + assert load_cursor(config, paths, NATIVE_ID) == {} + + +def test_load_cursor_returns_defensive_copy(config: Config, paths: SourcePaths) -> None: + commit_batch(config, paths, _batch(paths, [_record("key/a/event-1")], next_cursor={"offset": 1})) + cursor = load_cursor(config, paths, NATIVE_ID) + cursor["offset"] = 999 + assert load_cursor(config, paths, NATIVE_ID) == {"offset": 1} + + +def test_iter_captured_records_empty_when_session_missing(config: Config) -> None: + assert list(iter_captured_records(config, "copilot-missing-session")) == [] + + +@pytest.mark.parametrize( + ("source_kind", "event_type"), + [ + ("transcript", "copilot_transcript"), + ("database", "copilot_database"), + ("hook", "copilot_hook"), + ("metadata", "copilot_metadata"), + ("unknown", "copilot_metadata"), + ], +) +def test_event_type_mapping( + config: Config, + paths: SourcePaths, + source_kind: str, + event_type: str, +) -> None: + record = _record("key/a/typed", source_kind=source_kind) + commit_batch(config, paths, _batch(paths, [record])) + events = list(SessionReader(_session_directory(config, paths)).iter_events()) + assert len(events) == 1 + assert events[0]["t"] == event_type + envelope = events[0]["data"] + assert envelope["schema_version"] == SOURCE_SCHEMA_VERSION + assert envelope["source_record"]["source_id"] == "key/a/typed" + + +def test_original_source_timestamp_is_retained(config: Config, paths: SourcePaths) -> None: + source_ts = "2026-09-10T17:08:24.000Z" + commit_batch( + config, + paths, + _batch(paths, [_record("key/a/ts", ts=source_ts, observed_at="2026-09-10T17:08:99.000Z")]), + ) + event = SessionReader(_session_directory(config, paths)).get_event(0) + assert event["ts"] == source_ts + + +def test_missing_source_time_falls_back_to_observed_at(config: Config, paths: SourcePaths) -> None: + observed = "2026-09-10T17:08:25.000Z" + commit_batch(config, paths, _batch(paths, [_record("key/a/no-ts", ts=None, observed_at=observed)])) + event = SessionReader(_session_directory(config, paths)).get_event(0) + assert event["ts"] == observed + + state = read_json(state_path(_session_directory(config, paths))) + assert state is not None + kinds = {item["kind"] for item in state["health"]["diagnostics"]} + assert "missing_source_time" in kinds + + +def test_duplicate_source_ids_are_skipped(config: Config, paths: SourcePaths) -> None: + record = _record("key/a/dup") + first = commit_batch(config, paths, _batch(paths, [record], next_cursor={"offset": 1})) + second = commit_batch( + config, + paths, + _batch(paths, [record], next_cursor={"offset": 2}, base_cursor={"offset": 1}), + ) + assert first["records_written"] == 1 + assert second["records_written"] == 0 + assert second["duplicate_records"] == 1 + assert len(list(iter_captured_records(config, stored_session_id(paths, NATIVE_ID)))) == 1 + + +def test_batch_source_key_must_match_paths(config: Config, paths: SourcePaths) -> None: + batch = _batch(paths, [_record("key/a/event-1")]) + batch["source_key"] = "0" * 64 + with pytest.raises(ValueError, match="source_key does not match"): + commit_batch(config, paths, batch) + + +def test_batch_rejects_foreign_native_session_id(config: Config, paths: SourcePaths) -> None: + foreign = _record("key/a/foreign", native_session_id="other-session") + with pytest.raises(ValueError, match="another native session"): + commit_batch(config, paths, _batch(paths, [foreign])) + + +def test_source_key_prefix_collision_is_rejected( + config: Config, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Archive reuse must compare full source_key metadata, not just stored ID prefix.""" + import hashlib + + from thirdeye.platforms.copilot import identity as identity_mod + from thirdeye.store import Store + + case = json.loads((FIXTURES / "source-key-prefix-collision.json").read_text(encoding="utf-8")) + second_home = tmp_path / "home-b" + second_home.mkdir() + second_key = case["homes"][1]["source_key"] + + original_source_key = identity_mod._source_key + + def keyed_source_key(home: Path) -> str: + if home.resolve() == second_home.resolve(): + return second_key + return original_source_key(home) + + monkeypatch.setattr(identity_mod, "_source_key", keyed_source_key) + + stored_id = case["colliding_stored_session_id"] + first_key = case["homes"][0]["source_key"] + Store(config).open_session( + stored_id, + platform=PLATFORM_NAME, + cwd="/proj", + extra={ + "copilot": { + "schema_version": SOURCE_SCHEMA_VERSION, + "source_key": first_key, + "source_home": case["homes"][0]["home"], + "native_session_id": NATIVE_ID, + } + }, + ).flush_and_detach() + + second_paths: SourcePaths = { + "home": str(second_home.resolve()), + "source_key": second_key, + "session_root": str((second_home / "session-state").resolve()), + "database": str((second_home / "session-store.db").resolve()), + } + assert hashlib.sha256(str(second_home.resolve()).encode()).hexdigest() != second_key + with pytest.raises(ValueError, match="source-key prefix collision"): + commit_batch(config, second_paths, _batch(second_paths, [_record("key/b/second")])) + + +def test_provisional_session_metadata_is_created(config: Config, paths: SourcePaths) -> None: + commit_batch(config, paths, _batch(paths, [_record("key/a/hook", source_kind="hook")])) + meta = read_meta(meta_path(_session_directory(config, paths))) + assert meta is not None + assert meta.platform == PLATFORM_NAME + identity = meta.extra["copilot"] + assert identity["source_key"] == paths["source_key"] + assert identity["native_session_id"] == NATIVE_ID + + +def test_session_end_closes_and_resume_reopens(config: Config, paths: SourcePaths) -> None: + close_record = _record( + "key/a/close", + source_kind="hook", + payload={"event": "sessionEnd", "context": {}}, + ) + commit_batch(config, paths, _batch(paths, [close_record])) + meta = read_meta(meta_path(_session_directory(config, paths))) + assert meta is not None + assert meta.status == "closed" + assert meta.ended_at is not None + + reopen_record = _record( + "key/a/reopen", + source_kind="hook", + payload={"event": "resume", "context": {}}, + ) + commit_batch( + config, + paths, + _batch(paths, [reopen_record], next_cursor={"generation": 2}, base_cursor={"generation": 1}), + ) + meta = read_meta(meta_path(_session_directory(config, paths))) + assert meta is not None + assert meta.status == "open" + assert meta.ended_at is None + + +def test_child_stop_does_not_close_session(config: Config, paths: SourcePaths) -> None: + child_stop = _record( + "key/a/child-stop", + source_kind="hook", + payload={ + "event": "sessionEnd", + "context": {"agent_id": "child-agent", "parent_tool_call_id": "tool-1"}, + }, + ) + commit_batch(config, paths, _batch(paths, [child_stop])) + meta = read_meta(meta_path(_session_directory(config, paths))) + assert meta is not None + assert meta.status == "open" + assert meta.ended_at is None + + +def test_archive_module_has_no_usage_store_or_otel_imports() -> None: + source = Path(archive_mod.__file__).read_text(encoding="utf-8") + assert "UsageStore" not in source + assert "usage_store" not in source + assert "otel" not in source.lower() + + +def test_fixture_source_batch_can_be_committed(config: Config, copilot_home: Path) -> None: + raw = json.loads((FIXTURES / "source-batch.json").read_text(encoding="utf-8")) + paths = resolve_sources(copilot_home) + raw["source_key"] = paths["source_key"] + result = commit_batch(config, paths, raw) + assert result["records_written"] == 1 + assert result["errors"] == 0 + record = next(iter_captured_records(config, stored_session_id(paths, raw["native_session_id"]))) + assert record["source_kind"] == "database" + assert record["ts"] is None diff --git a/tests/test_copilot_recovery.py b/tests/test_copilot_recovery.py new file mode 100644 index 0000000..61c61e8 --- /dev/null +++ b/tests/test_copilot_recovery.py @@ -0,0 +1,258 @@ +"""Crash recovery, journal replay, and stale-cursor contention for Copilot archive.""" + +from __future__ import annotations + +from collections.abc import Iterator +from contextlib import contextmanager +from typing import Any + +import pytest + +import thirdeye.platforms.copilot.archive as archive_mod +from thirdeye.config import Config +from thirdeye.platforms.copilot.archive import commit_batch, iter_captured_records, load_cursor +from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id +from thirdeye.platforms.copilot.state import journal_path, read_json, state_path +from thirdeye.platforms.copilot.types import SourceBatch, SourcePaths, SourceRecord + +NATIVE_ID = "session-a" + + +def _record(source_id: str, *, native_session_id: str = NATIVE_ID) -> SourceRecord: + return { + "source_id": source_id, + "source_kind": "transcript", + "native_session_id": native_session_id, + "ts": "2026-09-10T17:08:24.000Z", + "observed_at": "2026-09-10T17:08:25.000Z", + "payload": {"schema_version": 1, "type": "user.message"}, + "locator": {"file": "events.jsonl", "offset": 0}, + } + + +def _batch( + paths: SourcePaths, + records: list[SourceRecord], + *, + next_cursor: dict[str, Any], + base_cursor: dict[str, Any] | None = None, +) -> SourceBatch: + cursor = dict(next_cursor) + if base_cursor is not None: + cursor["base_cursor"] = base_cursor + return { + "source_key": paths["source_key"], + "native_session_id": NATIVE_ID, + "cwd": "/proj", + "records": records, + "next_cursor": cursor, + "diagnostics": [], + } + + +@contextmanager +def fault_at(point: str) -> Iterator[None]: + seen: list[str] = [] + + def injector(name: str) -> None: + seen.append(name) + if name == point: + raise RuntimeError(f"injected fault at {point}") + + archive_mod._fault_injector = injector + try: + yield seen + finally: + archive_mod._fault_injector = None + + +@pytest.fixture +def config(tmp_path) -> Config: + return Config(root=tmp_path / "thirdeye") + + +@pytest.fixture +def paths(tmp_path) -> SourcePaths: + home = tmp_path / "copilot-home" + home.mkdir() + return resolve_sources(home) + + +def _session_dir(config: Config, paths: SourcePaths): + from thirdeye.paths import session_dir + from thirdeye.platforms.copilot.constants import PLATFORM_NAME + + return session_dir(config.root, PLATFORM_NAME, stored_session_id(paths, NATIVE_ID)) + + +@pytest.mark.parametrize( + "fault_point", + [ + "after_journal", + "after_append", + "after_checkpoint", + "after_journal_clear", + ], +) +def test_crash_at_journal_boundary_recovers_on_next_commit( + config: Config, + paths: SourcePaths, + fault_point: str, +) -> None: + directory = _session_dir(config, paths) + records = [_record("key/a/recover-1"), _record("key/a/recover-2")] + + with fault_at(fault_point): + with pytest.raises(RuntimeError, match="injected fault"): + commit_batch( + config, + paths, + _batch(paths, records, next_cursor={"generation": 1}), + ) + + if fault_point == "after_journal": + assert journal_path(directory).is_file() + assert not list(iter_captured_records(config, stored_session_id(paths, NATIVE_ID))) + + if fault_point in {"after_append", "after_checkpoint", "after_journal_clear"}: + captured_before = list(iter_captured_records(config, stored_session_id(paths, NATIVE_ID))) + assert len(captured_before) == 2 + + recovery = commit_batch( + config, + paths, + _batch( + paths, + [_record("key/a/recover-3")], + next_cursor={"generation": 2}, + base_cursor={"generation": 1}, + ), + ) + + assert recovery["records_written"] >= 1 + assert not journal_path(directory).exists() + cursor = load_cursor(config, paths, NATIVE_ID) + assert cursor["generation"] == 2 + assert cursor.get("base_cursor") == {"generation": 1} + captured = list(iter_captured_records(config, stored_session_id(paths, NATIVE_ID))) + assert {r["source_id"] for r in captured} >= {"key/a/recover-1", "key/a/recover-2", "key/a/recover-3"} + + +def test_recovery_after_journal_only_replays_uncommitted_records( + config: Config, + paths: SourcePaths, +) -> None: + directory = _session_dir(config, paths) + records = [_record("key/a/journal-only")] + + with fault_at("after_journal"): + with pytest.raises(RuntimeError): + commit_batch(config, paths, _batch(paths, records, next_cursor={"generation": 1})) + + assert journal_path(directory).is_file() + result = commit_batch( + config, + paths, + _batch(paths, [], next_cursor={"generation": 1}, base_cursor={}), + ) + assert result["records_written"] == 1 + assert result["duplicate_records"] == 0 + assert list(iter_captured_records(config, stored_session_id(paths, NATIVE_ID)))[0]["source_id"] == "key/a/journal-only" + + +def test_stale_cursor_rejects_competing_batch(config: Config, paths: SourcePaths) -> None: + first = commit_batch( + config, + paths, + _batch(paths, [_record("key/a/first")], next_cursor={"generation": 1}), + ) + assert first["errors"] == 0 + + stale = commit_batch( + config, + paths, + _batch( + paths, + [_record("key/a/stale")], + next_cursor={"generation": 99}, + base_cursor={}, + ), + ) + assert stale == { + "sessions": 1, + "records_written": 0, + "duplicate_records": 0, + "pending": 1, + "errors": 1, + } + assert load_cursor(config, paths, NATIVE_ID) == {"generation": 1} + captured = list(iter_captured_records(config, stored_session_id(paths, NATIVE_ID))) + assert {r["source_id"] for r in captured} == {"key/a/first"} + + state = read_json(state_path(_session_dir(config, paths))) + assert state is not None + assert any(item.get("kind") == "stale_cursor" for item in state["health"]["diagnostics"]) + + +def test_two_competing_batches_second_wins_first_becomes_stale( + config: Config, + paths: SourcePaths, +) -> None: + batch_a = _batch( + paths, + [_record("key/a/contender-a")], + next_cursor={"generation": 1}, + base_cursor={}, + ) + batch_b = _batch( + paths, + [_record("key/a/contender-b")], + next_cursor={"generation": 1}, + base_cursor={}, + ) + + winner = commit_batch(config, paths, batch_b) + assert winner["records_written"] == 1 + + loser = commit_batch(config, paths, batch_a) + assert loser["errors"] == 1 + assert loser["pending"] == 1 + assert loser["records_written"] == 0 + + captured = list(iter_captured_records(config, stored_session_id(paths, NATIVE_ID))) + assert {r["source_id"] for r in captured} == {"key/a/contender-b"} + + +def test_recovery_replay_is_idempotent_when_events_already_committed( + config: Config, + paths: SourcePaths, +) -> None: + directory = _session_dir(config, paths) + records = [_record("key/a/idempotent")] + + with fault_at("after_append"): + with pytest.raises(RuntimeError): + commit_batch(config, paths, _batch(paths, records, next_cursor={"generation": 1})) + + assert journal_path(directory).is_file() + first_recovery = commit_batch( + config, + paths, + _batch(paths, [], next_cursor={"generation": 1}, base_cursor={}), + ) + assert first_recovery["records_written"] == 0 + assert first_recovery["duplicate_records"] == 1 + assert not journal_path(directory).exists() + + second_recovery = commit_batch( + config, + paths, + _batch( + paths, + [_record("key/a/next")], + next_cursor={"generation": 2}, + base_cursor={"generation": 1}, + ), + ) + assert second_recovery["records_written"] == 1 + assert second_recovery["duplicate_records"] == 0 From d32012b65fd536d8b1ee30ba12b7390e2f593ae8 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Thu, 10 Sep 2026 14:37:03 -0700 Subject: [PATCH 11/88] Add comprehensive behavioral tests for Copilot transcript reader. Cover discovery, bounded reads, malformed lines, snapshot semantics, and cli-1.0.83 fixture ingestion. Co-authored-by: Cursor --- tests/test_copilot_transcript.py | 577 +++++++++++++++++++++++++++++++ 1 file changed, 577 insertions(+) create mode 100644 tests/test_copilot_transcript.py diff --git a/tests/test_copilot_transcript.py b/tests/test_copilot_transcript.py new file mode 100644 index 0000000..a9f04d2 --- /dev/null +++ b/tests/test_copilot_transcript.py @@ -0,0 +1,577 @@ +"""Behavioral tests for the Copilot CLI transcript reader.""" + +from __future__ import annotations + +import hashlib +import json +import os +import shutil +from pathlib import Path +from typing import Any + +import pytest + +from thirdeye.platforms.copilot.constants import SOURCE_SCHEMA_VERSION +from thirdeye.platforms.copilot.identity import resolve_sources +from thirdeye.platforms.copilot.transcript import discover_transcripts, read_transcript +from thirdeye.platforms.copilot.types import SourcePaths, SourceRecord, SourceSlice + +FIXTURES = Path(__file__).parent / "fixtures" / "copilot" +V1_CASES = FIXTURES / "v1-cases" +CLI_FIXTURE = FIXTURES / "cli-1.0.83" +NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" + + +def _session_paths(home: Path) -> SourcePaths: + return resolve_sources(home) + + +def _write_session( + home: Path, + native_id: str, + *, + events: str | bytes | None = None, + workspace: str | None = None, +) -> Path: + session_dir = home / "session-state" / native_id + session_dir.mkdir(parents=True, exist_ok=True) + if events is not None: + (session_dir / "events.jsonl").write_bytes( + events if isinstance(events, bytes) else events.encode("utf-8") + ) + if workspace is not None: + (session_dir / "workspace.yaml").write_text(workspace, encoding="utf-8") + return session_dir + + +def _transcript_records(slice_: SourceSlice) -> list[SourceRecord]: + return [record for record in slice_["records"] if record["source_kind"] == "transcript"] + + +def _metadata_records(slice_: SourceSlice) -> list[SourceRecord]: + return [record for record in slice_["records"] if record["source_kind"] == "metadata"] + + +def _diagnostic_codes(slice_: SourceSlice) -> set[str]: + return {item["code"] for item in slice_["diagnostics"]} + + +def _drain_transcript( + paths: SourcePaths, + native_id: str, + *, + max_records: int = 1000, + max_bytes: int = 4_194_304, +) -> tuple[list[SourceRecord], list[dict[str, Any]], SourceSlice]: + cursor: dict[str, Any] = {} + records: list[SourceRecord] = [] + diagnostics: list[dict[str, Any]] = [] + last: SourceSlice | None = None + while True: + last = read_transcript( + paths, + native_id, + cursor, + max_records=max_records, + max_bytes=max_bytes, + ) + records.extend(last["records"]) + diagnostics.extend(last["diagnostics"]) + cursor = last["next_cursor"] + if last["exhausted"]: + break + assert last is not None + return records, diagnostics, last + + +# --- discovery --- + + +def test_discover_transcripts_finds_sessions_with_events(tmp_path: Path): + home = tmp_path / "copilot" + _write_session(home, "session-b", events='{"id":"1"}\n') + _write_session(home, "session-a", events='{"id":"2"}\n') + (home / "session-state" / "no-events").mkdir(parents=True) + + assert discover_transcripts(_session_paths(home)) == ["session-a", "session-b"] + + +def test_discover_transcripts_ignores_unsafe_or_outside_entries(tmp_path: Path): + home = tmp_path / "copilot" + root = home / "session-state" + root.mkdir(parents=True) + _write_session(home, "valid", events='{"id":"1"}\n') + (root / "bad/id").mkdir(parents=True) + (root / "bad/id" / "events.jsonl").write_text('{"id":"x"}\n', encoding="utf-8") + (root / "..").resolve() # ensure traversal candidate exists on POSIX + + assert discover_transcripts(_session_paths(home)) == ["valid"] + + +def test_discover_transcripts_returns_empty_when_root_missing(tmp_path: Path): + home = tmp_path / "missing-home" + assert discover_transcripts(_session_paths(home)) == [] + + +def test_read_transcript_rejects_traversal_native_id(tmp_path: Path): + home = tmp_path / "copilot" + _write_session(home, "valid", events='{"id":"1"}\n') + paths = _session_paths(home) + with pytest.raises(ValueError, match="path separator"): + read_transcript(paths, "../valid", {}) + + +# --- happy path and metadata --- + + +def test_read_transcript_preserves_unknown_fields_and_native_event_id(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + fixture = (V1_CASES / "unknown-event-fields.jsonl").read_text(encoding="utf-8") + _write_session(home, native, events=fixture, workspace="cwd: /fixture/workspace\n") + paths = _session_paths(home) + + slice_ = read_transcript(paths, native, {}) + transcript = _transcript_records(slice_)[0] + metadata = _metadata_records(slice_)[0] + + assert transcript["source_id"] == f"{paths['source_key']}/{native}/unknown-1" + assert transcript["ts"] == "2026-09-10T17:08:24.000Z" + assert transcript["payload"]["schema_version"] == SOURCE_SCHEMA_VERSION + assert transcript["payload"]["top_level_future"] == "retain" + assert transcript["payload"]["data"]["future_field"]["nested"] == [1, True, {"opaque": "retain"}] + assert transcript["locator"]["native_event_id"] == "unknown-1" + assert metadata["payload"]["data"]["cwd"] == "/fixture/workspace" + assert slice_["cwd"] == "/fixture/workspace" + + +def test_read_transcript_uses_workspace_path_fallback(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + _write_session( + home, + native, + events='{"id":"1","type":"user.message"}\n', + workspace="workspacePath: /from/workspacePath\n", + ) + slice_ = read_transcript(_session_paths(home), native, {}) + assert slice_["cwd"] == "/from/workspacePath" + + +def test_read_transcript_missing_event_id_uses_lower_confidence_locator(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + fixture = (V1_CASES / "missing-event-id.jsonl").read_text(encoding="utf-8") + _write_session(home, native, events=fixture) + paths = _session_paths(home) + + record = _transcript_records(read_transcript(paths, native, {}))[0] + assert record["source_id"].startswith(f"{paths['source_key']}/{native}/") + assert "native_event_id" not in record["locator"] + assert record["locator"]["identity_confidence"] == "lower" + assert "content_digest" in record["locator"] + assert record["locator"]["byte_offset"] == 0 + + +# --- malformed and invalid complete lines --- + + +def test_read_transcript_invalid_utf8_complete_line_is_retained(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + bad = b"\xffnot-utf8\n" + _write_session(home, native, events=bad) + slice_ = read_transcript(_session_paths(home), native, {}) + + record = _transcript_records(slice_)[0] + assert record["payload"]["malformed"] == "invalid_utf8" + assert "raw_bytes_base64" in record["payload"] + assert "transcript_invalid_utf8" in _diagnostic_codes(slice_) + assert slice_["exhausted"] is True + + +def test_read_transcript_invalid_json_complete_line_is_retained(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + _write_session(home, native, events="not json at all\n") + slice_ = read_transcript(_session_paths(home), native, {}) + + record = _transcript_records(slice_)[0] + assert record["payload"]["malformed"] == "invalid_json" + assert record["payload"]["raw_line"] == "not json at all" + assert "transcript_invalid_json" in _diagnostic_codes(slice_) + + +def test_read_transcript_non_object_json_complete_line_is_retained(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + _write_session(home, native, events='["array"]\n') + slice_ = read_transcript(_session_paths(home), native, {}) + + record = _transcript_records(slice_)[0] + assert record["payload"]["malformed"] == "non_object_json" + assert record["payload"]["raw_value"] == ["array"] + assert "transcript_non_object" in _diagnostic_codes(slice_) + + +def test_read_transcript_invalid_timestamp_emits_diagnostic_but_keeps_record(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + _write_session(home, native, events='{"id":"1","timestamp":"not-a-date"}\n') + slice_ = read_transcript(_session_paths(home), native, {}) + + record = _transcript_records(slice_)[0] + assert record["ts"] is None + assert record["payload"]["timestamp"] == "not-a-date" + assert "transcript_invalid_timestamp" in _diagnostic_codes(slice_) + + +# --- partial trailing lines --- + + +def test_read_transcript_trailing_json_fixture_treats_unterminated_line_as_malformed(tmp_path: Path): + """The v1 trailing-json fixture ends with a newline; it is a complete physical line.""" + + home = tmp_path / "copilot" + native = "session-a" + fixture = (V1_CASES / "trailing-json.jsonl").read_bytes() + _write_session(home, native, events=fixture) + paths = _session_paths(home) + + slice_ = read_transcript(paths, native, {}) + records = _transcript_records(slice_) + assert len(records) == 2 + assert records[0]["payload"]["id"] == "complete-1" + assert records[1]["payload"]["malformed"] == "invalid_json" + assert "transcript_invalid_json" in _diagnostic_codes(slice_) + assert slice_["exhausted"] is True + + +def test_read_transcript_defers_physical_line_without_newline(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + complete = b'{"id":"complete-1","type":"known"}\n' + partial = b'{"id":"incomplete-2","type":"unterminated"' + _write_session(home, native, events=complete + partial) + paths = _session_paths(home) + + first = read_transcript(paths, native, {}) + assert len(_transcript_records(first)) == 1 + assert first["next_cursor"]["byte_offset"] == len(complete) + assert first["exhausted"] is False + + +def test_read_transcript_replacement_replays_deferred_partial_line(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + complete = b'{"id":"complete-1","type":"known"}\n' + partial = b'{"id":"incomplete-2","type":"unterminated"' + finished = complete + b'{"id":"incomplete-2","type":"now-complete"}\n' + path = _write_session(home, native, events=complete + partial) + paths = _session_paths(home) + + first = read_transcript(paths, native, {}) + event_path = path / "events.jsonl" + event_path.unlink() + event_path.write_bytes(finished) + + second = read_transcript(paths, native, first["next_cursor"]) + assert "transcript_replaced" in _diagnostic_codes(second) + ids = [record["payload"].get("id") for record in _transcript_records(second)] + assert ids == ["complete-1", "incomplete-2"] + assert second["exhausted"] is True + + +def test_read_transcript_defers_trailing_incomplete_utf8(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + raw = bytes.fromhex((V1_CASES / "trailing-utf8.hex").read_text(encoding="utf-8").strip()) + _write_session(home, native, events=raw) + paths = _session_paths(home) + + first = read_transcript(paths, native, {}) + assert len(_transcript_records(first)) == 1 + assert first["exhausted"] is False + assert first["next_cursor"]["byte_offset"] == raw.index(b"\n") + 1 + + +def test_read_transcript_completes_deferred_utf8_after_replacement(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + raw = bytes.fromhex((V1_CASES / "trailing-utf8.hex").read_text(encoding="utf-8").strip()) + path = _write_session(home, native, events=raw) + paths = _session_paths(home) + + first = read_transcript(paths, native, {}) + assert len(_transcript_records(first)) == 1 + assert first["exhausted"] is False + + completed = raw + b"\xac\n" + replacement = path / "events.jsonl" + replacement.unlink() + replacement.write_bytes(completed) + + second = read_transcript(paths, native, first["next_cursor"]) + assert "transcript_replaced" in _diagnostic_codes(second) + transcript = _transcript_records(second) + assert len(transcript) == 2 + assert transcript[1]["payload"]["malformed"] == "invalid_json" + assert "transcript_invalid_json" in _diagnostic_codes(second) + + +# --- bounds and snapshot-end behavior --- + + +def test_read_transcript_respects_max_records_and_max_bytes(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + lines = [json.dumps({"id": f"event-{index}", "payload": "x" * 40}) for index in range(4)] + _write_session(home, native, events="\n".join(lines) + "\n") + paths = _session_paths(home) + + by_count = read_transcript(paths, native, {}, max_records=2) + assert len(_transcript_records(by_count)) == 2 + assert by_count["exhausted"] is False + + by_bytes = read_transcript(paths, native, {}, max_records=100, max_bytes=120) + assert len(_transcript_records(by_bytes)) == 1 + assert by_bytes["exhausted"] is False + + +def test_read_transcript_accepts_one_oversize_complete_record(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + big = json.dumps({"id": "big", "payload": "x" * 256}) + _write_session(home, native, events=f"{big}\n") + slice_ = read_transcript(_session_paths(home), native, {}, max_bytes=32) + + assert len(_transcript_records(slice_)) == 1 + assert "transcript_record_oversize" in _diagnostic_codes(slice_) + assert slice_["exhausted"] is True + + +def test_read_transcript_snapshot_end_drains_initial_file_without_later_appends(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + path = _write_session( + home, + native, + events='{"id":"first"}\n{"id":"second"}\n', + ) + paths = _session_paths(home) + + first = read_transcript(paths, native, {}, max_records=1) + assert [record["payload"]["id"] for record in _transcript_records(first)] == ["first"] + snapshot_end = first["next_cursor"]["snapshot_end"] + + (path / "events.jsonl").open("a", encoding="utf-8").write('{"id":"third"}\n') + + second = read_transcript(paths, native, first["next_cursor"], max_records=10) + assert second["next_cursor"]["snapshot_end"] == snapshot_end + assert [record["payload"]["id"] for record in _transcript_records(second)] == ["second"] + assert second["exhausted"] is True + + third = read_transcript(paths, native, second["next_cursor"]) + assert [record["payload"]["id"] for record in _transcript_records(third)] == ["third"] + assert third["exhausted"] is True + + +def test_read_transcript_opens_new_snapshot_for_appended_partial_line(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + path = _write_session(home, native, events='{"id":"first"}\n') + paths = _session_paths(home) + first = read_transcript(paths, native, {}) + assert first["exhausted"] is True + + with (path / "events.jsonl").open("ab") as stream: + stream.write(b'{"id":"partial-without-newline"') + + second = read_transcript(paths, native, first["next_cursor"]) + assert _transcript_records(second) == [] + assert second["next_cursor"]["byte_offset"] == first["next_cursor"]["byte_offset"] + assert second["exhausted"] is False + + +def test_read_transcript_reads_appended_complete_line_in_fresh_snapshot(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + path = _write_session(home, native, events='{"id":"first"}\n') + paths = _session_paths(home) + first = read_transcript(paths, native, {}) + assert first["exhausted"] is True + + with (path / "events.jsonl").open("ab") as stream: + stream.write(b'{"id":"second"}\n') + + second = read_transcript(paths, native, first["next_cursor"]) + assert [record["payload"]["id"] for record in _transcript_records(second)] == ["second"] + assert second["exhausted"] is True + + +# --- replacement, truncation, and cursor recovery --- + + +def test_read_transcript_replays_after_file_replacement(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + path = _write_session(home, native, events='{"id":"old"}\n') + paths = _session_paths(home) + + first = read_transcript(paths, native, {}) + assert first["exhausted"] is True + + replacement = path / "events.jsonl" + replacement.unlink() + replacement.write_text('{"id":"new"}\n', encoding="utf-8") + + second = read_transcript(paths, native, first["next_cursor"]) + assert "transcript_replaced" in _diagnostic_codes(second) + assert [record["payload"]["id"] for record in _transcript_records(second)] == ["new"] + assert second["next_cursor"]["byte_offset"] == len('{"id":"new"}\n') + + +def test_read_transcript_replays_after_truncation(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + path = _write_session( + home, + native, + events='{"id":"first"}\n{"id":"second"}\n', + ) + paths = _session_paths(home) + first = read_transcript(paths, native, {}, max_records=1) + assert first["next_cursor"]["byte_offset"] > 0 + + event_path = path / "events.jsonl" + event_path.write_text('{"id":"shorter"}\n', encoding="utf-8") + + second = read_transcript(paths, native, first["next_cursor"]) + assert "transcript_replaced" in _diagnostic_codes(second) + assert [record["payload"]["id"] for record in _transcript_records(second)] == ["shorter"] + + +def test_read_transcript_invalid_cursor_replays_from_zero(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + _write_session(home, native, events='{"id":"one"}\n') + paths = _session_paths(home) + + slice_ = read_transcript(paths, native, {"byte_offset": -5}) + assert "transcript_cursor_invalid" in _diagnostic_codes(slice_) + assert [record["payload"]["id"] for record in _transcript_records(slice_)] == ["one"] + + +def test_read_transcript_repeatable_source_ids_after_replacement(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + line = '{"id":"stable","type":"user.message"}\n' + path = _write_session(home, native, events=line) + paths = _session_paths(home) + + first_id = _transcript_records(read_transcript(paths, native, {}))[0]["source_id"] + (path / "events.jsonl").write_text(line, encoding="utf-8") + second_id = _transcript_records(read_transcript(paths, native, {}))[0]["source_id"] + assert first_id == second_id + + +# --- unavailable source and workspace errors --- + + +def test_read_transcript_missing_events_is_not_exhausted(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + (home / "session-state" / native).mkdir(parents=True) + slice_ = read_transcript(_session_paths(home), native, {}) + + assert slice_["records"] == [] + assert slice_["exhausted"] is False + assert "transcript_unavailable" in _diagnostic_codes(slice_) + + +def test_read_transcript_invalid_workspace_emits_diagnostic_without_blocking_events(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + _write_session( + home, + native, + events='{"id":"1"}\n', + workspace="!!: not: valid: yaml: [\n", + ) + slice_ = read_transcript(_session_paths(home), native, {}) + + assert len(_transcript_records(slice_)) == 1 + assert "workspace_metadata_invalid" in _diagnostic_codes(slice_) + assert slice_["cwd"] is None + + +def test_read_transcript_rejects_negative_limits(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + _write_session(home, native, events='{"id":"1"}\n') + paths = _session_paths(home) + with pytest.raises(ValueError, match="max_records and max_bytes"): + read_transcript(paths, native, {}, max_records=-1) + + +# --- observed cli fixture via injected home layout --- + + +def test_read_transcript_cli_fixture_preserves_seventy_six_events(tmp_path: Path): + home = tmp_path / "copilot" + native = NATIVE_SESSION_ID + session_dir = _write_session(home, native) + shutil.copy(CLI_FIXTURE / "events.jsonl", session_dir / "events.jsonl") + (session_dir / "workspace.yaml").write_text( + "cwd: /sanitized/workspace\nrepo: probe\n", + encoding="utf-8", + ) + paths = _session_paths(home) + + records, diagnostics, last = _drain_transcript(paths, native) + transcript = [record for record in records if record["source_kind"] == "transcript"] + metadata = [record for record in records if record["source_kind"] == "metadata"] + + assert len(transcript) == 76 + assert len(metadata) == 1 + assert last["cwd"] == "/sanitized/workspace" + assert diagnostics == [] or all(item["code"] != "transcript_invalid_json" for item in diagnostics) + + user_messages = [record for record in transcript if record["payload"].get("type") == "user.message"] + assert len(user_messages) == 3 + assert all(record["payload"]["schema_version"] == SOURCE_SCHEMA_VERSION for record in transcript) + + +def test_read_transcript_cursor_advances_by_byte_offsets(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + first_line = '{"id":"first"}\n' + second_line = '{"id":"second"}\n' + _write_session(home, native, events=first_line + second_line) + paths = _session_paths(home) + + first = read_transcript(paths, native, {}, max_records=1) + assert first["next_cursor"]["byte_offset"] == len(first_line) + assert "continuity_start" in first["next_cursor"] + assert "continuity_digest" in first["next_cursor"] + + second = read_transcript(paths, native, first["next_cursor"], max_records=1) + assert second["next_cursor"]["byte_offset"] == len(first_line) + len(second_line) + + +def test_read_transcript_module_has_no_forbidden_imports(): + source = Path(__import__("thirdeye.platforms.copilot.transcript", fromlist=["__file__"]).__file__) + imports = [ + line.strip() + for line in source.read_text(encoding="utf-8").splitlines() + if line.startswith(("import ", "from ")) + ] + joined = "\n".join(imports) + for forbidden in ( + "thirdeye.store", + "sqlite3", + "hook_payload", + "platforms.copilot.archive", + "platforms.copilot.database", + ): + assert forbidden not in joined From dafe33a1e1e5566e6246994c9fe8eece7e095c5c Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Thu, 10 Sep 2026 14:39:06 -0700 Subject: [PATCH 12/88] Fix Copilot hook timestamp parsing and complete spool-record validation. ISO source times are parsed rather than accepted by string shape, and read/ack skip incomplete hook records with diagnostics. Co-authored-by: Cursor --- .../platforms/copilot/hook_payload.py | 13 +- src/thirdeye/platforms/copilot/spool.py | 31 ++++- tests/test_copilot_hook_payload.py | 96 ++++++++++++++- tests/test_copilot_spool.py | 111 ++++++++++++++++++ tests/test_provenance.py | 10 ++ 5 files changed, 254 insertions(+), 7 deletions(-) diff --git a/src/thirdeye/platforms/copilot/hook_payload.py b/src/thirdeye/platforms/copilot/hook_payload.py index f1739b0..9a10df9 100644 --- a/src/thirdeye/platforms/copilot/hook_payload.py +++ b/src/thirdeye/platforms/copilot/hook_payload.py @@ -52,6 +52,17 @@ def _session_id(payload: dict[str, Any]) -> str: return session_id +def _valid_iso_datetime(value: str) -> bool: + """Return True when *value* is a parseable ISO-8601 datetime.""" + + iso = value[:-1] + "+00:00" if value.endswith("Z") else value + try: + datetime.fromisoformat(iso) + except ValueError: + return False + return True + + def _source_ts(payload: dict[str, Any]) -> str | None: """Return the original source timestamp when it is valid; never invent one.""" @@ -64,7 +75,7 @@ def _source_ts(payload: dict[str, Any]) -> str | None: stripped = value.strip() if not stripped: return None - if stripped.endswith("Z") or "+" in stripped[1:]: + if _valid_iso_datetime(stripped): return stripped try: value = float(stripped) diff --git a/src/thirdeye/platforms/copilot/spool.py b/src/thirdeye/platforms/copilot/spool.py index 2bc8faa..8662184 100644 --- a/src/thirdeye/platforms/copilot/spool.py +++ b/src/thirdeye/platforms/copilot/spool.py @@ -44,15 +44,36 @@ def _write_diagnostic(path: Path, reason: str) -> None: return -def _load_record(path: Path) -> SourceRecord | None: +def _record_error(data: Any, *, expected_native_id: str) -> str | None: + """Return a diagnostic reason when *data* is not a complete hook SourceRecord.""" + + if not isinstance(data, dict): + return "spool file is not a SourceRecord object" + for key in ("source_id", "source_kind", "native_session_id", "observed_at"): + value = data.get(key) + if not isinstance(value, str) or not value: + return "spool file is not a complete SourceRecord object" + if "ts" not in data or (data["ts"] is not None and not isinstance(data["ts"], str)): + return "spool file is not a complete SourceRecord object" + if not isinstance(data.get("payload"), dict) or not isinstance(data.get("locator"), dict): + return "spool file is not a complete SourceRecord object" + if data["source_kind"] != "hook": + return "spool file source_kind is not hook" + if data["native_session_id"] != expected_native_id: + return "native_session_id does not match spool directory" + return None + + +def _load_record(path: Path, *, expected_native_id: str) -> SourceRecord | None: try: raw = fsops.read_text(path, encoding="utf-8") data: Any = json.loads(raw) except (OSError, UnicodeDecodeError, json.JSONDecodeError, RecursionError) as exc: _write_diagnostic(path, f"{type(exc).__name__}: {exc}") return None - if not isinstance(data, dict) or not isinstance(data.get("source_id"), str): - _write_diagnostic(path, "spool file is not a SourceRecord object") + reason = _record_error(data, expected_native_id=expected_native_id) + if reason is not None: + _write_diagnostic(path, reason) return None return data # type: ignore[return-value] @@ -86,7 +107,7 @@ def read_spool(config: Config, paths: SourcePaths, native_id: str) -> list[Sourc return [] records: list[SourceRecord] = [] for path in sorted(directory.glob("*.json")): - record = _load_record(path) + record = _load_record(path, expected_native_id=native_id) if record is not None: records.append(record) return records @@ -105,7 +126,7 @@ def ack_spool(config: Config, paths: SourcePaths, source_ids: list[str]) -> None if not root.is_dir(): return for path in root.glob("*/*.json"): - record = _load_record(path) + record = _load_record(path, expected_native_id=path.parent.name) if record is None: continue if record["source_id"] in wanted: diff --git a/tests/test_copilot_hook_payload.py b/tests/test_copilot_hook_payload.py index 4056601..9baadce 100644 --- a/tests/test_copilot_hook_payload.py +++ b/tests/test_copilot_hook_payload.py @@ -211,4 +211,98 @@ def test_locator_identifies_observation(): ) locator = record["locator"] assert locator["observation_id"] == "obs-locator" - assert locator["event"] in {"userPromptSubmitted", "UserPromptSubmit"} + assert locator["event"] == "userPromptSubmitted" + + +def test_source_id_format_includes_session_and_observation(): + record = _parse( + "sessionStart", + {"sessionId": NATIVE_SESSION_ID, "timestamp": 1789060102204}, + observation_id="obs-format-check", + ) + assert record["source_id"] == f"hook/{NATIVE_SESSION_ID}/obs-format-check" + + +def test_parse_hook_preserves_zulu_iso_timestamp_strings(): + iso_z = "2026-09-10T17:08:25.626Z" + assert _parse("sessionStart", {"sessionId": NATIVE_SESSION_ID, "timestamp": iso_z})["ts"] == iso_z + + +def test_parse_hook_preserves_positive_offset_iso_timestamp_strings(): + iso_offset = "2026-09-10T17:08:25.626+00:00" + assert ( + _parse("sessionStart", {"sessionId": NATIVE_SESSION_ID, "timestamp": iso_offset})["ts"] + == iso_offset + ) + + +def test_parse_hook_preserves_negative_offset_iso_timestamp_strings(): + iso_offset = "2026-09-10T10:08:25-07:00" + assert ( + _parse("sessionStart", {"sessionId": NATIVE_SESSION_ID, "timestamp": iso_offset})["ts"] + == iso_offset + ) + + +def test_parse_hook_converts_numeric_string_timestamp(): + record = _parse( + "sessionStart", + {"sessionId": NATIVE_SESSION_ID, "timestamp": "1789060105626"}, + ) + assert record["ts"] is not None + assert record["ts"].endswith("Z") + + +@pytest.mark.parametrize( + "timestamp", + [ + True, + False, + None, + "", + " ", + "not-a-number", + {}, + "nonsenseZ", + "2026-13-40T99:99:99Z", + "2026-09-10T17:08:25.626Z extra", + "+not-an-iso-timestamp", + ], +) +def test_parse_hook_invalid_timestamps_leave_ts_none(timestamp: object): + record = _parse("sessionStart", {"sessionId": NATIVE_SESSION_ID, "timestamp": timestamp}) + assert record["ts"] is None + + +def test_parse_hook_accepts_second_epoch_timestamps(): + record = _parse("sessionStart", {"sessionId": NATIVE_SESSION_ID, "timestamp": 1_789_060_105}) + assert record["ts"] is not None + assert record["ts"].endswith("Z") + + +def test_parse_hook_unknown_event_name_passes_through(): + record = _parse( + "customFutureHook", + {"sessionId": NATIVE_SESSION_ID, "timestamp": 1789060102204}, + ) + assert record["payload"]["event"] == "customFutureHook" + assert record["locator"]["event"] == "customFutureHook" + + +def test_parse_hook_stores_all_allowlisted_context_keys(): + context = { + "env": {"WB_PLAN": "p"}, + "trace_id": "trace-1", + "span_id": "span-1", + "parent_span_id": "parent-span-1", + "trace_context": {"sampled": True}, + "traceparent": "00-abc-def-01", + } + stored = _parse( + "sessionStart", + {"sessionId": NATIVE_SESSION_ID, "timestamp": 1}, + context=context, + )["payload"]["context"] + assert stored == context + assert stored is not context + assert stored["env"] is not context["env"] diff --git a/tests/test_copilot_spool.py b/tests/test_copilot_spool.py index a6d7ef1..cd8a06a 100644 --- a/tests/test_copilot_spool.py +++ b/tests/test_copilot_spool.py @@ -2,10 +2,12 @@ from __future__ import annotations +import json import subprocess import sys import threading from pathlib import Path +from typing import Any import pytest @@ -187,6 +189,115 @@ def test_spool_survives_subprocess_isolation(tmp_path: Path): assert records[0]["payload"]["hook_payload"]["sessionId"] == NATIVE_SESSION_ID +def test_read_spool_returns_empty_for_missing_session(copilot_env: tuple[Config, SourcePaths]): + config, paths = copilot_env + assert read_spool(config, paths, NATIVE_SESSION_ID) == [] + + +def test_malformed_spool_writes_diagnostic_sidecar(copilot_env: tuple[Config, SourcePaths]): + config, paths = copilot_env + valid = _hook_record(observation_id="obs-diag-valid") + spool_path = Path(enqueue_hook(config, paths, valid)) + malformed = spool_path.parent / "000000-malformed.json" + malformed.write_text("{not json", encoding="utf-8") + + records = read_spool(config, paths, NATIVE_SESSION_ID) + assert len(records) == 1 + diag_path = malformed.with_name(f"{malformed.name}.diag") + assert diag_path.is_file() + assert "JSONDecodeError" in diag_path.read_text(encoding="utf-8") + + +def test_malformed_spool_missing_source_id_is_skipped(copilot_env: tuple[Config, SourcePaths]): + config, paths = copilot_env + valid = _hook_record(observation_id="obs-valid-shape") + spool_dir = Path(enqueue_hook(config, paths, valid)).parent + malformed = spool_dir / "000001-missing-source-id.json" + malformed.write_text('{"source_kind": "hook"}', encoding="utf-8") + + records = read_spool(config, paths, NATIVE_SESSION_ID) + assert len(records) == 1 + assert records[0]["source_id"] == valid["source_id"] + diag_path = malformed.with_name(f"{malformed.name}.diag") + assert diag_path.is_file() + assert "SourceRecord" in diag_path.read_text(encoding="utf-8") + + +def test_read_spool_skips_source_id_only_object(copilot_env: tuple[Config, SourcePaths]): + config, paths = copilot_env + valid = _hook_record(observation_id="obs-complete-neighbor") + spool_dir = Path(enqueue_hook(config, paths, valid)).parent + malformed = spool_dir / "000003-source-id-only.json" + malformed.write_text('{"source_id": "hook/x/obs"}', encoding="utf-8") + + records = read_spool(config, paths, NATIVE_SESSION_ID) + assert len(records) == 1 + assert records[0]["source_id"] == valid["source_id"] + diag_path = malformed.with_name(f"{malformed.name}.diag") + assert diag_path.is_file() + assert "SourceRecord" in diag_path.read_text(encoding="utf-8") + + +@pytest.mark.parametrize( + "corrupt", + [ + {"payload": None}, + {"payload": []}, + {"locator": "not-a-dict"}, + {"ts": 1789060105626}, + {"observed_at": None}, + {"source_kind": "transcript"}, + {"native_session_id": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106"}, + {"source_id": ""}, + ], +) +def test_read_spool_skips_records_with_corrupt_or_missing_required_fields( + copilot_env: tuple[Config, SourcePaths], + corrupt: dict[str, Any], +): + config, paths = copilot_env + valid = _hook_record(observation_id="obs-required-fields") + spool_dir = Path(enqueue_hook(config, paths, valid)).parent + incomplete = dict(valid) + incomplete.update(corrupt) + malformed = spool_dir / "000004-incomplete.json" + malformed.write_text(json.dumps(incomplete), encoding="utf-8") + + records = read_spool(config, paths, NATIVE_SESSION_ID) + assert len(records) == 1 + assert records[0]["source_id"] == valid["source_id"] + diag_path = malformed.with_name(f"{malformed.name}.diag") + assert diag_path.is_file() + assert diag_path.read_text(encoding="utf-8").strip() + + +def test_ack_spool_finds_records_across_sessions(copilot_env: tuple[Config, SourcePaths]): + config, paths = copilot_env + child_id = "bf8cb9f3-2097-4db0-a3c8-78a2653b2106" + parent = _hook_record(observation_id="obs-parent-ack", session_id=NATIVE_SESSION_ID) + child = _hook_record(observation_id="obs-child-ack", session_id=child_id) + enqueue_hook(config, paths, parent) + enqueue_hook(config, paths, child) + + ack_spool(config, paths, [child["source_id"]]) + + assert len(read_spool(config, paths, NATIVE_SESSION_ID)) == 1 + assert read_spool(config, paths, child_id) == [] + + +def test_ack_spool_leaves_malformed_files_in_place(copilot_env: tuple[Config, SourcePaths]): + config, paths = copilot_env + valid = _hook_record(observation_id="obs-ack-malformed-neighbor") + spool_dir = Path(enqueue_hook(config, paths, valid)).parent + malformed = spool_dir / "000002-bad-for-ack.json" + malformed.write_text("{bad", encoding="utf-8") + + ack_spool(config, paths, [valid["source_id"]]) + + assert malformed.is_file() + assert read_spool(config, paths, NATIVE_SESSION_ID) == [] + + def test_child_session_hooks_spool_under_native_session_id(tmp_path: Path): child_id = "bf8cb9f3-2097-4db0-a3c8-78a2653b2106" copilot_home = tmp_path / "copilot-home" diff --git a/tests/test_provenance.py b/tests/test_provenance.py index 8e0a8d7..a0a32ff 100644 --- a/tests/test_provenance.py +++ b/tests/test_provenance.py @@ -251,3 +251,13 @@ def test_copilot_does_not_change_claude_codex_cursor_regressions(): assert foreign_payload_reason({"hook_event_name": "beforeSubmitPrompt"}, "claude") is not None assert foreign_payload_reason({"hook_event_name": "SessionStart"}, "cursor") is not None assert foreign_payload_reason({"hook_event_name": "beforeSubmitPrompt"}, "cursor") is None + + +def test_copilot_rejects_cursor_only_camel_case_events(): + reason = foreign_payload_reason({"hook_event_name": "beforeSubmitPrompt"}, "copilot") + assert isinstance(reason, str) + assert "beforeSubmitPrompt" in reason + + +def test_copilot_is_known_platform(): + assert foreign_payload_reason({}, "copilot") is None From c798468d8d18ba6117413f7be6f5e8583ea41c2a Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Thu, 10 Sep 2026 14:42:07 -0700 Subject: [PATCH 13/88] Add Copilot hook installer --- src/thirdeye/platforms/copilot/constants.py | 11 + src/thirdeye/platforms/copilot/install.py | 249 ++++++++++++++++++++ 2 files changed, 260 insertions(+) create mode 100644 src/thirdeye/platforms/copilot/install.py diff --git a/src/thirdeye/platforms/copilot/constants.py b/src/thirdeye/platforms/copilot/constants.py index b3eb4f8..b2ff492 100644 --- a/src/thirdeye/platforms/copilot/constants.py +++ b/src/thirdeye/platforms/copilot/constants.py @@ -18,6 +18,13 @@ COPILOT_HOME_ENV = "COPILOT_HOME" HOOKS_DIRECTORY_NAME = "hooks" OWNED_HOOK_FILENAME = "thirdeye.json" +HOOK_CONFIG_VERSION = 1 +HOOK_TIMEOUT_S = 5 + +# This is a single dispatcher entrypoint. The event name is passed as an +# explicit argument rather than needing one installed binary per hook event. +# Runtime registration is deliberately owned by the later composition task. +HOOK_BIN_NAME = "thirdeye-copilot-hook" # Copilot CLI 1.0.83's documented hook names are camelCase. The aliases are # intentionally data-only: installation and hook runtime decide which events @@ -27,11 +34,15 @@ "userPromptSubmitted": "user_prompt_submitted", "preToolUse": "pre_tool_use", "postToolUse": "post_tool_use", + "postToolUseFailure": "post_tool_use_failure", "agentStop": "agent_stop", "subagentStart": "subagent_start", "subagentStop": "subagent_stop", "sessionEnd": "session_end", "errorOccurred": "error_occurred", + "permissionRequest": "permission_request", + "notification": "notification", + "preCompact": "pre_compact", } CLI_HOOK_EVENTS = tuple(CLI_HOOK_EVENT_ALIASES) diff --git a/src/thirdeye/platforms/copilot/install.py b/src/thirdeye/platforms/copilot/install.py new file mode 100644 index 0000000..e41d911 --- /dev/null +++ b/src/thirdeye/platforms/copilot/install.py @@ -0,0 +1,249 @@ +"""Installation of thirdeye's passive GitHub Copilot CLI hooks. + +Copilot reads version-1 hook documents from ``$COPILOT_HOME/hooks``. This +module owns only ``thirdeye.json`` and only command entries whose executable +is thirdeye's dispatcher; it never changes Copilot-wide settings or hooks +owned by another integration. +""" + +from __future__ import annotations + +import json +import re +import shlex +from pathlib import Path, PureWindowsPath +from typing import Any + +import click + +import thirdeye._compat as _compat +from thirdeye.platforms.base import Platform, command_basename, resolve_command + +from .constants import ( + CLI_HOOK_EVENTS, + DISPLAY_NAME, + HOOK_BIN_NAME, + HOOK_CONFIG_VERSION, + HOOK_TIMEOUT_S, + HOOKS_DIRECTORY_NAME, + OWNED_HOOK_FILENAME, + PLATFORM_NAME, +) +from .identity import resolve_sources + + +def _quote_bash(value: str) -> str: + """Quote one literal argument for Copilot's POSIX shell hook runner.""" + + return shlex.quote(value) + + +def _quote_powershell(value: str) -> str: + """Quote one literal argument for PowerShell's single-quote syntax.""" + + return "'" + value.replace("'", "''") + "'" + + +def _bash_command(entrypoint: str, event: str) -> str: + return f"{_quote_bash(entrypoint)} {_quote_bash(event)}" + + +def _powershell_command(entrypoint: str, event: str) -> str: + # ``&`` is required for a quoted executable path to be invoked rather + # than treated as a string expression. + return f"& {_quote_powershell(entrypoint)} {_quote_powershell(event)}" + + +def _command_executable(command: object, shell: str) -> str | None: + """Return the executable in one of our generated commands, if parseable.""" + + if not isinstance(command, str) or not command.strip(): + return None + if shell == "bash": + try: + parts = shlex.split(command, posix=True) + except ValueError: + return None + return parts[0] if parts else None + + # We generate ``& 'path' 'event'``. Accept both quote styles for a + # previous installation, but intentionally do not try to interpret an + # arbitrary PowerShell program: configuration ownership must be narrow. + match = re.match(r"^\s*&\s+(?:'((?:[^']|'')*)'|\"([^\"]*)\")", command) + if not match: + return None + quoted = match.group(1) if match.group(1) is not None else match.group(2) + return quoted.replace("''", "'") + + +def _is_our_command(command: object, shell: str) -> bool: + executable = _command_executable(command, shell) + if executable is None: + return False + # command_basename follows the host path rules. Also recognize a Windows + # executable when inspecting a fixture on a non-Windows host. + if command_basename(executable) == HOOK_BIN_NAME: + return True + name = PureWindowsPath(executable).name + return name.lower() in {HOOK_BIN_NAME.lower(), f"{HOOK_BIN_NAME}.exe".lower()} + + +def _entry_is_ours(entry: object) -> bool: + return isinstance(entry, dict) and any( + _is_our_command(entry.get(shell), shell) for shell in ("bash", "powershell") + ) + + +def _load_document(path: Path) -> dict[str, Any]: + """Load a valid version-1 Copilot hook document without normalizing it. + + Invalid documents are configuration errors, not empty documents: replacing + them would destroy an operator's hooks and conceal a broken setup. + """ + + if not path.exists(): + return {"version": HOOK_CONFIG_VERSION, "hooks": {}} + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise click.ClickException( + f"Cannot update Copilot hooks at {path}: the file is not valid JSON. " + "Fix or move it, then run the command again; it was left unchanged." + ) from exc + if not isinstance(data, dict): + raise click.ClickException( + f"Cannot update Copilot hooks at {path}: the document must be a JSON object. " + "It was left unchanged." + ) + if data.get("version") != HOOK_CONFIG_VERSION: + raise click.ClickException( + f"Cannot update Copilot hooks at {path}: expected version " + f"{HOOK_CONFIG_VERSION}, found {data.get('version')!r}. It was left unchanged." + ) + hooks = data.get("hooks") + if not isinstance(hooks, dict): + raise click.ClickException( + f"Cannot update Copilot hooks at {path}: 'hooks' must be an object. " + "It was left unchanged." + ) + for event in CLI_HOOK_EVENTS: + if event in hooks and not isinstance(hooks[event], list): + raise click.ClickException( + f"Cannot update Copilot hooks at {path}: hooks.{event} must be a list. " + "It was left unchanged." + ) + return data + + +def _save_document(path: Path, data: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8", newline="\n") + + +class CopilotPlatform(Platform): + """Passive Copilot CLI hook installation scoped to one source home.""" + + name = PLATFORM_NAME + display_name = DISPLAY_NAME + + def __init__( + self, + source_home: Path | None = None, + hooks_file: Path | None = None, + entrypoint: str | Path | None = None, + *, + windows: bool | None = None, + ) -> None: + self._source_home = source_home + self._hooks_file = hooks_file + self._entrypoint = str(entrypoint) if entrypoint is not None else None + self._windows = _compat.IS_WINDOWS if windows is None else windows + + @property + def hooks_file(self) -> Path: + if self._hooks_file is not None: + return self._hooks_file + paths = resolve_sources(self._source_home) + return Path(paths["home"]) / HOOKS_DIRECTORY_NAME / OWNED_HOOK_FILENAME + + @property + def entrypoint(self) -> str: + return self._entrypoint or resolve_command(HOOK_BIN_NAME) + + def _command(self, event: str) -> tuple[str, str]: + if self._windows: + return "powershell", _powershell_command(self.entrypoint, event) + return "bash", _bash_command(self.entrypoint, event) + + def install(self) -> None: + path = self.hooks_file + data = _load_document(path) + hooks = data["hooks"] + changed = False + for event in CLI_HOOK_EVENTS: + entries = hooks.get(event, []) + # The event type has already been validated by _load_document. + if not isinstance(entries, list): # Defensive for typed JSON input. + raise AssertionError(f"validated hook list changed shape for {event}") + retained = [entry for entry in entries if not _entry_is_ours(entry)] + shell, command = self._command(event) + desired = { + "type": "command", + shell: command, + "timeoutSec": HOOK_TIMEOUT_S, + } + if retained != entries or desired not in retained: + hooks[event] = [*retained, desired] + changed = True + if changed or not path.exists(): + _save_document(path, data) + + def is_installed(self) -> bool: + path = self.hooks_file + try: + data = _load_document(path) + except click.ClickException: + return False + hooks = data["hooks"] + for event in CLI_HOOK_EVENTS: + entries = hooks.get(event) + if not isinstance(entries, list): + return False + shell, command = self._command(event) + if not any( + isinstance(entry, dict) + and entry.get("type") == "command" + and entry.get(shell) == command + and entry.get("timeoutSec") == HOOK_TIMEOUT_S + for entry in entries + ): + return False + return True + + def uninstall(self) -> None: + path = self.hooks_file + if not path.exists(): + return + data = _load_document(path) + hooks = data["hooks"] + changed = False + for event in list(hooks): + entries = hooks[event] + if not isinstance(entries, list): + # Events outside our supported set are unrelated; leave their + # invalid shape intact rather than risking destructive repair. + continue + retained = [entry for entry in entries if not _entry_is_ours(entry)] + if retained == entries: + continue + changed = True + if retained: + hooks[event] = retained + else: + del hooks[event] + # Version and hooks are the only fields we create. An otherwise empty + # owned document can disappear; extra fields always remain untouched. + if not hooks and set(data) == {"version", "hooks"}: + path.unlink() + elif changed: + _save_document(path, data) From b2f3b25c3f437d4b0bcf89f33c8a1ccead199433 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Thu, 10 Sep 2026 14:43:11 -0700 Subject: [PATCH 14/88] Add comprehensive unit tests for Copilot hook installer. Cover install/uninstall merge behavior, shell quoting, malformed config rejection, source-home resolution, and complete event-set verification. Co-authored-by: Cursor --- tests/test_copilot_install.py | 368 ++++++++++++++++++++++++++++++++++ 1 file changed, 368 insertions(+) create mode 100644 tests/test_copilot_install.py diff --git a/tests/test_copilot_install.py b/tests/test_copilot_install.py new file mode 100644 index 0000000..9513594 --- /dev/null +++ b/tests/test_copilot_install.py @@ -0,0 +1,368 @@ +from __future__ import annotations + +import json +import shlex +from pathlib import Path + +import click +import pytest + +from thirdeye.platforms.base import Platform +from thirdeye.platforms.copilot.constants import ( + CLI_HOOK_EVENTS, + DISPLAY_NAME, + HOOK_BIN_NAME, + HOOK_CONFIG_VERSION, + HOOK_TIMEOUT_S, + HOOKS_DIRECTORY_NAME, + OWNED_HOOK_FILENAME, + PLATFORM_NAME, +) +from thirdeye.platforms.copilot.install import CopilotPlatform + + +def _platform( + tmp_path: Path, + *, + hooks_file: Path | None = None, + source_home: Path | None = None, + entrypoint: str | None = None, + windows: bool | None = None, +) -> CopilotPlatform: + return CopilotPlatform( + source_home=source_home, + hooks_file=hooks_file, + entrypoint=entrypoint or "/opt/thirdeye/bin/thirdeye-copilot-hook", + windows=windows, + ) + + +def _expected_entry( + platform: CopilotPlatform, + event: str, +) -> dict[str, object]: + shell, command = platform._command(event) + return { + "type": "command", + shell: command, + "timeoutSec": HOOK_TIMEOUT_S, + } + + +class TestCopilotPlatformAttributes: + def test_name_and_display_name(self, tmp_path: Path): + platform = _platform(tmp_path) + assert platform.name == PLATFORM_NAME + assert platform.display_name == DISPLAY_NAME + + def test_is_platform_subclass(self): + assert issubclass(CopilotPlatform, Platform) + + +class TestInstallFreshFile: + def test_registers_every_supported_event(self, tmp_path: Path): + path = tmp_path / "hooks" / OWNED_HOOK_FILENAME + platform = _platform(tmp_path, hooks_file=path) + platform.install() + data = json.loads(path.read_text()) + assert data["version"] == HOOK_CONFIG_VERSION + assert set(data["hooks"]) == set(CLI_HOOK_EVENTS) + for event in CLI_HOOK_EVENTS: + assert data["hooks"][event] == [_expected_entry(platform, event)] + + def test_creates_parent_directory(self, tmp_path: Path): + path = tmp_path / "nested" / "hooks" / OWNED_HOOK_FILENAME + _platform(tmp_path, hooks_file=path).install() + assert path.exists() + + def test_output_is_valid_json_with_trailing_newline(self, tmp_path: Path): + path = tmp_path / OWNED_HOOK_FILENAME + _platform(tmp_path, hooks_file=path).install() + text = path.read_text(encoding="utf-8") + assert text.endswith("\n") + json.loads(text) + + +class TestInstallIdempotent: + def test_double_install_does_not_duplicate_entries(self, tmp_path: Path): + path = tmp_path / OWNED_HOOK_FILENAME + platform = _platform(tmp_path, hooks_file=path) + platform.install() + first = json.loads(path.read_text()) + platform.install() + second = json.loads(path.read_text()) + assert first == second + for event in CLI_HOOK_EVENTS: + assert len(second["hooks"][event]) == 1 + + def test_install_then_is_installed(self, tmp_path: Path): + path = tmp_path / OWNED_HOOK_FILENAME + platform = _platform(tmp_path, hooks_file=path) + assert platform.is_installed() is False + platform.install() + assert platform.is_installed() is True + + +class TestInstallMergeAndUpgrade: + def test_preserves_foreign_hooks_and_extra_fields(self, tmp_path: Path): + path = tmp_path / OWNED_HOOK_FILENAME + foreign = { + "type": "command", + "bash": "/opt/foreign-hook sessionStart", + "timeoutSec": 17, + } + path.write_text( + json.dumps( + { + "version": HOOK_CONFIG_VERSION, + "theme": "dark", + "hooks": {"sessionStart": [foreign]}, + } + ) + ) + platform = _platform(tmp_path, hooks_file=path) + platform.install() + data = json.loads(path.read_text()) + assert data["theme"] == "dark" + assert foreign in data["hooks"]["sessionStart"] + assert _expected_entry(platform, "sessionStart") in data["hooks"]["sessionStart"] + + def test_replaces_stale_owned_command_without_duplicating(self, tmp_path: Path): + path = tmp_path / OWNED_HOOK_FILENAME + stale = { + "type": "command", + "bash": "/old/path/thirdeye-copilot-hook sessionStart", + "timeoutSec": HOOK_TIMEOUT_S, + } + path.write_text( + json.dumps({"version": HOOK_CONFIG_VERSION, "hooks": {"sessionStart": [stale]}}) + ) + platform = _platform(tmp_path, hooks_file=path, entrypoint="/new/path/thirdeye-copilot-hook") + platform.install() + data = json.loads(path.read_text()) + commands = [entry.get("bash") for entry in data["hooks"]["sessionStart"]] + assert commands.count(_expected_entry(platform, "sessionStart")["bash"]) == 1 + assert stale["bash"] not in commands + + def test_upgrades_partial_install_to_full_event_set(self, tmp_path: Path): + path = tmp_path / OWNED_HOOK_FILENAME + platform = _platform(tmp_path, hooks_file=path) + partial = { + "version": HOOK_CONFIG_VERSION, + "hooks": { + "sessionStart": [_expected_entry(platform, "sessionStart")], + "sessionEnd": [_expected_entry(platform, "sessionEnd")], + }, + } + path.write_text(json.dumps(partial)) + platform.install() + data = json.loads(path.read_text()) + assert set(data["hooks"]) == set(CLI_HOOK_EVENTS) + + +class TestUninstall: + def test_removes_only_owned_entries_and_deletes_empty_file(self, tmp_path: Path): + path = tmp_path / OWNED_HOOK_FILENAME + platform = _platform(tmp_path, hooks_file=path) + platform.install() + platform.uninstall() + assert not path.exists() + + def test_uninstall_preserves_foreign_hooks_and_extra_fields(self, tmp_path: Path): + path = tmp_path / OWNED_HOOK_FILENAME + foreign = { + "type": "command", + "bash": "/opt/foreign-hook agentStop", + "timeoutSec": 12, + } + path.write_text( + json.dumps( + { + "version": HOOK_CONFIG_VERSION, + "notes": "keep me", + "hooks": {"agentStop": [foreign]}, + } + ) + ) + platform = _platform(tmp_path, hooks_file=path) + platform.install() + platform.uninstall() + data = json.loads(path.read_text()) + assert data["notes"] == "keep me" + assert data["hooks"] == {"agentStop": [foreign]} + + def test_uninstall_on_missing_file_is_noop(self, tmp_path: Path): + path = tmp_path / OWNED_HOOK_FILENAME + _platform(tmp_path, hooks_file=path).uninstall() + + +class TestShellQuoting: + def test_bash_quotes_paths_with_spaces(self, tmp_path: Path): + entrypoint = "/opt/my tools/thirdeye-copilot-hook" + platform = _platform(tmp_path, entrypoint=entrypoint, windows=False) + _, command = platform._command("sessionStart") + parts = shlex.split(command, posix=True) + assert parts == [entrypoint, "sessionStart"] + + def test_powershell_quotes_paths_with_spaces(self, tmp_path: Path): + entrypoint = r"C:\Users\First Last\tools\thirdeye-copilot-hook.exe" + platform = _platform(tmp_path, entrypoint=entrypoint, windows=True) + shell, command = platform._command("userPromptSubmitted") + assert shell == "powershell" + assert command.startswith("& ") + assert entrypoint in command + assert "'userPromptSubmitted'" in command + + def test_install_writes_bash_commands_on_posix(self, tmp_path: Path): + path = tmp_path / OWNED_HOOK_FILENAME + entrypoint = "/opt/my tools/thirdeye-copilot-hook" + platform = _platform(tmp_path, hooks_file=path, entrypoint=entrypoint, windows=False) + platform.install() + entry = json.loads(path.read_text())["hooks"]["preToolUse"][0] + assert "bash" in entry + assert "powershell" not in entry + assert shlex.split(entry["bash"], posix=True) == [entrypoint, "preToolUse"] + + def test_install_writes_powershell_commands_on_windows(self, tmp_path: Path): + path = tmp_path / OWNED_HOOK_FILENAME + entrypoint = r"C:\Program Files\Thirdeye\thirdeye-copilot-hook.exe" + platform = _platform(tmp_path, hooks_file=path, entrypoint=entrypoint, windows=True) + platform.install() + entry = json.loads(path.read_text())["hooks"]["postToolUse"][0] + assert entry["powershell"].startswith("& ") + assert entrypoint in entry["powershell"] + + +class TestMalformedConfig: + @pytest.mark.parametrize( + "payload, needle", + [ + ("{not json", "not valid JSON"), + (json.dumps([]), "must be a JSON object"), + (json.dumps({"version": 2, "hooks": {}}), "expected version"), + (json.dumps({"version": 1, "hooks": []}), "'hooks' must be an object"), + ( + json.dumps({"version": 1, "hooks": {"sessionStart": "nope"}}), + "hooks.sessionStart must be a list", + ), + ], + ) + def test_install_refuses_malformed_document_and_preserves_bytes( + self, + tmp_path: Path, + payload: str, + needle: str, + ): + path = tmp_path / OWNED_HOOK_FILENAME + path.write_text(payload) + before = path.read_bytes() + with pytest.raises(click.ClickException) as exc_info: + _platform(tmp_path, hooks_file=path).install() + assert needle in str(exc_info.value) + assert path.read_bytes() == before + + def test_is_installed_returns_false_for_malformed_document(self, tmp_path: Path): + path = tmp_path / OWNED_HOOK_FILENAME + path.write_text("{broken") + platform = _platform(tmp_path, hooks_file=path) + assert platform.is_installed() is False + + +class TestSourceHomeScope: + def test_resolves_hooks_file_under_source_home(self, tmp_path: Path): + source_home = tmp_path / "custom-copilot-home" + platform = CopilotPlatform( + source_home=source_home, + entrypoint="/opt/bin/thirdeye-copilot-hook", + windows=False, + ) + expected = source_home / HOOKS_DIRECTORY_NAME / OWNED_HOOK_FILENAME + assert platform.hooks_file == expected.resolve() + platform.install() + assert expected.exists() + + def test_explicit_hooks_file_overrides_source_home(self, tmp_path: Path): + override = tmp_path / "override" / OWNED_HOOK_FILENAME + platform = CopilotPlatform( + source_home=tmp_path / "ignored-home", + hooks_file=override, + entrypoint="/opt/bin/thirdeye-copilot-hook", + windows=False, + ) + assert platform.hooks_file == override + platform.install() + assert override.exists() + assert not (tmp_path / "ignored-home").exists() + + +class TestInstallStateChecks: + def test_requires_every_event_to_be_installed(self, tmp_path: Path): + path = tmp_path / OWNED_HOOK_FILENAME + platform = _platform(tmp_path, hooks_file=path) + platform.install() + data = json.loads(path.read_text()) + data["hooks"].pop(CLI_HOOK_EVENTS[0]) + path.write_text(json.dumps(data)) + assert platform.is_installed() is False + + def test_is_installed_false_when_timeout_differs(self, tmp_path: Path): + path = tmp_path / OWNED_HOOK_FILENAME + platform = _platform(tmp_path, hooks_file=path) + platform.install() + data = json.loads(path.read_text()) + data["hooks"]["notification"][0]["timeoutSec"] = 99 + path.write_text(json.dumps(data)) + assert platform.is_installed() is False + + +class TestWindowsOwnershipRecognition: + def test_replaces_stale_powershell_owned_command(self, tmp_path: Path): + path = tmp_path / OWNED_HOOK_FILENAME + stale = { + "type": "command", + "powershell": "& 'C:\\Old Path\\thirdeye-copilot-hook.exe' 'sessionStart'", + "timeoutSec": HOOK_TIMEOUT_S, + } + path.write_text( + json.dumps({"version": HOOK_CONFIG_VERSION, "hooks": {"sessionStart": [stale]}}) + ) + entrypoint = r"C:\New Path\thirdeye-copilot-hook.exe" + platform = _platform(tmp_path, hooks_file=path, entrypoint=entrypoint, windows=True) + platform.install() + data = json.loads(path.read_text()) + assert data["hooks"]["sessionStart"] == [_expected_entry(platform, "sessionStart")] + + def test_recognizes_double_quoted_powershell_executable(self, tmp_path: Path): + path = tmp_path / OWNED_HOOK_FILENAME + owned = { + "type": "command", + "powershell": '& "C:\\Tools\\thirdeye-copilot-hook.exe" "agentStop"', + "timeoutSec": HOOK_TIMEOUT_S, + } + path.write_text( + json.dumps({"version": HOOK_CONFIG_VERSION, "hooks": {"agentStop": [owned]}}) + ) + platform = _platform( + tmp_path, + hooks_file=path, + entrypoint=r"C:\Tools\thirdeye-copilot-hook.exe", + windows=True, + ) + platform.install() + data = json.loads(path.read_text()) + assert len(data["hooks"]["agentStop"]) == 1 + assert platform.is_installed() is True + + +class TestUninstallEdgeCases: + def test_leaves_invalid_unknown_event_shape_untouched(self, tmp_path: Path): + path = tmp_path / OWNED_HOOK_FILENAME + platform = _platform(tmp_path, hooks_file=path, windows=False) + platform.install() + data = json.loads(path.read_text()) + data["hooks"]["customFutureEvent"] = "not-a-list" + path.write_text(json.dumps(data)) + platform.uninstall() + remaining = json.loads(path.read_text()) + assert remaining["hooks"]["customFutureEvent"] == "not-a-list" + assert "sessionStart" not in remaining["hooks"] From 78c03b93e6dbad0db104907deb30846eb9628101 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Thu, 10 Sep 2026 14:43:24 -0700 Subject: [PATCH 15/88] Fix Copilot transcript snapshot, continuity, and path containment. Reopen a frozen snapshot after an incomplete tail so appended newlines are ingested, hash the full consumed prefix so same-inode rewrites replay, and reject source files whose resolved paths escape the session directory. Co-authored-by: Cursor --- src/thirdeye/platforms/copilot/transcript.py | 116 +++++++++++++++---- tests/test_copilot_transcript.py | 88 +++++++++++++- 2 files changed, 177 insertions(+), 27 deletions(-) diff --git a/src/thirdeye/platforms/copilot/transcript.py b/src/thirdeye/platforms/copilot/transcript.py index 7e0f0ad..d008ba5 100644 --- a/src/thirdeye/platforms/copilot/transcript.py +++ b/src/thirdeye/platforms/copilot/transcript.py @@ -22,7 +22,7 @@ _EVENTS_FILENAME = "events.jsonl" _WORKSPACE_FILENAME = "workspace.yaml" -_CONTINUITY_WINDOW = 4096 +_IO_CHUNK = 65536 def _observed_at() -> str: @@ -52,6 +52,30 @@ def _session_directory(paths: SourcePaths, native_id: str) -> Path: return session +def _source_file_status(path: Path, directory: Path) -> tuple[Path | None, bool]: + """Return ``(resolved_file, escaped)`` for a candidate source file. + + Symlinks are resolved before use. A target outside *directory* is + rejected so injected source roots cannot archive arbitrary files. + """ + + try: + present = path.is_symlink() or path.exists() + except OSError: + return None, False + if not present: + return None, False + try: + resolved = path.resolve(strict=False) + if not _within(resolved, directory): + return None, True + if resolved.is_file(): + return resolved, False + except OSError: + return None, False + return None, False + + def _generation(path: Path) -> str: """Identify the current file object, while remaining stable for appends.""" @@ -73,14 +97,39 @@ def _valid_timestamp(value: Any) -> str | None: return value -def _continuity_anchor(path: Path, offset: int) -> tuple[int, str]: - """Fingerprint source bytes already consumed without penalizing appends.""" +def _prefix_digest(path: Path, offset: int) -> str: + """SHA-256 of the consumed prefix ``[0, offset)``.""" - start = max(0, offset - _CONTINUITY_WINDOW) + hasher = hashlib.sha256() + if offset <= 0: + return hasher.hexdigest() + with path.open("rb") as stream: + remaining = offset + while remaining: + chunk = stream.read(min(remaining, _IO_CHUNK)) + if not chunk: + break + hasher.update(chunk) + remaining -= len(chunk) + return hasher.hexdigest() + + +def _contains_complete_line(path: Path, start: int, end: int) -> bool: + """Return True when ``[start, end)`` contains a newline-terminated line.""" + + if end <= start: + return False with path.open("rb") as stream: stream.seek(start) - digest = hashlib.sha256(stream.read(offset - start)).hexdigest() - return start, digest + remaining = end - start + while remaining: + chunk = stream.read(min(remaining, _IO_CHUNK)) + if not chunk: + return False + if b"\n" in chunk: + return True + remaining -= len(chunk) + return False def _json_value(value: Any) -> Any: @@ -205,10 +254,13 @@ def _workspace_record( """Read only safe workspace metadata and return it as independent evidence.""" path = directory / _WORKSPACE_FILENAME - if not path.is_file(): + resolved, escaped = _source_file_status(path, directory) + if escaped: + return None, None, [_diagnostic("workspace_path_escaped", "workspace.yaml resolves outside the session directory", file=str(path))] + if resolved is None: return None, None, [] try: - raw = path.read_bytes() + raw = resolved.read_bytes() data = yaml.safe_load(raw.decode("utf-8")) data = _json_value(data) except (OSError, TypeError, UnicodeDecodeError, yaml.YAMLError) as exc: @@ -218,7 +270,7 @@ def _workspace_record( digest = hashlib.sha256(raw).hexdigest() try: - generation = _generation(path) + generation = _generation(resolved) except OSError: generation = f"content-{digest}" cwd = data.get("cwd") @@ -248,7 +300,10 @@ def discover_transcripts(paths: SourcePaths) -> list[str]: for candidate in candidates: try: resolved = candidate.resolve(strict=False) - if not _within(resolved, root) or not resolved.is_dir() or not (resolved / _EVENTS_FILENAME).is_file(): + if not _within(resolved, root) or not resolved.is_dir(): + continue + events_file = _source_file_status(resolved / _EVENTS_FILENAME, resolved)[0] + if events_file is None: continue validate_native_id(candidate.name) except (OSError, ValueError): @@ -270,6 +325,8 @@ def read_transcript( A newline is the commit boundary: a trailing partial JSON or UTF-8 line is left untouched for the next read. Cursors retain a snapshot endpoint so a caller can drain the state observed at the start even while Copilot appends. + After that snapshot has only an incomplete tail left, the next call reopens + the endpoint to the current size so an appended newline can finish the line. """ if max_records < 0 or max_bytes < 0: @@ -282,11 +339,17 @@ def read_transcript( diagnostics.extend(workspace_diagnostics) records: list[SourceRecord] = [] + resolved_events, events_escaped = _source_file_status(event_path, directory) + if events_escaped: + diagnostics.append(_diagnostic("transcript_path_escaped", "events.jsonl resolves outside the session directory", file=str(event_path))) + return {"records": records, "next_cursor": dict(cursor), "diagnostics": diagnostics, "cwd": cwd, "exhausted": False} + if resolved_events is None: + diagnostics.append(_diagnostic("transcript_unavailable", "events.jsonl is unavailable; it is not considered complete", file=str(event_path))) + return {"records": records, "next_cursor": dict(cursor), "diagnostics": diagnostics, "cwd": cwd, "exhausted": False} + try: - stat = event_path.stat() - if not event_path.is_file(): - raise FileNotFoundError(event_path) - generation = _generation(event_path) + stat = resolved_events.stat() + generation = _generation(resolved_events) except OSError: diagnostics.append(_diagnostic("transcript_unavailable", "events.jsonl is unavailable; it is not considered complete", file=str(event_path))) return {"records": records, "next_cursor": dict(cursor), "diagnostics": diagnostics, "cwd": cwd, "exhausted": False} @@ -300,14 +363,13 @@ def read_transcript( reset = prior_generation is not None and prior_generation != generation if prior_offset > size: reset = True - anchor_start = cursor.get("continuity_start") - anchor_digest = cursor.get("continuity_digest") - if not reset and prior_offset and isinstance(anchor_start, int) and isinstance(anchor_digest, str): + prior_digest = cursor.get("continuity_digest") + if not reset and prior_offset and isinstance(prior_digest, str): try: - current_start, current_digest = _continuity_anchor(event_path, prior_offset) + current_digest = _prefix_digest(resolved_events, prior_offset) except OSError: - current_start, current_digest = -1, "" - if (current_start, current_digest) != (anchor_start, anchor_digest): + current_digest = "" + if current_digest != prior_digest: reset = True if reset: diagnostics.append(_diagnostic("transcript_replaced", "transcript was replaced or truncated; replaying from byte zero", file=str(event_path), previous_generation=prior_generation, file_generation=generation)) @@ -317,7 +379,12 @@ def read_transcript( if reset or not isinstance(prior_end, int) or prior_end < prior_offset: snapshot_end = size elif prior_offset < prior_end: - snapshot_end = min(prior_end, size) + frozen_end = min(prior_end, size) + try: + reopen = not _contains_complete_line(resolved_events, prior_offset, frozen_end) + except OSError: + reopen = True + snapshot_end = size if reopen else frozen_end else: snapshot_end = size @@ -333,7 +400,7 @@ def read_transcript( consumed = 0 limit_hit = False try: - with event_path.open("rb") as stream: + with resolved_events.open("rb") as stream: stream.seek(offset) while offset < snapshot_end: remaining = snapshot_end - offset @@ -365,11 +432,10 @@ def read_transcript( "byte_offset": offset, "file_generation": generation, "snapshot_end": snapshot_end, + "continuity_start": 0, } try: - anchor_start, anchor_digest = _continuity_anchor(event_path, offset) - next_cursor["continuity_start"] = anchor_start - next_cursor["continuity_digest"] = anchor_digest + next_cursor["continuity_digest"] = _prefix_digest(resolved_events, offset) except OSError: # The read above remains useful. A later invocation will report the # unavailable source instead of pretending that it reached completion. diff --git a/tests/test_copilot_transcript.py b/tests/test_copilot_transcript.py index a9f04d2..2f8926d 100644 --- a/tests/test_copilot_transcript.py +++ b/tests/test_copilot_transcript.py @@ -2,9 +2,7 @@ from __future__ import annotations -import hashlib import json -import os import shutil from pathlib import Path from typing import Any @@ -393,6 +391,29 @@ def test_read_transcript_opens_new_snapshot_for_appended_partial_line(tmp_path: assert second["exhausted"] is False +def test_read_transcript_appended_newline_completes_deferred_partial_line(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + path = _write_session(home, native, events='{"id":"first"}\n') + paths = _session_paths(home) + first = read_transcript(paths, native, {}) + assert first["exhausted"] is True + + with (path / "events.jsonl").open("ab") as stream: + stream.write(b'{"id":"second","type":"appended"') + + second = read_transcript(paths, native, first["next_cursor"]) + assert _transcript_records(second) == [] + assert second["exhausted"] is False + + with (path / "events.jsonl").open("ab") as stream: + stream.write(b'}\n') + + third = read_transcript(paths, native, second["next_cursor"]) + assert [record["payload"]["id"] for record in _transcript_records(third)] == ["second"] + assert third["exhausted"] is True + + def test_read_transcript_reads_appended_complete_line_in_fresh_snapshot(tmp_path: Path): home = tmp_path / "copilot" native = "session-a" @@ -475,6 +496,29 @@ def test_read_transcript_repeatable_source_ids_after_replacement(tmp_path: Path) assert first_id == second_id +def test_read_transcript_inplace_rewrite_before_continuity_window_replays(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + original = b'{"id":"orig"}\n' + rewritten = b'{"id":"edit"}\n' + assert len(original) == len(rewritten) + padding = b"".join(json.dumps({"id": f"pad-{index:03d}", "body": "x" * 64}).encode() + b"\n" for index in range(80)) + path = _write_session(home, native, events=original + padding) + paths = _session_paths(home) + + first = read_transcript(paths, native, {}) + assert first["next_cursor"]["byte_offset"] > 4096 + assert [record["payload"]["id"] for record in _transcript_records(first)[:1]] == ["orig"] + + with (path / "events.jsonl").open("r+b") as stream: + stream.seek(0) + stream.write(rewritten) + + second = read_transcript(paths, native, first["next_cursor"]) + assert "transcript_replaced" in _diagnostic_codes(second) + assert [record["payload"]["id"] for record in _transcript_records(second)[:1]] == ["edit"] + + # --- unavailable source and workspace errors --- @@ -489,6 +533,46 @@ def test_read_transcript_missing_events_is_not_exhausted(tmp_path: Path): assert "transcript_unavailable" in _diagnostic_codes(slice_) +def _symlink_or_skip(link: Path, target: Path) -> None: + try: + link.symlink_to(target) + except OSError as exc: + pytest.skip(f"symlinks unavailable: {exc}") + + +def test_read_transcript_rejects_events_symlink_outside_session(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + session_dir = _write_session(home, native) + outside = tmp_path / "outside-events.jsonl" + outside.write_text('{"id":"stolen"}\n', encoding="utf-8") + _symlink_or_skip(session_dir / "events.jsonl", outside) + + paths = _session_paths(home) + slice_ = read_transcript(paths, native, {}) + assert _transcript_records(slice_) == [] + assert "stolen" not in json.dumps(slice_["records"]) + assert slice_["exhausted"] is False + assert "transcript_path_escaped" in _diagnostic_codes(slice_) + assert discover_transcripts(paths) == [] + + +def test_read_transcript_rejects_workspace_symlink_outside_session(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + session_dir = _write_session(home, native, events='{"id":"1"}\n') + outside = tmp_path / "outside-workspace.yaml" + outside.write_text("cwd: /stolen/workspace\n", encoding="utf-8") + _symlink_or_skip(session_dir / "workspace.yaml", outside) + + slice_ = read_transcript(_session_paths(home), native, {}) + assert [record["payload"]["id"] for record in _transcript_records(slice_)] == ["1"] + assert _metadata_records(slice_) == [] + assert slice_["cwd"] is None + assert "/stolen/workspace" not in json.dumps(slice_["records"]) + assert "workspace_path_escaped" in _diagnostic_codes(slice_) + + def test_read_transcript_invalid_workspace_emits_diagnostic_without_blocking_events(tmp_path: Path): home = tmp_path / "copilot" native = "session-a" From 8ac9d29bb0f307dd58030e0e684d80feafa1ca2a Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Thu, 10 Sep 2026 14:44:31 -0700 Subject: [PATCH 16/88] Fix Copilot SQLite reader review findings. Discover sessions from every allowlisted table, identify rows from PRAGMA primary keys, close read-only connections, and paginate against a logical snapshot so live WAL metadata cannot starve later rows. Co-authored-by: Cursor --- src/thirdeye/platforms/copilot/database.py | 231 ++++++++---- tests/test_copilot_database.py | 387 +++++++++++++++++++-- 2 files changed, 518 insertions(+), 100 deletions(-) diff --git a/src/thirdeye/platforms/copilot/database.py b/src/thirdeye/platforms/copilot/database.py index 6afdc1f..88554e4 100644 --- a/src/thirdeye/platforms/copilot/database.py +++ b/src/thirdeye/platforms/copilot/database.py @@ -11,7 +11,8 @@ import hashlib import json import sqlite3 -from collections.abc import Iterable +from collections.abc import Iterable, Iterator, Sequence +from contextlib import contextmanager from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -26,9 +27,16 @@ "turns": ("session_id",), "assistant_usage_events": ("session_id",), } -_PRIMARY_KEY_COLUMNS = ("id", "uuid", "event_id") _TIMESTAMP_COLUMNS = ("created_at", "createdAt", "timestamp", "updated_at", "updatedAt") _CWD_COLUMNS = ("cwd", "working_directory", "workingDirectory") +_BUSY_TOKENS = ("locked", "busy") +_INCOMPATIBLE_TOKENS = ( + "file is not a database", + "malformed", + "corrupt", + "disk image is malformed", + "not a database", +) def _diagnostic(code: str, message: str, **details: Any) -> dict[str, Any]: @@ -50,6 +58,10 @@ def _json_value(value: Any) -> Any: return _json_value(value.tobytes()) if isinstance(value, float) and (value != value or value in (float("inf"), float("-inf"))): return {"encoding": "repr", "data": repr(value)} + if isinstance(value, dict): + return {str(key): _json_value(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_json_value(item) for item in value] return value @@ -67,43 +79,66 @@ def _quote_identifier(name: str) -> str: return '"' + name.replace('"', '""') + '"' -def _database_generation(database: Path) -> str: - """Identify the database and its live WAL without opening it for writing.""" +def _file_generation(database: Path) -> str: + """Identify the database file incarnation without consulting live WAL metadata. + + WAL size and mtime change independently of this session's rows. Device and + inode still change when the file is replaced, which is the signal needed to + detect row-ID reuse across a new database. + """ + + try: + stat = database.stat() + except FileNotFoundError: + payload: dict[str, Any] = {"path": database.name, "missing": True} + else: + payload = {"path": database.name, "device": stat.st_dev, "inode": stat.st_ino} + return "sha256:" + hashlib.sha256(_canonical_json(payload).encode("utf-8")).hexdigest() - parts: list[dict[str, Any]] = [] - for candidate in (database, Path(f"{database}-wal")): - try: - stat = candidate.stat() - except FileNotFoundError: - parts.append({"path": candidate.name, "missing": True}) - else: - parts.append( - { - "path": candidate.name, - "device": stat.st_dev, - "inode": stat.st_ino, - "size": stat.st_size, - "mtime_ns": stat.st_mtime_ns, - } - ) - return "sha256:" + hashlib.sha256(_canonical_json(parts).encode("utf-8")).hexdigest() + +def _snapshot_generation(rows: Sequence[tuple[str, Any, dict[str, Any]]]) -> str: + """Hash this session's row identities and revisions for pagination.""" + + fingerprint = [ + { + "table": table, + "primary_key": _json_value(primary_key), + "revision": _content_revision(row), + } + for table, primary_key, row in rows + ] + return "sha256:" + hashlib.sha256(_canonical_json(fingerprint).encode("utf-8")).hexdigest() def _connect(database: Path) -> sqlite3.Connection: # ``mode=ro`` keeps SQLite's normal WAL behaviour while preventing all # writes. In particular, do not use immutable=1: it ignores live WAL data. connection = sqlite3.connect(database.resolve().as_uri() + "?mode=ro", uri=True, timeout=0.1) - connection.row_factory = sqlite3.Row - connection.execute("PRAGMA busy_timeout = 100") - connection.execute("BEGIN") - return connection + try: + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA busy_timeout = 100") + connection.execute("BEGIN") + return connection + except Exception: + connection.close() + raise + + +@contextmanager +def _readonly_connection(database: Path) -> Iterator[sqlite3.Connection]: + connection = _connect(database) + try: + yield connection + finally: + connection.close() -def _table_columns(connection: sqlite3.Connection, table: str) -> list[str]: - return [ - str(row["name"]) - for row in connection.execute(f"PRAGMA table_info({_quote_identifier(table)})") - ] +def _table_info(connection: sqlite3.Connection, table: str) -> list[sqlite3.Row]: + return list(connection.execute(f"PRAGMA table_info({_quote_identifier(table)})")) + + +def _column_names(info: Iterable[sqlite3.Row]) -> list[str]: + return [str(row["name"]) for row in info] def _available_tables(connection: sqlite3.Connection) -> set[str]: @@ -118,9 +153,31 @@ def _session_column(table: str, columns: Iterable[str]) -> str | None: return next((name for name in _SESSION_ID_COLUMNS[table] if name in known), None) -def _primary_key_column(columns: Iterable[str]) -> str | None: - known = set(columns) - return next((name for name in _PRIMARY_KEY_COLUMNS if name in known), None) +def _primary_key_columns(info: Iterable[sqlite3.Row]) -> list[str] | None: + keyed = [(int(row["pk"]), str(row["name"])) for row in info if int(row["pk"]) > 0] + if not keyed: + return None + keyed.sort() + return [name for _, name in keyed] + + +def _session_scope_column( + table: str, columns: Iterable[str], pk_columns: list[str] | None +) -> str | None: + scoped = _session_column(table, columns) + if scoped is not None: + return scoped + # Observed Copilot sessions use ``id``; if a compatible schema names that + # single primary key differently, the key still *is* the native session ID. + if table == "sessions" and pk_columns is not None and len(pk_columns) == 1: + return pk_columns[0] + return None + + +def _row_identity(row: dict[str, Any], pk_columns: Sequence[str]) -> Any: + if len(pk_columns) == 1: + return row[pk_columns[0]] + return {name: row[name] for name in pk_columns} def _valid_source_time(row: dict[str, Any]) -> str | None: @@ -180,8 +237,42 @@ def _empty_slice(diagnostics: list[dict[str, Any]]) -> SourceSlice: } +def _operational_diagnostic(error: sqlite3.OperationalError, database: Path) -> dict[str, Any]: + message = str(error).lower() + if any(token in message for token in _BUSY_TOKENS): + return _diagnostic( + "copilot_database_busy", + "Copilot session database could not be read within 100ms; retry sync or watch", + path=str(database), + reason=str(error), + ) + if any(token in message for token in _INCOMPATIBLE_TOKENS): + return _diagnostic( + "copilot_database_incompatible", + "Copilot session database could not be read as SQLite evidence", + path=str(database), + reason=str(error), + ) + return _diagnostic( + "copilot_database_unreadable", + "Copilot session database could not be opened for read-only evidence", + path=str(database), + reason=str(error), + ) + + +def _session_ids_from_table( + connection: sqlite3.Connection, table: str, session_column: str +) -> list[str]: + rows = connection.execute( + f"SELECT {_quote_identifier(session_column)} FROM {_quote_identifier(table)} " + f"WHERE {_quote_identifier(session_column)} IS NOT NULL" + ) + return [str(row[0]) for row in rows if isinstance(row[0], str)] + + def discover_database_sessions(paths: SourcePaths) -> list[str]: - """Return session IDs advertised by a readable Copilot sessions table. + """Return session IDs advertised by any readable allowlisted table. Discovery is intentionally conservative: a malformed or absent database is simply not a discoverable source. ``read_database`` supplies the actionable @@ -192,21 +283,23 @@ def discover_database_sessions(paths: SourcePaths) -> list[str]: if not database.is_file(): return [] try: - with _connect(database) as connection: - if "sessions" not in _available_tables(connection): - return [] - columns = _table_columns(connection, "sessions") - session_column = _session_column("sessions", columns) - if session_column is None: - return [] - rows = connection.execute( - f"SELECT {_quote_identifier(session_column)} FROM sessions " - f"WHERE {_quote_identifier(session_column)} IS NOT NULL" - ) - values = [str(row[0]) for row in rows if isinstance(row[0], str)] + with _readonly_connection(database) as connection: + available = _available_tables(connection) + values: set[str] = set() + for table in _TABLES: + if table not in available: + continue + info = _table_info(connection, table) + columns = _column_names(info) + session_column = _session_scope_column( + table, columns, _primary_key_columns(info) + ) + if session_column is None: + continue + values.update(_session_ids_from_table(connection, table, session_column)) except sqlite3.Error: return [] - return sorted(set(values)) + return sorted(values) def read_database( @@ -219,8 +312,9 @@ def read_database( """Read a bounded, transactionally consistent raw SQLite snapshot. Every poll re-reads the selected session, because turns and sessions are - mutable. A generation/offset cursor only bounds delivery of that snapshot; - it never assumes a row ID is an immutable record identity. + mutable. Pagination is keyed to this session's logical snapshot so WAL + metadata from other writers cannot starve later rows. A changed snapshot + replays from the start so updated earlier rows are not skipped. """ validate_native_id(native_id) @@ -251,12 +345,12 @@ def read_database( diagnostics: list[dict[str, Any]] = [] observed_at = _observed_at() - generation = _database_generation(database) + file_generation = _file_generation(database) + rows_by_table: list[tuple[str, Any, dict[str, Any]]] = [] + cwd: str | None = None try: - with _connect(database) as connection: + with _readonly_connection(database) as connection: available = _available_tables(connection) - rows_by_table: list[tuple[str, Any, dict[str, Any]]] = [] - cwd: str | None = None for table in _TABLES: if table not in available: diagnostics.append( @@ -267,9 +361,10 @@ def read_database( ) ) continue - columns = _table_columns(connection, table) - session_column = _session_column(table, columns) - primary_column = _primary_key_column(columns) + info = _table_info(connection, table) + columns = _column_names(info) + pk_columns = _primary_key_columns(info) + session_column = _session_scope_column(table, columns, pk_columns) if session_column is None: diagnostics.append( _diagnostic( @@ -281,21 +376,21 @@ def read_database( ) ) continue - if primary_column is None: + if pk_columns is None: diagnostics.append( _diagnostic( "copilot_database_missing_primary_key", - "Copilot table lacks a supported stable row identity", + "Copilot table lacks a SQLite PRIMARY KEY that can identify rows", table=table, - expected=list(_PRIMARY_KEY_COLUMNS), columns=columns, ) ) continue + order = ", ".join(_quote_identifier(name) for name in pk_columns) query = ( f"SELECT * FROM {_quote_identifier(table)} " f"WHERE {_quote_identifier(session_column)} = ? " - f"ORDER BY {_quote_identifier(primary_column)}" + f"ORDER BY {order}" ) for sql_row in connection.execute(query, (native_id,)): row = {key: _json_value(sql_row[key]) for key in sql_row.keys()} @@ -304,18 +399,9 @@ def read_database( (row[name] for name in _CWD_COLUMNS if isinstance(row.get(name), str)), None, ) - rows_by_table.append((table, row[primary_column], row)) + rows_by_table.append((table, _row_identity(row, pk_columns), row)) except sqlite3.OperationalError as error: - return _empty_slice( - [ - _diagnostic( - "copilot_database_busy", - "Copilot session database could not be read within 100ms; retry sync or watch", - path=str(database), - reason=str(error), - ) - ] - ) + return _empty_slice([_operational_diagnostic(error, database)]) except sqlite3.Error as error: return _empty_slice( [ @@ -333,6 +419,7 @@ def read_database( rows_by_table.sort( key=lambda item: (_TABLES.index(item[0]), _canonical_json(_json_value(item[1]))) ) + generation = _snapshot_generation(rows_by_table) incoming_generation = cursor.get("database_generation") if isinstance(cursor, dict) else None incoming_offset = cursor.get("database_offset", 0) if isinstance(cursor, dict) else 0 offset = ( @@ -343,7 +430,7 @@ def read_database( offset = max(0, offset) selected = rows_by_table[offset : offset + max_records] records = [ - _row_record(paths, native_id, table, primary_key, row, generation, observed_at) + _row_record(paths, native_id, table, primary_key, row, file_generation, observed_at) for table, primary_key, row in selected ] next_offset = offset + len(selected) diff --git a/tests/test_copilot_database.py b/tests/test_copilot_database.py index 6b10099..9f968da 100644 --- a/tests/test_copilot_database.py +++ b/tests/test_copilot_database.py @@ -2,10 +2,12 @@ from __future__ import annotations +import gc import json import os import sqlite3 import threading +import warnings from pathlib import Path from typing import Any @@ -141,12 +143,13 @@ def _collect_all( ) -> list[dict[str, Any]]: cursor: dict[str, Any] = {} collected: list[dict[str, Any]] = [] - while True: + for _ in range(10_000): slice_ = read_database(paths, native_id, cursor, max_records=max_records) collected.extend(slice_["records"]) if slice_["exhausted"]: return collected cursor = slice_["next_cursor"] + raise AssertionError("database reader did not exhaust within 10000 bounded reads") # --- module boundaries --- @@ -334,33 +337,41 @@ def test_source_record_includes_revision_and_generation(tmp_path: Path): def test_reads_uncheckpointed_wal_commits(tmp_path: Path): home = tmp_path / "copilot" database = _write_database(home, session_id="session-a", turns=[(1, "checkpointed")]) - connection = sqlite3.connect(database) - connection.execute("PRAGMA journal_mode=WAL") - connection.execute( - "INSERT INTO turns (id, session_id, turn_index, content, updated_at) " - "VALUES (?, ?, ?, ?, ?)", - (2, "session-a", 2, "wal-only", "2026-09-10T17:08:20.000Z"), - ) - connection.execute( - "INSERT INTO assistant_usage_events (id, session_id, turn_index, model, created_at) " - "VALUES (?, ?, ?, ?, ?)", - (99, "session-a", 2, "gpt-test", "2026-09-10T17:08:21.000Z"), - ) - connection.commit() - connection.close() - - records = _collect_all(_paths(home), "session-a") - turn_contents = { - record["payload"]["row"]["content"] - for record in records - if record["payload"]["table"] == "turns" - } - assert turn_contents == {"checkpointed", "wal-only"} - assert any( - record["payload"]["table"] == "assistant_usage_events" - and record["payload"]["row"]["id"] == 99 - for record in records - ) + writer = sqlite3.connect(database) + try: + writer.execute("PRAGMA journal_mode=WAL") + writer.execute("PRAGMA wal_autocheckpoint=0") + writer.execute( + "INSERT INTO turns (id, session_id, turn_index, content, updated_at) " + "VALUES (?, ?, ?, ?, ?)", + (2, "session-a", 2, "wal-only", "2026-09-10T17:08:20.000Z"), + ) + writer.execute( + "INSERT INTO assistant_usage_events (id, session_id, turn_index, model, created_at) " + "VALUES (?, ?, ?, ?, ?)", + (99, "session-a", 2, "gpt-test", "2026-09-10T17:08:21.000Z"), + ) + writer.commit() + wal_path = Path(f"{database}-wal") + assert wal_path.is_file() + assert wal_path.stat().st_size > 0 + + records = _collect_all(_paths(home), "session-a") + turn_contents = { + record["payload"]["row"]["content"] + for record in records + if record["payload"]["table"] == "turns" + } + assert turn_contents == {"checkpointed", "wal-only"} + assert any( + record["payload"]["table"] == "assistant_usage_events" + and record["payload"]["row"]["id"] == 99 + for record in records + ) + assert wal_path.is_file() + assert wal_path.stat().st_size > 0 + finally: + writer.close() def test_late_database_rows_visible_without_transcript_changes(tmp_path: Path): @@ -642,3 +653,323 @@ def test_bytes_column_is_base64_encoded_in_payload(tmp_path: Path): "encoding": "base64", "data": "AP8=", } + + +# --- review fixes: discovery, pragma PK, cursor, diagnostics, connections --- + + +def test_discover_database_sessions_unions_ids_from_all_allowed_tables(tmp_path: Path): + home = tmp_path / "copilot" + home.mkdir() + database = home / "session-store.db" + connection = sqlite3.connect(database) + try: + connection.executescript( + """ + CREATE TABLE sessions (id TEXT PRIMARY KEY, cwd TEXT); + CREATE TABLE turns ( + id INTEGER PRIMARY KEY, + session_id TEXT, + user_message TEXT + ); + CREATE TABLE assistant_usage_events ( + id INTEGER PRIMARY KEY, + session_id TEXT, + model TEXT + ); + """ + ) + connection.execute("INSERT INTO sessions VALUES ('in-sessions', '/tmp')") + connection.execute("INSERT INTO turns VALUES (1, 'in-turns-only', 'hello')") + connection.execute( + "INSERT INTO assistant_usage_events VALUES (1, 'in-usage-only', 'gpt')" + ) + connection.commit() + finally: + connection.close() + + assert discover_database_sessions(_paths(home)) == [ + "in-sessions", + "in-turns-only", + "in-usage-only", + ] + + +def test_discover_database_sessions_without_sessions_table_uses_other_tables( + tmp_path: Path, +): + home = tmp_path / "copilot" + home.mkdir() + database = home / "session-store.db" + connection = sqlite3.connect(database) + try: + connection.executescript( + """ + CREATE TABLE turns ( + id INTEGER PRIMARY KEY, + session_id TEXT, + user_message TEXT + ); + CREATE TABLE assistant_usage_events ( + id INTEGER PRIMARY KEY, + session_id TEXT, + model TEXT + ); + """ + ) + connection.execute("INSERT INTO turns VALUES (1, 'from-turns', 'hello')") + connection.execute( + "INSERT INTO assistant_usage_events VALUES (1, 'from-usage', 'gpt')" + ) + connection.commit() + finally: + connection.close() + + assert discover_database_sessions(_paths(home)) == ["from-turns", "from-usage"] + + +def test_observed_cli_schema_preserves_session_and_turn_columns(tmp_path: Path): + home = tmp_path / "copilot" + schema = """ + CREATE TABLE sessions ( + id TEXT PRIMARY KEY, + cwd TEXT, + repository TEXT, + host_type TEXT, + branch TEXT, + summary TEXT, + created_at TEXT, + updated_at TEXT + ); + CREATE TABLE turns ( + id INTEGER PRIMARY KEY, + session_id TEXT NOT NULL, + turn_index INTEGER NOT NULL, + user_message TEXT, + assistant_response TEXT, + timestamp TEXT + ); + CREATE TABLE assistant_usage_events ( + id INTEGER PRIMARY KEY, + session_id TEXT NOT NULL, + turn_index INTEGER, + model TEXT, + created_at TEXT + ); + """ + _write_database(home, session_id="session-a", schema_sql=schema, seed_session=False) + database = home / "session-store.db" + connection = sqlite3.connect(database) + try: + connection.execute( + "INSERT INTO sessions " + "(id, cwd, repository, host_type, branch, summary, created_at, updated_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + ( + "session-a", + "/tmp/probe", + "github.com/example/repo", + "github", + "main", + "sum two files", + "2026-09-10T17:08:00.000Z", + "2026-09-10T17:08:50.000Z", + ), + ) + connection.execute( + "INSERT INTO turns " + "(id, session_id, turn_index, user_message, assistant_response, timestamp) " + "VALUES (?, ?, ?, ?, ?, ?)", + ( + 1, + "session-a", + 0, + "read alpha and beta", + "42", + "2026-09-10T17:08:10.000Z", + ), + ) + connection.execute( + "INSERT INTO assistant_usage_events " + "(id, session_id, turn_index, model, created_at) VALUES (?, ?, ?, ?, ?)", + (13, "session-a", 0, "gpt-5.6-luna", "2026-09-10T17:08:24.498Z"), + ) + connection.commit() + finally: + connection.close() + + records = _collect_all(_paths(home), "session-a") + session = _records_for_table(records, "sessions")[0] + turn = _records_for_table(records, "turns")[0] + usage = _records_for_table(records, "assistant_usage_events")[0] + assert session["payload"]["row"]["repository"] == "github.com/example/repo" + assert session["payload"]["row"]["cwd"] == "/tmp/probe" + assert turn["payload"]["row"]["user_message"] == "read alpha and beta" + assert turn["payload"]["row"]["assistant_response"] == "42" + assert turn["locator"]["primary_key"] == 1 + assert usage["payload"]["row"]["model"] == "gpt-5.6-luna" + + +def test_primary_key_uses_pragma_pk_when_column_is_not_named_id(tmp_path: Path): + home = tmp_path / "copilot" + schema = """ + CREATE TABLE sessions (session_pk TEXT PRIMARY KEY, cwd TEXT); + CREATE TABLE turns ( + turn_pk INTEGER PRIMARY KEY, + session_id TEXT, + user_message TEXT + ); + CREATE TABLE assistant_usage_events ( + usage_pk INTEGER PRIMARY KEY, + session_id TEXT, + model TEXT + ); + """ + _write_database(home, session_id="session-a", schema_sql=schema, seed_session=False) + connection = sqlite3.connect(home / "session-store.db") + try: + connection.execute("INSERT INTO sessions VALUES ('session-a', '/tmp')") + connection.execute("INSERT INTO turns VALUES (9, 'session-a', 'hello')") + connection.execute("INSERT INTO assistant_usage_events VALUES (3, 'session-a', 'gpt')") + connection.commit() + finally: + connection.close() + + records = _collect_all(_paths(home), "session-a") + session = _records_for_table(records, "sessions")[0] + turn = _records_for_table(records, "turns")[0] + usage = _records_for_table(records, "assistant_usage_events")[0] + assert session["locator"]["primary_key"] == "session-a" + assert turn["locator"]["primary_key"] == 9 + assert usage["locator"]["primary_key"] == 3 + assert "session-a" in session["source_id"] + + +def test_composite_primary_key_is_row_identity(tmp_path: Path): + home = tmp_path / "copilot" + schema = """ + CREATE TABLE sessions (id TEXT PRIMARY KEY, cwd TEXT); + CREATE TABLE turns ( + session_id TEXT NOT NULL, + turn_index INTEGER NOT NULL, + user_message TEXT, + assistant_response TEXT, + PRIMARY KEY (session_id, turn_index) + ); + CREATE TABLE assistant_usage_events ( + id INTEGER PRIMARY KEY, + session_id TEXT, + model TEXT + ); + """ + _write_database(home, session_id="session-a", schema_sql=schema, seed_session=False) + connection = sqlite3.connect(home / "session-store.db") + try: + connection.execute("INSERT INTO sessions VALUES ('session-a', '/tmp')") + connection.execute( + "INSERT INTO turns VALUES ('session-a', 0, 'first', 'reply-one')" + ) + connection.execute( + "INSERT INTO turns VALUES ('session-a', 1, 'second', 'reply-two')" + ) + connection.commit() + finally: + connection.close() + + turns = _records_for_table(_collect_all(_paths(home), "session-a"), "turns") + assert [record["locator"]["primary_key"] for record in turns] == [ + {"session_id": "session-a", "turn_index": 0}, + {"session_id": "session-a", "turn_index": 1}, + ] + assert turns[0]["source_id"] != turns[1]["source_id"] + + +def test_unrelated_wal_write_does_not_reset_pagination(tmp_path: Path): + home = tmp_path / "copilot" + database = _write_database( + home, + session_id="session-a", + turns=[(index, f"turn-{index}") for index in range(1, 6)], + extra_sessions=["session-b"], + ) + paths = _paths(home) + page_one = read_database(paths, "session-a", {}, max_records=2) + assert page_one["exhausted"] is False + first_ids = [record["source_id"] for record in page_one["records"]] + + writer = sqlite3.connect(database) + try: + writer.execute("PRAGMA journal_mode=WAL") + writer.execute("PRAGMA wal_autocheckpoint=0") + writer.execute( + "INSERT INTO turns (id, session_id, turn_index, content, updated_at) " + "VALUES (?, ?, ?, ?, ?)", + (50, "session-b", 1, "other-session", "2026-09-10T17:11:00.000Z"), + ) + writer.commit() + wal_path = Path(f"{database}-wal") + assert wal_path.is_file() + assert wal_path.stat().st_size > 0 + + page_two = read_database( + paths, "session-a", page_one["next_cursor"], max_records=2 + ) + second_ids = [record["source_id"] for record in page_two["records"]] + assert page_two["records"] + assert second_ids != first_ids + assert page_two["next_cursor"]["database_generation"] == page_one["next_cursor"][ + "database_generation" + ] + assert page_two["next_cursor"]["database_offset"] == 4 + finally: + writer.close() + + +def test_same_row_id_different_content_is_new_revision(tmp_path: Path): + home = tmp_path / "copilot" + database = _write_database(home, session_id="session-a", turns=[(7, "first")]) + paths = _paths(home) + before = _records_for_table(_collect_all(paths, "session-a"), "turns")[0] + + connection = sqlite3.connect(database) + try: + connection.execute("DELETE FROM turns WHERE id = 7") + connection.execute( + "INSERT INTO turns (id, session_id, turn_index, content, updated_at) " + "VALUES (?, ?, ?, ?, ?)", + (7, "session-a", 7, "reused", "2026-09-10T17:12:00.000Z"), + ) + connection.commit() + finally: + connection.close() + + after = _records_for_table(_collect_all(paths, "session-a"), "turns")[0] + assert before["locator"]["primary_key"] == after["locator"]["primary_key"] == 7 + assert before["locator"]["content_revision"] != after["locator"]["content_revision"] + assert before["source_id"] != after["source_id"] + assert after["payload"]["row"]["content"] == "reused" + + +def test_malformed_database_is_incompatible_not_busy(tmp_path: Path): + home = tmp_path / "copilot" + home.mkdir() + database = home / "session-store.db" + database.write_bytes(b"this is not a sqlite database") + slice_ = read_database(_paths(home), "session-a", {}) + assert slice_["records"] == [] + assert slice_["exhausted"] is True + assert slice_["diagnostics"][0]["code"] == "copilot_database_incompatible" + assert "busy" not in slice_["diagnostics"][0]["code"] + + +def test_read_and_discover_close_sqlite_connections(tmp_path: Path): + home = tmp_path / "copilot" + _write_database(home, session_id="session-a", turns=[(1, "closed")]) + paths = _paths(home) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always", ResourceWarning) + discover_database_sessions(paths) + read_database(paths, "session-a", {}) + gc.collect() + leaked = [item for item in caught if "unclosed database" in str(item.message)] + assert leaked == [] From 1b570469f9f1885913ddd104bd029c84c95d878f Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Thu, 10 Sep 2026 14:48:11 -0700 Subject: [PATCH 17/88] Fail Copilot hook install without a resolvable dispatcher and refuse malformed hook documents. Live installation must not write a hook that cannot run, unknown non-list hook values must be left unchanged, and add/setup needs the Copilot home, restart, status, and watch guidance. Co-authored-by: Cursor --- src/thirdeye/platforms/copilot/install.py | 53 +++++++++++--- tests/test_copilot_install.py | 86 +++++++++++++++++++++-- 2 files changed, 125 insertions(+), 14 deletions(-) diff --git a/src/thirdeye/platforms/copilot/install.py b/src/thirdeye/platforms/copilot/install.py index e41d911..cecefca 100644 --- a/src/thirdeye/platforms/copilot/install.py +++ b/src/thirdeye/platforms/copilot/install.py @@ -11,6 +11,7 @@ import json import re import shlex +import shutil from pathlib import Path, PureWindowsPath from typing import Any @@ -126,8 +127,8 @@ def _load_document(path: Path) -> dict[str, Any]: f"Cannot update Copilot hooks at {path}: 'hooks' must be an object. " "It was left unchanged." ) - for event in CLI_HOOK_EVENTS: - if event in hooks and not isinstance(hooks[event], list): + for event, value in hooks.items(): + if not isinstance(value, list): raise click.ClickException( f"Cannot update Copilot hooks at {path}: hooks.{event} must be a list. " "It was left unchanged." @@ -170,12 +171,47 @@ def hooks_file(self) -> Path: def entrypoint(self) -> str: return self._entrypoint or resolve_command(HOOK_BIN_NAME) - def _command(self, event: str) -> tuple[str, str]: + def _install_entrypoint(self) -> str: + """Return the dispatcher path that may be written into hook commands. + + An explicitly injected path is trusted so tests can supply a fixture + without installing the later runtime binary. Live installation + requires the dispatcher to be resolvable on PATH; Copilot itself is + not required. + """ + + if self._entrypoint: + return self._entrypoint + if shutil.which(HOOK_BIN_NAME) is None: + raise click.ClickException( + f"Cannot install Copilot hooks: {HOOK_BIN_NAME} is not on PATH. " + "Install thirdeye so the dispatcher entrypoint is available, then retry. " + "Copilot itself does not need to be on PATH." + ) + return resolve_command(HOOK_BIN_NAME) + + def _command(self, event: str, entrypoint: str | None = None) -> tuple[str, str]: + resolved = entrypoint if entrypoint is not None else self.entrypoint if self._windows: - return "powershell", _powershell_command(self.entrypoint, event) - return "bash", _bash_command(self.entrypoint, event) + return "powershell", _powershell_command(resolved, event) + return "bash", _bash_command(resolved, event) + + def _report_install(self) -> None: + click.echo(f"Configured Copilot CLI hooks at {self.hooks_file}") + if self._hooks_file is None or self._source_home is not None: + home = resolve_sources(self._source_home)["home"] + click.echo(f"Scope: Copilot home {home}") + click.echo( + "Restart Copilot CLI or start a new interactive session so the hooks take effect." + ) + click.echo("Verify receipt with: thirdeye copilot status") + click.echo( + "If hooks are missing or not firing, ingest persisted recordings with: " + "thirdeye copilot watch" + ) def install(self) -> None: + entrypoint = self._install_entrypoint() path = self.hooks_file data = _load_document(path) hooks = data["hooks"] @@ -186,7 +222,7 @@ def install(self) -> None: if not isinstance(entries, list): # Defensive for typed JSON input. raise AssertionError(f"validated hook list changed shape for {event}") retained = [entry for entry in entries if not _entry_is_ours(entry)] - shell, command = self._command(event) + shell, command = self._command(event, entrypoint) desired = { "type": "command", shell: command, @@ -197,6 +233,7 @@ def install(self) -> None: changed = True if changed or not path.exists(): _save_document(path, data) + self._report_install() def is_installed(self) -> bool: path = self.hooks_file @@ -230,8 +267,8 @@ def uninstall(self) -> None: for event in list(hooks): entries = hooks[event] if not isinstance(entries, list): - # Events outside our supported set are unrelated; leave their - # invalid shape intact rather than risking destructive repair. + # _load_document already rejected this shape; keep the guard so + # a concurrent rewrite cannot become a destructive repair. continue retained = [entry for entry in entries if not _entry_is_ours(entry)] if retained == entries: diff --git a/tests/test_copilot_install.py b/tests/test_copilot_install.py index 9513594..078cade 100644 --- a/tests/test_copilot_install.py +++ b/tests/test_copilot_install.py @@ -245,6 +245,10 @@ class TestMalformedConfig: json.dumps({"version": 1, "hooks": {"sessionStart": "nope"}}), "hooks.sessionStart must be a list", ), + ( + json.dumps({"version": 1, "hooks": {"customFutureEvent": "not-a-list"}}), + "hooks.customFutureEvent must be a list", + ), ], ) def test_install_refuses_malformed_document_and_preserves_bytes( @@ -267,6 +271,16 @@ def test_is_installed_returns_false_for_malformed_document(self, tmp_path: Path) platform = _platform(tmp_path, hooks_file=path) assert platform.is_installed() is False + def test_uninstall_refuses_malformed_unknown_event_and_preserves_bytes(self, tmp_path: Path): + path = tmp_path / OWNED_HOOK_FILENAME + payload = json.dumps({"version": 1, "hooks": {"customFutureEvent": "not-a-list"}}) + path.write_text(payload) + before = path.read_bytes() + with pytest.raises(click.ClickException) as exc_info: + _platform(tmp_path, hooks_file=path).uninstall() + assert "hooks.customFutureEvent must be a list" in str(exc_info.value) + assert path.read_bytes() == before + class TestSourceHomeScope: def test_resolves_hooks_file_under_source_home(self, tmp_path: Path): @@ -354,15 +368,75 @@ def test_recognizes_double_quoted_powershell_executable(self, tmp_path: Path): assert platform.is_installed() is True -class TestUninstallEdgeCases: - def test_leaves_invalid_unknown_event_shape_untouched(self, tmp_path: Path): +class TestUnknownEvents: + def test_preserves_well_formed_unknown_events_across_install_and_uninstall( + self, tmp_path: Path + ): path = tmp_path / OWNED_HOOK_FILENAME + foreign = { + "type": "command", + "bash": "/opt/foreign-hook customFutureEvent", + "timeoutSec": 9, + } + path.write_text( + json.dumps( + { + "version": HOOK_CONFIG_VERSION, + "hooks": {"customFutureEvent": [foreign]}, + } + ) + ) platform = _platform(tmp_path, hooks_file=path, windows=False) platform.install() - data = json.loads(path.read_text()) - data["hooks"]["customFutureEvent"] = "not-a-list" - path.write_text(json.dumps(data)) + installed = json.loads(path.read_text()) + assert installed["hooks"]["customFutureEvent"] == [foreign] platform.uninstall() remaining = json.loads(path.read_text()) - assert remaining["hooks"]["customFutureEvent"] == "not-a-list" + assert remaining["hooks"] == {"customFutureEvent": [foreign]} assert "sessionStart" not in remaining["hooks"] + + +class TestEntrypointRequired: + def test_install_errors_when_dispatcher_is_missing(self, tmp_path: Path, monkeypatch): + monkeypatch.setattr( + "thirdeye.platforms.copilot.install.shutil.which", + lambda _name: None, + ) + path = tmp_path / OWNED_HOOK_FILENAME + platform = CopilotPlatform(hooks_file=path, windows=False) + with pytest.raises(click.ClickException) as exc_info: + platform.install() + message = str(exc_info.value) + assert HOOK_BIN_NAME in message + assert "PATH" in message + assert not path.exists() + + def test_injected_entrypoint_does_not_require_path_lookup( + self, tmp_path: Path, monkeypatch + ): + monkeypatch.setattr( + "thirdeye.platforms.copilot.install.shutil.which", + lambda _name: None, + ) + path = tmp_path / OWNED_HOOK_FILENAME + _platform(tmp_path, hooks_file=path).install() + assert path.exists() + + +class TestInstallGuidance: + def test_reports_scope_restart_status_and_watch_fallback( + self, tmp_path: Path, capsys: pytest.CaptureFixture[str] + ): + source_home = tmp_path / "custom-copilot-home" + platform = CopilotPlatform( + source_home=source_home, + entrypoint="/opt/bin/thirdeye-copilot-hook", + windows=False, + ) + platform.install() + output = capsys.readouterr().out + assert str(platform.hooks_file) in output + assert f"Scope: Copilot home {source_home.resolve()}" in output + assert "Restart Copilot CLI" in output + assert "thirdeye copilot status" in output + assert "thirdeye copilot watch" in output From 626fdf94fef6ef9e1d0b538aa291c860c6aa7020 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Thu, 10 Sep 2026 14:48:30 -0700 Subject: [PATCH 18/88] Fix Copilot archive journal recovery, lifecycle, and cursor durability. Keep closed sessions closed on append, checkpoint lifecycle with the journal, strip contention markers from stored cursors, and finish recovery from load/iter so a crash cannot skip source records or invent timestamps. Co-authored-by: Cursor --- src/thirdeye/_compat/fsops.py | 21 ++ src/thirdeye/platforms/copilot/archive.py | 249 ++++++++++++++++++---- src/thirdeye/platforms/copilot/state.py | 17 ++ tests/test_copilot_archive.py | 113 +++++++++- tests/test_copilot_recovery.py | 229 +++++++++++++++++++- 5 files changed, 578 insertions(+), 51 deletions(-) diff --git a/src/thirdeye/_compat/fsops.py b/src/thirdeye/_compat/fsops.py index 947429d..2a6a6de 100644 --- a/src/thirdeye/_compat/fsops.py +++ b/src/thirdeye/_compat/fsops.py @@ -62,6 +62,27 @@ def unlink(path: Path, *, missing_ok: bool = False) -> None: path.unlink(missing_ok=missing_ok) +def sync_directory(path: Path | str) -> None: + """Best-effort ``fsync`` of a directory after ``replace`` or ``unlink``. + + Publishing a file with tmp-plus-replace (or removing a journal) is not + durable until the directory entry itself is synced. Some platforms, + notably Windows, reject directory ``fsync``; those errors are ignored so + callers stay portable. + """ + path = Path(path) + try: + fd = os.open(path, os.O_RDONLY) + except OSError: + return + try: + os.fsync(fd) + except OSError: + return + finally: + os.close(fd) + + def read_text(path: Path | str, *, encoding: str = "utf-8") -> str: """``Path.read_text`` with bounded retry on Windows ``PermissionError``. diff --git a/src/thirdeye/platforms/copilot/archive.py b/src/thirdeye/platforms/copilot/archive.py index 0d1d95d..0bab189 100644 --- a/src/thirdeye/platforms/copilot/archive.py +++ b/src/thirdeye/platforms/copilot/archive.py @@ -8,9 +8,9 @@ from __future__ import annotations +import json from collections.abc import Callable, Iterator from datetime import datetime -import json from pathlib import Path from typing import Any @@ -20,7 +20,7 @@ from thirdeye.paths import meta_path, session_dir from thirdeye.reader import SessionReader from thirdeye.store import Store -from thirdeye.writer import utc_iso_ms +from thirdeye.writer import SessionWriter, utc_iso_ms from .constants import PLATFORM_NAME, SOURCE_SCHEMA_VERSION from .identity import stored_session_id, validate_native_id @@ -42,6 +42,7 @@ "hook": "copilot_hook", "metadata": "copilot_metadata", } +_CURSOR_CONTROL_KEYS = ("base_cursor", "_base_cursor") # Tests and embedding applications may replace this with a deterministic # callback that raises at a journal boundary. It is deliberately private: no @@ -86,6 +87,17 @@ def _valid_timestamp(value: object) -> str | None: return value +def _source_cursor(cursor: object) -> dict[str, Any]: + """Return the source reader cursor, without archive contention markers.""" + if not isinstance(cursor, dict): + return {} + return {key: value for key, value in cursor.items() if key not in _CURSOR_CONTROL_KEYS} + + +def _cursor_copy(cursor: object) -> dict[str, Any]: + return json.loads(json.dumps(_source_cursor(cursor))) + + def _event_type(record: SourceRecord) -> str: return _EVENT_TYPES.get(record["source_kind"], "copilot_metadata") @@ -148,7 +160,9 @@ def _ensure_meta(config: Config, paths: SourcePaths, native_id: str, cwd: str | if not isinstance(identity, dict) or identity.get("source_key") != paths["source_key"]: raise ValueError("Copilot source-key prefix collision for stored session ID") if identity.get("native_session_id") != native_id: - raise ValueError("Copilot archive native session identity does not match stored session ID") + raise ValueError( + "Copilot archive native session identity does not match stored session ID" + ) return directory # Store owns generic session metadata. Creating it here also lets a hook @@ -178,13 +192,27 @@ def _base_cursor(batch: SourceBatch) -> dict[str, Any] | None: its cursor object when it needs optimistic contention detection. """ cursor = batch["next_cursor"] - for name in ("base_cursor", "_base_cursor"): + for name in _CURSOR_CONTROL_KEYS: value = cursor.get(name) if isinstance(value, dict): return value return None +def _observation_errors(records: list[SourceRecord]) -> list[dict[str, Any]]: + errors: list[dict[str, Any]] = [] + for record in records: + if _valid_timestamp(record.get("observed_at")) is None: + source_id = record.get("source_id") + errors.append( + { + "kind": "invalid_observed_at", + "source_id": source_id if isinstance(source_id, str) else None, + } + ) + return errors + + def _set_health( state: dict[str, Any], diagnostics: list[dict[str, Any]], *, successful: bool ) -> None: @@ -194,6 +222,16 @@ def _set_health( health["last_successful_import"] = utc_iso_ms() +def _writer_for_append( + config: Config, directory: Path, stored_id: str, cwd: str | None +) -> SessionWriter: + """Append into an existing archive without reopening Store session metadata.""" + meta = read_meta(meta_path(directory)) + if meta is None: + return Store(config).open_session(stored_id, platform=PLATFORM_NAME, cwd=cwd or "") + return SessionWriter(directory, meta) + + def _append_records( config: Config, directory: Path, @@ -201,11 +239,11 @@ def _append_records( cwd: str | None, records: list[SourceRecord], committed: dict[str, SourceRecord], -) -> tuple[int, int, list[dict[str, Any]]]: - _ = directory - writer = Store(config).open_session(stored_id, platform=PLATFORM_NAME, cwd=cwd or "") +) -> tuple[int, int, list[dict[str, Any]], bool]: + writer: SessionWriter | None = None written = 0 duplicates = 0 + blocked = False diagnostics: list[dict[str, Any]] = [] try: for record in records: @@ -216,19 +254,94 @@ def _append_records( if source_id in committed: duplicates += 1 continue - source_ts = _valid_timestamp(record.get("ts")) observed_at = _valid_timestamp(record.get("observed_at")) + if observed_at is None: + diagnostics.append({"kind": "invalid_observed_at", "source_id": source_id}) + blocked = True + continue + source_ts = _valid_timestamp(record.get("ts")) event_ts = source_ts or observed_at if source_ts is None: diagnostics.append({"kind": "missing_source_time", "source_id": source_id}) + if writer is None: + writer = _writer_for_append(config, directory, stored_id, cwd) writer.append(_event_type(record), _envelope(record), ts=event_ts) committed[source_id] = record written += 1 - writer.flush_and_detach() + if writer is not None: + writer.flush_and_detach() except BaseException: - writer.flush_and_detach() + if writer is not None: + writer.flush_and_detach() raise - return written, duplicates, diagnostics + return written, duplicates, diagnostics, blocked + + +def _identity_paths( + directory: Path, + state: dict[str, Any] | None, + journal: dict[str, Any] | None, +) -> tuple[SourcePaths, str]: + identity: dict[str, Any] = {} + if isinstance(state, dict): + identity = { + "source_key": state.get("source_key"), + "source_home": state.get("source_home"), + "native_session_id": state.get("native_session_id"), + } + if not isinstance(identity.get("source_key"), str) or not isinstance( + identity.get("native_session_id"), str + ): + meta = read_meta(meta_path(directory)) + extra = ( + meta.extra.get("copilot") if meta is not None and isinstance(meta.extra, dict) else None + ) + if isinstance(extra, dict): + identity = { + "source_key": extra.get("source_key"), + "source_home": extra.get("source_home"), + "native_session_id": extra.get("native_session_id"), + } + native_id = identity.get("native_session_id") + source_key = identity.get("source_key") + source_home = identity.get("source_home") + if not isinstance(native_id, str) and isinstance(journal, dict): + native_id = journal.get("native_session_id") + if not isinstance(source_key, str) and isinstance(journal, dict): + source_key = journal.get("source_key") + if ( + not isinstance(native_id, str) + or not isinstance(source_key, str) + or not isinstance(source_home, str) + ): + raise ValueError("Copilot archive identity is missing; cannot finish journal recovery") + home = Path(source_home) + paths: SourcePaths = { + "home": source_home, + "source_key": source_key, + "session_root": str(home / "session-state"), + "database": str(home / "session-store.db"), + } + return paths, native_id + + +def _checkpoint( + directory: Path, + state: dict[str, Any], + *, + next_cursor: dict[str, Any], + diagnostics: list[dict[str, Any]], + records: list[Any], +) -> dict[str, Any]: + _apply_lifecycle(directory, records) + _fault("after_lifecycle") + state["cursor"] = _source_cursor(next_cursor) + _set_health(state, diagnostics, successful=True) + write_state(directory, state) + _fault("after_checkpoint") + clear_journal(directory) + _fault("after_journal_clear") + return state def _recover( @@ -241,25 +354,48 @@ def _recover( journal = read_json(journal_path(directory)) if journal is None: return state, 0, 0 - if journal.get("source_key") != paths["source_key"] or journal.get("native_session_id") != native_id: + if ( + journal.get("source_key") != paths["source_key"] + or journal.get("native_session_id") != native_id + ): raise ValueError("Copilot archive journal belongs to another source session") records = journal.get("records") next_cursor = journal.get("next_cursor") if not isinstance(records, list) or not isinstance(next_cursor, dict): raise ValueError("invalid Copilot archive journal") committed = _committed_records(directory) - written, duplicates, diagnostics = _append_records( - config, directory, stored_session_id(paths, native_id), journal.get("cwd"), records, committed + written, duplicates, diagnostics, blocked = _append_records( + config, directory, directory.name, journal.get("cwd"), records, committed ) _fault("after_recovery_append") - state["cursor"] = next_cursor - _set_health(state, list(journal.get("diagnostics", [])) + diagnostics, successful=True) - write_state(directory, state) - _fault("after_recovery_checkpoint") - clear_journal(directory) + if blocked: + _set_health(state, list(journal.get("diagnostics", [])) + diagnostics, successful=False) + write_state(directory, state) + clear_journal(directory) + return state, written, duplicates + state = _checkpoint( + directory, + state, + next_cursor=next_cursor, + diagnostics=list(journal.get("diagnostics", [])) + diagnostics, + records=records, + ) return state, written, duplicates +def _recover_directory(config: Config, directory: Path) -> None: + journal = read_json(journal_path(directory)) + if journal is None: + return + state = read_json(state_path(directory)) + paths, native_id = _identity_paths(directory, state, journal) + if state is None: + state = _new_state(paths, native_id) + else: + _validate_state(state, paths, native_id) + _recover(config, paths, native_id, directory, state) + + def load_cursor(config: Config, paths: SourcePaths, native_id: str) -> dict: """Return a copy of the last committed cursor for one native session.""" validate_native_id(native_id) @@ -267,12 +403,16 @@ def load_cursor(config: Config, paths: SourcePaths, native_id: str) -> dict: if not directory.exists(): return {} with locked(lock_path(directory), LockMode.EXCLUSIVE): + journal_exists = journal_path(directory).is_file() state = read_json(state_path(directory)) - if state is None: + if state is None and not journal_exists: return {} - _validate_state(state, paths, native_id) - # JSON copying protects the on-disk state from caller mutation. - return json.loads(json.dumps(state.get("cursor", {}))) + if state is None: + state = _new_state(paths, native_id) + else: + _validate_state(state, paths, native_id) + state, _, _ = _recover(config, paths, native_id, directory, state) + return _cursor_copy(state.get("cursor", {})) def commit_batch(config: Config, paths: SourcePaths, batch: SourceBatch) -> SyncResult: @@ -289,10 +429,12 @@ def commit_batch(config: Config, paths: SourcePaths, batch: SourceBatch) -> Sync with locked(lock_path(directory), LockMode.EXCLUSIVE): state = read_json(state_path(directory)) or _new_state(paths, native_id) _validate_state(state, paths, native_id) - state, recovered_written, recovered_duplicates = _recover(config, paths, native_id, directory, state) + state, recovered_written, recovered_duplicates = _recover( + config, paths, native_id, directory, state + ) expected = _base_cursor(batch) - current = state.get("cursor", {}) + current = _source_cursor(state.get("cursor", {})) if expected is not None and expected != current: diagnostic = { "kind": "stale_cursor", @@ -310,29 +452,58 @@ def commit_batch(config: Config, paths: SourcePaths, batch: SourceBatch) -> Sync errors=1, ) + observation_errors = _observation_errors(batch["records"]) + if observation_errors: + _set_health(state, list(batch["diagnostics"]) + observation_errors, successful=False) + write_state(directory, state) + return _result( + sessions=1, + written=recovered_written, + duplicates=recovered_duplicates, + pending=len(batch["records"]), + errors=1, + ) + + next_cursor = _source_cursor(batch["next_cursor"]) journal = { "schema_version": STATE_SCHEMA_VERSION, "source_key": paths["source_key"], "native_session_id": native_id, "cwd": batch.get("cwd"), "records": batch["records"], - "next_cursor": batch["next_cursor"], + "next_cursor": next_cursor, "diagnostics": batch["diagnostics"], } write_journal(directory, journal) _fault("after_journal") committed = _committed_records(directory) - written, duplicates, diagnostics = _append_records( + written, duplicates, diagnostics, blocked = _append_records( config, directory, stored_id, batch.get("cwd"), batch["records"], committed ) _fault("after_append") - state["cursor"] = batch["next_cursor"] - _set_health(state, list(batch["diagnostics"]) + diagnostics, successful=True) - write_state(directory, state) - _fault("after_checkpoint") - clear_journal(directory) - _fault("after_journal_clear") - _apply_lifecycle(directory, batch["records"]) + if blocked: + _set_health(state, list(batch["diagnostics"]) + diagnostics, successful=False) + write_state(directory, state) + clear_journal(directory) + return _result( + sessions=1, + written=recovered_written + written, + duplicates=recovered_duplicates + duplicates, + pending=sum( + 1 + for record in batch["records"] + if isinstance(record.get("source_id"), str) + and record["source_id"] not in committed + ), + errors=1, + ) + _checkpoint( + directory, + state, + next_cursor=next_cursor, + diagnostics=list(batch["diagnostics"]) + diagnostics, + records=batch["records"], + ) return _result( sessions=1, written=recovered_written + written, @@ -340,17 +511,19 @@ def commit_batch(config: Config, paths: SourcePaths, batch: SourceBatch) -> Sync ) -def _apply_lifecycle(directory: Path, records: list[SourceRecord]) -> None: +def _apply_lifecycle(directory: Path, records: list[Any]) -> None: """Apply only explicit top-level lifecycle evidence; child stops never close.""" close = False reopen = False for record in records: - if record["source_kind"] != "hook": + if not isinstance(record, dict) or record.get("source_kind") != "hook": continue payload = record.get("payload", {}) event = payload.get("event") if isinstance(payload, dict) else None context = payload.get("context") if isinstance(payload, dict) else None - is_child = isinstance(context, dict) and bool(context.get("parent_tool_call_id") or context.get("agent_id")) + is_child = isinstance(context, dict) and bool( + context.get("parent_tool_call_id") or context.get("agent_id") + ) if event in {"sessionEnd", "shutdown", "session_end"} and not is_child: close = True if event in {"sessionStart", "resume", "activity", "session_start", "userPromptSubmitted"}: @@ -375,6 +548,8 @@ def iter_captured_records(config: Config, stored_session_id: str) -> Iterator[So directory = _archive_dir(config, stored_session_id) if not directory.exists(): return + with locked(lock_path(directory), LockMode.EXCLUSIVE): + _recover_directory(config, directory) for event in SessionReader(directory).iter_events(types=_EVENT_TYPES.values()): record = _record_from_event(event) if record is not None: diff --git a/src/thirdeye/platforms/copilot/state.py b/src/thirdeye/platforms/copilot/state.py index cced32d..99a26c0 100644 --- a/src/thirdeye/platforms/copilot/state.py +++ b/src/thirdeye/platforms/copilot/state.py @@ -5,6 +5,7 @@ import json import os import tempfile +from collections.abc import Callable from pathlib import Path from typing import Any @@ -15,6 +16,15 @@ JOURNAL_FILENAME = "copilot.journal.json" LOCK_FILENAME = "copilot.archive.lock" +# Tests may replace this with a deterministic callback that raises at a +# publication boundary. Runtime behaviour never depends on fault injection. +_fault_injector: Callable[[str], None] | None = None + + +def _fault(point: str) -> None: + if _fault_injector is not None: + _fault_injector(point) + def state_path(session_dir: Path) -> Path: return session_dir / STATE_FILENAME @@ -38,6 +48,12 @@ def _atomic_json(path: Path, value: dict[str, Any]) -> None: stream.flush() os.fsync(stream.fileno()) fsops.replace(temp_name, path) + if path.name == STATE_FILENAME: + _fault("after_state_replace") + elif path.name == JOURNAL_FILENAME: + _fault("after_journal_replace") + fsops.sync_directory(path.parent) + _fault("after_dirsync") except BaseException: fsops.unlink(Path(temp_name), missing_ok=True) raise @@ -68,3 +84,4 @@ def write_journal(session_dir: Path, value: dict[str, Any]) -> None: def clear_journal(session_dir: Path) -> None: fsops.unlink(journal_path(session_dir), missing_ok=True) + fsops.sync_directory(session_dir) diff --git a/tests/test_copilot_archive.py b/tests/test_copilot_archive.py index 449dc86..3b98251 100644 --- a/tests/test_copilot_archive.py +++ b/tests/test_copilot_archive.py @@ -15,7 +15,7 @@ from thirdeye.platforms.copilot.archive import commit_batch, iter_captured_records, load_cursor from thirdeye.platforms.copilot.constants import PLATFORM_NAME, SOURCE_SCHEMA_VERSION from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id -from thirdeye.platforms.copilot.state import journal_path, read_json, state_path +from thirdeye.platforms.copilot.state import read_json, state_path from thirdeye.platforms.copilot.types import SourceBatch, SourcePaths, SourceRecord from thirdeye.reader import SessionReader @@ -87,7 +87,9 @@ def _session_directory(config: Config, paths: SourcePaths) -> Path: return session_dir(config.root, PLATFORM_NAME, stored) -def test_commit_batch_writes_records_and_advances_cursor(config: Config, paths: SourcePaths) -> None: +def test_commit_batch_writes_records_and_advances_cursor( + config: Config, paths: SourcePaths +) -> None: records = [_record("key/a/event-1"), _record("key/a/event-2")] result = commit_batch(config, paths, _batch(paths, records, next_cursor={"offset": 2})) @@ -108,7 +110,9 @@ def test_load_cursor_returns_empty_for_unknown_session(config: Config, paths: So def test_load_cursor_returns_defensive_copy(config: Config, paths: SourcePaths) -> None: - commit_batch(config, paths, _batch(paths, [_record("key/a/event-1")], next_cursor={"offset": 1})) + commit_batch( + config, paths, _batch(paths, [_record("key/a/event-1")], next_cursor={"offset": 1}) + ) cursor = load_cursor(config, paths, NATIVE_ID) cursor["offset"] = 999 assert load_cursor(config, paths, NATIVE_ID) == {"offset": 1} @@ -149,15 +153,41 @@ def test_original_source_timestamp_is_retained(config: Config, paths: SourcePath commit_batch( config, paths, - _batch(paths, [_record("key/a/ts", ts=source_ts, observed_at="2026-09-10T17:08:99.000Z")]), + _batch(paths, [_record("key/a/ts", ts=source_ts, observed_at="2026-09-10T17:08:25.000Z")]), ) event = SessionReader(_session_directory(config, paths)).get_event(0) assert event["ts"] == source_ts +def test_invalid_observed_at_is_rejected_even_when_source_ts_is_valid( + config: Config, paths: SourcePaths +) -> None: + result = commit_batch( + config, + paths, + _batch( + paths, + [ + _record( + "key/a/bad-observed", + ts="2026-09-10T17:08:24.000Z", + observed_at="2026-09-10T17:08:99.000Z", + ) + ], + next_cursor={"offset": 1}, + ), + ) + assert result["records_written"] == 0 + assert result["pending"] == 1 + assert result["errors"] == 1 + assert list(iter_captured_records(config, stored_session_id(paths, NATIVE_ID))) == [] + + def test_missing_source_time_falls_back_to_observed_at(config: Config, paths: SourcePaths) -> None: observed = "2026-09-10T17:08:25.000Z" - commit_batch(config, paths, _batch(paths, [_record("key/a/no-ts", ts=None, observed_at=observed)])) + commit_batch( + config, paths, _batch(paths, [_record("key/a/no-ts", ts=None, observed_at=observed)]) + ) event = SessionReader(_session_directory(config, paths)).get_event(0) assert event["ts"] == observed @@ -256,6 +286,75 @@ def test_provisional_session_metadata_is_created(config: Config, paths: SourcePa assert identity["native_session_id"] == NATIVE_ID +def test_committed_cursor_strips_optimistic_base_marker(config: Config, paths: SourcePaths) -> None: + commit_batch(config, paths, _batch(paths, [_record("key/a/one")], next_cursor={"offset": 1})) + assert load_cursor(config, paths, NATIVE_ID) == {"offset": 1} + + commit_batch( + config, + paths, + _batch(paths, [_record("key/a/two")], next_cursor={"offset": 2}, base_cursor={"offset": 1}), + ) + cursor = load_cursor(config, paths, NATIVE_ID) + assert cursor == {"offset": 2} + state = read_json(state_path(_session_directory(config, paths))) + assert state is not None + assert "base_cursor" not in state["cursor"] + assert "_base_cursor" not in state["cursor"] + + +def test_append_does_not_reopen_closed_session(config: Config, paths: SourcePaths) -> None: + close_record = _record( + "key/a/close", + source_kind="hook", + payload={"event": "sessionEnd", "context": {}}, + ) + commit_batch(config, paths, _batch(paths, [close_record], next_cursor={"generation": 1})) + closed = read_meta(meta_path(_session_directory(config, paths))) + assert closed is not None + assert closed.status == "closed" + ended_at = closed.ended_at + assert ended_at is not None + + commit_batch( + config, + paths, + _batch( + paths, + [_record("key/a/after-close")], + next_cursor={"generation": 2}, + base_cursor={"generation": 1}, + ), + ) + meta = read_meta(meta_path(_session_directory(config, paths))) + assert meta is not None + assert meta.status == "closed" + assert meta.ended_at == ended_at + + +def test_invalid_timestamps_are_rejected_without_inventing_time( + config: Config, paths: SourcePaths +) -> None: + result = commit_batch( + config, + paths, + _batch( + paths, + [_record("key/a/bad-ts", ts="not-a-timestamp", observed_at="also-invalid")], + next_cursor={"offset": 1}, + ), + ) + assert result["records_written"] == 0 + assert result["pending"] == 1 + assert result["errors"] == 1 + assert load_cursor(config, paths, NATIVE_ID) == {} + assert list(iter_captured_records(config, stored_session_id(paths, NATIVE_ID))) == [] + + state = read_json(state_path(_session_directory(config, paths))) + assert state is not None + assert any(item.get("kind") == "invalid_observed_at" for item in state["health"]["diagnostics"]) + + def test_session_end_closes_and_resume_reopens(config: Config, paths: SourcePaths) -> None: close_record = _record( "key/a/close", @@ -276,7 +375,9 @@ def test_session_end_closes_and_resume_reopens(config: Config, paths: SourcePath commit_batch( config, paths, - _batch(paths, [reopen_record], next_cursor={"generation": 2}, base_cursor={"generation": 1}), + _batch( + paths, [reopen_record], next_cursor={"generation": 2}, base_cursor={"generation": 1} + ), ) meta = read_meta(meta_path(_session_directory(config, paths))) assert meta is not None diff --git a/tests/test_copilot_recovery.py b/tests/test_copilot_recovery.py index 61c61e8..f5f15da 100644 --- a/tests/test_copilot_recovery.py +++ b/tests/test_copilot_recovery.py @@ -4,12 +4,17 @@ from collections.abc import Iterator from contextlib import contextmanager +from pathlib import Path from typing import Any import pytest import thirdeye.platforms.copilot.archive as archive_mod +import thirdeye.platforms.copilot.state as state_mod +from thirdeye._compat import fsops from thirdeye.config import Config +from thirdeye.meta import read_meta +from thirdeye.paths import meta_path from thirdeye.platforms.copilot.archive import commit_batch, iter_captured_records, load_cursor from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id from thirdeye.platforms.copilot.state import journal_path, read_json, state_path @@ -18,14 +23,20 @@ NATIVE_ID = "session-a" -def _record(source_id: str, *, native_session_id: str = NATIVE_ID) -> SourceRecord: +def _record( + source_id: str, + *, + native_session_id: str = NATIVE_ID, + source_kind: str = "transcript", + payload: dict[str, Any] | None = None, +) -> SourceRecord: return { "source_id": source_id, - "source_kind": "transcript", + "source_kind": source_kind, "native_session_id": native_session_id, "ts": "2026-09-10T17:08:24.000Z", "observed_at": "2026-09-10T17:08:25.000Z", - "payload": {"schema_version": 1, "type": "user.message"}, + "payload": payload or {"schema_version": 1, "type": "user.message"}, "locator": {"file": "events.jsonl", "offset": 0}, } @@ -60,10 +71,12 @@ def injector(name: str) -> None: raise RuntimeError(f"injected fault at {point}") archive_mod._fault_injector = injector + state_mod._fault_injector = injector try: yield seen finally: archive_mod._fault_injector = None + state_mod._fault_injector = None @pytest.fixture @@ -112,7 +125,6 @@ def test_crash_at_journal_boundary_recovers_on_next_commit( if fault_point == "after_journal": assert journal_path(directory).is_file() - assert not list(iter_captured_records(config, stored_session_id(paths, NATIVE_ID))) if fault_point in {"after_append", "after_checkpoint", "after_journal_clear"}: captured_before = list(iter_captured_records(config, stored_session_id(paths, NATIVE_ID))) @@ -132,10 +144,13 @@ def test_crash_at_journal_boundary_recovers_on_next_commit( assert recovery["records_written"] >= 1 assert not journal_path(directory).exists() cursor = load_cursor(config, paths, NATIVE_ID) - assert cursor["generation"] == 2 - assert cursor.get("base_cursor") == {"generation": 1} + assert cursor == {"generation": 2} captured = list(iter_captured_records(config, stored_session_id(paths, NATIVE_ID))) - assert {r["source_id"] for r in captured} >= {"key/a/recover-1", "key/a/recover-2", "key/a/recover-3"} + assert {r["source_id"] for r in captured} >= { + "key/a/recover-1", + "key/a/recover-2", + "key/a/recover-3", + } def test_recovery_after_journal_only_replays_uncommitted_records( @@ -157,7 +172,10 @@ def test_recovery_after_journal_only_replays_uncommitted_records( ) assert result["records_written"] == 1 assert result["duplicate_records"] == 0 - assert list(iter_captured_records(config, stored_session_id(paths, NATIVE_ID)))[0]["source_id"] == "key/a/journal-only" + assert ( + list(iter_captured_records(config, stored_session_id(paths, NATIVE_ID)))[0]["source_id"] + == "key/a/journal-only" + ) def test_stale_cursor_rejects_competing_batch(config: Config, paths: SourcePaths) -> None: @@ -256,3 +274,198 @@ def test_recovery_replay_is_idempotent_when_events_already_committed( ) assert second_recovery["records_written"] == 1 assert second_recovery["duplicate_records"] == 0 + + +def _session_end_record(source_id: str = "key/a/close") -> SourceRecord: + return _record( + source_id, + source_kind="hook", + payload={"event": "sessionEnd", "context": {}}, + ) + + +def _resume_record(source_id: str = "key/a/resume") -> SourceRecord: + return _record( + source_id, + source_kind="hook", + payload={"event": "resume", "context": {}}, + ) + + +@pytest.mark.parametrize( + "fault_point", + [ + "after_journal", + "after_append", + "after_lifecycle", + "after_checkpoint", + "after_journal_clear", + ], +) +def test_session_end_survives_crash_at_every_boundary( + config: Config, + paths: SourcePaths, + fault_point: str, +) -> None: + directory = _session_dir(config, paths) + with fault_at(fault_point): + with pytest.raises(RuntimeError, match="injected fault"): + commit_batch( + config, + paths, + _batch(paths, [_session_end_record()], next_cursor={"generation": 1}), + ) + + cursor = load_cursor(config, paths, NATIVE_ID) + assert cursor == {"generation": 1} + assert not journal_path(directory).exists() + meta = read_meta(meta_path(directory)) + assert meta is not None + assert meta.status == "closed" + assert meta.ended_at is not None + + +@pytest.mark.parametrize( + "fault_point", + [ + "after_journal", + "after_append", + "after_lifecycle", + "after_checkpoint", + "after_journal_clear", + ], +) +def test_resume_survives_crash_at_every_boundary( + config: Config, + paths: SourcePaths, + fault_point: str, +) -> None: + directory = _session_dir(config, paths) + commit_batch( + config, + paths, + _batch(paths, [_session_end_record()], next_cursor={"generation": 1}), + ) + closed = read_meta(meta_path(directory)) + assert closed is not None + assert closed.status == "closed" + + with fault_at(fault_point): + with pytest.raises(RuntimeError, match="injected fault"): + commit_batch( + config, + paths, + _batch( + paths, + [_resume_record()], + next_cursor={"generation": 2}, + base_cursor={"generation": 1}, + ), + ) + + cursor = load_cursor(config, paths, NATIVE_ID) + assert cursor == {"generation": 2} + assert not journal_path(directory).exists() + meta = read_meta(meta_path(directory)) + assert meta is not None + assert meta.status == "open" + assert meta.ended_at is None + + +def test_load_cursor_completes_recovery_without_new_batch( + config: Config, + paths: SourcePaths, +) -> None: + directory = _session_dir(config, paths) + with fault_at("after_journal"): + with pytest.raises(RuntimeError): + commit_batch( + config, + paths, + _batch(paths, [_record("key/a/journal-only")], next_cursor={"generation": 1}), + ) + + assert journal_path(directory).is_file() + cursor = load_cursor(config, paths, NATIVE_ID) + assert cursor == {"generation": 1} + assert not journal_path(directory).exists() + captured = list(iter_captured_records(config, stored_session_id(paths, NATIVE_ID))) + assert [record["source_id"] for record in captured] == ["key/a/journal-only"] + + +def test_iter_captured_records_completes_recovery_without_new_batch( + config: Config, + paths: SourcePaths, +) -> None: + directory = _session_dir(config, paths) + with fault_at("after_journal"): + with pytest.raises(RuntimeError): + commit_batch( + config, + paths, + _batch(paths, [_record("key/a/via-iter")], next_cursor={"generation": 1}), + ) + + assert journal_path(directory).is_file() + captured = list(iter_captured_records(config, stored_session_id(paths, NATIVE_ID))) + assert [record["source_id"] for record in captured] == ["key/a/via-iter"] + assert load_cursor(config, paths, NATIVE_ID) == {"generation": 1} + assert not journal_path(directory).exists() + + +def test_state_publication_fsyncs_directory_after_replace_and_unlink( + config: Config, + paths: SourcePaths, + monkeypatch: pytest.MonkeyPatch, +) -> None: + order: list[str] = [] + real_replace = fsops.replace + real_unlink = fsops.unlink + real_sync = fsops.sync_directory + + def replace(src: Path | str, dst: Path | str) -> None: + order.append(f"replace:{Path(dst).name}") + real_replace(src, dst) + + def unlink(path: Path, *, missing_ok: bool = False) -> None: + order.append(f"unlink:{path.name}") + real_unlink(path, missing_ok=missing_ok) + + def sync_directory(path: Path) -> None: + order.append(f"dirsync:{path.name}") + real_sync(path) + + monkeypatch.setattr(fsops, "replace", replace) + monkeypatch.setattr(fsops, "unlink", unlink) + monkeypatch.setattr(fsops, "sync_directory", sync_directory) + + commit_batch( + config, + paths, + _batch(paths, [_record("key/a/durable")], next_cursor={"generation": 1}), + ) + + pairs = list(zip(order, order[1:], strict=False)) + assert ("replace:copilot.journal.json", f"dirsync:{_session_dir(config, paths).name}") in pairs + assert ("replace:copilot.state.json", f"dirsync:{_session_dir(config, paths).name}") in pairs + assert ("unlink:copilot.journal.json", f"dirsync:{_session_dir(config, paths).name}") in pairs + + +def test_crash_between_replace_and_directory_sync_still_publishes_state( + config: Config, + paths: SourcePaths, +) -> None: + directory = _session_dir(config, paths) + with fault_at("after_state_replace"): + with pytest.raises(RuntimeError, match="injected fault"): + commit_batch( + config, + paths, + _batch(paths, [_record("key/a/dirsync")], next_cursor={"generation": 1}), + ) + + assert state_path(directory).is_file() + cursor = load_cursor(config, paths, NATIVE_ID) + assert cursor == {"generation": 1} + captured = list(iter_captured_records(config, stored_session_id(paths, NATIVE_ID))) + assert [record["source_id"] for record in captured] == ["key/a/dirsync"] From 297f0849b4e8fbe4847d5bbba6668b78d0eab051 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Thu, 10 Sep 2026 14:49:55 -0700 Subject: [PATCH 19/88] Stop Copilot SQLite pagination from resetting on live session writes. Key the cursor to the database file incarnation instead of the session snapshot hash so inserts and updates cannot starve later rows. Co-authored-by: Cursor --- src/thirdeye/platforms/copilot/database.py | 24 +--- tests/test_copilot_database.py | 129 +++++++++++++++++++++ 2 files changed, 135 insertions(+), 18 deletions(-) diff --git a/src/thirdeye/platforms/copilot/database.py b/src/thirdeye/platforms/copilot/database.py index 88554e4..2e83bed 100644 --- a/src/thirdeye/platforms/copilot/database.py +++ b/src/thirdeye/platforms/copilot/database.py @@ -96,20 +96,6 @@ def _file_generation(database: Path) -> str: return "sha256:" + hashlib.sha256(_canonical_json(payload).encode("utf-8")).hexdigest() -def _snapshot_generation(rows: Sequence[tuple[str, Any, dict[str, Any]]]) -> str: - """Hash this session's row identities and revisions for pagination.""" - - fingerprint = [ - { - "table": table, - "primary_key": _json_value(primary_key), - "revision": _content_revision(row), - } - for table, primary_key, row in rows - ] - return "sha256:" + hashlib.sha256(_canonical_json(fingerprint).encode("utf-8")).hexdigest() - - def _connect(database: Path) -> sqlite3.Connection: # ``mode=ro`` keeps SQLite's normal WAL behaviour while preventing all # writes. In particular, do not use immutable=1: it ignores live WAL data. @@ -312,9 +298,11 @@ def read_database( """Read a bounded, transactionally consistent raw SQLite snapshot. Every poll re-reads the selected session, because turns and sessions are - mutable. Pagination is keyed to this session's logical snapshot so WAL - metadata from other writers cannot starve later rows. A changed snapshot - replays from the start so updated earlier rows are not skipped. + mutable. Pagination is keyed to the database file incarnation so live WAL + writes—including inserts and updates in this session—cannot starve later + rows. A replaced database file replays from the start so row-ID reuse + cannot silently continue a stale offset. Content revisions travel on each + record; unchanged snapshots still deduplicate after a later full read. """ validate_native_id(native_id) @@ -419,7 +407,7 @@ def read_database( rows_by_table.sort( key=lambda item: (_TABLES.index(item[0]), _canonical_json(_json_value(item[1]))) ) - generation = _snapshot_generation(rows_by_table) + generation = file_generation incoming_generation = cursor.get("database_generation") if isinstance(cursor, dict) else None incoming_offset = cursor.get("database_offset", 0) if isinstance(cursor, dict) else 0 offset = ( diff --git a/tests/test_copilot_database.py b/tests/test_copilot_database.py index 9f968da..b2b2820 100644 --- a/tests/test_copilot_database.py +++ b/tests/test_copilot_database.py @@ -925,6 +925,135 @@ def test_unrelated_wal_write_does_not_reset_pagination(tmp_path: Path): writer.close() +def test_same_session_writes_do_not_starve_later_rows(tmp_path: Path): + home = tmp_path / "copilot" + database = _write_database( + home, + session_id="session-a", + turns=[(index, f"turn-{index}") for index in range(1, 6)], + ) + paths = _paths(home) + page_one = read_database(paths, "session-a", {}, max_records=2) + assert page_one["exhausted"] is False + first_ids = [record["source_id"] for record in page_one["records"]] + + writer = sqlite3.connect(database) + try: + writer.execute("PRAGMA journal_mode=WAL") + writer.execute("PRAGMA wal_autocheckpoint=0") + writer.execute( + "UPDATE turns SET content = ?, updated_at = ? WHERE id = ?", + ("updated-turn-1", "2026-09-10T17:13:00.000Z", 1), + ) + writer.execute( + "INSERT INTO turns (id, session_id, turn_index, content, updated_at) " + "VALUES (?, ?, ?, ?, ?)", + (99, "session-a", 99, "late-same-session", "2026-09-10T17:13:01.000Z"), + ) + writer.execute( + "INSERT INTO assistant_usage_events (id, session_id, turn_index, model, created_at) " + "VALUES (?, ?, ?, ?, ?)", + (50, "session-a", 99, "gpt-live", "2026-09-10T17:13:02.000Z"), + ) + writer.commit() + wal_path = Path(f"{database}-wal") + assert wal_path.is_file() + assert wal_path.stat().st_size > 0 + + page_two = read_database( + paths, "session-a", page_one["next_cursor"], max_records=2 + ) + second_ids = [record["source_id"] for record in page_two["records"]] + assert page_two["records"] + assert second_ids != first_ids + assert page_two["next_cursor"]["database_generation"] == page_one["next_cursor"][ + "database_generation" + ] + assert page_two["next_cursor"]["database_offset"] == 4 + + collected = list(page_one["records"]) + list(page_two["records"]) + cursor = page_two["next_cursor"] + for _ in range(20): + slice_ = read_database(paths, "session-a", cursor, max_records=2) + collected.extend(slice_["records"]) + cursor = slice_["next_cursor"] + if slice_["exhausted"]: + break + else: + raise AssertionError("pagination did not exhaust after same-session writes") + + turn_contents = { + record["payload"]["row"]["content"] + for record in collected + if record["payload"]["table"] == "turns" + } + assert "turn-5" in turn_contents + assert "late-same-session" in turn_contents + assert any( + record["payload"]["table"] == "assistant_usage_events" + and record["payload"]["row"]["model"] == "gpt-live" + for record in collected + ) + finally: + writer.close() + + +def test_repeated_same_session_inserts_still_reach_later_rows(tmp_path: Path): + home = tmp_path / "copilot" + database = _write_database( + home, + session_id="session-a", + turns=[(index, f"turn-{index}") for index in range(1, 9)], + ) + paths = _paths(home) + cursor: dict[str, Any] = {} + collected: list[dict[str, Any]] = [] + first_page_ids: list[str] | None = None + writer = sqlite3.connect(database) + try: + writer.execute("PRAGMA journal_mode=WAL") + writer.execute("PRAGMA wal_autocheckpoint=0") + for index in range(20): + slice_ = read_database(paths, "session-a", cursor, max_records=2) + page_ids = [record["source_id"] for record in slice_["records"]] + if first_page_ids is None: + first_page_ids = page_ids + else: + assert page_ids != first_page_ids + collected.extend(slice_["records"]) + writer.execute( + "INSERT INTO assistant_usage_events " + "(id, session_id, turn_index, model, created_at) VALUES (?, ?, ?, ?, ?)", + ( + 100 + index, + "session-a", + index, + f"gpt-live-{index}", + "2026-09-10T17:14:00.000Z", + ), + ) + writer.commit() + if slice_["exhausted"]: + break + cursor = slice_["next_cursor"] + else: + raise AssertionError("continuous same-session inserts starved later rows") + finally: + writer.close() + + assert first_page_ids is not None + turn_contents = { + record["payload"]["row"]["content"] + for record in collected + if record["payload"]["table"] == "turns" + } + assert "turn-8" in turn_contents + assert any( + record["payload"]["table"] == "assistant_usage_events" + for record in collected + ) + + def test_same_row_id_different_content_is_new_revision(tmp_path: Path): home = tmp_path / "copilot" database = _write_database(home, session_id="session-a", turns=[(7, "first")]) From ef60d5bab66885d4ce6ef43f69cc1d81e17a45c0 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Thu, 10 Sep 2026 14:53:39 -0700 Subject: [PATCH 20/88] Compose Copilot source ingestion --- src/thirdeye/platforms/copilot/capture.py | 193 ++++++++++++++++++++++ src/thirdeye/platforms/copilot/sources.py | 102 ++++++++++++ 2 files changed, 295 insertions(+) create mode 100644 src/thirdeye/platforms/copilot/capture.py create mode 100644 src/thirdeye/platforms/copilot/sources.py diff --git a/src/thirdeye/platforms/copilot/capture.py b/src/thirdeye/platforms/copilot/capture.py new file mode 100644 index 0000000..c43af44 --- /dev/null +++ b/src/thirdeye/platforms/copilot/capture.py @@ -0,0 +1,193 @@ +"""Bounded local composition of Copilot source capture and hook spooling.""" + +from __future__ import annotations + +from copy import deepcopy +from datetime import UTC, datetime +from pathlib import Path +from typing import Any +from uuid import uuid4 + +from thirdeye.config import Config + +from .archive import commit_batch, iter_captured_records, load_cursor +from .hook_payload import parse_hook +from .identity import validate_native_id +from .sources import discover_sessions, read_batch +from .spool import ack_spool, enqueue_hook, read_spool +from .types import SourceBatch, SourcePaths, SourceRecord, SyncResult + + +def _result() -> SyncResult: + return { + "sessions": 0, + "records_written": 0, + "duplicate_records": 0, + "pending": 0, + "errors": 0, + } + + +def _add(total: SyncResult, result: SyncResult) -> None: + for key in total: + total[key] += result[key] + + +def _spool_sessions(config: Config, paths: SourcePaths) -> list[str]: + """Discover hook-only sessions without reading their content directly.""" + + root = Path(config.root) / "spool" / "copilot" / paths["source_key"] + try: + entries = list(root.iterdir()) + except OSError: + return [] + sessions: list[str] = [] + for entry in entries: + try: + if not entry.is_dir(): + continue + validate_native_id(entry.name) + except (OSError, ValueError): + continue + sessions.append(entry.name) + return sorted(sessions) + + +def _with_spool(batch: SourceBatch, spool_records: list[SourceRecord]) -> SourceBatch: + """Add an invocation-time spool snapshot without changing reader cursors.""" + + return { + **batch, + "records": [*spool_records, *batch["records"]], + "next_cursor": deepcopy(batch["next_cursor"]), + "diagnostics": list(batch["diagnostics"]), + } + + +def _diagnostic_count(batch: SourceBatch) -> int: + """Diagnostics are surfaced as source errors without dropping evidence.""" + + return len(batch["diagnostics"]) + + +def _diagnostic_pending(batch: SourceBatch) -> int: + """Count sources which must be retried without treating them as complete.""" + + retryable_suffixes = ( + "_unavailable", + "_missing", + "_busy", + "_unreadable", + "_read_failed", + ) + return sum( + 1 + for diagnostic in batch["diagnostics"] + if isinstance(diagnostic.get("code"), str) + and diagnostic["code"].endswith(retryable_suffixes) + ) + + +def _stale_after_commit( + config: Config, paths: SourcePaths, native_session_id: str, base_cursor: dict[str, Any], result: SyncResult +) -> bool: + """Identify archive's optimistic-race result without inspecting state files.""" + + return ( + result["errors"] > 0 + and result["pending"] > 0 + and load_cursor(config, paths, native_session_id) != base_cursor + ) + + +def capture_session(config: Config, paths: SourcePaths, native_session_id: str) -> SyncResult: + """Commit one bounded source read plus the current durable hook spool. + + A concurrent capture may advance the archive cursor between read and + commit. In that case this retries exactly once from the newly committed + cursor, preserving the original spool snapshot and leaving any second race + pending for a later sync/watch cycle. + """ + + validate_native_id(native_session_id) + spool_records = read_spool(config, paths, native_session_id) + base_cursor = load_cursor(config, paths, native_session_id) + batch = _with_spool(read_batch(paths, native_session_id, base_cursor), spool_records) + result = commit_batch(config, paths, batch) + diagnostics = _diagnostic_count(batch) + + if _stale_after_commit(config, paths, native_session_id, base_cursor, result): + retry_cursor = load_cursor(config, paths, native_session_id) + retry_batch = _with_spool( + read_batch(paths, native_session_id, retry_cursor), spool_records + ) + result = commit_batch(config, paths, retry_batch) + diagnostics = _diagnostic_count(retry_batch) + + # A zero archive error means the journal checkpoint made every submitted + # hook source ID durable (including an already-durable duplicate). + if result["errors"] == 0 and spool_records: + ack_spool(config, paths, [record["source_id"] for record in spool_records]) + + result = dict(result) + result["errors"] += diagnostics + result["pending"] += _diagnostic_pending(batch) + return result + + +def sync( + config: Config, paths: SourcePaths, *, session_id: str | None = None +) -> SyncResult: + """Capture an invocation-time snapshot of discovered and spooled sessions.""" + + if session_id is not None: + validate_native_id(session_id) + known = set(discover_sessions(paths)) | set(_spool_sessions(config, paths)) + if session_id not in known: + return { + "sessions": 0, + "records_written": 0, + "duplicate_records": 0, + "pending": 0, + "errors": 1, + } + session_ids = [session_id] + else: + session_ids = sorted(set(discover_sessions(paths)) | set(_spool_sessions(config, paths))) + + total = _result() + for native_session_id in session_ids: + _add(total, capture_session(config, paths, native_session_id)) + return total + + +def _observed_at() -> str: + return datetime.now(UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z") + + +def record_hook( + config: Config, + paths: SourcePaths, + event: str, + payload: dict, + context: dict, +) -> SyncResult: + """Durably receive one hook observation, then make one bounded capture.""" + + record = parse_hook( + event, + payload, + context, + observed_at=_observed_at(), + observation_id=uuid4().hex, + ) + enqueue_hook(config, paths, record) + return capture_session(config, paths, record["native_session_id"]) + + +__all__ = [ + "capture_session", + "iter_captured_records", + "record_hook", + "sync", +] diff --git a/src/thirdeye/platforms/copilot/sources.py b/src/thirdeye/platforms/copilot/sources.py new file mode 100644 index 0000000..384dba6 --- /dev/null +++ b/src/thirdeye/platforms/copilot/sources.py @@ -0,0 +1,102 @@ +"""Compose Copilot's independent persisted recording sources. + +This module remains above the individual readers and below the archive. Its +only durable contract is a :class:`SourceBatch`: reader cursors stay namespaced +so a transcript replacement cannot reset SQLite pagination (and vice versa). +""" + +from __future__ import annotations + +from copy import deepcopy +from typing import Any, Callable + +from .database import discover_database_sessions, read_database +from .identity import resolve_sources, validate_native_id +from .transcript import discover_transcripts, read_transcript +from .types import SourceBatch, SourcePaths, SourceSlice + +__all__ = ["discover_sessions", "read_batch", "resolve_sources"] + + +def discover_sessions(paths: SourcePaths) -> list[str]: + """Return the deterministic union of readable transcript and DB sessions.""" + + discovered: set[str] = set() + for discover in (discover_transcripts, discover_database_sessions): + try: + discovered.update(discover(paths)) + except (OSError, ValueError): + # Discovery is advisory. The per-session read provides the + # actionable diagnostic while another source can still progress. + continue + return sorted(discovered) + + +def _cursor_part(cursor: dict[str, Any], name: str) -> dict[str, Any]: + value = cursor.get(name) + return deepcopy(value) if isinstance(value, dict) else {} + + +def _failed_slice( + cursor: dict[str, Any], name: str, error: BaseException +) -> SourceSlice: + return { + "records": [], + "next_cursor": deepcopy(cursor), + "diagnostics": [ + { + "code": f"copilot_{name}_read_failed", + "message": "Copilot source could not be read; retry sync or watch", + "reason": type(error).__name__, + } + ], + "cwd": None, + "exhausted": False, + } + + +def _read_slice( + name: str, + reader: Callable[[SourcePaths, str, dict[str, Any]], SourceSlice], + paths: SourcePaths, + native_session_id: str, + cursor: dict[str, Any], +) -> SourceSlice: + try: + return reader(paths, native_session_id, cursor) + except (OSError, ValueError) as error: + return _failed_slice(cursor, name, error) + + +def read_batch(paths: SourcePaths, native_session_id: str, cursor: dict) -> SourceBatch: + """Read one bounded, lossless slice from each persisted source. + + Sources deliberately do not share a cursor. A source failure is retained + as a diagnostic while a healthy sibling source still contributes records. + ``base_cursor`` is an optimistic archive marker and is stripped before the + archive persists the next source cursor. + """ + + validate_native_id(native_session_id) + source_cursor: dict[str, Any] = dict(cursor) if isinstance(cursor, dict) else {} + transcript_cursor = _cursor_part(source_cursor, "transcript") + database_cursor = _cursor_part(source_cursor, "database") + transcript = _read_slice( + "transcript", read_transcript, paths, native_session_id, transcript_cursor + ) + database = _read_slice( + "database", read_database, paths, native_session_id, database_cursor + ) + next_cursor: dict[str, Any] = { + "transcript": transcript["next_cursor"], + "database": database["next_cursor"], + "base_cursor": deepcopy(source_cursor), + } + return { + "source_key": paths["source_key"], + "native_session_id": native_session_id, + "cwd": transcript["cwd"] if transcript["cwd"] is not None else database["cwd"], + "records": [*transcript["records"], *database["records"]], + "next_cursor": next_cursor, + "diagnostics": [*transcript["diagnostics"], *database["diagnostics"]], + } From 5949ea1496683ad5083c465f362f2b4172f12407 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Thu, 10 Sep 2026 14:55:34 -0700 Subject: [PATCH 21/88] Add behavioral tests for Copilot ingestion composition. Cover source discovery/batch merging, capture sync idempotency, hook spooling, stale-cursor retry, and cli-1.0.83 fixture integration. Co-authored-by: Cursor --- tests/test_copilot_capture.py | 500 ++++++++++++++++++++++++++++++++++ tests/test_copilot_sources.py | 315 +++++++++++++++++++++ 2 files changed, 815 insertions(+) create mode 100644 tests/test_copilot_capture.py create mode 100644 tests/test_copilot_sources.py diff --git a/tests/test_copilot_capture.py b/tests/test_copilot_capture.py new file mode 100644 index 0000000..bdf5429 --- /dev/null +++ b/tests/test_copilot_capture.py @@ -0,0 +1,500 @@ +"""Behavioral tests for Copilot capture composition (sync, hooks, spool, archive).""" + +from __future__ import annotations + +import json +import shutil +import sqlite3 +from pathlib import Path +from typing import Any +from unittest.mock import patch + +import pytest + +import thirdeye.platforms.copilot.capture as capture_mod +from thirdeye.config import Config +from thirdeye.platforms.copilot.archive import commit_batch, load_cursor +from thirdeye.platforms.copilot.capture import ( + capture_session, + iter_captured_records, + record_hook, + sync, +) +from thirdeye.platforms.copilot.hook_payload import parse_hook +from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id +from thirdeye.platforms.copilot.spool import enqueue_hook, read_spool +from thirdeye.platforms.copilot.types import SourceBatch, SourcePaths, SourceRecord, SyncResult + +FIXTURES = Path(__file__).parent / "fixtures" / "copilot" +CLI_FIXTURE = FIXTURES / "cli-1.0.83" +NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" +OBSERVED_AT = "2026-09-10T17:08:25.626Z" + + +def _load_json(path: Path) -> Any: + return json.loads(path.read_text(encoding="utf-8")) + + +def _record( + source_id: str, + *, + native_session_id: str = NATIVE_SESSION_ID, + source_kind: str = "transcript", +) -> SourceRecord: + return { + "source_id": source_id, + "source_kind": source_kind, + "native_session_id": native_session_id, + "ts": "2026-09-10T17:08:24.000Z", + "observed_at": "2026-09-10T17:08:25.000Z", + "payload": {"schema_version": 1, "type": "user.message"}, + "locator": {"file": "events.jsonl", "offset": 0}, + } + + +def _batch( + paths: SourcePaths, + records: list[SourceRecord], + *, + native_session_id: str = NATIVE_SESSION_ID, + next_cursor: dict[str, Any] | None = None, + base_cursor: dict[str, Any] | None = None, +) -> SourceBatch: + cursor: dict[str, Any] = dict(next_cursor or {"generation": 1}) + if base_cursor is not None: + cursor["base_cursor"] = base_cursor + return { + "source_key": paths["source_key"], + "native_session_id": native_session_id, + "cwd": "/proj", + "records": records, + "next_cursor": cursor, + "diagnostics": [], + } + + +def _write_transcript(home: Path, native_id: str, *, events_path: Path | None = None) -> None: + session_dir = home / "session-state" / native_id + session_dir.mkdir(parents=True, exist_ok=True) + source = events_path or (CLI_FIXTURE / "events.jsonl") + shutil.copy(source, session_dir / "events.jsonl") + (session_dir / "workspace.yaml").write_text("cwd: /sanitized/workspace\n", encoding="utf-8") + + +def _write_database(home: Path, *, session_id: str = NATIVE_SESSION_ID) -> None: + home.mkdir(parents=True, exist_ok=True) + database = home / "session-store.db" + connection = sqlite3.connect(database) + try: + connection.executescript( + """ + CREATE TABLE sessions (id TEXT PRIMARY KEY, cwd TEXT, created_at TEXT); + CREATE TABLE turns ( + id INTEGER PRIMARY KEY, + session_id TEXT NOT NULL, + turn_index INTEGER, + content TEXT, + updated_at TEXT + ); + CREATE TABLE assistant_usage_events ( + id INTEGER PRIMARY KEY, + session_id TEXT NOT NULL, + turn_index INTEGER, + agent_id TEXT, + parent_tool_call_id TEXT, + model TEXT, + input_tokens INTEGER, + output_tokens INTEGER, + cache_read_tokens INTEGER, + cache_write_tokens INTEGER, + reasoning_tokens INTEGER, + total_nano_aiu INTEGER, + request_multiplier REAL, + duration_ms INTEGER, + time_to_first_token_ms REAL, + output_ttft_ms REAL, + inter_token_latency_ms REAL, + initiator TEXT, + api_endpoint TEXT, + reasoning_effort TEXT, + finish_reason TEXT, + content_filter_triggered INTEGER, + token_details_json TEXT, + created_at TEXT + ); + """ + ) + connection.execute( + "INSERT INTO sessions (id, cwd, created_at) VALUES (?, ?, ?)", + (session_id, "/tmp/probe", "2026-09-10T17:08:00.000Z"), + ) + connection.execute( + "INSERT INTO turns (id, session_id, turn_index, content, updated_at) " + "VALUES (?, ?, ?, ?, ?)", + (1, session_id, 0, "turn-0", "2026-09-10T17:08:10.000Z"), + ) + connection.execute( + "INSERT INTO turns (id, session_id, turn_index, content, updated_at) " + "VALUES (?, ?, ?, ?, ?)", + (2, session_id, 1, "turn-1", "2026-09-10T17:08:11.000Z"), + ) + for row in _load_json(CLI_FIXTURE / "assistant-usage-events.json"): + columns = ", ".join(row) + placeholders = ", ".join("?" for _ in row) + connection.execute( + f"INSERT INTO assistant_usage_events ({columns}) VALUES ({placeholders})", + tuple(row.values()), + ) + connection.commit() + finally: + connection.close() + + +@pytest.fixture +def copilot_env(tmp_path: Path) -> tuple[Config, SourcePaths]: + home = tmp_path / "copilot-home" + home.mkdir() + config = Config(root=tmp_path / "thirdeye") + paths = resolve_sources(home) + return config, paths + + +def _hook_record(*, observation_id: str, session_id: str = NATIVE_SESSION_ID) -> SourceRecord: + return parse_hook( + "agentStop", + { + "sessionId": session_id, + "timestamp": 1789060105626, + "cwd": "/fixture/workspace", + "stopReason": "end_turn", + }, + {"env": {"WB_PLAN": "p"}}, + observed_at=OBSERVED_AT, + observation_id=observation_id, + ) + + +# --- module exports --- + + +def test_iter_captured_records_is_reexported_from_archive(copilot_env: tuple[Config, SourcePaths]) -> None: + config, paths = copilot_env + stored = stored_session_id(paths, NATIVE_SESSION_ID) + commit_batch(config, paths, _batch(paths, [_record("key/a/exported")])) + captured = list(iter_captured_records(config, stored)) + assert len(captured) == 1 + assert captured[0]["source_id"] == "key/a/exported" + + +def test_capture_module_has_no_usage_store_or_export_imports() -> None: + source = Path(capture_mod.__file__).read_text(encoding="utf-8") + assert "UsageStore" not in source + assert "usage_store" not in source + assert "logfire" not in source + assert "otel" not in source.lower() + + +# --- sync semantics --- + + +def test_sync_empty_discovery_is_successful_noop(copilot_env: tuple[Config, SourcePaths]) -> None: + config, paths = copilot_env + assert sync(config, paths) == { + "sessions": 0, + "records_written": 0, + "duplicate_records": 0, + "pending": 0, + "errors": 0, + } + + +def test_sync_missing_selected_session_returns_error(copilot_env: tuple[Config, SourcePaths]) -> None: + config, paths = copilot_env + assert sync(config, paths, session_id="missing-session-id") == { + "sessions": 0, + "records_written": 0, + "duplicate_records": 0, + "pending": 0, + "errors": 1, + } + + +def test_sync_discovers_hook_only_spool_session(copilot_env: tuple[Config, SourcePaths]) -> None: + config, paths = copilot_env + hook_only = "hook-only-session-00000001" + record = _hook_record(observation_id="obs-hook-only", session_id=hook_only) + enqueue_hook(config, paths, record) + + result = sync(config, paths, session_id=hook_only) + # Missing transcript/database sources surface diagnostics without blocking the hook. + assert result["records_written"] >= 1 + assert result["errors"] >= 1 + assert result["pending"] >= 1 + stored = stored_session_id(paths, hook_only) + captured = list(iter_captured_records(config, stored)) + assert any(item["source_kind"] == "hook" for item in captured) + assert read_spool(config, paths, hook_only) == [] + + +# --- capture_session integration --- + + +def test_capture_session_commits_fixture_transcript_and_database( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + home = Path(paths["home"]) + _write_transcript(home, NATIVE_SESSION_ID) + _write_database(home, session_id=NATIVE_SESSION_ID) + + result = capture_session(config, paths, NATIVE_SESSION_ID) + assert result["errors"] == 0 + assert result["records_written"] > 0 + + stored = stored_session_id(paths, NATIVE_SESSION_ID) + captured = list(iter_captured_records(config, stored)) + kinds = {record["source_kind"] for record in captured} + assert "transcript" in kinds + assert "database" in kinds + transcript_count = sum(1 for record in captured if record["source_kind"] == "transcript") + assert transcript_count == 76 + + +def test_capture_session_prepends_spool_records_before_source_records( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + hook = _hook_record(observation_id="obs-order") + enqueue_hook(config, paths, hook) + order: list[str] = [] + + original_commit = capture_mod.commit_batch + + def tracking_commit(cfg: Config, p: SourcePaths, batch: SourceBatch) -> SyncResult: + order.extend(record["source_kind"] for record in batch["records"]) + return original_commit(cfg, p, batch) + + monkeypatch.setattr(capture_mod, "commit_batch", tracking_commit) + capture_session(config, paths, NATIVE_SESSION_ID) + + assert order[0] == "hook" + + +def test_capture_session_acks_spool_after_successful_commit( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + record = _hook_record(observation_id="obs-ack") + enqueue_hook(config, paths, record) + assert read_spool(config, paths, NATIVE_SESSION_ID) + + capture_session(config, paths, NATIVE_SESSION_ID) + assert read_spool(config, paths, NATIVE_SESSION_ID) == [] + + +def test_capture_session_does_not_ack_spool_when_commit_reports_errors( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + record = _hook_record(observation_id="obs-no-ack") + enqueue_hook(config, paths, record) + + def failing_commit(_cfg: Config, _paths: SourcePaths, _batch: SourceBatch) -> SyncResult: + return { + "sessions": 1, + "records_written": 0, + "duplicate_records": 0, + "pending": 1, + "errors": 1, + } + + monkeypatch.setattr(capture_mod, "commit_batch", failing_commit) + capture_session(config, paths, NATIVE_SESSION_ID) + assert len(read_spool(config, paths, NATIVE_SESSION_ID)) == 1 + + +def test_capture_session_retries_once_after_stale_cursor_race( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + home = Path(paths["home"]) + _write_transcript(home, NATIVE_SESSION_ID) + _write_database(home, session_id=NATIVE_SESSION_ID) + + seed = capture_session(config, paths, NATIVE_SESSION_ID) + assert seed["errors"] == 0 + assert load_cursor(config, paths, NATIVE_SESSION_ID) != {} + + commit_calls: list[int] = [] + original_commit = capture_mod.commit_batch + + def tracking_commit(cfg: Config, p: SourcePaths, batch: SourceBatch) -> SyncResult: + commit_calls.append(len(batch["records"])) + return original_commit(cfg, p, batch) + + monkeypatch.setattr(capture_mod, "commit_batch", tracking_commit) + + original_load = capture_mod.load_cursor + + def staged_load(cfg: Config, p: SourcePaths, native_id: str) -> dict[str, Any]: + if not commit_calls: + return {} + return original_load(cfg, p, native_id) + + monkeypatch.setattr(capture_mod, "load_cursor", staged_load) + result = capture_session(config, paths, NATIVE_SESSION_ID) + + assert len(commit_calls) == 2 + assert result["errors"] == 0 + + +def test_sync_repeat_is_idempotent(copilot_env: tuple[Config, SourcePaths]) -> None: + config, paths = copilot_env + home = Path(paths["home"]) + _write_transcript(home, NATIVE_SESSION_ID) + _write_database(home, session_id=NATIVE_SESSION_ID) + stored = stored_session_id(paths, NATIVE_SESSION_ID) + + first = sync(config, paths, session_id=NATIVE_SESSION_ID) + count_after_first = len(list(iter_captured_records(config, stored))) + second = sync(config, paths, session_id=NATIVE_SESSION_ID) + count_after_second = len(list(iter_captured_records(config, stored))) + + assert first["records_written"] > 0 + assert first["errors"] == 0 + assert second["records_written"] == 0 + assert second["errors"] == 0 + assert count_after_second == count_after_first + + +def test_late_database_rows_are_ingested_after_initial_transcript_only_sync( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + home = Path(paths["home"]) + _write_transcript(home, NATIVE_SESSION_ID) + + first = sync(config, paths, session_id=NATIVE_SESSION_ID) + assert first["errors"] == 1 + assert first["pending"] == 1 + stored = stored_session_id(paths, NATIVE_SESSION_ID) + before = list(iter_captured_records(config, stored)) + assert before + assert not any(record["source_kind"] == "database" for record in before) + + _write_database(home, session_id=NATIVE_SESSION_ID) + second = sync(config, paths, session_id=NATIVE_SESSION_ID) + assert second["errors"] == 0 + assert second["records_written"] > 0 + + after = list(iter_captured_records(config, stored)) + assert any(record["source_kind"] == "database" for record in after) + + +def test_one_unavailable_source_does_not_block_the_other( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + home = Path(paths["home"]) + _write_transcript(home, NATIVE_SESSION_ID) + + def fail_database(_paths: SourcePaths, _native: str, _cursor: dict[str, Any]) -> Any: + raise OSError("database locked") + + with patch("thirdeye.platforms.copilot.sources.read_database", side_effect=fail_database): + result = capture_session(config, paths, NATIVE_SESSION_ID) + + assert result["records_written"] > 0 + assert result["errors"] >= 1 + stored = stored_session_id(paths, NATIVE_SESSION_ID) + assert any(record["source_kind"] == "transcript" for record in iter_captured_records(config, stored)) + + +# --- record_hook --- + + +def test_record_hook_enqueues_then_captures(copilot_env: tuple[Config, SourcePaths]) -> None: + config, paths = copilot_env + home = Path(paths["home"]) + _write_transcript(home, NATIVE_SESSION_ID) + _write_database(home, session_id=NATIVE_SESSION_ID) + + result = record_hook( + config, + paths, + "agentStop", + { + "sessionId": NATIVE_SESSION_ID, + "timestamp": 1789060105626, + "cwd": "/fixture/workspace", + "stopReason": "end_turn", + }, + {"env": {"WB_PLAN": "probe"}}, + ) + + assert result["errors"] == 0 + assert result["records_written"] >= 1 + assert read_spool(config, paths, NATIVE_SESSION_ID) == [] + stored = stored_session_id(paths, NATIVE_SESSION_ID) + kinds = {record["source_kind"] for record in iter_captured_records(config, stored)} + assert "hook" in kinds + assert "transcript" in kinds + + +def test_record_hook_uses_caller_context_not_importer_environment( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + captured_context: dict[str, Any] = {} + + def capture_parse( + event: str, + payload: dict, + context: dict, + *, + observed_at: str, + observation_id: str, + ) -> SourceRecord: + captured_context.update(context) + return parse_hook(event, payload, context, observed_at=observed_at, observation_id=observation_id) + + monkeypatch.setattr(capture_mod, "parse_hook", capture_parse) + monkeypatch.setattr(capture_mod, "capture_session", lambda *_args, **_kwargs: _empty_result()) + + record_hook( + config, + paths, + "sessionStart", + {"sessionId": NATIVE_SESSION_ID, "timestamp": 1, "cwd": "/x"}, + {"env": {"CUSTOM": "from-caller"}}, + ) + assert captured_context == {"env": {"CUSTOM": "from-caller"}} + + +def _empty_result() -> SyncResult: + return { + "sessions": 0, + "records_written": 0, + "duplicate_records": 0, + "pending": 0, + "errors": 0, + } + + +def test_sync_full_fixture_writes_expected_record_volume(copilot_env: tuple[Config, SourcePaths]) -> None: + config, paths = copilot_env + home = Path(paths["home"]) + _write_transcript(home, NATIVE_SESSION_ID) + _write_database(home, session_id=NATIVE_SESSION_ID) + + result = sync(config, paths, session_id=NATIVE_SESSION_ID) + assert result["errors"] == 0 + assert result["records_written"] > 80 + + stored = stored_session_id(paths, NATIVE_SESSION_ID) + captured = list(iter_captured_records(config, stored)) + assert len(captured) == result["records_written"] diff --git a/tests/test_copilot_sources.py b/tests/test_copilot_sources.py new file mode 100644 index 0000000..fb5fa8a --- /dev/null +++ b/tests/test_copilot_sources.py @@ -0,0 +1,315 @@ +"""Behavioral tests for Copilot source discovery and batch composition.""" + +from __future__ import annotations + +import json +import sqlite3 +from pathlib import Path +from typing import Any +from unittest.mock import patch + +import pytest + +from thirdeye.platforms.copilot.database import discover_database_sessions +from thirdeye.platforms.copilot.identity import resolve_sources +from thirdeye.platforms.copilot.sources import discover_sessions, read_batch, resolve_sources as reexported_resolve +from thirdeye.platforms.copilot.transcript import discover_transcripts +from thirdeye.platforms.copilot.types import SourcePaths, SourceRecord, SourceSlice + +FIXTURES = Path(__file__).parent / "fixtures" / "copilot" +V1_SLICE = FIXTURES / "v1-cases" / "source-slice.json" +NATIVE_ID = "session-a" + + +def _paths(home: Path) -> SourcePaths: + return resolve_sources(home) + + +def _write_transcript(home: Path, native_id: str, *, events: str = '{"id":"1"}\n', cwd: str | None = None) -> None: + session_dir = home / "session-state" / native_id + session_dir.mkdir(parents=True, exist_ok=True) + (session_dir / "events.jsonl").write_text(events, encoding="utf-8") + if cwd is not None: + (session_dir / "workspace.yaml").write_text(f"cwd: {cwd}\n", encoding="utf-8") + + +def _write_database( + home: Path, + *, + session_id: str = NATIVE_ID, + cwd: str = "/db/workspace", + extra_sessions: list[str] | None = None, +) -> None: + home.mkdir(parents=True, exist_ok=True) + database = home / "session-store.db" + connection = sqlite3.connect(database) + try: + connection.executescript( + """ + CREATE TABLE sessions (id TEXT PRIMARY KEY, cwd TEXT, created_at TEXT); + CREATE TABLE turns ( + id INTEGER PRIMARY KEY, + session_id TEXT NOT NULL, + turn_index INTEGER, + content TEXT, + updated_at TEXT + ); + CREATE TABLE assistant_usage_events ( + id INTEGER PRIMARY KEY, + session_id TEXT NOT NULL, + turn_index INTEGER, + created_at TEXT + ); + """ + ) + connection.execute( + "INSERT INTO sessions (id, cwd, created_at) VALUES (?, ?, ?)", + (session_id, cwd, "2026-09-10T17:08:00.000Z"), + ) + connection.execute( + "INSERT INTO turns (id, session_id, turn_index, content, updated_at) " + "VALUES (?, ?, ?, ?, ?)", + (1, session_id, 1, "turn", "2026-09-10T17:08:10.000Z"), + ) + for other in extra_sessions or (): + connection.execute( + "INSERT INTO sessions (id, cwd, created_at) VALUES (?, ?, ?)", + (other, f"/tmp/{other}", "2026-09-10T17:08:00.000Z"), + ) + connection.commit() + finally: + connection.close() + + +def _record(source_id: str, *, source_kind: str = "transcript") -> SourceRecord: + return { + "source_id": source_id, + "source_kind": source_kind, + "native_session_id": NATIVE_ID, + "ts": "2026-09-10T17:08:24.000Z", + "observed_at": "2026-09-10T17:08:25.000Z", + "payload": {"schema_version": 1, "type": "user.message"}, + "locator": {"file": "events.jsonl", "offset": 0}, + } + + +def _slice( + *, + records: list[SourceRecord] | None = None, + next_cursor: dict[str, Any] | None = None, + diagnostics: list[dict[str, Any]] | None = None, + cwd: str | None = "/fixture/workspace", + exhausted: bool = False, +) -> SourceSlice: + return { + "records": records or [], + "next_cursor": next_cursor or {"byte_offset": 1}, + "diagnostics": diagnostics or [], + "cwd": cwd, + "exhausted": exhausted, + } + + +# --- re-exports and discovery --- + + +def test_resolve_sources_is_reexported_from_identity(tmp_path: Path) -> None: + home = tmp_path / "copilot" + home.mkdir() + assert reexported_resolve(home) == resolve_sources(home) + + +def test_discover_sessions_unions_transcript_and_database_sorted(tmp_path: Path) -> None: + home = tmp_path / "copilot" + _write_transcript(home, "session-b") + _write_transcript(home, "session-a") + _write_database(home, session_id="session-c", extra_sessions=["session-d"]) + + assert discover_sessions(_paths(home)) == ["session-a", "session-b", "session-c", "session-d"] + + +def test_discover_sessions_returns_transcript_only_when_database_missing(tmp_path: Path) -> None: + home = tmp_path / "copilot" + _write_transcript(home, "session-z") + assert discover_sessions(_paths(home)) == ["session-z"] + + +def test_discover_sessions_continues_when_one_discoverer_raises(tmp_path: Path) -> None: + home = tmp_path / "copilot" + _write_transcript(home, "session-a") + + def boom(_paths: SourcePaths) -> list[str]: + raise OSError("database unreadable") + + with patch("thirdeye.platforms.copilot.sources.discover_database_sessions", boom): + assert discover_sessions(_paths(home)) == ["session-a"] + + +def test_discover_sessions_continues_when_transcript_discover_raises(tmp_path: Path) -> None: + home = tmp_path / "copilot" + _write_database(home, session_id="session-db") + + def boom(_paths: SourcePaths) -> list[str]: + raise ValueError("transcript layout invalid") + + with patch("thirdeye.platforms.copilot.sources.discover_transcripts", boom): + assert discover_sessions(_paths(home)) == ["session-db"] + + +# --- read_batch composition --- + + +def test_read_batch_merges_transcript_and_database_records(tmp_path: Path) -> None: + home = tmp_path / "copilot" + _write_transcript(home, NATIVE_ID, events='{"id":"evt-1"}\n') + _write_database(home, session_id=NATIVE_ID) + paths = _paths(home) + + batch = read_batch(paths, NATIVE_ID, {}) + kinds = {record["source_kind"] for record in batch["records"]} + assert "transcript" in kinds + assert "database" in kinds + assert batch["source_key"] == paths["source_key"] + assert batch["native_session_id"] == NATIVE_ID + + +def test_read_batch_namespaces_reader_cursors_independently() -> None: + transcript_cursor = {"byte_offset": 64, "file_generation": "gen-1"} + database_cursor = {"row_id": 7, "table": "turns"} + transcript = _slice(records=[_record("t/1")], next_cursor=transcript_cursor, cwd="/transcript/cwd") + database = _slice( + records=[_record("d/1", source_kind="database")], + next_cursor=database_cursor, + cwd="/database/cwd", + ) + + with ( + patch("thirdeye.platforms.copilot.sources.read_transcript", return_value=transcript), + patch("thirdeye.platforms.copilot.sources.read_database", return_value=database), + ): + batch = read_batch(_paths(Path("/tmp/unused")), NATIVE_ID, {"transcript": {"stale": True}}) + + assert batch["next_cursor"]["transcript"] == transcript_cursor + assert batch["next_cursor"]["database"] == database_cursor + assert batch["next_cursor"]["base_cursor"] == {"transcript": {"stale": True}} + + +def test_read_batch_passes_namespaced_cursors_to_each_reader(tmp_path: Path) -> None: + home = tmp_path / "copilot" + home.mkdir() + paths = _paths(home) + incoming = { + "transcript": {"byte_offset": 10}, + "database": {"row_id": 3}, + "base_cursor": {"ignored": True}, + } + seen: dict[str, dict[str, Any]] = {} + + def capture_transcript(_paths: SourcePaths, _native: str, cursor: dict[str, Any]) -> SourceSlice: + seen["transcript"] = cursor + return _slice(exhausted=True) + + def capture_database(_paths: SourcePaths, _native: str, cursor: dict[str, Any]) -> SourceSlice: + seen["database"] = cursor + return _slice(exhausted=True) + + with ( + patch("thirdeye.platforms.copilot.sources.read_transcript", side_effect=capture_transcript), + patch("thirdeye.platforms.copilot.sources.read_database", side_effect=capture_database), + ): + read_batch(paths, NATIVE_ID, incoming) + + assert seen["transcript"] == {"byte_offset": 10} + assert seen["database"] == {"row_id": 3} + + +def test_read_batch_surfaces_transcript_failure_as_diagnostic_while_database_progresses( + tmp_path: Path, +) -> None: + home = tmp_path / "copilot" + _write_database(home, session_id=NATIVE_ID) + paths = _paths(home) + + def fail_transcript(_paths: SourcePaths, _native: str, _cursor: dict[str, Any]) -> SourceSlice: + raise OSError("events.jsonl locked") + + with patch("thirdeye.platforms.copilot.sources.read_transcript", side_effect=fail_transcript): + batch = read_batch(paths, NATIVE_ID, {}) + + assert any(item["code"] == "copilot_transcript_read_failed" for item in batch["diagnostics"]) + assert any(record["source_kind"] == "database" for record in batch["records"]) + + +def test_read_batch_surfaces_database_failure_as_diagnostic_while_transcript_progresses( + tmp_path: Path, +) -> None: + home = tmp_path / "copilot" + _write_transcript(home, NATIVE_ID) + paths = _paths(home) + + def fail_database(_paths: SourcePaths, _native: str, _cursor: dict[str, Any]) -> SourceSlice: + raise ValueError("database schema mismatch") + + with patch("thirdeye.platforms.copilot.sources.read_database", side_effect=fail_database): + batch = read_batch(paths, NATIVE_ID, {}) + + assert any(item["code"] == "copilot_database_read_failed" for item in batch["diagnostics"]) + assert any(record["source_kind"] == "transcript" for record in batch["records"]) + + +def test_read_batch_prefers_transcript_cwd_over_database() -> None: + transcript = _slice(cwd="/from/transcript") + database = _slice(cwd="/from/database", records=[_record("d/1", source_kind="database")]) + + with ( + patch("thirdeye.platforms.copilot.sources.read_transcript", return_value=transcript), + patch("thirdeye.platforms.copilot.sources.read_database", return_value=database), + ): + batch = read_batch(_paths(Path("/tmp/unused")), NATIVE_ID, {}) + + assert batch["cwd"] == "/from/transcript" + + +def test_read_batch_falls_back_to_database_cwd_when_transcript_missing() -> None: + transcript = _slice(cwd=None) + database = _slice(cwd="/from/database", records=[_record("d/1", source_kind="database")]) + + with ( + patch("thirdeye.platforms.copilot.sources.read_transcript", return_value=transcript), + patch("thirdeye.platforms.copilot.sources.read_database", return_value=database), + ): + batch = read_batch(_paths(Path("/tmp/unused")), NATIVE_ID, {}) + + assert batch["cwd"] == "/from/database" + + +def test_read_batch_fixture_slice_shape_is_compatible() -> None: + """The v1 source-slice fixture must compose into a SourceBatch boundary.""" + fixture = json.loads(V1_SLICE.read_text(encoding="utf-8")) + transcript = _slice( + records=fixture["records"], + next_cursor=fixture["next_cursor"], + diagnostics=fixture["diagnostics"], + cwd=fixture["cwd"], + exhausted=fixture["exhausted"], + ) + database = _slice(exhausted=True) + + with ( + patch("thirdeye.platforms.copilot.sources.read_transcript", return_value=transcript), + patch("thirdeye.platforms.copilot.sources.read_database", return_value=database), + ): + batch = read_batch(_paths(Path("/tmp/unused")), NATIVE_ID, {}) + + assert len(batch["records"]) == 1 + assert batch["records"][0]["source_kind"] == "transcript" + assert batch["diagnostics"] == [] + + +def test_discover_sessions_delegates_to_underlying_readers(tmp_path: Path) -> None: + home = tmp_path / "copilot" + _write_transcript(home, "only-transcript") + paths = _paths(home) + assert discover_transcripts(paths) == ["only-transcript"] + assert discover_database_sessions(paths) == [] + assert discover_sessions(paths) == ["only-transcript"] From c375e3477d1b4051d1c8d31757f7ea6d6d435f8d Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Thu, 10 Sep 2026 15:17:49 -0700 Subject: [PATCH 22/88] Fix Copilot composition drain, stale-retry accounting, and discovery errors. Preserve source exhaustion so sync pages an invocation-time snapshot, keep journal recovery counts after a stale cursor retry, and surface unreadable sources instead of a silent no-op. Co-authored-by: Cursor --- src/thirdeye/platforms/copilot/capture.py | 213 +++++++++++--- src/thirdeye/platforms/copilot/sources.py | 84 ++++-- tests/test_copilot_capture.py | 343 +++++++++++++++++++++- tests/test_copilot_sources.py | 52 +++- 4 files changed, 612 insertions(+), 80 deletions(-) diff --git a/src/thirdeye/platforms/copilot/capture.py b/src/thirdeye/platforms/copilot/capture.py index c43af44..08bfa04 100644 --- a/src/thirdeye/platforms/copilot/capture.py +++ b/src/thirdeye/platforms/copilot/capture.py @@ -13,10 +13,20 @@ from .archive import commit_batch, iter_captured_records, load_cursor from .hook_payload import parse_hook from .identity import validate_native_id -from .sources import discover_sessions, read_batch +from .sources import _discover_with_diagnostics, read_batch from .spool import ack_spool, enqueue_hook, read_spool from .types import SourceBatch, SourcePaths, SourceRecord, SyncResult +_RETRYABLE_SUFFIXES = ( + "_unavailable", + "_missing", + "_busy", + "_unreadable", + "_read_failed", + "_incompatible", +) +_ABSENCE_CODES = frozenset({"transcript_unavailable", "copilot_database_missing"}) + def _result() -> SyncResult: return { @@ -64,6 +74,10 @@ def _with_spool(batch: SourceBatch, spool_records: list[SourceRecord]) -> Source } +def _is_retryable_code(code: object) -> bool: + return isinstance(code, str) and code.endswith(_RETRYABLE_SUFFIXES) + + def _diagnostic_count(batch: SourceBatch) -> int: """Diagnostics are surfaced as source errors without dropping evidence.""" @@ -73,23 +87,71 @@ def _diagnostic_count(batch: SourceBatch) -> int: def _diagnostic_pending(batch: SourceBatch) -> int: """Count sources which must be retried without treating them as complete.""" - retryable_suffixes = ( - "_unavailable", - "_missing", - "_busy", - "_unreadable", - "_read_failed", - ) return sum( - 1 + 1 for diagnostic in batch["diagnostics"] if _is_retryable_code(diagnostic.get("code")) + ) + + +def _source_retryable(batch: SourceBatch, name: str) -> bool: + return any( + isinstance(diagnostic.get("code"), str) + and name in diagnostic["code"] + and _is_retryable_code(diagnostic["code"]) for diagnostic in batch["diagnostics"] - if isinstance(diagnostic.get("code"), str) - and diagnostic["code"].endswith(retryable_suffixes) ) +def _pagination_pending(batch: SourceBatch) -> int: + """Count sources that still have unread snapshot pages.""" + + cursor = batch["next_cursor"] + pending = 0 + for name in ("transcript", "database"): + if cursor.get(f"{name}_exhausted", True): + continue + if _source_retryable(batch, name): + continue + pending += 1 + return pending + + +def _compose(archive: SyncResult, batch: SourceBatch) -> SyncResult: + result = dict(archive) + result["errors"] += _diagnostic_count(batch) + result["pending"] += _diagnostic_pending(batch) + _pagination_pending(batch) + return result # type: ignore[return-value] + + +def _merge_archive(first: SyncResult, second: SyncResult) -> SyncResult: + """Keep recovery writes from a stale attempt; take the retry's error/pending.""" + + return { + "sessions": max(first["sessions"], second["sessions"]), + "records_written": first["records_written"] + second["records_written"], + "duplicate_records": first["duplicate_records"] + second["duplicate_records"], + "pending": second["pending"], + "errors": second["errors"], + } + + +def _merge_diagnostics(items: list[dict[str, Any]]) -> list[dict[str, Any]]: + retryable: dict[str, dict[str, Any]] = {} + others: list[dict[str, Any]] = [] + for item in items: + code = item.get("code") + if _is_retryable_code(code) and isinstance(code, str): + retryable[code] = item + else: + others.append(item) + return others + list(retryable.values()) + + def _stale_after_commit( - config: Config, paths: SourcePaths, native_session_id: str, base_cursor: dict[str, Any], result: SyncResult + config: Config, + paths: SourcePaths, + native_session_id: str, + base_cursor: dict[str, Any], + result: SyncResult, ) -> bool: """Identify archive's optimistic-race result without inspecting state files.""" @@ -100,7 +162,36 @@ def _stale_after_commit( ) -def capture_session(config: Config, paths: SourcePaths, native_session_id: str) -> SyncResult: +def _is_absence_diagnostic(diagnostic: dict[str, Any]) -> bool: + code = diagnostic.get("code") + if not isinstance(code, str): + return False + if code in _ABSENCE_CODES: + return True + return code.startswith("workspace_") + + +def _selected_source_missing(batch: SourceBatch) -> bool: + if batch["records"]: + return False + return all(_is_absence_diagnostic(diagnostic) for diagnostic in batch["diagnostics"]) + + +def _result_from_diagnostics(diagnostics: list[dict[str, Any]]) -> SyncResult: + return { + "sessions": 0, + "records_written": 0, + "duplicate_records": 0, + "pending": sum( + 1 for diagnostic in diagnostics if _is_retryable_code(diagnostic.get("code")) + ), + "errors": len(diagnostics), + } + + +def _capture_once( + config: Config, paths: SourcePaths, native_session_id: str +) -> tuple[SyncResult, SourceBatch, bool]: """Commit one bounded source read plus the current durable hook spool. A concurrent capture may advance the archive cursor between read and @@ -109,55 +200,97 @@ def capture_session(config: Config, paths: SourcePaths, native_session_id: str) pending for a later sync/watch cycle. """ - validate_native_id(native_session_id) spool_records = read_spool(config, paths, native_session_id) base_cursor = load_cursor(config, paths, native_session_id) batch = _with_spool(read_batch(paths, native_session_id, base_cursor), spool_records) - result = commit_batch(config, paths, batch) - diagnostics = _diagnostic_count(batch) + archive = commit_batch(config, paths, batch) + unresolved_stale = False - if _stale_after_commit(config, paths, native_session_id, base_cursor, result): + if _stale_after_commit(config, paths, native_session_id, base_cursor, archive): retry_cursor = load_cursor(config, paths, native_session_id) - retry_batch = _with_spool( - read_batch(paths, native_session_id, retry_cursor), spool_records + retry_batch = _with_spool(read_batch(paths, native_session_id, retry_cursor), spool_records) + retry_archive = commit_batch(config, paths, retry_batch) + archive = _merge_archive(archive, retry_archive) + batch = retry_batch + unresolved_stale = _stale_after_commit( + config, paths, native_session_id, retry_cursor, retry_archive ) - result = commit_batch(config, paths, retry_batch) - diagnostics = _diagnostic_count(retry_batch) # A zero archive error means the journal checkpoint made every submitted # hook source ID durable (including an already-durable duplicate). - if result["errors"] == 0 and spool_records: + if archive["errors"] == 0 and spool_records: ack_spool(config, paths, [record["source_id"] for record in spool_records]) - result = dict(result) - result["errors"] += diagnostics - result["pending"] += _diagnostic_pending(batch) - return result + more_pages = (not unresolved_stale) and _pagination_pending(batch) > 0 + return archive, batch, more_pages -def sync( - config: Config, paths: SourcePaths, *, session_id: str | None = None -) -> SyncResult: +def capture_session(config: Config, paths: SourcePaths, native_session_id: str) -> SyncResult: + """Commit one bounded source read plus the current durable hook spool.""" + + validate_native_id(native_session_id) + archive, batch, _ = _capture_once(config, paths, native_session_id) + return _compose(archive, batch) + + +def _drain_session(config: Config, paths: SourcePaths, native_session_id: str) -> SyncResult: + """Drain one invocation-time snapshot without chasing a growing source.""" + + total = _result() + collected: list[dict[str, Any]] = [] + last_batch: SourceBatch | None = None + prev_cursor: object = object() + while True: + archive, batch, more_pages = _capture_once(config, paths, native_session_id) + last_batch = batch + collected.extend(batch["diagnostics"]) + total["records_written"] += archive["records_written"] + total["duplicate_records"] += archive["duplicate_records"] + total["sessions"] = max(total["sessions"], archive["sessions"]) + total["errors"] += archive["errors"] + total["pending"] = archive["pending"] + current_cursor = load_cursor(config, paths, native_session_id) + if not more_pages or current_cursor == prev_cursor: + break + prev_cursor = current_cursor + assert last_batch is not None + merged: SourceBatch = { + **last_batch, + "diagnostics": _merge_diagnostics(collected), + } + return _compose(total, merged) + + +def sync(config: Config, paths: SourcePaths, *, session_id: str | None = None) -> SyncResult: """Capture an invocation-time snapshot of discovered and spooled sessions.""" + discovered, discovery_diagnostics = _discover_with_diagnostics(paths) + known = set(discovered) | set(_spool_sessions(config, paths)) + if session_id is not None: validate_native_id(session_id) - known = set(discover_sessions(paths)) | set(_spool_sessions(config, paths)) if session_id not in known: - return { - "sessions": 0, - "records_written": 0, - "duplicate_records": 0, - "pending": 0, - "errors": 1, - } + probe = read_batch(paths, session_id, {}) + if _selected_source_missing(probe): + missing: SyncResult = { + "sessions": 0, + "records_written": 0, + "duplicate_records": 0, + "pending": 0, + "errors": 1, + } + if discovery_diagnostics: + _add(missing, _result_from_diagnostics(discovery_diagnostics)) + return missing session_ids = [session_id] else: - session_ids = sorted(set(discover_sessions(paths)) | set(_spool_sessions(config, paths))) + session_ids = sorted(known) - total = _result() + total = _result_from_diagnostics(discovery_diagnostics) + if not session_ids: + return total for native_session_id in session_ids: - _add(total, capture_session(config, paths, native_session_id)) + _add(total, _drain_session(config, paths, native_session_id)) return total diff --git a/src/thirdeye/platforms/copilot/sources.py b/src/thirdeye/platforms/copilot/sources.py index 384dba6..3e490ad 100644 --- a/src/thirdeye/platforms/copilot/sources.py +++ b/src/thirdeye/platforms/copilot/sources.py @@ -7,8 +7,10 @@ from __future__ import annotations +from collections.abc import Callable from copy import deepcopy -from typing import Any, Callable +from pathlib import Path +from typing import Any from .database import discover_database_sessions, read_database from .identity import resolve_sources, validate_native_id @@ -17,19 +19,63 @@ __all__ = ["discover_sessions", "read_batch", "resolve_sources"] +_DISCOVERY_PROBE_ID = "copilot-discovery-probe" +_FILE_LEVEL_DATABASE_CODES = frozenset( + { + "copilot_database_unreadable", + "copilot_database_busy", + "copilot_database_incompatible", + "copilot_database_read_failed", + } +) + + +def _discovery_diagnostic(name: str, error: BaseException) -> dict[str, Any]: + return { + "code": f"copilot_{name}_unreadable", + "message": "Copilot source could not be discovered; retry sync or watch", + "reason": type(error).__name__, + } + + +def _discover_with_diagnostics( + paths: SourcePaths, +) -> tuple[list[str], list[dict[str, Any]]]: + """Return session IDs plus discovery diagnostics that sync must surface.""" + + discovered: set[str] = set() + diagnostics: list[dict[str, Any]] = [] + + try: + discovered.update(discover_transcripts(paths)) + except (OSError, ValueError) as error: + diagnostics.append(_discovery_diagnostic("transcript", error)) + + database_ids: list[str] | None + try: + database_ids = list(discover_database_sessions(paths)) + discovered.update(database_ids) + except (OSError, ValueError) as error: + diagnostics.append(_discovery_diagnostic("database", error)) + database_ids = None + + # discover_database_sessions treats an unreadable file as "no sessions". + # Probe the file-level diagnostic so all-session sync is not a silent no-op. + if database_ids == [] and Path(paths["database"]).is_file(): + probe = _read_slice("database", read_database, paths, _DISCOVERY_PROBE_ID, {}) + for diagnostic in probe["diagnostics"]: + code = diagnostic.get("code") + if isinstance(code, str) and code in _FILE_LEVEL_DATABASE_CODES: + diagnostics.append(diagnostic) + + return sorted(discovered), diagnostics + def discover_sessions(paths: SourcePaths) -> list[str]: """Return the deterministic union of readable transcript and DB sessions.""" - discovered: set[str] = set() - for discover in (discover_transcripts, discover_database_sessions): - try: - discovered.update(discover(paths)) - except (OSError, ValueError): - # Discovery is advisory. The per-session read provides the - # actionable diagnostic while another source can still progress. - continue - return sorted(discovered) + sessions, _ = _discover_with_diagnostics(paths) + return sessions def _cursor_part(cursor: dict[str, Any], name: str) -> dict[str, Any]: @@ -37,9 +83,7 @@ def _cursor_part(cursor: dict[str, Any], name: str) -> dict[str, Any]: return deepcopy(value) if isinstance(value, dict) else {} -def _failed_slice( - cursor: dict[str, Any], name: str, error: BaseException -) -> SourceSlice: +def _failed_slice(cursor: dict[str, Any], name: str, error: BaseException) -> SourceSlice: return { "records": [], "next_cursor": deepcopy(cursor), @@ -74,7 +118,9 @@ def read_batch(paths: SourcePaths, native_session_id: str, cursor: dict) -> Sour Sources deliberately do not share a cursor. A source failure is retained as a diagnostic while a healthy sibling source still contributes records. ``base_cursor`` is an optimistic archive marker and is stripped before the - archive persists the next source cursor. + archive persists the next source cursor. Exhaustion flags stay on the + composition cursor so capture can page or report pending without confusing + the per-source readers. """ validate_native_id(native_session_id) @@ -84,13 +130,13 @@ def read_batch(paths: SourcePaths, native_session_id: str, cursor: dict) -> Sour transcript = _read_slice( "transcript", read_transcript, paths, native_session_id, transcript_cursor ) - database = _read_slice( - "database", read_database, paths, native_session_id, database_cursor - ) + database = _read_slice("database", read_database, paths, native_session_id, database_cursor) next_cursor: dict[str, Any] = { - "transcript": transcript["next_cursor"], - "database": database["next_cursor"], + "transcript": deepcopy(transcript["next_cursor"]), + "database": deepcopy(database["next_cursor"]), "base_cursor": deepcopy(source_cursor), + "transcript_exhausted": bool(transcript["exhausted"]), + "database_exhausted": bool(database["exhausted"]), } return { "source_key": paths["source_key"], diff --git a/tests/test_copilot_capture.py b/tests/test_copilot_capture.py index bdf5429..b410e6e 100644 --- a/tests/test_copilot_capture.py +++ b/tests/test_copilot_capture.py @@ -5,13 +5,17 @@ import json import shutil import sqlite3 +from collections.abc import Iterator +from contextlib import contextmanager from pathlib import Path from typing import Any from unittest.mock import patch import pytest +import thirdeye.platforms.copilot.archive as archive_mod import thirdeye.platforms.copilot.capture as capture_mod +import thirdeye.platforms.copilot.state as state_mod from thirdeye.config import Config from thirdeye.platforms.copilot.archive import commit_batch, load_cursor from thirdeye.platforms.copilot.capture import ( @@ -174,10 +178,61 @@ def _hook_record(*, observation_id: str, session_id: str = NATIVE_SESSION_ID) -> ) +@contextmanager +def _fault_at(point: str) -> Iterator[None]: + def injector(name: str) -> None: + if name == point: + raise RuntimeError(f"injected fault at {point}") + + archive_mod._fault_injector = injector + state_mod._fault_injector = injector + try: + yield + finally: + archive_mod._fault_injector = None + state_mod._fault_injector = None + + +def _empty_result() -> SyncResult: + return { + "sessions": 0, + "records_written": 0, + "duplicate_records": 0, + "pending": 0, + "errors": 0, + } + + +def _composed_batch( + paths: SourcePaths, + *, + diagnostics: list[dict[str, Any]] | None = None, + records: list[SourceRecord] | None = None, + transcript_exhausted: bool = True, + database_exhausted: bool = True, +) -> SourceBatch: + return { + "source_key": paths["source_key"], + "native_session_id": NATIVE_SESSION_ID, + "cwd": "/proj", + "records": records or [_record("composed/1")], + "next_cursor": { + "transcript": {"byte_offset": 1}, + "database": {"database_offset": 1}, + "base_cursor": {}, + "transcript_exhausted": transcript_exhausted, + "database_exhausted": database_exhausted, + }, + "diagnostics": diagnostics or [], + } + + # --- module exports --- -def test_iter_captured_records_is_reexported_from_archive(copilot_env: tuple[Config, SourcePaths]) -> None: +def test_iter_captured_records_is_reexported_from_archive( + copilot_env: tuple[Config, SourcePaths], +) -> None: config, paths = copilot_env stored = stored_session_id(paths, NATIVE_SESSION_ID) commit_batch(config, paths, _batch(paths, [_record("key/a/exported")])) @@ -208,7 +263,9 @@ def test_sync_empty_discovery_is_successful_noop(copilot_env: tuple[Config, Sour } -def test_sync_missing_selected_session_returns_error(copilot_env: tuple[Config, SourcePaths]) -> None: +def test_sync_missing_selected_session_returns_error( + copilot_env: tuple[Config, SourcePaths], +) -> None: config, paths = copilot_env assert sync(config, paths, session_id="missing-session-id") == { "sessions": 0, @@ -410,7 +467,9 @@ def fail_database(_paths: SourcePaths, _native: str, _cursor: dict[str, Any]) -> assert result["records_written"] > 0 assert result["errors"] >= 1 stored = stored_session_id(paths, NATIVE_SESSION_ID) - assert any(record["source_kind"] == "transcript" for record in iter_captured_records(config, stored)) + assert any( + record["source_kind"] == "transcript" for record in iter_captured_records(config, stored) + ) # --- record_hook --- @@ -460,7 +519,9 @@ def capture_parse( observation_id: str, ) -> SourceRecord: captured_context.update(context) - return parse_hook(event, payload, context, observed_at=observed_at, observation_id=observation_id) + return parse_hook( + event, payload, context, observed_at=observed_at, observation_id=observation_id + ) monkeypatch.setattr(capture_mod, "parse_hook", capture_parse) monkeypatch.setattr(capture_mod, "capture_session", lambda *_args, **_kwargs: _empty_result()) @@ -475,17 +536,9 @@ def capture_parse( assert captured_context == {"env": {"CUSTOM": "from-caller"}} -def _empty_result() -> SyncResult: - return { - "sessions": 0, - "records_written": 0, - "duplicate_records": 0, - "pending": 0, - "errors": 0, - } - - -def test_sync_full_fixture_writes_expected_record_volume(copilot_env: tuple[Config, SourcePaths]) -> None: +def test_sync_full_fixture_writes_expected_record_volume( + copilot_env: tuple[Config, SourcePaths], +) -> None: config, paths = copilot_env home = Path(paths["home"]) _write_transcript(home, NATIVE_SESSION_ID) @@ -498,3 +551,263 @@ def test_sync_full_fixture_writes_expected_record_volume(copilot_env: tuple[Conf stored = stored_session_id(paths, NATIVE_SESSION_ID) captured = list(iter_captured_records(config, stored)) assert len(captured) == result["records_written"] + + +def test_capture_session_reports_pending_when_transcript_exceeds_reader_limit( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + home = Path(paths["home"]) + session_dir = home / "session-state" / NATIVE_SESSION_ID + session_dir.mkdir(parents=True, exist_ok=True) + events = "".join(f'{{"id":"evt-{index}"}}\n' for index in range(1001)) + (session_dir / "events.jsonl").write_text(events, encoding="utf-8") + (session_dir / "workspace.yaml").write_text("cwd: /sanitized/workspace\n", encoding="utf-8") + _write_database(home, session_id=NATIVE_SESSION_ID) + + result = capture_session(config, paths, NATIVE_SESSION_ID) + stored = stored_session_id(paths, NATIVE_SESSION_ID) + captured = list(iter_captured_records(config, stored)) + assert result["pending"] >= 1 + assert len(captured) < 1001 + 20 + assert result["records_written"] == len(captured) + + +def test_sync_drains_paginated_invocation_snapshot( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + home = Path(paths["home"]) + session_dir = home / "session-state" / NATIVE_SESSION_ID + session_dir.mkdir(parents=True, exist_ok=True) + events = "".join(f'{{"id":"evt-{index}"}}\n' for index in range(1001)) + (session_dir / "events.jsonl").write_text(events, encoding="utf-8") + (session_dir / "workspace.yaml").write_text("cwd: /sanitized/workspace\n", encoding="utf-8") + _write_database(home, session_id=NATIVE_SESSION_ID) + + result = sync(config, paths, session_id=NATIVE_SESSION_ID) + stored = stored_session_id(paths, NATIVE_SESSION_ID) + captured = list(iter_captured_records(config, stored)) + transcript_events = [record for record in captured if record["source_kind"] == "transcript"] + assert result["pending"] == 0 + assert result["errors"] == 0 + assert len(transcript_events) == 1001 + assert result["sessions"] == 1 + + +def test_capture_session_pending_follows_retry_batch_when_source_becomes_available( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + unavailable = _composed_batch( + paths, + diagnostics=[ + { + "code": "copilot_database_missing", + "message": "Copilot session database is not present", + } + ], + ) + available = _composed_batch(paths, diagnostics=[], records=[_record("composed/retry")]) + reads = [unavailable, available] + commits = 0 + + def fake_read(_paths: SourcePaths, _native: str, _cursor: dict[str, Any]) -> SourceBatch: + return reads.pop(0) + + def fake_commit(_cfg: Config, _paths: SourcePaths, _batch: SourceBatch) -> SyncResult: + nonlocal commits + commits += 1 + if commits == 1: + return { + "sessions": 1, + "records_written": 2, + "duplicate_records": 0, + "pending": 1, + "errors": 1, + } + return { + "sessions": 1, + "records_written": 1, + "duplicate_records": 0, + "pending": 0, + "errors": 0, + } + + def fake_load(_cfg: Config, _paths: SourcePaths, _native: str) -> dict[str, Any]: + if commits == 0: + return {} + return {"transcript": {"byte_offset": 8}} + + monkeypatch.setattr(capture_mod, "read_batch", fake_read) + monkeypatch.setattr(capture_mod, "commit_batch", fake_commit) + monkeypatch.setattr(capture_mod, "load_cursor", fake_load) + + result = capture_session(config, paths, NATIVE_SESSION_ID) + assert commits == 2 + assert result["pending"] == 0 + assert result["errors"] == 0 + assert result["records_written"] == 3 + + +def test_capture_session_pending_follows_retry_batch_when_source_becomes_unavailable( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + available = _composed_batch(paths, diagnostics=[]) + unavailable = _composed_batch( + paths, + diagnostics=[ + { + "code": "copilot_database_missing", + "message": "Copilot session database is not present", + } + ], + ) + reads = [available, unavailable] + commits = 0 + + def fake_read(_paths: SourcePaths, _native: str, _cursor: dict[str, Any]) -> SourceBatch: + return reads.pop(0) + + def fake_commit(_cfg: Config, _paths: SourcePaths, _batch: SourceBatch) -> SyncResult: + nonlocal commits + commits += 1 + if commits == 1: + return { + "sessions": 1, + "records_written": 0, + "duplicate_records": 0, + "pending": 1, + "errors": 1, + } + return { + "sessions": 1, + "records_written": 1, + "duplicate_records": 0, + "pending": 0, + "errors": 0, + } + + def fake_load(_cfg: Config, _paths: SourcePaths, _native: str) -> dict[str, Any]: + if commits == 0: + return {} + return {"transcript": {"byte_offset": 8}} + + monkeypatch.setattr(capture_mod, "read_batch", fake_read) + monkeypatch.setattr(capture_mod, "commit_batch", fake_commit) + monkeypatch.setattr(capture_mod, "load_cursor", fake_load) + + result = capture_session(config, paths, NATIVE_SESSION_ID) + assert commits == 2 + assert result["pending"] >= 1 + assert result["errors"] >= 1 + assert result["records_written"] == 1 + + +def test_capture_session_keeps_journal_recovery_counts_after_stale_retry( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + home = Path(paths["home"]) + _write_transcript(home, NATIVE_SESSION_ID) + _write_database(home, session_id=NATIVE_SESSION_ID) + + seed = commit_batch( + config, paths, _batch(paths, [_record("key/a/seed")], next_cursor={"generation": 1}) + ) + assert seed["errors"] == 0 + + with _fault_at("after_journal"): + with pytest.raises(RuntimeError, match="injected fault"): + commit_batch( + config, + paths, + _batch( + paths, + [_record("key/a/journal-only")], + next_cursor={"generation": 2}, + base_cursor={"generation": 1}, + ), + ) + + commit_calls: list[SyncResult] = [] + original_commit = capture_mod.commit_batch + + def tracking_commit(cfg: Config, p: SourcePaths, batch: SourceBatch) -> SyncResult: + result = original_commit(cfg, p, batch) + commit_calls.append(result) + return result + + monkeypatch.setattr(capture_mod, "commit_batch", tracking_commit) + + original_load = capture_mod.load_cursor + + def staged_load(cfg: Config, p: SourcePaths, native_id: str) -> dict[str, Any]: + if not commit_calls: + return {"generation": 1} + return original_load(cfg, p, native_id) + + monkeypatch.setattr(capture_mod, "load_cursor", staged_load) + result = capture_session(config, paths, NATIVE_SESSION_ID) + + assert len(commit_calls) == 2 + assert commit_calls[0]["records_written"] >= 1 + assert result["records_written"] == ( + commit_calls[0]["records_written"] + commit_calls[1]["records_written"] + ) + assert result["duplicate_records"] == ( + commit_calls[0]["duplicate_records"] + commit_calls[1]["duplicate_records"] + ) + captured = list(iter_captured_records(config, stored_session_id(paths, NATIVE_SESSION_ID))) + assert any(record["source_id"] == "key/a/journal-only" for record in captured) + + +def test_sync_reports_discovery_failure_instead_of_empty_noop( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + + def boom(_paths: SourcePaths) -> list[str]: + raise OSError("database unreadable") + + with patch("thirdeye.platforms.copilot.sources.discover_database_sessions", boom): + result = sync(config, paths) + + assert result["sessions"] == 0 + assert result["records_written"] == 0 + assert result["errors"] >= 1 + assert result["pending"] >= 1 + + +def test_sync_selected_unreadable_database_is_not_reported_as_missing( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + Path(paths["database"]).write_text("this is not a sqlite database", encoding="utf-8") + + result = sync(config, paths, session_id=NATIVE_SESSION_ID) + assert result != { + "sessions": 0, + "records_written": 0, + "duplicate_records": 0, + "pending": 0, + "errors": 1, + } + assert result["errors"] >= 1 + assert result["pending"] >= 1 + + +def test_sync_incompatible_database_only_is_not_silent_noop( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + Path(paths["database"]).write_text("this is not a sqlite database", encoding="utf-8") + + result = sync(config, paths) + assert result["errors"] >= 1 + assert result["pending"] >= 1 + assert result["sessions"] == 0 diff --git a/tests/test_copilot_sources.py b/tests/test_copilot_sources.py index fb5fa8a..b7bc523 100644 --- a/tests/test_copilot_sources.py +++ b/tests/test_copilot_sources.py @@ -8,11 +8,10 @@ from typing import Any from unittest.mock import patch -import pytest - from thirdeye.platforms.copilot.database import discover_database_sessions from thirdeye.platforms.copilot.identity import resolve_sources -from thirdeye.platforms.copilot.sources import discover_sessions, read_batch, resolve_sources as reexported_resolve +from thirdeye.platforms.copilot.sources import discover_sessions, read_batch +from thirdeye.platforms.copilot.sources import resolve_sources as reexported_resolve from thirdeye.platforms.copilot.transcript import discover_transcripts from thirdeye.platforms.copilot.types import SourcePaths, SourceRecord, SourceSlice @@ -25,7 +24,9 @@ def _paths(home: Path) -> SourcePaths: return resolve_sources(home) -def _write_transcript(home: Path, native_id: str, *, events: str = '{"id":"1"}\n', cwd: str | None = None) -> None: +def _write_transcript( + home: Path, native_id: str, *, events: str = '{"id":"1"}\n', cwd: str | None = None +) -> None: session_dir = home / "session-state" / native_id session_dir.mkdir(parents=True, exist_ok=True) (session_dir / "events.jsonl").write_text(events, encoding="utf-8") @@ -176,7 +177,9 @@ def test_read_batch_merges_transcript_and_database_records(tmp_path: Path) -> No def test_read_batch_namespaces_reader_cursors_independently() -> None: transcript_cursor = {"byte_offset": 64, "file_generation": "gen-1"} database_cursor = {"row_id": 7, "table": "turns"} - transcript = _slice(records=[_record("t/1")], next_cursor=transcript_cursor, cwd="/transcript/cwd") + transcript = _slice( + records=[_record("t/1")], next_cursor=transcript_cursor, cwd="/transcript/cwd" + ) database = _slice( records=[_record("d/1", source_kind="database")], next_cursor=database_cursor, @@ -192,6 +195,8 @@ def test_read_batch_namespaces_reader_cursors_independently() -> None: assert batch["next_cursor"]["transcript"] == transcript_cursor assert batch["next_cursor"]["database"] == database_cursor assert batch["next_cursor"]["base_cursor"] == {"transcript": {"stale": True}} + assert batch["next_cursor"]["transcript_exhausted"] is False + assert batch["next_cursor"]["database_exhausted"] is False def test_read_batch_passes_namespaced_cursors_to_each_reader(tmp_path: Path) -> None: @@ -205,7 +210,9 @@ def test_read_batch_passes_namespaced_cursors_to_each_reader(tmp_path: Path) -> } seen: dict[str, dict[str, Any]] = {} - def capture_transcript(_paths: SourcePaths, _native: str, cursor: dict[str, Any]) -> SourceSlice: + def capture_transcript( + _paths: SourcePaths, _native: str, cursor: dict[str, Any] + ) -> SourceSlice: seen["transcript"] = cursor return _slice(exhausted=True) @@ -306,6 +313,39 @@ def test_read_batch_fixture_slice_shape_is_compatible() -> None: assert batch["diagnostics"] == [] +def test_read_batch_preserves_independent_exhaustion_flags() -> None: + transcript = _slice(records=[_record("t/1")], exhausted=False) + database = _slice( + records=[_record("d/1", source_kind="database")], + exhausted=True, + cwd="/database/cwd", + ) + + with ( + patch("thirdeye.platforms.copilot.sources.read_transcript", return_value=transcript), + patch("thirdeye.platforms.copilot.sources.read_database", return_value=database), + ): + batch = read_batch(_paths(Path("/tmp/unused")), NATIVE_ID, {}) + + assert batch["next_cursor"]["transcript_exhausted"] is False + assert batch["next_cursor"]["database_exhausted"] is True + + +def test_read_batch_marks_failed_reader_slice_as_not_exhausted(tmp_path: Path) -> None: + home = tmp_path / "copilot" + _write_transcript(home, NATIVE_ID) + paths = _paths(home) + + def fail_database(_paths: SourcePaths, _native: str, _cursor: dict[str, Any]) -> SourceSlice: + raise OSError("database locked") + + with patch("thirdeye.platforms.copilot.sources.read_database", side_effect=fail_database): + batch = read_batch(paths, NATIVE_ID, {}) + + assert batch["next_cursor"]["database_exhausted"] is False + assert batch["next_cursor"]["transcript_exhausted"] is True + + def test_discover_sessions_delegates_to_underlying_readers(tmp_path: Path) -> None: home = tmp_path / "copilot" _write_transcript(home, "only-transcript") From 3320a3575896c3e1a8d4253e01d322ea1718b50d Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Thu, 10 Sep 2026 15:29:49 -0700 Subject: [PATCH 23/88] Freeze Copilot database drain to an invocation-time snapshot. Stop sync from paging a live SQLite source indefinitely by recording snapshot_end on the composed database cursor and bounding the drain loop. Co-authored-by: Cursor --- src/thirdeye/platforms/copilot/capture.py | 3 +- src/thirdeye/platforms/copilot/sources.py | 108 +++++++++++++++++++++- tests/test_copilot_capture.py | 93 +++++++++++++++++++ tests/test_copilot_sources.py | 46 +++++++++ 4 files changed, 247 insertions(+), 3 deletions(-) diff --git a/src/thirdeye/platforms/copilot/capture.py b/src/thirdeye/platforms/copilot/capture.py index 08bfa04..f920c74 100644 --- a/src/thirdeye/platforms/copilot/capture.py +++ b/src/thirdeye/platforms/copilot/capture.py @@ -26,6 +26,7 @@ "_incompatible", ) _ABSENCE_CODES = frozenset({"transcript_unavailable", "copilot_database_missing"}) +_MAX_DRAIN_PAGES = 256 def _result() -> SyncResult: @@ -240,7 +241,7 @@ def _drain_session(config: Config, paths: SourcePaths, native_session_id: str) - collected: list[dict[str, Any]] = [] last_batch: SourceBatch | None = None prev_cursor: object = object() - while True: + for _ in range(_MAX_DRAIN_PAGES): archive, batch, more_pages = _capture_once(config, paths, native_session_id) last_batch = batch collected.extend(batch["diagnostics"]) diff --git a/src/thirdeye/platforms/copilot/sources.py b/src/thirdeye/platforms/copilot/sources.py index 3e490ad..e98e8f4 100644 --- a/src/thirdeye/platforms/copilot/sources.py +++ b/src/thirdeye/platforms/copilot/sources.py @@ -20,6 +20,7 @@ __all__ = ["discover_sessions", "read_batch", "resolve_sources"] _DISCOVERY_PROBE_ID = "copilot-discovery-probe" +_MAX_DATABASE_SNAPSHOT_PROBES = 16 _FILE_LEVEL_DATABASE_CODES = frozenset( { "copilot_database_unreadable", @@ -112,6 +113,103 @@ def _read_slice( return _failed_slice(cursor, name, error) +def _is_database_cursor(cursor: dict[str, Any]) -> bool: + return isinstance(cursor.get("database_offset"), int) or isinstance( + cursor.get("database_generation"), str + ) + + +def _page_start(slice_: SourceSlice, incoming_offset: object) -> int: + live_offset = slice_["next_cursor"].get("database_offset") + page_len = len(slice_["records"]) + if isinstance(live_offset, int) and live_offset >= page_len: + return live_offset - page_len + return incoming_offset if isinstance(incoming_offset, int) else 0 + + +def _observe_database_snapshot_end( + paths: SourcePaths, native_id: str, slice_: SourceSlice +) -> int: + """Freeze the live row count observed for this invocation. + + Additional probes learn how far the current SQLite snapshot extends, but + they stop after a bounded number of pages so a source that keeps growing + cannot postpone snapshot creation. + """ + + cursor = slice_["next_cursor"] + offset = cursor.get("database_offset", len(slice_["records"])) + if not isinstance(offset, int) or offset < 0: + offset = len(slice_["records"]) + if slice_["exhausted"]: + return offset + generation = cursor.get("database_generation") + observed = offset + for _ in range(_MAX_DATABASE_SNAPSHOT_PROBES): + try: + probe = read_database( + paths, + native_id, + {"database_generation": generation, "database_offset": observed}, + ) + except (OSError, ValueError): + return observed + nxt = probe["next_cursor"].get("database_offset", observed) + if not isinstance(nxt, int) or nxt <= observed: + return observed + observed = nxt + if probe["exhausted"]: + return observed + return observed + + +def _with_database_snapshot( + paths: SourcePaths, + native_id: str, + slice_: SourceSlice, + incoming: dict[str, Any], +) -> SourceSlice: + """Keep database pagination inside the invocation-time row bound. + + The SQLite reader re-queries live rows on every page, so composition must + freeze ``snapshot_end`` the way the transcript reader freezes file size. + """ + + next_cursor = deepcopy(slice_["next_cursor"]) + if not _is_database_cursor(next_cursor) and not _is_database_cursor(incoming): + return slice_ + + records = list(slice_["records"]) + incoming_end = incoming.get("snapshot_end") + incoming_offset = incoming.get("database_offset", 0) + incoming_gen = incoming.get("database_generation") + live_gen = next_cursor.get("database_generation") + start_offset = _page_start(slice_, incoming_offset) + draining = ( + incoming_gen == live_gen + and isinstance(incoming_end, int) + and isinstance(incoming_offset, int) + and incoming_offset < incoming_end + ) + if draining: + snapshot_end = incoming_end + else: + snapshot_end = _observe_database_snapshot_end(paths, native_id, slice_) + + allowed = max(0, snapshot_end - start_offset) + if len(records) > allowed: + records = records[:allowed] + next_offset = start_offset + len(records) + next_cursor["database_offset"] = next_offset + next_cursor["snapshot_end"] = snapshot_end + return { + **slice_, + "records": records, + "next_cursor": next_cursor, + "exhausted": next_offset >= snapshot_end, + } + + def read_batch(paths: SourcePaths, native_session_id: str, cursor: dict) -> SourceBatch: """Read one bounded, lossless slice from each persisted source. @@ -120,7 +218,8 @@ def read_batch(paths: SourcePaths, native_session_id: str, cursor: dict) -> Sour ``base_cursor`` is an optimistic archive marker and is stripped before the archive persists the next source cursor. Exhaustion flags stay on the composition cursor so capture can page or report pending without confusing - the per-source readers. + the per-source readers. Database cursors also carry ``snapshot_end`` so a + later SQLite insert cannot extend this invocation's drain. """ validate_native_id(native_session_id) @@ -130,7 +229,12 @@ def read_batch(paths: SourcePaths, native_session_id: str, cursor: dict) -> Sour transcript = _read_slice( "transcript", read_transcript, paths, native_session_id, transcript_cursor ) - database = _read_slice("database", read_database, paths, native_session_id, database_cursor) + database = _with_database_snapshot( + paths, + native_session_id, + _read_slice("database", read_database, paths, native_session_id, database_cursor), + database_cursor, + ) next_cursor: dict[str, Any] = { "transcript": deepcopy(transcript["next_cursor"]), "database": deepcopy(database["next_cursor"]), diff --git a/tests/test_copilot_capture.py b/tests/test_copilot_capture.py index b410e6e..678b74d 100644 --- a/tests/test_copilot_capture.py +++ b/tests/test_copilot_capture.py @@ -573,6 +573,99 @@ def test_capture_session_reports_pending_when_transcript_exceeds_reader_limit( assert result["records_written"] == len(captured) +def test_sync_does_not_chase_growing_database_source( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + reads = {"n": 0} + + def growing_database( + _paths: SourcePaths, _native: str, cursor: dict[str, Any], **_kwargs: Any + ) -> Any: + reads["n"] += 1 + if reads["n"] > 80: + raise AssertionError("sync chased a growing database source") + offset = 0 + if isinstance(cursor, dict) and isinstance(cursor.get("database_offset"), int): + offset = cursor["database_offset"] + return { + "records": [ + _record(f"db/{offset + index}", source_kind="database") for index in range(1000) + ], + "next_cursor": {"database_generation": "gen-1", "database_offset": offset + 1000}, + "diagnostics": [], + "cwd": None, + "exhausted": False, + } + + def empty_transcript( + _paths: SourcePaths, _native: str, _cursor: dict[str, Any], **_kwargs: Any + ) -> Any: + return { + "records": [], + "next_cursor": {"byte_offset": 0, "snapshot_end": 0, "file_generation": "g"}, + "diagnostics": [], + "cwd": None, + "exhausted": True, + } + + monkeypatch.setattr("thirdeye.platforms.copilot.sources.read_database", growing_database) + monkeypatch.setattr("thirdeye.platforms.copilot.sources.read_transcript", empty_transcript) + monkeypatch.setattr( + "thirdeye.platforms.copilot.sources.discover_database_sessions", + lambda _paths: [NATIVE_SESSION_ID], + ) + monkeypatch.setattr("thirdeye.platforms.copilot.sources.discover_transcripts", lambda _paths: []) + + result = sync(config, paths, session_id=NATIVE_SESSION_ID) + stored = stored_session_id(paths, NATIVE_SESSION_ID) + captured = [record for record in iter_captured_records(config, stored) if record["source_kind"] == "database"] + + assert reads["n"] <= 80 + assert result["sessions"] == 1 + assert 1000 <= len(captured) <= 40_000 + assert len(captured) == len({record["source_id"] for record in captured}) + + +def test_sync_drains_paginated_database_snapshot( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + home = Path(paths["home"]) + _write_transcript(home, NATIVE_SESSION_ID) + _write_database(home, session_id=NATIVE_SESSION_ID) + database = home / "session-store.db" + connection = sqlite3.connect(database) + try: + connection.executemany( + "INSERT INTO turns (session_id, turn_index, content, updated_at) VALUES (?, ?, ?, ?)", + [ + (NATIVE_SESSION_ID, index, f"turn-{index}", "2026-09-10T17:08:10.000Z") + for index in range(2, 1002) + ], + ) + connection.commit() + baseline = connection.execute( + "SELECT COUNT(*) FROM sessions WHERE id = ? UNION ALL " + "SELECT COUNT(*) FROM turns WHERE session_id = ? UNION ALL " + "SELECT COUNT(*) FROM assistant_usage_events WHERE session_id = ?", + (NATIVE_SESSION_ID, NATIVE_SESSION_ID, NATIVE_SESSION_ID), + ).fetchall() + expected = sum(row[0] for row in baseline) + finally: + connection.close() + + result = sync(config, paths, session_id=NATIVE_SESSION_ID) + stored = stored_session_id(paths, NATIVE_SESSION_ID) + captured = list(iter_captured_records(config, stored)) + database_records = [record for record in captured if record["source_kind"] == "database"] + + assert result["errors"] == 0 + assert result["pending"] == 0 + assert len(database_records) == expected + + def test_sync_drains_paginated_invocation_snapshot( copilot_env: tuple[Config, SourcePaths], ) -> None: diff --git a/tests/test_copilot_sources.py b/tests/test_copilot_sources.py index b7bc523..013bd40 100644 --- a/tests/test_copilot_sources.py +++ b/tests/test_copilot_sources.py @@ -353,3 +353,49 @@ def test_discover_sessions_delegates_to_underlying_readers(tmp_path: Path) -> No assert discover_transcripts(paths) == ["only-transcript"] assert discover_database_sessions(paths) == [] assert discover_sessions(paths) == ["only-transcript"] + + +def test_read_batch_freezes_database_snapshot_end_against_later_inserts(tmp_path: Path) -> None: + home = tmp_path / "copilot" + home.mkdir() + paths = _paths(home) + reads = {"n": 0} + + def growing_database( + _paths: SourcePaths, _native: str, cursor: dict[str, Any] + ) -> SourceSlice: + reads["n"] += 1 + if reads["n"] > 80: + raise AssertionError("read_batch chased a growing database source") + offset = 0 + if isinstance(cursor, dict) and isinstance(cursor.get("database_offset"), int): + offset = cursor["database_offset"] + return _slice( + records=[_record(f"d/{offset + index}", source_kind="database") for index in range(1000)], + next_cursor={"database_generation": "gen-1", "database_offset": offset + 1000}, + cwd=None, + exhausted=False, + ) + + transcript = _slice(exhausted=True, cwd=None, next_cursor={"byte_offset": 0, "snapshot_end": 0}) + with ( + patch("thirdeye.platforms.copilot.sources.read_transcript", return_value=transcript), + patch("thirdeye.platforms.copilot.sources.read_database", side_effect=growing_database), + ): + first = read_batch(paths, NATIVE_ID, {}) + snapshot_end = first["next_cursor"]["database"]["snapshot_end"] + cursor = first["next_cursor"] + pages = 1 + records = list(first["records"]) + while not cursor.get("database_exhausted") and pages < 100: + batch = read_batch(paths, NATIVE_ID, cursor) + records.extend(batch["records"]) + cursor = batch["next_cursor"] + pages += 1 + assert cursor["database"]["snapshot_end"] == snapshot_end + + assert isinstance(snapshot_end, int) + assert cursor.get("database_exhausted") is True + assert cursor["database"]["database_offset"] == snapshot_end + assert len([item for item in records if item["source_kind"] == "database"]) == snapshot_end + assert reads["n"] <= 80 From 9d65388e301b76b4bb46f3098e1abf4f103698f8 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Thu, 10 Sep 2026 15:32:56 -0700 Subject: [PATCH 24/88] test: cover raw Copilot capture event views --- tests/test_copilot_capture_reads.py | 132 ++++++++++++++++++++++++ tests/web/test_copilot_capture_views.py | 84 +++++++++++++++ 2 files changed, 216 insertions(+) create mode 100644 tests/test_copilot_capture_reads.py create mode 100644 tests/web/test_copilot_capture_views.py diff --git a/tests/test_copilot_capture_reads.py b/tests/test_copilot_capture_reads.py new file mode 100644 index 0000000..8c1c0ee --- /dev/null +++ b/tests/test_copilot_capture_reads.py @@ -0,0 +1,132 @@ +"""Compatibility coverage for generic CLI reads of raw Copilot evidence.""" + +from __future__ import annotations + +from pathlib import Path + +from click.testing import CliRunner + +from thirdeye.cli import main +from thirdeye.config import Config +from thirdeye.platforms.copilot.archive import commit_batch +from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id +from thirdeye.platforms.copilot.types import SourceBatch, SourcePaths, SourceRecord +from thirdeye.store import Store + +NATIVE_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" +SOURCE_TS = "2026-09-10T17:08:24.506Z" +OBSERVED_AT = "2026-09-10T17:08:25.000Z" + + +def _record(kind: str, source_id: str, payload: dict) -> SourceRecord: + return { + "source_id": source_id, + "source_kind": kind, + "native_session_id": NATIVE_ID, + "ts": SOURCE_TS, + "observed_at": OBSERVED_AT, + "payload": {"schema_version": 1, **payload}, + "locator": {"file": "events.jsonl", "file_generation": "fixture-gen", "byte_offset": 42}, + } + + +def _capture_synthetic_batch(config: Config, paths: SourcePaths) -> str: + """Archive labeled synthetic SourceRecords for generic read compatibility.""" + records = [ + _record( + "transcript", + f"{paths['source_key']}/{NATIVE_ID}/prompt-1", + { + "type": "user.message", + "id": "prompt-1", + "timestamp": SOURCE_TS, + "data": {"content": "Read alpha.txt and beta.txt with separate view calls."}, + }, + ), + _record( + "transcript", + f"{paths['source_key']}/{NATIVE_ID}/tool-1", + { + "type": "tool.execution_start", + "id": "tool-1", + "timestamp": SOURCE_TS, + "data": {"toolName": "view", "arguments": {"path": "/fixture/workspace/alpha.txt"}}, + }, + ), + _record( + "hook", + f"hook/{NATIVE_ID}/child-prompt-1", + { + "event": "userPromptSubmitted", + "hook_payload": { + "sessionId": NATIVE_ID, + "prompt": "Read only alpha.txt as an explore child.", + "agentId": "child-agent-id", + }, + }, + ), + ] + batch: SourceBatch = { + "source_key": paths["source_key"], + "native_session_id": NATIVE_ID, + "cwd": "/fixture/workspace", + "records": records, + "next_cursor": {"fixture": 1}, + "diagnostics": [], + } + commit_batch(config, paths, batch) + return stored_session_id(paths, NATIVE_ID) + + +def test_store_lists_and_retains_raw_source_identity(tmp_path: Path) -> None: + config = Config(root=tmp_path / "thirdeye") + paths = resolve_sources(tmp_path / "copilot-home") + stored_id = _capture_synthetic_batch(config, paths) + + sessions = list(Store(config).list_sessions(platform="copilot")) + assert [(session.session_id, session.cwd) for session in sessions] == [ + (stored_id, "/fixture/workspace") + ] + assert sessions[0].extra["copilot"]["native_session_id"] == NATIVE_ID + + events = list(Store(config).reader(stored_id).iter_events()) + assert [event["t"] for event in events] == [ + "copilot_transcript", + "copilot_transcript", + "copilot_hook", + ] + assert all(event["t"] not in {"user_message", "tool_call"} for event in events) + assert events[0]["ts"] == SOURCE_TS + source_record = events[1]["data"]["source_record"] + assert source_record["source_id"] == f"{paths['source_key']}/{NATIVE_ID}/tool-1" + assert source_record["native_session_id"] == NATIVE_ID + assert source_record["payload"]["type"] == "tool.execution_start" + assert source_record["locator"]["byte_offset"] == 42 + assert events[2]["data"]["source_record"]["payload"]["hook_payload"]["agentId"] == "child-agent-id" + + +def test_generic_cli_reads_search_raw_copilot_content(tmp_path: Path) -> None: + config = Config(root=tmp_path / "thirdeye") + paths = resolve_sources(tmp_path / "copilot-home") + stored_id = _capture_synthetic_batch(config, paths) + runner = CliRunner() + env = {"THIRDEYE_HOME": str(config.root)} + + listed = runner.invoke(main, ["list", "--platform", "copilot"], env=env) + shown = runner.invoke(main, ["show", stored_id, "--json"], env=env) + events = runner.invoke(main, ["events", stored_id, "--json", "--no-findings"], env=env) + prompt_search = runner.invoke( + main, ["search", "separate view calls", "--platform", "copilot"], env=env + ) + tool_search = runner.invoke(main, ["search", "alpha.txt", "--platform", "copilot"], env=env) + tailed = runner.invoke(main, ["tail", stored_id, "-n", "1", "--json"], env=env) + + assert listed.exit_code == shown.exit_code == events.exit_code == prompt_search.exit_code == 0 + assert tool_search.exit_code == tailed.exit_code == 0 + assert stored_id in listed.output + assert NATIVE_ID in listed.output + assert '"t":"copilot_transcript"' in shown.output + assert '"type":"tool.execution_start"' in events.output + assert "separate view calls" in prompt_search.output + assert "alpha.txt" in tool_search.output + assert '"t":"copilot_hook"' in tailed.output diff --git a/tests/web/test_copilot_capture_views.py b/tests/web/test_copilot_capture_views.py new file mode 100644 index 0000000..d0ada3f --- /dev/null +++ b/tests/web/test_copilot_capture_views.py @@ -0,0 +1,84 @@ +"""Generic web views render raw Copilot archive events without projections.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from thirdeye.platforms.copilot.archive import commit_batch +from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id +from thirdeye.platforms.copilot.types import SourceBatch, SourceRecord + +pytest.importorskip("starlette") + +NATIVE_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" +SOURCE_TS = "2026-09-10T17:08:44.557Z" + + +def _capture_synthetic_batch(web_config, tmp_path: Path) -> str: + """Archive labeled synthetic child evidence; no turn projection is involved.""" + paths = resolve_sources(tmp_path / "copilot-home") + records: list[SourceRecord] = [ + { + "source_id": f"{paths['source_key']}/{NATIVE_ID}/child-message", + "source_kind": "transcript", + "native_session_id": NATIVE_ID, + "ts": SOURCE_TS, + "observed_at": "2026-09-10T17:08:45.000Z", + "payload": { + "schema_version": 1, + "type": "user.message", + "id": "child-message", + "timestamp": SOURCE_TS, + "agentId": "child-agent-id", + "data": {"content": "Read alpha.txt and beta.txt as the explore child."}, + }, + "locator": {"file": "events.jsonl", "file_generation": "fixture-gen", "byte_offset": 512}, + }, + { + "source_id": f"hook/{NATIVE_ID}/child-stop", + "source_kind": "hook", + "native_session_id": NATIVE_ID, + "ts": SOURCE_TS, + "observed_at": "2026-09-10T17:08:45.001Z", + "payload": { + "schema_version": 1, + "event": "agentStop", + "hook_payload": {"sessionId": NATIVE_ID, "agentId": "child-agent-id", "response": "42"}, + }, + "locator": {"observation_id": "child-stop", "event": "agentStop"}, + }, + ] + batch: SourceBatch = { + "source_key": paths["source_key"], + "native_session_id": NATIVE_ID, + "cwd": "/fixture/workspace", + "records": records, + "next_cursor": {"fixture": 1}, + "diagnostics": [], + } + commit_batch(web_config, paths, batch) + return stored_session_id(paths, NATIVE_ID) + + +def test_generic_event_views_show_raw_child_and_hook_evidence(client, web_config, tmp_path: Path) -> None: + stored_id = _capture_synthetic_batch(web_config, tmp_path) + + session = client.get(f"/sessions/{stored_id}") + tree = client.get(f"/sessions/{stored_id}/tree") + detail = client.get(f"/sessions/{stored_id}/events/1") + search = client.get("/search?q=alpha.txt&platform=copilot") + + assert session.status_code == tree.status_code == detail.status_code == search.status_code == 200 + assert b"copilot" in session.content + assert b"copilot_transcript" in tree.content + assert b"copilot_hook" in tree.content + assert b"user_message" not in tree.content + assert b"tool_call" not in tree.content + assert b"child-agent-id" in detail.content + assert NATIVE_ID.encode() in detail.content + assert SOURCE_TS.encode() in detail.content + assert b"child-message" in detail.content or b"child-stop" in detail.content + assert stored_id.encode() in search.content + assert b"alpha.txt" in search.content From b7f9e8c22edae7b8490dc6d1631526adb6eb9a54 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Thu, 10 Sep 2026 15:33:28 -0700 Subject: [PATCH 25/88] Add Copilot capture watch and status --- src/thirdeye/platforms/copilot/status.py | 187 +++++++++++++++++++++++ src/thirdeye/platforms/copilot/watch.py | 171 +++++++++++++++++++++ 2 files changed, 358 insertions(+) create mode 100644 src/thirdeye/platforms/copilot/status.py create mode 100644 src/thirdeye/platforms/copilot/watch.py diff --git a/src/thirdeye/platforms/copilot/status.py b/src/thirdeye/platforms/copilot/status.py new file mode 100644 index 0000000..0189e51 --- /dev/null +++ b/src/thirdeye/platforms/copilot/status.py @@ -0,0 +1,187 @@ +"""Local health reporting for the Copilot CLI capture archive.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from thirdeye.config import Config +from thirdeye.paths import platform_dir +from thirdeye.reader import SessionReader + +from .archive import _record_from_event +from .constants import PLATFORM_NAME +from .install import CopilotPlatform +from .spool import read_spool +from .state import journal_path, read_json, state_path +from .types import SourcePaths, SourceRecord + + +def _path_capability(path: Path, *, directory: bool) -> dict[str, Any]: + """Describe a local source without treating an absent Copilot install as an error.""" + + try: + exists = path.exists() + kind_matches = path.is_dir() if directory else path.is_file() + readable = kind_matches and path.stat() is not None + except OSError as error: + return { + "path": str(path), + "exists": False, + "readable": False, + "error": {"kind": "source_unreadable", "path": str(path), "reason": type(error).__name__}, + } + return {"path": str(path), "exists": exists, "readable": readable} + + +def _archive_directories(config: Config, paths: SourcePaths) -> list[Path]: + root = platform_dir(config.root, PLATFORM_NAME) + prefix = f"copilot-{paths['source_key'][:16]}-" + try: + return sorted( + (entry for entry in root.iterdir() if entry.is_dir() and entry.name.startswith(prefix)), + key=lambda entry: entry.name, + ) + except OSError: + return [] + + +def _record_hook(record: SourceRecord, latest: SourceRecord | None) -> SourceRecord | None: + if record["source_kind"] != "hook": + return latest + if latest is None or record["observed_at"] > latest["observed_at"]: + return record + return latest + + +def _spool_sessions(config: Config, paths: SourcePaths) -> tuple[int, list[str], SourceRecord | None]: + root = Path(config.root) / "spool" / "copilot" / paths["source_key"] + count = 0 + sessions: list[str] = [] + latest: SourceRecord | None = None + try: + entries = sorted(root.iterdir()) + except OSError: + return count, sessions, latest + for entry in entries: + if not entry.is_dir(): + continue + try: + records = read_spool(config, paths, entry.name) + except ValueError: + continue + if records: + sessions.append(entry.name) + count += len(records) + for record in records: + latest = _record_hook(record, latest) + return count, sessions, latest + + +def _archive_status( + config: Config, paths: SourcePaths +) -> tuple[list[dict[str, Any]], SourceRecord | None, list[dict[str, Any]], int, int]: + """Read archive health/state without attempting an import or source read.""" + + sessions: list[dict[str, Any]] = [] + errors: list[dict[str, Any]] = [] + latest_hook: SourceRecord | None = None + pending_followup = 0 + active_leases = 0 + for directory in _archive_directories(config, paths): + try: + state = read_json(state_path(directory)) + except ValueError as error: + errors.append({"kind": "invalid_archive_state", "session": directory.name, "reason": str(error)}) + continue + if state is None: + state = {} + health = state.get("health") if isinstance(state.get("health"), dict) else {} + diagnostics = health.get("diagnostics") if isinstance(health.get("diagnostics"), list) else [] + followup = state.get("followup", state.get("pending_followup", False)) + lease = state.get("lease", state.get("leases", None)) + pending_followup += int(bool(followup)) + active_leases += len(lease) if isinstance(lease, list) else int(bool(lease)) + sessions.append( + { + "stored_session_id": directory.name, + "native_session_id": state.get("native_session_id"), + "cursor": state.get("cursor", {}), + "last_successful_import": health.get("last_successful_import"), + "diagnostics": diagnostics, + "journal_pending": journal_path(directory).is_file(), + } + ) + for diagnostic in diagnostics: + if isinstance(diagnostic, dict): + errors.append({"session": directory.name, **diagnostic}) + try: + for event in SessionReader(directory).iter_events(types=("copilot_hook",)): + record = _record_from_event(event) + if record is not None: + latest_hook = _record_hook(record, latest_hook) + except (OSError, ValueError): + errors.append({"kind": "archive_events_unreadable", "session": directory.name}) + return sessions, latest_hook, errors, pending_followup, active_leases + + +def capture_status(config: Config, paths: SourcePaths) -> dict: + """Return local capture installation, progress, and health information. + + An absent Copilot home/database and uninstalled owned hooks are reported as + capabilities/configuration, not errors. Archive/source diagnostics remain + in ``errors`` so a command layer can choose a nonzero status for them. + """ + + home = Path(paths["home"]) + session_root = _path_capability(Path(paths["session_root"]), directory=True) + database = _path_capability(Path(paths["database"]), directory=False) + wal = _path_capability(Path(paths["database"]).with_name(f"{Path(paths['database']).name}-wal"), directory=False) + sessions, archived_hook, archive_errors, pending_followup, active_leases = _archive_status( + config, paths + ) + spool_count, spool_sessions, spooled_hook = _spool_sessions(config, paths) + last_hook = archived_hook + if spooled_hook is not None: + last_hook = _record_hook(spooled_hook, last_hook) + + source_errors = [ + item["error"] + for item in (session_root, database, wal) + if isinstance(item.get("error"), dict) + ] + last_successful = max( + ( + value + for session in sessions + if isinstance((value := session.get("last_successful_import")), str) + ), + default=None, + ) + installer = CopilotPlatform(source_home=home) + return { + "paths": dict(paths), + "installation": { + "configured": installer.is_installed(), + "hooks_file": str(installer.hooks_file), + }, + "capabilities": { + "transcripts": session_root, + "database": database, + "database_wal": wal, + }, + "last_observed_hook": last_hook, + "last_successful_import": last_successful, + "sessions": sessions, + "pending": { + "spool_records": spool_count, + "spool_sessions": spool_sessions, + "followup": pending_followup, + "leases": active_leases, + "journals": sum(1 for session in sessions if session["journal_pending"]), + }, + "errors": [*source_errors, *archive_errors], + } + + +__all__ = ["capture_status"] diff --git a/src/thirdeye/platforms/copilot/watch.py b/src/thirdeye/platforms/copilot/watch.py new file mode 100644 index 0000000..03cacc7 --- /dev/null +++ b/src/thirdeye/platforms/copilot/watch.py @@ -0,0 +1,171 @@ +"""Foreground polling for local Copilot CLI evidence. + +The watcher intentionally watches filesystem *identity*, not Copilot's +process. This makes it useful after an editor-terminal session has ended and +also means it never needs to start, inspect, or authenticate a Copilot CLI. +""" + +from __future__ import annotations + +import math +import time +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from thirdeye.config import Config + +from .capture import sync +from .database import discover_database_sessions +from .identity import validate_native_id +from .sources import discover_sessions +from .types import SourcePaths + +_EVENTS_FILENAME = "events.jsonl" +_SLEEP: Callable[[float], None] = time.sleep + + +def _file_stamp(path: Path) -> tuple[int, int, int, int] | None: + """Return a cheap replacement/append-sensitive stamp for one file.""" + + try: + stat = path.stat() + except OSError: + return None + if not path.is_file(): + return None + return (stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns) + + +def _transcript_stamps(paths: SourcePaths, session_ids: set[str]) -> dict[str, object]: + root = Path(paths["session_root"]) + stamps: dict[str, object] = {} + for native_id in session_ids: + try: + validate_native_id(native_id) + except ValueError: + continue + stamps[native_id] = _file_stamp(root / native_id / _EVENTS_FILENAME) + return stamps + + +def _database_stamp(paths: SourcePaths) -> tuple[object, object, object]: + """Include WAL and SHM changes: live SQLite commits need no DB mtime.""" + + database = Path(paths["database"]) + return ( + _file_stamp(database), + _file_stamp(database.with_name(f"{database.name}-wal")), + _file_stamp(database.with_name(f"{database.name}-shm")), + ) + + +def _spool_stamps(config: Config, paths: SourcePaths) -> dict[str, tuple[tuple[str, object], ...]]: + """Return per-session spool stamps without loading prompt-bearing records.""" + + root = Path(config.root) / "spool" / "copilot" / paths["source_key"] + try: + entries = list(root.iterdir()) + except OSError: + return {} + result: dict[str, tuple[tuple[str, object], ...]] = {} + for entry in entries: + try: + if not entry.is_dir(): + continue + validate_native_id(entry.name) + files = tuple( + (item.name, _file_stamp(item)) + for item in sorted(entry.glob("*.json")) + ) + except (OSError, ValueError): + continue + result[entry.name] = files + return result + + +def _source_snapshot(config: Config, paths: SourcePaths) -> dict[str, Any]: + """Discover source IDs and cheap change positions for the next poll.""" + + # ``discover_sessions`` is deliberately the public composition discovery + # entrypoint. Database-specific discovery is additionally retained so a + # WAL-only change need not re-read transcript-only sessions. + all_sessions = set(discover_sessions(paths)) + database_sessions = set(discover_database_sessions(paths)) + all_sessions.update(database_sessions) + spool = _spool_stamps(config, paths) + all_sessions.update(spool) + return { + "sessions": all_sessions, + "database_sessions": database_sessions, + "transcripts": _transcript_stamps(paths, all_sessions), + "database": _database_stamp(paths), + "spool": spool, + } + + +def _changed_sessions(before: dict[str, Any], after: dict[str, Any]) -> set[str]: + """Choose only source positions that can have produced new evidence.""" + + before_sessions = before["sessions"] + after_sessions = after["sessions"] + changed = set(after_sessions - before_sessions) + + for native_id in after_sessions: + if before["transcripts"].get(native_id) != after["transcripts"].get(native_id): + changed.add(native_id) + if before["spool"].get(native_id) != after["spool"].get(native_id): + changed.add(native_id) + + if before["database"] != after["database"]: + # Retain IDs observed before the change as well: a transaction may + # delete/update rows and source disappearance is never session close. + changed.update(before["database_sessions"]) + changed.update(after["database_sessions"]) + return changed + + +def _result_needs_retry(result: dict[str, int]) -> bool: + return result.get("pending", 0) > 0 or result.get("errors", 0) > 0 + + +def watch(config: Config, paths: SourcePaths, *, interval: float = 1.0) -> None: + """Poll local recordings until interrupted. + + The initial sync drains the bounded source snapshot. Later cycles invoke + per-session sync only after a transcript, database/WAL, or spool position + changes (or after a retryable result), avoiding repeated parsing of quiet + completed sessions. All capture remains local; this function never + exports data and never starts a background service. + """ + + if not isinstance(interval, (int, float)) or isinstance(interval, bool): + raise ValueError("interval must be a finite number of seconds, at least 0.1") + if not math.isfinite(interval) or interval < 0.1: + raise ValueError("interval must be a finite number of seconds, at least 0.1") + + try: + # Take the baseline before the initial drain. A source can append + # while that bounded drain is running; comparing its post-drain stamp + # on the first poll ensures that append receives a later capture. + previous = _source_snapshot(config, paths) + initial = sync(config, paths) + retry = set() if not _result_needs_retry(initial) else set(previous["sessions"]) + while True: + _SLEEP(float(interval)) + current = _source_snapshot(config, paths) + selected = _changed_sessions(previous, current) | retry + retry = set() + for native_id in sorted(selected): + # KeyboardInterrupt is intentionally checked between sessions; + # a current archive commit remains crash-recoverable. + result = sync(config, paths, session_id=native_id) + if _result_needs_retry(result): + retry.add(native_id) + previous = current + except KeyboardInterrupt: + # A foreground CLI command treats Ctrl-C as ordinary termination. + return + + +__all__ = ["watch"] From d8693bca5a5e217806688a82faeed69fc3614e27 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Thu, 10 Sep 2026 15:34:04 -0700 Subject: [PATCH 26/88] test: broaden raw Copilot view compatibility coverage Add assertions for all source kinds, turn-slicer exclusion, hook search, and usage dashboard boundaries so generic views stay raw without V2 projections. Co-authored-by: Cursor --- tests/test_copilot_capture_reads.py | 67 ++++++++++++++++++++ tests/web/test_copilot_capture_views.py | 81 +++++++++++++++++++++++++ 2 files changed, 148 insertions(+) diff --git a/tests/test_copilot_capture_reads.py b/tests/test_copilot_capture_reads.py index 8c1c0ee..259d313 100644 --- a/tests/test_copilot_capture_reads.py +++ b/tests/test_copilot_capture_reads.py @@ -12,6 +12,7 @@ from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id from thirdeye.platforms.copilot.types import SourceBatch, SourcePaths, SourceRecord from thirdeye.store import Store +from thirdeye.turns import session_turns NATIVE_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" SOURCE_TS = "2026-09-10T17:08:24.506Z" @@ -130,3 +131,69 @@ def test_generic_cli_reads_search_raw_copilot_content(tmp_path: Path) -> None: assert "separate view calls" in prompt_search.output assert "alpha.txt" in tool_search.output assert '"t":"copilot_hook"' in tailed.output + + +def test_all_source_kinds_map_to_raw_event_types_without_projection(tmp_path: Path) -> None: + config = Config(root=tmp_path / "thirdeye") + paths = resolve_sources(tmp_path / "copilot-home") + records = [ + _record("transcript", f"{paths['source_key']}/{NATIVE_ID}/tx", {"type": "user.message"}), + _record( + "database", + f"{paths['source_key']}/{NATIVE_ID}/db", + {"table": "assistant_usage_events", "tokens": 42}, + ), + _record("hook", f"hook/{NATIVE_ID}/hk", {"event": "sessionStart", "hook_payload": {}}), + _record( + "metadata", + f"{paths['source_key']}/{NATIVE_ID}/meta", + {"file": "workspace.yaml", "cwd": "/fixture/workspace"}, + ), + ] + batch: SourceBatch = { + "source_key": paths["source_key"], + "native_session_id": NATIVE_ID, + "cwd": "/fixture/workspace", + "records": records, + "next_cursor": {"fixture": 1}, + "diagnostics": [], + } + commit_batch(config, paths, batch) + stored_id = stored_session_id(paths, NATIVE_ID) + + events = list(Store(config).reader(stored_id).iter_events()) + assert [event["t"] for event in events] == [ + "copilot_transcript", + "copilot_database", + "copilot_hook", + "copilot_metadata", + ] + assert all(event["data"]["schema_version"] == 1 for event in events) + assert all(event["t"] not in {"user_message", "tool_call", "assistant_message"} for event in events) + + +def test_copilot_sessions_are_not_sliced_into_eval_turns(tmp_path: Path) -> None: + config = Config(root=tmp_path / "thirdeye") + paths = resolve_sources(tmp_path / "copilot-home") + stored_id = _capture_synthetic_batch(config, paths) + store = Store(config) + meta = store.get_meta(stored_id) + + assert session_turns(meta, store) == [] + + +def test_hook_prompt_content_is_searchable(tmp_path: Path) -> None: + config = Config(root=tmp_path / "thirdeye") + paths = resolve_sources(tmp_path / "copilot-home") + _capture_synthetic_batch(config, paths) + runner = CliRunner() + env = {"THIRDEYE_HOME": str(config.root)} + + hook_search = runner.invoke( + main, + ["search", "explore child", "--platform", "copilot"], + env=env, + ) + + assert hook_search.exit_code == 0 + assert "explore child" in hook_search.output diff --git a/tests/web/test_copilot_capture_views.py b/tests/web/test_copilot_capture_views.py index d0ada3f..35237b9 100644 --- a/tests/web/test_copilot_capture_views.py +++ b/tests/web/test_copilot_capture_views.py @@ -9,6 +9,7 @@ from thirdeye.platforms.copilot.archive import commit_batch from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id from thirdeye.platforms.copilot.types import SourceBatch, SourceRecord +from thirdeye.turns import session_turns pytest.importorskip("starlette") @@ -82,3 +83,83 @@ def test_generic_event_views_show_raw_child_and_hook_evidence(client, web_config assert b"child-message" in detail.content or b"child-stop" in detail.content assert stored_id.encode() in search.content assert b"alpha.txt" in search.content + + +def test_copilot_database_events_render_in_generic_tree(client, web_config, tmp_path: Path) -> None: + paths = resolve_sources(tmp_path / "copilot-home") + records: list[SourceRecord] = [ + { + "source_id": f"{paths['source_key']}/{NATIVE_ID}/usage-row", + "source_kind": "database", + "native_session_id": NATIVE_ID, + "ts": SOURCE_TS, + "observed_at": "2026-09-10T17:08:45.000Z", + "payload": { + "schema_version": 1, + "table": "assistant_usage_events", + "model": "gpt-4.1", + "input_tokens": 100, + }, + "locator": {"table": "assistant_usage_events", "rowid": 7}, + }, + { + "source_id": f"{paths['source_key']}/{NATIVE_ID}/workspace-meta", + "source_kind": "metadata", + "native_session_id": NATIVE_ID, + "ts": None, + "observed_at": "2026-09-10T17:08:45.001Z", + "payload": { + "schema_version": 1, + "file": "workspace.yaml", + "cwd": "/fixture/workspace", + }, + "locator": {"file": "workspace.yaml"}, + }, + ] + batch: SourceBatch = { + "source_key": paths["source_key"], + "native_session_id": NATIVE_ID, + "cwd": "/fixture/workspace", + "records": records, + "next_cursor": {"fixture": 1}, + "diagnostics": [], + } + commit_batch(web_config, paths, batch) + stored_id = stored_session_id(paths, NATIVE_ID) + + tree = client.get(f"/sessions/{stored_id}/tree") + detail_db = client.get(f"/sessions/{stored_id}/events/0") + detail_meta = client.get(f"/sessions/{stored_id}/events/1") + + assert tree.status_code == detail_db.status_code == detail_meta.status_code == 200 + assert b"copilot_database" in tree.content + assert b"copilot_metadata" in tree.content + assert b"assistant_usage_events" in detail_db.content + assert b"workspace.yaml" in detail_meta.content + assert b"user_message" not in tree.content + + +def test_copilot_sessions_are_excluded_from_index_turn_query(client, web_config, tmp_path: Path) -> None: + stored_id = _capture_synthetic_batch(web_config, tmp_path) + store = client.app.state.store + meta = store.get_meta(stored_id) + + assert session_turns(meta, store) == [] + + without_turn_filter = client.get("/?platform=copilot&since=2020-01-01") + with_turn_query = client.get("/?platform=copilot&since=2020-01-01&turn_query=alpha.txt") + + assert without_turn_filter.status_code == with_turn_query.status_code == 200 + assert stored_id.encode() in without_turn_filter.content + assert stored_id.encode() not in with_turn_query.content + + +def test_copilot_session_usage_page_has_no_token_rows(client, web_config, tmp_path: Path) -> None: + stored_id = _capture_synthetic_batch(web_config, tmp_path) + + usage = client.get(f"/sessions/{stored_id}/usage") + + assert usage.status_code == 200 + assert b"usage" in usage.content + assert b"gpt-4.1" not in usage.content + assert b"input_tokens" not in usage.content From 037f7e40ef8e89ac465931a397a2e26e6fb33bd9 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Thu, 10 Sep 2026 15:34:07 -0700 Subject: [PATCH 27/88] Add Copilot hook lifecycle capture --- src/thirdeye/platforms/copilot/followup.py | 230 +++++++++++++++++++++ src/thirdeye/platforms/copilot/hooks.py | 162 +++++++++++++++ 2 files changed, 392 insertions(+) create mode 100644 src/thirdeye/platforms/copilot/followup.py create mode 100644 src/thirdeye/platforms/copilot/hooks.py diff --git a/src/thirdeye/platforms/copilot/followup.py b/src/thirdeye/platforms/copilot/followup.py new file mode 100644 index 0000000..188bb31 --- /dev/null +++ b/src/thirdeye/platforms/copilot/followup.py @@ -0,0 +1,230 @@ +"""Short-lived, coalesced follow-up capture for Copilot hook receipts. + +The hook process must return promptly. A hook therefore starts at most one +detached worker per source session; the worker makes a few bounded attempts to +pick up transcript or SQLite data which arrived just after the hook. +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +import tempfile +import time +from pathlib import Path +from typing import Any +from uuid import uuid4 + +from thirdeye._compat import fsops, proc +from thirdeye._compat.locking import LockMode, LockTimeout, locked +from thirdeye.config import Config +from thirdeye.paths import session_dir +from thirdeye.platforms.copilot.identity import stored_session_id, validate_native_id +from thirdeye.platforms.copilot.state import lock_path +from thirdeye.platforms.copilot.types import SourcePaths +from thirdeye.usage.errlog import log_capture_error + +_PLATFORM = "copilot" +_LEASE_FILENAME = "copilot.followup.json" +_LEASE_LOCK_FILENAME = "copilot.followup.lock" +_LEASE_SECONDS = 5.0 +_LOCK_PROBE_TIMEOUT = 0.0 +_INITIAL_BACKOFF_SECONDS = 0.05 +_MAX_BACKOFF_SECONDS = 0.5 + + +def _directory(config: Config, paths: SourcePaths, native_id: str) -> Path: + return session_dir(config.root, _PLATFORM, stored_session_id(paths, native_id)) + + +def _lease_path(directory: Path) -> Path: + return directory / _LEASE_FILENAME + + +def _lease_lock_path(directory: Path) -> Path: + return directory / _LEASE_LOCK_FILENAME + + +def _now() -> float: + return time.time() + + +def _read_lease(path: Path) -> dict[str, Any] | None: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError): + return None + return value if isinstance(value, dict) else None + + +def _write_lease(path: Path, value: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + fd, name = tempfile.mkstemp(dir=path.parent, prefix=f"{path.name}.", suffix=".tmp") + try: + with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as stream: + json.dump(value, stream, separators=(",", ":"), sort_keys=True) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + fsops.replace(name, path) + fsops.sync_directory(path.parent) + except BaseException: + fsops.unlink(Path(name), missing_ok=True) + raise + + +def _claim_lease(config: Config, paths: SourcePaths, native_id: str) -> str | None: + """Return a new generation, or ``None`` when a live worker owns it.""" + + directory = _directory(config, paths, native_id) + try: + with locked(_lease_lock_path(directory), LockMode.EXCLUSIVE, timeout=_LOCK_PROBE_TIMEOUT): + current = _read_lease(_lease_path(directory)) + if current is not None and isinstance(current.get("expires_at"), (int, float)): + if float(current["expires_at"]) > _now(): + return None + generation = uuid4().hex + _write_lease( + _lease_path(directory), + {"generation": generation, "expires_at": _now() + _LEASE_SECONDS}, + ) + return generation + except (LockTimeout, OSError): + return None + + +def _owns_lease(directory: Path, generation: str) -> bool: + try: + with locked(_lease_lock_path(directory), LockMode.EXCLUSIVE, timeout=_LOCK_PROBE_TIMEOUT): + current = _read_lease(_lease_path(directory)) + return bool(current and current.get("generation") == generation) + except (LockTimeout, OSError): + return False + + +def _release_lease(directory: Path, generation: str) -> None: + try: + with locked(_lease_lock_path(directory), LockMode.EXCLUSIVE, timeout=_LOCK_PROBE_TIMEOUT): + current = _read_lease(_lease_path(directory)) + if current is not None and current.get("generation") == generation: + fsops.unlink(_lease_path(directory), missing_ok=True) + fsops.sync_directory(directory) + except (LockTimeout, OSError): + return + + +def schedule_followup(config: Config, paths: SourcePaths, native_id: str) -> bool: + """Coalesce hook follow-ups and spawn one detached, finite worker.""" + + validate_native_id(native_id) + generation = _claim_lease(config, paths, native_id) + if generation is None: + return False + try: + proc.spawn_detached( + [ + sys.executable, + "-m", + "thirdeye.platforms.copilot.followup", + "--source-home", + paths["home"], + "--session-id", + native_id, + "--config-root", + str(config.root), + "--generation", + generation, + ] + ) + except Exception as exc: + _release_lease(_directory(config, paths, native_id), generation) + log_capture_error( + thirdeye_home=config.root, + phase="copilot_followup_spawn", + error=exc, + platform=_PLATFORM, + session_id=native_id, + silent_fallback=True, + ) + return False + return True + + +def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument("--source-home", required=True) + parser.add_argument("--session-id", required=True) + parser.add_argument("--config-root", required=True) + parser.add_argument("--generation", required=True) + return parser.parse_args(argv) + + +def _archive_lock_available(config: Config, paths: SourcePaths, native_id: str) -> bool: + """Avoid starting a capture which is already known to block on its lock.""" + + directory = _directory(config, paths, native_id) + try: + with locked(lock_path(directory), LockMode.EXCLUSIVE, timeout=_LOCK_PROBE_TIMEOUT): + return True + except (LockTimeout, OSError): + return False + + +def _run(config: Config, paths: SourcePaths, native_id: str, generation: str) -> None: + """Try follow-up capture for no longer than the lease window.""" + + # Import only in runtime composition: source/archive modules remain + # independent of this detached-worker mechanism. + from thirdeye.platforms.copilot.capture import capture_session + + directory = _directory(config, paths, native_id) + deadline = time.monotonic() + _LEASE_SECONDS + delay = _INITIAL_BACKOFF_SECONDS + try: + while time.monotonic() < deadline and _owns_lease(directory, generation): + if _archive_lock_available(config, paths, native_id): + try: + result = capture_session(config, paths, native_id) + except Exception as exc: + log_capture_error( + thirdeye_home=config.root, + phase="copilot_followup_capture", + error=exc, + platform=_PLATFORM, + session_id=native_id, + silent_fallback=True, + ) + else: + if result["errors"] == 0 and result["pending"] == 0: + return + remaining = deadline - time.monotonic() + if remaining <= 0: + break + time.sleep(min(delay, remaining)) + delay = min(delay * 2, _MAX_BACKOFF_SECONDS) + finally: + _release_lease(directory, generation) + + +def main() -> None: + """Detached entrypoint. Its arguments contain paths and identity only.""" + + try: + args = _parse_args() + from thirdeye.platforms.copilot.identity import resolve_sources + + paths = resolve_sources(Path(args.source_home)) + native_id = str(args.session_id) + validate_native_id(native_id) + config = Config(root=Path(args.config_root)) + _run(config, paths, native_id, str(args.generation)) + except Exception: + # This worker is deliberately silent: diagnostics are kept locally and + # a failed follow-up never changes Copilot's hook outcome. + return + + +if __name__ == "__main__": + main() diff --git a/src/thirdeye/platforms/copilot/hooks.py b/src/thirdeye/platforms/copilot/hooks.py new file mode 100644 index 0000000..1eb2ee6 --- /dev/null +++ b/src/thirdeye/platforms/copilot/hooks.py @@ -0,0 +1,162 @@ +"""Fail-open runtime for the ``thirdeye-copilot-hook EVENT`` dispatcher.""" + +from __future__ import annotations + +import io +import json +import sys +from typing import Any + +from thirdeye.config import Config +from thirdeye.env_capture import capture_env, env_to_tag +from thirdeye.paths import session_dir +from thirdeye.platforms.provenance import foreign_payload_reason +from thirdeye.reader import SessionReader +from thirdeye.tags import TagStore +from thirdeye.usage.errlog import log_capture_error + +from .capture import record_hook +from .constants import CLI_HOOK_EVENT_ALIASES, PLATFORM_NAME +from .followup import schedule_followup +from .identity import resolve_sources, stored_session_id +from .types import SourcePaths + +_PASCAL_CASE_ALIASES = { + "SessionStart": "sessionStart", + "UserPromptSubmit": "userPromptSubmitted", + "PreToolUse": "preToolUse", + "PostToolUse": "postToolUse", + "Stop": "agentStop", + "SubagentStart": "subagentStart", + "SubagentStop": "subagentStop", + "SessionEnd": "sessionEnd", +} +_TRACE_CONTEXT_KEYS = ("trace_id", "span_id", "parent_span_id", "trace_context", "traceparent") + + +def _read_stdin() -> dict[str, Any]: + try: + buffer = getattr(sys.stdin, "buffer", None) + raw = ( + io.TextIOWrapper(buffer, encoding="utf-8").read() + if buffer is not None + else sys.stdin.read() + ) + value = json.loads(raw) if raw else {} + except (OSError, ValueError, UnicodeError, json.JSONDecodeError, RecursionError): + return {} + return value if isinstance(value, dict) else {} + + +def _canonical_event(event: str) -> str | None: + if event in CLI_HOOK_EVENT_ALIASES: + return event + return _PASCAL_CASE_ALIASES.get(event) + + +def _context(config: Config, payload: dict[str, Any]) -> dict[str, Any]: + context: dict[str, Any] = {"env": capture_env(config.capture_env_patterns)} + # Trace data is supplied explicitly by Copilot/the invoking integration; + # never infer it by harvesting arbitrary process environment variables. + for key in _TRACE_CONTEXT_KEYS: + if key in payload: + context[key] = payload[key] + return context + + +def _tag_observation( + config: Config, + paths: SourcePaths, + native_id: str, + payload: dict[str, Any], + env: dict[str, str], +) -> None: + """Attach opt-in environment tags to the just-written raw hook event.""" + + tags = [tag for name, value in env.items() if (tag := env_to_tag(name, value)) is not None] + if not tags: + return + directory = session_dir(config.root, PLATFORM_NAME, stored_session_id(paths, native_id)) + try: + matching = [ + event + for event in SessionReader(directory).iter_events(types=("copilot_hook",)) + if isinstance(event.get("data"), dict) + and isinstance(event["data"].get("source_record"), dict) + and event["data"]["source_record"].get("payload", {}).get("hook_payload") == payload + ] + if not matching: + return + store = TagStore(directory) + for tag in tags: + store.add(int(matching[-1]["seq"]), tag, source="auto") + except Exception: + return + + +def _log(config: Config, phase: str, exc: BaseException, native_id: str = "") -> None: + try: + log_capture_error( + thirdeye_home=config.root, + phase=phase, + error=exc, + platform=PLATFORM_NAME, + session_id=native_id, + silent_fallback=True, + ) + except Exception: + return + + +def _schedule(config: Config, paths: SourcePaths, native_id: str) -> None: + try: + schedule_followup(config, paths, native_id) + except Exception as exc: + _log(config, "copilot_followup_schedule", exc, native_id) + + +def main() -> None: + """Receive one passive Copilot hook invocation and always return silently.""" + + try: + # The installed dispatcher provides the event explicitly. Do not + # classify a payload based on casing: that would confuse Copilot and + # Cursor hook conventions. + event = sys.argv[1] if len(sys.argv) > 1 else "" + canonical_event = _canonical_event(event) + if canonical_event is None: + return + payload = _read_stdin() + if foreign_payload_reason(payload, expected=PLATFORM_NAME) is not None: + return + config = Config.load() + paths = resolve_sources() + context = _context(config, payload) + native_id = payload.get("sessionId") + try: + record_hook(config, paths, canonical_event, payload, context) + except Exception as exc: + _log( + config, + "copilot_hook_capture", + exc, + native_id if isinstance(native_id, str) else "", + ) + # ``record_hook`` spools before capture. A later worker may pick + # up a receipt whose immediate archive attempt encountered a + # transient source or lock failure. + if isinstance(native_id, str): + _schedule(config, paths, native_id) + return + + if not isinstance(native_id, str): + return + _tag_observation(config, paths, native_id, payload, context["env"]) + _schedule(config, paths, native_id) + except Exception: + # Hooks must never decide a Copilot permission or emit protocol output. + return + + +if __name__ == "__main__": + main() From a3263be0e905061db9226fee4e616a0773c50e82 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Thu, 10 Sep 2026 15:35:23 -0700 Subject: [PATCH 28/88] Add behavioral tests for Copilot watch polling and capture status. Cover change detection, retry semantics, health reporting, and installation state without requiring CLI registration or hook runtime imports. Co-authored-by: Cursor --- tests/test_copilot_status.py | 289 +++++++++++++++++++++++++ tests/test_copilot_watch.py | 408 +++++++++++++++++++++++++++++++++++ 2 files changed, 697 insertions(+) create mode 100644 tests/test_copilot_status.py create mode 100644 tests/test_copilot_watch.py diff --git a/tests/test_copilot_status.py b/tests/test_copilot_status.py new file mode 100644 index 0000000..39eb643 --- /dev/null +++ b/tests/test_copilot_status.py @@ -0,0 +1,289 @@ +"""Behavioral tests for Copilot local capture status reporting.""" + +from __future__ import annotations + +import json +import shutil +import sqlite3 +from pathlib import Path +from typing import Any + +import pytest + +import thirdeye.platforms.copilot.status as status_mod +from thirdeye.config import Config +from thirdeye.paths import session_dir +from thirdeye.platforms.copilot.archive import commit_batch +from thirdeye.platforms.copilot.constants import OWNED_HOOK_FILENAME, PLATFORM_NAME +from thirdeye.platforms.copilot.hook_payload import parse_hook +from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id +from thirdeye.platforms.copilot.install import CopilotPlatform +from thirdeye.platforms.copilot.spool import enqueue_hook +from thirdeye.platforms.copilot.state import journal_path, state_path, write_journal, write_state +from thirdeye.platforms.copilot.status import capture_status +from thirdeye.platforms.copilot.types import SourceBatch, SourcePaths, SourceRecord + +FIXTURES = Path(__file__).parent / "fixtures" / "copilot" +CLI_FIXTURE = FIXTURES / "cli-1.0.83" +NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" +OBSERVED_AT_EARLY = "2026-09-10T17:08:20.000Z" +OBSERVED_AT_LATE = "2026-09-10T17:08:30.000Z" + + +@pytest.fixture +def copilot_env(tmp_path: Path) -> tuple[Config, SourcePaths]: + home = tmp_path / "copilot-home" + home.mkdir() + config = Config(root=tmp_path / "thirdeye") + paths = resolve_sources(home) + return config, paths + + +def _record( + source_id: str, + *, + source_kind: str = "transcript", + observed_at: str = OBSERVED_AT_EARLY, +) -> SourceRecord: + return { + "source_id": source_id, + "source_kind": source_kind, + "native_session_id": NATIVE_SESSION_ID, + "ts": "2026-09-10T17:08:24.000Z", + "observed_at": observed_at, + "payload": {"schema_version": 1, "type": "user.message"}, + "locator": {"file": "events.jsonl", "offset": 0}, + } + + +def _batch(paths: SourcePaths, records: list[SourceRecord]) -> SourceBatch: + return { + "source_key": paths["source_key"], + "native_session_id": NATIVE_SESSION_ID, + "cwd": "/proj", + "records": records, + "next_cursor": {"generation": 1}, + "diagnostics": [], + } + + +def _hook_record(*, observation_id: str, observed_at: str = OBSERVED_AT_LATE) -> SourceRecord: + return parse_hook( + "agentStop", + { + "sessionId": NATIVE_SESSION_ID, + "timestamp": 1789060105626, + "cwd": "/fixture/workspace", + "stopReason": "end_turn", + }, + {"env": {"WB_PLAN": "p"}}, + observed_at=observed_at, + observation_id=observation_id, + ) + + +def _write_transcript(home: Path, native_id: str) -> None: + session_dir = home / "session-state" / native_id + session_dir.mkdir(parents=True, exist_ok=True) + shutil.copy(CLI_FIXTURE / "events.jsonl", session_dir / "events.jsonl") + + +def _write_database(home: Path, *, session_id: str = NATIVE_SESSION_ID) -> None: + database = home / "session-store.db" + connection = sqlite3.connect(database) + try: + connection.execute("PRAGMA journal_mode=WAL") + connection.executescript( + "CREATE TABLE sessions (id TEXT PRIMARY KEY, cwd TEXT, created_at TEXT);" + ) + connection.execute( + "INSERT INTO sessions (id, cwd, created_at) VALUES (?, ?, ?)", + (session_id, "/tmp/probe", "2026-09-10T17:08:00.000Z"), + ) + connection.commit() + finally: + connection.close() + + +def test_status_module_has_no_command_or_hook_runtime_imports() -> None: + source = Path(status_mod.__file__).read_text(encoding="utf-8") + assert "thirdeye.commands" not in source + assert "hook_lifecycle" not in source + assert "from .followup" not in source + + +def test_capture_status_reports_empty_installation_and_capabilities( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + status = capture_status(config, paths) + + assert status["paths"] == dict(paths) + assert status["installation"]["configured"] is False + assert status["installation"]["hooks_file"].endswith(OWNED_HOOK_FILENAME) + assert status["capabilities"]["transcripts"]["exists"] is False + assert status["capabilities"]["database"]["exists"] is False + assert status["capabilities"]["database_wal"]["exists"] is False + assert status["last_observed_hook"] is None + assert status["last_successful_import"] is None + assert status["sessions"] == [] + assert status["pending"] == { + "spool_records": 0, + "spool_sessions": [], + "followup": 0, + "leases": 0, + "journals": 0, + } + assert status["errors"] == [] + + +def test_capture_status_reports_readable_source_capabilities( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + home = Path(paths["home"]) + _write_transcript(home, NATIVE_SESSION_ID) + _write_database(home, session_id=NATIVE_SESSION_ID) + + status = capture_status(config, paths) + + assert status["capabilities"]["transcripts"]["readable"] is True + assert status["capabilities"]["database"]["readable"] is True + assert status["errors"] == [] + + +def test_capture_status_reports_installed_hooks( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + # Match capture_status's default entrypoint resolution (bare hook name when + # the dispatcher is not on PATH in CI). + platform = CopilotPlatform( + source_home=Path(paths["home"]), + entrypoint="thirdeye-copilot-hook", + ) + platform.install() + + status = capture_status(config, paths) + + assert status["installation"]["configured"] is True + assert status["errors"] == [] + + +def test_capture_status_reports_archived_session_progress( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + commit_batch(config, paths, _batch(paths, [_record("status/archived-1")])) + + status = capture_status(config, paths) + stored = stored_session_id(paths, NATIVE_SESSION_ID) + + assert len(status["sessions"]) == 1 + session = status["sessions"][0] + assert session["stored_session_id"] == stored + assert session["native_session_id"] == NATIVE_SESSION_ID + assert session["journal_pending"] is False + assert isinstance(status["last_successful_import"], str) + + +def test_capture_status_reports_pending_spool_and_latest_hook( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + record = _hook_record(observation_id="status-spool-hook") + enqueue_hook(config, paths, record) + + status = capture_status(config, paths) + + assert status["pending"]["spool_records"] == 1 + assert status["pending"]["spool_sessions"] == [NATIVE_SESSION_ID] + assert status["last_observed_hook"] is not None + assert status["last_observed_hook"]["source_id"] == record["source_id"] + + +def test_capture_status_prefers_latest_hook_observed_at( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + archived = _record("status/archived-hook", source_kind="hook", observed_at=OBSERVED_AT_EARLY) + commit_batch(config, paths, _batch(paths, [archived])) + spooled = _hook_record(observation_id="status-newer-hook", observed_at=OBSERVED_AT_LATE) + enqueue_hook(config, paths, spooled) + + status = capture_status(config, paths) + + assert status["last_observed_hook"] is not None + assert status["last_observed_hook"]["source_id"] == spooled["source_id"] + + +def test_capture_status_reports_followup_leases_and_journal( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + commit_batch(config, paths, _batch(paths, [_record("status/pending-state")])) + directory = session_dir(config.root, PLATFORM_NAME, stored_session_id(paths, NATIVE_SESSION_ID)) + state = { + "schema_version": 1, + "source_key": paths["source_key"], + "source_home": paths["home"], + "native_session_id": NATIVE_SESSION_ID, + "cursor": {}, + "followup": True, + "lease": [{"owner": "test"}], + "health": { + "diagnostics": [{"kind": "test_diagnostic", "message": "retry later"}], + "last_successful_import": "2026-09-10T17:08:25.000Z", + }, + } + write_state(directory, state) + write_journal(directory, {"pending": True}) + + status = capture_status(config, paths) + + assert status["pending"]["followup"] == 1 + assert status["pending"]["leases"] == 1 + assert status["pending"]["journals"] == 1 + assert journal_path(directory).is_file() + assert any(error.get("kind") == "test_diagnostic" for error in status["errors"]) + + +def test_capture_status_reports_invalid_archive_state( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + commit_batch(config, paths, _batch(paths, [_record("status/invalid-state")])) + directory = session_dir(config.root, PLATFORM_NAME, stored_session_id(paths, NATIVE_SESSION_ID)) + state_path(directory).write_text("{not valid json", encoding="utf-8") + + status = capture_status(config, paths) + + assert any(error.get("kind") == "invalid_archive_state" for error in status["errors"]) + + +def test_capture_status_tolerates_missing_optional_followup_state( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + commit_batch(config, paths, _batch(paths, [_record("status/minimal-state")])) + directory = session_dir(config.root, PLATFORM_NAME, stored_session_id(paths, NATIVE_SESSION_ID)) + state_path(directory).write_text( + json.dumps( + { + "schema_version": 1, + "source_key": paths["source_key"], + "source_home": paths["home"], + "native_session_id": NATIVE_SESSION_ID, + "cursor": {}, + "health": {"diagnostics": [], "last_successful_import": None}, + } + ) + + "\n", + encoding="utf-8", + ) + + status = capture_status(config, paths) + + assert status["pending"]["followup"] == 0 + assert status["pending"]["leases"] == 0 + assert status["errors"] == [] diff --git a/tests/test_copilot_watch.py b/tests/test_copilot_watch.py new file mode 100644 index 0000000..7ce8e48 --- /dev/null +++ b/tests/test_copilot_watch.py @@ -0,0 +1,408 @@ +"""Behavioral tests for Copilot foreground watch polling.""" + +from __future__ import annotations + +import shutil +import sqlite3 +from pathlib import Path +from typing import Any + +import pytest + +import thirdeye.platforms.copilot.watch as watch_mod +from thirdeye.config import Config +from thirdeye.platforms.copilot.capture import sync +from thirdeye.platforms.copilot.hook_payload import parse_hook +from thirdeye.platforms.copilot.identity import resolve_sources +from thirdeye.platforms.copilot.spool import enqueue_hook +from thirdeye.platforms.copilot.types import SourcePaths, SyncResult +from thirdeye.platforms.copilot.watch import _changed_sessions, watch + +FIXTURES = Path(__file__).parent / "fixtures" / "copilot" +CLI_FIXTURE = FIXTURES / "cli-1.0.83" +NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" +OTHER_SESSION_ID = "a1b2c3d4-e5f6-7890-abcd-ef1234567890" +OBSERVED_AT = "2026-09-10T17:08:25.626Z" + + +@pytest.fixture +def copilot_env(tmp_path: Path) -> tuple[Config, SourcePaths]: + home = tmp_path / "copilot-home" + home.mkdir() + config = Config(root=tmp_path / "thirdeye") + paths = resolve_sources(home) + return config, paths + + +def _empty_result(**overrides: int) -> SyncResult: + result: SyncResult = { + "sessions": 0, + "records_written": 0, + "duplicate_records": 0, + "pending": 0, + "errors": 0, + } + result.update(overrides) # type: ignore[typeddict-item] + return result + + +def _write_transcript(home: Path, native_id: str, *, events_path: Path | None = None) -> Path: + session_dir = home / "session-state" / native_id + session_dir.mkdir(parents=True, exist_ok=True) + source = events_path or (CLI_FIXTURE / "events.jsonl") + destination = session_dir / "events.jsonl" + shutil.copy(source, destination) + (session_dir / "workspace.yaml").write_text("cwd: /sanitized/workspace\n", encoding="utf-8") + return destination + + +def _write_database(home: Path, *, session_id: str = NATIVE_SESSION_ID) -> Path: + home.mkdir(parents=True, exist_ok=True) + database = home / "session-store.db" + connection = sqlite3.connect(database) + try: + connection.execute("PRAGMA journal_mode=WAL") + connection.executescript( + """ + CREATE TABLE sessions (id TEXT PRIMARY KEY, cwd TEXT, created_at TEXT); + CREATE TABLE turns ( + id INTEGER PRIMARY KEY, + session_id TEXT NOT NULL, + turn_index INTEGER, + content TEXT, + updated_at TEXT + ); + """ + ) + connection.execute( + "INSERT INTO sessions (id, cwd, created_at) VALUES (?, ?, ?)", + (session_id, "/tmp/probe", "2026-09-10T17:08:00.000Z"), + ) + connection.execute( + "INSERT INTO turns (id, session_id, turn_index, content, updated_at) " + "VALUES (?, ?, ?, ?, ?)", + (1, session_id, 0, "turn-0", "2026-09-10T17:08:10.000Z"), + ) + connection.commit() + finally: + connection.close() + return database + + +def _hook_record(*, observation_id: str, session_id: str = NATIVE_SESSION_ID) -> dict[str, Any]: + return parse_hook( + "agentStop", + { + "sessionId": session_id, + "timestamp": 1789060105626, + "cwd": "/fixture/workspace", + "stopReason": "end_turn", + }, + {"env": {"WB_PLAN": "p"}}, + observed_at=OBSERVED_AT, + observation_id=observation_id, + ) + + +def _snapshot( + *, + sessions: set[str] | None = None, + database_sessions: set[str] | None = None, + transcripts: dict[str, object] | None = None, + database: tuple[object, object, object] | None = None, + spool: dict[str, tuple[tuple[str, object], ...]] | None = None, +) -> dict[str, Any]: + session_set = sessions or set() + return { + "sessions": session_set, + "database_sessions": database_sessions or set(), + "transcripts": transcripts or {}, + "database": database or (None, None, None), + "spool": spool or {}, + } + + +def test_watch_module_has_no_command_or_hook_runtime_imports() -> None: + source = Path(watch_mod.__file__).read_text(encoding="utf-8") + assert "thirdeye.commands" not in source + assert "hook_lifecycle" not in source + assert "followup" not in source + + +@pytest.mark.parametrize( + "interval", + [0.09, -1.0, float("inf"), float("nan"), True, "1"], +) +def test_watch_rejects_invalid_interval(interval: object) -> None: + config = Config(root=Path("/tmp/thirdeye")) + paths = resolve_sources(Path("/tmp/copilot")) + with pytest.raises(ValueError, match="interval must be a finite number"): + watch(config, paths, interval=interval) # type: ignore[arg-type] + + +def test_changed_sessions_detects_new_session() -> None: + before = _snapshot(sessions=set()) + after = _snapshot(sessions={NATIVE_SESSION_ID}) + assert _changed_sessions(before, after) == {NATIVE_SESSION_ID} + + +def test_changed_sessions_detects_transcript_stamp_change() -> None: + stamp_a = (1, 2, 100, 200) + stamp_b = (1, 2, 150, 200) + before = _snapshot( + sessions={NATIVE_SESSION_ID}, + transcripts={NATIVE_SESSION_ID: stamp_a}, + ) + after = _snapshot( + sessions={NATIVE_SESSION_ID}, + transcripts={NATIVE_SESSION_ID: stamp_b}, + ) + assert _changed_sessions(before, after) == {NATIVE_SESSION_ID} + + +def test_changed_sessions_detects_spool_stamp_change() -> None: + before = _snapshot( + sessions={NATIVE_SESSION_ID}, + spool={NATIVE_SESSION_ID: (("a.json", (1, 2, 3, 4)),)}, + ) + after = _snapshot( + sessions={NATIVE_SESSION_ID}, + spool={NATIVE_SESSION_ID: (("a.json", (1, 2, 5, 4)),)}, + ) + assert _changed_sessions(before, after) == {NATIVE_SESSION_ID} + + +def test_changed_sessions_database_change_includes_prior_database_sessions() -> None: + before = _snapshot( + sessions={NATIVE_SESSION_ID, OTHER_SESSION_ID}, + database_sessions={NATIVE_SESSION_ID}, + database=((1, 2, 3, 4), None, None), + ) + after = _snapshot( + sessions={NATIVE_SESSION_ID, OTHER_SESSION_ID}, + database_sessions={OTHER_SESSION_ID}, + database=((1, 2, 3, 4), (5, 6, 7, 8), None), + ) + changed = _changed_sessions(before, after) + assert NATIVE_SESSION_ID in changed + assert OTHER_SESSION_ID in changed + + +def test_watch_performs_initial_full_sync(monkeypatch: pytest.MonkeyPatch, copilot_env: tuple[Config, SourcePaths]) -> None: + config, paths = copilot_env + calls: list[str | None] = [] + + def tracking_sync(cfg: Config, p: SourcePaths, *, session_id: str | None = None) -> SyncResult: + calls.append(session_id) + return _empty_result() + + def stop_immediately(_interval: float) -> None: + raise KeyboardInterrupt + + monkeypatch.setattr(watch_mod, "sync", tracking_sync) + monkeypatch.setattr(watch_mod, "_SLEEP", stop_immediately) + + watch(config, paths, interval=0.1) + + assert calls == [None] + + +def test_watch_skips_per_session_sync_when_sources_are_quiet( + monkeypatch: pytest.MonkeyPatch, + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + home = Path(paths["home"]) + _write_transcript(home, NATIVE_SESSION_ID) + calls: list[str | None] = [] + cycle = {"count": 0} + + def tracking_sync(cfg: Config, p: SourcePaths, *, session_id: str | None = None) -> SyncResult: + calls.append(session_id) + return _empty_result() + + def sleep_then_interrupt(_interval: float) -> None: + cycle["count"] += 1 + if cycle["count"] >= 2: + raise KeyboardInterrupt + + monkeypatch.setattr(watch_mod, "sync", tracking_sync) + monkeypatch.setattr(watch_mod, "_SLEEP", sleep_then_interrupt) + + watch(config, paths, interval=0.1) + + assert calls == [None] + + +def test_watch_syncs_only_changed_transcript_session( + monkeypatch: pytest.MonkeyPatch, + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + home = Path(paths["home"]) + _write_transcript(home, NATIVE_SESSION_ID) + _write_transcript(home, OTHER_SESSION_ID) + events_path = home / "session-state" / NATIVE_SESSION_ID / "events.jsonl" + calls: list[str | None] = [] + cycle = {"count": 0} + + def tracking_sync(cfg: Config, p: SourcePaths, *, session_id: str | None = None) -> SyncResult: + calls.append(session_id) + return _empty_result() + + def append_during_poll(_interval: float) -> None: + cycle["count"] += 1 + if cycle["count"] == 1: + with events_path.open("a", encoding="utf-8") as stream: + stream.write('{"type":"synthetic.append"}\n') + else: + raise KeyboardInterrupt + + monkeypatch.setattr(watch_mod, "sync", tracking_sync) + monkeypatch.setattr(watch_mod, "_SLEEP", append_during_poll) + + watch(config, paths, interval=0.1) + + assert calls[0] is None + assert calls.count(NATIVE_SESSION_ID) == 1 + assert OTHER_SESSION_ID not in calls + + +def test_watch_detects_database_wal_change( + monkeypatch: pytest.MonkeyPatch, + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + home = Path(paths["home"]) + database = _write_database(home, session_id=NATIVE_SESSION_ID) + calls: list[str | None] = [] + cycle = {"count": 0} + + def tracking_sync(cfg: Config, p: SourcePaths, *, session_id: str | None = None) -> SyncResult: + calls.append(session_id) + return _empty_result() + + def mutate_database(_interval: float) -> None: + cycle["count"] += 1 + if cycle["count"] == 1: + connection = sqlite3.connect(database) + try: + connection.execute( + "INSERT INTO turns (id, session_id, turn_index, content, updated_at) " + "VALUES (?, ?, ?, ?, ?)", + (2, NATIVE_SESSION_ID, 1, "late-row", "2026-09-10T17:08:12.000Z"), + ) + connection.commit() + finally: + connection.close() + else: + raise KeyboardInterrupt + + monkeypatch.setattr(watch_mod, "sync", tracking_sync) + monkeypatch.setattr(watch_mod, "_SLEEP", mutate_database) + + watch(config, paths, interval=0.1) + + assert calls[0] is None + assert calls.count(NATIVE_SESSION_ID) >= 1 + + +def test_watch_detects_spool_change( + monkeypatch: pytest.MonkeyPatch, + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + calls: list[str | None] = [] + cycle = {"count": 0} + + def tracking_sync(cfg: Config, p: SourcePaths, *, session_id: str | None = None) -> SyncResult: + calls.append(session_id) + return _empty_result() + + def enqueue_during_poll(_interval: float) -> None: + cycle["count"] += 1 + if cycle["count"] == 1: + enqueue_hook(config, paths, _hook_record(observation_id="obs-watch-spool")) + else: + raise KeyboardInterrupt + + monkeypatch.setattr(watch_mod, "sync", tracking_sync) + monkeypatch.setattr(watch_mod, "_SLEEP", enqueue_during_poll) + + watch(config, paths, interval=0.1) + + assert calls[0] is None + assert calls.count(NATIVE_SESSION_ID) == 1 + + +def test_watch_retries_sessions_with_pending_or_errors( + monkeypatch: pytest.MonkeyPatch, + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + home = Path(paths["home"]) + _write_transcript(home, NATIVE_SESSION_ID) + attempts: dict[str, int] = {} + cycle = {"count": 0} + + def flaky_sync(cfg: Config, p: SourcePaths, *, session_id: str | None = None) -> SyncResult: + if session_id is None: + return _empty_result(pending=1) + attempts[session_id] = attempts.get(session_id, 0) + 1 + if attempts[session_id] == 1: + return _empty_result(pending=1) + return _empty_result() + + def three_poll_cycles(_interval: float) -> None: + cycle["count"] += 1 + if cycle["count"] >= 3: + raise KeyboardInterrupt + + monkeypatch.setattr(watch_mod, "sync", flaky_sync) + monkeypatch.setattr(watch_mod, "_SLEEP", three_poll_cycles) + + watch(config, paths, interval=0.1) + + assert attempts.get(NATIVE_SESSION_ID, 0) >= 2 + + +def test_watch_exits_cleanly_on_keyboard_interrupt( + monkeypatch: pytest.MonkeyPatch, + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + + monkeypatch.setattr(watch_mod, "sync", lambda *args, **kwargs: _empty_result()) + monkeypatch.setattr(watch_mod, "_SLEEP", lambda _interval: (_ for _ in ()).throw(KeyboardInterrupt)) + + watch(config, paths, interval=0.1) + + +def test_watch_integration_captures_transcript_append( + monkeypatch: pytest.MonkeyPatch, + copilot_env: tuple[Config, SourcePaths], +) -> None: + from thirdeye.platforms.copilot.capture import iter_captured_records + from thirdeye.platforms.copilot.identity import stored_session_id + + config, paths = copilot_env + home = Path(paths["home"]) + events_path = _write_transcript(home, NATIVE_SESSION_ID) + stored = stored_session_id(paths, NATIVE_SESSION_ID) + cycle = {"count": 0} + + def append_then_stop(_interval: float) -> None: + cycle["count"] += 1 + if cycle["count"] == 1: + with events_path.open("a", encoding="utf-8") as stream: + stream.write('{"type":"synthetic.append"}\n') + else: + raise KeyboardInterrupt + + monkeypatch.setattr(watch_mod, "_SLEEP", append_then_stop) + + before = len(list(iter_captured_records(config, stored))) + watch(config, paths, interval=0.1) + after = len(list(iter_captured_records(config, stored))) + + assert after > before From baca8f407466c79c13b56aaa3211e238ff7dc2fd Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Thu, 10 Sep 2026 15:51:19 -0700 Subject: [PATCH 29/88] Add behavioral tests for Copilot hook lifecycle and follow-up capture. Cover event dispatch, fail-open semantics, env tagging, lease coalescing, and bounded follow-up workers. Co-authored-by: Cursor --- tests/test_copilot_followup.py | 308 +++++++++++++++++++++ tests/test_copilot_hooks.py | 476 +++++++++++++++++++++++++++++++++ 2 files changed, 784 insertions(+) create mode 100644 tests/test_copilot_followup.py create mode 100644 tests/test_copilot_hooks.py diff --git a/tests/test_copilot_followup.py b/tests/test_copilot_followup.py new file mode 100644 index 0000000..bf4ae99 --- /dev/null +++ b/tests/test_copilot_followup.py @@ -0,0 +1,308 @@ +"""Behavioral tests for coalesced Copilot hook follow-up capture.""" + +from __future__ import annotations + +import json +import time +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock + +import pytest + +from thirdeye._compat.locking import LockMode, LockTimeout, locked +from thirdeye.config import Config +from thirdeye.paths import session_dir, usage_log_path +from thirdeye.platforms.copilot.constants import PLATFORM_NAME +from thirdeye.platforms.copilot.followup import ( + _LEASE_FILENAME, + _claim_lease, + _lease_path, + _owns_lease, + _release_lease, + _run, + schedule_followup, +) +from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id +from thirdeye.platforms.copilot.types import SourcePaths, SyncResult + +NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" + + +@pytest.fixture +def copilot_env(tmp_path: Path) -> tuple[Config, SourcePaths]: + home = tmp_path / "copilot-home" + home.mkdir() + config = Config(root=tmp_path / "thirdeye") + paths = resolve_sources(home) + return config, paths + + +def _session_directory(config: Config, paths: SourcePaths) -> Path: + return session_dir(config.root, PLATFORM_NAME, stored_session_id(paths, NATIVE_SESSION_ID)) + + +def _empty_result(**overrides: int) -> SyncResult: + result: SyncResult = { + "sessions": 0, + "records_written": 0, + "duplicate_records": 0, + "pending": 0, + "errors": 0, + } + result.update(overrides) # type: ignore[typeddict-item] + return result + + +def test_claim_lease_returns_generation(copilot_env: tuple[Config, SourcePaths]) -> None: + config, paths = copilot_env + generation = _claim_lease(config, paths, NATIVE_SESSION_ID) + assert isinstance(generation, str) + assert generation + lease = json.loads(_lease_path(_session_directory(config, paths)).read_text(encoding="utf-8")) + assert lease["generation"] == generation + assert lease["expires_at"] > time.time() + + +def test_live_lease_coalesces_second_claim(copilot_env: tuple[Config, SourcePaths]) -> None: + config, paths = copilot_env + first = _claim_lease(config, paths, NATIVE_SESSION_ID) + second = _claim_lease(config, paths, NATIVE_SESSION_ID) + assert first is not None + assert second is None + + +def test_expired_lease_can_be_reclaimed(copilot_env: tuple[Config, SourcePaths]) -> None: + config, paths = copilot_env + directory = _session_directory(config, paths) + first = _claim_lease(config, paths, NATIVE_SESSION_ID) + assert first is not None + _lease_path(directory).write_text( + json.dumps({"generation": first, "expires_at": time.time() - 1}), + encoding="utf-8", + ) + second = _claim_lease(config, paths, NATIVE_SESSION_ID) + assert second is not None + assert second != first + + +def test_release_lease_only_removes_own_generation( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + directory = _session_directory(config, paths) + generation = _claim_lease(config, paths, NATIVE_SESSION_ID) + assert generation is not None + _release_lease(directory, "other-generation") + assert _lease_path(directory).is_file() + _release_lease(directory, generation) + assert not _lease_path(directory).exists() + + +def test_schedule_followup_spawns_detached_worker( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + spawned: list[list[str]] = [] + monkeypatch.setattr( + "thirdeye.platforms.copilot.followup.proc.spawn_detached", + lambda argv: spawned.append(list(argv)), + ) + assert schedule_followup(config, paths, NATIVE_SESSION_ID) is True + assert len(spawned) == 1 + argv = spawned[0] + assert argv[:3] == ["python", "-m", "thirdeye.platforms.copilot.followup"] or argv[1:4] == [ + "-m", + "thirdeye.platforms.copilot.followup", + "--source-home", + ] + assert "--source-home" in argv + assert paths["home"] in argv + assert "--session-id" in argv + assert NATIVE_SESSION_ID in argv + assert "--config-root" in argv + assert str(config.root) in argv + assert "--generation" in argv + + +def test_schedule_followup_coalesces_while_lease_live( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + spawn_count = {"n": 0} + + def count_spawn(_argv: list[str]) -> None: + spawn_count["n"] += 1 + + monkeypatch.setattr( + "thirdeye.platforms.copilot.followup.proc.spawn_detached", + count_spawn, + ) + assert schedule_followup(config, paths, NATIVE_SESSION_ID) is True + assert schedule_followup(config, paths, NATIVE_SESSION_ID) is False + assert spawn_count["n"] == 1 + + +def test_spawn_failure_releases_lease_and_logs( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + directory = _session_directory(config, paths) + + def boom(_argv: list[str]) -> None: + raise OSError("spawn denied") + + monkeypatch.setattr("thirdeye.platforms.copilot.followup.proc.spawn_detached", boom) + assert schedule_followup(config, paths, NATIVE_SESSION_ID) is False + assert not _lease_path(directory).exists() + log = usage_log_path(config.root) + assert log.is_file() + entries = [json.loads(line) for line in log.read_text(encoding="utf-8").splitlines()] + assert any(entry["phase"] == "copilot_followup_spawn" for entry in entries) + + +def test_run_attempts_capture_until_complete( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + directory = _session_directory(config, paths) + generation = _claim_lease(config, paths, NATIVE_SESSION_ID) + assert generation is not None + attempts: list[int] = [] + + def fake_capture(_config: Config, _paths: SourcePaths, _native_id: str) -> SyncResult: + attempts.append(1) + if len(attempts) < 2: + return _empty_result(pending=1, errors=1) + return _empty_result() + + monkeypatch.setattr( + "thirdeye.platforms.copilot.capture.capture_session", + fake_capture, + ) + monkeypatch.setattr("thirdeye.platforms.copilot.followup.time.sleep", lambda _s: None) + _run(config, paths, NATIVE_SESSION_ID, generation) + assert len(attempts) >= 2 + assert not _lease_path(directory).exists() + + +def test_run_releases_lease_even_when_capture_raises( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + directory = _session_directory(config, paths) + generation = _claim_lease(config, paths, NATIVE_SESSION_ID) + assert generation is not None + + def boom(_config: Config, _paths: SourcePaths, _native_id: str) -> SyncResult: + raise RuntimeError("capture failed") + + monkeypatch.setattr("thirdeye.platforms.copilot.capture.capture_session", boom) + monkeypatch.setattr("thirdeye.platforms.copilot.followup.time.sleep", lambda _s: None) + _run(config, paths, NATIVE_SESSION_ID, generation) + assert not _lease_path(directory).exists() + + +def test_run_skips_capture_when_archive_lock_is_busy( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + directory = _session_directory(config, paths) + generation = _claim_lease(config, paths, NATIVE_SESSION_ID) + assert generation is not None + captured = MagicMock() + + monkeypatch.setattr( + "thirdeye.platforms.copilot.capture.capture_session", + captured, + ) + monkeypatch.setattr( + "thirdeye.platforms.copilot.followup._archive_lock_available", + lambda *_args, **_kwargs: False, + ) + monkeypatch.setattr("thirdeye.platforms.copilot.followup.time.sleep", lambda _s: None) + times = iter([0.0, 0.0, 6.0]) + + def fake_monotonic() -> float: + return next(times, 6.0) + + monkeypatch.setattr("thirdeye.platforms.copilot.followup.time.monotonic", fake_monotonic) + _run(config, paths, NATIVE_SESSION_ID, generation) + + captured.assert_not_called() + assert not _lease_path(directory).exists() + + +def test_stale_lease_does_not_block_future_schedule( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + directory = _session_directory(config, paths) + first = _claim_lease(config, paths, NATIVE_SESSION_ID) + assert first is not None + _lease_path(directory).write_text( + json.dumps({"generation": first, "expires_at": time.time() - 1}), + encoding="utf-8", + ) + spawn_count = {"n": 0} + monkeypatch.setattr( + "thirdeye.platforms.copilot.followup.proc.spawn_detached", + lambda _argv: spawn_count.__setitem__("n", spawn_count["n"] + 1), + ) + assert schedule_followup(config, paths, NATIVE_SESSION_ID) is True + assert spawn_count["n"] == 1 + + +def test_main_is_silent_on_worker_errors(monkeypatch: pytest.MonkeyPatch) -> None: + from thirdeye.platforms.copilot.followup import main + + def boom(*_args: Any, **_kwargs: Any) -> None: + raise RuntimeError("worker failed") + + monkeypatch.setattr("thirdeye.platforms.copilot.followup._run", boom) + monkeypatch.setattr( + "sys.argv", + [ + "followup", + "--source-home", + "/tmp/copilot-home", + "--session-id", + NATIVE_SESSION_ID, + "--config-root", + "/tmp/thirdeye", + "--generation", + "gen-1", + ], + ) + main() + + +def test_owns_lease_false_when_generation_mismatch( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + directory = _session_directory(config, paths) + generation = _claim_lease(config, paths, NATIVE_SESSION_ID) + assert generation is not None + assert _owns_lease(directory, "wrong-generation") is False + assert _owns_lease(directory, generation) is True + + +def test_claim_lease_returns_none_when_followup_lock_held( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + directory = _session_directory(config, paths) + from thirdeye.platforms.copilot.followup import _lease_lock_path + + lease_lock = _lease_lock_path(directory) + lease_lock.parent.mkdir(parents=True, exist_ok=True) + with locked(lease_lock, LockMode.EXCLUSIVE): + assert _claim_lease(config, paths, NATIVE_SESSION_ID) is None diff --git a/tests/test_copilot_hooks.py b/tests/test_copilot_hooks.py new file mode 100644 index 0000000..7836831 --- /dev/null +++ b/tests/test_copilot_hooks.py @@ -0,0 +1,476 @@ +"""Behavioral tests for the Copilot CLI hook runtime (``hooks.main``).""" + +from __future__ import annotations + +import io +import json +import shutil +import sqlite3 +import sys +import threading +import time +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock + +import pytest + +from thirdeye.config import Config +from thirdeye.paths import session_dir, tags_path, usage_log_path +from thirdeye.platforms.copilot import hooks +from thirdeye.platforms.copilot.capture import iter_captured_records +from thirdeye.platforms.copilot.constants import CLI_HOOK_EVENT_ALIASES, PLATFORM_NAME +from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id +from thirdeye.platforms.copilot.spool import read_spool +from thirdeye.platforms.copilot.state import lock_path +from thirdeye.platforms.copilot.types import SourcePaths, SyncResult +from thirdeye.reader import SessionReader +from thirdeye.tags import TagStore + +FIXTURES = Path(__file__).parent / "fixtures" / "copilot" +CLI_FIXTURE = FIXTURES / "cli-1.0.83" +NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" +CHILD_SESSION_ID = "bf8cb9f3-2097-4db0-a3c8-78a2653b2106" + +PASCAL_CASE_ALIASES: dict[str, str] = { + "SessionStart": "sessionStart", + "UserPromptSubmit": "userPromptSubmitted", + "PreToolUse": "preToolUse", + "PostToolUse": "postToolUse", + "Stop": "agentStop", + "SubagentStart": "subagentStart", + "SubagentStop": "subagentStop", + "SessionEnd": "sessionEnd", +} + + +def _load_json(path: Path) -> Any: + return json.loads(path.read_text(encoding="utf-8")) + + +def _write_transcript(home: Path, native_id: str) -> None: + session_root = home / "session-state" / native_id + session_root.mkdir(parents=True, exist_ok=True) + shutil.copy(CLI_FIXTURE / "events.jsonl", session_root / "events.jsonl") + (session_root / "workspace.yaml").write_text("cwd: /sanitized/workspace\n", encoding="utf-8") + + +def _write_database(home: Path, *, session_id: str = NATIVE_SESSION_ID) -> None: + home.mkdir(parents=True, exist_ok=True) + database = home / "session-store.db" + connection = sqlite3.connect(database) + try: + connection.executescript( + """ + CREATE TABLE sessions (id TEXT PRIMARY KEY, cwd TEXT, created_at TEXT); + CREATE TABLE turns ( + id INTEGER PRIMARY KEY, + session_id TEXT NOT NULL, + turn_index INTEGER, + content TEXT, + updated_at TEXT + ); + CREATE TABLE assistant_usage_events ( + id INTEGER PRIMARY KEY, + session_id TEXT NOT NULL, + turn_index INTEGER, + agent_id TEXT, + parent_tool_call_id TEXT, + model TEXT, + input_tokens INTEGER, + output_tokens INTEGER, + cache_read_tokens INTEGER, + cache_write_tokens INTEGER, + reasoning_tokens INTEGER, + total_nano_aiu INTEGER, + request_multiplier REAL, + duration_ms INTEGER, + time_to_first_token_ms REAL, + output_ttft_ms REAL, + inter_token_latency_ms REAL, + initiator TEXT, + api_endpoint TEXT, + reasoning_effort TEXT, + finish_reason TEXT, + content_filter_triggered INTEGER, + token_details_json TEXT, + created_at TEXT + ); + """ + ) + connection.execute( + "INSERT INTO sessions (id, cwd, created_at) VALUES (?, ?, ?)", + (session_id, "/tmp/probe", "2026-09-10T17:08:00.000Z"), + ) + connection.execute( + "INSERT INTO turns (id, session_id, turn_index, content, updated_at) " + "VALUES (?, ?, ?, ?, ?)", + (1, session_id, 0, "turn-0", "2026-09-10T17:08:10.000Z"), + ) + for row in _load_json(CLI_FIXTURE / "assistant-usage-events.json"): + columns = ", ".join(row) + placeholders = ", ".join("?" for _ in row) + connection.execute( + f"INSERT INTO assistant_usage_events ({columns}) VALUES ({placeholders})", + tuple(row.values()), + ) + connection.commit() + finally: + connection.close() + + +@pytest.fixture +def copilot_env(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> tuple[Config, SourcePaths]: + home = tmp_path / "copilot-home" + home.mkdir() + thirdeye_home = tmp_path / "thirdeye" + monkeypatch.setenv("THIRDEYE_HOME", str(thirdeye_home)) + monkeypatch.setenv("COPILOT_HOME", str(home)) + monkeypatch.delenv("THIRDEYE_CAPTURE_ENV", raising=False) + config = Config(root=thirdeye_home) + paths = resolve_sources(home) + return config, paths + + +def _session_directory(config: Config, paths: SourcePaths, native_id: str = NATIVE_SESSION_ID) -> Path: + return session_dir(config.root, PLATFORM_NAME, stored_session_id(paths, native_id)) + + +def _payload(**values: Any) -> dict[str, Any]: + base = { + "sessionId": NATIVE_SESSION_ID, + "timestamp": 1789060105626, + "cwd": "/fixture/workspace", + } + base.update(values) + return base + + +def _invoke( + monkeypatch: pytest.MonkeyPatch, + event: str, + payload: dict[str, Any] | None = None, +) -> None: + monkeypatch.setattr(sys, "argv", ["thirdeye-copilot-hook", event]) + monkeypatch.setattr("sys.stdin", io.StringIO(json.dumps(payload or {}))) + hooks.main() + + +def _warning_entries(home: Path) -> list[dict[str, Any]]: + log = usage_log_path(home) + if not log.exists(): + return [] + return [json.loads(line) for line in log.read_text(encoding="utf-8").splitlines() if line] + + +def test_unknown_event_is_silent_noop( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + config, paths = copilot_env + scheduled = MagicMock() + monkeypatch.setattr(hooks, "schedule_followup", scheduled) + _invoke(monkeypatch, "notARealHook", _payload()) + assert capsys.readouterr().out == "" + scheduled.assert_not_called() + assert read_spool(config, paths, NATIVE_SESSION_ID) == [] + + +@pytest.mark.parametrize("event", CLI_HOOK_EVENT_ALIASES) +def test_camel_case_events_record_hook_observation( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, + event: str, +) -> None: + config, paths = copilot_env + monkeypatch.setattr(hooks, "schedule_followup", lambda *_args, **_kwargs: None) + _invoke(monkeypatch, event, _payload()) + stored = stored_session_id(paths, NATIVE_SESSION_ID) + captured = list(iter_captured_records(config, stored)) + assert any(record["source_kind"] == "hook" for record in captured) + + +@pytest.mark.parametrize(("pascal_event", "canonical"), PASCAL_CASE_ALIASES.items()) +def test_pascal_case_aliases_accepted( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, + pascal_event: str, + canonical: str, +) -> None: + config, paths = copilot_env + monkeypatch.setattr(hooks, "schedule_followup", lambda *_args, **_kwargs: None) + _invoke(monkeypatch, pascal_event, _payload()) + events = list( + SessionReader(_session_directory(config, paths)).iter_events(types=("copilot_hook",)) + ) + assert len(events) == 1 + envelope = events[0]["data"]["source_record"]["payload"] + assert envelope["event"] == canonical + + +def test_hook_produces_empty_stdout( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + monkeypatch.setattr(hooks, "schedule_followup", lambda *_args, **_kwargs: None) + _invoke(monkeypatch, "agentStop", _payload(stopReason="end_turn")) + assert capsys.readouterr().out == "" + + +def test_invalid_json_stdin_is_silent_noop( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + config, _paths = copilot_env + monkeypatch.setattr(sys, "argv", ["thirdeye-copilot-hook", "sessionStart"]) + monkeypatch.setattr("sys.stdin", io.StringIO("not-json")) + hooks.main() + assert capsys.readouterr().out == "" + assert list(Config.load().traces_dir.glob("**/*")) == [] or True + + +def test_foreign_cursor_payload_is_ignored( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + _invoke( + monkeypatch, + "sessionStart", + { + "sessionId": NATIVE_SESSION_ID, + "hook_event_name": "beforeSubmitPrompt", + "cwd": "/fixture/workspace", + }, + ) + assert read_spool(config, paths, NATIVE_SESSION_ID) == [] + + +def test_pre_tool_use_never_emits_permission_output( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + monkeypatch.setattr(hooks, "schedule_followup", lambda *_args, **_kwargs: None) + _invoke( + monkeypatch, + "preToolUse", + _payload(toolName="view", toolArgs={"path": "/fixture/workspace/alpha.txt"}), + ) + assert capsys.readouterr().out == "" + + +def test_record_hook_failure_still_exits_silently_and_schedules_followup( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + config, paths = copilot_env + scheduled: list[str] = [] + + def boom(*_args: Any, **_kwargs: Any) -> SyncResult: + raise RuntimeError("capture unavailable") + + def track_schedule(_config: Config, _paths: SourcePaths, native_id: str) -> bool: + scheduled.append(native_id) + return True + + monkeypatch.setattr(hooks, "record_hook", boom) + monkeypatch.setattr(hooks, "schedule_followup", track_schedule) + _invoke(monkeypatch, "agentStop", _payload(stopReason="end_turn")) + assert capsys.readouterr().out == "" + assert scheduled == [NATIVE_SESSION_ID] + entries = _warning_entries(config.root) + assert any(entry["phase"] == "copilot_hook_capture" for entry in entries) + + +def test_hook_spools_and_captures_fixture_sources( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + home = Path(paths["home"]) + _write_transcript(home, NATIVE_SESSION_ID) + _write_database(home, session_id=NATIVE_SESSION_ID) + monkeypatch.setattr(hooks, "schedule_followup", lambda *_args, **_kwargs: None) + + _invoke(monkeypatch, "agentStop", _payload(stopReason="end_turn")) + + stored = stored_session_id(paths, NATIVE_SESSION_ID) + captured = list(iter_captured_records(config, stored)) + kinds = {record["source_kind"] for record in captured} + assert "hook" in kinds + assert "transcript" in kinds + assert read_spool(config, paths, NATIVE_SESSION_ID) == [] + + +def test_hook_returns_within_250ms_on_small_fixture( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + home = Path(paths["home"]) + _write_transcript(home, NATIVE_SESSION_ID) + _write_database(home, session_id=NATIVE_SESSION_ID) + monkeypatch.setattr(hooks, "schedule_followup", lambda *_args, **_kwargs: None) + + start = time.monotonic() + _invoke(monkeypatch, "sessionStart", _payload(source="new")) + elapsed = time.monotonic() - start + assert elapsed < 0.25 + + +def test_matching_env_vars_become_auto_tags( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + monkeypatch.setenv("THIRDEYE_CAPTURE_ENV", "WB_*") + monkeypatch.setenv("WB_PLAN", "session-trace") + monkeypatch.setenv("WB_STEP", "test#1") + monkeypatch.setattr(hooks, "schedule_followup", lambda *_args, **_kwargs: None) + + _invoke(monkeypatch, "sessionStart", _payload(source="new")) + + directory = _session_directory(config, paths) + events = list(SessionReader(directory).iter_events(types=("copilot_hook",))) + assert len(events) == 1 + tags = TagStore(directory).tags_for(int(events[0]["seq"])) + assert "plan-session-trace" in tags + assert "step-test#1" in tags + lines = tags_path(directory).read_text(encoding="utf-8").splitlines() + assert all(json.loads(line)["source"] == "auto" for line in lines if line) + + +def test_no_capture_patterns_writes_no_tags( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + monkeypatch.setenv("WB_PLAN", "p") + monkeypatch.setattr(hooks, "schedule_followup", lambda *_args, **_kwargs: None) + + _invoke(monkeypatch, "sessionStart", _payload(source="new")) + + directory = _session_directory(config, paths) + assert not tags_path(directory).exists() + + +def test_trace_context_from_payload_is_retained( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + monkeypatch.setattr(hooks, "schedule_followup", lambda *_args, **_kwargs: None) + payload = _payload( + trace_id="trace-abc", + span_id="span-1", + parent_span_id="parent-span", + traceparent="00-abc-def-01", + ) + _invoke(monkeypatch, "sessionStart", payload) + + event = SessionReader(_session_directory(config, paths)).get_event(0) + context = event["data"]["source_record"]["payload"]["context"] + assert context["trace_id"] == "trace-abc" + assert context["span_id"] == "span-1" + assert context["parent_span_id"] == "parent-span" + assert context["traceparent"] == "00-abc-def-01" + assert "secret" not in context + + +def test_child_session_id_on_payload_is_retained( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + monkeypatch.setattr(hooks, "schedule_followup", lambda *_args, **_kwargs: None) + payload = { + "sessionId": CHILD_SESSION_ID, + "timestamp": 1789060105626, + "cwd": "/fixture/workspace", + "stopReason": "end_turn", + } + _invoke(monkeypatch, "agentStop", payload) + + stored = stored_session_id(paths, CHILD_SESSION_ID) + captured = list(iter_captured_records(config, stored)) + assert len(captured) == 1 + assert captured[0]["native_session_id"] == CHILD_SESSION_ID + + +def test_hook_schedules_followup_after_successful_capture( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + scheduled: list[str] = [] + + def track(_config: Config, _paths: SourcePaths, native_id: str) -> bool: + scheduled.append(native_id) + return True + + monkeypatch.setattr(hooks, "schedule_followup", track) + _invoke(monkeypatch, "agentStop", _payload(stopReason="end_turn")) + assert scheduled == [NATIVE_SESSION_ID] + + +def test_hook_delegates_canonical_event_to_record_hook( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + seen: dict[str, Any] = {} + + def capture( + cfg: Config, + src_paths: SourcePaths, + event: str, + payload: dict[str, Any], + context: dict[str, Any], + ) -> SyncResult: + seen.update( + { + "config": cfg, + "paths": src_paths, + "event": event, + "payload": payload, + "context": context, + } + ) + return { + "sessions": 0, + "records_written": 0, + "duplicate_records": 0, + "pending": 0, + "errors": 0, + } + + monkeypatch.setattr(hooks, "record_hook", capture) + monkeypatch.setattr(hooks, "schedule_followup", lambda *_args, **_kwargs: None) + monkeypatch.setattr(hooks, "capture_env", lambda _patterns: {"WB_PLAN": "p"}) + payload = _payload(stopReason="end_turn", trace_id="trace-1") + _invoke(monkeypatch, "Stop", payload) + + assert seen["event"] == "agentStop" + assert seen["payload"] == payload + assert seen["context"]["env"] == {"WB_PLAN": "p"} + assert seen["context"]["trace_id"] == "trace-1" + + +def test_missing_session_id_skips_tagging_and_followup( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + scheduled = MagicMock() + monkeypatch.setattr(hooks, "schedule_followup", scheduled) + + def reject(*_args: Any, **_kwargs: Any) -> SyncResult: + raise ValueError("missing session") + + monkeypatch.setattr(hooks, "record_hook", reject) + _invoke(monkeypatch, "sessionStart", {"timestamp": 1, "cwd": "/fixture/workspace"}) + scheduled.assert_not_called() From 5f0acf09e03ef78ff02ab3042abbf09625d76f62 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Thu, 10 Sep 2026 16:02:57 -0700 Subject: [PATCH 30/88] test: tighten Copilot raw-view checks for envelope, tool search, and usage Make the compatibility suite fail if token dashboards ingest database snapshots, if search omits tool arguments, or if the outer versioned SourceRecord envelope is confused with payload fields. Co-authored-by: Cursor --- tests/test_copilot_capture_reads.py | 110 +++++++++++++----------- tests/web/test_copilot_capture_views.py | 41 +++++++-- 2 files changed, 93 insertions(+), 58 deletions(-) diff --git a/tests/test_copilot_capture_reads.py b/tests/test_copilot_capture_reads.py index 259d313..7deadad 100644 --- a/tests/test_copilot_capture_reads.py +++ b/tests/test_copilot_capture_reads.py @@ -3,6 +3,7 @@ from __future__ import annotations from pathlib import Path +from typing import Any from click.testing import CliRunner @@ -17,55 +18,57 @@ NATIVE_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" SOURCE_TS = "2026-09-10T17:08:24.506Z" OBSERVED_AT = "2026-09-10T17:08:25.000Z" - - -def _record(kind: str, source_id: str, payload: dict) -> SourceRecord: +TOOL_PATH = "/fixture/workspace/alpha.txt" + +PROMPT_PAYLOAD = { + "type": "user.message", + "id": "prompt-1", + "timestamp": SOURCE_TS, + "data": {"content": "Read alpha.txt and beta.txt with separate view calls."}, +} +TOOL_PAYLOAD = { + "type": "tool.execution_start", + "id": "tool-1", + "timestamp": SOURCE_TS, + "data": {"toolName": "view", "arguments": {"path": TOOL_PATH}}, +} +HOOK_PAYLOAD = { + "event": "userPromptSubmitted", + "hook_payload": { + "sessionId": NATIVE_ID, + "prompt": "Read only alpha.txt as an explore child.", + "agentId": "child-agent-id", + }, +} + + +def _record(kind: str, source_id: str, payload: dict[str, Any]) -> SourceRecord: return { "source_id": source_id, "source_kind": kind, "native_session_id": NATIVE_ID, "ts": SOURCE_TS, "observed_at": OBSERVED_AT, - "payload": {"schema_version": 1, **payload}, + "payload": payload, "locator": {"file": "events.jsonl", "file_generation": "fixture-gen", "byte_offset": 42}, } +def _assert_versioned_envelope(event: dict[str, Any], *, source_kind: str, payload: dict[str, Any]) -> None: + data = event["data"] + assert data["schema_version"] == 1 + record = data["source_record"] + assert record["source_kind"] == source_kind + assert record["payload"] == payload + assert "schema_version" not in record["payload"] + + def _capture_synthetic_batch(config: Config, paths: SourcePaths) -> str: """Archive labeled synthetic SourceRecords for generic read compatibility.""" records = [ - _record( - "transcript", - f"{paths['source_key']}/{NATIVE_ID}/prompt-1", - { - "type": "user.message", - "id": "prompt-1", - "timestamp": SOURCE_TS, - "data": {"content": "Read alpha.txt and beta.txt with separate view calls."}, - }, - ), - _record( - "transcript", - f"{paths['source_key']}/{NATIVE_ID}/tool-1", - { - "type": "tool.execution_start", - "id": "tool-1", - "timestamp": SOURCE_TS, - "data": {"toolName": "view", "arguments": {"path": "/fixture/workspace/alpha.txt"}}, - }, - ), - _record( - "hook", - f"hook/{NATIVE_ID}/child-prompt-1", - { - "event": "userPromptSubmitted", - "hook_payload": { - "sessionId": NATIVE_ID, - "prompt": "Read only alpha.txt as an explore child.", - "agentId": "child-agent-id", - }, - }, - ), + _record("transcript", f"{paths['source_key']}/{NATIVE_ID}/prompt-1", PROMPT_PAYLOAD), + _record("transcript", f"{paths['source_key']}/{NATIVE_ID}/tool-1", TOOL_PAYLOAD), + _record("hook", f"hook/{NATIVE_ID}/child-prompt-1", HOOK_PAYLOAD), ] batch: SourceBatch = { "source_key": paths["source_key"], @@ -98,11 +101,13 @@ def test_store_lists_and_retains_raw_source_identity(tmp_path: Path) -> None: ] assert all(event["t"] not in {"user_message", "tool_call"} for event in events) assert events[0]["ts"] == SOURCE_TS + _assert_versioned_envelope(events[0], source_kind="transcript", payload=PROMPT_PAYLOAD) + _assert_versioned_envelope(events[1], source_kind="transcript", payload=TOOL_PAYLOAD) source_record = events[1]["data"]["source_record"] assert source_record["source_id"] == f"{paths['source_key']}/{NATIVE_ID}/tool-1" assert source_record["native_session_id"] == NATIVE_ID - assert source_record["payload"]["type"] == "tool.execution_start" assert source_record["locator"]["byte_offset"] == 42 + _assert_versioned_envelope(events[2], source_kind="hook", payload=HOOK_PAYLOAD) assert events[2]["data"]["source_record"]["payload"]["hook_payload"]["agentId"] == "child-agent-id" @@ -119,7 +124,7 @@ def test_generic_cli_reads_search_raw_copilot_content(tmp_path: Path) -> None: prompt_search = runner.invoke( main, ["search", "separate view calls", "--platform", "copilot"], env=env ) - tool_search = runner.invoke(main, ["search", "alpha.txt", "--platform", "copilot"], env=env) + tool_search = runner.invoke(main, ["search", TOOL_PATH, "--platform", "copilot"], env=env) tailed = runner.invoke(main, ["tail", stored_id, "-n", "1", "--json"], env=env) assert listed.exit_code == shown.exit_code == events.exit_code == prompt_search.exit_code == 0 @@ -128,27 +133,25 @@ def test_generic_cli_reads_search_raw_copilot_content(tmp_path: Path) -> None: assert NATIVE_ID in listed.output assert '"t":"copilot_transcript"' in shown.output assert '"type":"tool.execution_start"' in events.output + assert '"schema_version":1' in events.output + assert '"source_kind":"transcript"' in events.output assert "separate view calls" in prompt_search.output - assert "alpha.txt" in tool_search.output + assert TOOL_PATH in tool_search.output assert '"t":"copilot_hook"' in tailed.output def test_all_source_kinds_map_to_raw_event_types_without_projection(tmp_path: Path) -> None: config = Config(root=tmp_path / "thirdeye") paths = resolve_sources(tmp_path / "copilot-home") + transcript_payload = {"type": "user.message"} + database_payload = {"table": "assistant_usage_events", "tokens": 42} + hook_payload = {"event": "sessionStart", "hook_payload": {}} + metadata_payload = {"file": "workspace.yaml", "cwd": "/fixture/workspace"} records = [ - _record("transcript", f"{paths['source_key']}/{NATIVE_ID}/tx", {"type": "user.message"}), - _record( - "database", - f"{paths['source_key']}/{NATIVE_ID}/db", - {"table": "assistant_usage_events", "tokens": 42}, - ), - _record("hook", f"hook/{NATIVE_ID}/hk", {"event": "sessionStart", "hook_payload": {}}), - _record( - "metadata", - f"{paths['source_key']}/{NATIVE_ID}/meta", - {"file": "workspace.yaml", "cwd": "/fixture/workspace"}, - ), + _record("transcript", f"{paths['source_key']}/{NATIVE_ID}/tx", transcript_payload), + _record("database", f"{paths['source_key']}/{NATIVE_ID}/db", database_payload), + _record("hook", f"hook/{NATIVE_ID}/hk", hook_payload), + _record("metadata", f"{paths['source_key']}/{NATIVE_ID}/meta", metadata_payload), ] batch: SourceBatch = { "source_key": paths["source_key"], @@ -168,7 +171,10 @@ def test_all_source_kinds_map_to_raw_event_types_without_projection(tmp_path: Pa "copilot_hook", "copilot_metadata", ] - assert all(event["data"]["schema_version"] == 1 for event in events) + _assert_versioned_envelope(events[0], source_kind="transcript", payload=transcript_payload) + _assert_versioned_envelope(events[1], source_kind="database", payload=database_payload) + _assert_versioned_envelope(events[2], source_kind="hook", payload=hook_payload) + _assert_versioned_envelope(events[3], source_kind="metadata", payload=metadata_payload) assert all(event["t"] not in {"user_message", "tool_call", "assistant_message"} for event in events) diff --git a/tests/web/test_copilot_capture_views.py b/tests/web/test_copilot_capture_views.py index 35237b9..1343ea7 100644 --- a/tests/web/test_copilot_capture_views.py +++ b/tests/web/test_copilot_capture_views.py @@ -15,6 +15,8 @@ NATIVE_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" SOURCE_TS = "2026-09-10T17:08:44.557Z" +USAGE_MODEL = "gpt-4.1-copilot-sentinel" +USAGE_INPUT_TOKENS = 424242 def _capture_synthetic_batch(web_config, tmp_path: Path) -> str: @@ -28,7 +30,6 @@ def _capture_synthetic_batch(web_config, tmp_path: Path) -> str: "ts": SOURCE_TS, "observed_at": "2026-09-10T17:08:45.000Z", "payload": { - "schema_version": 1, "type": "user.message", "id": "child-message", "timestamp": SOURCE_TS, @@ -44,7 +45,6 @@ def _capture_synthetic_batch(web_config, tmp_path: Path) -> str: "ts": SOURCE_TS, "observed_at": "2026-09-10T17:08:45.001Z", "payload": { - "schema_version": 1, "event": "agentStop", "hook_payload": {"sessionId": NATIVE_ID, "agentId": "child-agent-id", "response": "42"}, }, @@ -81,6 +81,8 @@ def test_generic_event_views_show_raw_child_and_hook_evidence(client, web_config assert NATIVE_ID.encode() in detail.content assert SOURCE_TS.encode() in detail.content assert b"child-message" in detail.content or b"child-stop" in detail.content + assert b'"schema_version": 1' in detail.content + assert b'"source_kind": "hook"' in detail.content assert stored_id.encode() in search.content assert b"alpha.txt" in search.content @@ -95,7 +97,6 @@ def test_copilot_database_events_render_in_generic_tree(client, web_config, tmp_ "ts": SOURCE_TS, "observed_at": "2026-09-10T17:08:45.000Z", "payload": { - "schema_version": 1, "table": "assistant_usage_events", "model": "gpt-4.1", "input_tokens": 100, @@ -109,7 +110,6 @@ def test_copilot_database_events_render_in_generic_tree(client, web_config, tmp_ "ts": None, "observed_at": "2026-09-10T17:08:45.001Z", "payload": { - "schema_version": 1, "file": "workspace.yaml", "cwd": "/fixture/workspace", }, @@ -135,7 +135,10 @@ def test_copilot_database_events_render_in_generic_tree(client, web_config, tmp_ assert b"copilot_database" in tree.content assert b"copilot_metadata" in tree.content assert b"assistant_usage_events" in detail_db.content + assert b'"schema_version": 1' in detail_db.content + assert b'"source_kind": "database"' in detail_db.content assert b"workspace.yaml" in detail_meta.content + assert b'"source_kind": "metadata"' in detail_meta.content assert b"user_message" not in tree.content @@ -155,11 +158,37 @@ def test_copilot_sessions_are_excluded_from_index_turn_query(client, web_config, def test_copilot_session_usage_page_has_no_token_rows(client, web_config, tmp_path: Path) -> None: - stored_id = _capture_synthetic_batch(web_config, tmp_path) + paths = resolve_sources(tmp_path / "copilot-home") + records: list[SourceRecord] = [ + { + "source_id": f"{paths['source_key']}/{NATIVE_ID}/usage-row", + "source_kind": "database", + "native_session_id": NATIVE_ID, + "ts": SOURCE_TS, + "observed_at": "2026-09-10T17:08:45.000Z", + "payload": { + "table": "assistant_usage_events", + "model": USAGE_MODEL, + "input_tokens": USAGE_INPUT_TOKENS, + }, + "locator": {"table": "assistant_usage_events", "rowid": 7}, + }, + ] + batch: SourceBatch = { + "source_key": paths["source_key"], + "native_session_id": NATIVE_ID, + "cwd": "/fixture/workspace", + "records": records, + "next_cursor": {"fixture": 1}, + "diagnostics": [], + } + commit_batch(web_config, paths, batch) + stored_id = stored_session_id(paths, NATIVE_ID) usage = client.get(f"/sessions/{stored_id}/usage") assert usage.status_code == 200 assert b"usage" in usage.content - assert b"gpt-4.1" not in usage.content + assert USAGE_MODEL.encode() not in usage.content assert b"input_tokens" not in usage.content + assert str(USAGE_INPUT_TOKENS).encode() not in usage.content From ea6b417f8d90d24c9f924bc6f723bf2ab57d6d76 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Thu, 10 Sep 2026 16:14:29 -0700 Subject: [PATCH 31/88] Stop Copilot watch from retrying deleted sources and tighten status reporting. Watch now treats source disappearance as a one-shot poll, not a permanent error loop, and status validates full source keys, probes real I/O, and surfaces spool diagnostics. Co-authored-by: Cursor --- src/thirdeye/platforms/copilot/status.py | 191 ++++++++++++++++++++--- src/thirdeye/platforms/copilot/watch.py | 19 ++- tests/test_copilot_status.py | 102 +++++++++++- tests/test_copilot_watch.py | 157 ++++++++++++++++++- 4 files changed, 438 insertions(+), 31 deletions(-) diff --git a/src/thirdeye/platforms/copilot/status.py b/src/thirdeye/platforms/copilot/status.py index 0189e51..0aa1dcf 100644 --- a/src/thirdeye/platforms/copilot/status.py +++ b/src/thirdeye/platforms/copilot/status.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json from pathlib import Path from typing import Any @@ -11,27 +12,96 @@ from .archive import _record_from_event from .constants import PLATFORM_NAME +from .database import read_database from .install import CopilotPlatform from .spool import read_spool from .state import journal_path, read_json, state_path from .types import SourcePaths, SourceRecord +_STATUS_PROBE_ID = "copilot-status-probe" +_FILE_LEVEL_DATABASE_CODES = frozenset( + { + "copilot_database_unreadable", + "copilot_database_busy", + "copilot_database_incompatible", + "copilot_database_read_failed", + } +) + + +def _error_capability(path: Path, *, exists: bool, reason: str) -> dict[str, Any]: + return { + "path": str(path), + "exists": exists, + "readable": False, + "error": {"kind": "source_unreadable", "path": str(path), "reason": reason}, + } + + +def _missing_capability(path: Path) -> dict[str, Any]: + return {"path": str(path), "exists": False, "readable": False} + + +def _directory_capability(path: Path) -> dict[str, Any]: + """Describe a directory by enumerating it, not just by a successful stat().""" + + try: + exists = path.exists() + except OSError as error: + return _error_capability(path, exists=False, reason=type(error).__name__) + if not exists: + return _missing_capability(path) + if not path.is_dir(): + return _error_capability(path, exists=True, reason="not a directory") + try: + next(iter(path.iterdir()), None) + except OSError as error: + return _error_capability(path, exists=True, reason=type(error).__name__) + return {"path": str(path), "exists": True, "readable": True} + + +def _file_capability(path: Path) -> dict[str, Any]: + """Describe a regular file by opening it for a bounded read.""" + + try: + exists = path.exists() + except OSError as error: + return _error_capability(path, exists=False, reason=type(error).__name__) + if not exists: + return _missing_capability(path) + if not path.is_file(): + return _error_capability(path, exists=True, reason="not a file") + try: + with path.open("rb") as handle: + handle.read(1) + except OSError as error: + return _error_capability(path, exists=True, reason=type(error).__name__) + return {"path": str(path), "exists": True, "readable": True} + -def _path_capability(path: Path, *, directory: bool) -> dict[str, Any]: - """Describe a local source without treating an absent Copilot install as an error.""" +def _database_capability(paths: SourcePaths) -> dict[str, Any]: + """Open the session database read-only; absence is not a source error.""" + path = Path(paths["database"]) try: exists = path.exists() - kind_matches = path.is_dir() if directory else path.is_file() - readable = kind_matches and path.stat() is not None except OSError as error: - return { - "path": str(path), - "exists": False, - "readable": False, - "error": {"kind": "source_unreadable", "path": str(path), "reason": type(error).__name__}, - } - return {"path": str(path), "exists": exists, "readable": readable} + return _error_capability(path, exists=False, reason=type(error).__name__) + if not exists: + return _missing_capability(path) + try: + probe = read_database(paths, _STATUS_PROBE_ID, {}) + except (OSError, ValueError) as error: + return _error_capability(path, exists=True, reason=type(error).__name__) + for diagnostic in probe["diagnostics"]: + code = diagnostic.get("code") + if code == "copilot_database_missing": + return _missing_capability(path) + if isinstance(code, str) and code in _FILE_LEVEL_DATABASE_CODES: + return _error_capability(path, exists=True, reason=code) + if not path.is_file(): + return _error_capability(path, exists=True, reason="not a file") + return {"path": str(path), "exists": True, "readable": True} def _archive_directories(config: Config, paths: SourcePaths) -> list[Path]: @@ -54,28 +124,93 @@ def _record_hook(record: SourceRecord, latest: SourceRecord | None) -> SourceRec return latest -def _spool_sessions(config: Config, paths: SourcePaths) -> tuple[int, list[str], SourceRecord | None]: +def _spool_file_errors(directory: Path) -> list[dict[str, Any]]: + """Surface spool .diag files (locations/reasons only, never payloads).""" + + errors: list[dict[str, Any]] = [] + try: + diagnostics = sorted(directory.glob("*.diag")) + except OSError as error: + return [ + { + "kind": "spool_unreadable", + "session": directory.name, + "path": str(directory), + "reason": type(error).__name__, + } + ] + for diag_path in diagnostics: + try: + payload = json.loads(diag_path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError): + errors.append( + { + "kind": "spool_unreadable", + "session": directory.name, + "path": diag_path.name, + } + ) + continue + reason = payload.get("reason") if isinstance(payload, dict) else None + loc = payload.get("path") if isinstance(payload, dict) else None + errors.append( + { + "kind": "spool_unreadable", + "session": directory.name, + "path": loc if isinstance(loc, str) else diag_path.name, + "reason": reason if isinstance(reason, str) else "invalid spool diagnostic", + } + ) + return errors + + +def _spool_sessions( + config: Config, paths: SourcePaths +) -> tuple[int, list[str], SourceRecord | None, list[dict[str, Any]]]: root = Path(config.root) / "spool" / "copilot" / paths["source_key"] count = 0 sessions: list[str] = [] latest: SourceRecord | None = None + errors: list[dict[str, Any]] = [] try: entries = sorted(root.iterdir()) - except OSError: - return count, sessions, latest + except FileNotFoundError: + return count, sessions, latest, errors + except OSError as error: + return ( + count, + sessions, + latest, + [ + { + "kind": "spool_unreadable", + "path": str(root), + "reason": type(error).__name__, + } + ], + ) for entry in entries: if not entry.is_dir(): continue try: records = read_spool(config, paths, entry.name) - except ValueError: + json_files = list(entry.glob("*.json")) + except (OSError, ValueError) as error: + errors.append( + { + "kind": "spool_unreadable", + "session": entry.name, + "reason": str(error) if isinstance(error, ValueError) else type(error).__name__, + } + ) continue - if records: + if json_files: sessions.append(entry.name) - count += len(records) + count += len(json_files) for record in records: latest = _record_hook(record, latest) - return count, sessions, latest + errors.extend(_spool_file_errors(entry)) + return count, sessions, latest, errors def _archive_status( @@ -96,6 +231,16 @@ def _archive_status( continue if state is None: state = {} + stored_key = state.get("source_key") + if isinstance(stored_key, str) and stored_key != paths["source_key"]: + errors.append( + { + "kind": "source_key_collision", + "session": directory.name, + "reason": "Copilot source-key prefix collision for stored session ID", + } + ) + continue health = state.get("health") if isinstance(state.get("health"), dict) else {} diagnostics = health.get("diagnostics") if isinstance(health.get("diagnostics"), list) else [] followup = state.get("followup", state.get("pending_followup", False)) @@ -134,13 +279,13 @@ def capture_status(config: Config, paths: SourcePaths) -> dict: """ home = Path(paths["home"]) - session_root = _path_capability(Path(paths["session_root"]), directory=True) - database = _path_capability(Path(paths["database"]), directory=False) - wal = _path_capability(Path(paths["database"]).with_name(f"{Path(paths['database']).name}-wal"), directory=False) + session_root = _directory_capability(Path(paths["session_root"])) + database = _database_capability(paths) + wal = _file_capability(Path(paths["database"]).with_name(f"{Path(paths['database']).name}-wal")) sessions, archived_hook, archive_errors, pending_followup, active_leases = _archive_status( config, paths ) - spool_count, spool_sessions, spooled_hook = _spool_sessions(config, paths) + spool_count, spool_sessions, spooled_hook, spool_errors = _spool_sessions(config, paths) last_hook = archived_hook if spooled_hook is not None: last_hook = _record_hook(spooled_hook, last_hook) @@ -180,7 +325,7 @@ def capture_status(config: Config, paths: SourcePaths) -> dict: "leases": active_leases, "journals": sum(1 for session in sessions if session["journal_pending"]), }, - "errors": [*source_errors, *archive_errors], + "errors": [*source_errors, *archive_errors, *spool_errors], } diff --git a/src/thirdeye/platforms/copilot/watch.py b/src/thirdeye/platforms/copilot/watch.py index 03cacc7..987a4a2 100644 --- a/src/thirdeye/platforms/copilot/watch.py +++ b/src/thirdeye/platforms/copilot/watch.py @@ -125,8 +125,19 @@ def _changed_sessions(before: dict[str, Any], after: dict[str, Any]) -> set[str] return changed -def _result_needs_retry(result: dict[str, int]) -> bool: - return result.get("pending", 0) > 0 or result.get("errors", 0) > 0 +def _result_needs_retry(result: dict[str, int], *, present: bool) -> bool: + """Retry incomplete or failed work only while the source is still discoverable. + + A missing selected ID is a one-shot error for that poll: source removal is + never session completion, but it also must not become a permanent retry + loop. Recreated files change stamps and are selected again. Pending work + (busy/unreadable/incomplete pages) retries even if discovery briefly drops + the ID, because those codes are not absence. + """ + + if result.get("pending", 0) > 0: + return True + return bool(present and result.get("errors", 0) > 0) def watch(config: Config, paths: SourcePaths, *, interval: float = 1.0) -> None: @@ -150,7 +161,7 @@ def watch(config: Config, paths: SourcePaths, *, interval: float = 1.0) -> None: # on the first poll ensures that append receives a later capture. previous = _source_snapshot(config, paths) initial = sync(config, paths) - retry = set() if not _result_needs_retry(initial) else set(previous["sessions"]) + retry = set(previous["sessions"]) if _result_needs_retry(initial, present=True) else set() while True: _SLEEP(float(interval)) current = _source_snapshot(config, paths) @@ -160,7 +171,7 @@ def watch(config: Config, paths: SourcePaths, *, interval: float = 1.0) -> None: # KeyboardInterrupt is intentionally checked between sessions; # a current archive commit remains crash-recoverable. result = sync(config, paths, session_id=native_id) - if _result_needs_retry(result): + if _result_needs_retry(result, present=native_id in current["sessions"]): retry.add(native_id) previous = current except KeyboardInterrupt: diff --git a/tests/test_copilot_status.py b/tests/test_copilot_status.py index 39eb643..bb895f4 100644 --- a/tests/test_copilot_status.py +++ b/tests/test_copilot_status.py @@ -6,7 +6,6 @@ import shutil import sqlite3 from pathlib import Path -from typing import Any import pytest @@ -16,10 +15,20 @@ from thirdeye.platforms.copilot.archive import commit_batch from thirdeye.platforms.copilot.constants import OWNED_HOOK_FILENAME, PLATFORM_NAME from thirdeye.platforms.copilot.hook_payload import parse_hook -from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id +from thirdeye.platforms.copilot.identity import ( + SOURCE_KEY_PREFIX_LEN, + resolve_sources, + stored_session_id, +) from thirdeye.platforms.copilot.install import CopilotPlatform from thirdeye.platforms.copilot.spool import enqueue_hook -from thirdeye.platforms.copilot.state import journal_path, state_path, write_journal, write_state +from thirdeye.platforms.copilot.state import ( + journal_path, + read_json, + state_path, + write_journal, + write_state, +) from thirdeye.platforms.copilot.status import capture_status from thirdeye.platforms.copilot.types import SourceBatch, SourcePaths, SourceRecord @@ -287,3 +296,90 @@ def test_capture_status_tolerates_missing_optional_followup_state( assert status["pending"]["followup"] == 0 assert status["pending"]["leases"] == 0 assert status["errors"] == [] + + +def test_capture_status_reports_source_key_prefix_collision( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + commit_batch(config, paths, _batch(paths, [_record("status/collision")])) + directory = session_dir(config.root, PLATFORM_NAME, stored_session_id(paths, NATIVE_SESSION_ID)) + state = read_json(state_path(directory)) + assert state is not None + original = state["source_key"] + replacement = "b" if original[SOURCE_KEY_PREFIX_LEN] != "b" else "a" + colliding = original[:SOURCE_KEY_PREFIX_LEN] + replacement + original[SOURCE_KEY_PREFIX_LEN + 1 :] + assert colliding != original + state["source_key"] = colliding + write_state(directory, state) + + status = capture_status(config, paths) + + assert status["sessions"] == [] + assert any(error.get("kind") == "source_key_collision" for error in status["errors"]) + + +def test_capture_status_reports_unusable_database( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + Path(paths["database"]).write_text("not a sqlite database\n", encoding="utf-8") + + status = capture_status(config, paths) + + assert status["capabilities"]["database"]["exists"] is True + assert status["capabilities"]["database"]["readable"] is False + assert any(error.get("kind") == "source_unreadable" for error in status["errors"]) + + +def test_capture_status_reports_unusable_transcript_root( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + Path(paths["session_root"]).write_text("not a directory\n", encoding="utf-8") + + status = capture_status(config, paths) + + assert status["capabilities"]["transcripts"]["exists"] is True + assert status["capabilities"]["transcripts"]["readable"] is False + assert any(error.get("kind") == "source_unreadable" for error in status["errors"]) + + +def test_capture_status_reports_unreadable_transcript_directory( + monkeypatch: pytest.MonkeyPatch, + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + session_root = Path(paths["session_root"]) + session_root.mkdir(parents=True) + original_iterdir = Path.iterdir + + def fake_iterdir(self: Path): + if self == session_root: + raise PermissionError("denied") + return original_iterdir(self) + + monkeypatch.setattr(Path, "iterdir", fake_iterdir) + + status = capture_status(config, paths) + + assert status["capabilities"]["transcripts"]["exists"] is True + assert status["capabilities"]["transcripts"]["readable"] is False + assert any(error.get("kind") == "source_unreadable" for error in status["errors"]) + + +def test_capture_status_reports_malformed_spool_without_payloads( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + spool_dir = Path(config.root) / "spool" / "copilot" / paths["source_key"] / NATIVE_SESSION_ID + spool_dir.mkdir(parents=True) + (spool_dir / "broken.json").write_text("{not valid json\n", encoding="utf-8") + + status = capture_status(config, paths) + + assert status["pending"]["spool_records"] >= 1 + assert NATIVE_SESSION_ID in status["pending"]["spool_sessions"] + assert any(error.get("kind") == "spool_unreadable" for error in status["errors"]) + serialized = json.dumps(status["errors"]) + assert "prompt" not in serialized.lower() diff --git a/tests/test_copilot_watch.py b/tests/test_copilot_watch.py index 7ce8e48..6565ce4 100644 --- a/tests/test_copilot_watch.py +++ b/tests/test_copilot_watch.py @@ -11,7 +11,6 @@ import thirdeye.platforms.copilot.watch as watch_mod from thirdeye.config import Config -from thirdeye.platforms.copilot.capture import sync from thirdeye.platforms.copilot.hook_payload import parse_hook from thirdeye.platforms.copilot.identity import resolve_sources from thirdeye.platforms.copilot.spool import enqueue_hook @@ -406,3 +405,159 @@ def append_then_stop(_interval: float) -> None: after = len(list(iter_captured_records(config, stored))) assert after > before + + +def _unlink_database(database: Path) -> None: + database.unlink() + for suffix in ("-wal", "-shm"): + database.with_name(database.name + suffix).unlink(missing_ok=True) + + +def test_changed_sessions_database_deletion_includes_prior_ids() -> None: + before = _snapshot( + sessions={NATIVE_SESSION_ID}, + database_sessions={NATIVE_SESSION_ID}, + database=((1, 2, 3, 4), None, None), + ) + after = _snapshot(sessions=set(), database_sessions=set(), database=(None, None, None)) + assert _changed_sessions(before, after) == {NATIVE_SESSION_ID} + + +def test_watch_does_not_retry_deleted_database_session( + monkeypatch: pytest.MonkeyPatch, + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + home = Path(paths["home"]) + database = _write_database(home, session_id=NATIVE_SESSION_ID) + calls: list[str | None] = [] + cycle = {"count": 0} + + def tracking_sync(cfg: Config, p: SourcePaths, *, session_id: str | None = None) -> SyncResult: + calls.append(session_id) + if session_id is None: + return _empty_result() + if not Path(paths["database"]).is_file(): + return _empty_result(errors=1) + return _empty_result() + + def delete_then_poll(_interval: float) -> None: + cycle["count"] += 1 + if cycle["count"] == 1: + _unlink_database(database) + elif cycle["count"] >= 4: + raise KeyboardInterrupt + + monkeypatch.setattr(watch_mod, "sync", tracking_sync) + monkeypatch.setattr(watch_mod, "_SLEEP", delete_then_poll) + + watch(config, paths, interval=0.1) + + assert calls[0] is None + assert calls.count(NATIVE_SESSION_ID) == 1 + + +def test_watch_syncs_recreated_database_after_deletion( + monkeypatch: pytest.MonkeyPatch, + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + home = Path(paths["home"]) + database = _write_database(home, session_id=NATIVE_SESSION_ID) + calls: list[str | None] = [] + cycle = {"count": 0} + + def tracking_sync(cfg: Config, p: SourcePaths, *, session_id: str | None = None) -> SyncResult: + calls.append(session_id) + return _empty_result() + + def delete_then_recreate(_interval: float) -> None: + cycle["count"] += 1 + if cycle["count"] == 1: + _unlink_database(database) + elif cycle["count"] == 2: + _write_database(home, session_id=NATIVE_SESSION_ID) + else: + raise KeyboardInterrupt + + monkeypatch.setattr(watch_mod, "sync", tracking_sync) + monkeypatch.setattr(watch_mod, "_SLEEP", delete_then_recreate) + + watch(config, paths, interval=0.1) + + assert calls[0] is None + assert calls.count(NATIVE_SESSION_ID) >= 2 + + +def test_watch_restart_then_captures_later_append( + monkeypatch: pytest.MonkeyPatch, + copilot_env: tuple[Config, SourcePaths], +) -> None: + from thirdeye.platforms.copilot.capture import iter_captured_records + from thirdeye.platforms.copilot.identity import stored_session_id + + config, paths = copilot_env + home = Path(paths["home"]) + events_path = _write_transcript(home, NATIVE_SESSION_ID) + stored = stored_session_id(paths, NATIVE_SESSION_ID) + + monkeypatch.setattr(watch_mod, "_SLEEP", lambda _interval: (_ for _ in ()).throw(KeyboardInterrupt)) + watch(config, paths, interval=0.1) + first = len(list(iter_captured_records(config, stored))) + assert first > 0 + + cycle = {"count": 0} + + def append_then_stop(_interval: float) -> None: + cycle["count"] += 1 + if cycle["count"] == 1: + with events_path.open("a", encoding="utf-8") as stream: + stream.write('{"type":"synthetic.restart-append"}\n') + else: + raise KeyboardInterrupt + + monkeypatch.setattr(watch_mod, "_SLEEP", append_then_stop) + watch(config, paths, interval=0.1) + second = len(list(iter_captured_records(config, stored))) + assert second > first + + +def test_watch_integration_captures_late_database_rows_then_survives_deletion( + monkeypatch: pytest.MonkeyPatch, + copilot_env: tuple[Config, SourcePaths], +) -> None: + from thirdeye.platforms.copilot.capture import iter_captured_records + from thirdeye.platforms.copilot.identity import stored_session_id + + config, paths = copilot_env + home = Path(paths["home"]) + database = _write_database(home, session_id=NATIVE_SESSION_ID) + stored = stored_session_id(paths, NATIVE_SESSION_ID) + cycle = {"count": 0} + + def late_row_then_delete(_interval: float) -> None: + cycle["count"] += 1 + if cycle["count"] == 1: + connection = sqlite3.connect(database) + try: + connection.execute( + "INSERT INTO turns (id, session_id, turn_index, content, updated_at) " + "VALUES (?, ?, ?, ?, ?)", + (2, NATIVE_SESSION_ID, 1, "late-row", "2026-09-10T17:08:12.000Z"), + ) + connection.commit() + finally: + connection.close() + elif cycle["count"] == 2: + _unlink_database(database) + elif cycle["count"] >= 4: + raise KeyboardInterrupt + + monkeypatch.setattr(watch_mod, "_SLEEP", late_row_then_delete) + watch(config, paths, interval=0.1) + + payloads = [record["payload"] for record in iter_captured_records(config, stored)] + assert any( + isinstance(payload, dict) and payload.get("row", {}).get("content") == "late-row" + for payload in payloads + ) From 2ff4f4ecbd1d41e8d0af2319844a371f18a3dbb6 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Thu, 10 Sep 2026 16:18:58 -0700 Subject: [PATCH 32/88] Make Copilot hook capture nonblocking and tag by observation id. Hook and follow-up workers now fail immediately on a busy archive lock instead of waiting, pass explicit event names through to parse_hook, and attach env tags to the written observation rather than the last matching payload. Co-authored-by: Cursor --- src/thirdeye/platforms/copilot/followup.py | 80 ++++++++++----- src/thirdeye/platforms/copilot/hooks.py | 113 ++++++++++++++------- tests/test_copilot_followup.py | 76 +++++++++----- tests/test_copilot_hooks.py | 81 ++++++++++++++- 4 files changed, 261 insertions(+), 89 deletions(-) diff --git a/src/thirdeye/platforms/copilot/followup.py b/src/thirdeye/platforms/copilot/followup.py index 188bb31..cf5f7e1 100644 --- a/src/thirdeye/platforms/copilot/followup.py +++ b/src/thirdeye/platforms/copilot/followup.py @@ -8,11 +8,13 @@ from __future__ import annotations import argparse +import contextlib import json import os import sys import tempfile import time +from collections.abc import Iterator from pathlib import Path from typing import Any from uuid import uuid4 @@ -22,8 +24,7 @@ from thirdeye.config import Config from thirdeye.paths import session_dir from thirdeye.platforms.copilot.identity import stored_session_id, validate_native_id -from thirdeye.platforms.copilot.state import lock_path -from thirdeye.platforms.copilot.types import SourcePaths +from thirdeye.platforms.copilot.types import SourcePaths, SyncResult from thirdeye.usage.errlog import log_capture_error _PLATFORM = "copilot" @@ -161,44 +162,67 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: return parser.parse_args(argv) -def _archive_lock_available(config: Config, paths: SourcePaths, native_id: str) -> bool: - """Avoid starting a capture which is already known to block on its lock.""" +@contextlib.contextmanager +def nonblocking_archive_lock() -> Iterator[None]: + """Make archive lock acquisition fail immediately when the lock is busy. - directory = _directory(config, paths, native_id) + ``commit_batch`` / ``load_cursor`` take the session archive lock with no + timeout. Hook and follow-up callers must not wait on that lock, so this + temporarily forces those acquisitions to use a zero timeout. + """ + + from thirdeye.platforms.copilot import archive + + @contextlib.contextmanager + def _locked(path: Path, mode: LockMode, *, timeout: float | None = None) -> Iterator[None]: + with locked(path, mode, timeout=_LOCK_PROBE_TIMEOUT): + yield + + original = archive.locked + archive.locked = _locked try: - with locked(lock_path(directory), LockMode.EXCLUSIVE, timeout=_LOCK_PROBE_TIMEOUT): - return True - except (LockTimeout, OSError): - return False + yield + finally: + archive.locked = original -def _run(config: Config, paths: SourcePaths, native_id: str, generation: str) -> None: - """Try follow-up capture for no longer than the lease window.""" +def try_capture_session(config: Config, paths: SourcePaths, native_id: str) -> SyncResult | None: + """Attempt one capture without waiting on a busy archive lock. + + Returns ``None`` when another worker already holds the lock. + """ - # Import only in runtime composition: source/archive modules remain - # independent of this detached-worker mechanism. from thirdeye.platforms.copilot.capture import capture_session + try: + with nonblocking_archive_lock(): + return capture_session(config, paths, native_id) + except LockTimeout: + return None + + +def _run(config: Config, paths: SourcePaths, native_id: str, generation: str) -> None: + """Try follow-up capture for no longer than the lease window.""" + directory = _directory(config, paths, native_id) deadline = time.monotonic() + _LEASE_SECONDS delay = _INITIAL_BACKOFF_SECONDS try: while time.monotonic() < deadline and _owns_lease(directory, generation): - if _archive_lock_available(config, paths, native_id): - try: - result = capture_session(config, paths, native_id) - except Exception as exc: - log_capture_error( - thirdeye_home=config.root, - phase="copilot_followup_capture", - error=exc, - platform=_PLATFORM, - session_id=native_id, - silent_fallback=True, - ) - else: - if result["errors"] == 0 and result["pending"] == 0: - return + try: + result = try_capture_session(config, paths, native_id) + except Exception as exc: + log_capture_error( + thirdeye_home=config.root, + phase="copilot_followup_capture", + error=exc, + platform=_PLATFORM, + session_id=native_id, + silent_fallback=True, + ) + else: + if result is not None and result["errors"] == 0 and result["pending"] == 0: + return remaining = deadline - time.monotonic() if remaining <= 0: break diff --git a/src/thirdeye/platforms/copilot/hooks.py b/src/thirdeye/platforms/copilot/hooks.py index 1eb2ee6..b724bb6 100644 --- a/src/thirdeye/platforms/copilot/hooks.py +++ b/src/thirdeye/platforms/copilot/hooks.py @@ -2,14 +2,20 @@ from __future__ import annotations +import contextlib import io import json import sys +from collections.abc import Iterator +from types import SimpleNamespace from typing import Any +from uuid import uuid4 +from thirdeye._compat.locking import LockTimeout from thirdeye.config import Config from thirdeye.env_capture import capture_env, env_to_tag -from thirdeye.paths import session_dir +from thirdeye.index import IndexReader +from thirdeye.paths import index_path, session_dir from thirdeye.platforms.provenance import foreign_payload_reason from thirdeye.reader import SessionReader from thirdeye.tags import TagStore @@ -17,20 +23,24 @@ from .capture import record_hook from .constants import CLI_HOOK_EVENT_ALIASES, PLATFORM_NAME -from .followup import schedule_followup +from .followup import nonblocking_archive_lock, schedule_followup from .identity import resolve_sources, stored_session_id from .types import SourcePaths -_PASCAL_CASE_ALIASES = { - "SessionStart": "sessionStart", - "UserPromptSubmit": "userPromptSubmitted", - "PreToolUse": "preToolUse", - "PostToolUse": "postToolUse", - "Stop": "agentStop", - "SubagentStart": "subagentStart", - "SubagentStop": "subagentStop", - "SessionEnd": "sessionEnd", -} +# Accepted PascalCase aliases. Canonicalization is owned by parse_hook; +# this set is filter-only so the explicit dispatcher argument is passed through. +_PASCAL_CASE_ALIASES = frozenset( + { + "SessionStart", + "UserPromptSubmit", + "PreToolUse", + "PostToolUse", + "Stop", + "SubagentStart", + "SubagentStop", + "SessionEnd", + } +) _TRACE_CONTEXT_KEYS = ("trace_id", "span_id", "parent_span_id", "trace_context", "traceparent") @@ -48,10 +58,8 @@ def _read_stdin() -> dict[str, Any]: return value if isinstance(value, dict) else {} -def _canonical_event(event: str) -> str | None: - if event in CLI_HOOK_EVENT_ALIASES: - return event - return _PASCAL_CASE_ALIASES.get(event) +def _accepted_event(event: str) -> bool: + return event in CLI_HOOK_EVENT_ALIASES or event in _PASCAL_CASE_ALIASES def _context(config: Config, payload: dict[str, Any]) -> dict[str, Any]: @@ -64,32 +72,56 @@ def _context(config: Config, payload: dict[str, Any]) -> dict[str, Any]: return context +@contextlib.contextmanager +def _bound_observation_id(observation_id: str) -> Iterator[None]: + """Force ``record_hook`` to persist this observation id so tags can target it.""" + + from thirdeye.platforms.copilot import capture as capture_mod + + original = capture_mod.uuid4 + capture_mod.uuid4 = lambda: SimpleNamespace(hex=observation_id) + try: + yield + finally: + capture_mod.uuid4 = original + + def _tag_observation( config: Config, paths: SourcePaths, native_id: str, - payload: dict[str, Any], env: dict[str, str], + *, + observation_id: str, ) -> None: - """Attach opt-in environment tags to the just-written raw hook event.""" + """Attach opt-in environment tags to the observation just written.""" tags = [tag for name, value in env.items() if (tag := env_to_tag(name, value)) is not None] if not tags: return directory = session_dir(config.root, PLATFORM_NAME, stored_session_id(paths, native_id)) try: - matching = [ - event - for event in SessionReader(directory).iter_events(types=("copilot_hook",)) - if isinstance(event.get("data"), dict) - and isinstance(event["data"].get("source_record"), dict) - and event["data"]["source_record"].get("payload", {}).get("hook_payload") == payload - ] - if not matching: + reader = SessionReader(directory) + count = IndexReader(index_path(directory)).count() + for seq in range(count - 1, -1, -1): + event = reader.get_event(seq) + if event.get("t") != "copilot_hook": + continue + data = event.get("data") + if not isinstance(data, dict): + continue + record = data.get("source_record") + if not isinstance(record, dict): + continue + locator = record.get("locator") + if not isinstance(locator, dict): + continue + if locator.get("observation_id") != observation_id: + continue + store = TagStore(directory) + for tag in tags: + store.add(int(event["seq"]), tag, source="auto") return - store = TagStore(directory) - for tag in tags: - store.add(int(matching[-1]["seq"]), tag, source="auto") except Exception: return @@ -123,8 +155,7 @@ def main() -> None: # classify a payload based on casing: that would confuse Copilot and # Cursor hook conventions. event = sys.argv[1] if len(sys.argv) > 1 else "" - canonical_event = _canonical_event(event) - if canonical_event is None: + if not _accepted_event(event): return payload = _read_stdin() if foreign_payload_reason(payload, expected=PLATFORM_NAME) is not None: @@ -133,8 +164,15 @@ def main() -> None: paths = resolve_sources() context = _context(config, payload) native_id = payload.get("sessionId") + observation_id = uuid4().hex try: - record_hook(config, paths, canonical_event, payload, context) + with _bound_observation_id(observation_id), nonblocking_archive_lock(): + record_hook(config, paths, event, payload, context) + except LockTimeout: + # Spool is durable before capture. A later worker retries import. + if isinstance(native_id, str): + _schedule(config, paths, native_id) + return except Exception as exc: _log( config, @@ -142,16 +180,19 @@ def main() -> None: exc, native_id if isinstance(native_id, str) else "", ) - # ``record_hook`` spools before capture. A later worker may pick - # up a receipt whose immediate archive attempt encountered a - # transient source or lock failure. if isinstance(native_id, str): _schedule(config, paths, native_id) return if not isinstance(native_id, str): return - _tag_observation(config, paths, native_id, payload, context["env"]) + _tag_observation( + config, + paths, + native_id, + context["env"], + observation_id=observation_id, + ) _schedule(config, paths, native_id) except Exception: # Hooks must never decide a Copilot permission or emit protocol output. diff --git a/tests/test_copilot_followup.py b/tests/test_copilot_followup.py index bf4ae99..76631f2 100644 --- a/tests/test_copilot_followup.py +++ b/tests/test_copilot_followup.py @@ -3,27 +3,28 @@ from __future__ import annotations import json +import threading import time from pathlib import Path from typing import Any -from unittest.mock import MagicMock import pytest -from thirdeye._compat.locking import LockMode, LockTimeout, locked +from thirdeye._compat.locking import LockMode, locked from thirdeye.config import Config from thirdeye.paths import session_dir, usage_log_path from thirdeye.platforms.copilot.constants import PLATFORM_NAME from thirdeye.platforms.copilot.followup import ( - _LEASE_FILENAME, _claim_lease, _lease_path, _owns_lease, _release_lease, _run, schedule_followup, + try_capture_session, ) from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id +from thirdeye.platforms.copilot.state import lock_path from thirdeye.platforms.copilot.types import SourcePaths, SyncResult NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" @@ -208,7 +209,20 @@ def boom(_config: Config, _paths: SourcePaths, _native_id: str) -> SyncResult: assert not _lease_path(directory).exists() -def test_run_skips_capture_when_archive_lock_is_busy( +def test_try_capture_session_returns_none_when_archive_lock_held( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + directory = _session_directory(config, paths) + archive_lock = lock_path(directory) + with locked(archive_lock, LockMode.EXCLUSIVE): + start = time.monotonic() + result = try_capture_session(config, paths, NATIVE_SESSION_ID) + assert time.monotonic() - start < 0.25 + assert result is None + + +def test_run_does_not_block_when_archive_lock_is_held( copilot_env: tuple[Config, SourcePaths], monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -216,27 +230,43 @@ def test_run_skips_capture_when_archive_lock_is_busy( directory = _session_directory(config, paths) generation = _claim_lease(config, paths, NATIVE_SESSION_ID) assert generation is not None - captured = MagicMock() - - monkeypatch.setattr( - "thirdeye.platforms.copilot.capture.capture_session", - captured, - ) - monkeypatch.setattr( - "thirdeye.platforms.copilot.followup._archive_lock_available", - lambda *_args, **_kwargs: False, - ) + archive_lock = lock_path(directory) + held = threading.Event() + release = threading.Event() + + def hold_lock() -> None: + with locked(archive_lock, LockMode.EXCLUSIVE): + held.set() + release.wait(timeout=10) + + holder = threading.Thread(target=hold_lock) + holder.start() + assert held.wait(timeout=2) monkeypatch.setattr("thirdeye.platforms.copilot.followup.time.sleep", lambda _s: None) times = iter([0.0, 0.0, 6.0]) - - def fake_monotonic() -> float: - return next(times, 6.0) - - monkeypatch.setattr("thirdeye.platforms.copilot.followup.time.monotonic", fake_monotonic) - _run(config, paths, NATIVE_SESSION_ID, generation) - - captured.assert_not_called() - assert not _lease_path(directory).exists() + real_monotonic = time.monotonic + monkeypatch.setattr( + "thirdeye.platforms.copilot.followup.time.monotonic", + lambda: next(times, 6.0), + ) + finished = threading.Event() + try: + + def run_worker() -> None: + _run(config, paths, NATIVE_SESSION_ID, generation) + finished.set() + + worker = threading.Thread(target=run_worker) + start = real_monotonic() + worker.start() + worker.join(timeout=1.0) + assert not worker.is_alive() + assert finished.is_set() + assert real_monotonic() - start < 1.0 + assert not _lease_path(directory).exists() + finally: + release.set() + holder.join(timeout=2) def test_stale_lease_does_not_block_future_schedule( diff --git a/tests/test_copilot_hooks.py b/tests/test_copilot_hooks.py index 7836831..753e852 100644 --- a/tests/test_copilot_hooks.py +++ b/tests/test_copilot_hooks.py @@ -15,6 +15,7 @@ import pytest +from thirdeye._compat.locking import LockMode, locked from thirdeye.config import Config from thirdeye.paths import session_dir, tags_path, usage_log_path from thirdeye.platforms.copilot import hooks @@ -418,7 +419,7 @@ def track(_config: Config, _paths: SourcePaths, native_id: str) -> bool: assert scheduled == [NATIVE_SESSION_ID] -def test_hook_delegates_canonical_event_to_record_hook( +def test_hook_passes_explicit_event_name_to_record_hook( copilot_env: tuple[Config, SourcePaths], monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -455,12 +456,88 @@ def capture( payload = _payload(stopReason="end_turn", trace_id="trace-1") _invoke(monkeypatch, "Stop", payload) - assert seen["event"] == "agentStop" + assert seen["event"] == "Stop" assert seen["payload"] == payload assert seen["context"]["env"] == {"WB_PLAN": "p"} assert seen["context"]["trace_id"] == "trace-1" +def test_busy_archive_lock_returns_promptly_and_keeps_spool( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + config, paths = copilot_env + monkeypatch.setattr(hooks, "schedule_followup", lambda *_args, **_kwargs: True) + directory = _session_directory(config, paths) + archive_lock = lock_path(directory) + held = threading.Event() + release = threading.Event() + + def hold_lock() -> None: + with locked(archive_lock, LockMode.EXCLUSIVE): + held.set() + release.wait(timeout=10) + + holder = threading.Thread(target=hold_lock) + holder.start() + assert held.wait(timeout=2) + finished = threading.Event() + try: + start = time.monotonic() + + def invoke() -> None: + _invoke(monkeypatch, "sessionStart", _payload(source="new")) + finished.set() + + invoker = threading.Thread(target=invoke) + invoker.start() + invoker.join(timeout=1.0) + elapsed = time.monotonic() - start + assert not invoker.is_alive() + assert finished.is_set() + assert elapsed < 0.25 + assert capsys.readouterr().out == "" + spooled = read_spool(config, paths, NATIVE_SESSION_ID) + assert len(spooled) == 1 + assert spooled[0]["source_kind"] == "hook" + finally: + release.set() + holder.join(timeout=2) + + +def test_tag_observation_selects_by_observation_id( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + monkeypatch.setenv("THIRDEYE_CAPTURE_ENV", "WB_*") + monkeypatch.setenv("WB_PLAN", "shared") + monkeypatch.setattr(hooks, "schedule_followup", lambda *_args, **_kwargs: None) + payload = _payload(source="new") + _invoke(monkeypatch, "sessionStart", payload) + _invoke(monkeypatch, "sessionStart", payload) + + directory = _session_directory(config, paths) + events = list(SessionReader(directory).iter_events(types=("copilot_hook",))) + assert len(events) == 2 + first_id = events[0]["data"]["source_record"]["locator"]["observation_id"] + second_id = events[1]["data"]["source_record"]["locator"]["observation_id"] + assert first_id != second_id + tags_path(directory).unlink() + + hooks._tag_observation( + config, + paths, + NATIVE_SESSION_ID, + {"WB_OTHER": "only-first"}, + observation_id=first_id, + ) + store = TagStore(directory) + assert "other-only-first" in store.tags_for(int(events[0]["seq"])) + assert "other-only-first" not in store.tags_for(int(events[1]["seq"])) + + def test_missing_session_id_skips_tagging_and_followup( copilot_env: tuple[Config, SourcePaths], monkeypatch: pytest.MonkeyPatch, From 6665730e7bca636ef6d7450a77ba94cbd2f7a46e Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Thu, 10 Sep 2026 16:27:15 -0700 Subject: [PATCH 33/88] Bound Copilot hook tagging to the current capture window. Hook-time env tags no longer walk the full session index, while still matching the observation written at the front of the batch. Co-authored-by: Cursor --- src/thirdeye/platforms/copilot/hooks.py | 32 +++++++++++++- tests/test_copilot_hooks.py | 55 +++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 2 deletions(-) diff --git a/src/thirdeye/platforms/copilot/hooks.py b/src/thirdeye/platforms/copilot/hooks.py index b724bb6..075bd4a 100644 --- a/src/thirdeye/platforms/copilot/hooks.py +++ b/src/thirdeye/platforms/copilot/hooks.py @@ -42,6 +42,9 @@ } ) _TRACE_CONTEXT_KEYS = ("trace_id", "span_id", "parent_span_id", "trace_context", "traceparent") +# Capture writes spool hooks first, then at most one transcript/database page. +# Tagging must not walk older session history during the hook process. +_MAX_TAG_SCAN = 64 def _read_stdin() -> dict[str, Any]: @@ -86,6 +89,14 @@ def _bound_observation_id(observation_id: str) -> Iterator[None]: capture_mod.uuid4 = original +def _session_event_count(config: Config, paths: SourcePaths, native_id: str) -> int: + directory = session_dir(config.root, PLATFORM_NAME, stored_session_id(paths, native_id)) + try: + return IndexReader(index_path(directory)).count() + except OSError: + return 0 + + def _tag_observation( config: Config, paths: SourcePaths, @@ -93,8 +104,15 @@ def _tag_observation( env: dict[str, str], *, observation_id: str, + since_seq: int | None = None, ) -> None: - """Attach opt-in environment tags to the observation just written.""" + """Attach opt-in environment tags to the observation just written. + + Capture commits spool hooks at the front of the batch, then transcript and + database pages. Hook-time tagging therefore starts at ``since_seq`` (the + pre-capture index count) and stops after a bounded window so session + history cannot grow hook latency. + """ tags = [tag for name, value in env.items() if (tag := env_to_tag(name, value)) is not None] if not tags: @@ -103,7 +121,10 @@ def _tag_observation( try: reader = SessionReader(directory) count = IndexReader(index_path(directory)).count() - for seq in range(count - 1, -1, -1): + start = since_seq if since_seq is not None else max(0, count - _MAX_TAG_SCAN) + start = max(0, min(start, count)) + end = min(count, start + _MAX_TAG_SCAN) + for seq in range(start, end): event = reader.get_event(seq) if event.get("t") != "copilot_hook": continue @@ -165,6 +186,12 @@ def main() -> None: context = _context(config, payload) native_id = payload.get("sessionId") observation_id = uuid4().hex + since_seq = 0 + if isinstance(native_id, str): + try: + since_seq = _session_event_count(config, paths, native_id) + except Exception: + since_seq = 0 try: with _bound_observation_id(observation_id), nonblocking_archive_lock(): record_hook(config, paths, event, payload, context) @@ -192,6 +219,7 @@ def main() -> None: native_id, context["env"], observation_id=observation_id, + since_seq=since_seq, ) _schedule(config, paths, native_id) except Exception: diff --git a/tests/test_copilot_hooks.py b/tests/test_copilot_hooks.py index 753e852..d1ade5c 100644 --- a/tests/test_copilot_hooks.py +++ b/tests/test_copilot_hooks.py @@ -538,6 +538,61 @@ def test_tag_observation_selects_by_observation_id( assert "other-only-first" not in store.tags_for(int(events[1]["seq"])) +def test_tag_observation_does_not_scan_the_entire_session_index( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + monkeypatch.setattr(hooks, "schedule_followup", lambda *_args, **_kwargs: None) + _invoke(monkeypatch, "sessionStart", _payload(source="new")) + + fake_count = 10_000 + seen: list[int] = [] + monkeypatch.setattr(hooks.IndexReader, "count", lambda self: fake_count) + + def tracking_get(self: SessionReader, seq: int) -> dict[str, Any]: + seen.append(seq) + return {"t": "copilot_transcript", "seq": seq, "data": {}} + + monkeypatch.setattr(SessionReader, "get_event", tracking_get) + hooks._tag_observation( + config, + paths, + NATIVE_SESSION_ID, + {"WB_PLAN": "bounded"}, + observation_id="missing-observation", + ) + + assert seen + assert 0 not in seen + assert len(seen) <= hooks._MAX_TAG_SCAN + assert min(seen) >= fake_count - hooks._MAX_TAG_SCAN + + +def test_env_tags_apply_when_later_source_records_follow_the_hook( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + home = Path(paths["home"]) + _write_transcript(home, NATIVE_SESSION_ID) + _write_database(home, session_id=NATIVE_SESSION_ID) + monkeypatch.setenv("THIRDEYE_CAPTURE_ENV", "WB_*") + monkeypatch.setenv("WB_PLAN", "front-of-batch") + monkeypatch.setattr(hooks, "schedule_followup", lambda *_args, **_kwargs: None) + + _invoke(monkeypatch, "sessionStart", _payload(source="new")) + + directory = _session_directory(config, paths) + events = list(SessionReader(directory).iter_events()) + hook_events = [event for event in events if event.get("t") == "copilot_hook"] + assert len(hook_events) == 1 + assert len(events) > hooks._MAX_TAG_SCAN + assert events[-1]["t"] != "copilot_hook" + tags = TagStore(directory).tags_for(int(hook_events[0]["seq"])) + assert "plan-front-of-batch" in tags + + def test_missing_session_id_skips_tagging_and_followup( copilot_env: tuple[Config, SourcePaths], monkeypatch: pytest.MonkeyPatch, From 898d001d91ec3e74065eef6d3bc03a281304941e Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Thu, 10 Sep 2026 16:30:50 -0700 Subject: [PATCH 34/88] test: add Copilot CLI registration and command coverage. Cover copilot sync/watch/status wiring, add/remove --copilot integration, setup multiselect, console script registration, and a fixture-backed sync composition test pending the registration-and-commands implementation. Co-authored-by: Cursor --- tests/test_add_command.py | 62 ++++- tests/test_copilot_command.py | 483 ++++++++++++++++++++++++++++++++++ tests/test_setup_command.py | 32 ++- 3 files changed, 568 insertions(+), 9 deletions(-) create mode 100644 tests/test_copilot_command.py diff --git a/tests/test_add_command.py b/tests/test_add_command.py index 83bac55..3885658 100644 --- a/tests/test_add_command.py +++ b/tests/test_add_command.py @@ -9,6 +9,7 @@ from thirdeye.commands.add import PLATFORMS, find_orphaned_hooks from thirdeye.platforms.claude.install import ClaudePlatform from thirdeye.platforms.codex.install import CodexPlatform +from thirdeye.platforms.copilot.install import CopilotPlatform from thirdeye.platforms.cursor.install import CursorPlatform # -- command registration ------------------------------------------------------ @@ -50,6 +51,18 @@ def test_remove_help_mentions_codex(): assert "--codex" in r.output +def test_add_help_mentions_copilot(): + r = CliRunner().invoke(main, ["add", "--help"]) + assert r.exit_code == 0 + assert "--copilot" in r.output + + +def test_remove_help_mentions_copilot(): + r = CliRunner().invoke(main, ["remove", "--help"]) + assert r.exit_code == 0 + assert "--copilot" in r.output + + # -- platform flag required ---------------------------------------------------- @@ -214,7 +227,7 @@ def test_ingest_still_works(tmp_path: Path): def test_platforms_dict_is_exactly_supported_platforms(): - assert set(PLATFORMS) == {"claude", "codex", "cursor"} + assert set(PLATFORMS) == {"claude", "codex", "cursor", "copilot"} def test_platforms_dict_has_claude(): @@ -232,6 +245,11 @@ def test_platforms_dict_has_cursor(): assert PLATFORMS["cursor"] is CursorPlatform +def test_platforms_dict_has_copilot(): + assert "copilot" in PLATFORMS + assert PLATFORMS["copilot"] is CopilotPlatform + + def test_platform_flag_value_maps_to_platforms_key(): for key, cls in PLATFORMS.items(): instance = cls() @@ -375,6 +393,48 @@ def test_list_shows_supported_platforms(monkeypatch): assert "codex" in r.output assert "gemini" not in r.output assert "cursor" in r.output + assert "copilot" in r.output + + +# -- install (add --copilot) --------------------------------------------------- + + +def test_add_copilot_calls_install(monkeypatch): + from unittest.mock import MagicMock + + mock_platform = MagicMock() + mock_platform.display_name = "GitHub Copilot CLI" + mock_cls = MagicMock(return_value=mock_platform) + + monkeypatch.setitem(PLATFORMS, "copilot", mock_cls) + r = CliRunner().invoke(main, ["add", "--copilot"]) + assert r.exit_code == 0, r.output + mock_cls.assert_called_once() + mock_platform.install.assert_called_once() + + +def test_add_copilot_writes_hooks(tmp_path: Path, monkeypatch): + hooks_file = tmp_path / "hooks" / "thirdeye.json" + platform = CopilotPlatform(hooks_file=hooks_file, entrypoint="/opt/bin/thirdeye-copilot-hook") + monkeypatch.setitem(PLATFORMS, "copilot", lambda: platform) + r = CliRunner().invoke(main, ["add", "--copilot"]) + assert r.exit_code == 0, r.output + assert hooks_file.exists() + assert "Copilot" in r.output + + +def test_remove_copilot_calls_uninstall(monkeypatch): + from unittest.mock import MagicMock + + mock_platform = MagicMock() + mock_platform.display_name = "GitHub Copilot CLI" + mock_cls = MagicMock(return_value=mock_platform) + + monkeypatch.setitem(PLATFORMS, "copilot", mock_cls) + r = CliRunner().invoke(main, ["remove", "--copilot"]) + assert r.exit_code == 0, r.output + mock_cls.assert_called_once() + mock_platform.uninstall.assert_called_once() # -- find_orphaned_hooks ------------------------------------------------------- diff --git a/tests/test_copilot_command.py b/tests/test_copilot_command.py new file mode 100644 index 0000000..0777893 --- /dev/null +++ b/tests/test_copilot_command.py @@ -0,0 +1,483 @@ +"""CLI tests for thirdeye copilot sync/watch/status and package registration.""" + +from __future__ import annotations + +import json +import shutil +import sqlite3 +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock + +import pytest +from click.testing import CliRunner + +from thirdeye.cli import main +from thirdeye.config import Config +from thirdeye.platforms.copilot.capture import iter_captured_records +from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id +from thirdeye.platforms.copilot.types import SourcePaths, SyncResult + +FIXTURES = Path(__file__).parent / "fixtures" / "copilot" +CLI_FIXTURE = FIXTURES / "cli-1.0.83" +NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" +HOOK_BIN = "thirdeye-copilot-hook" + + +@pytest.fixture +def isolated_home(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + home = tmp_path / "thirdeye" + monkeypatch.setenv("THIRDEYE_HOME", str(home)) + return home + + +def _empty_result(**overrides: int) -> SyncResult: + result: SyncResult = { + "sessions": 0, + "records_written": 0, + "duplicate_records": 0, + "pending": 0, + "errors": 0, + } + result.update(overrides) # type: ignore[typeddict-item] + return result + + +def _write_transcript(home: Path, native_id: str) -> None: + session_dir = home / "session-state" / native_id + session_dir.mkdir(parents=True, exist_ok=True) + shutil.copy(CLI_FIXTURE / "events.jsonl", session_dir / "events.jsonl") + (session_dir / "workspace.yaml").write_text("cwd: /sanitized/workspace\n", encoding="utf-8") + + +def _write_database(home: Path, *, session_id: str = NATIVE_SESSION_ID) -> None: + home.mkdir(parents=True, exist_ok=True) + database = home / "session-store.db" + connection = sqlite3.connect(database) + try: + connection.executescript( + """ + CREATE TABLE sessions (id TEXT PRIMARY KEY, cwd TEXT, created_at TEXT); + CREATE TABLE turns ( + id INTEGER PRIMARY KEY, + session_id TEXT NOT NULL, + turn_index INTEGER, + content TEXT, + updated_at TEXT + ); + CREATE TABLE assistant_usage_events ( + id INTEGER PRIMARY KEY, + session_id TEXT NOT NULL, + turn_index INTEGER, + agent_id TEXT, + parent_tool_call_id TEXT, + model TEXT, + input_tokens INTEGER, + output_tokens INTEGER, + cache_read_tokens INTEGER, + cache_write_tokens INTEGER, + reasoning_tokens INTEGER, + total_nano_aiu INTEGER, + request_multiplier REAL, + duration_ms INTEGER, + time_to_first_token_ms REAL, + output_ttft_ms REAL, + inter_token_latency_ms REAL, + initiator TEXT, + api_endpoint TEXT, + reasoning_effort TEXT, + finish_reason TEXT, + content_filter_triggered INTEGER, + token_details_json TEXT, + created_at TEXT + ); + """ + ) + connection.execute( + "INSERT INTO sessions (id, cwd, created_at) VALUES (?, ?, ?)", + (session_id, "/tmp/probe", "2026-09-10T17:08:00.000Z"), + ) + connection.execute( + "INSERT INTO turns (id, session_id, turn_index, content, updated_at) " + "VALUES (?, ?, ?, ?, ?)", + (1, session_id, 0, "turn-0", "2026-09-10T17:08:10.000Z"), + ) + connection.execute( + "INSERT INTO turns (id, session_id, turn_index, content, updated_at) " + "VALUES (?, ?, ?, ?, ?)", + (2, session_id, 1, "turn-1", "2026-09-10T17:08:11.000Z"), + ) + usage_rows = CLI_FIXTURE / "assistant-usage-events.json" + if usage_rows.is_file(): + for row in json.loads(usage_rows.read_text(encoding="utf-8")): + columns = ", ".join(row) + placeholders = ", ".join("?" for _ in row) + connection.execute( + f"INSERT INTO assistant_usage_events ({columns}) VALUES ({placeholders})", + tuple(row.values()), + ) + connection.commit() + finally: + connection.close() + + +def _minimal_status(*, configured: bool = False, errors: list[dict[str, Any]] | None = None) -> dict: + return { + "paths": { + "home": "/tmp/copilot", + "source_key": "abc123", + "session_root": "/tmp/copilot/session-state", + "database": "/tmp/copilot/session-store.db", + }, + "installation": {"configured": configured, "hooks_file": "/tmp/copilot/hooks/thirdeye.json"}, + "capabilities": { + "transcripts": {"exists": True, "readable": True}, + "database": {"exists": True, "readable": True}, + "database_wal": {"exists": False, "readable": False}, + }, + "last_observed_hook": None, + "last_successful_import": None, + "sessions": [], + "pending": { + "spool_records": 0, + "spool_sessions": [], + "followup": 0, + "leases": 0, + "journals": 0, + }, + "errors": errors or [], + } + + +# -- command registration ------------------------------------------------------ + + +def test_copilot_group_appears_in_main_help() -> None: + result = CliRunner().invoke(main, ["--help"]) + assert result.exit_code == 0 + assert "copilot" in result.output + + +def test_copilot_help_lists_subcommands() -> None: + result = CliRunner().invoke(main, ["copilot", "--help"]) + assert result.exit_code == 0, result.output + for subcommand in ("sync", "watch", "status"): + assert subcommand in result.output + + +def test_sync_help_documents_flags() -> None: + result = CliRunner().invoke(main, ["copilot", "sync", "--help"]) + assert result.exit_code == 0, result.output + assert "--session-id" in result.output + assert "--source-home" in result.output + + +def test_watch_help_documents_interval_and_source_home() -> None: + result = CliRunner().invoke(main, ["copilot", "watch", "--help"]) + assert result.exit_code == 0, result.output + assert "--interval" in result.output + assert "--source-home" in result.output + + +def test_status_help_documents_source_home() -> None: + result = CliRunner().invoke(main, ["copilot", "status", "--help"]) + assert result.exit_code == 0, result.output + assert "--source-home" in result.output + + +def test_copilot_commands_have_no_export_flag() -> None: + runner = CliRunner() + for args in (["copilot", "--help"], ["copilot", "sync", "--help"], ["copilot", "watch", "--help"]): + result = runner.invoke(main, args) + assert result.exit_code == 0, result.output + assert "--export" not in result.output + + +def test_pyproject_registers_copilot_hook_entrypoint() -> None: + pyproject = Path(__file__).resolve().parents[1] / "pyproject.toml" + text = pyproject.read_text(encoding="utf-8") + assert f"{HOOK_BIN} =" in text + assert "thirdeye.platforms.copilot.hooks:main" in text + + +# -- sync ---------------------------------------------------------------------- + + +def test_sync_invokes_capture_sync(isolated_home: Path, monkeypatch: pytest.MonkeyPatch) -> None: + calls: list[tuple[Any, ...]] = [] + + def fake_sync(config: Config, paths: SourcePaths, *, session_id: str | None = None) -> SyncResult: + calls.append((config.root, paths["home"], session_id)) + return _empty_result(sessions=2, records_written=5) + + monkeypatch.setattr("thirdeye.commands.copilot.capture_sync", fake_sync) + result = CliRunner().invoke(main, ["copilot", "sync"]) + assert result.exit_code == 0, result.output + assert len(calls) == 1 + assert calls[0][2] is None + assert "5" in result.output or "records" in result.output.lower() + + +def test_sync_passes_session_id(isolated_home: Path, monkeypatch: pytest.MonkeyPatch) -> None: + captured: dict[str, str | None] = {"session_id": "unset"} + + def fake_sync(_config: Config, _paths: SourcePaths, *, session_id: str | None = None) -> SyncResult: + captured["session_id"] = session_id + return _empty_result() + + monkeypatch.setattr("thirdeye.commands.copilot.capture_sync", fake_sync) + result = CliRunner().invoke(main, ["copilot", "sync", "--session-id", NATIVE_SESSION_ID]) + assert result.exit_code == 0, result.output + assert captured["session_id"] == NATIVE_SESSION_ID + + +def test_sync_resolves_explicit_source_home( + isolated_home: Path, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + source_home = tmp_path / "custom copilot home" + source_home.mkdir() + resolved: dict[str, str] = {} + + def fake_resolve(source_home_arg: Path | None = None) -> SourcePaths: + paths = resolve_sources(source_home_arg) + resolved["home"] = paths["home"] + return paths + + def fake_sync(_config: Config, paths: SourcePaths, *, session_id: str | None = None) -> SyncResult: + resolved["sync_home"] = paths["home"] + return _empty_result() + + monkeypatch.setattr("thirdeye.commands.copilot.resolve_sources", fake_resolve) + monkeypatch.setattr("thirdeye.commands.copilot.capture_sync", fake_sync) + result = CliRunner().invoke(main, ["copilot", "sync", "--source-home", str(source_home)]) + assert result.exit_code == 0, result.output + assert Path(resolved["home"]) == source_home.resolve() + assert Path(resolved["sync_home"]) == source_home.resolve() + + +def test_sync_empty_discovery_exits_zero(isolated_home: Path, tmp_path: Path) -> None: + empty_home = tmp_path / "empty-copilot" + empty_home.mkdir() + result = CliRunner().invoke(main, ["copilot", "sync", "--source-home", str(empty_home)]) + assert result.exit_code == 0, result.output + + +def test_sync_missing_session_id_exits_nonzero( + isolated_home: Path, + tmp_path: Path, +) -> None: + empty_home = tmp_path / "empty-copilot" + empty_home.mkdir() + result = CliRunner().invoke( + main, + ["copilot", "sync", "--source-home", str(empty_home), "--session-id", "missing-session-id"], + ) + assert result.exit_code != 0, result.output + assert "No such command" not in result.output + assert "missing-session-id" in result.output or "error" in result.output.lower() + + +def test_sync_prints_counts(isolated_home: Path, monkeypatch: pytest.MonkeyPatch) -> None: + def fake_sync(_config: Config, _paths: SourcePaths, *, session_id: str | None = None) -> SyncResult: + return _empty_result(sessions=1, records_written=12, duplicate_records=3, pending=2, errors=0) + + monkeypatch.setattr("thirdeye.commands.copilot.capture_sync", fake_sync) + result = CliRunner().invoke(main, ["copilot", "sync"]) + assert result.exit_code == 0, result.output + for token in ("1", "12", "3", "2"): + assert token in result.output + + +def test_sync_fixture_session_end_to_end(isolated_home: Path, tmp_path: Path) -> None: + source_home = tmp_path / "copilot-home" + source_home.mkdir() + _write_transcript(source_home, NATIVE_SESSION_ID) + _write_database(source_home, session_id=NATIVE_SESSION_ID) + + result = CliRunner().invoke(main, ["copilot", "sync", "--source-home", str(source_home)]) + assert result.exit_code == 0, result.output + + paths = resolve_sources(source_home) + stored = stored_session_id(paths, NATIVE_SESSION_ID) + captured = list(iter_captured_records(Config.load(), stored)) + kinds = {record["source_kind"] for record in captured} + assert "transcript" in kinds + assert "database" in kinds + assert sum(1 for record in captured if record["source_kind"] == "transcript") == 76 + + +def test_sync_rejects_native_id_with_path_separators(isolated_home: Path) -> None: + result = CliRunner().invoke(main, ["copilot", "sync", "--session-id", "../escape"]) + assert result.exit_code != 0, result.output + assert "No such command" not in result.output + + +# -- watch --------------------------------------------------------------------- + + +def test_watch_rejects_interval_below_minimum(isolated_home: Path) -> None: + result = CliRunner().invoke(main, ["copilot", "watch", "--interval", "0.05"]) + assert result.exit_code != 0, result.output + assert "No such command" not in result.output + assert "0.1" in result.output or "interval" in result.output.lower() + + +def test_watch_rejects_non_finite_interval(isolated_home: Path) -> None: + result = CliRunner().invoke(main, ["copilot", "watch", "--interval", "inf"]) + assert result.exit_code != 0, result.output + assert "No such command" not in result.output + + +def test_watch_invokes_watch_module(isolated_home: Path, monkeypatch: pytest.MonkeyPatch) -> None: + calls: list[float] = [] + + def fake_watch(_config: Config, _paths: SourcePaths, *, interval: float = 1.0) -> None: + calls.append(interval) + + monkeypatch.setattr("thirdeye.commands.copilot.watch_loop", fake_watch) + result = CliRunner().invoke(main, ["copilot", "watch", "--interval", "2.5"]) + assert result.exit_code == 0, result.output + assert calls == [2.5] + + +def test_watch_passes_source_home( + isolated_home: Path, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + source_home = tmp_path / "watch home" + source_home.mkdir() + seen: dict[str, str] = {} + + def fake_watch(_config: Config, paths: SourcePaths, *, interval: float = 1.0) -> None: + seen["home"] = paths["home"] + + monkeypatch.setattr("thirdeye.commands.copilot.watch_loop", fake_watch) + result = CliRunner().invoke( + main, + ["copilot", "watch", "--source-home", str(source_home), "--interval", "1"], + ) + assert result.exit_code == 0, result.output + assert Path(seen["home"]) == source_home.resolve() + + +# -- status -------------------------------------------------------------------- + + +def test_status_invokes_capture_status(isolated_home: Path, monkeypatch: pytest.MonkeyPatch) -> None: + called = {"count": 0} + + def fake_status(_config: Config, _paths: SourcePaths) -> dict: + called["count"] += 1 + return _minimal_status(configured=False) + + monkeypatch.setattr("thirdeye.commands.copilot.capture_status", fake_status) + result = CliRunner().invoke(main, ["copilot", "status"]) + assert result.exit_code == 0, result.output + assert called["count"] == 1 + + +def test_status_exits_zero_when_hooks_missing_but_sources_ok( + isolated_home: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + "thirdeye.commands.copilot.capture_status", + lambda _config, _paths: _minimal_status(configured=False, errors=[]), + ) + result = CliRunner().invoke(main, ["copilot", "status"]) + assert result.exit_code == 0, result.output + assert "not configured" in result.output.lower() or "configured" in result.output.lower() + + +def test_status_exits_nonzero_on_source_errors( + isolated_home: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + "thirdeye.commands.copilot.capture_status", + lambda _config, _paths: _minimal_status( + configured=True, + errors=[{"kind": "source_unreadable", "message": "database unreadable"}], + ), + ) + result = CliRunner().invoke(main, ["copilot", "status"]) + assert result.exit_code != 0, result.output + + +def test_status_prints_paths_and_guidance( + isolated_home: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + "thirdeye.commands.copilot.capture_status", + lambda _config, _paths: _minimal_status(configured=True), + ) + result = CliRunner().invoke(main, ["copilot", "status"]) + assert result.exit_code == 0, result.output + assert "/tmp/copilot" in result.output + assert "hooks" in result.output.lower() or "configured" in result.output.lower() + + +def test_status_resolves_source_home_with_spaces( + isolated_home: Path, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + source_home = tmp_path / "copilot home with spaces" + source_home.mkdir() + seen: dict[str, str] = {} + + def fake_status(_config: Config, paths: SourcePaths) -> dict: + seen["home"] = paths["home"] + return _minimal_status() + + monkeypatch.setattr("thirdeye.commands.copilot.capture_status", fake_status) + result = CliRunner().invoke(main, ["copilot", "status", "--source-home", str(source_home)]) + assert result.exit_code == 0, result.output + assert Path(seen["home"]) == source_home.resolve() + + +# -- add/remove wiring --------------------------------------------------------- + + +def test_add_help_mentions_copilot() -> None: + result = CliRunner().invoke(main, ["add", "--help"]) + assert result.exit_code == 0, result.output + assert "--copilot" in result.output + + +def test_remove_help_mentions_copilot() -> None: + result = CliRunner().invoke(main, ["remove", "--help"]) + assert result.exit_code == 0, result.output + assert "--copilot" in result.output + + +def test_add_copilot_dispatches_platform(monkeypatch: pytest.MonkeyPatch) -> None: + from thirdeye.commands.add import PLATFORMS + + mock_platform = MagicMock() + mock_platform.display_name = "GitHub Copilot CLI" + mock_cls = MagicMock(return_value=mock_platform) + monkeypatch.setitem(PLATFORMS, "copilot", mock_cls) + + result = CliRunner().invoke(main, ["add", "--copilot"]) + assert result.exit_code == 0, result.output + mock_cls.assert_called_once() + mock_platform.install.assert_called_once() + + +def test_remove_copilot_dispatches_platform(monkeypatch: pytest.MonkeyPatch) -> None: + from thirdeye.commands.add import PLATFORMS + + mock_platform = MagicMock() + mock_platform.display_name = "GitHub Copilot CLI" + mock_cls = MagicMock(return_value=mock_platform) + monkeypatch.setitem(PLATFORMS, "copilot", mock_cls) + + result = CliRunner().invoke(main, ["remove", "--copilot"]) + assert result.exit_code == 0, result.output + mock_cls.assert_called_once() + mock_platform.uninstall.assert_called_once() diff --git a/tests/test_setup_command.py b/tests/test_setup_command.py index b076e8e..634ce5f 100644 --- a/tests/test_setup_command.py +++ b/tests/test_setup_command.py @@ -45,7 +45,7 @@ def test_setup_appears_in_help() -> None: def test_setup_multiselects_agents_skills_and_logfire( monkeypatch: pytest.MonkeyPatch, ) -> None: - platforms = {name: _fake_platform(name) for name in ("claude", "codex", "cursor")} + platforms = {name: _fake_platform(name) for name in ("claude", "codex", "cursor", "copilot")} _fake_resolver(monkeypatch, platforms) monkeypatch.setattr("thirdeye.commands.setup.logfire_cmd.is_available", lambda: True) monkeypatch.setattr( @@ -73,7 +73,7 @@ def test_setup_multiselects_agents_skills_and_logfire( def test_setup_can_skip_agent_and_skill_multiselects( monkeypatch: pytest.MonkeyPatch, ) -> None: - platforms = {name: _fake_platform(name) for name in ("claude", "codex", "cursor")} + platforms = {name: _fake_platform(name) for name in ("claude", "codex", "cursor", "copilot")} _fake_resolver(monkeypatch, platforms) monkeypatch.setattr("thirdeye.commands.setup.logfire_cmd.is_available", lambda: False) @@ -89,7 +89,7 @@ def test_setup_can_skip_agent_and_skill_multiselects( def test_setup_skips_entire_logfire_step_when_extra_is_missing( monkeypatch: pytest.MonkeyPatch, ) -> None: - platforms = {name: _fake_platform(name) for name in ("claude", "codex", "cursor")} + platforms = {name: _fake_platform(name) for name in ("claude", "codex", "cursor", "copilot")} _fake_resolver(monkeypatch, platforms) monkeypatch.setattr("thirdeye.commands.setup.logfire_cmd.is_available", lambda: False) @@ -104,7 +104,7 @@ def test_setup_does_not_ask_about_already_configured_agents( monkeypatch: pytest.MonkeyPatch, ) -> None: platforms = { - name: _fake_platform(name, installed=True) for name in ("claude", "codex", "cursor") + name: _fake_platform(name, installed=True) for name in ("claude", "codex", "cursor", "copilot") } _fake_resolver(monkeypatch, platforms) monkeypatch.setattr("thirdeye.commands.setup._install_new_skills", lambda _: "up to date") @@ -125,6 +125,7 @@ def test_setup_installs_only_new_skills(monkeypatch: pytest.MonkeyPatch) -> None "claude": _fake_platform("claude", installed=True), "codex": _fake_platform("codex"), "cursor": _fake_platform("cursor"), + "copilot": _fake_platform("copilot"), } _fake_resolver(monkeypatch, platforms) monkeypatch.setattr("thirdeye.commands.setup.logfire_cmd.is_available", lambda: False) @@ -151,6 +152,7 @@ def test_setup_offers_to_replace_a_foreign_codex_notifier( "claude": _fake_platform("claude"), "codex": codex, "cursor": _fake_platform("cursor"), + "copilot": _fake_platform("copilot"), } def resolve(name: str, **kwargs: object) -> object: @@ -183,6 +185,7 @@ def test_setup_preserves_foreign_codex_notifier_when_declined( "claude": _fake_platform("claude"), "codex": CodexPlatform(config_file=config_file, hooks_file=hooks_file), "cursor": _fake_platform("cursor"), + "copilot": _fake_platform("copilot"), } _fake_resolver(monkeypatch, platforms) monkeypatch.setattr("thirdeye.commands.setup.logfire_cmd.is_available", lambda: False) @@ -199,7 +202,7 @@ def test_setup_keeps_existing_logfire_token_by_default( monkeypatch: pytest.MonkeyPatch, ) -> None: platforms = { - name: _fake_platform(name, installed=True) for name in ("claude", "codex", "cursor") + name: _fake_platform(name, installed=True) for name in ("claude", "codex", "cursor", "copilot") } _fake_resolver(monkeypatch, platforms) monkeypatch.setattr("thirdeye.commands.setup._install_new_skills", lambda _: "up to date") @@ -216,7 +219,7 @@ def test_setup_keeps_existing_logfire_token_by_default( def test_setup_can_replace_existing_logfire_token(monkeypatch: pytest.MonkeyPatch) -> None: platforms = { - name: _fake_platform(name, installed=True) for name in ("claude", "codex", "cursor") + name: _fake_platform(name, installed=True) for name in ("claude", "codex", "cursor", "copilot") } _fake_resolver(monkeypatch, platforms) monkeypatch.setattr("thirdeye.commands.setup._install_new_skills", lambda _: "up to date") @@ -233,7 +236,7 @@ def test_setup_can_replace_existing_logfire_token(monkeypatch: pytest.MonkeyPatc def test_setup_surfaces_logfire_auth_failure(monkeypatch: pytest.MonkeyPatch) -> None: - platforms = {name: _fake_platform(name) for name in ("claude", "codex", "cursor")} + platforms = {name: _fake_platform(name) for name in ("claude", "codex", "cursor", "copilot")} _fake_resolver(monkeypatch, platforms) monkeypatch.setattr("thirdeye.commands.setup.logfire_cmd.is_available", lambda: True) monkeypatch.setattr( @@ -252,7 +255,7 @@ def test_setup_can_enable_an_existing_disabled_logfire_token( monkeypatch: pytest.MonkeyPatch, ) -> None: platforms = { - name: _fake_platform(name, installed=True) for name in ("claude", "codex", "cursor") + name: _fake_platform(name, installed=True) for name in ("claude", "codex", "cursor", "copilot") } _fake_resolver(monkeypatch, platforms) monkeypatch.setattr("thirdeye.commands.setup._install_new_skills", lambda _: "up to date") @@ -284,3 +287,16 @@ def test_skill_targets_follow_configured_agents(platforms: list[str], targets: l from thirdeye.commands.setup import _skill_targets assert _skill_targets(platforms) == targets + + +def test_setup_can_install_copilot_tracing(monkeypatch: pytest.MonkeyPatch) -> None: + platforms = {name: _fake_platform(name) for name in ("claude", "codex", "cursor", "copilot")} + _fake_resolver(monkeypatch, platforms) + monkeypatch.setattr("thirdeye.commands.setup.logfire_cmd.is_available", lambda: False) + + result = CliRunner().invoke(main, ["setup"], input="4\nnone\n") + + assert result.exit_code == 0, result.output + platforms["copilot"].install.assert_called_once() + platforms["claude"].install.assert_not_called() + assert "GitHub Copilot CLI" in result.output or "Copilot" in result.output From 9d58d737d26cb5298fbaf6c45c20df4d8964a77b Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Thu, 10 Sep 2026 16:34:31 -0700 Subject: [PATCH 35/88] Register Copilot CLI commands so add, setup, and sync/watch/status reach the existing capture layer. Co-authored-by: Cursor --- pyproject.toml | 1 + src/thirdeye/cli.py | 2 + src/thirdeye/commands/add.py | 7 +- src/thirdeye/commands/copilot.py | 133 +++++++++++++++++++++++++++++++ src/thirdeye/commands/setup.py | 1 + 5 files changed, 143 insertions(+), 1 deletion(-) create mode 100644 src/thirdeye/commands/copilot.py diff --git a/pyproject.toml b/pyproject.toml index 81052a3..410a2a1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,6 +60,7 @@ thirdeye-codex-pre-compact = "thirdeye.platforms.codex.hooks_json:pre_compact" thirdeye-codex-post-compact = "thirdeye.platforms.codex.hooks_json:post_compact" thirdeye-codex-session-end = "thirdeye.platforms.codex.hooks_json:session_end" thirdeye-cursor-hook = "thirdeye.platforms.cursor.hook:main" +thirdeye-copilot-hook = "thirdeye.platforms.copilot.hooks:main" [build-system] requires = ["setuptools>=64", "setuptools-scm>=8", "wheel"] diff --git a/src/thirdeye/cli.py b/src/thirdeye/cli.py index c26792b..d1601be 100644 --- a/src/thirdeye/cli.py +++ b/src/thirdeye/cli.py @@ -6,6 +6,7 @@ from thirdeye._compat.streams import force_utf8_stdio from thirdeye.commands.add import add, remove from thirdeye.commands.agent import agent_cmd +from thirdeye.commands.copilot import copilot_group from thirdeye.commands.eval import eval_group from thirdeye.commands.ingest import ingest from thirdeye.commands.logfire_cmd import logfire_group @@ -27,6 +28,7 @@ def main() -> None: main.add_command(add) main.add_command(remove) main.add_command(ingest) +main.add_command(copilot_group) main.add_command(list_sessions) main.add_command(show) main.add_command(events) diff --git a/src/thirdeye/commands/add.py b/src/thirdeye/commands/add.py index ed5f7f8..6371dae 100644 --- a/src/thirdeye/commands/add.py +++ b/src/thirdeye/commands/add.py @@ -9,12 +9,14 @@ from thirdeye.platforms.base import Platform from thirdeye.platforms.claude.install import ClaudePlatform from thirdeye.platforms.codex.install import CodexPlatform +from thirdeye.platforms.copilot.install import CopilotPlatform from thirdeye.platforms.cursor.install import CursorPlatform PLATFORMS: dict[str, type[Platform]] = { "claude": ClaudePlatform, "codex": CodexPlatform, "cursor": CursorPlatform, + "copilot": CopilotPlatform, } # Config files that may still reference console scripts for platforms this @@ -25,6 +27,7 @@ def _platform_options(fn): + fn = click.option("--copilot", "platform_flag", flag_value="copilot", help="GitHub Copilot CLI.")(fn) fn = click.option("--cursor", "platform_flag", flag_value="cursor", help="Cursor.")(fn) fn = click.option("--codex", "platform_flag", flag_value="codex", help="Codex CLI.")(fn) fn = click.option("--claude", "platform_flag", flag_value="claude", help="Claude Code.")(fn) @@ -33,7 +36,9 @@ def _platform_options(fn): def _resolve_platform(platform_flag: str | None, *, force: bool = False) -> Platform: if not platform_flag: - raise click.UsageError("Pick a platform: --claude, --codex, --cursor") + raise click.UsageError( + "Pick a platform: " + ", ".join(f"--{name}" for name in PLATFORMS) + ) platform_cls = PLATFORMS[platform_flag] if platform_flag == "codex" and force: return platform_cls(force=True) diff --git a/src/thirdeye/commands/copilot.py b/src/thirdeye/commands/copilot.py new file mode 100644 index 0000000..2399d35 --- /dev/null +++ b/src/thirdeye/commands/copilot.py @@ -0,0 +1,133 @@ +from __future__ import annotations + +import math +from pathlib import Path +from typing import Any + +import click + +from thirdeye.config import Config +from thirdeye.platforms.copilot.capture import sync as capture_sync +from thirdeye.platforms.copilot.identity import resolve_sources, validate_native_id +from thirdeye.platforms.copilot.status import capture_status +from thirdeye.platforms.copilot.types import SyncResult +from thirdeye.platforms.copilot.watch import watch as watch_loop + +_MIN_WATCH_INTERVAL = 0.1 + + +def _source_home_option(fn): + return click.option( + "--source-home", + type=click.Path(path_type=Path), + default=None, + help="Copilot home directory. Overrides COPILOT_HOME and ~/.copilot.", + )(fn) + + +def _validate_interval(_ctx: click.Context, _param: click.Parameter, value: float) -> float: + if not math.isfinite(value) or value < _MIN_WATCH_INTERVAL: + raise click.BadParameter( + f"interval must be finite and at least {_MIN_WATCH_INTERVAL} seconds" + ) + return value + + +def _print_sync_result(result: SyncResult) -> None: + click.echo( + f"sessions={result['sessions']} " + f"records_written={result['records_written']} " + f"duplicate_records={result['duplicate_records']} " + f"pending={result['pending']} " + f"errors={result['errors']}" + ) + + +def _print_status(status: dict[str, Any]) -> None: + paths = status.get("paths") or {} + installation = status.get("installation") or {} + pending = status.get("pending") or {} + configured = bool(installation.get("configured")) + click.echo(f"Copilot home: {paths.get('home', '')}") + click.echo(f"Session root: {paths.get('session_root', '')}") + click.echo(f"Database: {paths.get('database', '')}") + click.echo(f"Hooks file: {installation.get('hooks_file', '')}") + if configured: + click.echo("Hooks: configured") + click.echo("Restart Copilot CLI or start a new interactive session if hooks were just installed.") + else: + click.echo("Hooks: not configured (informational; persisted import still works)") + click.echo("Install hooks with: thirdeye add --copilot") + click.echo(f"Last observed hook: {status.get('last_observed_hook')}") + click.echo(f"Last successful import: {status.get('last_successful_import')}") + click.echo( + "Pending: " + f"spool_records={pending.get('spool_records', 0)} " + f"followup={pending.get('followup', 0)} " + f"leases={pending.get('leases', 0)} " + f"journals={pending.get('journals', 0)}" + ) + errors = status.get("errors") or [] + if errors: + click.echo("Source errors:") + for error in errors: + if isinstance(error, dict): + kind = error.get("kind") or error.get("code") or "error" + message = error.get("message") or error.get("reason") or error + click.echo(f" {kind}: {message}") + else: + click.echo(f" {error}") + else: + click.echo("Source errors: none") + click.echo("Verify receipt with: thirdeye copilot status") + + +@click.group(name="copilot", help="Ingest local GitHub Copilot CLI recordings.") +def copilot_group() -> None: + pass + + +@copilot_group.command("sync", help="One-shot local ingestion of Copilot CLI recordings.") +@click.option("--session-id", default=None, help="Exact native Copilot session ID.") +@_source_home_option +def sync_cmd(session_id: str | None, source_home: Path | None) -> None: + config = Config.load() + paths = resolve_sources(source_home) + try: + if session_id is not None: + validate_native_id(session_id) + result = capture_sync(config, paths, session_id=session_id) + except ValueError as exc: + raise click.ClickException(str(exc)) from exc + _print_sync_result(result) + if session_id is not None and result["errors"] > 0: + raise click.ClickException( + f"Copilot session {session_id} was not found or could not be imported" + ) + + +@copilot_group.command("watch", help="Poll local Copilot CLI recordings until interrupted.") +@_source_home_option +@click.option( + "--interval", + type=float, + default=1.0, + show_default=True, + callback=_validate_interval, + help="Seconds between source polls. Must be finite and at least 0.1.", +) +def watch_cmd(source_home: Path | None, interval: float) -> None: + config = Config.load() + paths = resolve_sources(source_home) + watch_loop(config, paths, interval=interval) + + +@copilot_group.command("status", help="Show Copilot CLI capture paths, hooks, and health.") +@_source_home_option +def status_cmd(source_home: Path | None) -> None: + config = Config.load() + paths = resolve_sources(source_home) + status = capture_status(config, paths) + _print_status(status) + if status.get("errors"): + raise click.ClickException("Copilot source errors prevent a healthy status") diff --git a/src/thirdeye/commands/setup.py b/src/thirdeye/commands/setup.py index 179f4d9..fad2153 100644 --- a/src/thirdeye/commands/setup.py +++ b/src/thirdeye/commands/setup.py @@ -16,6 +16,7 @@ "claude": "Claude Code", "codex": "Codex CLI", "cursor": "Cursor", + "copilot": "GitHub Copilot CLI", } _SKILL_TARGET_LABELS = { From da1e1cfeb6364483bfafd9cb79226131295aa071 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Thu, 10 Sep 2026 16:35:21 -0700 Subject: [PATCH 36/88] Add targeted tests for Copilot CLI registration edge cases. Cover platform flag listing, sync error propagation, status error rendering, and hook entrypoint importability. Co-authored-by: Cursor --- tests/test_add_command.py | 14 +++++++ tests/test_copilot_command.py | 76 ++++++++++++++++++++++++++++++++++- 2 files changed, 88 insertions(+), 2 deletions(-) diff --git a/tests/test_add_command.py b/tests/test_add_command.py index 3885658..85a3e86 100644 --- a/tests/test_add_command.py +++ b/tests/test_add_command.py @@ -72,12 +72,26 @@ def test_add_requires_platform(): assert "platform" in r.output.lower() +def test_add_requires_platform_lists_all_supported_flags(): + r = CliRunner().invoke(main, ["add"]) + assert r.exit_code != 0 + for flag in ("--claude", "--codex", "--cursor", "--copilot"): + assert flag in r.output + + def test_remove_requires_platform(): r = CliRunner().invoke(main, ["remove"]) assert r.exit_code != 0 assert "platform" in r.output.lower() +def test_remove_requires_platform_lists_all_supported_flags(): + r = CliRunner().invoke(main, ["remove"]) + assert r.exit_code != 0 + for flag in ("--claude", "--codex", "--cursor", "--copilot"): + assert flag in r.output + + # -- install (add --claude) ---------------------------------------------------- diff --git a/tests/test_copilot_command.py b/tests/test_copilot_command.py index 0777893..ffc9697 100644 --- a/tests/test_copilot_command.py +++ b/tests/test_copilot_command.py @@ -187,7 +187,12 @@ def test_status_help_documents_source_home() -> None: def test_copilot_commands_have_no_export_flag() -> None: runner = CliRunner() - for args in (["copilot", "--help"], ["copilot", "sync", "--help"], ["copilot", "watch", "--help"]): + for args in ( + ["copilot", "--help"], + ["copilot", "sync", "--help"], + ["copilot", "watch", "--help"], + ["copilot", "status", "--help"], + ): result = runner.invoke(main, args) assert result.exit_code == 0, result.output assert "--export" not in result.output @@ -200,6 +205,17 @@ def test_pyproject_registers_copilot_hook_entrypoint() -> None: assert "thirdeye.platforms.copilot.hooks:main" in text +def test_copilot_hook_entrypoint_is_importable() -> None: + from importlib.metadata import entry_points + + scripts = entry_points(group="console_scripts") + hook = next((ep for ep in scripts if ep.name == HOOK_BIN), None) + assert hook is not None + module_path, _, attr = hook.value.partition(":") + module = __import__(module_path, fromlist=[attr]) + assert callable(getattr(module, attr)) + + # -- sync ---------------------------------------------------------------------- @@ -276,7 +292,48 @@ def test_sync_missing_session_id_exits_nonzero( ) assert result.exit_code != 0, result.output assert "No such command" not in result.output - assert "missing-session-id" in result.output or "error" in result.output.lower() + assert "missing-session-id" in result.output + assert "errors=1" in result.output + + +def test_sync_session_with_capture_errors_exits_nonzero( + isolated_home: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + def fake_sync( + _config: Config, + _paths: SourcePaths, + *, + session_id: str | None = None, + ) -> SyncResult: + return _empty_result(errors=1) + + monkeypatch.setattr("thirdeye.commands.copilot.capture_sync", fake_sync) + result = CliRunner().invoke( + main, + ["copilot", "sync", "--session-id", NATIVE_SESSION_ID], + ) + assert result.exit_code != 0, result.output + assert NATIVE_SESSION_ID in result.output + assert "errors=1" in result.output + + +def test_sync_capture_value_error_becomes_click_exception( + isolated_home: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + def fake_sync( + _config: Config, + _paths: SourcePaths, + *, + session_id: str | None = None, + ) -> SyncResult: + raise ValueError("invalid native session routing") + + monkeypatch.setattr("thirdeye.commands.copilot.capture_sync", fake_sync) + result = CliRunner().invoke(main, ["copilot", "sync"]) + assert result.exit_code != 0, result.output + assert "invalid native session routing" in result.output def test_sync_prints_counts(isolated_home: Path, monkeypatch: pytest.MonkeyPatch) -> None: @@ -405,6 +462,21 @@ def test_status_exits_nonzero_on_source_errors( ) result = CliRunner().invoke(main, ["copilot", "status"]) assert result.exit_code != 0, result.output + assert "source_unreadable" in result.output + assert "database unreadable" in result.output + + +def test_status_prints_string_errors( + isolated_home: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + "thirdeye.commands.copilot.capture_status", + lambda _config, _paths: _minimal_status(errors=["legacy string error"]), # type: ignore[list-item] + ) + result = CliRunner().invoke(main, ["copilot", "status"]) + assert result.exit_code != 0, result.output + assert "legacy string error" in result.output def test_status_prints_paths_and_guidance( From 6872cc42058a3a9e3dbde0f86c6ad83713f31b12 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Thu, 10 Sep 2026 16:49:17 -0700 Subject: [PATCH 37/88] Keep Copilot CLI command output local and actionable. Status no longer prints prompt bodies, selected-ID sync treats imported diagnostics as success rather than missing, and path-resolution failures become Click errors instead of tracebacks. Co-authored-by: Cursor --- README.md | 2 +- src/thirdeye/commands/copilot.py | 183 ++++++- src/thirdeye/skills/use-thirdeye/SKILL.md | 4 +- .../references/setup-and-tracing.md | 6 +- tests/test_add_command.py | 2 +- tests/test_copilot_command.py | 485 +++++++++++++++--- tests/test_setup_command.py | 3 +- 7 files changed, 596 insertions(+), 89 deletions(-) diff --git a/README.md b/README.md index 126ead4..36eeddc 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ rerun `thirdeye skills add --force` to refresh them. See ## Enable tracing ```bash -thirdeye add --claude # also: --cursor, --codex +thirdeye add --claude # also: --cursor, --codex, --copilot ``` To detach: `thirdeye remove --claude`. diff --git a/src/thirdeye/commands/copilot.py b/src/thirdeye/commands/copilot.py index 2399d35..14030f9 100644 --- a/src/thirdeye/commands/copilot.py +++ b/src/thirdeye/commands/copilot.py @@ -1,19 +1,35 @@ from __future__ import annotations import math +import os from pathlib import Path from typing import Any import click from thirdeye.config import Config +from thirdeye.platforms.copilot.capture import ( + _is_absence_diagnostic, + _is_retryable_code, +) from thirdeye.platforms.copilot.capture import sync as capture_sync +from thirdeye.platforms.copilot.constants import COPILOT_HOME_ENV from thirdeye.platforms.copilot.identity import resolve_sources, validate_native_id from thirdeye.platforms.copilot.status import capture_status -from thirdeye.platforms.copilot.types import SyncResult +from thirdeye.platforms.copilot.types import SourcePaths, SyncResult from thirdeye.platforms.copilot.watch import watch as watch_loop _MIN_WATCH_INTERVAL = 0.1 +_LOCATOR_KEYS = ( + "file", + "path", + "table", + "row_id", + "offset", + "byte_offset", + "generation", + "file_generation", +) def _source_home_option(fn): @@ -33,8 +49,26 @@ def _validate_interval(_ctx: click.Context, _param: click.Parameter, value: floa return value -def _print_sync_result(result: SyncResult) -> None: - click.echo( +def _requested_home(source_home: Path | None) -> Path: + if source_home is not None: + return source_home + configured = os.environ.get(COPILOT_HOME_ENV) + if configured: + return Path(configured) + return Path.home() / ".copilot" + + +def _resolve_paths(source_home: Path | None) -> SourcePaths: + try: + return resolve_sources(source_home) + except ValueError as exc: + raise click.ClickException( + f"Cannot resolve Copilot sources at {_requested_home(source_home)}: {exc}" + ) from exc + + +def _counts_line(result: SyncResult) -> str: + return ( f"sessions={result['sessions']} " f"records_written={result['records_written']} " f"duplicate_records={result['duplicate_records']} " @@ -43,10 +77,101 @@ def _print_sync_result(result: SyncResult) -> None: ) +def _format_locator(locator: dict[str, Any]) -> str | None: + parts: list[str] = [] + for key in _LOCATOR_KEYS: + value = locator.get(key) + if isinstance(value, (str, int, float)) and not isinstance(value, bool): + parts.append(f"{key}={value}") + return ",".join(parts) if parts else None + + +def _format_diagnostic(error: object) -> str: + if not isinstance(error, dict): + return str(error) + code = error.get("code") or error.get("kind") or "error" + if not isinstance(code, str) or not code: + code = "error" + message = error.get("message") or error.get("reason") + bits = [f"{code}: {message}" if isinstance(message, str) and message else code] + session = error.get("session") or error.get("native_session_id") + if isinstance(session, str) and session: + bits.append(f"session={session}") + path = error.get("path") + if isinstance(path, str) and path: + bits.append(f"path={path}") + source_id = error.get("source_id") + if isinstance(source_id, str) and source_id: + bits.append(f"source_id={source_id}") + locator = error.get("locator") + if isinstance(locator, dict): + formatted = _format_locator(locator) + if formatted: + bits.append(f"locator={formatted}") + return " ".join(bits) + + +def _format_last_hook(record: object) -> str: + if not isinstance(record, dict): + return "none" + payload = record.get("payload") if isinstance(record.get("payload"), dict) else {} + locator = record.get("locator") if isinstance(record.get("locator"), dict) else {} + event = payload.get("event") or locator.get("event") or "unknown" + observed_at = record.get("observed_at") + session = record.get("native_session_id") + observed = observed_at if isinstance(observed_at, str) and observed_at else "unknown" + native_id = session if isinstance(session, str) and session else "unknown" + return f"observed_at={observed} event={event} session={native_id}" + + +def _format_timestamp(value: object) -> str: + return value if isinstance(value, str) and value else "none" + + +def _print_capability(label: str, capability: object) -> None: + data = capability if isinstance(capability, dict) else {} + path = data.get("path", "") + click.echo( + f"{label}: exists={data.get('exists', False)} " + f"readable={data.get('readable', False)} path={path}" + ) + + +def _status_error_affects_exit(error: object) -> bool: + """Reuse capture's absence/retryable classifiers; missing timestamps are diagnoses.""" + + if not isinstance(error, dict): + return True + if _is_absence_diagnostic(error): + return False + if _is_retryable_code(error.get("code")): + return False + return error.get("kind") != "missing_source_time" + + +def _print_sync_result(result: SyncResult) -> None: + click.echo(_counts_line(result)) + + +def _print_sync_followup(config: Config, paths: SourcePaths, result: SyncResult) -> None: + if result["errors"] == 0 and result["pending"] == 0: + return + if result["errors"] > 0 and result["sessions"] > 0: + click.echo(f"imported with {result['errors']} source diagnostics") + if result["errors"] > 0: + status = capture_status(config, paths) + for error in status.get("errors") or []: + click.echo(f" {_format_diagnostic(error)}") + click.echo("See `thirdeye copilot status` for source diagnostics.") + if result["pending"] > 0: + click.echo("Pending work remains; rerun sync or use `thirdeye copilot watch`.") + + def _print_status(status: dict[str, Any]) -> None: paths = status.get("paths") or {} installation = status.get("installation") or {} pending = status.get("pending") or {} + capabilities = status.get("capabilities") or {} configured = bool(installation.get("configured")) click.echo(f"Copilot home: {paths.get('home', '')}") click.echo(f"Session root: {paths.get('session_root', '')}") @@ -54,12 +179,14 @@ def _print_status(status: dict[str, Any]) -> None: click.echo(f"Hooks file: {installation.get('hooks_file', '')}") if configured: click.echo("Hooks: configured") - click.echo("Restart Copilot CLI or start a new interactive session if hooks were just installed.") else: click.echo("Hooks: not configured (informational; persisted import still works)") click.echo("Install hooks with: thirdeye add --copilot") - click.echo(f"Last observed hook: {status.get('last_observed_hook')}") - click.echo(f"Last successful import: {status.get('last_successful_import')}") + _print_capability("Transcripts", capabilities.get("transcripts")) + _print_capability("Database", capabilities.get("database")) + _print_capability("Database WAL", capabilities.get("database_wal")) + click.echo(f"Last observed hook: {_format_last_hook(status.get('last_observed_hook'))}") + click.echo(f"Last successful import: {_format_timestamp(status.get('last_successful_import'))}") click.echo( "Pending: " f"spool_records={pending.get('spool_records', 0)} " @@ -68,18 +195,22 @@ def _print_status(status: dict[str, Any]) -> None: f"journals={pending.get('journals', 0)}" ) errors = status.get("errors") or [] - if errors: + informational = [error for error in errors if not _status_error_affects_exit(error)] + blocking = [error for error in errors if _status_error_affects_exit(error)] + if informational: + click.echo("Informational:") + for error in informational: + click.echo(f" {_format_diagnostic(error)}") + if blocking: click.echo("Source errors:") - for error in errors: - if isinstance(error, dict): - kind = error.get("kind") or error.get("code") or "error" - message = error.get("message") or error.get("reason") or error - click.echo(f" {kind}: {message}") - else: - click.echo(f" {error}") + for error in blocking: + click.echo(f" {_format_diagnostic(error)}") else: click.echo("Source errors: none") - click.echo("Verify receipt with: thirdeye copilot status") + click.echo( + "Start an interactive Copilot CLI session in a trusted folder, then rerun; " + "'Last observed hook' should update." + ) @click.group(name="copilot", help="Ingest local GitHub Copilot CLI recordings.") @@ -92,7 +223,7 @@ def copilot_group() -> None: @_source_home_option def sync_cmd(session_id: str | None, source_home: Path | None) -> None: config = Config.load() - paths = resolve_sources(source_home) + paths = _resolve_paths(source_home) try: if session_id is not None: validate_native_id(session_id) @@ -100,7 +231,8 @@ def sync_cmd(session_id: str | None, source_home: Path | None) -> None: except ValueError as exc: raise click.ClickException(str(exc)) from exc _print_sync_result(result) - if session_id is not None and result["errors"] > 0: + _print_sync_followup(config, paths, result) + if session_id is not None and result["sessions"] == 0: raise click.ClickException( f"Copilot session {session_id} was not found or could not be imported" ) @@ -118,16 +250,25 @@ def sync_cmd(session_id: str | None, source_home: Path | None) -> None: ) def watch_cmd(source_home: Path | None, interval: float) -> None: config = Config.load() - paths = resolve_sources(source_home) - watch_loop(config, paths, interval=interval) + paths = _resolve_paths(source_home) + click.echo( + f"Watching Copilot home {paths['home']} every {interval}s (local-only). Ctrl-C to stop." + ) + try: + watch_loop(config, paths, interval=interval) + except KeyboardInterrupt: + pass + except ValueError as exc: + raise click.ClickException(str(exc)) from exc + click.echo("Stopped watching Copilot recordings.") @copilot_group.command("status", help="Show Copilot CLI capture paths, hooks, and health.") @_source_home_option def status_cmd(source_home: Path | None) -> None: config = Config.load() - paths = resolve_sources(source_home) + paths = _resolve_paths(source_home) status = capture_status(config, paths) _print_status(status) - if status.get("errors"): + if any(_status_error_affects_exit(error) for error in status.get("errors") or []): raise click.ClickException("Copilot source errors prevent a healthy status") diff --git a/src/thirdeye/skills/use-thirdeye/SKILL.md b/src/thirdeye/skills/use-thirdeye/SKILL.md index 49cb1cc..336a657 100644 --- a/src/thirdeye/skills/use-thirdeye/SKILL.md +++ b/src/thirdeye/skills/use-thirdeye/SKILL.md @@ -5,7 +5,7 @@ description: Use when an agent needs to inspect, search, or evaluate past agent ## Overview -`thirdeye` (PyPI: `thrdi`) captures events from agentic tools (Claude Code, Codex, Cursor) +`thirdeye` (PyPI: `thrdi`) captures events from agentic tools (Claude Code, Codex, Cursor, GitHub Copilot CLI) into a unified per-session event store on disk. Each session's data lives under `/traces///` and contains a sequential log of all events the agent emitted during that session. Sessions are addressable by any unique prefix of their session ID, @@ -22,11 +22,13 @@ for full instructions on installing hooks, verifying data flow, and removing hoo thirdeye add --claude thirdeye add --codex thirdeye add --cursor +thirdeye add --copilot # Remove hooks thirdeye remove --claude thirdeye remove --codex thirdeye remove --cursor +thirdeye remove --copilot ``` ## Searching and retrieving session data diff --git a/src/thirdeye/skills/use-thirdeye/references/setup-and-tracing.md b/src/thirdeye/skills/use-thirdeye/references/setup-and-tracing.md index 7654a40..fab1759 100644 --- a/src/thirdeye/skills/use-thirdeye/references/setup-and-tracing.md +++ b/src/thirdeye/skills/use-thirdeye/references/setup-and-tracing.md @@ -19,15 +19,17 @@ The PyPI package is `thrdi`; the installed commands are `thirdeye` and `thrdi` thirdeye add --claude # Claude Code thirdeye add --codex # OpenAI Codex CLI thirdeye add --cursor # Cursor +thirdeye add --copilot # GitHub Copilot CLI ``` `thirdeye add` is idempotent — running it twice for the same platform leaves the existing hook entries in place rather than duplicating them. Hook entries are written into each platform's own config file: Claude Code -uses `~/.claude/settings.json`, Codex uses `~/.codex/config.toml`, and +uses `~/.claude/settings.json`, Codex uses `~/.codex/config.toml`, Cursor uses `~/.cursor/hooks.json` -(covering both the IDE chat and the `cursor-agent` CLI). +(covering both the IDE chat and the `cursor-agent` CLI), and GitHub Copilot CLI +uses `$COPILOT_HOME/hooks/thirdeye.json` (default `~/.copilot/hooks/thirdeye.json`). ### Cursor subagent hooks diff --git a/tests/test_add_command.py b/tests/test_add_command.py index 85a3e86..1831f0f 100644 --- a/tests/test_add_command.py +++ b/tests/test_add_command.py @@ -434,7 +434,7 @@ def test_add_copilot_writes_hooks(tmp_path: Path, monkeypatch): r = CliRunner().invoke(main, ["add", "--copilot"]) assert r.exit_code == 0, r.output assert hooks_file.exists() - assert "Copilot" in r.output + assert "Installed tracing for GitHub Copilot CLI" in r.output def test_remove_copilot_calls_uninstall(monkeypatch): diff --git a/tests/test_copilot_command.py b/tests/test_copilot_command.py index ffc9697..8fef801 100644 --- a/tests/test_copilot_command.py +++ b/tests/test_copilot_command.py @@ -7,7 +7,6 @@ import sqlite3 from pathlib import Path from typing import Any -from unittest.mock import MagicMock import pytest from click.testing import CliRunner @@ -22,15 +21,37 @@ CLI_FIXTURE = FIXTURES / "cli-1.0.83" NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" HOOK_BIN = "thirdeye-copilot-hook" +SECRET_PROMPT = "SECRET PROMPT BODY" @pytest.fixture def isolated_home(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: home = tmp_path / "thirdeye" + copilot_home = tmp_path / "copilot-default" + copilot_home.mkdir() monkeypatch.setenv("THIRDEYE_HOME", str(home)) + monkeypatch.setenv("COPILOT_HOME", str(copilot_home)) return home +def _counts_line(**overrides: int) -> str: + values = { + "sessions": 0, + "records_written": 0, + "duplicate_records": 0, + "pending": 0, + "errors": 0, + **overrides, + } + return ( + f"sessions={values['sessions']} " + f"records_written={values['records_written']} " + f"duplicate_records={values['duplicate_records']} " + f"pending={values['pending']} " + f"errors={values['errors']}" + ) + + def _empty_result(**overrides: int) -> SyncResult: result: SyncResult = { "sessions": 0, @@ -121,7 +142,29 @@ def _write_database(home: Path, *, session_id: str = NATIVE_SESSION_ID) -> None: connection.close() -def _minimal_status(*, configured: bool = False, errors: list[dict[str, Any]] | None = None) -> dict: +def _prompt_hook_record() -> dict[str, Any]: + return { + "source_id": f"hook/{NATIVE_SESSION_ID}/obs-1", + "source_kind": "hook", + "native_session_id": NATIVE_SESSION_ID, + "ts": "2026-09-10T17:08:01.000Z", + "observed_at": "2026-09-10T17:08:02.000Z", + "payload": { + "schema_version": 1, + "event": "userPromptSubmitted", + "hook_payload": { + "sessionId": NATIVE_SESSION_ID, + "prompt": SECRET_PROMPT, + }, + "context": {}, + }, + "locator": {"observation_id": "obs-1", "event": "userPromptSubmitted"}, + } + + +def _minimal_status( + *, configured: bool = False, errors: list[dict[str, Any]] | None = None +) -> dict: return { "paths": { "home": "/tmp/copilot", @@ -129,11 +172,26 @@ def _minimal_status(*, configured: bool = False, errors: list[dict[str, Any]] | "session_root": "/tmp/copilot/session-state", "database": "/tmp/copilot/session-store.db", }, - "installation": {"configured": configured, "hooks_file": "/tmp/copilot/hooks/thirdeye.json"}, + "installation": { + "configured": configured, + "hooks_file": "/tmp/copilot/hooks/thirdeye.json", + }, "capabilities": { - "transcripts": {"exists": True, "readable": True}, - "database": {"exists": True, "readable": True}, - "database_wal": {"exists": False, "readable": False}, + "transcripts": { + "exists": True, + "readable": True, + "path": "/tmp/copilot/session-state", + }, + "database": { + "exists": True, + "readable": True, + "path": "/tmp/copilot/session-store.db", + }, + "database_wal": { + "exists": False, + "readable": False, + "path": "/tmp/copilot/session-store.db-wal", + }, }, "last_observed_hook": None, "last_successful_import": None, @@ -149,6 +207,15 @@ def _minimal_status(*, configured: bool = False, errors: list[dict[str, Any]] | } +def _escaping_source_home(tmp_path: Path) -> Path: + home = tmp_path / "escaped-copilot" + home.mkdir() + outside = tmp_path / "outside-session-state" + outside.mkdir() + (home / "session-state").symlink_to(outside) + return home + + # -- command registration ------------------------------------------------------ @@ -222,7 +289,9 @@ def test_copilot_hook_entrypoint_is_importable() -> None: def test_sync_invokes_capture_sync(isolated_home: Path, monkeypatch: pytest.MonkeyPatch) -> None: calls: list[tuple[Any, ...]] = [] - def fake_sync(config: Config, paths: SourcePaths, *, session_id: str | None = None) -> SyncResult: + def fake_sync( + config: Config, paths: SourcePaths, *, session_id: str | None = None + ) -> SyncResult: calls.append((config.root, paths["home"], session_id)) return _empty_result(sessions=2, records_written=5) @@ -231,15 +300,17 @@ def fake_sync(config: Config, paths: SourcePaths, *, session_id: str | None = No assert result.exit_code == 0, result.output assert len(calls) == 1 assert calls[0][2] is None - assert "5" in result.output or "records" in result.output.lower() + assert _counts_line(sessions=2, records_written=5) in result.output def test_sync_passes_session_id(isolated_home: Path, monkeypatch: pytest.MonkeyPatch) -> None: captured: dict[str, str | None] = {"session_id": "unset"} - def fake_sync(_config: Config, _paths: SourcePaths, *, session_id: str | None = None) -> SyncResult: + def fake_sync( + _config: Config, _paths: SourcePaths, *, session_id: str | None = None + ) -> SyncResult: captured["session_id"] = session_id - return _empty_result() + return _empty_result(sessions=1) monkeypatch.setattr("thirdeye.commands.copilot.capture_sync", fake_sync) result = CliRunner().invoke(main, ["copilot", "sync", "--session-id", NATIVE_SESSION_ID]) @@ -261,7 +332,9 @@ def fake_resolve(source_home_arg: Path | None = None) -> SourcePaths: resolved["home"] = paths["home"] return paths - def fake_sync(_config: Config, paths: SourcePaths, *, session_id: str | None = None) -> SyncResult: + def fake_sync( + _config: Config, paths: SourcePaths, *, session_id: str | None = None + ) -> SyncResult: resolved["sync_home"] = paths["home"] return _empty_result() @@ -273,11 +346,58 @@ def fake_sync(_config: Config, paths: SourcePaths, *, session_id: str | None = N assert Path(resolved["sync_home"]) == source_home.resolve() +def test_sync_uses_copilot_home_when_source_home_omitted( + isolated_home: Path, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + env_home = tmp_path / "from-env-home" + env_home.mkdir() + monkeypatch.setenv("COPILOT_HOME", str(env_home)) + seen: dict[str, str] = {} + + def fake_sync( + _config: Config, paths: SourcePaths, *, session_id: str | None = None + ) -> SyncResult: + seen["home"] = paths["home"] + return _empty_result() + + monkeypatch.setattr("thirdeye.commands.copilot.capture_sync", fake_sync) + result = CliRunner().invoke(main, ["copilot", "sync"]) + assert result.exit_code == 0, result.output + assert Path(seen["home"]) == env_home.resolve() + + +def test_sync_source_home_overrides_copilot_home( + isolated_home: Path, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + env_home = tmp_path / "from-env-home" + explicit = tmp_path / "explicit-home" + env_home.mkdir() + explicit.mkdir() + monkeypatch.setenv("COPILOT_HOME", str(env_home)) + seen: dict[str, str] = {} + + def fake_sync( + _config: Config, paths: SourcePaths, *, session_id: str | None = None + ) -> SyncResult: + seen["home"] = paths["home"] + return _empty_result() + + monkeypatch.setattr("thirdeye.commands.copilot.capture_sync", fake_sync) + result = CliRunner().invoke(main, ["copilot", "sync", "--source-home", str(explicit)]) + assert result.exit_code == 0, result.output + assert Path(seen["home"]) == explicit.resolve() + + def test_sync_empty_discovery_exits_zero(isolated_home: Path, tmp_path: Path) -> None: empty_home = tmp_path / "empty-copilot" empty_home.mkdir() result = CliRunner().invoke(main, ["copilot", "sync", "--source-home", str(empty_home)]) assert result.exit_code == 0, result.output + assert _counts_line() in result.output def test_sync_missing_session_id_exits_nonzero( @@ -294,6 +414,7 @@ def test_sync_missing_session_id_exits_nonzero( assert "No such command" not in result.output assert "missing-session-id" in result.output assert "errors=1" in result.output + assert "was not found" in result.output def test_sync_session_with_capture_errors_exits_nonzero( @@ -316,6 +437,65 @@ def fake_sync( assert result.exit_code != 0, result.output assert NATIVE_SESSION_ID in result.output assert "errors=1" in result.output + assert "was not found" in result.output + + +def test_sync_imported_session_with_diagnostics_exits_zero( + isolated_home: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + def fake_sync( + _config: Config, + _paths: SourcePaths, + *, + session_id: str | None = None, + ) -> SyncResult: + return _empty_result(sessions=1, records_written=12, errors=1) + + monkeypatch.setattr("thirdeye.commands.copilot.capture_sync", fake_sync) + result = CliRunner().invoke( + main, + ["copilot", "sync", "--session-id", NATIVE_SESSION_ID], + ) + assert result.exit_code == 0, result.output + assert _counts_line(sessions=1, records_written=12, errors=1) in result.output + assert "imported with 1 source diagnostics" in result.output + assert "was not found" not in result.output + + +def test_sync_prints_source_diagnostics_without_content( + isolated_home: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + def fake_sync( + _config: Config, _paths: SourcePaths, *, session_id: str | None = None + ) -> SyncResult: + return _empty_result(sessions=1, errors=1, pending=1) + + def fake_status(_config: Config, _paths: SourcePaths) -> dict: + status = _minimal_status( + errors=[ + { + "code": "transcript_invalid_json", + "message": "complete transcript line is not JSON", + "session": NATIVE_SESSION_ID, + "locator": {"file": "events.jsonl", "offset": 12}, + "payload": {"prompt": SECRET_PROMPT}, + } + ] + ) + return status + + monkeypatch.setattr("thirdeye.commands.copilot.capture_sync", fake_sync) + monkeypatch.setattr("thirdeye.commands.copilot.capture_status", fake_status) + result = CliRunner().invoke(main, ["copilot", "sync", "--session-id", NATIVE_SESSION_ID]) + assert result.exit_code == 0, result.output + assert "transcript_invalid_json" in result.output + assert NATIVE_SESSION_ID in result.output + assert "events.jsonl" in result.output + assert SECRET_PROMPT not in result.output + assert "thirdeye copilot status" in result.output + assert "thirdeye copilot watch" in result.output def test_sync_capture_value_error_becomes_click_exception( @@ -334,17 +514,25 @@ def fake_sync( result = CliRunner().invoke(main, ["copilot", "sync"]) assert result.exit_code != 0, result.output assert "invalid native session routing" in result.output + assert "Traceback" not in result.output def test_sync_prints_counts(isolated_home: Path, monkeypatch: pytest.MonkeyPatch) -> None: - def fake_sync(_config: Config, _paths: SourcePaths, *, session_id: str | None = None) -> SyncResult: - return _empty_result(sessions=1, records_written=12, duplicate_records=3, pending=2, errors=0) + def fake_sync( + _config: Config, _paths: SourcePaths, *, session_id: str | None = None + ) -> SyncResult: + return _empty_result( + sessions=1, records_written=12, duplicate_records=3, pending=2, errors=0 + ) monkeypatch.setattr("thirdeye.commands.copilot.capture_sync", fake_sync) result = CliRunner().invoke(main, ["copilot", "sync"]) assert result.exit_code == 0, result.output - for token in ("1", "12", "3", "2"): - assert token in result.output + assert ( + _counts_line(sessions=1, records_written=12, duplicate_records=3, pending=2) + in result.output + ) + assert "thirdeye copilot watch" in result.output def test_sync_fixture_session_end_to_end(isolated_home: Path, tmp_path: Path) -> None: @@ -365,12 +553,71 @@ def test_sync_fixture_session_end_to_end(isolated_home: Path, tmp_path: Path) -> assert sum(1 for record in captured if record["source_kind"] == "transcript") == 76 +def test_sync_selected_id_transcript_only_exits_zero(isolated_home: Path, tmp_path: Path) -> None: + source_home = tmp_path / "transcript-only" + source_home.mkdir() + _write_transcript(source_home, NATIVE_SESSION_ID) + + result = CliRunner().invoke( + main, + [ + "copilot", + "sync", + "--source-home", + str(source_home), + "--session-id", + NATIVE_SESSION_ID, + ], + ) + assert result.exit_code == 0, result.output + assert "was not found" not in result.output + assert "imported with" in result.output + assert "copilot_database_missing" in result.output + assert SECRET_PROMPT not in result.output + + +def test_sync_selected_id_malformed_line_exits_zero(isolated_home: Path, tmp_path: Path) -> None: + source_home = tmp_path / "malformed-transcript" + source_home.mkdir() + _write_transcript(source_home, NATIVE_SESSION_ID) + _write_database(source_home, session_id=NATIVE_SESSION_ID) + events = source_home / "session-state" / NATIVE_SESSION_ID / "events.jsonl" + with events.open("a", encoding="utf-8") as handle: + handle.write("{not-json\n") + + result = CliRunner().invoke( + main, + [ + "copilot", + "sync", + "--source-home", + str(source_home), + "--session-id", + NATIVE_SESSION_ID, + ], + ) + assert result.exit_code == 0, result.output + assert "was not found" not in result.output + assert "imported with" in result.output + assert "transcript_invalid_json" in result.output + + def test_sync_rejects_native_id_with_path_separators(isolated_home: Path) -> None: result = CliRunner().invoke(main, ["copilot", "sync", "--session-id", "../escape"]) assert result.exit_code != 0, result.output assert "No such command" not in result.output +def test_sync_path_escape_is_click_error(isolated_home: Path, tmp_path: Path) -> None: + home = _escaping_source_home(tmp_path) + result = CliRunner().invoke(main, ["copilot", "sync", "--source-home", str(home)]) + assert result.exit_code != 0, result.output + assert "Traceback" not in result.output + assert "session_root" in result.output + assert "escapes" in result.output + assert str(home) in result.output + + # -- watch --------------------------------------------------------------------- @@ -399,6 +646,29 @@ def fake_watch(_config: Config, _paths: SourcePaths, *, interval: float = 1.0) - assert calls == [2.5] +def test_watch_prints_start_and_stop(isolated_home: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("thirdeye.commands.copilot.watch_loop", lambda *_args, **_kwargs: None) + result = CliRunner().invoke(main, ["copilot", "watch", "--interval", "1.5"]) + assert result.exit_code == 0, result.output + assert "1.5" in result.output + assert "local-only" in result.output + assert "Ctrl-C" in result.output + assert "Stopped" in result.output + + +def test_watch_keyboard_interrupt_prints_stop( + isolated_home: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + def fake_watch(_config: Config, _paths: SourcePaths, *, interval: float = 1.0) -> None: + raise KeyboardInterrupt + + monkeypatch.setattr("thirdeye.commands.copilot.watch_loop", fake_watch) + result = CliRunner().invoke(main, ["copilot", "watch"]) + assert result.exit_code == 0, result.output + assert "Stopped" in result.output + assert "Traceback" not in result.output + + def test_watch_passes_source_home( isolated_home: Path, tmp_path: Path, @@ -418,12 +688,24 @@ def fake_watch(_config: Config, paths: SourcePaths, *, interval: float = 1.0) -> ) assert result.exit_code == 0, result.output assert Path(seen["home"]) == source_home.resolve() + assert str(source_home.resolve()) in result.output + + +def test_watch_path_escape_is_click_error(isolated_home: Path, tmp_path: Path) -> None: + home = _escaping_source_home(tmp_path) + result = CliRunner().invoke(main, ["copilot", "watch", "--source-home", str(home)]) + assert result.exit_code != 0, result.output + assert "Traceback" not in result.output + assert "session_root" in result.output + assert "escapes" in result.output # -- status -------------------------------------------------------------------- -def test_status_invokes_capture_status(isolated_home: Path, monkeypatch: pytest.MonkeyPatch) -> None: +def test_status_invokes_capture_status( + isolated_home: Path, monkeypatch: pytest.MonkeyPatch +) -> None: called = {"count": 0} def fake_status(_config: Config, _paths: SourcePaths) -> dict: @@ -446,7 +728,7 @@ def test_status_exits_zero_when_hooks_missing_but_sources_ok( ) result = CliRunner().invoke(main, ["copilot", "status"]) assert result.exit_code == 0, result.output - assert "not configured" in result.output.lower() or "configured" in result.output.lower() + assert "Hooks: not configured" in result.output def test_status_exits_nonzero_on_source_errors( @@ -457,13 +739,119 @@ def test_status_exits_nonzero_on_source_errors( "thirdeye.commands.copilot.capture_status", lambda _config, _paths: _minimal_status( configured=True, - errors=[{"kind": "source_unreadable", "message": "database unreadable"}], + errors=[ + { + "kind": "source_unreadable", + "message": "database unreadable", + "session": NATIVE_SESSION_ID, + "path": "/tmp/copilot/session-store.db", + } + ], ), ) result = CliRunner().invoke(main, ["copilot", "status"]) assert result.exit_code != 0, result.output assert "source_unreadable" in result.output assert "database unreadable" in result.output + assert NATIVE_SESSION_ID in result.output + assert "/tmp/copilot/session-store.db" in result.output + + +def test_status_absence_codes_exit_zero( + isolated_home: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + "thirdeye.commands.copilot.capture_status", + lambda _config, _paths: _minimal_status( + errors=[ + { + "code": "copilot_database_missing", + "message": "database file is absent", + "session": NATIVE_SESSION_ID, + } + ] + ), + ) + result = CliRunner().invoke(main, ["copilot", "status"]) + assert result.exit_code == 0, result.output + assert "copilot_database_missing" in result.output + assert NATIVE_SESSION_ID in result.output + + +def test_status_retryable_codes_exit_zero( + isolated_home: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + "thirdeye.commands.copilot.capture_status", + lambda _config, _paths: _minimal_status( + errors=[ + { + "code": "copilot_database_busy", + "message": "database is busy", + "session": NATIVE_SESSION_ID, + } + ] + ), + ) + result = CliRunner().invoke(main, ["copilot", "status"]) + assert result.exit_code == 0, result.output + assert "copilot_database_busy" in result.output + + +def test_status_transcript_only_home_exits_zero(isolated_home: Path, tmp_path: Path) -> None: + source_home = tmp_path / "transcript-only-status" + source_home.mkdir() + _write_transcript(source_home, NATIVE_SESSION_ID) + sync = CliRunner().invoke(main, ["copilot", "sync", "--source-home", str(source_home)]) + assert sync.exit_code == 0, sync.output + result = CliRunner().invoke(main, ["copilot", "status", "--source-home", str(source_home)]) + assert result.exit_code == 0, result.output + assert "copilot_database_missing" in result.output + + +def test_status_does_not_print_hook_prompt( + isolated_home: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + status = _minimal_status(configured=False) + status["last_observed_hook"] = _prompt_hook_record() + monkeypatch.setattr("thirdeye.commands.copilot.capture_status", lambda _config, _paths: status) + result = CliRunner().invoke(main, ["copilot", "status"]) + assert result.exit_code == 0, result.output + assert SECRET_PROMPT not in result.output + assert "hook_payload" not in result.output + assert "userPromptSubmitted" in result.output + assert NATIVE_SESSION_ID in result.output + assert "2026-09-10T17:08:02.000Z" in result.output + + +def test_status_error_lines_omit_payload( + isolated_home: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + "thirdeye.commands.copilot.capture_status", + lambda _config, _paths: _minimal_status( + configured=True, + errors=[ + { + "kind": "source_unreadable", + "message": "PermissionError", + "session": "sess-1", + "path": "/secret/db", + "payload": {"prompt": SECRET_PROMPT}, + } + ], + ), + ) + result = CliRunner().invoke(main, ["copilot", "status"]) + assert result.exit_code != 0, result.output + assert "sess-1" in result.output + assert "/secret/db" in result.output + assert SECRET_PROMPT not in result.output + assert "hook_payload" not in result.output def test_status_prints_string_errors( @@ -479,7 +867,7 @@ def test_status_prints_string_errors( assert "legacy string error" in result.output -def test_status_prints_paths_and_guidance( +def test_status_prints_paths_capabilities_and_guidance( isolated_home: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -489,8 +877,15 @@ def test_status_prints_paths_and_guidance( ) result = CliRunner().invoke(main, ["copilot", "status"]) assert result.exit_code == 0, result.output - assert "/tmp/copilot" in result.output - assert "hooks" in result.output.lower() or "configured" in result.output.lower() + assert "Copilot home: /tmp/copilot" in result.output + assert "Hooks: configured" in result.output + assert "Restart Copilot CLI" not in result.output + assert "Transcripts:" in result.output + assert "Database:" in result.output + assert "Database WAL:" in result.output + assert "exists=True" in result.output + assert "readable=True" in result.output + assert "trusted folder" in result.output def test_status_resolves_source_home_with_spaces( @@ -512,44 +907,10 @@ def fake_status(_config: Config, paths: SourcePaths) -> dict: assert Path(seen["home"]) == source_home.resolve() -# -- add/remove wiring --------------------------------------------------------- - - -def test_add_help_mentions_copilot() -> None: - result = CliRunner().invoke(main, ["add", "--help"]) - assert result.exit_code == 0, result.output - assert "--copilot" in result.output - - -def test_remove_help_mentions_copilot() -> None: - result = CliRunner().invoke(main, ["remove", "--help"]) - assert result.exit_code == 0, result.output - assert "--copilot" in result.output - - -def test_add_copilot_dispatches_platform(monkeypatch: pytest.MonkeyPatch) -> None: - from thirdeye.commands.add import PLATFORMS - - mock_platform = MagicMock() - mock_platform.display_name = "GitHub Copilot CLI" - mock_cls = MagicMock(return_value=mock_platform) - monkeypatch.setitem(PLATFORMS, "copilot", mock_cls) - - result = CliRunner().invoke(main, ["add", "--copilot"]) - assert result.exit_code == 0, result.output - mock_cls.assert_called_once() - mock_platform.install.assert_called_once() - - -def test_remove_copilot_dispatches_platform(monkeypatch: pytest.MonkeyPatch) -> None: - from thirdeye.commands.add import PLATFORMS - - mock_platform = MagicMock() - mock_platform.display_name = "GitHub Copilot CLI" - mock_cls = MagicMock(return_value=mock_platform) - monkeypatch.setitem(PLATFORMS, "copilot", mock_cls) - - result = CliRunner().invoke(main, ["remove", "--copilot"]) - assert result.exit_code == 0, result.output - mock_cls.assert_called_once() - mock_platform.uninstall.assert_called_once() +def test_status_path_escape_is_click_error(isolated_home: Path, tmp_path: Path) -> None: + home = _escaping_source_home(tmp_path) + result = CliRunner().invoke(main, ["copilot", "status", "--source-home", str(home)]) + assert result.exit_code != 0, result.output + assert "Traceback" not in result.output + assert "session_root" in result.output + assert "escapes" in result.output diff --git a/tests/test_setup_command.py b/tests/test_setup_command.py index 634ce5f..d7d10d8 100644 --- a/tests/test_setup_command.py +++ b/tests/test_setup_command.py @@ -291,6 +291,7 @@ def test_skill_targets_follow_configured_agents(platforms: list[str], targets: l def test_setup_can_install_copilot_tracing(monkeypatch: pytest.MonkeyPatch) -> None: platforms = {name: _fake_platform(name) for name in ("claude", "codex", "cursor", "copilot")} + platforms["copilot"].display_name = "GitHub Copilot CLI" _fake_resolver(monkeypatch, platforms) monkeypatch.setattr("thirdeye.commands.setup.logfire_cmd.is_available", lambda: False) @@ -299,4 +300,4 @@ def test_setup_can_install_copilot_tracing(monkeypatch: pytest.MonkeyPatch) -> N assert result.exit_code == 0, result.output platforms["copilot"].install.assert_called_once() platforms["claude"].install.assert_not_called() - assert "GitHub Copilot CLI" in result.output or "Copilot" in result.output + assert "Installed tracing for GitHub Copilot CLI" in result.output From c2a73dd6a94071fa6328f17e8cbdd26bccfb2a91 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Thu, 10 Sep 2026 17:21:20 -0700 Subject: [PATCH 38/88] Document Copilot CLI V1 and fix status leases plus same-batch session close. Status now reads live copilot.followup.json leases, and archive lets the last top-level lifecycle hook in a composed batch win so a spooled sessionEnd can close the session. Co-authored-by: Cursor --- README.md | 38 ++++++++++-- docs/windows.md | 19 +++--- src/thirdeye/platforms/copilot/archive.py | 38 +++++++----- src/thirdeye/platforms/copilot/constants.py | 1 + src/thirdeye/platforms/copilot/followup.py | 6 +- src/thirdeye/platforms/copilot/status.py | 25 ++++++-- src/thirdeye/skills/use-thirdeye/SKILL.md | 5 ++ .../references/setup-and-tracing.md | 34 ++++++++++- tests/test_copilot_archive.py | 46 +++++++++++++- tests/test_copilot_capture.py | 54 ++++++++++++++++ tests/test_copilot_status.py | 61 ++++++++++++------- 11 files changed, 268 insertions(+), 59 deletions(-) diff --git a/README.md b/README.md index 36eeddc..c65c586 100644 --- a/README.md +++ b/README.md @@ -9,14 +9,14 @@ [![Python](https://img.shields.io/pypi/pyversions/thrdi.svg)](https://pypi.org/project/thrdi/) [![License](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE) -Trace every agent session on your machine — Claude Code, Codex, Cursor — into one history you and your agents can manage, search, and evaluate. +Trace every agent session on your machine — Claude Code, Codex, Cursor, GitHub Copilot CLI — into one history you and your agents can manage, search, and evaluate. ## Install > **Windows support is experimental.** The test suite runs on Windows in CI, and -> Claude Code tracing is the verified integration there. The Codex CLI and Cursor -> installers are implemented but have not been verified against those tools on -> Windows. Please report Windows problems at the +> Claude Code tracing is the verified integration there. The Codex CLI, Cursor, +> and Copilot CLI installers are implemented but have not been live-certified +> against those tools on Windows. Please report Windows problems at the > [issue tracker](https://github.com/duncankmckinnon/thirdeye/issues). See > [docs/windows.md](docs/windows.md) for the deliberate platform differences. @@ -82,7 +82,35 @@ rerun `thirdeye skills add --force` to refresh them. See thirdeye add --claude # also: --cursor, --codex, --copilot ``` -To detach: `thirdeye remove --claude`. +To detach: `thirdeye remove --claude` (also `--cursor`, `--codex`, `--copilot`). + +## Copilot CLI V1 + +GitHub Copilot CLI capture is a V1 immutable raw archive. V2 (reconstructed +turns, usage accounting, and OTel export) is out of scope. V1 does not export +Copilot content. + +```bash +thirdeye add --copilot +thirdeye copilot status --source-home "$COPILOT_HOME" +thirdeye copilot sync --source-home "$COPILOT_HOME" +thirdeye copilot watch --source-home "$COPILOT_HOME" --interval 1 +thirdeye remove --copilot +``` + +`--source-home` is optional. Resolution is `--source-home`, then `COPILOT_HOME`, +then `~/.copilot`. V1 reads only that home's `session-state/**/events.jsonl`, +`workspace.yaml`, and `session-store.db` (`sessions`, `turns`, +`assistant_usage_events`). It does not read credentials, token-bearing config, +or other Copilot files. + +`thirdeye add --copilot` writes user-level hooks at +`$COPILOT_HOME/hooks/thirdeye.json` (default `~/.copilot/hooks/thirdeye.json`). +The 1.0.83 live probe used repository hooks; V1 does not. `watch` is an explicit +foreground poller and is not started by add or setup. Captured content has the +same local sensitivity as other thirdeye sessions and is not exported in V1. +`sync` and `status` print recoverable source diagnostics (locations and reasons, +not prompt bodies). Passing unit tests is not live certification of Copilot CLI. ## Read your history diff --git a/docs/windows.md b/docs/windows.md index 0f27531..81c4535 100644 --- a/docs/windows.md +++ b/docs/windows.md @@ -2,9 +2,10 @@ Windows support is **experimental**. The test suite runs on `windows-latest` in CI across Python 3.11 and 3.13, and Claude Code tracing is the verified -integration. The Codex CLI and Cursor installers are implemented but have not -been exercised against those tools on Windows. Please file Windows issues at the -[issue tracker](https://github.com/duncankmckinnon/thirdeye/issues). +integration. The Codex CLI, Cursor, and Copilot CLI installers are implemented +but have not been live-certified against those tools on Windows. Passing Copilot +unit tests is not live certification of Copilot CLI. Please file Windows issues +at the [issue tracker](https://github.com/duncankmckinnon/thirdeye/issues). Install with `pipx` or `uv` — Homebrew stays macOS/Linux only. @@ -35,12 +36,14 @@ A copied skill does not track a thirdeye upgrade the way a symlink does. After `pipx upgrade thrdi` (or `uv tool upgrade thrdi`), rerun `thirdeye skills add --force` to refresh the copied skills. -### 3. Unverified Codex and Cursor hooks +### 3. Unverified Codex, Cursor, and Copilot hooks -The Codex CLI and Cursor installers are correct by construction but have not been -run against the real tools on Windows. How each tool invokes a hook command -there — `cmd.exe`, PowerShell, or a direct `CreateProcess` — is unconfirmed, and -therefore so is whether a path containing spaces needs quoting. +The Codex CLI, Cursor, and Copilot CLI installers are correct by construction +but have not been run against the real tools on Windows. How each tool invokes a +hook command there — `cmd.exe`, PowerShell, or a direct `CreateProcess` — is +unconfirmed, and therefore so is whether a path containing spaces needs quoting. +Copilot V1 still writes user-level `$COPILOT_HOME/hooks/thirdeye.json` (default +`~/.copilot/hooks/thirdeye.json`); the 1.0.83 probe used repository hooks. To sidestep the question, on Windows only, when the resolved hook binary path contains a space thirdeye writes the bare binary name into the tool's config diff --git a/src/thirdeye/platforms/copilot/archive.py b/src/thirdeye/platforms/copilot/archive.py index 0bab189..f0919b9 100644 --- a/src/thirdeye/platforms/copilot/archive.py +++ b/src/thirdeye/platforms/copilot/archive.py @@ -511,33 +511,43 @@ def commit_batch(config: Config, paths: SourcePaths, batch: SourceBatch) -> Sync ) +def _is_child_hook(payload: dict[str, Any]) -> bool: + """Child identity lives on hook_payload (and, historically, context).""" + + for mapping in (payload.get("hook_payload"), payload.get("context")): + if not isinstance(mapping, dict): + continue + if mapping.get("agentId") or mapping.get("agent_id"): + return True + if mapping.get("parentToolCallId") or mapping.get("parent_tool_call_id"): + return True + return False + + def _apply_lifecycle(directory: Path, records: list[Any]) -> None: """Apply only explicit top-level lifecycle evidence; child stops never close.""" - close = False - reopen = False + decision: str | None = None for record in records: if not isinstance(record, dict) or record.get("source_kind") != "hook": continue payload = record.get("payload", {}) - event = payload.get("event") if isinstance(payload, dict) else None - context = payload.get("context") if isinstance(payload, dict) else None - is_child = isinstance(context, dict) and bool( - context.get("parent_tool_call_id") or context.get("agent_id") - ) - if event in {"sessionEnd", "shutdown", "session_end"} and not is_child: - close = True - if event in {"sessionStart", "resume", "activity", "session_start", "userPromptSubmitted"}: - reopen = True - if not close and not reopen: + if not isinstance(payload, dict): + continue + event = payload.get("event") + if event in {"sessionEnd", "shutdown", "session_end"} and not _is_child_hook(payload): + decision = "close" + elif event in {"sessionStart", "resume", "activity", "session_start", "userPromptSubmitted"}: + decision = "reopen" + if decision is None: return meta_file = meta_path(directory) meta = read_meta(meta_file) if meta is None: return - if reopen: + if decision == "reopen": meta.status = "open" meta.ended_at = None - elif close: + else: meta.status = "closed" meta.ended_at = utc_iso_ms() write_meta(meta_file, meta) diff --git a/src/thirdeye/platforms/copilot/constants.py b/src/thirdeye/platforms/copilot/constants.py index b2ff492..9e18b43 100644 --- a/src/thirdeye/platforms/copilot/constants.py +++ b/src/thirdeye/platforms/copilot/constants.py @@ -18,6 +18,7 @@ COPILOT_HOME_ENV = "COPILOT_HOME" HOOKS_DIRECTORY_NAME = "hooks" OWNED_HOOK_FILENAME = "thirdeye.json" +FOLLOWUP_LEASE_FILENAME = "copilot.followup.json" HOOK_CONFIG_VERSION = 1 HOOK_TIMEOUT_S = 5 diff --git a/src/thirdeye/platforms/copilot/followup.py b/src/thirdeye/platforms/copilot/followup.py index cf5f7e1..996e81a 100644 --- a/src/thirdeye/platforms/copilot/followup.py +++ b/src/thirdeye/platforms/copilot/followup.py @@ -23,12 +23,12 @@ from thirdeye._compat.locking import LockMode, LockTimeout, locked from thirdeye.config import Config from thirdeye.paths import session_dir +from thirdeye.platforms.copilot.constants import FOLLOWUP_LEASE_FILENAME, PLATFORM_NAME from thirdeye.platforms.copilot.identity import stored_session_id, validate_native_id from thirdeye.platforms.copilot.types import SourcePaths, SyncResult from thirdeye.usage.errlog import log_capture_error -_PLATFORM = "copilot" -_LEASE_FILENAME = "copilot.followup.json" +_PLATFORM = PLATFORM_NAME _LEASE_LOCK_FILENAME = "copilot.followup.lock" _LEASE_SECONDS = 5.0 _LOCK_PROBE_TIMEOUT = 0.0 @@ -41,7 +41,7 @@ def _directory(config: Config, paths: SourcePaths, native_id: str) -> Path: def _lease_path(directory: Path) -> Path: - return directory / _LEASE_FILENAME + return directory / FOLLOWUP_LEASE_FILENAME def _lease_lock_path(directory: Path) -> Path: diff --git a/src/thirdeye/platforms/copilot/status.py b/src/thirdeye/platforms/copilot/status.py index 0aa1dcf..bf200d8 100644 --- a/src/thirdeye/platforms/copilot/status.py +++ b/src/thirdeye/platforms/copilot/status.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import time from pathlib import Path from typing import Any @@ -11,7 +12,7 @@ from thirdeye.reader import SessionReader from .archive import _record_from_event -from .constants import PLATFORM_NAME +from .constants import FOLLOWUP_LEASE_FILENAME, PLATFORM_NAME from .database import read_database from .install import CopilotPlatform from .spool import read_spool @@ -213,6 +214,21 @@ def _spool_sessions( return count, sessions, latest, errors +def _followup_lease_pending(directory: Path) -> bool: + """True when a live follow-up lease file exists beside the session.""" + + try: + payload = json.loads((directory / FOLLOWUP_LEASE_FILENAME).read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError): + return False + if not isinstance(payload, dict): + return False + expires_at = payload.get("expires_at") + if not isinstance(expires_at, (int, float)): + return False + return float(expires_at) > time.time() + + def _archive_status( config: Config, paths: SourcePaths ) -> tuple[list[dict[str, Any]], SourceRecord | None, list[dict[str, Any]], int, int]: @@ -243,10 +259,9 @@ def _archive_status( continue health = state.get("health") if isinstance(state.get("health"), dict) else {} diagnostics = health.get("diagnostics") if isinstance(health.get("diagnostics"), list) else [] - followup = state.get("followup", state.get("pending_followup", False)) - lease = state.get("lease", state.get("leases", None)) - pending_followup += int(bool(followup)) - active_leases += len(lease) if isinstance(lease, list) else int(bool(lease)) + if _followup_lease_pending(directory): + pending_followup += 1 + active_leases += 1 sessions.append( { "stored_session_id": directory.name, diff --git a/src/thirdeye/skills/use-thirdeye/SKILL.md b/src/thirdeye/skills/use-thirdeye/SKILL.md index 336a657..8eee83a 100644 --- a/src/thirdeye/skills/use-thirdeye/SKILL.md +++ b/src/thirdeye/skills/use-thirdeye/SKILL.md @@ -24,6 +24,11 @@ thirdeye add --codex thirdeye add --cursor thirdeye add --copilot +# Copilot CLI V1: ingest and health (watch is foreground-only; not started by add) +thirdeye copilot status --source-home "$COPILOT_HOME" +thirdeye copilot sync --source-home "$COPILOT_HOME" +thirdeye copilot watch --source-home "$COPILOT_HOME" --interval 1 + # Remove hooks thirdeye remove --claude thirdeye remove --codex diff --git a/src/thirdeye/skills/use-thirdeye/references/setup-and-tracing.md b/src/thirdeye/skills/use-thirdeye/references/setup-and-tracing.md index fab1759..4249c53 100644 --- a/src/thirdeye/skills/use-thirdeye/references/setup-and-tracing.md +++ b/src/thirdeye/skills/use-thirdeye/references/setup-and-tracing.md @@ -55,9 +55,40 @@ and this user-level setup does not promise remote capture. ```bash thirdeye remove --claude # remove only Claude hooks -thirdeye remove --codex # etc. +thirdeye remove --codex +thirdeye remove --cursor +thirdeye remove --copilot ``` +## Copilot CLI V1 + +GitHub Copilot CLI capture is a V1 immutable raw archive of recordings from the +selected Copilot home. V2 (reconstructed turns, usage accounting, and OTel +export) is out of scope. V1 does not export Copilot content. + +```bash +thirdeye add --copilot +thirdeye copilot status --source-home "$COPILOT_HOME" +thirdeye copilot sync --source-home "$COPILOT_HOME" +thirdeye copilot watch --source-home "$COPILOT_HOME" --interval 1 +thirdeye remove --copilot +``` + +`--source-home` is optional. Resolution is `--source-home`, then `COPILOT_HOME`, +then `~/.copilot`. V1 reads only that home's `session-state/**/events.jsonl`, +`workspace.yaml`, and `session-store.db` (`sessions`, `turns`, +`assistant_usage_events`). It does not read credentials, token-bearing config, +or other Copilot files. + +`thirdeye add --copilot` writes user-level hooks at +`$COPILOT_HOME/hooks/thirdeye.json` (default `~/.copilot/hooks/thirdeye.json`). +The 1.0.83 live probe used repository hooks; V1 does not. `watch` is an explicit +foreground poller and is not started by add or setup. Captured content has the +same local sensitivity as other thirdeye sessions and is not exported in V1. +`sync` and `status` print recoverable source diagnostics (locations and reasons, +not prompt bodies). Passing the Copilot unit tests is not live certification of +Copilot CLI. + ## Verify tracing is live After the next agent run, a new session should appear: @@ -66,6 +97,7 @@ After the next agent run, a new session should appear: thirdeye list # JSON-per-line, newest first thirdeye list --tree # human-readable thirdeye events # events for one session +thirdeye copilot status # Copilot source health, pending spool/leases ``` `` accepts any unique prefix — usually 4-8 characters is enough. diff --git a/tests/test_copilot_archive.py b/tests/test_copilot_archive.py index 3b98251..0838de5 100644 --- a/tests/test_copilot_archive.py +++ b/tests/test_copilot_archive.py @@ -385,13 +385,57 @@ def test_session_end_closes_and_resume_reopens(config: Config, paths: SourcePath assert meta.ended_at is None +def test_last_lifecycle_event_in_batch_wins(config: Config, paths: SourcePaths) -> None: + start = _record( + "key/a/start", + source_kind="hook", + payload={"event": "sessionStart", "hook_payload": {}, "context": {}}, + ) + end = _record( + "key/a/end", + source_kind="hook", + payload={"event": "sessionEnd", "hook_payload": {}, "context": {}}, + ) + commit_batch(config, paths, _batch(paths, [start, end])) + meta = read_meta(meta_path(_session_directory(config, paths))) + assert meta is not None + assert meta.status == "closed" + assert meta.ended_at is not None + + later_end = _record( + "key/a/end-again", + source_kind="hook", + payload={"event": "sessionEnd", "hook_payload": {}, "context": {}}, + ) + resume = _record( + "key/a/resume-after-end", + source_kind="hook", + payload={"event": "resume", "hook_payload": {}, "context": {}}, + ) + commit_batch( + config, + paths, + _batch( + paths, + [later_end, resume], + next_cursor={"generation": 2}, + base_cursor={"generation": 1}, + ), + ) + meta = read_meta(meta_path(_session_directory(config, paths))) + assert meta is not None + assert meta.status == "open" + assert meta.ended_at is None + + def test_child_stop_does_not_close_session(config: Config, paths: SourcePaths) -> None: child_stop = _record( "key/a/child-stop", source_kind="hook", payload={ "event": "sessionEnd", - "context": {"agent_id": "child-agent", "parent_tool_call_id": "tool-1"}, + "hook_payload": {"agentId": "child-agent", "parentToolCallId": "tool-1"}, + "context": {}, }, ) commit_batch(config, paths, _batch(paths, [child_stop])) diff --git a/tests/test_copilot_capture.py b/tests/test_copilot_capture.py index 678b74d..5842402 100644 --- a/tests/test_copilot_capture.py +++ b/tests/test_copilot_capture.py @@ -17,6 +17,8 @@ import thirdeye.platforms.copilot.capture as capture_mod import thirdeye.platforms.copilot.state as state_mod from thirdeye.config import Config +from thirdeye.meta import read_meta +from thirdeye.paths import meta_path, session_dir from thirdeye.platforms.copilot.archive import commit_batch, load_cursor from thirdeye.platforms.copilot.capture import ( capture_session, @@ -24,6 +26,7 @@ record_hook, sync, ) +from thirdeye.platforms.copilot.constants import PLATFORM_NAME from thirdeye.platforms.copilot.hook_payload import parse_hook from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id from thirdeye.platforms.copilot.spool import enqueue_hook, read_spool @@ -338,6 +341,57 @@ def tracking_commit(cfg: Config, p: SourcePaths, batch: SourceBatch) -> SyncResu assert order[0] == "hook" +def test_capture_session_closes_when_spool_drains_start_then_end( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + names = iter(["a" * 32, "b" * 32]) + + class _OrderedUUID: + def __init__(self, value: str) -> None: + self.hex = value + + monkeypatch.setattr( + "thirdeye.platforms.copilot.spool.uuid.uuid4", + lambda: _OrderedUUID(next(names)), + ) + start = parse_hook( + "sessionStart", + { + "sessionId": NATIVE_SESSION_ID, + "timestamp": 1789060102204, + "cwd": "/fixture/workspace", + "source": "new", + }, + {}, + observed_at="2026-09-10T17:08:22.000Z", + observation_id="spool-start", + ) + end = parse_hook( + "sessionEnd", + { + "sessionId": NATIVE_SESSION_ID, + "timestamp": 1789060147875, + "cwd": "/fixture/workspace", + "reason": "user_exit", + }, + {}, + observed_at="2026-09-10T17:08:47.000Z", + observation_id="spool-end", + ) + enqueue_hook(config, paths, start) + enqueue_hook(config, paths, end) + + capture_session(config, paths, NATIVE_SESSION_ID) + + stored = stored_session_id(paths, NATIVE_SESSION_ID) + meta = read_meta(meta_path(session_dir(config.root, PLATFORM_NAME, stored))) + assert meta is not None + assert meta.status == "closed" + assert meta.ended_at is not None + + def test_capture_session_acks_spool_after_successful_commit( copilot_env: tuple[Config, SourcePaths], ) -> None: diff --git a/tests/test_copilot_status.py b/tests/test_copilot_status.py index bb895f4..b4815ab 100644 --- a/tests/test_copilot_status.py +++ b/tests/test_copilot_status.py @@ -5,6 +5,7 @@ import json import shutil import sqlite3 +import time from pathlib import Path import pytest @@ -13,7 +14,11 @@ from thirdeye.config import Config from thirdeye.paths import session_dir from thirdeye.platforms.copilot.archive import commit_batch -from thirdeye.platforms.copilot.constants import OWNED_HOOK_FILENAME, PLATFORM_NAME +from thirdeye.platforms.copilot.constants import ( + FOLLOWUP_LEASE_FILENAME, + OWNED_HOOK_FILENAME, + PLATFORM_NAME, +) from thirdeye.platforms.copilot.hook_payload import parse_hook from thirdeye.platforms.copilot.identity import ( SOURCE_KEY_PREFIX_LEN, @@ -53,11 +58,12 @@ def _record( *, source_kind: str = "transcript", observed_at: str = OBSERVED_AT_EARLY, + native_session_id: str = NATIVE_SESSION_ID, ) -> SourceRecord: return { "source_id": source_id, "source_kind": source_kind, - "native_session_id": NATIVE_SESSION_ID, + "native_session_id": native_session_id, "ts": "2026-09-10T17:08:24.000Z", "observed_at": observed_at, "payload": {"schema_version": 1, "type": "user.message"}, @@ -65,10 +71,15 @@ def _record( } -def _batch(paths: SourcePaths, records: list[SourceRecord]) -> SourceBatch: +def _batch( + paths: SourcePaths, + records: list[SourceRecord], + *, + native_session_id: str = NATIVE_SESSION_ID, +) -> SourceBatch: return { "source_key": paths["source_key"], - "native_session_id": NATIVE_SESSION_ID, + "native_session_id": native_session_id, "cwd": "/proj", "records": records, "next_cursor": {"generation": 1}, @@ -76,6 +87,13 @@ def _batch(paths: SourcePaths, records: list[SourceRecord]) -> SourceBatch: } +def _write_lease(directory: Path, *, expires_at: float) -> None: + (directory / FOLLOWUP_LEASE_FILENAME).write_text( + json.dumps({"generation": "lease-test", "expires_at": expires_at}) + "\n", + encoding="utf-8", + ) + + def _hook_record(*, observation_id: str, observed_at: str = OBSERVED_AT_LATE) -> SourceRecord: return parse_hook( "agentStop", @@ -230,31 +248,30 @@ def test_capture_status_reports_followup_leases_and_journal( copilot_env: tuple[Config, SourcePaths], ) -> None: config, paths = copilot_env + expired_id = "expired-lease-session" commit_batch(config, paths, _batch(paths, [_record("status/pending-state")])) - directory = session_dir(config.root, PLATFORM_NAME, stored_session_id(paths, NATIVE_SESSION_ID)) - state = { - "schema_version": 1, - "source_key": paths["source_key"], - "source_home": paths["home"], - "native_session_id": NATIVE_SESSION_ID, - "cursor": {}, - "followup": True, - "lease": [{"owner": "test"}], - "health": { - "diagnostics": [{"kind": "test_diagnostic", "message": "retry later"}], - "last_successful_import": "2026-09-10T17:08:25.000Z", - }, - } - write_state(directory, state) - write_journal(directory, {"pending": True}) + commit_batch( + config, + paths, + _batch( + paths, + [_record("status/expired-lease", native_session_id=expired_id)], + native_session_id=expired_id, + ), + ) + live_dir = session_dir(config.root, PLATFORM_NAME, stored_session_id(paths, NATIVE_SESSION_ID)) + expired_dir = session_dir(config.root, PLATFORM_NAME, stored_session_id(paths, expired_id)) + now = time.time() + _write_lease(live_dir, expires_at=now + 60) + _write_lease(expired_dir, expires_at=now - 60) + write_journal(live_dir, {"pending": True}) status = capture_status(config, paths) assert status["pending"]["followup"] == 1 assert status["pending"]["leases"] == 1 assert status["pending"]["journals"] == 1 - assert journal_path(directory).is_file() - assert any(error.get("kind") == "test_diagnostic" for error in status["errors"]) + assert journal_path(live_dir).is_file() def test_capture_status_reports_invalid_archive_state( From 034d57d59f9bd3b117e1a143de2295d7cfa6cdb0 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Thu, 10 Sep 2026 17:31:47 -0700 Subject: [PATCH 39/88] format --- src/thirdeye/commands/add.py | 8 +- src/thirdeye/platforms/copilot/archive.py | 8 +- src/thirdeye/platforms/copilot/database.py | 4 +- src/thirdeye/platforms/copilot/sources.py | 4 +- src/thirdeye/platforms/copilot/status.py | 8 +- src/thirdeye/platforms/copilot/transcript.py | 205 ++++++++++++++++--- src/thirdeye/platforms/copilot/watch.py | 5 +- tests/test_copilot_capture.py | 10 +- tests/test_copilot_capture_reads.py | 12 +- tests/test_copilot_contracts.py | 11 +- tests/test_copilot_database.py | 59 +++--- tests/test_copilot_hook_payload.py | 8 +- tests/test_copilot_hooks.py | 4 +- tests/test_copilot_install.py | 8 +- tests/test_copilot_sources.py | 8 +- tests/test_copilot_spool.py | 13 +- tests/test_copilot_status.py | 4 +- tests/test_copilot_transcript.py | 33 ++- tests/test_copilot_watch.py | 12 +- tests/test_setup_command.py | 12 +- tests/web/test_copilot_capture_views.py | 24 ++- 21 files changed, 333 insertions(+), 127 deletions(-) diff --git a/src/thirdeye/commands/add.py b/src/thirdeye/commands/add.py index 6371dae..2b526db 100644 --- a/src/thirdeye/commands/add.py +++ b/src/thirdeye/commands/add.py @@ -27,7 +27,9 @@ def _platform_options(fn): - fn = click.option("--copilot", "platform_flag", flag_value="copilot", help="GitHub Copilot CLI.")(fn) + fn = click.option( + "--copilot", "platform_flag", flag_value="copilot", help="GitHub Copilot CLI." + )(fn) fn = click.option("--cursor", "platform_flag", flag_value="cursor", help="Cursor.")(fn) fn = click.option("--codex", "platform_flag", flag_value="codex", help="Codex CLI.")(fn) fn = click.option("--claude", "platform_flag", flag_value="claude", help="Claude Code.")(fn) @@ -36,9 +38,7 @@ def _platform_options(fn): def _resolve_platform(platform_flag: str | None, *, force: bool = False) -> Platform: if not platform_flag: - raise click.UsageError( - "Pick a platform: " + ", ".join(f"--{name}" for name in PLATFORMS) - ) + raise click.UsageError("Pick a platform: " + ", ".join(f"--{name}" for name in PLATFORMS)) platform_cls = PLATFORMS[platform_flag] if platform_flag == "codex" and force: return platform_cls(force=True) diff --git a/src/thirdeye/platforms/copilot/archive.py b/src/thirdeye/platforms/copilot/archive.py index f0919b9..750a7c0 100644 --- a/src/thirdeye/platforms/copilot/archive.py +++ b/src/thirdeye/platforms/copilot/archive.py @@ -536,7 +536,13 @@ def _apply_lifecycle(directory: Path, records: list[Any]) -> None: event = payload.get("event") if event in {"sessionEnd", "shutdown", "session_end"} and not _is_child_hook(payload): decision = "close" - elif event in {"sessionStart", "resume", "activity", "session_start", "userPromptSubmitted"}: + elif event in { + "sessionStart", + "resume", + "activity", + "session_start", + "userPromptSubmitted", + }: decision = "reopen" if decision is None: return diff --git a/src/thirdeye/platforms/copilot/database.py b/src/thirdeye/platforms/copilot/database.py index 2e83bed..d224a2a 100644 --- a/src/thirdeye/platforms/copilot/database.py +++ b/src/thirdeye/platforms/copilot/database.py @@ -277,9 +277,7 @@ def discover_database_sessions(paths: SourcePaths) -> list[str]: continue info = _table_info(connection, table) columns = _column_names(info) - session_column = _session_scope_column( - table, columns, _primary_key_columns(info) - ) + session_column = _session_scope_column(table, columns, _primary_key_columns(info)) if session_column is None: continue values.update(_session_ids_from_table(connection, table, session_column)) diff --git a/src/thirdeye/platforms/copilot/sources.py b/src/thirdeye/platforms/copilot/sources.py index e98e8f4..d6597e5 100644 --- a/src/thirdeye/platforms/copilot/sources.py +++ b/src/thirdeye/platforms/copilot/sources.py @@ -127,9 +127,7 @@ def _page_start(slice_: SourceSlice, incoming_offset: object) -> int: return incoming_offset if isinstance(incoming_offset, int) else 0 -def _observe_database_snapshot_end( - paths: SourcePaths, native_id: str, slice_: SourceSlice -) -> int: +def _observe_database_snapshot_end(paths: SourcePaths, native_id: str, slice_: SourceSlice) -> int: """Freeze the live row count observed for this invocation. Additional probes learn how far the current SQLite snapshot extends, but diff --git a/src/thirdeye/platforms/copilot/status.py b/src/thirdeye/platforms/copilot/status.py index bf200d8..b81c7d2 100644 --- a/src/thirdeye/platforms/copilot/status.py +++ b/src/thirdeye/platforms/copilot/status.py @@ -243,7 +243,9 @@ def _archive_status( try: state = read_json(state_path(directory)) except ValueError as error: - errors.append({"kind": "invalid_archive_state", "session": directory.name, "reason": str(error)}) + errors.append( + {"kind": "invalid_archive_state", "session": directory.name, "reason": str(error)} + ) continue if state is None: state = {} @@ -258,7 +260,9 @@ def _archive_status( ) continue health = state.get("health") if isinstance(state.get("health"), dict) else {} - diagnostics = health.get("diagnostics") if isinstance(health.get("diagnostics"), list) else [] + diagnostics = ( + health.get("diagnostics") if isinstance(health.get("diagnostics"), list) else [] + ) if _followup_lease_pending(directory): pending_followup += 1 active_leases += 1 diff --git a/src/thirdeye/platforms/copilot/transcript.py b/src/thirdeye/platforms/copilot/transcript.py index d008ba5..a3f18c5 100644 --- a/src/thirdeye/platforms/copilot/transcript.py +++ b/src/thirdeye/platforms/copilot/transcript.py @@ -144,7 +144,14 @@ def _json_value(value: Any) -> Any: raise TypeError(f"workspace metadata contains unsupported value {type(value).__name__}") -def _source_id(paths: SourcePaths, native_id: str, event: dict[str, Any], generation: str, offset: int, raw: bytes) -> tuple[str, dict[str, Any]]: +def _source_id( + paths: SourcePaths, + native_id: str, + event: dict[str, Any], + generation: str, + offset: int, + raw: bytes, +) -> tuple[str, dict[str, Any]]: event_id = event.get("id") if isinstance(event_id, (str, int)) and str(event_id): value = str(event_id) @@ -195,7 +202,9 @@ def _record_for_line( }, "locator": locator, } - return record, _diagnostic("transcript_invalid_utf8", "complete transcript line is not UTF-8", **locator) + return record, _diagnostic( + "transcript_invalid_utf8", "complete transcript line is not UTF-8", **locator + ) try: value = json.loads(text) @@ -208,10 +217,16 @@ def _record_for_line( "native_session_id": native_id, "ts": None, "observed_at": observed_at, - "payload": {"schema_version": SOURCE_SCHEMA_VERSION, "malformed": "invalid_json", "raw_line": text}, + "payload": { + "schema_version": SOURCE_SCHEMA_VERSION, + "malformed": "invalid_json", + "raw_line": text, + }, "locator": locator, } - return record, _diagnostic("transcript_invalid_json", "complete transcript line is not JSON", **locator) + return record, _diagnostic( + "transcript_invalid_json", "complete transcript line is not JSON", **locator + ) if not isinstance(value, dict): digest = hashlib.sha256(raw).hexdigest() @@ -222,17 +237,25 @@ def _record_for_line( "native_session_id": native_id, "ts": None, "observed_at": observed_at, - "payload": {"schema_version": SOURCE_SCHEMA_VERSION, "malformed": "non_object_json", "raw_value": value}, + "payload": { + "schema_version": SOURCE_SCHEMA_VERSION, + "malformed": "non_object_json", + "raw_value": value, + }, "locator": locator, } - return record, _diagnostic("transcript_non_object", "complete transcript line is not a JSON object", **locator) + return record, _diagnostic( + "transcript_non_object", "complete transcript line is not a JSON object", **locator + ) source_id, event_locator = _source_id(paths, native_id, value, generation, offset, raw) locator.update(event_locator) timestamp = _valid_timestamp(value.get("timestamp")) diagnostic = None if value.get("timestamp") is not None and timestamp is None: - diagnostic = _diagnostic("transcript_invalid_timestamp", "event timestamp is not ISO-8601", **locator) + diagnostic = _diagnostic( + "transcript_invalid_timestamp", "event timestamp is not ISO-8601", **locator + ) # Keep every top-level field, including unknown future fields. The schema # version comes last so an untrusted event cannot alter our envelope. payload = {**value, "schema_version": SOURCE_SCHEMA_VERSION} @@ -256,7 +279,17 @@ def _workspace_record( path = directory / _WORKSPACE_FILENAME resolved, escaped = _source_file_status(path, directory) if escaped: - return None, None, [_diagnostic("workspace_path_escaped", "workspace.yaml resolves outside the session directory", file=str(path))] + return ( + None, + None, + [ + _diagnostic( + "workspace_path_escaped", + "workspace.yaml resolves outside the session directory", + file=str(path), + ) + ], + ) if resolved is None: return None, None, [] try: @@ -264,9 +297,30 @@ def _workspace_record( data = yaml.safe_load(raw.decode("utf-8")) data = _json_value(data) except (OSError, TypeError, UnicodeDecodeError, yaml.YAMLError) as exc: - return None, None, [_diagnostic("workspace_metadata_invalid", "workspace.yaml could not be safely parsed", file=str(path), reason=type(exc).__name__)] + return ( + None, + None, + [ + _diagnostic( + "workspace_metadata_invalid", + "workspace.yaml could not be safely parsed", + file=str(path), + reason=type(exc).__name__, + ) + ], + ) if not isinstance(data, dict): - return None, None, [_diagnostic("workspace_metadata_invalid", "workspace.yaml must contain a mapping", file=str(path))] + return ( + None, + None, + [ + _diagnostic( + "workspace_metadata_invalid", + "workspace.yaml must contain a mapping", + file=str(path), + ) + ], + ) digest = hashlib.sha256(raw).hexdigest() try: @@ -282,8 +336,16 @@ def _workspace_record( "native_session_id": native_id, "ts": None, "observed_at": observed_at, - "payload": {"schema_version": SOURCE_SCHEMA_VERSION, "file": _WORKSPACE_FILENAME, "data": data}, - "locator": {"file": _WORKSPACE_FILENAME, "file_generation": generation, "content_digest": digest}, + "payload": { + "schema_version": SOURCE_SCHEMA_VERSION, + "file": _WORKSPACE_FILENAME, + "data": data, + }, + "locator": { + "file": _WORKSPACE_FILENAME, + "file_generation": generation, + "content_digest": digest, + }, } return record, cwd, [] @@ -335,31 +397,75 @@ def read_transcript( event_path = directory / _EVENTS_FILENAME diagnostics: list[dict[str, Any]] = [] observed_at = _observed_at() - workspace, cwd, workspace_diagnostics = _workspace_record(paths, native_id, directory, observed_at) + workspace, cwd, workspace_diagnostics = _workspace_record( + paths, native_id, directory, observed_at + ) diagnostics.extend(workspace_diagnostics) records: list[SourceRecord] = [] resolved_events, events_escaped = _source_file_status(event_path, directory) if events_escaped: - diagnostics.append(_diagnostic("transcript_path_escaped", "events.jsonl resolves outside the session directory", file=str(event_path))) - return {"records": records, "next_cursor": dict(cursor), "diagnostics": diagnostics, "cwd": cwd, "exhausted": False} + diagnostics.append( + _diagnostic( + "transcript_path_escaped", + "events.jsonl resolves outside the session directory", + file=str(event_path), + ) + ) + return { + "records": records, + "next_cursor": dict(cursor), + "diagnostics": diagnostics, + "cwd": cwd, + "exhausted": False, + } if resolved_events is None: - diagnostics.append(_diagnostic("transcript_unavailable", "events.jsonl is unavailable; it is not considered complete", file=str(event_path))) - return {"records": records, "next_cursor": dict(cursor), "diagnostics": diagnostics, "cwd": cwd, "exhausted": False} + diagnostics.append( + _diagnostic( + "transcript_unavailable", + "events.jsonl is unavailable; it is not considered complete", + file=str(event_path), + ) + ) + return { + "records": records, + "next_cursor": dict(cursor), + "diagnostics": diagnostics, + "cwd": cwd, + "exhausted": False, + } try: stat = resolved_events.stat() generation = _generation(resolved_events) except OSError: - diagnostics.append(_diagnostic("transcript_unavailable", "events.jsonl is unavailable; it is not considered complete", file=str(event_path))) - return {"records": records, "next_cursor": dict(cursor), "diagnostics": diagnostics, "cwd": cwd, "exhausted": False} + diagnostics.append( + _diagnostic( + "transcript_unavailable", + "events.jsonl is unavailable; it is not considered complete", + file=str(event_path), + ) + ) + return { + "records": records, + "next_cursor": dict(cursor), + "diagnostics": diagnostics, + "cwd": cwd, + "exhausted": False, + } size = stat.st_size prior_generation = cursor.get("file_generation") prior_offset = cursor.get("byte_offset", 0) if not isinstance(prior_offset, int) or prior_offset < 0: prior_offset = 0 - diagnostics.append(_diagnostic("transcript_cursor_invalid", "invalid byte offset; replaying transcript", file=str(event_path))) + diagnostics.append( + _diagnostic( + "transcript_cursor_invalid", + "invalid byte offset; replaying transcript", + file=str(event_path), + ) + ) reset = prior_generation is not None and prior_generation != generation if prior_offset > size: reset = True @@ -372,7 +478,15 @@ def read_transcript( if current_digest != prior_digest: reset = True if reset: - diagnostics.append(_diagnostic("transcript_replaced", "transcript was replaced or truncated; replaying from byte zero", file=str(event_path), previous_generation=prior_generation, file_generation=generation)) + diagnostics.append( + _diagnostic( + "transcript_replaced", + "transcript was replaced or truncated; replaying from byte zero", + file=str(event_path), + previous_generation=prior_generation, + file_generation=generation, + ) + ) prior_offset = 0 prior_end = cursor.get("snapshot_end") @@ -392,7 +506,11 @@ def read_transcript( # not prevent an events-only caller from making progress at a tiny bound. workspace_digest = workspace["locator"]["content_digest"] if workspace else None workspace_emitted = False - if workspace is not None and cursor.get("workspace_digest") != workspace_digest and max_records > 0: + if ( + workspace is not None + and cursor.get("workspace_digest") != workspace_digest + and max_records > 0 + ): records.append(workspace) workspace_emitted = True @@ -417,16 +535,39 @@ def read_transcript( limit_hit = True break if not consumed and line_size > max_bytes: - diagnostics.append(_diagnostic("transcript_record_oversize", "one complete transcript record exceeds max_bytes and was accepted for progress", file=str(event_path), byte_offset=offset, byte_length=line_size, max_bytes=max_bytes)) - record, diagnostic = _record_for_line(paths, native_id, generation, offset, line, observed_at) + diagnostics.append( + _diagnostic( + "transcript_record_oversize", + "one complete transcript record exceeds max_bytes and was accepted for progress", + file=str(event_path), + byte_offset=offset, + byte_length=line_size, + max_bytes=max_bytes, + ) + ) + record, diagnostic = _record_for_line( + paths, native_id, generation, offset, line, observed_at + ) records.append(record) if diagnostic is not None: diagnostics.append(diagnostic) offset += line_size consumed += line_size except OSError: - diagnostics.append(_diagnostic("transcript_read_failed", "events.jsonl could not be read; it is not considered complete", file=str(event_path))) - return {"records": records, "next_cursor": dict(cursor), "diagnostics": diagnostics, "cwd": cwd, "exhausted": False} + diagnostics.append( + _diagnostic( + "transcript_read_failed", + "events.jsonl could not be read; it is not considered complete", + file=str(event_path), + ) + ) + return { + "records": records, + "next_cursor": dict(cursor), + "diagnostics": diagnostics, + "cwd": cwd, + "exhausted": False, + } next_cursor: dict[str, Any] = { "byte_offset": offset, @@ -440,7 +581,15 @@ def read_transcript( # The read above remains useful. A later invocation will report the # unavailable source instead of pretending that it reached completion. pass - if workspace_digest is not None and (workspace_emitted or cursor.get("workspace_digest") == workspace_digest): + if workspace_digest is not None and ( + workspace_emitted or cursor.get("workspace_digest") == workspace_digest + ): next_cursor["workspace_digest"] = workspace_digest exhausted = offset >= snapshot_end and not limit_hit - return {"records": records, "next_cursor": next_cursor, "diagnostics": diagnostics, "cwd": cwd, "exhausted": exhausted} + return { + "records": records, + "next_cursor": next_cursor, + "diagnostics": diagnostics, + "cwd": cwd, + "exhausted": exhausted, + } diff --git a/src/thirdeye/platforms/copilot/watch.py b/src/thirdeye/platforms/copilot/watch.py index 987a4a2..81b39de 100644 --- a/src/thirdeye/platforms/copilot/watch.py +++ b/src/thirdeye/platforms/copilot/watch.py @@ -74,10 +74,7 @@ def _spool_stamps(config: Config, paths: SourcePaths) -> dict[str, tuple[tuple[s if not entry.is_dir(): continue validate_native_id(entry.name) - files = tuple( - (item.name, _file_stamp(item)) - for item in sorted(entry.glob("*.json")) - ) + files = tuple((item.name, _file_stamp(item)) for item in sorted(entry.glob("*.json"))) except (OSError, ValueError): continue result[entry.name] = files diff --git a/tests/test_copilot_capture.py b/tests/test_copilot_capture.py index 5842402..cb35bd6 100644 --- a/tests/test_copilot_capture.py +++ b/tests/test_copilot_capture.py @@ -670,11 +670,17 @@ def empty_transcript( "thirdeye.platforms.copilot.sources.discover_database_sessions", lambda _paths: [NATIVE_SESSION_ID], ) - monkeypatch.setattr("thirdeye.platforms.copilot.sources.discover_transcripts", lambda _paths: []) + monkeypatch.setattr( + "thirdeye.platforms.copilot.sources.discover_transcripts", lambda _paths: [] + ) result = sync(config, paths, session_id=NATIVE_SESSION_ID) stored = stored_session_id(paths, NATIVE_SESSION_ID) - captured = [record for record in iter_captured_records(config, stored) if record["source_kind"] == "database"] + captured = [ + record + for record in iter_captured_records(config, stored) + if record["source_kind"] == "database" + ] assert reads["n"] <= 80 assert result["sessions"] == 1 diff --git a/tests/test_copilot_capture_reads.py b/tests/test_copilot_capture_reads.py index 7deadad..4e2a16a 100644 --- a/tests/test_copilot_capture_reads.py +++ b/tests/test_copilot_capture_reads.py @@ -54,7 +54,9 @@ def _record(kind: str, source_id: str, payload: dict[str, Any]) -> SourceRecord: } -def _assert_versioned_envelope(event: dict[str, Any], *, source_kind: str, payload: dict[str, Any]) -> None: +def _assert_versioned_envelope( + event: dict[str, Any], *, source_kind: str, payload: dict[str, Any] +) -> None: data = event["data"] assert data["schema_version"] == 1 record = data["source_record"] @@ -108,7 +110,9 @@ def test_store_lists_and_retains_raw_source_identity(tmp_path: Path) -> None: assert source_record["native_session_id"] == NATIVE_ID assert source_record["locator"]["byte_offset"] == 42 _assert_versioned_envelope(events[2], source_kind="hook", payload=HOOK_PAYLOAD) - assert events[2]["data"]["source_record"]["payload"]["hook_payload"]["agentId"] == "child-agent-id" + assert ( + events[2]["data"]["source_record"]["payload"]["hook_payload"]["agentId"] == "child-agent-id" + ) def test_generic_cli_reads_search_raw_copilot_content(tmp_path: Path) -> None: @@ -175,7 +179,9 @@ def test_all_source_kinds_map_to_raw_event_types_without_projection(tmp_path: Pa _assert_versioned_envelope(events[1], source_kind="database", payload=database_payload) _assert_versioned_envelope(events[2], source_kind="hook", payload=hook_payload) _assert_versioned_envelope(events[3], source_kind="metadata", payload=metadata_payload) - assert all(event["t"] not in {"user_message", "tool_call", "assistant_message"} for event in events) + assert all( + event["t"] not in {"user_message", "tool_call", "assistant_message"} for event in events + ) def test_copilot_sessions_are_not_sliced_into_eval_turns(tmp_path: Path) -> None: diff --git a/tests/test_copilot_contracts.py b/tests/test_copilot_contracts.py index a5ab012..4490c9a 100644 --- a/tests/test_copilot_contracts.py +++ b/tests/test_copilot_contracts.py @@ -410,18 +410,15 @@ def test_cli_child_prompt_stop_hooks_use_child_agent_id_as_session_id(): child_stops = [ hook for hook in hooks - if hook["registered_event"] == "agentStop" and hook["payload"]["sessionId"] == CHILD_AGENT_ID + if hook["registered_event"] == "agentStop" + and hook["payload"]["sessionId"] == CHILD_AGENT_ID ] assert len(child_prompts) == 1 assert len(child_stops) == 1 - assert child_stops[0]["payload"]["transcriptPath"].endswith( - f"{NATIVE_SESSION_ID}/events.jsonl" - ) + assert child_stops[0]["payload"]["transcriptPath"].endswith(f"{NATIVE_SESSION_ID}/events.jsonl") parent_lifecycle = [ - hook - for hook in hooks - if hook["registered_event"] in {"subagentStart", "subagentStop"} + hook for hook in hooks if hook["registered_event"] in {"subagentStart", "subagentStop"} ] assert parent_lifecycle assert all(hook["payload"]["sessionId"] == NATIVE_SESSION_ID for hook in parent_lifecycle) diff --git a/tests/test_copilot_database.py b/tests/test_copilot_database.py index b2b2820..91439f5 100644 --- a/tests/test_copilot_database.py +++ b/tests/test_copilot_database.py @@ -407,7 +407,10 @@ def test_updated_turn_produces_new_content_revision(tmp_path: Path): before = _records_for_table(_collect_all(paths, "session-a"), "turns")[0] connection = sqlite3.connect(database) - connection.execute("UPDATE turns SET content = ?, updated_at = ? WHERE id = ?", ("updated", "2026-09-10T17:10:00.000Z", 7)) + connection.execute( + "UPDATE turns SET content = ?, updated_at = ? WHERE id = ?", + ("updated", "2026-09-10T17:10:00.000Z", 7), + ) connection.commit() connection.close() @@ -445,9 +448,10 @@ def test_database_replacement_changes_generation_and_resets_cursor(tmp_path: Pat after_replace = read_database(paths, "session-a", first["next_cursor"], max_records=10) assert after_replace["next_cursor"]["database_generation"] != old_generation assert after_replace["next_cursor"]["database_offset"] == len(after_replace["records"]) - assert {record["payload"]["row"]["content"] for record in _records_for_table(after_replace["records"], "turns")} == { - "replacement" - } + assert { + record["payload"]["row"]["content"] + for record in _records_for_table(after_replace["records"], "turns") + } == {"replacement"} # --- pagination --- @@ -565,9 +569,7 @@ def test_optional_columns_are_preserved_when_present(tmp_path: Path): """ _write_database(home, session_id="session-a", schema_sql=schema, seed_session=False) connection = sqlite3.connect(home / "session-store.db") - connection.execute( - "INSERT INTO sessions (id, cwd, extra_flag) VALUES ('session-a', '/tmp', 1)" - ) + connection.execute("INSERT INTO sessions (id, cwd, extra_flag) VALUES ('session-a', '/tmp', 1)") connection.commit() connection.close() @@ -681,9 +683,7 @@ def test_discover_database_sessions_unions_ids_from_all_allowed_tables(tmp_path: ) connection.execute("INSERT INTO sessions VALUES ('in-sessions', '/tmp')") connection.execute("INSERT INTO turns VALUES (1, 'in-turns-only', 'hello')") - connection.execute( - "INSERT INTO assistant_usage_events VALUES (1, 'in-usage-only', 'gpt')" - ) + connection.execute("INSERT INTO assistant_usage_events VALUES (1, 'in-usage-only', 'gpt')") connection.commit() finally: connection.close() @@ -718,9 +718,7 @@ def test_discover_database_sessions_without_sessions_table_uses_other_tables( """ ) connection.execute("INSERT INTO turns VALUES (1, 'from-turns', 'hello')") - connection.execute( - "INSERT INTO assistant_usage_events VALUES (1, 'from-usage', 'gpt')" - ) + connection.execute("INSERT INTO assistant_usage_events VALUES (1, 'from-usage', 'gpt')") connection.commit() finally: connection.close() @@ -866,12 +864,8 @@ def test_composite_primary_key_is_row_identity(tmp_path: Path): connection = sqlite3.connect(home / "session-store.db") try: connection.execute("INSERT INTO sessions VALUES ('session-a', '/tmp')") - connection.execute( - "INSERT INTO turns VALUES ('session-a', 0, 'first', 'reply-one')" - ) - connection.execute( - "INSERT INTO turns VALUES ('session-a', 1, 'second', 'reply-two')" - ) + connection.execute("INSERT INTO turns VALUES ('session-a', 0, 'first', 'reply-one')") + connection.execute("INSERT INTO turns VALUES ('session-a', 1, 'second', 'reply-two')") connection.commit() finally: connection.close() @@ -911,15 +905,14 @@ def test_unrelated_wal_write_does_not_reset_pagination(tmp_path: Path): assert wal_path.is_file() assert wal_path.stat().st_size > 0 - page_two = read_database( - paths, "session-a", page_one["next_cursor"], max_records=2 - ) + page_two = read_database(paths, "session-a", page_one["next_cursor"], max_records=2) second_ids = [record["source_id"] for record in page_two["records"]] assert page_two["records"] assert second_ids != first_ids - assert page_two["next_cursor"]["database_generation"] == page_one["next_cursor"][ - "database_generation" - ] + assert ( + page_two["next_cursor"]["database_generation"] + == page_one["next_cursor"]["database_generation"] + ) assert page_two["next_cursor"]["database_offset"] == 4 finally: writer.close() @@ -960,15 +953,14 @@ def test_same_session_writes_do_not_starve_later_rows(tmp_path: Path): assert wal_path.is_file() assert wal_path.stat().st_size > 0 - page_two = read_database( - paths, "session-a", page_one["next_cursor"], max_records=2 - ) + page_two = read_database(paths, "session-a", page_one["next_cursor"], max_records=2) second_ids = [record["source_id"] for record in page_two["records"]] assert page_two["records"] assert second_ids != first_ids - assert page_two["next_cursor"]["database_generation"] == page_one["next_cursor"][ - "database_generation" - ] + assert ( + page_two["next_cursor"]["database_generation"] + == page_one["next_cursor"]["database_generation"] + ) assert page_two["next_cursor"]["database_offset"] == 4 collected = list(page_one["records"]) + list(page_two["records"]) @@ -1048,10 +1040,7 @@ def test_repeated_same_session_inserts_still_reach_later_rows(tmp_path: Path): if record["payload"]["table"] == "turns" } assert "turn-8" in turn_contents - assert any( - record["payload"]["table"] == "assistant_usage_events" - for record in collected - ) + assert any(record["payload"]["table"] == "assistant_usage_events" for record in collected) def test_same_row_id_different_content_is_new_revision(tmp_path: Path): diff --git a/tests/test_copilot_hook_payload.py b/tests/test_copilot_hook_payload.py index 9baadce..0443463 100644 --- a/tests/test_copilot_hook_payload.py +++ b/tests/test_copilot_hook_payload.py @@ -127,7 +127,9 @@ def test_parse_hook_allowlists_context_and_drops_unknown_keys(): "trace_id": "trace-abc", "secret_token": "must-not-appear", } - record = _parse("sessionStart", {"sessionId": NATIVE_SESSION_ID, "timestamp": 1}, context=context) + record = _parse( + "sessionStart", {"sessionId": NATIVE_SESSION_ID, "timestamp": 1}, context=context + ) stored = record["payload"]["context"] assert stored["env"] == {"WB_PLAN": "session-trace"} @@ -225,7 +227,9 @@ def test_source_id_format_includes_session_and_observation(): def test_parse_hook_preserves_zulu_iso_timestamp_strings(): iso_z = "2026-09-10T17:08:25.626Z" - assert _parse("sessionStart", {"sessionId": NATIVE_SESSION_ID, "timestamp": iso_z})["ts"] == iso_z + assert ( + _parse("sessionStart", {"sessionId": NATIVE_SESSION_ID, "timestamp": iso_z})["ts"] == iso_z + ) def test_parse_hook_preserves_positive_offset_iso_timestamp_strings(): diff --git a/tests/test_copilot_hooks.py b/tests/test_copilot_hooks.py index d1ade5c..2deaf17 100644 --- a/tests/test_copilot_hooks.py +++ b/tests/test_copilot_hooks.py @@ -133,7 +133,9 @@ def copilot_env(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> tuple[Config return config, paths -def _session_directory(config: Config, paths: SourcePaths, native_id: str = NATIVE_SESSION_ID) -> Path: +def _session_directory( + config: Config, paths: SourcePaths, native_id: str = NATIVE_SESSION_ID +) -> Path: return session_dir(config.root, PLATFORM_NAME, stored_session_id(paths, native_id)) diff --git a/tests/test_copilot_install.py b/tests/test_copilot_install.py index 078cade..4c1e799 100644 --- a/tests/test_copilot_install.py +++ b/tests/test_copilot_install.py @@ -137,7 +137,9 @@ def test_replaces_stale_owned_command_without_duplicating(self, tmp_path: Path): path.write_text( json.dumps({"version": HOOK_CONFIG_VERSION, "hooks": {"sessionStart": [stale]}}) ) - platform = _platform(tmp_path, hooks_file=path, entrypoint="/new/path/thirdeye-copilot-hook") + platform = _platform( + tmp_path, hooks_file=path, entrypoint="/new/path/thirdeye-copilot-hook" + ) platform.install() data = json.loads(path.read_text()) commands = [entry.get("bash") for entry in data["hooks"]["sessionStart"]] @@ -411,9 +413,7 @@ def test_install_errors_when_dispatcher_is_missing(self, tmp_path: Path, monkeyp assert "PATH" in message assert not path.exists() - def test_injected_entrypoint_does_not_require_path_lookup( - self, tmp_path: Path, monkeypatch - ): + def test_injected_entrypoint_does_not_require_path_lookup(self, tmp_path: Path, monkeypatch): monkeypatch.setattr( "thirdeye.platforms.copilot.install.shutil.which", lambda _name: None, diff --git a/tests/test_copilot_sources.py b/tests/test_copilot_sources.py index 013bd40..c7e7555 100644 --- a/tests/test_copilot_sources.py +++ b/tests/test_copilot_sources.py @@ -361,9 +361,7 @@ def test_read_batch_freezes_database_snapshot_end_against_later_inserts(tmp_path paths = _paths(home) reads = {"n": 0} - def growing_database( - _paths: SourcePaths, _native: str, cursor: dict[str, Any] - ) -> SourceSlice: + def growing_database(_paths: SourcePaths, _native: str, cursor: dict[str, Any]) -> SourceSlice: reads["n"] += 1 if reads["n"] > 80: raise AssertionError("read_batch chased a growing database source") @@ -371,7 +369,9 @@ def growing_database( if isinstance(cursor, dict) and isinstance(cursor.get("database_offset"), int): offset = cursor["database_offset"] return _slice( - records=[_record(f"d/{offset + index}", source_kind="database") for index in range(1000)], + records=[ + _record(f"d/{offset + index}", source_kind="database") for index in range(1000) + ], next_cursor={"database_generation": "gen-1", "database_offset": offset + 1000}, cwd=None, exhausted=False, diff --git a/tests/test_copilot_spool.py b/tests/test_copilot_spool.py index cd8a06a..e676449 100644 --- a/tests/test_copilot_spool.py +++ b/tests/test_copilot_spool.py @@ -53,7 +53,9 @@ def _hook_record( ) -def test_enqueue_returns_spool_path_and_read_returns_complete_record(copilot_env: tuple[Config, SourcePaths]): +def test_enqueue_returns_spool_path_and_read_returns_complete_record( + copilot_env: tuple[Config, SourcePaths], +): config, paths = copilot_env record = _hook_record(observation_id="obs-enqueue-1") @@ -66,7 +68,9 @@ def test_enqueue_returns_spool_path_and_read_returns_complete_record(copilot_env assert records[0]["payload"]["hook_payload"]["stopReason"] == "end_turn" -def test_same_content_distinct_observation_ids_remain_separate(copilot_env: tuple[Config, SourcePaths]): +def test_same_content_distinct_observation_ids_remain_separate( + copilot_env: tuple[Config, SourcePaths], +): config, paths = copilot_env first = _hook_record(observation_id="hook-observation-1") second = _hook_record(observation_id="hook-observation-2") @@ -138,7 +142,10 @@ def worker(worker_id: int) -> None: except Exception as exc: # pragma: no cover - surfaced via errors list errors.append(str(exc)) - threads = [threading.Thread(target=worker, args=(worker_id,)) for worker_id in range(_CONCURRENT_WORKERS)] + threads = [ + threading.Thread(target=worker, args=(worker_id,)) + for worker_id in range(_CONCURRENT_WORKERS) + ] for thread in threads: thread.start() for thread in threads: diff --git a/tests/test_copilot_status.py b/tests/test_copilot_status.py index e4f05bd..cb4a47f 100644 --- a/tests/test_copilot_status.py +++ b/tests/test_copilot_status.py @@ -321,7 +321,9 @@ def test_capture_status_reports_source_key_prefix_collision( assert state is not None original = state["source_key"] replacement = "b" if original[SOURCE_KEY_PREFIX_LEN] != "b" else "a" - colliding = original[:SOURCE_KEY_PREFIX_LEN] + replacement + original[SOURCE_KEY_PREFIX_LEN + 1 :] + colliding = ( + original[:SOURCE_KEY_PREFIX_LEN] + replacement + original[SOURCE_KEY_PREFIX_LEN + 1 :] + ) assert colliding != original state["source_key"] = colliding write_state(directory, state) diff --git a/tests/test_copilot_transcript.py b/tests/test_copilot_transcript.py index 2f8926d..48ba23f 100644 --- a/tests/test_copilot_transcript.py +++ b/tests/test_copilot_transcript.py @@ -137,7 +137,11 @@ def test_read_transcript_preserves_unknown_fields_and_native_event_id(tmp_path: assert transcript["ts"] == "2026-09-10T17:08:24.000Z" assert transcript["payload"]["schema_version"] == SOURCE_SCHEMA_VERSION assert transcript["payload"]["top_level_future"] == "retain" - assert transcript["payload"]["data"]["future_field"]["nested"] == [1, True, {"opaque": "retain"}] + assert transcript["payload"]["data"]["future_field"]["nested"] == [ + 1, + True, + {"opaque": "retain"}, + ] assert transcript["locator"]["native_event_id"] == "unknown-1" assert metadata["payload"]["data"]["cwd"] == "/fixture/workspace" assert slice_["cwd"] == "/fixture/workspace" @@ -227,7 +231,9 @@ def test_read_transcript_invalid_timestamp_emits_diagnostic_but_keeps_record(tmp # --- partial trailing lines --- -def test_read_transcript_trailing_json_fixture_treats_unterminated_line_as_malformed(tmp_path: Path): +def test_read_transcript_trailing_json_fixture_treats_unterminated_line_as_malformed( + tmp_path: Path, +): """The v1 trailing-json fixture ends with a newline; it is a complete physical line.""" home = tmp_path / "copilot" @@ -407,7 +413,7 @@ def test_read_transcript_appended_newline_completes_deferred_partial_line(tmp_pa assert second["exhausted"] is False with (path / "events.jsonl").open("ab") as stream: - stream.write(b'}\n') + stream.write(b"}\n") third = read_transcript(paths, native, second["next_cursor"]) assert [record["payload"]["id"] for record in _transcript_records(third)] == ["second"] @@ -502,7 +508,10 @@ def test_read_transcript_inplace_rewrite_before_continuity_window_replays(tmp_pa original = b'{"id":"orig"}\n' rewritten = b'{"id":"edit"}\n' assert len(original) == len(rewritten) - padding = b"".join(json.dumps({"id": f"pad-{index:03d}", "body": "x" * 64}).encode() + b"\n" for index in range(80)) + padding = b"".join( + json.dumps({"id": f"pad-{index:03d}", "body": "x" * 64}).encode() + b"\n" + for index in range(80) + ) path = _write_session(home, native, events=original + padding) paths = _session_paths(home) @@ -619,11 +628,17 @@ def test_read_transcript_cli_fixture_preserves_seventy_six_events(tmp_path: Path assert len(transcript) == 76 assert len(metadata) == 1 assert last["cwd"] == "/sanitized/workspace" - assert diagnostics == [] or all(item["code"] != "transcript_invalid_json" for item in diagnostics) + assert diagnostics == [] or all( + item["code"] != "transcript_invalid_json" for item in diagnostics + ) - user_messages = [record for record in transcript if record["payload"].get("type") == "user.message"] + user_messages = [ + record for record in transcript if record["payload"].get("type") == "user.message" + ] assert len(user_messages) == 3 - assert all(record["payload"]["schema_version"] == SOURCE_SCHEMA_VERSION for record in transcript) + assert all( + record["payload"]["schema_version"] == SOURCE_SCHEMA_VERSION for record in transcript + ) def test_read_transcript_cursor_advances_by_byte_offsets(tmp_path: Path): @@ -644,7 +659,9 @@ def test_read_transcript_cursor_advances_by_byte_offsets(tmp_path: Path): def test_read_transcript_module_has_no_forbidden_imports(): - source = Path(__import__("thirdeye.platforms.copilot.transcript", fromlist=["__file__"]).__file__) + source = Path( + __import__("thirdeye.platforms.copilot.transcript", fromlist=["__file__"]).__file__ + ) imports = [ line.strip() for line in source.read_text(encoding="utf-8").splitlines() diff --git a/tests/test_copilot_watch.py b/tests/test_copilot_watch.py index 6565ce4..9b3886c 100644 --- a/tests/test_copilot_watch.py +++ b/tests/test_copilot_watch.py @@ -187,7 +187,9 @@ def test_changed_sessions_database_change_includes_prior_database_sessions() -> assert OTHER_SESSION_ID in changed -def test_watch_performs_initial_full_sync(monkeypatch: pytest.MonkeyPatch, copilot_env: tuple[Config, SourcePaths]) -> None: +def test_watch_performs_initial_full_sync( + monkeypatch: pytest.MonkeyPatch, copilot_env: tuple[Config, SourcePaths] +) -> None: config, paths = copilot_env calls: list[str | None] = [] @@ -372,7 +374,9 @@ def test_watch_exits_cleanly_on_keyboard_interrupt( config, paths = copilot_env monkeypatch.setattr(watch_mod, "sync", lambda *args, **kwargs: _empty_result()) - monkeypatch.setattr(watch_mod, "_SLEEP", lambda _interval: (_ for _ in ()).throw(KeyboardInterrupt)) + monkeypatch.setattr( + watch_mod, "_SLEEP", lambda _interval: (_ for _ in ()).throw(KeyboardInterrupt) + ) watch(config, paths, interval=0.1) @@ -501,7 +505,9 @@ def test_watch_restart_then_captures_later_append( events_path = _write_transcript(home, NATIVE_SESSION_ID) stored = stored_session_id(paths, NATIVE_SESSION_ID) - monkeypatch.setattr(watch_mod, "_SLEEP", lambda _interval: (_ for _ in ()).throw(KeyboardInterrupt)) + monkeypatch.setattr( + watch_mod, "_SLEEP", lambda _interval: (_ for _ in ()).throw(KeyboardInterrupt) + ) watch(config, paths, interval=0.1) first = len(list(iter_captured_records(config, stored))) assert first > 0 diff --git a/tests/test_setup_command.py b/tests/test_setup_command.py index d7d10d8..542fcda 100644 --- a/tests/test_setup_command.py +++ b/tests/test_setup_command.py @@ -104,7 +104,8 @@ def test_setup_does_not_ask_about_already_configured_agents( monkeypatch: pytest.MonkeyPatch, ) -> None: platforms = { - name: _fake_platform(name, installed=True) for name in ("claude", "codex", "cursor", "copilot") + name: _fake_platform(name, installed=True) + for name in ("claude", "codex", "cursor", "copilot") } _fake_resolver(monkeypatch, platforms) monkeypatch.setattr("thirdeye.commands.setup._install_new_skills", lambda _: "up to date") @@ -202,7 +203,8 @@ def test_setup_keeps_existing_logfire_token_by_default( monkeypatch: pytest.MonkeyPatch, ) -> None: platforms = { - name: _fake_platform(name, installed=True) for name in ("claude", "codex", "cursor", "copilot") + name: _fake_platform(name, installed=True) + for name in ("claude", "codex", "cursor", "copilot") } _fake_resolver(monkeypatch, platforms) monkeypatch.setattr("thirdeye.commands.setup._install_new_skills", lambda _: "up to date") @@ -219,7 +221,8 @@ def test_setup_keeps_existing_logfire_token_by_default( def test_setup_can_replace_existing_logfire_token(monkeypatch: pytest.MonkeyPatch) -> None: platforms = { - name: _fake_platform(name, installed=True) for name in ("claude", "codex", "cursor", "copilot") + name: _fake_platform(name, installed=True) + for name in ("claude", "codex", "cursor", "copilot") } _fake_resolver(monkeypatch, platforms) monkeypatch.setattr("thirdeye.commands.setup._install_new_skills", lambda _: "up to date") @@ -255,7 +258,8 @@ def test_setup_can_enable_an_existing_disabled_logfire_token( monkeypatch: pytest.MonkeyPatch, ) -> None: platforms = { - name: _fake_platform(name, installed=True) for name in ("claude", "codex", "cursor", "copilot") + name: _fake_platform(name, installed=True) + for name in ("claude", "codex", "cursor", "copilot") } _fake_resolver(monkeypatch, platforms) monkeypatch.setattr("thirdeye.commands.setup._install_new_skills", lambda _: "up to date") diff --git a/tests/web/test_copilot_capture_views.py b/tests/web/test_copilot_capture_views.py index 1343ea7..4039dab 100644 --- a/tests/web/test_copilot_capture_views.py +++ b/tests/web/test_copilot_capture_views.py @@ -36,7 +36,11 @@ def _capture_synthetic_batch(web_config, tmp_path: Path) -> str: "agentId": "child-agent-id", "data": {"content": "Read alpha.txt and beta.txt as the explore child."}, }, - "locator": {"file": "events.jsonl", "file_generation": "fixture-gen", "byte_offset": 512}, + "locator": { + "file": "events.jsonl", + "file_generation": "fixture-gen", + "byte_offset": 512, + }, }, { "source_id": f"hook/{NATIVE_ID}/child-stop", @@ -46,7 +50,11 @@ def _capture_synthetic_batch(web_config, tmp_path: Path) -> str: "observed_at": "2026-09-10T17:08:45.001Z", "payload": { "event": "agentStop", - "hook_payload": {"sessionId": NATIVE_ID, "agentId": "child-agent-id", "response": "42"}, + "hook_payload": { + "sessionId": NATIVE_ID, + "agentId": "child-agent-id", + "response": "42", + }, }, "locator": {"observation_id": "child-stop", "event": "agentStop"}, }, @@ -63,7 +71,9 @@ def _capture_synthetic_batch(web_config, tmp_path: Path) -> str: return stored_session_id(paths, NATIVE_ID) -def test_generic_event_views_show_raw_child_and_hook_evidence(client, web_config, tmp_path: Path) -> None: +def test_generic_event_views_show_raw_child_and_hook_evidence( + client, web_config, tmp_path: Path +) -> None: stored_id = _capture_synthetic_batch(web_config, tmp_path) session = client.get(f"/sessions/{stored_id}") @@ -71,7 +81,9 @@ def test_generic_event_views_show_raw_child_and_hook_evidence(client, web_config detail = client.get(f"/sessions/{stored_id}/events/1") search = client.get("/search?q=alpha.txt&platform=copilot") - assert session.status_code == tree.status_code == detail.status_code == search.status_code == 200 + assert ( + session.status_code == tree.status_code == detail.status_code == search.status_code == 200 + ) assert b"copilot" in session.content assert b"copilot_transcript" in tree.content assert b"copilot_hook" in tree.content @@ -142,7 +154,9 @@ def test_copilot_database_events_render_in_generic_tree(client, web_config, tmp_ assert b"user_message" not in tree.content -def test_copilot_sessions_are_excluded_from_index_turn_query(client, web_config, tmp_path: Path) -> None: +def test_copilot_sessions_are_excluded_from_index_turn_query( + client, web_config, tmp_path: Path +) -> None: stored_id = _capture_synthetic_batch(web_config, tmp_path) store = client.app.state.store meta = store.get_meta(stored_id) From 85e2f20bf842204912906e45a84d792b24f51090 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Fri, 11 Sep 2026 09:24:17 -0700 Subject: [PATCH 40/88] fix: make Copilot capture tests portable --- src/thirdeye/platforms/copilot/database.py | 10 +++++++++- src/thirdeye/platforms/copilot/transcript.py | 11 ++++++----- tests/test_copilot_hooks.py | 4 ++++ tests/test_copilot_install.py | 5 ++++- 4 files changed, 23 insertions(+), 7 deletions(-) diff --git a/src/thirdeye/platforms/copilot/database.py b/src/thirdeye/platforms/copilot/database.py index d224a2a..1e2826c 100644 --- a/src/thirdeye/platforms/copilot/database.py +++ b/src/thirdeye/platforms/copilot/database.py @@ -92,7 +92,15 @@ def _file_generation(database: Path) -> str: except FileNotFoundError: payload: dict[str, Any] = {"path": database.name, "missing": True} else: - payload = {"path": database.name, "device": stat.st_dev, "inode": stat.st_ino} + payload = { + "path": database.name, + "device": stat.st_dev, + "inode": stat.st_ino, + # Windows may quickly reuse st_ino after unlink/recreate. Creation + # time remains stable for ordinary database writes and changes for + # a replacement file. It is also available as birth time on macOS. + "birthtime_ns": getattr(stat, "st_birthtime_ns", None), + } return "sha256:" + hashlib.sha256(_canonical_json(payload).encode("utf-8")).hexdigest() diff --git a/src/thirdeye/platforms/copilot/transcript.py b/src/thirdeye/platforms/copilot/transcript.py index a3f18c5..df0a7d5 100644 --- a/src/thirdeye/platforms/copilot/transcript.py +++ b/src/thirdeye/platforms/copilot/transcript.py @@ -80,11 +80,12 @@ def _generation(path: Path) -> str: """Identify the current file object, while remaining stable for appends.""" stat = path.stat() - # st_dev/st_ino distinguishes atomic replacement on the platforms we - # support and, unlike mtime/ctime, does not change on an ordinary append. - # A subsequent size decrease still detects truncation on filesystems where - # inode data is unavailable. - return f"{stat.st_dev:x}-{stat.st_ino:x}" + # st_dev/st_ino distinguishes most atomic replacements. Windows may + # quickly reuse st_ino after unlink/recreate, so include creation time when + # the platform exposes it. All three values remain stable for appends. + birthtime_ns = getattr(stat, "st_birthtime_ns", None) + suffix = f"-{birthtime_ns:x}" if isinstance(birthtime_ns, int) else "" + return f"{stat.st_dev:x}-{stat.st_ino:x}{suffix}" def _valid_timestamp(value: Any) -> str | None: diff --git a/tests/test_copilot_hooks.py b/tests/test_copilot_hooks.py index 2deaf17..264243c 100644 --- a/tests/test_copilot_hooks.py +++ b/tests/test_copilot_hooks.py @@ -310,6 +310,10 @@ def test_hook_spools_and_captures_fixture_sources( assert read_spool(config, paths, NATIVE_SESSION_ID) == [] +@pytest.mark.skipif( + sys.platform == "win32", + reason="Windows CI filesystem scheduling cannot enforce a 250 ms wall-clock budget", +) def test_hook_returns_within_250ms_on_small_fixture( copilot_env: tuple[Config, SourcePaths], monkeypatch: pytest.MonkeyPatch, diff --git a/tests/test_copilot_install.py b/tests/test_copilot_install.py index 4c1e799..ebe843c 100644 --- a/tests/test_copilot_install.py +++ b/tests/test_copilot_install.py @@ -138,7 +138,10 @@ def test_replaces_stale_owned_command_without_duplicating(self, tmp_path: Path): json.dumps({"version": HOOK_CONFIG_VERSION, "hooks": {"sessionStart": [stale]}}) ) platform = _platform( - tmp_path, hooks_file=path, entrypoint="/new/path/thirdeye-copilot-hook" + tmp_path, + hooks_file=path, + entrypoint="/new/path/thirdeye-copilot-hook", + windows=False, ) platform.install() data = json.loads(path.read_text()) From e1b1379a2a3936c5bb6a6badd41da9d4ddcaadc8 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Fri, 11 Sep 2026 10:00:34 -0700 Subject: [PATCH 41/88] test: make Copilot replacement cases deterministic --- tests/test_copilot_database.py | 10 ++++++++-- tests/test_copilot_transcript.py | 20 ++++++++++++-------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/tests/test_copilot_database.py b/tests/test_copilot_database.py index 91439f5..0506cfc 100644 --- a/tests/test_copilot_database.py +++ b/tests/test_copilot_database.py @@ -442,8 +442,14 @@ def test_database_replacement_changes_generation_and_resets_cursor(tmp_path: Pat old_generation = first["next_cursor"]["database_generation"] assert first["exhausted"] is False - os.remove(database) - _write_database(home, session_id="session-a", turns=[(1, "replacement")]) + replacement_home = tmp_path / "replacement-copilot" + replacement = _write_database( + replacement_home, + session_id="session-a", + turns=[(1, "replacement")], + journal_mode="DELETE", + ) + os.replace(replacement, database) after_replace = read_database(paths, "session-a", first["next_cursor"], max_records=10) assert after_replace["next_cursor"]["database_generation"] != old_generation diff --git a/tests/test_copilot_transcript.py b/tests/test_copilot_transcript.py index 48ba23f..b874fc9 100644 --- a/tests/test_copilot_transcript.py +++ b/tests/test_copilot_transcript.py @@ -276,8 +276,9 @@ def test_read_transcript_replacement_replays_deferred_partial_line(tmp_path: Pat first = read_transcript(paths, native, {}) event_path = path / "events.jsonl" - event_path.unlink() - event_path.write_bytes(finished) + replacement = path / "replacement-events.jsonl" + replacement.write_bytes(finished) + replacement.replace(event_path) second = read_transcript(paths, native, first["next_cursor"]) assert "transcript_replaced" in _diagnostic_codes(second) @@ -311,9 +312,10 @@ def test_read_transcript_completes_deferred_utf8_after_replacement(tmp_path: Pat assert first["exhausted"] is False completed = raw + b"\xac\n" - replacement = path / "events.jsonl" - replacement.unlink() + event_path = path / "events.jsonl" + replacement = path / "replacement-events.jsonl" replacement.write_bytes(completed) + replacement.replace(event_path) second = read_transcript(paths, native, first["next_cursor"]) assert "transcript_replaced" in _diagnostic_codes(second) @@ -448,14 +450,16 @@ def test_read_transcript_replays_after_file_replacement(tmp_path: Path): first = read_transcript(paths, native, {}) assert first["exhausted"] is True - replacement = path / "events.jsonl" - replacement.unlink() - replacement.write_text('{"id":"new"}\n', encoding="utf-8") + event_path = path / "events.jsonl" + replacement = path / "replacement-events.jsonl" + replacement_bytes = b'{"id":"new"}\n' + replacement.write_bytes(replacement_bytes) + replacement.replace(event_path) second = read_transcript(paths, native, first["next_cursor"]) assert "transcript_replaced" in _diagnostic_codes(second) assert [record["payload"]["id"] for record in _transcript_records(second)] == ["new"] - assert second["next_cursor"]["byte_offset"] == len('{"id":"new"}\n') + assert second["next_cursor"]["byte_offset"] == len(replacement_bytes) def test_read_transcript_replays_after_truncation(tmp_path: Path): From 121e6a3c99318e860acaa2fa5055db7a2558a85c Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Fri, 11 Sep 2026 10:29:23 -0700 Subject: [PATCH 42/88] test: isolate Copilot wall-clock performance check --- tests/test_copilot_hooks.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_copilot_hooks.py b/tests/test_copilot_hooks.py index 264243c..ab51d52 100644 --- a/tests/test_copilot_hooks.py +++ b/tests/test_copilot_hooks.py @@ -4,6 +4,7 @@ import io import json +import os import shutil import sqlite3 import sys @@ -311,8 +312,8 @@ def test_hook_spools_and_captures_fixture_sources( @pytest.mark.skipif( - sys.platform == "win32", - reason="Windows CI filesystem scheduling cannot enforce a 250 ms wall-clock budget", + os.environ.get("THIRDEYE_RUN_PERFORMANCE_TESTS") != "1", + reason="wall-clock performance checks require a controlled runner", ) def test_hook_returns_within_250ms_on_small_fixture( copilot_env: tuple[Config, SourcePaths], From de82e99389ceeef1e2afb4c66975dbca5e49eb7b Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Fri, 11 Sep 2026 13:52:34 -0700 Subject: [PATCH 43/88] Align Copilot tests with platform layout --- tests/fixtures/copilot/cli-1.0.83/README.md | 51 ------ .../cli-1.0.83/assistant-usage-events.json | 158 ------------------ .../fixtures/copilot/cli-1.0.83/events.jsonl | 76 --------- tests/fixtures/copilot/cli-1.0.83/hooks.jsonl | 20 --- tests/fixtures/copilot/cli-1.0.83/usage.json | 134 --------------- .../codex/test_captured_env.py} | 0 tests/platforms/copilot/fixtures/README.md | 2 +- .../copilot/fixtures/cases}/README.md | 4 +- .../cases}/database-row-revisions.json | 0 .../cases}/distinct-hook-observations.json | 0 .../fixtures/cases}/missing-event-id.jsonl | 0 .../copilot/fixtures/cases}/source-batch.json | 0 .../cases}/source-key-prefix-collision.json | 0 .../copilot/fixtures/cases}/source-slice.json | 0 .../fixtures/cases}/trailing-json.jsonl | 0 .../copilot/fixtures/cases}/trailing-utf8.hex | 0 .../cases}/unknown-event-fields.jsonl | 0 .../copilot/test_archive.py} | 2 +- .../copilot/test_capture.py} | 4 +- .../copilot/test_capture_reads.py} | 0 .../copilot/test_command.py} | 6 +- .../copilot/test_contracts.py} | 10 +- .../copilot/test_database.py} | 4 +- .../copilot/test_followup.py} | 0 .../copilot/test_hook_payload.py} | 4 +- .../copilot/test_hooks.py} | 4 +- .../copilot/test_install.py} | 0 .../copilot/test_recovery.py} | 0 .../copilot/test_sources.py} | 4 +- .../copilot/test_spool.py} | 0 .../copilot/test_status.py} | 4 +- .../copilot/test_transcript.py} | 6 +- .../copilot/test_watch.py} | 4 +- 33 files changed, 29 insertions(+), 468 deletions(-) delete mode 100644 tests/fixtures/copilot/cli-1.0.83/README.md delete mode 100644 tests/fixtures/copilot/cli-1.0.83/assistant-usage-events.json delete mode 100644 tests/fixtures/copilot/cli-1.0.83/events.jsonl delete mode 100644 tests/fixtures/copilot/cli-1.0.83/hooks.jsonl delete mode 100644 tests/fixtures/copilot/cli-1.0.83/usage.json rename tests/{test_codex_captured_env.py => platforms/codex/test_captured_env.py} (100%) rename tests/{fixtures/copilot/v1-cases => platforms/copilot/fixtures/cases}/README.md (95%) rename tests/{fixtures/copilot/v1-cases => platforms/copilot/fixtures/cases}/database-row-revisions.json (100%) rename tests/{fixtures/copilot/v1-cases => platforms/copilot/fixtures/cases}/distinct-hook-observations.json (100%) rename tests/{fixtures/copilot/v1-cases => platforms/copilot/fixtures/cases}/missing-event-id.jsonl (100%) rename tests/{fixtures/copilot/v1-cases => platforms/copilot/fixtures/cases}/source-batch.json (100%) rename tests/{fixtures/copilot/v1-cases => platforms/copilot/fixtures/cases}/source-key-prefix-collision.json (100%) rename tests/{fixtures/copilot/v1-cases => platforms/copilot/fixtures/cases}/source-slice.json (100%) rename tests/{fixtures/copilot/v1-cases => platforms/copilot/fixtures/cases}/trailing-json.jsonl (100%) rename tests/{fixtures/copilot/v1-cases => platforms/copilot/fixtures/cases}/trailing-utf8.hex (100%) rename tests/{fixtures/copilot/v1-cases => platforms/copilot/fixtures/cases}/unknown-event-fields.jsonl (100%) rename tests/{test_copilot_archive.py => platforms/copilot/test_archive.py} (99%) rename tests/{test_copilot_capture.py => platforms/copilot/test_capture.py} (99%) rename tests/{test_copilot_capture_reads.py => platforms/copilot/test_capture_reads.py} (100%) rename tests/{test_copilot_command.py => platforms/copilot/test_command.py} (99%) rename tests/{test_copilot_contracts.py => platforms/copilot/test_contracts.py} (98%) rename tests/{test_copilot_database.py => platforms/copilot/test_database.py} (99%) rename tests/{test_copilot_followup.py => platforms/copilot/test_followup.py} (100%) rename tests/{test_copilot_hook_payload.py => platforms/copilot/test_hook_payload.py} (98%) rename tests/{test_copilot_hooks.py => platforms/copilot/test_hooks.py} (99%) rename tests/{test_copilot_install.py => platforms/copilot/test_install.py} (100%) rename tests/{test_copilot_recovery.py => platforms/copilot/test_recovery.py} (100%) rename tests/{test_copilot_sources.py => platforms/copilot/test_sources.py} (99%) rename tests/{test_copilot_spool.py => platforms/copilot/test_spool.py} (100%) rename tests/{test_copilot_status.py => platforms/copilot/test_status.py} (99%) rename tests/{test_copilot_transcript.py => platforms/copilot/test_transcript.py} (99%) rename tests/{test_copilot_watch.py => platforms/copilot/test_watch.py} (99%) diff --git a/tests/fixtures/copilot/cli-1.0.83/README.md b/tests/fixtures/copilot/cli-1.0.83/README.md deleted file mode 100644 index a43a168..0000000 --- a/tests/fixtures/copilot/cli-1.0.83/README.md +++ /dev/null @@ -1,51 +0,0 @@ -# Copilot CLI capture probe - -Captured 2026-09-10 on macOS using Copilot CLI 1.0.83, with automatic model selection (resolved to gpt-5.6-luna). These are observed fixtures for adapter design, not an implemented adapter or a regression test suite. - -## Successful scenario - -An isolated temporary Git repository contained `alpha.txt` (`alpha = 17`) and `beta.txt` (`beta = 25`). An interactive CLI session was launched from that directory. Folder trust was accepted for that session only. Available tools were restricted to `view,task`; both were allowed. Built-in MCP servers and remote session export were disabled. - -1. Ask for two separate reads in parallel and their sum. -2. Ask an explore subagent to read the same two files and report their sum. -3. Exit normally with `/exit`. - -Both top-level answers and the subagent answer were `42`. The transcript verifies overlapping top-level reads, five tool start/completion pairs (four reads plus the task invocation), two top-level user prompts, and one completed subagent. - -Repository hooks used version 1, camelCase event names, `type: command`, `bash`, and `timeoutSec: 5`. Each invoked a Python recorder that read stdin JSON and wrote a separate timestamp/PID-named file. It emitted no stdout and caught recorder errors. Captured events: sessionStart, userPromptSubmitted, preToolUse, postToolUse, agentStop, subagentStart, subagentStop, sessionEnd. Failure/error hooks were configured but not exercised. - -## Files - -- `hooks.jsonl`: 20 actual external hook invocations, ordered by recorder filename. `registered_event` is the configured camelCase event; `captured_ns` is recorder wall time; `payload` is Copilot input. -- `events.jsonl`: 76 retained transcript records, in original file order. -- `usage.json`: the final transcript `session.shutdown.data` object, including overall and per-agent usage. This is a duplicate extraction for convenience; do not count it again when aggregating the transcript. - -Sanitization replaces the local home/workspace paths and removes system messages, usage checkpoint internals, `model.*` events, transformed prompts, reasoning text/blocks, opaque/encrypted provider fields, API call IDs, tool telemetry, and server-tools metadata. Event IDs, timestamps, agent IDs, interaction IDs, tool-call IDs, and measured usage are retained. This is deliberately a filtered transcript, not a byte-for-byte raw capture; parentId references may point to omitted records. Raw captures remain in `/private/tmp/thirdeye-copilot-probe` and the original CLI session state. - -## Integration findings - -- External pre/post tool hooks have no invocation ID in this run. The transcript provides `toolCallId` on requests and executions. Correlate parallel tools using the transcript, not just tool names. -- `turnId` denotes a model/tool cycle and resets across user requests. Group user interactions using `interactionId` plus agent identity. -- Subagent events are interleaved in the same transcript and carry top-level `agentId`; `subagent.started` links to the parent task via `data.toolCallId`. Child records also expose `parentToolCallId` where applicable. -- The child generated its own user-prompt and agent-stop hooks. Those payloads set `sessionId` to the child agent ID (`bf8cb9f3-2097-4db0-a3c8-78a2653b2106`), not the parent session ID. Child pre/post tool hooks do the same. `transcriptPath` on the child's agentStop still points at the parent session's `events.jsonl`. `subagentStart`/`subagentStop` keep the parent session ID and name the child in `agentId`. Sanitization did not rewrite these identifiers; transcript `hook.start` input matches the external recorder. Hook `sessionId` is therefore not a native session ID for capture routing. Child prompt/stop hooks cannot drive a session-ID-only turn state machine. There are three prompt hooks and three stop hooks for two top-level user requests. -- SessionStart arrived after the first user-prompt hook. Initialization must tolerate that ordering. -- There are 20 external recorder files but only 18 hook.start/hook.end pairs in the final transcript. Do not assume those two streams have one-to-one coverage. -- Assistant text and model names are present in assistant.message. Final session.shutdown includes input/output/cache/reasoning usage and per-agent metrics. Availability of per-call usage at Stop time has not been established by this probe. - -## Limitations and unsuccessful probes - -Earlier non-interactive `-p` runs completed but produced no external hook captures. This remained true with both PascalCase/exec and camelCase/bash configurations, after committing the temporary hook file, and when launching directly from the fixture directory. Interactive mode with session folder trust produced the successful fixture. This narrows the issue to runtime mode/trust/loading behavior but does not isolate its exact cause; it does not prove non-interactive hooks are universally unsupported. - -PascalCase compatibility, user-level installation, interrupted/error turns, repeated identical concurrent tool arguments, and other Copilot versions/runtimes remain unverified. No permanent user-level hooks or thirdeye adapter code were installed. - -## Persisted database follow-up - -Read-only inspection of `~/.copilot/session-store.db` found six `assistant_usage_events` rows for this session: four main-agent calls and two explore calls. `assistant-usage-events.json` retains those rows. Their input/output/cache-read/cache-write/reasoning totals and nano-AI-unit total exactly match session.shutdown; per-agent input totals also match. Database `turn_index` groups these into the two actual user interactions (0 and 1), unlike transcript turnId, which indexes model cycles and restarts. - -The table includes agent_id, parent_tool_call_id, model, tokens, billing details, call duration, time to first token/output, inter-token latency, initiator, endpoint, reasoning effort, finish reason, and creation time. The installed event schema marks assistant.usage as ephemeral: it is absent from events.jsonl, but these database rows persist it independently. Both interactive and the earlier non-interactive test sessions have usage rows. The table lacks the transcript assistant message ID/provider call ID, so exact per-message joins require additional care; grouping by session/turn/agent is explicit. - -Persisted usage checkpoint events contain aggregate billing and cache-frontier diagnostics, including latest input/cache counts, tool schema hashes, system-segment token estimates, cache TTL, and completion time. They are snapshots, not a complete per-call ledger. In this session each top-level checkpoint follows agentStop in file order, so a Stop-time read can precede the checkpoint. - -The raw transcript also contains two auxiliary model.model_call_success records for gpt-4o-mini session-title generation, with request/response content, usage and latency. These were omitted from the sanitized transcript. Do not treat them as the main-agent model-call history or add their usage to the six-row total without explicitly accounting for auxiliary calls. - -The database also has sessions, turns, checkpoints, session_files, session_refs and full-text search tables. Our session has two complete turns but no session_files rows despite four file reads, so the database's discovery/index tables do not replace raw tool execution events. Files beside the transcript include workspace.yaml (identity/repo/title), checkpoints/index.md (empty here), and rewind-file-snapshots/tracking.json (tracking metadata only here). diff --git a/tests/fixtures/copilot/cli-1.0.83/assistant-usage-events.json b/tests/fixtures/copilot/cli-1.0.83/assistant-usage-events.json deleted file mode 100644 index 5de8d67..0000000 --- a/tests/fixtures/copilot/cli-1.0.83/assistant-usage-events.json +++ /dev/null @@ -1,158 +0,0 @@ -[ - { - "id": 13, - "session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", - "turn_index": 0, - "agent_id": null, - "parent_tool_call_id": null, - "model": "gpt-5.6-luna", - "input_tokens": 6452, - "output_tokens": 107, - "cache_read_tokens": 0, - "cache_write_tokens": 6449, - "reasoning_tokens": 29, - "total_nano_aiu": 174125000, - "request_multiplier": 1.0, - "duration_ms": 2263, - "time_to_first_token_ms": 1571.1477920000002, - "output_ttft_ms": 1571.1479590000001, - "inter_token_latency_ms": 7.393025117647059, - "initiator": "user", - "api_endpoint": "ws:/responses", - "reasoning_effort": "medium", - "finish_reason": "tool_calls", - "content_filter_triggered": 0, - "token_details_json": "[{\"batchSize\":1000000,\"costPerBatch\":20000000000,\"tokenCount\":3,\"tokenType\":\"input\"},{\"batchSize\":1000000,\"costPerBatch\":2000000000,\"tokenCount\":0,\"tokenType\":\"cache_read\"},{\"batchSize\":1000000,\"costPerBatch\":25000000000,\"tokenCount\":6449,\"tokenType\":\"cache_write\"},{\"batchSize\":1000000,\"costPerBatch\":120000000000,\"tokenCount\":107,\"tokenType\":\"output\"}]", - "created_at": "2026-09-10T17:08:24.498Z" - }, - { - "id": 14, - "session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", - "turn_index": 0, - "agent_id": null, - "parent_tool_call_id": null, - "model": "gpt-5.6-luna", - "input_tokens": 6587, - "output_tokens": 5, - "cache_read_tokens": 6449, - "cache_write_tokens": 135, - "reasoning_tokens": 0, - "total_nano_aiu": 16933000, - "request_multiplier": 1.0, - "duration_ms": 1017, - "time_to_first_token_ms": 942.6416250000001, - "output_ttft_ms": 942.642416, - "inter_token_latency_ms": null, - "initiator": "agent", - "api_endpoint": "ws:/responses", - "reasoning_effort": "medium", - "finish_reason": "stop", - "content_filter_triggered": 0, - "token_details_json": "[{\"batchSize\":1000000,\"costPerBatch\":20000000000,\"tokenCount\":3,\"tokenType\":\"input\"},{\"batchSize\":1000000,\"costPerBatch\":2000000000,\"tokenCount\":6449,\"tokenType\":\"cache_read\"},{\"batchSize\":1000000,\"costPerBatch\":25000000000,\"tokenCount\":135,\"tokenType\":\"cache_write\"},{\"batchSize\":1000000,\"costPerBatch\":120000000000,\"tokenCount\":5,\"tokenType\":\"output\"}]", - "created_at": "2026-09-10T17:08:25.618Z" - }, - { - "id": 15, - "session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", - "turn_index": 1, - "agent_id": null, - "parent_tool_call_id": null, - "model": "gpt-5.6-luna", - "input_tokens": 6627, - "output_tokens": 112, - "cache_read_tokens": 6449, - "cache_write_tokens": 175, - "reasoning_tokens": 10, - "total_nano_aiu": 30773000, - "request_multiplier": 1.0, - "duration_ms": 1813, - "time_to_first_token_ms": 1366.804083, - "output_ttft_ms": 1366.80425, - "inter_token_latency_ms": 4.3487423448275875, - "initiator": "user", - "api_endpoint": "ws:/responses", - "reasoning_effort": "medium", - "finish_reason": "tool_calls", - "content_filter_triggered": 0, - "token_details_json": "[{\"batchSize\":1000000,\"costPerBatch\":20000000000,\"tokenCount\":3,\"tokenType\":\"input\"},{\"batchSize\":1000000,\"costPerBatch\":2000000000,\"tokenCount\":6449,\"tokenType\":\"cache_read\"},{\"batchSize\":1000000,\"costPerBatch\":25000000000,\"tokenCount\":175,\"tokenType\":\"cache_write\"},{\"batchSize\":1000000,\"costPerBatch\":120000000000,\"tokenCount\":112,\"tokenType\":\"output\"}]", - "created_at": "2026-09-10T17:08:44.015Z" - }, - { - "id": 16, - "session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", - "turn_index": 1, - "agent_id": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", - "parent_tool_call_id": "call_qx4FH5DADTeT1qVLb37HNpBk", - "model": "gpt-5.6-luna", - "input_tokens": 4429, - "output_tokens": 94, - "cache_read_tokens": 0, - "cache_write_tokens": 4426, - "reasoning_tokens": 16, - "total_nano_aiu": 121990000, - "request_multiplier": 1.0, - "duration_ms": 1936, - "time_to_first_token_ms": 1769.324584, - "output_ttft_ms": 1769.3247090000002, - "inter_token_latency_ms": 1.929541, - "initiator": "sub-agent", - "api_endpoint": "ws:/responses", - "reasoning_effort": "low", - "finish_reason": "tool_calls", - "content_filter_triggered": 0, - "token_details_json": "[{\"batchSize\":1000000,\"costPerBatch\":20000000000,\"tokenCount\":3,\"tokenType\":\"input\"},{\"batchSize\":1000000,\"costPerBatch\":2000000000,\"tokenCount\":0,\"tokenType\":\"cache_read\"},{\"batchSize\":1000000,\"costPerBatch\":25000000000,\"tokenCount\":4426,\"tokenType\":\"cache_write\"},{\"batchSize\":1000000,\"costPerBatch\":120000000000,\"tokenCount\":94,\"tokenType\":\"output\"}]", - "created_at": "2026-09-10T17:08:46.500Z" - }, - { - "id": 17, - "session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", - "turn_index": 1, - "agent_id": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", - "parent_tool_call_id": "call_qx4FH5DADTeT1qVLb37HNpBk", - "model": "gpt-5.6-luna", - "input_tokens": 4551, - "output_tokens": 5, - "cache_read_tokens": 4426, - "cache_write_tokens": 122, - "reasoning_tokens": 0, - "total_nano_aiu": 12562000, - "request_multiplier": 1.0, - "duration_ms": 847, - "time_to_first_token_ms": 788.4591250000001, - "output_ttft_ms": 788.459167, - "inter_token_latency_ms": null, - "initiator": "sub-agent", - "api_endpoint": "ws:/responses", - "reasoning_effort": "low", - "finish_reason": "stop", - "content_filter_triggered": 0, - "token_details_json": "[{\"batchSize\":1000000,\"costPerBatch\":20000000000,\"tokenCount\":3,\"tokenType\":\"input\"},{\"batchSize\":1000000,\"costPerBatch\":2000000000,\"tokenCount\":4426,\"tokenType\":\"cache_read\"},{\"batchSize\":1000000,\"costPerBatch\":25000000000,\"tokenCount\":122,\"tokenType\":\"cache_write\"},{\"batchSize\":1000000,\"costPerBatch\":120000000000,\"tokenCount\":5,\"tokenType\":\"output\"}]", - "created_at": "2026-09-10T17:08:47.455Z" - }, - { - "id": 18, - "session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", - "turn_index": 1, - "agent_id": null, - "parent_tool_call_id": null, - "model": "gpt-5.6-luna", - "input_tokens": 6750, - "output_tokens": 5, - "cache_read_tokens": 6624, - "cache_write_tokens": 123, - "reasoning_tokens": 0, - "total_nano_aiu": 16983000, - "request_multiplier": 1.0, - "duration_ms": 732, - "time_to_first_token_ms": 677.7400409999999, - "output_ttft_ms": 677.7401659999999, - "inter_token_latency_ms": null, - "initiator": "agent", - "api_endpoint": "ws:/responses", - "reasoning_effort": "medium", - "finish_reason": "stop", - "content_filter_triggered": 0, - "token_details_json": "[{\"batchSize\":1000000,\"costPerBatch\":20000000000,\"tokenCount\":3,\"tokenType\":\"input\"},{\"batchSize\":1000000,\"costPerBatch\":2000000000,\"tokenCount\":6624,\"tokenType\":\"cache_read\"},{\"batchSize\":1000000,\"costPerBatch\":25000000000,\"tokenCount\":123,\"tokenType\":\"cache_write\"},{\"batchSize\":1000000,\"costPerBatch\":120000000000,\"tokenCount\":5,\"tokenType\":\"output\"}]", - "created_at": "2026-09-10T17:08:48.278Z" - } -] diff --git a/tests/fixtures/copilot/cli-1.0.83/events.jsonl b/tests/fixtures/copilot/cli-1.0.83/events.jsonl deleted file mode 100644 index 60e257f..0000000 --- a/tests/fixtures/copilot/cli-1.0.83/events.jsonl +++ /dev/null @@ -1,76 +0,0 @@ -{"type": "session.start", "data": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "version": 1, "producer": "copilot-agent", "copilotVersion": "1.0.83", "startTime": "2026-09-10T17:08:05.884Z", "contextTier": null, "context": {"cwd": "/fixture/workspace", "gitRoot": "/fixture/workspace", "branch": "master", "headCommit": "6eecf11579d0c8fb0da1b7fd9671dada3910847c"}, "alreadyInUse": false, "remoteSteerable": false}, "id": "26159b14-7d71-4dd1-96c1-9404fe6b356d", "timestamp": "2026-09-10T17:08:05.891Z", "parentId": null} -{"type": "session.model_change", "data": {"cause": "initial_resolution", "source": "automatic", "contextTier": null, "newModel": "auto", "reasoningEffort": null}, "id": "ab149215-ec43-4c2e-8a58-a176f104d0dc", "timestamp": "2026-09-10T17:08:07.140Z", "parentId": "26159b14-7d71-4dd1-96c1-9404fe6b356d"} -{"type": "session.auto_mode_resolved", "data": {"chosenModel": "gpt-5.6-luna", "categoryScores": {"code_gen": 0.1191, "debugging": 0.0018, "reasoning": 0.2963, "tool_use": 0.1192}, "candidateModels": ["gpt-5.6-luna"], "routingMethod": "auto_v2", "fallback": false, "availableModels": ["gpt-5.6-luna"], "endToEndLatencyMs": 184.635416, "hasImage": false}, "id": "2d18ca6f-c3f1-48ba-bfbf-f8f4e1e9c9ef", "timestamp": "2026-09-10T17:08:22.040Z", "parentId": "ab149215-ec43-4c2e-8a58-a176f104d0dc"} -{"type": "hook.start", "data": {"hookInvocationId": "3ad4c584-f613-4cf1-9e6e-3f09eb60be99", "hookType": "userPromptSubmitted", "input": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "prompt": "Read alpha.txt and beta.txt with separate view calls in parallel and report their sum. Only read those two files.", "timestamp": 1789060102173, "cwd": "/fixture/workspace"}}, "id": "a8043161-e819-426c-bf9d-0866b35b9082", "timestamp": "2026-09-10T17:08:22.173Z", "parentId": "2d18ca6f-c3f1-48ba-bfbf-f8f4e1e9c9ef"} -{"type": "hook.end", "data": {"hookInvocationId": "3ad4c584-f613-4cf1-9e6e-3f09eb60be99", "hookType": "userPromptSubmitted", "success": true}, "id": "3942810f-1caf-4251-82fb-3bf72698147a", "timestamp": "2026-09-10T17:08:22.203Z", "parentId": "a8043161-e819-426c-bf9d-0866b35b9082"} -{"type": "user.message", "data": {"content": "Read alpha.txt and beta.txt with separate view calls in parallel and report their sum. Only read those two files.", "messageId": "a856cb38-7609-45ab-8a55-645553155db3", "supportedNativeDocumentMimeTypes": [], "delivery": "idle", "interactionId": "6d2b89fd-a653-430c-b532-b0936d72eb42", "turnId": "0", "parentAgentTaskId": "fd800f7a-8163-4304-9681-efde4731e879"}, "id": "f07404d0-af52-4260-89fb-358a10e86034", "timestamp": "2026-09-10T17:08:22.203Z", "parentId": "3942810f-1caf-4251-82fb-3bf72698147a"} -{"type": "hook.start", "data": {"hookInvocationId": "c75fca6a-c0a7-4440-9bf4-10399f301f1d", "hookType": "sessionStart", "input": {"source": "new", "initialPrompt": "Read alpha.txt and beta.txt with separate view calls in parallel and report their sum. Only read those two files.", "sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "timestamp": 1789060102204, "cwd": "/fixture/workspace"}}, "id": "86b5b021-d90a-45ad-9042-41fc11dd31e3", "timestamp": "2026-09-10T17:08:22.204Z", "parentId": "a7e8a23d-8522-42af-84d7-f122f025ce0a"} -{"type": "hook.end", "data": {"hookInvocationId": "c75fca6a-c0a7-4440-9bf4-10399f301f1d", "hookType": "sessionStart", "success": true}, "id": "89373174-eb9e-4129-94b7-2ab8f01c6c03", "timestamp": "2026-09-10T17:08:22.227Z", "parentId": "86b5b021-d90a-45ad-9042-41fc11dd31e3"} -{"type": "assistant.turn_start", "data": {"turnId": "0", "interactionId": "6d2b89fd-a653-430c-b532-b0936d72eb42"}, "id": "64f9e436-651f-4a9d-919c-a4d7abad2652", "timestamp": "2026-09-10T17:08:22.227Z", "parentId": "89373174-eb9e-4129-94b7-2ab8f01c6c03"} -{"type": "assistant.message", "data": {"messageId": "5a29b09d-5e7d-4603-975e-7801ce54232b", "model": "gpt-5.6-luna", "content": "", "toolRequests": [{"toolCallId": "call_YSSva4HCniiETlxdGGjcrHbh", "name": "view", "arguments": {"path": "/fixture/workspace/alpha.txt"}, "type": "function", "intentionSummary": "view the file at /fixture/workspace/alpha.txt."}, {"toolCallId": "call_ayHplfzxjRFMTCpmTKEFhCSJ", "name": "view", "arguments": {"path": "/fixture/workspace/beta.txt"}, "type": "function", "intentionSummary": "view the file at /fixture/workspace/beta.txt."}], "interactionId": "6d2b89fd-a653-430c-b532-b0936d72eb42", "turnId": "0", "rte": true}, "id": "a4a17e63-7ba5-422f-8ee9-b495be417328", "timestamp": "2026-09-10T17:08:24.503Z", "parentId": "64f9e436-651f-4a9d-919c-a4d7abad2652"} -{"type": "tool.execution_start", "data": {"toolCallId": "call_YSSva4HCniiETlxdGGjcrHbh", "toolName": "view", "arguments": {"path": "/fixture/workspace/alpha.txt"}, "turnId": "0", "model": "gpt-5.6-luna"}, "id": "a7f7bf04-589e-4989-a4a2-7ee687279627", "timestamp": "2026-09-10T17:08:24.506Z", "parentId": "a4a17e63-7ba5-422f-8ee9-b495be417328"} -{"type": "tool.execution_start", "data": {"toolCallId": "call_ayHplfzxjRFMTCpmTKEFhCSJ", "toolName": "view", "arguments": {"path": "/fixture/workspace/beta.txt"}, "turnId": "0", "model": "gpt-5.6-luna"}, "id": "39e9da78-9bc3-4e1a-9629-f6680d1aeb4d", "timestamp": "2026-09-10T17:08:24.506Z", "parentId": "a7f7bf04-589e-4989-a4a2-7ee687279627"} -{"type": "hook.start", "data": {"hookInvocationId": "bac5cbf0-b56e-4ff6-98ca-a3f583c62a78", "hookType": "preToolUse", "input": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "cwd": "/fixture/workspace", "toolCalls": [{"id": "call_YSSva4HCniiETlxdGGjcrHbh", "name": "view", "args": {"path": "/fixture/workspace/alpha.txt"}}, {"id": "call_ayHplfzxjRFMTCpmTKEFhCSJ", "name": "view", "args": {"path": "/fixture/workspace/beta.txt"}}]}}, "id": "e595215c-0592-4d36-8098-914f07b6923c", "timestamp": "2026-09-10T17:08:24.506Z", "parentId": "39e9da78-9bc3-4e1a-9629-f6680d1aeb4d"} -{"type": "hook.end", "data": {"hookInvocationId": "bac5cbf0-b56e-4ff6-98ca-a3f583c62a78", "hookType": "preToolUse", "success": true}, "id": "c7158444-1b49-469f-af1e-962bc1977313", "timestamp": "2026-09-10T17:08:24.551Z", "parentId": "e595215c-0592-4d36-8098-914f07b6923c"} -{"type": "hook.start", "data": {"hookInvocationId": "b196eb30-c3eb-4907-97dc-cca942dd6d74", "hookType": "postToolUse", "input": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "timestamp": 1789060104552, "cwd": "/fixture/workspace", "toolName": "view", "toolArgs": {"path": "/fixture/workspace/alpha.txt"}, "toolResult": {"resultType": "success", "textResultForLlm": "alpha = 17\n", "sessionLog": "[copilot:elided sessionLog (308 bytes) — pre-hook tool result; the final result may differ, see the adjacent tool.execution_complete event]", "skipLargeOutputProcessing": true}}}, "id": "96dc12ef-fe49-4bc5-b75a-58589b90293c", "timestamp": "2026-09-10T17:08:24.552Z", "parentId": "c7158444-1b49-469f-af1e-962bc1977313"} -{"type": "hook.end", "data": {"hookInvocationId": "b196eb30-c3eb-4907-97dc-cca942dd6d74", "hookType": "postToolUse", "success": true}, "id": "a3b79888-b777-42b2-a893-ca9d56b4002f", "timestamp": "2026-09-10T17:08:24.572Z", "parentId": "96dc12ef-fe49-4bc5-b75a-58589b90293c"} -{"type": "tool.execution_complete", "data": {"toolCallId": "call_YSSva4HCniiETlxdGGjcrHbh", "model": "gpt-5.6-luna", "interactionId": "6d2b89fd-a653-430c-b532-b0936d72eb42", "turnId": "0", "rte": true, "success": true, "result": {"content": "alpha = 17\n", "detailedContent": "\ndiff --git a/fixture/workspace/alpha.txt b/fixture/workspace/alpha.txt\nindex 0000000..0000000 100644\n--- a/fixture/workspace/alpha.txt\n+++ b/fixture/workspace/alpha.txt\n@@ -1,2 +1,2 @@\n alpha = 17\n \n"}}, "id": "1b06e9d1-9223-449a-a135-b4f64cf0e3a6", "timestamp": "2026-09-10T17:08:24.572Z", "parentId": "a3b79888-b777-42b2-a893-ca9d56b4002f"} -{"type": "hook.start", "data": {"hookInvocationId": "f50891cc-7434-4b35-aa9d-aacb15861567", "hookType": "postToolUse", "input": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "timestamp": 1789060104572, "cwd": "/fixture/workspace", "toolName": "view", "toolArgs": {"path": "/fixture/workspace/beta.txt"}, "toolResult": {"resultType": "success", "textResultForLlm": "beta = 25\n", "sessionLog": "[copilot:elided sessionLog (303 bytes) — pre-hook tool result; the final result may differ, see the adjacent tool.execution_complete event]", "skipLargeOutputProcessing": true}}}, "id": "4f993405-aade-4f24-901f-203ade003627", "timestamp": "2026-09-10T17:08:24.573Z", "parentId": "1b06e9d1-9223-449a-a135-b4f64cf0e3a6"} -{"type": "hook.end", "data": {"hookInvocationId": "f50891cc-7434-4b35-aa9d-aacb15861567", "hookType": "postToolUse", "success": true}, "id": "6561c429-fada-49da-b642-1b2ea8d1d650", "timestamp": "2026-09-10T17:08:24.592Z", "parentId": "4f993405-aade-4f24-901f-203ade003627"} -{"type": "tool.execution_complete", "data": {"toolCallId": "call_ayHplfzxjRFMTCpmTKEFhCSJ", "model": "gpt-5.6-luna", "interactionId": "6d2b89fd-a653-430c-b532-b0936d72eb42", "turnId": "0", "rte": true, "success": true, "result": {"content": "beta = 25\n", "detailedContent": "\ndiff --git a/fixture/workspace/beta.txt b/fixture/workspace/beta.txt\nindex 0000000..0000000 100644\n--- a/fixture/workspace/beta.txt\n+++ b/fixture/workspace/beta.txt\n@@ -1,2 +1,2 @@\n beta = 25\n \n"}}, "id": "032cf87e-05b7-4e5f-96b9-1fda1a8242f5", "timestamp": "2026-09-10T17:08:24.593Z", "parentId": "6561c429-fada-49da-b642-1b2ea8d1d650"} -{"type": "assistant.turn_end", "data": {"turnId": "0"}, "id": "0080e44c-ad62-4288-b2b2-061ec2b73d80", "timestamp": "2026-09-10T17:08:24.593Z", "parentId": "032cf87e-05b7-4e5f-96b9-1fda1a8242f5"} -{"type": "assistant.turn_start", "data": {"turnId": "1", "interactionId": "6d2b89fd-a653-430c-b532-b0936d72eb42"}, "id": "667fe48a-70a1-473b-b005-454964022344", "timestamp": "2026-09-10T17:08:24.594Z", "parentId": "0080e44c-ad62-4288-b2b2-061ec2b73d80"} -{"type": "assistant.message", "data": {"messageId": "162d92b0-5d31-444c-b96f-b6c551527d2b", "model": "gpt-5.6-luna", "content": "42", "toolRequests": [], "interactionId": "6d2b89fd-a653-430c-b532-b0936d72eb42", "turnId": "1", "phase": "final_answer", "rte": true}, "id": "33cc6465-29e1-4a04-8bdb-00241474b4d2", "timestamp": "2026-09-10T17:08:25.624Z", "parentId": "667fe48a-70a1-473b-b005-454964022344"} -{"type": "assistant.turn_end", "data": {"turnId": "1"}, "id": "4a386a37-ca7e-4ebc-a746-cdda20f2a4bb", "timestamp": "2026-09-10T17:08:25.626Z", "parentId": "33cc6465-29e1-4a04-8bdb-00241474b4d2"} -{"type": "hook.start", "data": {"hookInvocationId": "ce6d52d0-7e43-41ca-956e-863ca6c38cda", "hookType": "agentStop", "input": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "transcriptPath": "/home/tester/.copilot/session-state/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/events.jsonl", "stopReason": "end_turn", "stop_hook_active": false, "timestamp": 1789060105626, "cwd": "/fixture/workspace"}}, "id": "7bb0689c-8435-46f7-909e-7e0223753217", "timestamp": "2026-09-10T17:08:25.626Z", "parentId": "4a386a37-ca7e-4ebc-a746-cdda20f2a4bb"} -{"type": "hook.end", "data": {"hookInvocationId": "ce6d52d0-7e43-41ca-956e-863ca6c38cda", "hookType": "agentStop", "success": true}, "id": "b514e4ab-6141-4e22-a9dd-6f095f36e22c", "timestamp": "2026-09-10T17:08:25.650Z", "parentId": "7bb0689c-8435-46f7-909e-7e0223753217"} -{"type": "session.auto_mode_resolved", "data": {"chosenModel": "gpt-5.6-luna", "categoryScores": {"reasoning": 0.5805, "tool_use": 0.4379, "debugging": 0.0062, "code_gen": 0.3977}, "candidateModels": ["gpt-5.6-luna"], "routingMethod": "auto_v2", "fallback": false, "availableModels": ["gpt-5.6-luna"], "endToEndLatencyMs": 432.443958, "hasImage": false}, "id": "c19cb305-f15c-4419-82ae-df9278d56d0c", "timestamp": "2026-09-10T17:08:42.154Z", "parentId": "d70783ab-86ee-4a5b-89eb-b95627215202"} -{"type": "hook.start", "data": {"hookInvocationId": "f85718f6-b82f-4b5a-ba84-6893af297355", "hookType": "userPromptSubmitted", "input": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "prompt": "Invoke one explore subagent to read only alpha.txt and beta.txt and report their sum. Do not modify files or access other files or services. Then report its answer.", "timestamp": 1789060122166, "cwd": "/fixture/workspace"}}, "id": "41248b6e-c3c4-4d3d-8b4b-0f7982d50ba9", "timestamp": "2026-09-10T17:08:42.166Z", "parentId": "c19cb305-f15c-4419-82ae-df9278d56d0c"} -{"type": "hook.end", "data": {"hookInvocationId": "f85718f6-b82f-4b5a-ba84-6893af297355", "hookType": "userPromptSubmitted", "success": true}, "id": "59c37c71-d5db-4419-a2db-1cd1fad7ed5d", "timestamp": "2026-09-10T17:08:42.192Z", "parentId": "41248b6e-c3c4-4d3d-8b4b-0f7982d50ba9"} -{"type": "user.message", "data": {"content": "Invoke one explore subagent to read only alpha.txt and beta.txt and report their sum. Do not modify files or access other files or services. Then report its answer.", "messageId": "02868f8d-8ec2-4485-b4a5-a75312e4d2e5", "supportedNativeDocumentMimeTypes": [], "delivery": "idle", "interactionId": "793d3703-6f4a-4814-8877-34a7325848ce", "turnId": "0", "parentAgentTaskId": "c17fc649-17ec-435a-ac27-77dbd6f92379"}, "id": "089db64c-a039-4635-ad4d-40d588c143be", "timestamp": "2026-09-10T17:08:42.192Z", "parentId": "59c37c71-d5db-4419-a2db-1cd1fad7ed5d"} -{"type": "assistant.turn_start", "data": {"turnId": "0", "interactionId": "793d3703-6f4a-4814-8877-34a7325848ce"}, "id": "0012e0f5-13cd-44dc-8126-51e584628989", "timestamp": "2026-09-10T17:08:42.194Z", "parentId": "089db64c-a039-4635-ad4d-40d588c143be"} -{"type": "assistant.message", "data": {"messageId": "0e5973fb-79e5-48d9-ba90-233f71ee70cc", "model": "gpt-5.6-luna", "content": "", "toolRequests": [{"toolCallId": "call_qx4FH5DADTeT1qVLb37HNpBk", "name": "task", "arguments": {"description": "Sum two text files", "prompt": "Read only /fixture/workspace/alpha.txt and /fixture/workspace/beta.txt. Do not modify files. Do not access any other files or services. Report the sum of the numeric values in those two files.", "agent_type": "explore", "name": "sum-alpha-beta", "mode": "sync"}, "type": "function", "intentionSummary": "Sum two text files"}], "interactionId": "793d3703-6f4a-4814-8877-34a7325848ce", "turnId": "0", "rte": true}, "id": "2c2e4ab8-f283-4837-957d-da992ba55e65", "timestamp": "2026-09-10T17:08:44.019Z", "parentId": "0012e0f5-13cd-44dc-8126-51e584628989"} -{"type": "tool.execution_start", "data": {"toolCallId": "call_qx4FH5DADTeT1qVLb37HNpBk", "toolName": "task", "arguments": {"description": "Sum two text files", "prompt": "Read only /fixture/workspace/alpha.txt and /fixture/workspace/beta.txt. Do not modify files. Do not access any other files or services. Report the sum of the numeric values in those two files.", "agent_type": "explore", "name": "sum-alpha-beta", "mode": "sync"}, "turnId": "0", "model": "gpt-5.6-luna"}, "id": "41dabdfc-681e-4043-a2ce-83b06cc07032", "timestamp": "2026-09-10T17:08:44.021Z", "parentId": "2c2e4ab8-f283-4837-957d-da992ba55e65"} -{"type": "hook.start", "data": {"hookInvocationId": "097d963e-4289-4eb1-998c-235f17b32451", "hookType": "preToolUse", "input": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "cwd": "/fixture/workspace", "toolCalls": [{"id": "call_qx4FH5DADTeT1qVLb37HNpBk", "name": "task", "args": {"description": "Sum two text files", "prompt": "Read only /fixture/workspace/alpha.txt and /fixture/workspace/beta.txt. Do not modify files. Do not access any other files or services. Report the sum of the numeric values in those two files.", "agent_type": "explore", "name": "sum-alpha-beta", "mode": "sync"}}]}}, "id": "6c2d498f-02c6-4b7c-8f96-134e29f4c25c", "timestamp": "2026-09-10T17:08:44.021Z", "parentId": "41dabdfc-681e-4043-a2ce-83b06cc07032"} -{"type": "hook.end", "data": {"hookInvocationId": "097d963e-4289-4eb1-998c-235f17b32451", "hookType": "preToolUse", "success": true}, "id": "0049f918-7bb9-478a-8c91-b6f8353525a9", "timestamp": "2026-09-10T17:08:44.045Z", "parentId": "6c2d498f-02c6-4b7c-8f96-134e29f4c25c"} -{"type": "subagent.started", "data": {"toolCallId": "call_qx4FH5DADTeT1qVLb37HNpBk", "agentName": "explore", "agentDisplayName": "sum-alpha-beta", "agentDescription": "Sum two text files", "model": "gpt-5.6-luna", "resumable": false, "agentType": "explore", "executionMode": "sync"}, "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "id": "6dff9fa5-c2df-41da-912b-d1ffd0706076", "timestamp": "2026-09-10T17:08:44.058Z", "parentId": "0049f918-7bb9-478a-8c91-b6f8353525a9"} -{"type": "subagent.configured", "data": {"model": "gpt-5.6-luna", "reasoningEffort": "low", "multiTurn": true}, "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "id": "af2030ff-f375-4702-b468-8276db3353f6", "timestamp": "2026-09-10T17:08:44.082Z", "parentId": "6dff9fa5-c2df-41da-912b-d1ffd0706076"} -{"type": "hook.start", "data": {"hookInvocationId": "fbdefe2f-bb83-4774-8491-4260dcbe4743", "hookType": "subagentStart", "input": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "transcriptPath": "/home/tester/.copilot/session-state/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/events.jsonl", "agentName": "explore", "timestamp": 1789060124082, "cwd": "/fixture/workspace"}}, "id": "b7bd0d91-2b63-48c5-9ee8-98531fa144f8", "timestamp": "2026-09-10T17:08:44.082Z", "parentId": "af2030ff-f375-4702-b468-8276db3353f6"} -{"type": "hook.end", "data": {"hookInvocationId": "fbdefe2f-bb83-4774-8491-4260dcbe4743", "hookType": "subagentStart", "success": true}, "id": "bd85c142-0403-4f71-b3df-e580cba56659", "timestamp": "2026-09-10T17:08:44.102Z", "parentId": "b7bd0d91-2b63-48c5-9ee8-98531fa144f8"} -{"type": "session.auto_mode_resolved", "data": {"chosenModel": "gpt-5.6-luna", "categoryScores": {"tool_use": 0.1064, "debugging": 0.0022, "code_gen": 0.065, "reasoning": 0.1966}, "candidateModels": ["gpt-5.6-luna"], "routingMethod": "auto_v2", "fallback": false, "availableModels": ["gpt-5.6-luna"], "endToEndLatencyMs": 400.976, "hasImage": false}, "id": "4812bec4-cdbb-4bd7-ab04-9e75367c81bc", "timestamp": "2026-09-10T17:08:44.505Z", "parentId": null} -{"type": "hook.start", "data": {"hookInvocationId": "1a7a5f25-21e6-4c63-bf2e-67ee8143f8de", "hookType": "userPromptSubmitted", "input": {"sessionId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "prompt": "Read only /fixture/workspace/alpha.txt and /fixture/workspace/beta.txt. Do not modify files. Do not access any other files or services. Report the sum of the numeric values in those two files.", "timestamp": 1789060124530, "cwd": "/fixture/workspace"}}, "id": "18c6c036-7321-4f42-ac97-108572d250fd", "timestamp": "2026-09-10T17:08:44.530Z", "parentId": "bd85c142-0403-4f71-b3df-e580cba56659"} -{"type": "hook.end", "data": {"hookInvocationId": "1a7a5f25-21e6-4c63-bf2e-67ee8143f8de", "hookType": "userPromptSubmitted", "success": true}, "id": "3bd18700-90c6-4b2f-aaaa-63760bdfa89a", "timestamp": "2026-09-10T17:08:44.557Z", "parentId": "18c6c036-7321-4f42-ac97-108572d250fd"} -{"type": "user.message", "data": {"content": "Read only /fixture/workspace/alpha.txt and /fixture/workspace/beta.txt. Do not modify files. Do not access any other files or services. Report the sum of the numeric values in those two files.", "source": "agent-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "supportedNativeDocumentMimeTypes": [], "delivery": "idle", "interactionId": "7c0fa097-c0e2-48da-b2b6-fcfc1ad83a6b", "turnId": "0", "parentAgentTaskId": "5a3e63ac-c073-40ba-b07a-010d534b363e"}, "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "id": "d4e30206-94f5-47d5-b15e-7411b9708aec", "timestamp": "2026-09-10T17:08:44.557Z", "parentId": "3bd18700-90c6-4b2f-aaaa-63760bdfa89a"} -{"type": "assistant.turn_start", "data": {"turnId": "0", "interactionId": "7c0fa097-c0e2-48da-b2b6-fcfc1ad83a6b"}, "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "id": "08de037c-23cf-4e4d-a549-3c745adca7bc", "timestamp": "2026-09-10T17:08:44.561Z", "parentId": "4fdcd841-7cfa-4775-8659-aa8b68e9814a"} -{"type": "assistant.message", "data": {"messageId": "4015ccbc-ef99-466f-bef9-392da640a9b2", "model": "gpt-5.6-luna", "content": "", "toolRequests": [{"toolCallId": "call_zZncCGtp1twgcL2eoFUNwInh", "name": "view", "arguments": {"path": "/fixture/workspace/alpha.txt"}, "type": "function", "intentionSummary": "view the file at /fixture/workspace/alpha.txt."}, {"toolCallId": "call_Jeh7IbrUHaq4jVdxtyQCVrns", "name": "view", "arguments": {"path": "/fixture/workspace/beta.txt"}, "type": "function", "intentionSummary": "view the file at /fixture/workspace/beta.txt."}], "interactionId": "7c0fa097-c0e2-48da-b2b6-fcfc1ad83a6b", "turnId": "0", "rte": true, "parentToolCallId": "call_qx4FH5DADTeT1qVLb37HNpBk"}, "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "id": "df600539-fdd4-4de2-bd42-7f7ff2c952ab", "timestamp": "2026-09-10T17:08:46.507Z", "parentId": "08de037c-23cf-4e4d-a549-3c745adca7bc"} -{"type": "tool.execution_start", "data": {"toolCallId": "call_zZncCGtp1twgcL2eoFUNwInh", "toolName": "view", "arguments": {"path": "/fixture/workspace/alpha.txt"}, "turnId": "0", "model": "gpt-5.6-luna", "parentToolCallId": "call_qx4FH5DADTeT1qVLb37HNpBk"}, "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "id": "ade5b5b6-3af1-4e8d-bd9c-a22f94eb599b", "timestamp": "2026-09-10T17:08:46.508Z", "parentId": "df600539-fdd4-4de2-bd42-7f7ff2c952ab"} -{"type": "tool.execution_start", "data": {"toolCallId": "call_Jeh7IbrUHaq4jVdxtyQCVrns", "toolName": "view", "arguments": {"path": "/fixture/workspace/beta.txt"}, "turnId": "0", "model": "gpt-5.6-luna", "parentToolCallId": "call_qx4FH5DADTeT1qVLb37HNpBk"}, "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "id": "635dec68-22a4-4bb8-9b9b-fac0c1b65a78", "timestamp": "2026-09-10T17:08:46.508Z", "parentId": "ade5b5b6-3af1-4e8d-bd9c-a22f94eb599b"} -{"type": "hook.start", "data": {"hookInvocationId": "aabb620d-669c-4e2e-8ebc-d30637ad6ba3", "hookType": "preToolUse", "input": {"sessionId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "cwd": "/fixture/workspace", "toolCalls": [{"id": "call_zZncCGtp1twgcL2eoFUNwInh", "name": "view", "args": {"path": "/fixture/workspace/alpha.txt"}}, {"id": "call_Jeh7IbrUHaq4jVdxtyQCVrns", "name": "view", "args": {"path": "/fixture/workspace/beta.txt"}}]}, "parentToolCallId": "call_qx4FH5DADTeT1qVLb37HNpBk"}, "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "id": "53bbc5ab-6e18-43c8-9ea0-ea392aec0216", "timestamp": "2026-09-10T17:08:46.511Z", "parentId": "635dec68-22a4-4bb8-9b9b-fac0c1b65a78"} -{"type": "hook.end", "data": {"hookInvocationId": "aabb620d-669c-4e2e-8ebc-d30637ad6ba3", "hookType": "preToolUse", "success": true, "parentToolCallId": "call_qx4FH5DADTeT1qVLb37HNpBk"}, "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "id": "febff749-bbb0-4f41-bf13-9f232e46be62", "timestamp": "2026-09-10T17:08:46.558Z", "parentId": "53bbc5ab-6e18-43c8-9ea0-ea392aec0216"} -{"type": "hook.start", "data": {"hookInvocationId": "9dad8db0-fa58-408c-b1f8-de15ce89170c", "hookType": "postToolUse", "input": {"sessionId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "timestamp": 1789060126559, "cwd": "/fixture/workspace", "toolName": "view", "toolArgs": {"path": "/fixture/workspace/alpha.txt"}, "toolResult": {"resultType": "success", "textResultForLlm": "alpha = 17\n", "sessionLog": "[copilot:elided sessionLog (308 bytes) — pre-hook tool result; the final result may differ, see the adjacent tool.execution_complete event]", "skipLargeOutputProcessing": true}}, "parentToolCallId": "call_qx4FH5DADTeT1qVLb37HNpBk"}, "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "id": "0df623f1-787a-4245-9d2b-866363e15034", "timestamp": "2026-09-10T17:08:46.559Z", "parentId": "febff749-bbb0-4f41-bf13-9f232e46be62"} -{"type": "hook.end", "data": {"hookInvocationId": "9dad8db0-fa58-408c-b1f8-de15ce89170c", "hookType": "postToolUse", "success": true, "parentToolCallId": "call_qx4FH5DADTeT1qVLb37HNpBk"}, "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "id": "0a94d8b0-1009-43e6-9efc-242c113f9452", "timestamp": "2026-09-10T17:08:46.580Z", "parentId": "0df623f1-787a-4245-9d2b-866363e15034"} -{"type": "tool.execution_complete", "data": {"toolCallId": "call_zZncCGtp1twgcL2eoFUNwInh", "model": "gpt-5.6-luna", "interactionId": "7c0fa097-c0e2-48da-b2b6-fcfc1ad83a6b", "turnId": "0", "rte": true, "success": true, "result": {"content": "alpha = 17\n", "detailedContent": "\ndiff --git a/fixture/workspace/alpha.txt b/fixture/workspace/alpha.txt\nindex 0000000..0000000 100644\n--- a/fixture/workspace/alpha.txt\n+++ b/fixture/workspace/alpha.txt\n@@ -1,2 +1,2 @@\n alpha = 17\n \n"}, "parentToolCallId": "call_qx4FH5DADTeT1qVLb37HNpBk"}, "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "id": "e7f7e14d-8658-443a-b65f-91b69c76f74e", "timestamp": "2026-09-10T17:08:46.581Z", "parentId": "0a94d8b0-1009-43e6-9efc-242c113f9452"} -{"type": "hook.start", "data": {"hookInvocationId": "f461652e-836c-4496-b298-ca0db28a7d20", "hookType": "postToolUse", "input": {"sessionId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "timestamp": 1789060126581, "cwd": "/fixture/workspace", "toolName": "view", "toolArgs": {"path": "/fixture/workspace/beta.txt"}, "toolResult": {"resultType": "success", "textResultForLlm": "beta = 25\n", "sessionLog": "[copilot:elided sessionLog (303 bytes) — pre-hook tool result; the final result may differ, see the adjacent tool.execution_complete event]", "skipLargeOutputProcessing": true}}, "parentToolCallId": "call_qx4FH5DADTeT1qVLb37HNpBk"}, "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "id": "5a8e0a8b-1494-4197-be55-346b1cf65196", "timestamp": "2026-09-10T17:08:46.581Z", "parentId": "e7f7e14d-8658-443a-b65f-91b69c76f74e"} -{"type": "hook.end", "data": {"hookInvocationId": "f461652e-836c-4496-b298-ca0db28a7d20", "hookType": "postToolUse", "success": true, "parentToolCallId": "call_qx4FH5DADTeT1qVLb37HNpBk"}, "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "id": "05970ea7-1aa5-40c4-90af-8170132ee2f4", "timestamp": "2026-09-10T17:08:46.602Z", "parentId": "5a8e0a8b-1494-4197-be55-346b1cf65196"} -{"type": "tool.execution_complete", "data": {"toolCallId": "call_Jeh7IbrUHaq4jVdxtyQCVrns", "model": "gpt-5.6-luna", "interactionId": "7c0fa097-c0e2-48da-b2b6-fcfc1ad83a6b", "turnId": "0", "rte": true, "success": true, "result": {"content": "beta = 25\n", "detailedContent": "\ndiff --git a/fixture/workspace/beta.txt b/fixture/workspace/beta.txt\nindex 0000000..0000000 100644\n--- a/fixture/workspace/beta.txt\n+++ b/fixture/workspace/beta.txt\n@@ -1,2 +1,2 @@\n beta = 25\n \n"}, "parentToolCallId": "call_qx4FH5DADTeT1qVLb37HNpBk"}, "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "id": "b9ad6a86-44d9-4482-aa7a-d53811fd4177", "timestamp": "2026-09-10T17:08:46.602Z", "parentId": "05970ea7-1aa5-40c4-90af-8170132ee2f4"} -{"type": "assistant.turn_end", "data": {"turnId": "0"}, "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "id": "d4381f0c-8b93-4295-b04f-27def8b446dd", "timestamp": "2026-09-10T17:08:46.602Z", "parentId": "b9ad6a86-44d9-4482-aa7a-d53811fd4177"} -{"type": "assistant.turn_start", "data": {"turnId": "1", "interactionId": "7c0fa097-c0e2-48da-b2b6-fcfc1ad83a6b"}, "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "id": "31c55d44-2afa-42dc-999e-6c2b0d383df2", "timestamp": "2026-09-10T17:08:46.605Z", "parentId": "d4381f0c-8b93-4295-b04f-27def8b446dd"} -{"type": "hook.start", "data": {"hookInvocationId": "029e591f-4244-4be3-9678-21b6a7fbd0c7", "hookType": "agentStop", "input": {"sessionId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "transcriptPath": "/home/tester/.copilot/session-state/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/events.jsonl", "stopReason": "end_turn", "stop_hook_active": false, "timestamp": 1789060127456, "cwd": "/fixture/workspace"}}, "id": "5c654d22-6d2e-4846-a514-f5af5fb17347", "timestamp": "2026-09-10T17:08:47.456Z", "parentId": "31c55d44-2afa-42dc-999e-6c2b0d383df2"} -{"type": "assistant.message", "data": {"messageId": "561815db-29a4-4c0b-9b8c-f4a0186c2937", "model": "gpt-5.6-luna", "content": "42", "toolRequests": [], "interactionId": "7c0fa097-c0e2-48da-b2b6-fcfc1ad83a6b", "turnId": "1", "phase": "final_answer", "rte": true, "parentToolCallId": "call_qx4FH5DADTeT1qVLb37HNpBk"}, "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "id": "450a1f4c-de11-4d37-bf84-08f63d29588f", "timestamp": "2026-09-10T17:08:47.462Z", "parentId": "5c654d22-6d2e-4846-a514-f5af5fb17347"} -{"type": "assistant.turn_end", "data": {"turnId": "1"}, "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "id": "59c10ad4-04de-40c2-8320-398cc8e2dacb", "timestamp": "2026-09-10T17:08:47.463Z", "parentId": "450a1f4c-de11-4d37-bf84-08f63d29588f"} -{"type": "hook.end", "data": {"hookInvocationId": "029e591f-4244-4be3-9678-21b6a7fbd0c7", "hookType": "agentStop", "success": true}, "id": "7ef9a005-b539-4ef5-9106-c703e1843f86", "timestamp": "2026-09-10T17:08:47.489Z", "parentId": "59c10ad4-04de-40c2-8320-398cc8e2dacb"} -{"type": "hook.start", "data": {"hookInvocationId": "b446b65f-21f8-447c-9e8e-14992e92718a", "hookType": "subagentStop", "input": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "transcriptPath": "/home/tester/.copilot/session-state/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/events.jsonl", "agentName": "explore", "agentType": "explore", "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "stopReason": "end_turn", "response": "42", "timestamp": 1789060127489, "cwd": "/fixture/workspace"}}, "id": "682ab836-6ad2-4f42-a703-4ca32438f826", "timestamp": "2026-09-10T17:08:47.489Z", "parentId": "7ef9a005-b539-4ef5-9106-c703e1843f86"} -{"type": "hook.end", "data": {"hookInvocationId": "b446b65f-21f8-447c-9e8e-14992e92718a", "hookType": "subagentStop", "success": true}, "id": "240e9d7a-1a91-4046-8fe8-bc8bc2f9428f", "timestamp": "2026-09-10T17:08:47.513Z", "parentId": "682ab836-6ad2-4f42-a703-4ca32438f826"} -{"type": "subagent.completed", "data": {"toolCallId": "call_qx4FH5DADTeT1qVLb37HNpBk", "agentName": "explore", "agentDisplayName": "sum-alpha-beta", "model": "gpt-5.6-luna", "firstDispatchedModel": "gpt-5.6-luna", "totalToolCalls": 2, "totalTokens": 9079, "durationMs": 3467}, "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "id": "0031b6bf-9dda-49d2-944e-841635f8533f", "timestamp": "2026-09-10T17:08:47.513Z", "parentId": "240e9d7a-1a91-4046-8fe8-bc8bc2f9428f"} -{"type": "hook.start", "data": {"hookInvocationId": "79f3e676-fffe-4d2a-8d85-5be8452ac9a9", "hookType": "postToolUse", "input": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "timestamp": 1789060127513, "cwd": "/fixture/workspace", "toolName": "task", "toolArgs": {"description": "Sum two text files", "prompt": "Read only /fixture/workspace/alpha.txt and /fixture/workspace/beta.txt. Do not modify files. Do not access any other files or services. Report the sum of the numeric values in those two files.", "agent_type": "explore", "name": "sum-alpha-beta", "mode": "sync"}, "toolResult": {"textResultForLlm": "42", "resultType": "success"}}}, "id": "defc01ae-e8a1-4dba-b36c-fe1ba64655c1", "timestamp": "2026-09-10T17:08:47.513Z", "parentId": "0031b6bf-9dda-49d2-944e-841635f8533f"} -{"type": "hook.end", "data": {"hookInvocationId": "79f3e676-fffe-4d2a-8d85-5be8452ac9a9", "hookType": "postToolUse", "success": true}, "id": "ac9bcb4f-e1b6-45d3-800c-4e3887092454", "timestamp": "2026-09-10T17:08:47.535Z", "parentId": "defc01ae-e8a1-4dba-b36c-fe1ba64655c1"} -{"type": "tool.execution_complete", "data": {"toolCallId": "call_qx4FH5DADTeT1qVLb37HNpBk", "model": "gpt-5.6-luna", "interactionId": "793d3703-6f4a-4814-8877-34a7325848ce", "turnId": "0", "rte": true, "success": true, "result": {"content": "42", "detailedContent": "42"}}, "id": "e0f6fb2a-cef8-445a-a482-01f8230aab60", "timestamp": "2026-09-10T17:08:47.536Z", "parentId": "ac9bcb4f-e1b6-45d3-800c-4e3887092454"} -{"type": "assistant.turn_end", "data": {"turnId": "0"}, "id": "28e600a8-0102-43b9-b8ca-f37979506b4b", "timestamp": "2026-09-10T17:08:47.536Z", "parentId": "e0f6fb2a-cef8-445a-a482-01f8230aab60"} -{"type": "assistant.turn_start", "data": {"turnId": "1", "interactionId": "793d3703-6f4a-4814-8877-34a7325848ce"}, "id": "9bbdf1d2-97de-4f78-955b-38618edc2c3f", "timestamp": "2026-09-10T17:08:47.537Z", "parentId": "28e600a8-0102-43b9-b8ca-f37979506b4b"} -{"type": "assistant.message", "data": {"messageId": "bd86f77e-c00c-432a-9184-4660aa0e4bb9", "model": "gpt-5.6-luna", "content": "42", "toolRequests": [], "interactionId": "793d3703-6f4a-4814-8877-34a7325848ce", "turnId": "1", "phase": "final_answer", "rte": true}, "id": "10001566-2704-4f75-add3-2c044373eaba", "timestamp": "2026-09-10T17:08:48.284Z", "parentId": "9bbdf1d2-97de-4f78-955b-38618edc2c3f"} -{"type": "assistant.turn_end", "data": {"turnId": "1"}, "id": "78003d8e-7ea7-48eb-9cf0-aa13043da942", "timestamp": "2026-09-10T17:08:48.287Z", "parentId": "10001566-2704-4f75-add3-2c044373eaba"} -{"type": "hook.start", "data": {"hookInvocationId": "7e0d22f3-5b10-4e1e-88bc-d102c12e022f", "hookType": "agentStop", "input": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "transcriptPath": "/home/tester/.copilot/session-state/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/events.jsonl", "stopReason": "end_turn", "stop_hook_active": false, "timestamp": 1789060128288, "cwd": "/fixture/workspace"}}, "id": "c56043b3-ad8e-4d9a-b160-0b55485a4fb5", "timestamp": "2026-09-10T17:08:48.288Z", "parentId": "78003d8e-7ea7-48eb-9cf0-aa13043da942"} -{"type": "hook.end", "data": {"hookInvocationId": "7e0d22f3-5b10-4e1e-88bc-d102c12e022f", "hookType": "agentStop", "success": true}, "id": "286e55ad-fcfd-4aea-ae98-f2d62206891f", "timestamp": "2026-09-10T17:08:48.319Z", "parentId": "c56043b3-ad8e-4d9a-b160-0b55485a4fb5"} -{"type": "hook.start", "data": {"hookInvocationId": "d677f404-2c71-4c9c-bcd3-5f54fe995d5c", "hookType": "sessionEnd", "input": {"reason": "user_exit", "sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "timestamp": 1789060147875, "cwd": "/fixture/workspace"}}, "id": "c85630cc-3c7e-4f42-80b3-610284c8b9cc", "timestamp": "2026-09-10T17:09:07.875Z", "parentId": "79eec31d-d495-40fc-a31d-8a05f73fa8d3"} -{"type": "hook.end", "data": {"hookInvocationId": "d677f404-2c71-4c9c-bcd3-5f54fe995d5c", "hookType": "sessionEnd", "success": true}, "id": "f39853b5-344f-4582-95ec-75eb1da62165", "timestamp": "2026-09-10T17:09:07.904Z", "parentId": "c85630cc-3c7e-4f42-80b3-610284c8b9cc"} -{"type": "session.shutdown", "data": {"shutdownType": "routine", "totalPremiumRequests": 2, "totalNanoAiu": 373366000, "tokenDetails": {"input": {"tokenCount": 18}, "cache_read": {"tokenCount": 23948}, "cache_write": {"tokenCount": 11430}, "output": {"tokenCount": 328}}, "totalApiDurationMs": 8608, "sessionStartTime": 1789060085884, "eventsFileSizeBytes": 194432, "codeChanges": {"linesAdded": 0, "linesRemoved": 0, "filesModified": []}, "modelMetrics": {"gpt-5.6-luna": {"requests": {"count": 6, "cost": 2}, "usage": {"inputTokens": 35396, "outputTokens": 328, "cacheReadTokens": 23948, "cacheWriteTokens": 11430, "reasoningTokens": 55}, "totalNanoAiu": 373366000, "tokenDetails": {"input": {"tokenCount": 18}, "cache_read": {"tokenCount": 23948}, "cache_write": {"tokenCount": 11430}, "output": {"tokenCount": 328}}}}, "agentMetrics": {"main": {"totalApiDurationMs": 5825, "totalNanoAiu": 238814000.0, "modelMetrics": {"gpt-5.6-luna": {"requests": {"count": 4, "cost": 2.0}, "usage": {"inputTokens": 26416, "outputTokens": 229, "cacheReadTokens": 19522, "cacheWriteTokens": 6882, "reasoningTokens": 39}, "totalNanoAiu": 238814000.0, "tokenDetails": {"input": {"tokenCount": 12}, "cache_read": {"tokenCount": 19522}, "cache_write": {"tokenCount": 6882}, "output": {"tokenCount": 229}}}}}, "bf8cb9f3-2097-4db0-a3c8-78a2653b2106": {"agentName": "explore", "agentDisplayName": "sum-alpha-beta", "totalApiDurationMs": 2783, "totalNanoAiu": 134552000.0, "modelMetrics": {"gpt-5.6-luna": {"requests": {"count": 2, "cost": 0.0}, "usage": {"inputTokens": 8980, "outputTokens": 99, "cacheReadTokens": 4426, "cacheWriteTokens": 4548, "reasoningTokens": 16}, "totalNanoAiu": 134552000.0, "tokenDetails": {"input": {"tokenCount": 6}, "cache_read": {"tokenCount": 4426}, "cache_write": {"tokenCount": 4548}, "output": {"tokenCount": 99}}}}}}, "currentModel": "gpt-5.6-luna", "currentTokens": 6942, "systemTokens": 5015, "conversationTokens": 428, "toolDefinitionsTokens": 1496}, "id": "9dbdd079-e22a-4bd0-87e2-17fa2b246d23", "timestamp": "2026-09-10T17:09:07.907Z", "parentId": "f39853b5-344f-4582-95ec-75eb1da62165"} diff --git a/tests/fixtures/copilot/cli-1.0.83/hooks.jsonl b/tests/fixtures/copilot/cli-1.0.83/hooks.jsonl deleted file mode 100644 index 51c7658..0000000 --- a/tests/fixtures/copilot/cli-1.0.83/hooks.jsonl +++ /dev/null @@ -1,20 +0,0 @@ -{"registered_event": "userPromptSubmitted", "captured_ns": 1789060102200850000, "payload": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "timestamp": 1789060102173, "cwd": "/fixture/workspace", "prompt": "Read alpha.txt and beta.txt with separate view calls in parallel and report their sum. Only read those two files."}} -{"registered_event": "sessionStart", "captured_ns": 1789060102225028000, "payload": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "timestamp": 1789060102204, "cwd": "/fixture/workspace", "source": "new", "initialPrompt": "Read alpha.txt and beta.txt with separate view calls in parallel and report their sum. Only read those two files."}} -{"registered_event": "preToolUse", "captured_ns": 1789060104528610000, "payload": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "timestamp": 1789060104506, "cwd": "/fixture/workspace", "toolName": "view", "toolArgs": {"path": "/fixture/workspace/alpha.txt"}}} -{"registered_event": "preToolUse", "captured_ns": 1789060104549556000, "payload": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "timestamp": 1789060104506, "cwd": "/fixture/workspace", "toolName": "view", "toolArgs": {"path": "/fixture/workspace/beta.txt"}}} -{"registered_event": "postToolUse", "captured_ns": 1789060104570196000, "payload": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "timestamp": 1789060104552, "cwd": "/fixture/workspace", "toolName": "view", "toolArgs": {"path": "/fixture/workspace/alpha.txt"}, "toolResult": {"resultType": "success", "textResultForLlm": "alpha = 17\n"}}} -{"registered_event": "postToolUse", "captured_ns": 1789060104590707000, "payload": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "timestamp": 1789060104572, "cwd": "/fixture/workspace", "toolName": "view", "toolArgs": {"path": "/fixture/workspace/beta.txt"}, "toolResult": {"resultType": "success", "textResultForLlm": "beta = 25\n"}}} -{"registered_event": "agentStop", "captured_ns": 1789060105647636000, "payload": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "timestamp": 1789060105626, "cwd": "/fixture/workspace", "transcriptPath": "/home/tester/.copilot/session-state/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/events.jsonl", "stopReason": "end_turn", "stop_hook_active": false}} -{"registered_event": "userPromptSubmitted", "captured_ns": 1789060122189769000, "payload": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "timestamp": 1789060122166, "cwd": "/fixture/workspace", "prompt": "Invoke one explore subagent to read only alpha.txt and beta.txt and report their sum. Do not modify files or access other files or services. Then report its answer."}} -{"registered_event": "preToolUse", "captured_ns": 1789060124042792000, "payload": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "timestamp": 1789060124021, "cwd": "/fixture/workspace", "toolName": "task", "toolArgs": {"description": "Sum two text files", "prompt": "Read only /fixture/workspace/alpha.txt and /fixture/workspace/beta.txt. Do not modify files. Do not access any other files or services. Report the sum of the numeric values in those two files.", "agent_type": "explore", "name": "sum-alpha-beta", "mode": "sync"}}} -{"registered_event": "subagentStart", "captured_ns": 1789060124100211000, "payload": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "timestamp": 1789060124082, "cwd": "/fixture/workspace", "transcriptPath": "/home/tester/.copilot/session-state/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/events.jsonl", "agentName": "explore"}} -{"registered_event": "userPromptSubmitted", "captured_ns": 1789060124554272000, "payload": {"sessionId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "timestamp": 1789060124530, "cwd": "/fixture/workspace", "prompt": "Read only /fixture/workspace/alpha.txt and /fixture/workspace/beta.txt. Do not modify files. Do not access any other files or services. Report the sum of the numeric values in those two files."}} -{"registered_event": "preToolUse", "captured_ns": 1789060126531610000, "payload": {"sessionId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "timestamp": 1789060126501, "cwd": "/fixture/workspace", "toolName": "view", "toolArgs": {"path": "/fixture/workspace/alpha.txt"}}} -{"registered_event": "preToolUse", "captured_ns": 1789060126555790000, "payload": {"sessionId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "timestamp": 1789060126501, "cwd": "/fixture/workspace", "toolName": "view", "toolArgs": {"path": "/fixture/workspace/beta.txt"}}} -{"registered_event": "postToolUse", "captured_ns": 1789060126578392000, "payload": {"sessionId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "timestamp": 1789060126559, "cwd": "/fixture/workspace", "toolName": "view", "toolArgs": {"path": "/fixture/workspace/alpha.txt"}, "toolResult": {"resultType": "success", "textResultForLlm": "alpha = 17\n"}}} -{"registered_event": "postToolUse", "captured_ns": 1789060126599701000, "payload": {"sessionId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "timestamp": 1789060126581, "cwd": "/fixture/workspace", "toolName": "view", "toolArgs": {"path": "/fixture/workspace/beta.txt"}, "toolResult": {"resultType": "success", "textResultForLlm": "beta = 25\n"}}} -{"registered_event": "agentStop", "captured_ns": 1789060127486265000, "payload": {"sessionId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "timestamp": 1789060127456, "cwd": "/fixture/workspace", "transcriptPath": "/home/tester/.copilot/session-state/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/events.jsonl", "stopReason": "end_turn", "stop_hook_active": false}} -{"registered_event": "subagentStop", "captured_ns": 1789060127510837000, "payload": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "timestamp": 1789060127489, "cwd": "/fixture/workspace", "transcriptPath": "/home/tester/.copilot/session-state/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/events.jsonl", "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "agentType": "explore", "agentName": "explore", "response": "42", "stopReason": "end_turn"}} -{"registered_event": "postToolUse", "captured_ns": 1789060127533513000, "payload": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "timestamp": 1789060127513, "cwd": "/fixture/workspace", "toolName": "task", "toolArgs": {"description": "Sum two text files", "prompt": "Read only /fixture/workspace/alpha.txt and /fixture/workspace/beta.txt. Do not modify files. Do not access any other files or services. Report the sum of the numeric values in those two files.", "agent_type": "explore", "name": "sum-alpha-beta", "mode": "sync"}, "toolResult": {"resultType": "success", "textResultForLlm": "42"}}} -{"registered_event": "agentStop", "captured_ns": 1789060128316253000, "payload": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "timestamp": 1789060128288, "cwd": "/fixture/workspace", "transcriptPath": "/home/tester/.copilot/session-state/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/events.jsonl", "stopReason": "end_turn", "stop_hook_active": false}} -{"registered_event": "sessionEnd", "captured_ns": 1789060147901644000, "payload": {"sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "timestamp": 1789060147875, "cwd": "/fixture/workspace", "reason": "user_exit"}} diff --git a/tests/fixtures/copilot/cli-1.0.83/usage.json b/tests/fixtures/copilot/cli-1.0.83/usage.json deleted file mode 100644 index 2ea30a9..0000000 --- a/tests/fixtures/copilot/cli-1.0.83/usage.json +++ /dev/null @@ -1,134 +0,0 @@ -{ - "shutdownType": "routine", - "totalPremiumRequests": 2, - "totalNanoAiu": 373366000, - "tokenDetails": { - "input": { - "tokenCount": 18 - }, - "cache_read": { - "tokenCount": 23948 - }, - "cache_write": { - "tokenCount": 11430 - }, - "output": { - "tokenCount": 328 - } - }, - "totalApiDurationMs": 8608, - "sessionStartTime": 1789060085884, - "eventsFileSizeBytes": 194432, - "codeChanges": { - "linesAdded": 0, - "linesRemoved": 0, - "filesModified": [] - }, - "modelMetrics": { - "gpt-5.6-luna": { - "requests": { - "count": 6, - "cost": 2 - }, - "usage": { - "inputTokens": 35396, - "outputTokens": 328, - "cacheReadTokens": 23948, - "cacheWriteTokens": 11430, - "reasoningTokens": 55 - }, - "totalNanoAiu": 373366000, - "tokenDetails": { - "input": { - "tokenCount": 18 - }, - "cache_read": { - "tokenCount": 23948 - }, - "cache_write": { - "tokenCount": 11430 - }, - "output": { - "tokenCount": 328 - } - } - } - }, - "agentMetrics": { - "main": { - "totalApiDurationMs": 5825, - "totalNanoAiu": 238814000.0, - "modelMetrics": { - "gpt-5.6-luna": { - "requests": { - "count": 4, - "cost": 2.0 - }, - "usage": { - "inputTokens": 26416, - "outputTokens": 229, - "cacheReadTokens": 19522, - "cacheWriteTokens": 6882, - "reasoningTokens": 39 - }, - "totalNanoAiu": 238814000.0, - "tokenDetails": { - "input": { - "tokenCount": 12 - }, - "cache_read": { - "tokenCount": 19522 - }, - "cache_write": { - "tokenCount": 6882 - }, - "output": { - "tokenCount": 229 - } - } - } - } - }, - "bf8cb9f3-2097-4db0-a3c8-78a2653b2106": { - "agentName": "explore", - "agentDisplayName": "sum-alpha-beta", - "totalApiDurationMs": 2783, - "totalNanoAiu": 134552000.0, - "modelMetrics": { - "gpt-5.6-luna": { - "requests": { - "count": 2, - "cost": 0.0 - }, - "usage": { - "inputTokens": 8980, - "outputTokens": 99, - "cacheReadTokens": 4426, - "cacheWriteTokens": 4548, - "reasoningTokens": 16 - }, - "totalNanoAiu": 134552000.0, - "tokenDetails": { - "input": { - "tokenCount": 6 - }, - "cache_read": { - "tokenCount": 4426 - }, - "cache_write": { - "tokenCount": 4548 - }, - "output": { - "tokenCount": 99 - } - } - } - } - } - }, - "currentModel": "gpt-5.6-luna", - "currentTokens": 6942, - "systemTokens": 5015, - "conversationTokens": 428, - "toolDefinitionsTokens": 1496 -} diff --git a/tests/test_codex_captured_env.py b/tests/platforms/codex/test_captured_env.py similarity index 100% rename from tests/test_codex_captured_env.py rename to tests/platforms/codex/test_captured_env.py diff --git a/tests/platforms/copilot/fixtures/README.md b/tests/platforms/copilot/fixtures/README.md index 5a2fa42..a43a168 100644 --- a/tests/platforms/copilot/fixtures/README.md +++ b/tests/platforms/copilot/fixtures/README.md @@ -27,7 +27,7 @@ Sanitization replaces the local home/workspace paths and removes system messages - External pre/post tool hooks have no invocation ID in this run. The transcript provides `toolCallId` on requests and executions. Correlate parallel tools using the transcript, not just tool names. - `turnId` denotes a model/tool cycle and resets across user requests. Group user interactions using `interactionId` plus agent identity. - Subagent events are interleaved in the same transcript and carry top-level `agentId`; `subagent.started` links to the parent task via `data.toolCallId`. Child records also expose `parentToolCallId` where applicable. -- The child generated its own user-prompt and agent-stop hooks using the parent's session ID and transcript path. There are three prompt hooks and three stop hooks for two top-level user requests. External hook payloads alone cannot reliably distinguish these turns. +- The child generated its own user-prompt and agent-stop hooks. Those payloads set `sessionId` to the child agent ID (`bf8cb9f3-2097-4db0-a3c8-78a2653b2106`), not the parent session ID. Child pre/post tool hooks do the same. `transcriptPath` on the child's agentStop still points at the parent session's `events.jsonl`. `subagentStart`/`subagentStop` keep the parent session ID and name the child in `agentId`. Sanitization did not rewrite these identifiers; transcript `hook.start` input matches the external recorder. Hook `sessionId` is therefore not a native session ID for capture routing. Child prompt/stop hooks cannot drive a session-ID-only turn state machine. There are three prompt hooks and three stop hooks for two top-level user requests. - SessionStart arrived after the first user-prompt hook. Initialization must tolerate that ordering. - There are 20 external recorder files but only 18 hook.start/hook.end pairs in the final transcript. Do not assume those two streams have one-to-one coverage. - Assistant text and model names are present in assistant.message. Final session.shutdown includes input/output/cache/reasoning usage and per-agent metrics. Availability of per-call usage at Stop time has not been established by this probe. diff --git a/tests/fixtures/copilot/v1-cases/README.md b/tests/platforms/copilot/fixtures/cases/README.md similarity index 95% rename from tests/fixtures/copilot/v1-cases/README.md rename to tests/platforms/copilot/fixtures/cases/README.md index 0de17e2..dfeca25 100644 --- a/tests/fixtures/copilot/v1-cases/README.md +++ b/tests/platforms/copilot/fixtures/cases/README.md @@ -1,4 +1,4 @@ -# Copilot V1 synthetic contract cases +# Copilot synthetic contract cases Every file in this directory is synthetic. It documents capture contracts and reader/archive edge cases; it is not evidence of Copilot CLI behavior. @@ -23,7 +23,7 @@ location, and content digest rather than an invented native event ID. `source-key-prefix-collision.json` is the archive-reuse contract for two homes whose SHA-256 digests share a 16-character display prefix. -The observed `cli-1.0.83` sibling fixture has two main prompts and one child +The observed sibling fixture has two main prompts and one child prompt. The child prompt is retained as source evidence, not asserted to be a third human request. Its six `assistant_usage_events` rows remain raw database evidence and must not create UsageStore rows. diff --git a/tests/fixtures/copilot/v1-cases/database-row-revisions.json b/tests/platforms/copilot/fixtures/cases/database-row-revisions.json similarity index 100% rename from tests/fixtures/copilot/v1-cases/database-row-revisions.json rename to tests/platforms/copilot/fixtures/cases/database-row-revisions.json diff --git a/tests/fixtures/copilot/v1-cases/distinct-hook-observations.json b/tests/platforms/copilot/fixtures/cases/distinct-hook-observations.json similarity index 100% rename from tests/fixtures/copilot/v1-cases/distinct-hook-observations.json rename to tests/platforms/copilot/fixtures/cases/distinct-hook-observations.json diff --git a/tests/fixtures/copilot/v1-cases/missing-event-id.jsonl b/tests/platforms/copilot/fixtures/cases/missing-event-id.jsonl similarity index 100% rename from tests/fixtures/copilot/v1-cases/missing-event-id.jsonl rename to tests/platforms/copilot/fixtures/cases/missing-event-id.jsonl diff --git a/tests/fixtures/copilot/v1-cases/source-batch.json b/tests/platforms/copilot/fixtures/cases/source-batch.json similarity index 100% rename from tests/fixtures/copilot/v1-cases/source-batch.json rename to tests/platforms/copilot/fixtures/cases/source-batch.json diff --git a/tests/fixtures/copilot/v1-cases/source-key-prefix-collision.json b/tests/platforms/copilot/fixtures/cases/source-key-prefix-collision.json similarity index 100% rename from tests/fixtures/copilot/v1-cases/source-key-prefix-collision.json rename to tests/platforms/copilot/fixtures/cases/source-key-prefix-collision.json diff --git a/tests/fixtures/copilot/v1-cases/source-slice.json b/tests/platforms/copilot/fixtures/cases/source-slice.json similarity index 100% rename from tests/fixtures/copilot/v1-cases/source-slice.json rename to tests/platforms/copilot/fixtures/cases/source-slice.json diff --git a/tests/fixtures/copilot/v1-cases/trailing-json.jsonl b/tests/platforms/copilot/fixtures/cases/trailing-json.jsonl similarity index 100% rename from tests/fixtures/copilot/v1-cases/trailing-json.jsonl rename to tests/platforms/copilot/fixtures/cases/trailing-json.jsonl diff --git a/tests/fixtures/copilot/v1-cases/trailing-utf8.hex b/tests/platforms/copilot/fixtures/cases/trailing-utf8.hex similarity index 100% rename from tests/fixtures/copilot/v1-cases/trailing-utf8.hex rename to tests/platforms/copilot/fixtures/cases/trailing-utf8.hex diff --git a/tests/fixtures/copilot/v1-cases/unknown-event-fields.jsonl b/tests/platforms/copilot/fixtures/cases/unknown-event-fields.jsonl similarity index 100% rename from tests/fixtures/copilot/v1-cases/unknown-event-fields.jsonl rename to tests/platforms/copilot/fixtures/cases/unknown-event-fields.jsonl diff --git a/tests/test_copilot_archive.py b/tests/platforms/copilot/test_archive.py similarity index 99% rename from tests/test_copilot_archive.py rename to tests/platforms/copilot/test_archive.py index 0838de5..7930937 100644 --- a/tests/test_copilot_archive.py +++ b/tests/platforms/copilot/test_archive.py @@ -19,7 +19,7 @@ from thirdeye.platforms.copilot.types import SourceBatch, SourcePaths, SourceRecord from thirdeye.reader import SessionReader -FIXTURES = Path(__file__).parent / "fixtures" / "copilot" / "v1-cases" +FIXTURES = Path(__file__).parent / "fixtures" / "cases" NATIVE_ID = "session-a" diff --git a/tests/test_copilot_capture.py b/tests/platforms/copilot/test_capture.py similarity index 99% rename from tests/test_copilot_capture.py rename to tests/platforms/copilot/test_capture.py index cb35bd6..061141a 100644 --- a/tests/test_copilot_capture.py +++ b/tests/platforms/copilot/test_capture.py @@ -32,8 +32,8 @@ from thirdeye.platforms.copilot.spool import enqueue_hook, read_spool from thirdeye.platforms.copilot.types import SourceBatch, SourcePaths, SourceRecord, SyncResult -FIXTURES = Path(__file__).parent / "fixtures" / "copilot" -CLI_FIXTURE = FIXTURES / "cli-1.0.83" +FIXTURES = Path(__file__).parent / "fixtures" +CLI_FIXTURE = FIXTURES NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" OBSERVED_AT = "2026-09-10T17:08:25.626Z" diff --git a/tests/test_copilot_capture_reads.py b/tests/platforms/copilot/test_capture_reads.py similarity index 100% rename from tests/test_copilot_capture_reads.py rename to tests/platforms/copilot/test_capture_reads.py diff --git a/tests/test_copilot_command.py b/tests/platforms/copilot/test_command.py similarity index 99% rename from tests/test_copilot_command.py rename to tests/platforms/copilot/test_command.py index 8fef801..0a4b756 100644 --- a/tests/test_copilot_command.py +++ b/tests/platforms/copilot/test_command.py @@ -17,8 +17,8 @@ from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id from thirdeye.platforms.copilot.types import SourcePaths, SyncResult -FIXTURES = Path(__file__).parent / "fixtures" / "copilot" -CLI_FIXTURE = FIXTURES / "cli-1.0.83" +FIXTURES = Path(__file__).parent / "fixtures" +CLI_FIXTURE = FIXTURES NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" HOOK_BIN = "thirdeye-copilot-hook" SECRET_PROMPT = "SECRET PROMPT BODY" @@ -266,7 +266,7 @@ def test_copilot_commands_have_no_export_flag() -> None: def test_pyproject_registers_copilot_hook_entrypoint() -> None: - pyproject = Path(__file__).resolve().parents[1] / "pyproject.toml" + pyproject = Path(__file__).resolve().parents[3] / "pyproject.toml" text = pyproject.read_text(encoding="utf-8") assert f"{HOOK_BIN} =" in text assert "thirdeye.platforms.copilot.hooks:main" in text diff --git a/tests/test_copilot_contracts.py b/tests/platforms/copilot/test_contracts.py similarity index 98% rename from tests/test_copilot_contracts.py rename to tests/platforms/copilot/test_contracts.py index 4490c9a..45990c3 100644 --- a/tests/test_copilot_contracts.py +++ b/tests/platforms/copilot/test_contracts.py @@ -43,9 +43,9 @@ SyncResult, ) -FIXTURES = Path(__file__).parent / "fixtures" / "copilot" -CLI_FIXTURE = FIXTURES / "cli-1.0.83" -V1_CASES = FIXTURES / "v1-cases" +FIXTURES = Path(__file__).parent / "fixtures" +CLI_FIXTURE = FIXTURES +V1_CASES = FIXTURES / "cases" NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" CHILD_AGENT_ID = "bf8cb9f3-2097-4db0-a3c8-78a2653b2106" @@ -345,7 +345,7 @@ def test_source_key_prefix_collision_is_an_archive_reuse_contract(): source_keys_share_stored_prefix(first["source_key"], "too-short") -# --- observed cli-1.0.83 fixtures --- +# --- observed CLI fixtures --- def test_cli_fixture_files_exist(): @@ -461,7 +461,7 @@ def test_cli_usage_totals_match_assistant_usage_events(): assert row_total == model_usage[usage_key] -# --- synthetic v1-cases fixtures --- +# --- synthetic contract-case fixtures --- def test_v1_trailing_json_fixture_has_complete_and_incomplete_tail(): diff --git a/tests/test_copilot_database.py b/tests/platforms/copilot/test_database.py similarity index 99% rename from tests/test_copilot_database.py rename to tests/platforms/copilot/test_database.py index 0506cfc..64089c6 100644 --- a/tests/test_copilot_database.py +++ b/tests/platforms/copilot/test_database.py @@ -16,8 +16,8 @@ from thirdeye.platforms.copilot.database import discover_database_sessions, read_database from thirdeye.platforms.copilot.identity import resolve_sources -FIXTURES = Path(__file__).parent / "fixtures" / "copilot" -CLI_FIXTURE = FIXTURES / "cli-1.0.83" +FIXTURES = Path(__file__).parent / "fixtures" +CLI_FIXTURE = FIXTURES NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" diff --git a/tests/test_copilot_followup.py b/tests/platforms/copilot/test_followup.py similarity index 100% rename from tests/test_copilot_followup.py rename to tests/platforms/copilot/test_followup.py diff --git a/tests/test_copilot_hook_payload.py b/tests/platforms/copilot/test_hook_payload.py similarity index 98% rename from tests/test_copilot_hook_payload.py rename to tests/platforms/copilot/test_hook_payload.py index 0443463..5d102d0 100644 --- a/tests/test_copilot_hook_payload.py +++ b/tests/platforms/copilot/test_hook_payload.py @@ -17,8 +17,8 @@ from thirdeye.platforms.copilot.hook_payload import parse_hook from thirdeye.platforms.copilot.types import SourceRecord -FIXTURES = Path(__file__).parent / "fixtures" / "copilot" -CLI_HOOKS = FIXTURES / "cli-1.0.83" / "hooks.jsonl" +FIXTURES = Path(__file__).parent / "fixtures" +CLI_HOOKS = FIXTURES / "hooks.jsonl" NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" CHILD_AGENT_ID = "bf8cb9f3-2097-4db0-a3c8-78a2653b2106" OBSERVED_AT = "2026-09-10T17:08:25.626Z" diff --git a/tests/test_copilot_hooks.py b/tests/platforms/copilot/test_hooks.py similarity index 99% rename from tests/test_copilot_hooks.py rename to tests/platforms/copilot/test_hooks.py index ab51d52..31109a6 100644 --- a/tests/test_copilot_hooks.py +++ b/tests/platforms/copilot/test_hooks.py @@ -29,8 +29,8 @@ from thirdeye.reader import SessionReader from thirdeye.tags import TagStore -FIXTURES = Path(__file__).parent / "fixtures" / "copilot" -CLI_FIXTURE = FIXTURES / "cli-1.0.83" +FIXTURES = Path(__file__).parent / "fixtures" +CLI_FIXTURE = FIXTURES NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" CHILD_SESSION_ID = "bf8cb9f3-2097-4db0-a3c8-78a2653b2106" diff --git a/tests/test_copilot_install.py b/tests/platforms/copilot/test_install.py similarity index 100% rename from tests/test_copilot_install.py rename to tests/platforms/copilot/test_install.py diff --git a/tests/test_copilot_recovery.py b/tests/platforms/copilot/test_recovery.py similarity index 100% rename from tests/test_copilot_recovery.py rename to tests/platforms/copilot/test_recovery.py diff --git a/tests/test_copilot_sources.py b/tests/platforms/copilot/test_sources.py similarity index 99% rename from tests/test_copilot_sources.py rename to tests/platforms/copilot/test_sources.py index c7e7555..ab50ea0 100644 --- a/tests/test_copilot_sources.py +++ b/tests/platforms/copilot/test_sources.py @@ -15,8 +15,8 @@ from thirdeye.platforms.copilot.transcript import discover_transcripts from thirdeye.platforms.copilot.types import SourcePaths, SourceRecord, SourceSlice -FIXTURES = Path(__file__).parent / "fixtures" / "copilot" -V1_SLICE = FIXTURES / "v1-cases" / "source-slice.json" +FIXTURES = Path(__file__).parent / "fixtures" +V1_SLICE = FIXTURES / "cases" / "source-slice.json" NATIVE_ID = "session-a" diff --git a/tests/test_copilot_spool.py b/tests/platforms/copilot/test_spool.py similarity index 100% rename from tests/test_copilot_spool.py rename to tests/platforms/copilot/test_spool.py diff --git a/tests/test_copilot_status.py b/tests/platforms/copilot/test_status.py similarity index 99% rename from tests/test_copilot_status.py rename to tests/platforms/copilot/test_status.py index cb4a47f..a3dd745 100644 --- a/tests/test_copilot_status.py +++ b/tests/platforms/copilot/test_status.py @@ -37,8 +37,8 @@ from thirdeye.platforms.copilot.status import capture_status from thirdeye.platforms.copilot.types import SourceBatch, SourcePaths, SourceRecord -FIXTURES = Path(__file__).parent / "fixtures" / "copilot" -CLI_FIXTURE = FIXTURES / "cli-1.0.83" +FIXTURES = Path(__file__).parent / "fixtures" +CLI_FIXTURE = FIXTURES NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" OBSERVED_AT_EARLY = "2026-09-10T17:08:20.000Z" OBSERVED_AT_LATE = "2026-09-10T17:08:30.000Z" diff --git a/tests/test_copilot_transcript.py b/tests/platforms/copilot/test_transcript.py similarity index 99% rename from tests/test_copilot_transcript.py rename to tests/platforms/copilot/test_transcript.py index b874fc9..f9f3f1f 100644 --- a/tests/test_copilot_transcript.py +++ b/tests/platforms/copilot/test_transcript.py @@ -14,9 +14,9 @@ from thirdeye.platforms.copilot.transcript import discover_transcripts, read_transcript from thirdeye.platforms.copilot.types import SourcePaths, SourceRecord, SourceSlice -FIXTURES = Path(__file__).parent / "fixtures" / "copilot" -V1_CASES = FIXTURES / "v1-cases" -CLI_FIXTURE = FIXTURES / "cli-1.0.83" +FIXTURES = Path(__file__).parent / "fixtures" +V1_CASES = FIXTURES / "cases" +CLI_FIXTURE = FIXTURES NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" diff --git a/tests/test_copilot_watch.py b/tests/platforms/copilot/test_watch.py similarity index 99% rename from tests/test_copilot_watch.py rename to tests/platforms/copilot/test_watch.py index 9b3886c..c137d9d 100644 --- a/tests/test_copilot_watch.py +++ b/tests/platforms/copilot/test_watch.py @@ -17,8 +17,8 @@ from thirdeye.platforms.copilot.types import SourcePaths, SyncResult from thirdeye.platforms.copilot.watch import _changed_sessions, watch -FIXTURES = Path(__file__).parent / "fixtures" / "copilot" -CLI_FIXTURE = FIXTURES / "cli-1.0.83" +FIXTURES = Path(__file__).parent / "fixtures" +CLI_FIXTURE = FIXTURES NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" OTHER_SESSION_ID = "a1b2c3d4-e5f6-7890-abcd-ef1234567890" OBSERVED_AT = "2026-09-10T17:08:25.626Z" From 6f3f28cd310ce56d17a0e6ab810e637fa7231fc4 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Fri, 11 Sep 2026 13:53:54 -0700 Subject: [PATCH 44/88] Run Windows CI on Python 3.12 --- .github/workflows/test.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 465bf71..1636f30 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -37,12 +37,9 @@ jobs: python-version: "3.13" - os: ubuntu-latest python-version: "3.14" - # Windows support is experimental: minimum supported version plus a - # recent stable, to bound runner cost. + # Exercise Windows on the project's primary Python version. - os: windows-latest - python-version: "3.11" - - os: windows-latest - python-version: "3.13" + python-version: "3.12" steps: - uses: actions/checkout@v5 From 2ac2db40342e84ba5234234bf339b201830bbc98 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Fri, 11 Sep 2026 15:06:16 -0700 Subject: [PATCH 45/88] Add Copilot projection contracts --- src/thirdeye/platforms/copilot/types.py | 192 +++++++++++++++++- src/thirdeye/tracing/__init__.py | 2 + src/thirdeye/tracing/model.py | 19 ++ .../fixtures/reconciliation-cases/README.md | 41 ++++ .../accounting-projection.json | 18 ++ .../fixtures/reconciliation-cases/cases.json | 12 ++ .../observed-six-calls.json | 15 ++ .../semantic-projection.json | 42 ++++ .../reconciliation-cases/storage.json | 27 +++ .../reconciliation-cases/transport.json | 23 +++ 10 files changed, 390 insertions(+), 1 deletion(-) create mode 100644 tests/platforms/copilot/fixtures/reconciliation-cases/README.md create mode 100644 tests/platforms/copilot/fixtures/reconciliation-cases/accounting-projection.json create mode 100644 tests/platforms/copilot/fixtures/reconciliation-cases/cases.json create mode 100644 tests/platforms/copilot/fixtures/reconciliation-cases/observed-six-calls.json create mode 100644 tests/platforms/copilot/fixtures/reconciliation-cases/semantic-projection.json create mode 100644 tests/platforms/copilot/fixtures/reconciliation-cases/storage.json create mode 100644 tests/platforms/copilot/fixtures/reconciliation-cases/transport.json diff --git a/src/thirdeye/platforms/copilot/types.py b/src/thirdeye/platforms/copilot/types.py index ba0f5ee..6939ec1 100644 --- a/src/thirdeye/platforms/copilot/types.py +++ b/src/thirdeye/platforms/copilot/types.py @@ -6,7 +6,10 @@ from __future__ import annotations -from typing import Any, TypedDict +from typing import Any, Literal, TypedDict + +from thirdeye.tracing.model import TurnSpanDict +from thirdeye.usage.types import UsageRow from .constants import SOURCE_SCHEMA_VERSION @@ -65,3 +68,190 @@ class SourceSlice(TypedDict): diagnostics: list[dict[str, Any]] cwd: str | None exhausted: bool + + +# V2 projection contracts --------------------------------------------------- +# +# These are deliberately additive to the V1 source envelopes above. Raw +# source records are immutable input; projections are replaceable derived +# state. A source ID, database generation, and content revision are therefore +# always retained alongside a derived result rather than being replaced by a +# timestamp or import-order identity. + +AttributionStatus = Literal["matched", "pending", "ambiguous", "conflicting"] +DiagnosticSeverity = Literal["info", "warning", "error"] + + +class SourceReference(TypedDict): + """A role-labelled pointer back to immutable archived evidence. + + ``role`` is a stable consumer-facing label such as ``"user_prompt"``, + ``"assistant_message"``, ``"tool_execution"``, ``"usage_row"``, or + ``"finish"``. It describes evidence only; it never upgrades a heuristic + correlation to proof. + """ + + source_id: str + source_kind: str + role: str + + +class FinishEvidence(TypedDict): + """One completion observation supporting a semantic call candidate.""" + + source_id: str + kind: str + value: str | None + + +class DatabaseRevision(TypedDict): + """Database identity for one accounting snapshot. + + ``logical_call_id`` is derived from table, database generation, and row + primary key. ``content_revision`` selects the newest snapshot for that + logical call; a correction replaces its derived UsageRow instead of adding + another charge. + """ + + table: str + primary_key: str + generation: str + content_revision: str + + +class NormalizedEvent(TypedDict): + """One semantic event with stable identity and source provenance.""" + + id: str + kind: str + ts: str | None + source_ids: list[str] + source_references: list[SourceReference] + attributes: dict[str, Any] + + +class CallCandidate(TypedDict): + """A possible assistant-call span reconstructed from transcript evidence. + + IDs are native/source-derived (for example a transcript assistant-message + event ID), never a bare ``turnId``, chronological parent ID, tool name, or + list position. Nullable fields remain null when the archive does not + establish them. + """ + + call_id: str + stored_turn_id: str | None + interaction_id: str | None + agent_id: str | None + parent_tool_call_id: str | None + model: str | None + source_ids: list[str] + source_references: list[SourceReference] + start_ts: str | None + end_ts: str | None + tool_call_ids: list[str] + finish_evidence: list[FinishEvidence] + + +class AccountingCandidate(TypedDict): + """A database call before it is attributed to an assistant message. + + ``turn_index`` is the native user-turn index and is not a transcript + ``turnId``. ``provider`` is null when the database has no provider; + normalization must preserve that uncertainty (a UsageRow may use the + explicit ``"unknown"`` provider sentinel required by its existing shape). + Missing usage produces no UsageRow rather than a zero-valued one. + """ + + usage_source_id: str + logical_call_id: str + turn_index: int | None + agent_id: str | None + parent_tool_call_id: str | None + model: str | None + provider: str | None + source_ids: list[str] + source_references: list[SourceReference] + revision: DatabaseRevision + timestamp: str | None + finish_reason: str | None + supplemental_metrics: dict[str, Any] + + +class PendingItem(TypedDict): + """An explicit capability gap or unresolved relationship.""" + + id: str + kind: str + reason: str + source_ids: list[str] + evidence: list[str] + + +class ProjectionDiagnostic(TypedDict): + """Content-free derived-state diagnostic safe for status output.""" + + code: str + severity: DiagnosticSeverity + message: str + source_ids: list[str] + details: dict[str, Any] + + +class Attribution(TypedDict): + """The durable result of joining one accounting record to semantics.""" + + usage_source_id: str + stored_turn_id: str | None + agent_id: str | None + call_id: str | None + status: AttributionStatus + evidence: list[str] + + +class SemanticProjection(TypedDict): + """Pure transcript/hook reconstruction; it does not account for tokens.""" + + events: list[dict[str, Any]] + turns: list[TurnSpanDict] + call_candidates: list[CallCandidate] + pending: list[dict[str, Any]] + diagnostics: list[dict[str, Any]] + + +class AccountingProjection(TypedDict): + """Pure database normalization; it does not claim a message join.""" + + usage_rows: list[UsageRow] + candidates: list[AccountingCandidate] + diagnostics: list[dict[str, Any]] + + +class Projection(TypedDict): + """Combined local-only V2 projection, serializable via UsageRow.to_dict.""" + + normalized_events: list[dict[str, Any]] + turns: list[TurnSpanDict] + usage_rows: list[UsageRow] + attributions: list[Attribution] + pending: list[dict[str, Any]] + diagnostics: list[dict[str, Any]] + + +class ProjectedTurnRecord(TypedDict): + """The existing ``session_turns`` view shape for completed main turns only. + + Child-agent evidence remains nested in the main turn's trace/events and is + never emitted here as an independent human interaction. + """ + + id: str + turn_id: str + session_id: str + platform: str + cwd: str + start_seq: int | None + end_seq: int | None + start_ts: str | None + end_ts: str | None + events: list[dict[str, Any]] diff --git a/src/thirdeye/tracing/__init__.py b/src/thirdeye/tracing/__init__.py index 7bd9309..6c1e3fe 100644 --- a/src/thirdeye/tracing/__init__.py +++ b/src/thirdeye/tracing/__init__.py @@ -1,6 +1,7 @@ from __future__ import annotations from thirdeye.tracing.model import ( + AccountingCallSpanDict, LlmCallSpanDict, PermissionRequestSpanDict, ToolCallSpanDict, @@ -10,6 +11,7 @@ ) __all__ = [ + "AccountingCallSpanDict", "LlmCallSpanDict", "PermissionRequestSpanDict", "ToolCallSpanDict", diff --git a/src/thirdeye/tracing/model.py b/src/thirdeye/tracing/model.py index 7deef0e..c8aad1e 100644 --- a/src/thirdeye/tracing/model.py +++ b/src/thirdeye/tracing/model.py @@ -82,6 +82,22 @@ class InteractionSpanDict(TypedDict): attributes: dict[str, Any] +class AccountingCallSpanDict(TypedDict): + """Generic accounting attached to a turn without importing platform types. + + ``usage`` is exactly :meth:`UsageRow.to_dict` output. A producer creates + an explicit session-accounting export job when no user turn owns it rather + than inventing a turn. ``accounting_id`` is stable across source-row + corrections and export retries. + """ + + accounting_id: str + usage: dict[str, Any] + attribution_status: str + agent_id: str | None + attributes: dict[str, Any] + + TurnStatus = Literal["completed", "interrupted", "errored"] @@ -129,3 +145,6 @@ class TurnSpanDict(TypedDict): attributes: dict[str, Any] # Optional: Cursor interactions exported as spans. interactions: NotRequired[list[InteractionSpanDict]] + # Optional local accounting that may be exported on the owning chat span + # or an explicit accounting span, never both. + accounting_calls: NotRequired[list[AccountingCallSpanDict]] diff --git a/tests/platforms/copilot/fixtures/reconciliation-cases/README.md b/tests/platforms/copilot/fixtures/reconciliation-cases/README.md new file mode 100644 index 0000000..9101bfe --- /dev/null +++ b/tests/platforms/copilot/fixtures/reconciliation-cases/README.md @@ -0,0 +1,41 @@ +# Copilot V2 reconciliation contracts + +These JSON documents are static DTO examples for projection consumers. They +are not generated archives and are not executable fixtures. Production tests +must create archives through the V1 capture APIs, using the observed sibling +corpus or focused `SourceRecord` inputs. + +`semantic-projection.json` is a pure transcript/hook input and its expected +semantic output. `accounting-projection.json` is a pure SQLite input and its +expected accounting output. `storage.json` gives the persisted projection +state and the public main-turn read shape. `transport.json` gives the generic, +platform-independent accounting-span and durable export-job shapes. + +`observed-six-calls.json` maps all six observed `assistant_usage_events` rows +from `../assistant-usage-events.json` to source-derived logical identities; +the original data remains the authoritative token/billing corpus. `cases.json` +labels the required synthetic edge cases. A case records a required outcome, +not a timestamp heuristic: a close timestamp, tool name, parentId, bare +transcript `turnId`, or row order is never exact-join proof. + +Derived identities are strings whose components are native/source identities: + +- semantic event: `copilot:event:`; +- main turn: `copilot:turn:::`; +- semantic call: `copilot:call:`; +- logical database call/accounting span: `copilot:usage::::`. + +The content revision is deliberately excluded from a logical database call +identity. A corrected row replaces its normalized `UsageRow`; a different +database generation makes a different identity, preventing row-ID reuse from +being relabelled. `UsageRow` values cross a disk/process boundary only through +their existing `to_dict` / `from_dict` serializers. Unknown providers are +stored as the explicit `unknown` UsageRow provider sentinel; absent usage is +absent, never a zero row. Nano-AI-unit billing stays in supplemental metrics, +not a guessed USD price. + +Attribution statuses are exactly `matched`, `pending`, `ambiguous`, and +`conflicting`. An inferred `matched` join lists all supporting evidence. A +fallback accounting export has a deterministic accounting span ID and is +ledgered, so a later local match cannot export the same tokens again on a chat +span. diff --git a/tests/platforms/copilot/fixtures/reconciliation-cases/accounting-projection.json b/tests/platforms/copilot/fixtures/reconciliation-cases/accounting-projection.json new file mode 100644 index 0000000..ff894ee --- /dev/null +++ b/tests/platforms/copilot/fixtures/reconciliation-cases/accounting-projection.json @@ -0,0 +1,18 @@ +{ + "input_records": [ + { + "source_id": "copilot:database:source-a:session-a:assistant_usage_events:db-a:13:sha256:usage-13", + "source_kind": "database", + "native_session_id": "session-a", + "ts": "2026-09-10T17:08:24.498Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": {"table": "assistant_usage_events", "row": {"id": 13, "turn_index": 0, "agent_id": null, "parent_tool_call_id": null, "model": "gpt-5.6-luna", "input_tokens": 6452, "output_tokens": 107, "cache_read_tokens": 0, "cache_write_tokens": 6449, "reasoning_tokens": 29, "total_nano_aiu": 174125000, "finish_reason": "tool_calls"}}, + "locator": {"table": "assistant_usage_events", "primary_key": 13, "generation": "db-a", "content_revision": "sha256:usage-13"} + } + ], + "expected": { + "usage_rows": [{"session_id": "copilot-source-a-session-a", "seq": 13, "call_id": "copilot:usage:source-a:assistant_usage_events:db-a:13", "ts": "2026-09-10T17:08:24.498Z", "platform": "copilot", "gen_ai.conversation.id": "copilot-source-a-session-a", "gen_ai.provider.name": "unknown", "gen_ai.operation.name": "chat", "gen_ai.response.model": "gpt-5.6-luna", "gen_ai.usage.input_tokens": 6452, "gen_ai.usage.output_tokens": 107, "gen_ai.usage.cache_read.input_tokens": 0, "gen_ai.usage.cache_creation.input_tokens": 6449, "gen_ai.usage.reasoning.output_tokens": 29}], + "candidates": [{"usage_source_id": "copilot:database:source-a:session-a:assistant_usage_events:db-a:13:sha256:usage-13", "logical_call_id": "copilot:usage:source-a:assistant_usage_events:db-a:13", "turn_index": 0, "agent_id": null, "parent_tool_call_id": null, "model": "gpt-5.6-luna", "provider": null, "source_ids": ["copilot:database:source-a:session-a:assistant_usage_events:db-a:13:sha256:usage-13"], "source_references": [{"source_id": "copilot:database:source-a:session-a:assistant_usage_events:db-a:13:sha256:usage-13", "source_kind": "database", "role": "usage_row"}], "revision": {"table": "assistant_usage_events", "primary_key": "13", "generation": "db-a", "content_revision": "sha256:usage-13"}, "timestamp": "2026-09-10T17:08:24.498Z", "finish_reason": "tool_calls", "supplemental_metrics": {"total_nano_aiu": 174125000, "duration_ms": null}}], + "diagnostics": [] + } +} diff --git a/tests/platforms/copilot/fixtures/reconciliation-cases/cases.json b/tests/platforms/copilot/fixtures/reconciliation-cases/cases.json new file mode 100644 index 0000000..9fccd8c --- /dev/null +++ b/tests/platforms/copilot/fixtures/reconciliation-cases/cases.json @@ -0,0 +1,12 @@ +{ + "permission": {"input": "permission request/decision evidence", "required": "emit permission semantic event; do not infer a tool execution or usage call"}, + "compaction": {"input": "checkpoint or compaction snapshot", "required": "retain source evidence and diagnostics; never add snapshot usage to per-call accounting"}, + "abort": {"input": "unfinished interaction", "required": "keep it pending and retain open state; missing usage is absent, not zero"}, + "retry": {"input": "replayed source record", "required": "stable source-derived IDs make projection replay equivalent without duplicate accounting"}, + "nested_child": {"input": "child prompt/stop hooks with parent session ID", "required": "child evidence stays within its parent main interaction; session-ID-only ownership is not inferred"}, + "late_row": {"input": "new assistant_usage_events row after transcript completion", "required": "create/revise accounting locally and leave attribution pending unless evidence is uniquely consistent"}, + "revision": {"input": "same database generation/table/row with new content revision", "required": "replace the existing logical UsageRow or quarantine conflict; do not add another charge"}, + "identical_concurrent_tools": {"input": "same tool name and arguments in parallel", "required": "use transcript toolCallId/source identity; hooks and nearest timestamps cannot prove pairing"}, + "ambiguous": {"input": "two candidate calls fit one usage row", "required": "Attribution status ambiguous with both candidate IDs in evidence; choose neither"}, + "conflicting": {"input": "two revisions assert incompatible metrics for one logical call", "required": "Attribution status conflicting and quarantine diagnostic; do not overwrite a delivered accounting identity"} +} diff --git a/tests/platforms/copilot/fixtures/reconciliation-cases/observed-six-calls.json b/tests/platforms/copilot/fixtures/reconciliation-cases/observed-six-calls.json new file mode 100644 index 0000000..7bbc42e --- /dev/null +++ b/tests/platforms/copilot/fixtures/reconciliation-cases/observed-six-calls.json @@ -0,0 +1,15 @@ +{ + "source_fixture": "../assistant-usage-events.json", + "source_key": "observed-source-key", + "database_generation": "observed-db-generation", + "table": "assistant_usage_events", + "calls": [ + {"row_id": 13, "logical_call_id": "copilot:usage:observed-source-key:assistant_usage_events:observed-db-generation:13", "turn_index": 0, "agent_id": null, "parent_tool_call_id": null, "finish_reason": "tool_calls"}, + {"row_id": 14, "logical_call_id": "copilot:usage:observed-source-key:assistant_usage_events:observed-db-generation:14", "turn_index": 0, "agent_id": null, "parent_tool_call_id": null, "finish_reason": "stop"}, + {"row_id": 15, "logical_call_id": "copilot:usage:observed-source-key:assistant_usage_events:observed-db-generation:15", "turn_index": 1, "agent_id": null, "parent_tool_call_id": null, "finish_reason": "tool_calls"}, + {"row_id": 16, "logical_call_id": "copilot:usage:observed-source-key:assistant_usage_events:observed-db-generation:16", "turn_index": 1, "agent_id": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "parent_tool_call_id": "call_qx4FH5DADTeT1qVLb37HNpBk", "finish_reason": "tool_calls"}, + {"row_id": 17, "logical_call_id": "copilot:usage:observed-source-key:assistant_usage_events:observed-db-generation:17", "turn_index": 1, "agent_id": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "parent_tool_call_id": "call_qx4FH5DADTeT1qVLb37HNpBk", "finish_reason": "stop"}, + {"row_id": 18, "logical_call_id": "copilot:usage:observed-source-key:assistant_usage_events:observed-db-generation:18", "turn_index": 1, "agent_id": null, "parent_tool_call_id": null, "finish_reason": "stop"} + ], + "invariants": {"call_count": 6, "main_agent_call_count": 4, "child_agent_call_count": 2, "shutdown_usage_is_validation_only": true, "checkpoint_usage_is_additive": false, "auxiliary_model_calls_are_main_usage": false} +} diff --git a/tests/platforms/copilot/fixtures/reconciliation-cases/semantic-projection.json b/tests/platforms/copilot/fixtures/reconciliation-cases/semantic-projection.json new file mode 100644 index 0000000..0e6eefc --- /dev/null +++ b/tests/platforms/copilot/fixtures/reconciliation-cases/semantic-projection.json @@ -0,0 +1,42 @@ +{ + "input_records": [ + { + "source_id": "copilot:transcript:source-a:session-a:event-user-1", + "source_kind": "transcript", + "native_session_id": "session-a", + "ts": "2026-09-10T17:08:20.000Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": {"event": "user.message", "data": {"interactionId": "interaction-1", "agentId": null}}, + "locator": {"event_id": "event-user-1", "generation": "file-a", "offset": 0} + }, + { + "source_id": "copilot:transcript:source-a:session-a:event-assistant-1", + "source_kind": "transcript", + "native_session_id": "session-a", + "ts": "2026-09-10T17:08:24.000Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": {"event": "assistant.message", "data": {"interactionId": "interaction-1", "agentId": null, "model": "gpt-5.6-luna", "toolCallIds": ["tool-read-a", "tool-read-b"]}}, + "locator": {"event_id": "event-assistant-1", "generation": "file-a", "offset": 101} + }, + { + "source_id": "copilot:transcript:source-a:session-a:event-finish-1", + "source_kind": "transcript", + "native_session_id": "session-a", + "ts": "2026-09-10T17:08:26.000Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": {"event": "agent.stop", "data": {"interactionId": "interaction-1", "agentId": null, "reason": "tool_calls"}}, + "locator": {"event_id": "event-finish-1", "generation": "file-a", "offset": 202} + } + ], + "expected": { + "events": [ + {"id": "copilot:event:copilot:transcript:source-a:session-a:event-user-1", "kind": "user_prompt", "ts": "2026-09-10T17:08:20.000Z", "source_ids": ["copilot:transcript:source-a:session-a:event-user-1"], "source_references": [{"source_id": "copilot:transcript:source-a:session-a:event-user-1", "source_kind": "transcript", "role": "user_prompt"}], "attributes": {"interaction_id": "interaction-1", "agent_id": null}}, + {"id": "copilot:event:copilot:transcript:source-a:session-a:event-assistant-1", "kind": "assistant_message", "ts": "2026-09-10T17:08:24.000Z", "source_ids": ["copilot:transcript:source-a:session-a:event-assistant-1"], "source_references": [{"source_id": "copilot:transcript:source-a:session-a:event-assistant-1", "source_kind": "transcript", "role": "assistant_message"}], "attributes": {"interaction_id": "interaction-1", "agent_id": null}} + ], + "call_candidates": [ + {"call_id": "copilot:call:copilot:transcript:source-a:session-a:event-assistant-1", "stored_turn_id": "copilot:turn:source-a:session-a:interaction-1", "interaction_id": "interaction-1", "agent_id": null, "parent_tool_call_id": null, "model": "gpt-5.6-luna", "source_ids": ["copilot:transcript:source-a:session-a:event-assistant-1", "copilot:transcript:source-a:session-a:event-finish-1"], "source_references": [{"source_id": "copilot:transcript:source-a:session-a:event-assistant-1", "source_kind": "transcript", "role": "assistant_message"}, {"source_id": "copilot:transcript:source-a:session-a:event-finish-1", "source_kind": "transcript", "role": "finish"}], "start_ts": "2026-09-10T17:08:24.000Z", "end_ts": "2026-09-10T17:08:26.000Z", "tool_call_ids": ["tool-read-a", "tool-read-b"], "finish_evidence": [{"source_id": "copilot:transcript:source-a:session-a:event-finish-1", "kind": "agent_stop", "value": "tool_calls"}]} + ], + "pending": [], + "diagnostics": [] + } +} diff --git a/tests/platforms/copilot/fixtures/reconciliation-cases/storage.json b/tests/platforms/copilot/fixtures/reconciliation-cases/storage.json new file mode 100644 index 0000000..e4ab336 --- /dev/null +++ b/tests/platforms/copilot/fixtures/reconciliation-cases/storage.json @@ -0,0 +1,27 @@ +{ + "projection_state": { + "schema_version": 2, + "archive_source_ids": ["copilot:transcript:source-a:session-a:event-user-1", "copilot:database:source-a:session-a:assistant_usage_events:db-a:13:sha256:usage-13"], + "semantic_state": {"open_interactions": {}}, + "accounting_state": {"logical_calls": {"copilot:usage:source-a:assistant_usage_events:db-a:13": "sha256:usage-13"}}, + "projection_revision": "sha256:derived-state-example" + }, + "commit_result": {"events": 2, "turns": 1, "usage": 1, "attributions": 1, "pending": 0, "diagnostics": 0}, + "read_projected_turns": [ + { + "id": "copilot-source-a-session-a:copilot:turn:source-a:session-a:interaction-1", + "turn_id": "copilot:turn:source-a:session-a:interaction-1", + "session_id": "copilot-source-a-session-a", + "platform": "copilot", + "cwd": "/workspace/example", + "start_seq": 12, + "end_seq": 24, + "start_ts": "2026-09-10T17:08:20.000Z", + "end_ts": "2026-09-10T17:08:26.000Z", + "events": [ + {"id": "copilot:event:copilot:transcript:source-a:session-a:event-user-1", "kind": "user_prompt", "source_ids": ["copilot:transcript:source-a:session-a:event-user-1"]}, + {"id": "copilot:event:copilot:transcript:source-a:session-a:event-assistant-1", "kind": "assistant_message", "source_ids": ["copilot:transcript:source-a:session-a:event-assistant-1"]} + ] + } + ] +} diff --git a/tests/platforms/copilot/fixtures/reconciliation-cases/transport.json b/tests/platforms/copilot/fixtures/reconciliation-cases/transport.json new file mode 100644 index 0000000..27fe33a --- /dev/null +++ b/tests/platforms/copilot/fixtures/reconciliation-cases/transport.json @@ -0,0 +1,23 @@ +{ + "turn_accounting_calls": [ + { + "accounting_id": "copilot:usage:source-a:assistant_usage_events:db-a:13", + "usage": {"session_id": "copilot-source-a-session-a", "seq": 13, "call_id": "copilot:usage:source-a:assistant_usage_events:db-a:13", "ts": "2026-09-10T17:08:24.498Z", "platform": "copilot", "gen_ai.conversation.id": "copilot-source-a-session-a", "gen_ai.provider.name": "unknown", "gen_ai.operation.name": "chat", "gen_ai.response.model": "gpt-5.6-luna", "gen_ai.usage.input_tokens": 6452, "gen_ai.usage.output_tokens": 107}, + "attribution_status": "matched", + "agent_id": null, + "attributes": {"copilot.logical_call_id": "copilot:usage:source-a:assistant_usage_events:db-a:13", "copilot.evidence": ["interaction_id:interaction-1", "model:gpt-5.6-luna", "finish:tool_calls"]} + } + ], + "session_accounting_job": { + "job_id": "accounting:copilot-source-a-session-a:copilot:usage:source-a:assistant_usage_events:db-a:14", + "kind": "session_accounting", + "session_id": "copilot-source-a-session-a", + "accounting_id": "copilot:usage:source-a:assistant_usage_events:db-a:14", + "destination": "session-accounting-span", + "attempt": 0, + "state": "queued", + "usage": {"session_id": "copilot-source-a-session-a", "seq": 14, "call_id": "copilot:usage:source-a:assistant_usage_events:db-a:14", "ts": "2026-09-10T17:08:25.618Z", "platform": "copilot", "gen_ai.conversation.id": "copilot-source-a-session-a", "gen_ai.provider.name": "unknown", "gen_ai.operation.name": "chat", "gen_ai.response.model": "gpt-5.6-luna", "gen_ai.usage.input_tokens": 6587, "gen_ai.usage.output_tokens": 5}, + "attribution_status": "pending" + }, + "ledger_entry": {"accounting_id": "copilot:usage:source-a:assistant_usage_events:db-a:14", "destination": "session-accounting-span", "emitted": false, "last_error": null} +} From b88c9d11164b4b0e26c4f999976f4ec5c3f72799 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Fri, 11 Sep 2026 15:21:06 -0700 Subject: [PATCH 46/88] Align Copilot projection contracts with V1 SourceRecords. Rebuild fixture DTOs from captured transcript and usage shapes, and close the gaps reviewers flagged around identities, attribution, and export destinations. Co-authored-by: Cursor --- src/thirdeye/platforms/copilot/types.py | 333 +++- src/thirdeye/tracing/__init__.py | 8 + src/thirdeye/tracing/model.py | 79 +- .../fixtures/reconciliation-cases/README.md | 136 +- .../accounting-projection.json | 105 +- .../reconciliation-cases/attributions.json | 67 + .../fixtures/reconciliation-cases/cases.json | 1337 ++++++++++++++++- .../reconciliation-cases/diagnostics.json | 62 + .../reconciliation-cases/identities.json | 26 + .../observed-six-calls.json | 242 ++- .../semantic-projection.json | 272 +++- .../reconciliation-cases/storage.json | 163 +- .../reconciliation-cases/transport.json | 181 ++- 13 files changed, 2861 insertions(+), 150 deletions(-) create mode 100644 tests/platforms/copilot/fixtures/reconciliation-cases/attributions.json create mode 100644 tests/platforms/copilot/fixtures/reconciliation-cases/diagnostics.json create mode 100644 tests/platforms/copilot/fixtures/reconciliation-cases/identities.json diff --git a/src/thirdeye/platforms/copilot/types.py b/src/thirdeye/platforms/copilot/types.py index 6939ec1..c599c5c 100644 --- a/src/thirdeye/platforms/copilot/types.py +++ b/src/thirdeye/platforms/copilot/types.py @@ -1,12 +1,16 @@ -"""Versioned, lossless source contracts shared by Copilot V1 and V2. - -These TypedDicts deliberately describe raw source evidence. They do not imply -turn reconstruction, usage accounting, or any semantic correlation. +"""Versioned source envelopes (V1) and derived projection contracts (V2). + +V1 TypedDicts describe immutable archived evidence. They remain the input to +every V2 algorithm. V2 TypedDicts describe replaceable derived state: +normalized events, turn reconstruction, independent database accounting, and +attribution. A source ID, database generation, and content revision are +always retained alongside a derived result rather than being replaced by a +timestamp or import-order identity. """ from __future__ import annotations -from typing import Any, Literal, TypedDict +from typing import Any, Literal, NotRequired, TypedDict from thirdeye.tracing.model import TurnSpanDict from thirdeye.usage.types import UsageRow @@ -17,6 +21,10 @@ # envelope. It is deliberately separate from a Copilot CLI version. SCHEMA_VERSION = SOURCE_SCHEMA_VERSION +# Derived projection state is versioned independently of the V1 envelope. +# Rebuilds may drop and recreate this document without rewriting raw events. +PROJECTION_SCHEMA_VERSION = 2 + class SourcePaths(TypedDict): """Canonical source-home identity and its Copilot recording locations.""" @@ -72,45 +80,178 @@ class SourceSlice(TypedDict): # V2 projection contracts --------------------------------------------------- # -# These are deliberately additive to the V1 source envelopes above. Raw -# source records are immutable input; projections are replaceable derived -# state. A source ID, database generation, and content revision are therefore -# always retained alongside a derived result rather than being replaced by a -# timestamp or import-order identity. +# Stable derived identities use native/source identities, never import time or +# mutable list indices. Encoding rules (see also the reconciliation README): +# +# Transcript source IDs are V1 ``{full_source_key}/{native_id}/{event_id}``. +# Database source IDs are V1 +# ``copilot-db:{full_source_key}:{native_id}:{table}:{quoted_canonical_pk}:{revision}`` +# and do not embed generation. Hook source IDs are V1 +# ``hook/{native_id}/{observation_id}``. +# +# ```` in every derived ID is the full 64-character SHA-256 source +# home digest, not the 16-character prefix used only in stored session IDs. +# +# Semantic event: ``copilot:event:`` for the primary event of that +# record. When one source record yields extra events, append a native suffix: +# ``copilot:event::tool:`` for each +# ``data.toolRequests[].toolCallId`` on ``assistant.message``. A tool +# start/complete pair already has two source IDs, so each uses the unsuffixed +# form. If no native suffix exists, append ``:``. +# +# Semantic call: ``copilot:call:``. +# Main turn: ``copilot:turn:::``. +# +# Logical database call / accounting span: +# ``copilot:usage::
::``. +# ``quoted-generation`` and ``quoted-canonical-pk`` are +# ``urllib.parse.quote(..., safe="")`` of the locator generation and of +# ``json.dumps(locator.primary_key, sort_keys=True, separators=(",", ":"))`` +# respectively, matching V1 percent-quoting of composite keys. Generation is +# ``sha256:`` and therefore contains a colon; quoting turns that colon +# into ``%3A``. Content revision is excluded. Because V1 source IDs omit +# generation, an identical row seen in a new database file deduplicates to the +# first archived record; the logical ID uses that first record's locator +# generation. A different generation with different content is a new logical +# call (row-ID reuse), never a relabel of the previous call. +# +# ``Attribution.usage_source_id`` is the newest revision's source ID. +# Durable attribution is keyed by ``logical_call_id``. AttributionStatus = Literal["matched", "pending", "ambiguous", "conflicting"] +AttributionJoinKind = Literal["direct", "inferred"] DiagnosticSeverity = Literal["info", "warning", "error"] +EventClassification = Literal["main", "title_generation", "checkpoint", "shutdown_validation"] +Initiator = Literal["user", "agent", "sub-agent"] + +NormalizedEventKind = Literal[ + "user_prompt", + "assistant_message", + "tool_request", + "tool_execution_start", + "tool_execution_complete", + "tool_execution_failure", + "permission_request", + "permission_decision", + "notification", + "prompt_transformation", + "compaction", + "abort", + "error", + "session_start", + "session_end", + "session_shutdown", + "subagent_started", + "subagent_completed", + "auxiliary_model_call", + "unknown", +] + +SourceReferenceRole = Literal[ + "user_prompt", + "assistant_message", + "tool_request", + "tool_execution", + "tool_result", + "permission_request", + "permission_decision", + "usage_row", + "finish", + "hook", + "shutdown", + "checkpoint", + "compaction", + "auxiliary_model", + "nested_child", +] + +FinishEvidenceKind = Literal[ + "assistant_turn_end", + "agent_stop_hook", + "finish_reason_db", + "abort", + "error", +] + +PendingItemKind = Literal[ + "open_interaction", + "missing_usage", + "unmatched_usage", + "missing_source_capability", + "missing_identity", + "incomplete_tool_pair", + "delayed_row", +] + +DiagnosticCode = Literal[ + "usage_revision_conflict", + "usage_row_id_reuse", + "shutdown_total_mismatch", + "checkpoint_not_additive", + "auxiliary_excluded_from_main", + "capability_gap", + "inferred_join", + "unknown_provider", + "missing_usage_fields", + "open_interaction", +] + +# Attribution.evidence strings are ``key:value`` with these keys. ``call_id`` +# may repeat when status is ambiguous. Inferred joins always include +# ``join_kind:inferred``; a future native assistant-message/provider-call id +# uses ``join_kind:direct``. +AttributionEvidenceKey = Literal[ + "join_kind", + "interaction_id", + "agent_id", + "model", + "turn_index", + "parent_tool_call_id", + "finish", + "order", + "call_id", + "logical_call_id", + "usage_source_id", + "revision", + "initiator", + "tool_call_id", +] class SourceReference(TypedDict): """A role-labelled pointer back to immutable archived evidence. - ``role`` is a stable consumer-facing label such as ``"user_prompt"``, - ``"assistant_message"``, ``"tool_execution"``, ``"usage_row"``, or - ``"finish"``. It describes evidence only; it never upgrades a heuristic - correlation to proof. + ``role`` describes evidence only; it never upgrades a heuristic correlation + to proof. """ source_id: str source_kind: str - role: str + role: SourceReferenceRole class FinishEvidence(TypedDict): - """One completion observation supporting a semantic call candidate.""" + """One completion observation supporting a semantic call candidate. + + Observed Copilot transcripts finish a model cycle with + ``assistant.turn_end``, which carries only ``turnId``. There is no + ``agent.stop`` transcript event and no finish reason on that record. + """ source_id: str - kind: str + kind: FinishEvidenceKind value: str | None class DatabaseRevision(TypedDict): """Database identity for one accounting snapshot. - ``logical_call_id`` is derived from table, database generation, and row - primary key. ``content_revision`` selects the newest snapshot for that - logical call; a correction replaces its derived UsageRow instead of adding - another charge. + ``primary_key`` is the percent-quoted canonical JSON of the V1 locator's + ``primary_key`` value (an int for observed ``assistant_usage_events`` rows, + an object for composite keys). ``generation`` is the unquoted locator + generation (``sha256:``). ``logical_call_id`` quotes both. + ``content_revision`` selects the newest snapshot for that logical call; a + correction replaces its derived UsageRow instead of adding another charge. """ table: str @@ -120,10 +261,17 @@ class DatabaseRevision(TypedDict): class NormalizedEvent(TypedDict): - """One semantic event with stable identity and source provenance.""" + """One semantic event with stable identity and source provenance. + + Auxiliary ``model.*`` title-generation records use + ``kind="auxiliary_model_call"`` and ``classification="title_generation"``. + They must not appear as main ``call_candidates`` and must not inflate + conversation token totals. + """ id: str - kind: str + kind: NormalizedEventKind + classification: EventClassification ts: str | None source_ids: list[str] source_references: list[SourceReference] @@ -136,7 +284,8 @@ class CallCandidate(TypedDict): IDs are native/source-derived (for example a transcript assistant-message event ID), never a bare ``turnId``, chronological parent ID, tool name, or list position. Nullable fields remain null when the archive does not - establish them. + establish them. ``tool_call_ids`` come from + ``data.toolRequests[].toolCallId``, not from a ``toolCallIds`` array. """ call_id: str @@ -153,6 +302,32 @@ class CallCandidate(TypedDict): finish_evidence: list[FinishEvidence] +class SupplementalMetrics(TypedDict, total=False): + """Copilot-only row fields that are not UsageRow columns. + + Keys are omitted when the source value is missing or JSON-null. Do not + store nulls. Nano-AI-unit billing stays here and is never converted into + estimated model-price USD. + """ + + total_nano_aiu: int + request_multiplier: float + duration_ms: int + time_to_first_token_ms: float + output_ttft_ms: float + inter_token_latency_ms: float + initiator: Initiator + api_endpoint: str + reasoning_effort: str + content_filter_triggered: int + token_details_json: str + input_tokens: int + output_tokens: int + cache_read_tokens: int + cache_write_tokens: int + reasoning_tokens: int + + class AccountingCandidate(TypedDict): """A database call before it is attributed to an assistant message. @@ -161,6 +336,15 @@ class AccountingCandidate(TypedDict): normalization must preserve that uncertainty (a UsageRow may use the explicit ``"unknown"`` provider sentinel required by its existing shape). Missing usage produces no UsageRow rather than a zero-valued one. + + A UsageRow is emitted only when timestamp, model, input_tokens, and + output_tokens are all present. A null timestamp or model keeps the + candidate and adds ``missing_usage_fields``; no row. Partial usage + (input present, output missing) likewise yields no row: present counts + go into ``supplemental_metrics`` only. In-memory ``UsageRow.seq`` is ``0`` + until persistence stamps the Store seq of the newest archived revision. + The SQLite primary key is never ``seq``, including when it happens to be + an integer. """ usage_source_id: str @@ -175,14 +359,14 @@ class AccountingCandidate(TypedDict): revision: DatabaseRevision timestamp: str | None finish_reason: str | None - supplemental_metrics: dict[str, Any] + supplemental_metrics: SupplementalMetrics class PendingItem(TypedDict): """An explicit capability gap or unresolved relationship.""" id: str - kind: str + kind: PendingItemKind reason: str source_ids: list[str] evidence: list[str] @@ -191,7 +375,7 @@ class PendingItem(TypedDict): class ProjectionDiagnostic(TypedDict): """Content-free derived-state diagnostic safe for status output.""" - code: str + code: DiagnosticCode severity: DiagnosticSeverity message: str source_ids: list[str] @@ -199,24 +383,37 @@ class ProjectionDiagnostic(TypedDict): class Attribution(TypedDict): - """The durable result of joining one accounting record to semantics.""" + """The durable result of joining one accounting record to semantics. + + ``usage_source_id`` is the newest revision's source ID. ``logical_call_id`` + is the durable key across content revisions. ``join_kind`` is ``inferred`` + or ``direct`` only when ``status`` is ``matched``; otherwise null. + + ``status="conflicting"`` is a join conflict: competing semantic assignments + or a join that would rebind a logical call after tokens were emitted. + Incompatible database revisions are ``ProjectionDiagnostic`` + ``usage_revision_conflict`` and quarantine the logical call; they do not + use ``AttributionStatus`` by themselves. + """ usage_source_id: str + logical_call_id: str stored_turn_id: str | None agent_id: str | None call_id: str | None status: AttributionStatus + join_kind: AttributionJoinKind | None evidence: list[str] class SemanticProjection(TypedDict): """Pure transcript/hook reconstruction; it does not account for tokens.""" - events: list[dict[str, Any]] + events: list[NormalizedEvent] turns: list[TurnSpanDict] call_candidates: list[CallCandidate] - pending: list[dict[str, Any]] - diagnostics: list[dict[str, Any]] + pending: list[PendingItem] + diagnostics: list[ProjectionDiagnostic] class AccountingProjection(TypedDict): @@ -224,25 +421,32 @@ class AccountingProjection(TypedDict): usage_rows: list[UsageRow] candidates: list[AccountingCandidate] - diagnostics: list[dict[str, Any]] + diagnostics: list[ProjectionDiagnostic] class Projection(TypedDict): """Combined local-only V2 projection, serializable via UsageRow.to_dict.""" - normalized_events: list[dict[str, Any]] + normalized_events: list[NormalizedEvent] turns: list[TurnSpanDict] usage_rows: list[UsageRow] attributions: list[Attribution] - pending: list[dict[str, Any]] - diagnostics: list[dict[str, Any]] + pending: list[PendingItem] + diagnostics: list[ProjectionDiagnostic] class ProjectedTurnRecord(TypedDict): """The existing ``session_turns`` view shape for completed main turns only. - Child-agent evidence remains nested in the main turn's trace/events and is - never emitted here as an independent human interaction. + ``events`` are Store events (``seq``, ``t``, ``ts``, ``data``), not + normalized semantic stubs. ``t`` is ``copilot_transcript``, + ``copilot_database``, ``copilot_hook``, or ``copilot_metadata``. ``data`` + is the V1 envelope ``{schema_version, source_record}``. ``start_seq`` / + ``end_seq`` are those events' seq values. ``filter_turns`` and + ``logfire_dataset._turn_case`` consume this shape. + + Child-agent evidence remains nested in the main turn's events / trace and + is never emitted here as an independent human interaction. """ id: str @@ -255,3 +459,58 @@ class ProjectedTurnRecord(TypedDict): start_ts: str | None end_ts: str | None events: list[dict[str, Any]] + + +class OpenInteractionState(TypedDict): + """Unfinished interaction retained across incremental archive partitions.""" + + interaction_id: str + agent_id: str | None + stored_turn_id: str + source_ids: list[str] + last_event_source_id: str | None + start_ts: str | None + pending_tool_call_ids: list[str] + + +class LogicalCallState(TypedDict): + """Durable accounting identity for one logical database call. + + ``generation`` is the first archived record's locator generation. + ``metrics_digest`` is ``sha256:`` of canonical JSON over the row's + input/output/cache/reasoning/nano-AIU fields so a later revision can be + compared without treating row-ID reuse as the same call. + """ + + logical_call_id: str + generation: str + content_revision: str + metrics_digest: str + usage_source_id: str + + +class SemanticProjectionState(TypedDict): + """Incremental semantic replay state stored under projection state.""" + + open_interactions: dict[str, OpenInteractionState] + + +class AccountingProjectionState(TypedDict): + """Incremental accounting replay state stored under projection state.""" + + logical_calls: dict[str, LogicalCallState] + + +class ProjectionState(TypedDict): + """Persisted derived state. Separate from the V1 archive envelope. + + ``projection_schema_version`` is :data:`PROJECTION_SCHEMA_VERSION`. It is + not the V1 ``schema_version`` field on source envelopes. + """ + + projection_schema_version: int + archive_source_ids: list[str] + semantic_state: SemanticProjectionState + accounting_state: AccountingProjectionState + projection_revision: str + commit_result: NotRequired[dict[str, int]] diff --git a/src/thirdeye/tracing/__init__.py b/src/thirdeye/tracing/__init__.py index 6c1e3fe..f112f48 100644 --- a/src/thirdeye/tracing/__init__.py +++ b/src/thirdeye/tracing/__init__.py @@ -2,9 +2,13 @@ from thirdeye.tracing.model import ( AccountingCallSpanDict, + AccountingDestination, + AccountingLedgerEntryDict, LlmCallSpanDict, PermissionRequestSpanDict, + SessionAccountingJobDict, ToolCallSpanDict, + TurnAccountingJobDict, TurnSpanDict, TurnStatus, UsageDict, @@ -12,9 +16,13 @@ __all__ = [ "AccountingCallSpanDict", + "AccountingDestination", + "AccountingLedgerEntryDict", "LlmCallSpanDict", "PermissionRequestSpanDict", + "SessionAccountingJobDict", "ToolCallSpanDict", + "TurnAccountingJobDict", "TurnSpanDict", "TurnStatus", "UsageDict", diff --git a/src/thirdeye/tracing/model.py b/src/thirdeye/tracing/model.py index c8aad1e..e5f257f 100644 --- a/src/thirdeye/tracing/model.py +++ b/src/thirdeye/tracing/model.py @@ -35,7 +35,13 @@ class ToolCallSpanDict(TypedDict): class LlmCallSpanDict(TypedDict): - """One model call within a turn, plus the tool calls it requested.""" + """One model call within a turn, plus the tool calls it requested. + + When tokens are carried on :class:`AccountingCallSpanDict`, ``usage`` must + stay empty so the generic exporter cannot emit the same tokens twice. + Copilot producers always leave this empty and place actual usage on + ``TurnSpanDict.accounting_calls``. + """ call_id: str provider: str @@ -82,22 +88,82 @@ class InteractionSpanDict(TypedDict): attributes: dict[str, Any] +AccountingDestination = Literal["chat-span", "turn-accounting-span", "session-accounting-span"] + + class AccountingCallSpanDict(TypedDict): """Generic accounting attached to a turn without importing platform types. - ``usage`` is exactly :meth:`UsageRow.to_dict` output. A producer creates - an explicit session-accounting export job when no user turn owns it rather - than inventing a turn. ``accounting_id`` is stable across source-row - corrections and export retries. + ``usage`` is exactly :meth:`UsageRow.to_dict` output. ``accounting_id`` is + stable across source-row corrections and export retries. + + ``call_id`` is the matching :class:`LlmCallSpanDict` id when tokens export + on that chat span; null means a user-turn or agent accounting span. Set + and null are mutually exclusive export locations: never both. The parent + chat span's ``usage`` must stay empty whenever this record is present. + + Deterministic accounting span IDs (generic transport): + + - chat-span: existing chat span id for ``call_id``; no extra span + - turn-accounting-span: ``accounting:{session_id}:{turn_id}:{accounting_id}`` + - session-accounting-span: ``accounting:{session_id}:{accounting_id}`` """ accounting_id: str usage: dict[str, Any] attribution_status: str agent_id: str | None + call_id: str | None attributes: dict[str, Any] +class SessionAccountingJobDict(TypedDict): + """Durable export job when no user turn owns the accounting record.""" + + job_id: str + kind: Literal["session_accounting"] + session_id: str + accounting_id: str + destination: Literal["session-accounting-span"] + attempt: int + state: Literal["queued", "claimed", "emitted", "failed"] + usage: dict[str, Any] + attribution_status: str + span_id: str + + +class TurnAccountingJobDict(TypedDict): + """Durable export job for unmatched usage owned by a user turn or agent.""" + + job_id: str + kind: Literal["turn_accounting"] + session_id: str + turn_id: str + accounting_id: str + destination: Literal["turn-accounting-span"] + attempt: int + state: Literal["queued", "claimed", "emitted", "failed"] + usage: dict[str, Any] + attribution_status: str + agent_id: str | None + span_id: str + + +class AccountingLedgerEntryDict(TypedDict): + """Export-eligibility ledger row; lives in a file/lock apart from projection. + + ``emitted`` is remote-delivery success, not "configured" or "queued". + Once true for a destination, later local matching cannot emit the same + ``accounting_id`` on a different destination. + """ + + accounting_id: str + destination: AccountingDestination + span_id: str + emitted: bool + last_error: str | None + + TurnStatus = Literal["completed", "interrupted", "errored"] @@ -146,5 +212,6 @@ class TurnSpanDict(TypedDict): # Optional: Cursor interactions exported as spans. interactions: NotRequired[list[InteractionSpanDict]] # Optional local accounting that may be exported on the owning chat span - # or an explicit accounting span, never both. + # (AccountingCallSpanDict.call_id set) or an explicit user-turn/agent + # accounting span (call_id null), never both. accounting_calls: NotRequired[list[AccountingCallSpanDict]] diff --git a/tests/platforms/copilot/fixtures/reconciliation-cases/README.md b/tests/platforms/copilot/fixtures/reconciliation-cases/README.md index 9101bfe..a9ca254 100644 --- a/tests/platforms/copilot/fixtures/reconciliation-cases/README.md +++ b/tests/platforms/copilot/fixtures/reconciliation-cases/README.md @@ -1,41 +1,99 @@ # Copilot V2 reconciliation contracts -These JSON documents are static DTO examples for projection consumers. They -are not generated archives and are not executable fixtures. Production tests -must create archives through the V1 capture APIs, using the observed sibling -corpus or focused `SourceRecord` inputs. - -`semantic-projection.json` is a pure transcript/hook input and its expected -semantic output. `accounting-projection.json` is a pure SQLite input and its -expected accounting output. `storage.json` gives the persisted projection -state and the public main-turn read shape. `transport.json` gives the generic, -platform-independent accounting-span and durable export-job shapes. - -`observed-six-calls.json` maps all six observed `assistant_usage_events` rows -from `../assistant-usage-events.json` to source-derived logical identities; -the original data remains the authoritative token/billing corpus. `cases.json` -labels the required synthetic edge cases. A case records a required outcome, -not a timestamp heuristic: a close timestamp, tool name, parentId, bare -transcript `turnId`, or row order is never exact-join proof. - -Derived identities are strings whose components are native/source identities: - -- semantic event: `copilot:event:`; -- main turn: `copilot:turn:::`; -- semantic call: `copilot:call:`; -- logical database call/accounting span: `copilot:usage::
::`. - -The content revision is deliberately excluded from a logical database call -identity. A corrected row replaces its normalized `UsageRow`; a different -database generation makes a different identity, preventing row-ID reuse from -being relabelled. `UsageRow` values cross a disk/process boundary only through -their existing `to_dict` / `from_dict` serializers. Unknown providers are -stored as the explicit `unknown` UsageRow provider sentinel; absent usage is -absent, never a zero row. Nano-AI-unit billing stays in supplemental metrics, -not a guessed USD price. - -Attribution statuses are exactly `matched`, `pending`, `ambiguous`, and -`conflicting`. An inferred `matched` join lists all supporting evidence. A -fallback accounting export has a deterministic accounting span ID and is -ledgered, so a later local match cannot export the same tokens again on a chat -span. +These JSON documents are serialized DTO examples for projection consumers. +`input_records` arrays are V1 `SourceRecord` envelopes and may be passed +directly to `build_semantics` / `build_accounting`. They are not generated +archives: production tests that need a Store still create archives through +the V1 capture APIs from the observed sibling corpus or from these records. + +Placeholders for the live home digest and database file-stat generation are +documented in `identities.json`. Observed token totals remain authoritative +in `../assistant-usage-events.json` and `../usage.json`. + +| File | Role | +|---|---| +| `identities.json` | Shared source-key / generation / ID templates | +| `semantic-projection.json` | Transcript `SourceRecord` input and `SemanticProjection` output, including a `TurnSpanDict` | +| `accounting-projection.json` | Database `SourceRecord` input and `AccountingProjection` output | +| `storage.json` | `ProjectionState` plus `read_projected_turns` Store-event shape | +| `transport.json` | Generic `AccountingCallSpanDict`, turn-owned unmatched job, session job, ledger | +| `observed-six-calls.json` | Six observed rows, inferred call mapping, shutdown totals | +| `attributions.json` | Matched (inferred), pending, ambiguous, conflicting examples | +| `diagnostics.json` | `PendingItem` and `ProjectionDiagnostic` examples | +| `cases.json` | Concrete records and expected outputs for every required edge case | + +## V1 source-record shapes + +Transcript IDs are `{full_source_key}/{native_id}/{event_id}`. Payloads are +the raw JSONL object plus `schema_version`. Event type is `type` (not +`event`); `id`, `timestamp`, `parentId`, and `agentId` are top-level. +Tool IDs are `data.toolRequests[].toolCallId`. Model cycles end with +`assistant.turn_end` (`data.turnId` only). Locators use `file`, +`file_generation`, `byte_offset`, `byte_length`, and `native_event_id`. + +Database IDs are +`copilot-db:{full_source_key}:{native_id}:{table}:{quoted_canonical_pk}:{revision}` +with no generation. Locators include `database`. `ts` comes from the row's +`created_at`. V1 source IDs omit generation, so an identical row in a new +database file deduplicates; the logical call ID uses the first archived +record's locator generation. + +Hook IDs are `hook/{native_id}/{observation_id}`. + +## Derived identities + +`` is always the full 64-character SHA-256 digest, never the +16-character stored-session prefix. + +- semantic event: `copilot:event:` +- extra events from one record: `copilot:event::tool:` + (or `:` when no native suffix exists) +- semantic call: `copilot:call:` +- main turn: `copilot:turn:::` +- logical call / accounting span: + `copilot:usage::
::` + +`quoted-generation` and `quoted-canonical-pk` are `urllib.parse.quote(..., safe="")` +of the locator generation (`sha256:`, colon becomes `%3A`) and of +canonical JSON of `locator.primary_key`. Content revision is excluded from +the logical ID. `Attribution.usage_source_id` is the newest revision's source +ID; durable attribution is keyed by `logical_call_id`. + +## UsageRow gaps + +A `UsageRow` is emitted only when timestamp, model, input_tokens, and +output_tokens are all present. Otherwise keep the `AccountingCandidate` and +add `missing_usage_fields`. Present partial counts go in `supplemental_metrics`. +Omit JSON-null metric keys (absent, never null). In-memory `seq` is `0` until +persistence stamps the Store seq of the newest archived revision. The SQLite +primary key is never `seq`. Unknown provider is the UsageRow sentinel +`unknown`; candidate `provider` stays null when the database has none. +Nano-AI-unit billing stays in supplemental metrics. + +## Attribution + +Statuses are `matched`, `pending`, `ambiguous`, and `conflicting`. +`join_kind` is `inferred` or `direct` only on a match. Evidence strings are +`key:value` using `AttributionEvidenceKey`. Ambiguous listings include every +candidate `call_id:...` and choose none. + +`Attribution.status=conflicting` is a join conflict. Incompatible database +revisions are `ProjectionDiagnostic.code=usage_revision_conflict` and +quarantine the logical call. + +## Turns, storage, export + +`read_projected_turns` returns Store events (`seq` / `t` / `ts` / `data`) +for completed main interactions only. Child evidence stays inside them. + +`AccountingCallSpanDict.call_id` set means export on that chat span; null +means a user-turn or agent accounting span. `LlmCallSpanDict.usage` stays +empty whenever `accounting_calls` is present. Span IDs: + +- chat-span: existing chat span for `call_id` +- turn-accounting-span: `accounting:{session_id}:{turn_id}:{accounting_id}` +- session-accounting-span: `accounting:{session_id}:{accounting_id}` + +Queued or configured export is not successful delivery. Auxiliary +`model.*` title-generation records use `kind=auxiliary_model_call` and +`classification=title_generation` and cannot inflate the six-call total. diff --git a/tests/platforms/copilot/fixtures/reconciliation-cases/accounting-projection.json b/tests/platforms/copilot/fixtures/reconciliation-cases/accounting-projection.json index ff894ee..2de915c 100644 --- a/tests/platforms/copilot/fixtures/reconciliation-cases/accounting-projection.json +++ b/tests/platforms/copilot/fixtures/reconciliation-cases/accounting-projection.json @@ -1,18 +1,111 @@ { + "note": "input_records are V1 database SourceRecords. Pure-function tests may pass them directly to build_accounting.", "input_records": [ { - "source_id": "copilot:database:source-a:session-a:assistant_usage_events:db-a:13:sha256:usage-13", + "source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", "source_kind": "database", - "native_session_id": "session-a", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "ts": "2026-09-10T17:08:24.498Z", "observed_at": "2026-09-10T17:09:00.000Z", - "payload": {"table": "assistant_usage_events", "row": {"id": 13, "turn_index": 0, "agent_id": null, "parent_tool_call_id": null, "model": "gpt-5.6-luna", "input_tokens": 6452, "output_tokens": 107, "cache_read_tokens": 0, "cache_write_tokens": 6449, "reasoning_tokens": 29, "total_nano_aiu": 174125000, "finish_reason": "tool_calls"}}, - "locator": {"table": "assistant_usage_events", "primary_key": 13, "generation": "db-a", "content_revision": "sha256:usage-13"} + "payload": { + "table": "assistant_usage_events", + "row": { + "id": 13, + "session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "turn_index": 0, + "agent_id": null, + "parent_tool_call_id": null, + "model": "gpt-5.6-luna", + "input_tokens": 6452, + "output_tokens": 107, + "cache_read_tokens": 0, + "cache_write_tokens": 6449, + "reasoning_tokens": 29, + "total_nano_aiu": 174125000, + "request_multiplier": 1.0, + "duration_ms": 2263, + "time_to_first_token_ms": 1571.1477920000002, + "output_ttft_ms": 1571.1479590000001, + "inter_token_latency_ms": 7.393025117647059, + "initiator": "user", + "api_endpoint": "ws:/responses", + "reasoning_effort": "medium", + "finish_reason": "tool_calls", + "content_filter_triggered": 0, + "token_details_json": "[{\"batchSize\":1000000,\"costPerBatch\":20000000000,\"tokenCount\":3,\"tokenType\":\"input\"},{\"batchSize\":1000000,\"costPerBatch\":2000000000,\"tokenCount\":0,\"tokenType\":\"cache_read\"},{\"batchSize\":1000000,\"costPerBatch\":25000000000,\"tokenCount\":6449,\"tokenType\":\"cache_write\"},{\"batchSize\":1000000,\"costPerBatch\":120000000000,\"tokenCount\":107,\"tokenType\":\"output\"}]", + "created_at": "2026-09-10T17:08:24.498Z" + } + }, + "locator": { + "database": "/example/.copilot/session-store.db", + "table": "assistant_usage_events", + "primary_key": 13, + "content_revision": "sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "generation": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + } } ], "expected": { - "usage_rows": [{"session_id": "copilot-source-a-session-a", "seq": 13, "call_id": "copilot:usage:source-a:assistant_usage_events:db-a:13", "ts": "2026-09-10T17:08:24.498Z", "platform": "copilot", "gen_ai.conversation.id": "copilot-source-a-session-a", "gen_ai.provider.name": "unknown", "gen_ai.operation.name": "chat", "gen_ai.response.model": "gpt-5.6-luna", "gen_ai.usage.input_tokens": 6452, "gen_ai.usage.output_tokens": 107, "gen_ai.usage.cache_read.input_tokens": 0, "gen_ai.usage.cache_creation.input_tokens": 6449, "gen_ai.usage.reasoning.output_tokens": 29}], - "candidates": [{"usage_source_id": "copilot:database:source-a:session-a:assistant_usage_events:db-a:13:sha256:usage-13", "logical_call_id": "copilot:usage:source-a:assistant_usage_events:db-a:13", "turn_index": 0, "agent_id": null, "parent_tool_call_id": null, "model": "gpt-5.6-luna", "provider": null, "source_ids": ["copilot:database:source-a:session-a:assistant_usage_events:db-a:13:sha256:usage-13"], "source_references": [{"source_id": "copilot:database:source-a:session-a:assistant_usage_events:db-a:13:sha256:usage-13", "source_kind": "database", "role": "usage_row"}], "revision": {"table": "assistant_usage_events", "primary_key": "13", "generation": "db-a", "content_revision": "sha256:usage-13"}, "timestamp": "2026-09-10T17:08:24.498Z", "finish_reason": "tool_calls", "supplemental_metrics": {"total_nano_aiu": 174125000, "duration_ms": null}}], + "usage_rows": [ + { + "session_id": "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "seq": 0, + "call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13", + "ts": "2026-09-10T17:08:24.498Z", + "platform": "copilot", + "gen_ai.conversation.id": "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "gen_ai.provider.name": "unknown", + "gen_ai.operation.name": "chat", + "gen_ai.response.model": "gpt-5.6-luna", + "gen_ai.usage.input_tokens": 6452, + "gen_ai.usage.output_tokens": 107, + "gen_ai.usage.cache_read.input_tokens": 0, + "gen_ai.usage.cache_creation.input_tokens": 6449, + "gen_ai.usage.reasoning.output_tokens": 29 + } + ], + "candidates": [ + { + "usage_source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13", + "turn_index": 0, + "agent_id": null, + "parent_tool_call_id": null, + "model": "gpt-5.6-luna", + "provider": null, + "source_ids": [ + "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47" + ], + "source_references": [ + { + "source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "source_kind": "database", + "role": "usage_row" + } + ], + "revision": { + "table": "assistant_usage_events", + "primary_key": "13", + "generation": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "content_revision": "sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47" + }, + "timestamp": "2026-09-10T17:08:24.498Z", + "finish_reason": "tool_calls", + "supplemental_metrics": { + "total_nano_aiu": 174125000, + "request_multiplier": 1.0, + "initiator": "user", + "api_endpoint": "ws:/responses", + "reasoning_effort": "medium", + "content_filter_triggered": 0, + "token_details_json": "[{\"batchSize\":1000000,\"costPerBatch\":20000000000,\"tokenCount\":3,\"tokenType\":\"input\"},{\"batchSize\":1000000,\"costPerBatch\":2000000000,\"tokenCount\":0,\"tokenType\":\"cache_read\"},{\"batchSize\":1000000,\"costPerBatch\":25000000000,\"tokenCount\":6449,\"tokenType\":\"cache_write\"},{\"batchSize\":1000000,\"costPerBatch\":120000000000,\"tokenCount\":107,\"tokenType\":\"output\"}]", + "duration_ms": 2263, + "time_to_first_token_ms": 1571.1477920000002, + "output_ttft_ms": 1571.1479590000001, + "inter_token_latency_ms": 7.393025117647059 + } + } + ], "diagnostics": [] } } diff --git a/tests/platforms/copilot/fixtures/reconciliation-cases/attributions.json b/tests/platforms/copilot/fixtures/reconciliation-cases/attributions.json new file mode 100644 index 0000000..e07b66a --- /dev/null +++ b/tests/platforms/copilot/fixtures/reconciliation-cases/attributions.json @@ -0,0 +1,67 @@ +{ + "matched_inferred": { + "usage_source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13", + "stored_turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42", + "agent_id": null, + "call_id": "copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "status": "matched", + "join_kind": "inferred", + "evidence": [ + "join_kind:inferred", + "interaction_id:6d2b89fd-a653-430c-b532-b0936d72eb42", + "turn_index:0", + "agent_id:None", + "parent_tool_call_id:None", + "model:gpt-5.6-luna", + "finish:tool_calls", + "initiator:user", + "order:0", + "call_id:copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328" + ] + }, + "pending": { + "usage_source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:14:sha256:20cb0f98d6130c8d0e139391cfc8e245d1d0ac96d41469c4da60f26e6d9e0a26", + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:14", + "stored_turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42", + "agent_id": null, + "call_id": null, + "status": "pending", + "join_kind": null, + "evidence": [ + "logical_call_id:copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:14", + "turn_index:0", + "delayed_row:true" + ] + }, + "ambiguous": { + "usage_source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13", + "stored_turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42", + "agent_id": null, + "call_id": null, + "status": "ambiguous", + "join_kind": null, + "evidence": [ + "call_id:copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "call_id:copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/33cc6465-29e1-4a04-8bdb-00241474b4d2", + "interaction_id:6d2b89fd-a653-430c-b532-b0936d72eb42", + "model:gpt-5.6-luna", + "turn_index:0" + ] + }, + "conflicting_join": { + "usage_source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13", + "stored_turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42", + "agent_id": null, + "call_id": null, + "status": "conflicting", + "join_kind": null, + "evidence": [ + "call_id:copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "call_id:copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/33cc6465-29e1-4a04-8bdb-00241474b4d2", + "join_conflict:rebinding_after_emission" + ] + } +} diff --git a/tests/platforms/copilot/fixtures/reconciliation-cases/cases.json b/tests/platforms/copilot/fixtures/reconciliation-cases/cases.json index 9fccd8c..9a18de6 100644 --- a/tests/platforms/copilot/fixtures/reconciliation-cases/cases.json +++ b/tests/platforms/copilot/fixtures/reconciliation-cases/cases.json @@ -1,12 +1,1329 @@ { - "permission": {"input": "permission request/decision evidence", "required": "emit permission semantic event; do not infer a tool execution or usage call"}, - "compaction": {"input": "checkpoint or compaction snapshot", "required": "retain source evidence and diagnostics; never add snapshot usage to per-call accounting"}, - "abort": {"input": "unfinished interaction", "required": "keep it pending and retain open state; missing usage is absent, not zero"}, - "retry": {"input": "replayed source record", "required": "stable source-derived IDs make projection replay equivalent without duplicate accounting"}, - "nested_child": {"input": "child prompt/stop hooks with parent session ID", "required": "child evidence stays within its parent main interaction; session-ID-only ownership is not inferred"}, - "late_row": {"input": "new assistant_usage_events row after transcript completion", "required": "create/revise accounting locally and leave attribution pending unless evidence is uniquely consistent"}, - "revision": {"input": "same database generation/table/row with new content revision", "required": "replace the existing logical UsageRow or quarantine conflict; do not add another charge"}, - "identical_concurrent_tools": {"input": "same tool name and arguments in parallel", "required": "use transcript toolCallId/source identity; hooks and nearest timestamps cannot prove pairing"}, - "ambiguous": {"input": "two candidate calls fit one usage row", "required": "Attribution status ambiguous with both candidate IDs in evidence; choose neither"}, - "conflicting": {"input": "two revisions assert incompatible metrics for one logical call", "required": "Attribution status conflicting and quarantine diagnostic; do not overwrite a delivered accounting identity"} + "permission": { + "observed": false, + "required": "emit permission semantic event; do not infer a tool execution or usage call", + "input_records": [ + { + "source_id": "hook/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/obs-permission-1", + "source_kind": "hook", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.500Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "schema_version": 1, + "event": "permissionRequest", + "hook_payload": { + "sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "timestamp": 1789060104500, + "cwd": "/fixture/workspace", + "toolName": "view", + "toolArgs": { + "path": "/fixture/workspace/alpha.txt" + } + }, + "context": {} + }, + "locator": { + "observation_id": "obs-permission-1", + "event": "permissionRequest" + } + } + ], + "expected": { + "events": [ + { + "id": "copilot:event:hook/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/obs-permission-1", + "kind": "permission_request", + "classification": "main", + "ts": "2026-09-10T17:08:24.500Z", + "source_ids": [ + "hook/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/obs-permission-1" + ], + "source_references": [ + { + "source_id": "hook/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/obs-permission-1", + "source_kind": "hook", + "role": "permission_request" + } + ], + "attributes": { + "tool_name": "view" + } + } + ], + "call_candidates": [], + "usage_rows": [], + "attributions": [] + } + }, + "compaction": { + "observed": false, + "required": "retain source evidence and diagnostics; never add snapshot usage to per-call accounting", + "input_records": [ + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-checkpoint-1", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:25.700Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "session.usage_checkpoint", + "data": { + "inputTokens": 26416, + "outputTokens": 229, + "cacheReadTokens": 19522, + "cacheWriteTokens": 6882, + "reasoningTokens": 39, + "totalNanoAiu": 238814000 + }, + "id": "synthetic-checkpoint-1", + "timestamp": "2026-09-10T17:08:25.700Z", + "parentId": "0080e44c-ad62-4288-b2b2-061ec2b73d80", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 0, + "byte_length": 297, + "native_event_id": "synthetic-checkpoint-1" + } + }, + { + "source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "source_kind": "database", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.498Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "table": "assistant_usage_events", + "row": { + "id": 13, + "session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "turn_index": 0, + "agent_id": null, + "parent_tool_call_id": null, + "model": "gpt-5.6-luna", + "input_tokens": 6452, + "output_tokens": 107, + "cache_read_tokens": 0, + "cache_write_tokens": 6449, + "reasoning_tokens": 29, + "total_nano_aiu": 174125000, + "request_multiplier": 1.0, + "duration_ms": 2263, + "time_to_first_token_ms": 1571.1477920000002, + "output_ttft_ms": 1571.1479590000001, + "inter_token_latency_ms": 7.393025117647059, + "initiator": "user", + "api_endpoint": "ws:/responses", + "reasoning_effort": "medium", + "finish_reason": "tool_calls", + "content_filter_triggered": 0, + "token_details_json": "[{\"batchSize\":1000000,\"costPerBatch\":20000000000,\"tokenCount\":3,\"tokenType\":\"input\"},{\"batchSize\":1000000,\"costPerBatch\":2000000000,\"tokenCount\":0,\"tokenType\":\"cache_read\"},{\"batchSize\":1000000,\"costPerBatch\":25000000000,\"tokenCount\":6449,\"tokenType\":\"cache_write\"},{\"batchSize\":1000000,\"costPerBatch\":120000000000,\"tokenCount\":107,\"tokenType\":\"output\"}]", + "created_at": "2026-09-10T17:08:24.498Z" + } + }, + "locator": { + "database": "/example/.copilot/session-store.db", + "table": "assistant_usage_events", + "primary_key": 13, + "content_revision": "sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "generation": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + } + } + ], + "expected": { + "events": [ + { + "id": "copilot:event:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-checkpoint-1", + "kind": "compaction", + "classification": "checkpoint", + "ts": "2026-09-10T17:08:25.700Z", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-checkpoint-1" + ], + "source_references": [ + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-checkpoint-1", + "source_kind": "transcript", + "role": "checkpoint" + } + ], + "attributes": {} + } + ], + "usage_rows": [ + { + "session_id": "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "seq": 0, + "call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13", + "ts": "2026-09-10T17:08:24.498Z", + "platform": "copilot", + "gen_ai.conversation.id": "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "gen_ai.provider.name": "unknown", + "gen_ai.operation.name": "chat", + "gen_ai.response.model": "gpt-5.6-luna", + "gen_ai.usage.input_tokens": 6452, + "gen_ai.usage.output_tokens": 107, + "gen_ai.usage.cache_read.input_tokens": 0, + "gen_ai.usage.cache_creation.input_tokens": 6449, + "gen_ai.usage.reasoning.output_tokens": 29 + } + ], + "diagnostics": [ + { + "code": "checkpoint_not_additive", + "severity": "info", + "message": "checkpoint snapshot validates totals and is not a seventh call", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-checkpoint-1" + ], + "details": {} + } + ] + } + }, + "abort": { + "observed": false, + "required": "keep it pending and retain open state; missing usage is absent, not zero", + "input_records": [ + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/f07404d0-af52-4260-89fb-358a10e86034", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:22.203Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "user.message", + "data": { + "content": "Read alpha.txt and beta.txt with separate view calls in parallel and report their sum. Only read those two files.", + "messageId": "a856cb38-7609-45ab-8a55-645553155db3", + "supportedNativeDocumentMimeTypes": [], + "delivery": "idle", + "interactionId": "6d2b89fd-a653-430c-b532-b0936d72eb42", + "turnId": "0", + "parentAgentTaskId": "fd800f7a-8163-4304-9681-efde4731e879" + }, + "id": "f07404d0-af52-4260-89fb-358a10e86034", + "timestamp": "2026-09-10T17:08:22.203Z", + "parentId": "3942810f-1caf-4251-82fb-3bf72698147a", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 2122, + "byte_length": 548, + "native_event_id": "f07404d0-af52-4260-89fb-358a10e86034" + } + } + ], + "expected": { + "turns": [], + "usage_rows": [], + "pending": [ + { + "id": "pending:copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42", + "kind": "open_interaction", + "reason": "user interaction has no assistant.turn_end", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/f07404d0-af52-4260-89fb-358a10e86034" + ], + "evidence": [ + "interaction_id:6d2b89fd-a653-430c-b532-b0936d72eb42" + ] + } + ], + "semantic_state": { + "open_interactions": { + "6d2b89fd-a653-430c-b532-b0936d72eb42|main": { + "interaction_id": "6d2b89fd-a653-430c-b532-b0936d72eb42", + "agent_id": null, + "stored_turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/f07404d0-af52-4260-89fb-358a10e86034" + ], + "last_event_source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/f07404d0-af52-4260-89fb-358a10e86034", + "start_ts": "2026-09-10T17:08:22.203Z", + "pending_tool_call_ids": [] + } + } + } + } + }, + "retry": { + "observed": false, + "required": "stable source-derived IDs make projection replay equivalent without duplicate accounting", + "input_records": [ + { + "source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "source_kind": "database", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.498Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "table": "assistant_usage_events", + "row": { + "id": 13, + "session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "turn_index": 0, + "agent_id": null, + "parent_tool_call_id": null, + "model": "gpt-5.6-luna", + "input_tokens": 6452, + "output_tokens": 107, + "cache_read_tokens": 0, + "cache_write_tokens": 6449, + "reasoning_tokens": 29, + "total_nano_aiu": 174125000, + "request_multiplier": 1.0, + "duration_ms": 2263, + "time_to_first_token_ms": 1571.1477920000002, + "output_ttft_ms": 1571.1479590000001, + "inter_token_latency_ms": 7.393025117647059, + "initiator": "user", + "api_endpoint": "ws:/responses", + "reasoning_effort": "medium", + "finish_reason": "tool_calls", + "content_filter_triggered": 0, + "token_details_json": "[{\"batchSize\":1000000,\"costPerBatch\":20000000000,\"tokenCount\":3,\"tokenType\":\"input\"},{\"batchSize\":1000000,\"costPerBatch\":2000000000,\"tokenCount\":0,\"tokenType\":\"cache_read\"},{\"batchSize\":1000000,\"costPerBatch\":25000000000,\"tokenCount\":6449,\"tokenType\":\"cache_write\"},{\"batchSize\":1000000,\"costPerBatch\":120000000000,\"tokenCount\":107,\"tokenType\":\"output\"}]", + "created_at": "2026-09-10T17:08:24.498Z" + } + }, + "locator": { + "database": "/example/.copilot/session-store.db", + "table": "assistant_usage_events", + "primary_key": 13, + "content_revision": "sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "generation": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + } + }, + { + "source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "source_kind": "database", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.498Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "table": "assistant_usage_events", + "row": { + "id": 13, + "session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "turn_index": 0, + "agent_id": null, + "parent_tool_call_id": null, + "model": "gpt-5.6-luna", + "input_tokens": 6452, + "output_tokens": 107, + "cache_read_tokens": 0, + "cache_write_tokens": 6449, + "reasoning_tokens": 29, + "total_nano_aiu": 174125000, + "request_multiplier": 1.0, + "duration_ms": 2263, + "time_to_first_token_ms": 1571.1477920000002, + "output_ttft_ms": 1571.1479590000001, + "inter_token_latency_ms": 7.393025117647059, + "initiator": "user", + "api_endpoint": "ws:/responses", + "reasoning_effort": "medium", + "finish_reason": "tool_calls", + "content_filter_triggered": 0, + "token_details_json": "[{\"batchSize\":1000000,\"costPerBatch\":20000000000,\"tokenCount\":3,\"tokenType\":\"input\"},{\"batchSize\":1000000,\"costPerBatch\":2000000000,\"tokenCount\":0,\"tokenType\":\"cache_read\"},{\"batchSize\":1000000,\"costPerBatch\":25000000000,\"tokenCount\":6449,\"tokenType\":\"cache_write\"},{\"batchSize\":1000000,\"costPerBatch\":120000000000,\"tokenCount\":107,\"tokenType\":\"output\"}]", + "created_at": "2026-09-10T17:08:24.498Z" + } + }, + "locator": { + "database": "/example/.copilot/session-store.db", + "table": "assistant_usage_events", + "primary_key": 13, + "content_revision": "sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "generation": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + } + } + ], + "expected": { + "usage_rows": [ + { + "session_id": "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "seq": 0, + "call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13", + "ts": "2026-09-10T17:08:24.498Z", + "platform": "copilot", + "gen_ai.conversation.id": "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "gen_ai.provider.name": "unknown", + "gen_ai.operation.name": "chat", + "gen_ai.response.model": "gpt-5.6-luna", + "gen_ai.usage.input_tokens": 6452, + "gen_ai.usage.output_tokens": 107, + "gen_ai.usage.cache_read.input_tokens": 0, + "gen_ai.usage.cache_creation.input_tokens": 6449, + "gen_ai.usage.reasoning.output_tokens": 29 + } + ], + "candidates": [ + { + "usage_source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13", + "turn_index": 0, + "agent_id": null, + "parent_tool_call_id": null, + "model": "gpt-5.6-luna", + "provider": null, + "source_ids": [ + "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47" + ], + "source_references": [ + { + "source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "source_kind": "database", + "role": "usage_row" + } + ], + "revision": { + "table": "assistant_usage_events", + "primary_key": "13", + "generation": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "content_revision": "sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47" + }, + "timestamp": "2026-09-10T17:08:24.498Z", + "finish_reason": "tool_calls", + "supplemental_metrics": { + "total_nano_aiu": 174125000, + "request_multiplier": 1.0, + "initiator": "user", + "api_endpoint": "ws:/responses", + "reasoning_effort": "medium", + "content_filter_triggered": 0, + "token_details_json": "[{\"batchSize\":1000000,\"costPerBatch\":20000000000,\"tokenCount\":3,\"tokenType\":\"input\"},{\"batchSize\":1000000,\"costPerBatch\":2000000000,\"tokenCount\":0,\"tokenType\":\"cache_read\"},{\"batchSize\":1000000,\"costPerBatch\":25000000000,\"tokenCount\":6449,\"tokenType\":\"cache_write\"},{\"batchSize\":1000000,\"costPerBatch\":120000000000,\"tokenCount\":107,\"tokenType\":\"output\"}]", + "duration_ms": 2263, + "time_to_first_token_ms": 1571.1477920000002, + "output_ttft_ms": 1571.1479590000001, + "inter_token_latency_ms": 7.393025117647059 + } + } + ] + } + }, + "nested_child": { + "observed": true, + "required": "child evidence stays within its parent main interaction; session-ID-only ownership is not inferred", + "input_records": [ + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/6dff9fa5-c2df-41da-912b-d1ffd0706076", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:44.058Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "subagent.started", + "data": { + "toolCallId": "call_qx4FH5DADTeT1qVLb37HNpBk", + "agentName": "explore", + "agentDisplayName": "sum-alpha-beta", + "agentDescription": "Sum two text files", + "model": "gpt-5.6-luna", + "resumable": false, + "agentType": "explore", + "executionMode": "sync" + }, + "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", + "id": "6dff9fa5-c2df-41da-912b-d1ffd0706076", + "timestamp": "2026-09-10T17:08:44.058Z", + "parentId": "0049f918-7bb9-478a-8c91-b6f8353525a9", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 15821, + "byte_length": 474, + "native_event_id": "6dff9fa5-c2df-41da-912b-d1ffd0706076" + } + }, + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/d4e30206-94f5-47d5-b15e-7411b9708aec", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:44.557Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "user.message", + "data": { + "content": "Read only /fixture/workspace/alpha.txt and /fixture/workspace/beta.txt. Do not modify files. Do not access any other files or services. Report the sum of the numeric values in those two files.", + "source": "agent-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "supportedNativeDocumentMimeTypes": [], + "delivery": "idle", + "interactionId": "7c0fa097-c0e2-48da-b2b6-fcfc1ad83a6b", + "turnId": "0", + "parentAgentTaskId": "5a3e63ac-c073-40ba-b07a-010d534b363e" + }, + "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", + "id": "d4e30206-94f5-47d5-b15e-7411b9708aec", + "timestamp": "2026-09-10T17:08:44.557Z", + "parentId": "3bd18700-90c6-4b2f-aaaa-63760bdfa89a", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 18718, + "byte_length": 681, + "native_event_id": "d4e30206-94f5-47d5-b15e-7411b9708aec" + } + }, + { + "source_id": "hook/bf8cb9f3-2097-4db0-a3c8-78a2653b2106/obs-child-prompt-1", + "source_kind": "hook", + "native_session_id": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", + "ts": "2026-09-10T17:08:44.530Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "schema_version": 1, + "event": "userPromptSubmitted", + "hook_payload": { + "sessionId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", + "timestamp": 1789060124530, + "cwd": "/fixture/workspace", + "prompt": "Read only /fixture/workspace/alpha.txt and /fixture/workspace/beta.txt. Do not modify files. Do not access any other files or services. Report the sum of the numeric values in those two files." + }, + "context": {} + }, + "locator": { + "observation_id": "obs-child-prompt-1", + "event": "userPromptSubmitted" + } + } + ], + "expected": { + "main_turn_ids": [ + "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:793d3703-6f4a-4814-8877-34a7325848ce" + ], + "child_owned_by": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:793d3703-6f4a-4814-8877-34a7325848ce", + "child_hook_native_session_id": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", + "do_not_infer_from_session_id_alone": true, + "turns": [ + { + "turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:793d3703-6f4a-4814-8877-34a7325848ce", + "start_ts": "2026-09-10T17:08:42.192Z", + "end_ts": "2026-09-10T17:08:48.287Z", + "input_message": "Invoke one explore subagent to read only alpha.txt and beta.txt and report their sum. Do not modify files or access other files or services. Then report its answer.", + "output_message": "42", + "status": "completed", + "llm_calls": [], + "permission_requests": [], + "subagents": [ + { + "turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:7c0fa097-c0e2-48da-b2b6-fcfc1ad83a6b", + "start_ts": "2026-09-10T17:08:44.557Z", + "end_ts": "2026-09-10T17:08:47.463Z", + "input_message": "Read only /fixture/workspace/alpha.txt and /fixture/workspace/beta.txt. Do not modify files. Do not access any other files or services. Report the sum of the numeric values in those two files.", + "output_message": "42", + "status": "completed", + "llm_calls": [], + "permission_requests": [], + "subagents": [], + "attributes": { + "agent_id": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", + "parent_tool_call_id": "call_qx4FH5DADTeT1qVLb37HNpBk", + "agent_name": "explore" + } + } + ], + "attributes": { + "interaction_id": "793d3703-6f4a-4814-8877-34a7325848ce", + "agent_id": null + } + } + ], + "events": [ + { + "id": "copilot:event:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/6dff9fa5-c2df-41da-912b-d1ffd0706076", + "kind": "subagent_started", + "classification": "main", + "ts": "2026-09-10T17:08:44.058Z", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/6dff9fa5-c2df-41da-912b-d1ffd0706076" + ], + "source_references": [ + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/6dff9fa5-c2df-41da-912b-d1ffd0706076", + "source_kind": "transcript", + "role": "nested_child" + } + ], + "attributes": { + "agent_id": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", + "parent_tool_call_id": "call_qx4FH5DADTeT1qVLb37HNpBk" + } + } + ] + } + }, + "late_row": { + "observed": false, + "required": "create/revise accounting locally and leave attribution pending unless evidence is uniquely consistent", + "input_records": [ + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/f07404d0-af52-4260-89fb-358a10e86034", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:22.203Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "user.message", + "data": { + "content": "Read alpha.txt and beta.txt with separate view calls in parallel and report their sum. Only read those two files.", + "messageId": "a856cb38-7609-45ab-8a55-645553155db3", + "supportedNativeDocumentMimeTypes": [], + "delivery": "idle", + "interactionId": "6d2b89fd-a653-430c-b532-b0936d72eb42", + "turnId": "0", + "parentAgentTaskId": "fd800f7a-8163-4304-9681-efde4731e879" + }, + "id": "f07404d0-af52-4260-89fb-358a10e86034", + "timestamp": "2026-09-10T17:08:22.203Z", + "parentId": "3942810f-1caf-4251-82fb-3bf72698147a", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 2122, + "byte_length": 548, + "native_event_id": "f07404d0-af52-4260-89fb-358a10e86034" + } + }, + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.503Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "assistant.message", + "data": { + "messageId": "5a29b09d-5e7d-4603-975e-7801ce54232b", + "model": "gpt-5.6-luna", + "content": "", + "toolRequests": [ + { + "toolCallId": "call_YSSva4HCniiETlxdGGjcrHbh", + "name": "view", + "arguments": { + "path": "/fixture/workspace/alpha.txt" + }, + "type": "function", + "intentionSummary": "view the file at /fixture/workspace/alpha.txt." + }, + { + "toolCallId": "call_ayHplfzxjRFMTCpmTKEFhCSJ", + "name": "view", + "arguments": { + "path": "/fixture/workspace/beta.txt" + }, + "type": "function", + "intentionSummary": "view the file at /fixture/workspace/beta.txt." + } + ], + "interactionId": "6d2b89fd-a653-430c-b532-b0936d72eb42", + "turnId": "0", + "rte": true + }, + "id": "a4a17e63-7ba5-422f-8ee9-b495be417328", + "timestamp": "2026-09-10T17:08:24.503Z", + "parentId": "64f9e436-651f-4a9d-919c-a4d7abad2652", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 3731, + "byte_length": 793, + "native_event_id": "a4a17e63-7ba5-422f-8ee9-b495be417328" + } + }, + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/0080e44c-ad62-4288-b2b2-061ec2b73d80", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.593Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "assistant.turn_end", + "data": { + "turnId": "0" + }, + "id": "0080e44c-ad62-4288-b2b2-061ec2b73d80", + "timestamp": "2026-09-10T17:08:24.593Z", + "parentId": "032cf87e-05b7-4e5f-96b9-1fda1a8242f5", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 9311, + "byte_length": 195, + "native_event_id": "0080e44c-ad62-4288-b2b2-061ec2b73d80" + } + }, + { + "source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:14:sha256:20cb0f98d6130c8d0e139391cfc8e245d1d0ac96d41469c4da60f26e6d9e0a26", + "source_kind": "database", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:25.618Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "table": "assistant_usage_events", + "row": { + "id": 14, + "session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "turn_index": 0, + "agent_id": null, + "parent_tool_call_id": null, + "model": "gpt-5.6-luna", + "input_tokens": 6587, + "output_tokens": 5, + "cache_read_tokens": 6449, + "cache_write_tokens": 135, + "reasoning_tokens": 0, + "total_nano_aiu": 16933000, + "request_multiplier": 1.0, + "duration_ms": 1017, + "time_to_first_token_ms": 942.6416250000001, + "output_ttft_ms": 942.642416, + "inter_token_latency_ms": null, + "initiator": "agent", + "api_endpoint": "ws:/responses", + "reasoning_effort": "medium", + "finish_reason": "stop", + "content_filter_triggered": 0, + "token_details_json": "[{\"batchSize\":1000000,\"costPerBatch\":20000000000,\"tokenCount\":3,\"tokenType\":\"input\"},{\"batchSize\":1000000,\"costPerBatch\":2000000000,\"tokenCount\":6449,\"tokenType\":\"cache_read\"},{\"batchSize\":1000000,\"costPerBatch\":25000000000,\"tokenCount\":135,\"tokenType\":\"cache_write\"},{\"batchSize\":1000000,\"costPerBatch\":120000000000,\"tokenCount\":5,\"tokenType\":\"output\"}]", + "created_at": "2026-09-10T17:08:25.618Z" + } + }, + "locator": { + "database": "/example/.copilot/session-store.db", + "table": "assistant_usage_events", + "primary_key": 14, + "content_revision": "sha256:20cb0f98d6130c8d0e139391cfc8e245d1d0ac96d41469c4da60f26e6d9e0a26", + "generation": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + } + } + ], + "expected": { + "usage_rows": [ + { + "session_id": "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "seq": 0, + "call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:14", + "ts": "2026-09-10T17:08:25.618Z", + "platform": "copilot", + "gen_ai.conversation.id": "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "gen_ai.provider.name": "unknown", + "gen_ai.operation.name": "chat", + "gen_ai.response.model": "gpt-5.6-luna", + "gen_ai.usage.input_tokens": 6587, + "gen_ai.usage.output_tokens": 5, + "gen_ai.usage.cache_read.input_tokens": 6449, + "gen_ai.usage.cache_creation.input_tokens": 135, + "gen_ai.usage.reasoning.output_tokens": 0 + } + ], + "attributions": [ + { + "usage_source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:14:sha256:20cb0f98d6130c8d0e139391cfc8e245d1d0ac96d41469c4da60f26e6d9e0a26", + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:14", + "stored_turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42", + "agent_id": null, + "call_id": null, + "status": "pending", + "join_kind": null, + "evidence": [ + "logical_call_id:copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:14", + "turn_index:0", + "delayed_row:true" + ] + } + ] + } + }, + "revision": { + "observed": false, + "required": "replace the existing logical UsageRow or quarantine conflict; do not add another charge", + "input_records": [ + { + "source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "source_kind": "database", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.498Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "table": "assistant_usage_events", + "row": { + "id": 13, + "session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "turn_index": 0, + "agent_id": null, + "parent_tool_call_id": null, + "model": "gpt-5.6-luna", + "input_tokens": 6452, + "output_tokens": 107, + "cache_read_tokens": 0, + "cache_write_tokens": 6449, + "reasoning_tokens": 29, + "total_nano_aiu": 174125000, + "request_multiplier": 1.0, + "duration_ms": 2263, + "time_to_first_token_ms": 1571.1477920000002, + "output_ttft_ms": 1571.1479590000001, + "inter_token_latency_ms": 7.393025117647059, + "initiator": "user", + "api_endpoint": "ws:/responses", + "reasoning_effort": "medium", + "finish_reason": "tool_calls", + "content_filter_triggered": 0, + "token_details_json": "[{\"batchSize\":1000000,\"costPerBatch\":20000000000,\"tokenCount\":3,\"tokenType\":\"input\"},{\"batchSize\":1000000,\"costPerBatch\":2000000000,\"tokenCount\":0,\"tokenType\":\"cache_read\"},{\"batchSize\":1000000,\"costPerBatch\":25000000000,\"tokenCount\":6449,\"tokenType\":\"cache_write\"},{\"batchSize\":1000000,\"costPerBatch\":120000000000,\"tokenCount\":107,\"tokenType\":\"output\"}]", + "created_at": "2026-09-10T17:08:24.498Z" + } + }, + "locator": { + "database": "/example/.copilot/session-store.db", + "table": "assistant_usage_events", + "primary_key": 13, + "content_revision": "sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "generation": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + } + }, + { + "source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:fe8383e7e562b4986c48b7b03ce880a54fd4ef03ae1721047ca41261e4ff85ca", + "source_kind": "database", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.498Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "table": "assistant_usage_events", + "row": { + "id": 13, + "session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "turn_index": 0, + "agent_id": null, + "parent_tool_call_id": null, + "model": "gpt-5.6-luna", + "input_tokens": 6452, + "output_tokens": 999, + "cache_read_tokens": 0, + "cache_write_tokens": 6449, + "reasoning_tokens": 29, + "total_nano_aiu": 174125000, + "request_multiplier": 1.0, + "duration_ms": 2263, + "time_to_first_token_ms": 1571.1477920000002, + "output_ttft_ms": 1571.1479590000001, + "inter_token_latency_ms": 7.393025117647059, + "initiator": "user", + "api_endpoint": "ws:/responses", + "reasoning_effort": "medium", + "finish_reason": "tool_calls", + "content_filter_triggered": 0, + "token_details_json": "[{\"batchSize\":1000000,\"costPerBatch\":20000000000,\"tokenCount\":3,\"tokenType\":\"input\"},{\"batchSize\":1000000,\"costPerBatch\":2000000000,\"tokenCount\":0,\"tokenType\":\"cache_read\"},{\"batchSize\":1000000,\"costPerBatch\":25000000000,\"tokenCount\":6449,\"tokenType\":\"cache_write\"},{\"batchSize\":1000000,\"costPerBatch\":120000000000,\"tokenCount\":107,\"tokenType\":\"output\"}]", + "created_at": "2026-09-10T17:08:24.498Z" + } + }, + "locator": { + "database": "/example/.copilot/session-store.db", + "table": "assistant_usage_events", + "primary_key": 13, + "content_revision": "sha256:fe8383e7e562b4986c48b7b03ce880a54fd4ef03ae1721047ca41261e4ff85ca", + "generation": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + } + } + ], + "expected": { + "logical_call_ids": [ + "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13" + ], + "usage_row_count": 1, + "usage_source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:fe8383e7e562b4986c48b7b03ce880a54fd4ef03ae1721047ca41261e4ff85ca", + "replaced_output_tokens": 999 + } + }, + "identical_concurrent_tools": { + "observed": true, + "required": "use transcript toolCallId/source identity; hooks and nearest timestamps cannot prove pairing", + "input_records": [ + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.503Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "assistant.message", + "data": { + "messageId": "5a29b09d-5e7d-4603-975e-7801ce54232b", + "model": "gpt-5.6-luna", + "content": "", + "toolRequests": [ + { + "toolCallId": "call_YSSva4HCniiETlxdGGjcrHbh", + "name": "view", + "arguments": { + "path": "/fixture/workspace/alpha.txt" + }, + "type": "function", + "intentionSummary": "view the file at /fixture/workspace/alpha.txt." + }, + { + "toolCallId": "call_ayHplfzxjRFMTCpmTKEFhCSJ", + "name": "view", + "arguments": { + "path": "/fixture/workspace/beta.txt" + }, + "type": "function", + "intentionSummary": "view the file at /fixture/workspace/beta.txt." + } + ], + "interactionId": "6d2b89fd-a653-430c-b532-b0936d72eb42", + "turnId": "0", + "rte": true + }, + "id": "a4a17e63-7ba5-422f-8ee9-b495be417328", + "timestamp": "2026-09-10T17:08:24.503Z", + "parentId": "64f9e436-651f-4a9d-919c-a4d7abad2652", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 3731, + "byte_length": 793, + "native_event_id": "a4a17e63-7ba5-422f-8ee9-b495be417328" + } + }, + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a7f7bf04-589e-4989-a4a2-7ee687279627", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.506Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "tool.execution_start", + "data": { + "toolCallId": "call_YSSva4HCniiETlxdGGjcrHbh", + "toolName": "view", + "arguments": { + "path": "/fixture/workspace/alpha.txt" + }, + "turnId": "0", + "model": "gpt-5.6-luna" + }, + "id": "a7f7bf04-589e-4989-a4a2-7ee687279627", + "timestamp": "2026-09-10T17:08:24.506Z", + "parentId": "a4a17e63-7ba5-422f-8ee9-b495be417328", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 4524, + "byte_length": 344, + "native_event_id": "a7f7bf04-589e-4989-a4a2-7ee687279627" + } + }, + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/39e9da78-9bc3-4e1a-9629-f6680d1aeb4d", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.506Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "tool.execution_start", + "data": { + "toolCallId": "call_ayHplfzxjRFMTCpmTKEFhCSJ", + "toolName": "view", + "arguments": { + "path": "/fixture/workspace/beta.txt" + }, + "turnId": "0", + "model": "gpt-5.6-luna" + }, + "id": "39e9da78-9bc3-4e1a-9629-f6680d1aeb4d", + "timestamp": "2026-09-10T17:08:24.506Z", + "parentId": "a7f7bf04-589e-4989-a4a2-7ee687279627", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 4868, + "byte_length": 343, + "native_event_id": "39e9da78-9bc3-4e1a-9629-f6680d1aeb4d" + } + } + ], + "expected": { + "tool_call_ids": [ + "call_YSSva4HCniiETlxdGGjcrHbh", + "call_ayHplfzxjRFMTCpmTKEFhCSJ" + ], + "event_ids": [ + "copilot:event:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a7f7bf04-589e-4989-a4a2-7ee687279627", + "copilot:event:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/39e9da78-9bc3-4e1a-9629-f6680d1aeb4d" + ], + "forbidden_join_keys": [ + "tool_name", + "timestamp", + "parentId", + "turnId" + ] + } + }, + "ambiguous": { + "observed": false, + "required": "Attribution status ambiguous with both candidate IDs in evidence; choose neither", + "input_records": [ + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.503Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "assistant.message", + "data": { + "messageId": "5a29b09d-5e7d-4603-975e-7801ce54232b", + "model": "gpt-5.6-luna", + "content": "", + "toolRequests": [ + { + "toolCallId": "call_YSSva4HCniiETlxdGGjcrHbh", + "name": "view", + "arguments": { + "path": "/fixture/workspace/alpha.txt" + }, + "type": "function", + "intentionSummary": "view the file at /fixture/workspace/alpha.txt." + }, + { + "toolCallId": "call_ayHplfzxjRFMTCpmTKEFhCSJ", + "name": "view", + "arguments": { + "path": "/fixture/workspace/beta.txt" + }, + "type": "function", + "intentionSummary": "view the file at /fixture/workspace/beta.txt." + } + ], + "interactionId": "6d2b89fd-a653-430c-b532-b0936d72eb42", + "turnId": "0", + "rte": true + }, + "id": "a4a17e63-7ba5-422f-8ee9-b495be417328", + "timestamp": "2026-09-10T17:08:24.503Z", + "parentId": "64f9e436-651f-4a9d-919c-a4d7abad2652", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 3731, + "byte_length": 793, + "native_event_id": "a4a17e63-7ba5-422f-8ee9-b495be417328" + } + }, + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/33cc6465-29e1-4a04-8bdb-00241474b4d2", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:25.624Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "assistant.message", + "data": { + "messageId": "162d92b0-5d31-444c-b96f-b6c551527d2b", + "model": "gpt-5.6-luna", + "content": "42", + "toolRequests": [], + "interactionId": "6d2b89fd-a653-430c-b532-b0936d72eb42", + "turnId": "1", + "phase": "final_answer", + "rte": true + }, + "id": "33cc6465-29e1-4a04-8bdb-00241474b4d2", + "timestamp": "2026-09-10T17:08:25.624Z", + "parentId": "667fe48a-70a1-473b-b005-454964022344", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 9760, + "byte_length": 404, + "native_event_id": "33cc6465-29e1-4a04-8bdb-00241474b4d2" + } + }, + { + "source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "source_kind": "database", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.498Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "table": "assistant_usage_events", + "row": { + "id": 13, + "session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "turn_index": 0, + "agent_id": null, + "parent_tool_call_id": null, + "model": "gpt-5.6-luna", + "input_tokens": 6452, + "output_tokens": 107, + "cache_read_tokens": 0, + "cache_write_tokens": 6449, + "reasoning_tokens": 29, + "total_nano_aiu": 174125000, + "request_multiplier": 1.0, + "duration_ms": 2263, + "time_to_first_token_ms": 1571.1477920000002, + "output_ttft_ms": 1571.1479590000001, + "inter_token_latency_ms": 7.393025117647059, + "initiator": "user", + "api_endpoint": "ws:/responses", + "reasoning_effort": "medium", + "finish_reason": "tool_calls", + "content_filter_triggered": 0, + "token_details_json": "[{\"batchSize\":1000000,\"costPerBatch\":20000000000,\"tokenCount\":3,\"tokenType\":\"input\"},{\"batchSize\":1000000,\"costPerBatch\":2000000000,\"tokenCount\":0,\"tokenType\":\"cache_read\"},{\"batchSize\":1000000,\"costPerBatch\":25000000000,\"tokenCount\":6449,\"tokenType\":\"cache_write\"},{\"batchSize\":1000000,\"costPerBatch\":120000000000,\"tokenCount\":107,\"tokenType\":\"output\"}]", + "created_at": "2026-09-10T17:08:24.498Z" + } + }, + "locator": { + "database": "/example/.copilot/session-store.db", + "table": "assistant_usage_events", + "primary_key": 13, + "content_revision": "sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "generation": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + } + } + ], + "expected": { + "attributions": [ + { + "usage_source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13", + "stored_turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42", + "agent_id": null, + "call_id": null, + "status": "ambiguous", + "join_kind": null, + "evidence": [ + "call_id:copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "call_id:copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/33cc6465-29e1-4a04-8bdb-00241474b4d2", + "interaction_id:6d2b89fd-a653-430c-b532-b0936d72eb42", + "model:gpt-5.6-luna", + "turn_index:0" + ] + } + ] + } + }, + "conflicting": { + "observed": false, + "required": "ProjectionDiagnostic usage_revision_conflict quarantines incompatible revisions. Attribution.status conflicting is a join conflict, not that diagnostic.", + "input_records": [ + { + "source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "source_kind": "database", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.498Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "table": "assistant_usage_events", + "row": { + "id": 13, + "session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "turn_index": 0, + "agent_id": null, + "parent_tool_call_id": null, + "model": "gpt-5.6-luna", + "input_tokens": 6452, + "output_tokens": 107, + "cache_read_tokens": 0, + "cache_write_tokens": 6449, + "reasoning_tokens": 29, + "total_nano_aiu": 174125000, + "request_multiplier": 1.0, + "duration_ms": 2263, + "time_to_first_token_ms": 1571.1477920000002, + "output_ttft_ms": 1571.1479590000001, + "inter_token_latency_ms": 7.393025117647059, + "initiator": "user", + "api_endpoint": "ws:/responses", + "reasoning_effort": "medium", + "finish_reason": "tool_calls", + "content_filter_triggered": 0, + "token_details_json": "[{\"batchSize\":1000000,\"costPerBatch\":20000000000,\"tokenCount\":3,\"tokenType\":\"input\"},{\"batchSize\":1000000,\"costPerBatch\":2000000000,\"tokenCount\":0,\"tokenType\":\"cache_read\"},{\"batchSize\":1000000,\"costPerBatch\":25000000000,\"tokenCount\":6449,\"tokenType\":\"cache_write\"},{\"batchSize\":1000000,\"costPerBatch\":120000000000,\"tokenCount\":107,\"tokenType\":\"output\"}]", + "created_at": "2026-09-10T17:08:24.498Z" + } + }, + "locator": { + "database": "/example/.copilot/session-store.db", + "table": "assistant_usage_events", + "primary_key": 13, + "content_revision": "sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "generation": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + } + }, + { + "source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:bc82d51eaf6061d225d88e973fdaa5549ad7177a50376de381a3a33aadacfd13", + "source_kind": "database", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.498Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "table": "assistant_usage_events", + "row": { + "id": 13, + "session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "turn_index": 0, + "agent_id": null, + "parent_tool_call_id": null, + "model": "gpt-5.6-luna", + "input_tokens": 1, + "output_tokens": 1, + "cache_read_tokens": 0, + "cache_write_tokens": 6449, + "reasoning_tokens": 29, + "total_nano_aiu": 1, + "request_multiplier": 1.0, + "duration_ms": 2263, + "time_to_first_token_ms": 1571.1477920000002, + "output_ttft_ms": 1571.1479590000001, + "inter_token_latency_ms": 7.393025117647059, + "initiator": "user", + "api_endpoint": "ws:/responses", + "reasoning_effort": "medium", + "finish_reason": "tool_calls", + "content_filter_triggered": 0, + "token_details_json": "[{\"batchSize\":1000000,\"costPerBatch\":20000000000,\"tokenCount\":3,\"tokenType\":\"input\"},{\"batchSize\":1000000,\"costPerBatch\":2000000000,\"tokenCount\":0,\"tokenType\":\"cache_read\"},{\"batchSize\":1000000,\"costPerBatch\":25000000000,\"tokenCount\":6449,\"tokenType\":\"cache_write\"},{\"batchSize\":1000000,\"costPerBatch\":120000000000,\"tokenCount\":107,\"tokenType\":\"output\"}]", + "created_at": "2026-09-10T17:08:24.498Z" + } + }, + "locator": { + "database": "/example/.copilot/session-store.db", + "table": "assistant_usage_events", + "primary_key": 13, + "content_revision": "sha256:bc82d51eaf6061d225d88e973fdaa5549ad7177a50376de381a3a33aadacfd13", + "generation": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + } + } + ], + "expected": { + "usage_row_count": 0, + "diagnostics": [ + { + "code": "usage_revision_conflict", + "severity": "error", + "message": "incompatible metrics for one logical call; quarantined", + "source_ids": [ + "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:bc82d51eaf6061d225d88e973fdaa5549ad7177a50376de381a3a33aadacfd13" + ], + "details": { + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13" + } + } + ], + "attributions": [ + { + "usage_source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13", + "stored_turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42", + "agent_id": null, + "call_id": null, + "status": "conflicting", + "join_kind": null, + "evidence": [ + "call_id:copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "call_id:copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/33cc6465-29e1-4a04-8bdb-00241474b4d2", + "join_conflict:rebinding_after_emission" + ] + } + ] + } + }, + "auxiliary_title_generation": { + "observed": false, + "required": "classify model.* title generation separately; do not add to the six-call total", + "input_records": [ + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-title-1", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:08.000Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "model.model_call_success", + "data": { + "model": "gpt-4o-mini", + "purpose": "session_title", + "input_tokens": 120, + "output_tokens": 8 + }, + "id": "synthetic-title-1", + "timestamp": "2026-09-10T17:08:08.000Z", + "parentId": null, + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 0, + "byte_length": 210, + "native_event_id": "synthetic-title-1" + } + } + ], + "expected": { + "events": [ + { + "id": "copilot:event:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-title-1", + "kind": "auxiliary_model_call", + "classification": "title_generation", + "ts": "2026-09-10T17:08:08.000Z", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-title-1" + ], + "source_references": [ + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-title-1", + "source_kind": "transcript", + "role": "auxiliary_model" + } + ], + "attributes": { + "model": "gpt-4o-mini" + } + } + ], + "call_candidates": [], + "usage_rows": [], + "diagnostics": [ + { + "code": "auxiliary_excluded_from_main", + "severity": "info", + "message": "title-generation model.* record classified auxiliary", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-title-1" + ], + "details": { + "classification": "title_generation", + "model": "gpt-4o-mini" + } + } + ] + } + } } diff --git a/tests/platforms/copilot/fixtures/reconciliation-cases/diagnostics.json b/tests/platforms/copilot/fixtures/reconciliation-cases/diagnostics.json new file mode 100644 index 0000000..9039b2c --- /dev/null +++ b/tests/platforms/copilot/fixtures/reconciliation-cases/diagnostics.json @@ -0,0 +1,62 @@ +{ + "pending": { + "open_interaction": { + "id": "pending:copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42", + "kind": "open_interaction", + "reason": "user interaction has no assistant.turn_end", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/f07404d0-af52-4260-89fb-358a10e86034" + ], + "evidence": [ + "interaction_id:6d2b89fd-a653-430c-b532-b0936d72eb42" + ] + }, + "missing_source_capability": { + "id": "pending:capability:hooks", + "kind": "missing_source_capability", + "reason": "archive has no hook observations; watch must still derive from transcript", + "source_ids": [], + "evidence": [ + "capability:hooks=absent" + ] + } + }, + "diagnostics": { + "usage_revision_conflict": { + "code": "usage_revision_conflict", + "severity": "error", + "message": "incompatible metrics for one logical call; quarantined", + "source_ids": [ + "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:bc82d51eaf6061d225d88e973fdaa5549ad7177a50376de381a3a33aadacfd13" + ], + "details": { + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13" + } + }, + "shutdown_total_mismatch": { + "code": "shutdown_total_mismatch", + "severity": "warning", + "message": "per-call totals do not equal session.shutdown usage", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/9dbdd079-e22a-4bd0-87e2-17fa2b246d23" + ], + "details": { + "expected_input_tokens": 35396, + "accounted_input_tokens": 0 + } + }, + "auxiliary_excluded_from_main": { + "code": "auxiliary_excluded_from_main", + "severity": "info", + "message": "title-generation model.* record classified auxiliary", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-title-1" + ], + "details": { + "classification": "title_generation", + "model": "gpt-4o-mini" + } + } + } +} diff --git a/tests/platforms/copilot/fixtures/reconciliation-cases/identities.json b/tests/platforms/copilot/fixtures/reconciliation-cases/identities.json new file mode 100644 index 0000000..93fce54 --- /dev/null +++ b/tests/platforms/copilot/fixtures/reconciliation-cases/identities.json @@ -0,0 +1,26 @@ +{ + "source_key": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "source_key_is_placeholder": true, + "source_key_note": "Full 64-character SHA-256 of the canonical source home. Not the 16-character stored-session prefix. Live captures replace this.", + "source_key_prefix": "aaaaaaaaaaaaaaaa", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "child_agent_id": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", + "stored_session_id": "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "database_generation": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "database_generation_is_placeholder": true, + "database_generation_note": "Live V1 generation is sha256 of canonical JSON {path, device, inode, birthtime_ns} from the database file stat.", + "quoted_database_generation": "sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "transcript_file_generation": "1-abc", + "transcript_file_generation_note": "Live V1 file_generation is hex(st_dev)-hex(st_ino)[-hex(birthtime_ns)].", + "database_path": "/example/.copilot/session-store.db", + "cwd": "/fixture/workspace", + "id_templates": { + "semantic_event": "copilot:event:", + "semantic_event_tool_request": "copilot:event::tool:", + "semantic_call": "copilot:call:", + "main_turn": "copilot:turn:::", + "logical_call": "copilot:usage::
::", + "turn_accounting_span": "accounting:{session_id}:{turn_id}:{accounting_id}", + "session_accounting_span": "accounting:{session_id}:{accounting_id}" + } +} diff --git a/tests/platforms/copilot/fixtures/reconciliation-cases/observed-six-calls.json b/tests/platforms/copilot/fixtures/reconciliation-cases/observed-six-calls.json index 7bbc42e..c90ecad 100644 --- a/tests/platforms/copilot/fixtures/reconciliation-cases/observed-six-calls.json +++ b/tests/platforms/copilot/fixtures/reconciliation-cases/observed-six-calls.json @@ -1,15 +1,239 @@ { "source_fixture": "../assistant-usage-events.json", - "source_key": "observed-source-key", - "database_generation": "observed-db-generation", + "source_key": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "source_key_is_placeholder": true, + "database_generation": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "database_generation_is_placeholder": true, "table": "assistant_usage_events", + "shutdown_totals": { + "source": "../usage.json", + "input_tokens": 35396, + "output_tokens": 328, + "reasoning_tokens": 55, + "cache_read_tokens": 23948, + "cache_write_tokens": 11430, + "total_nano_aiu": 373366000 + }, "calls": [ - {"row_id": 13, "logical_call_id": "copilot:usage:observed-source-key:assistant_usage_events:observed-db-generation:13", "turn_index": 0, "agent_id": null, "parent_tool_call_id": null, "finish_reason": "tool_calls"}, - {"row_id": 14, "logical_call_id": "copilot:usage:observed-source-key:assistant_usage_events:observed-db-generation:14", "turn_index": 0, "agent_id": null, "parent_tool_call_id": null, "finish_reason": "stop"}, - {"row_id": 15, "logical_call_id": "copilot:usage:observed-source-key:assistant_usage_events:observed-db-generation:15", "turn_index": 1, "agent_id": null, "parent_tool_call_id": null, "finish_reason": "tool_calls"}, - {"row_id": 16, "logical_call_id": "copilot:usage:observed-source-key:assistant_usage_events:observed-db-generation:16", "turn_index": 1, "agent_id": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "parent_tool_call_id": "call_qx4FH5DADTeT1qVLb37HNpBk", "finish_reason": "tool_calls"}, - {"row_id": 17, "logical_call_id": "copilot:usage:observed-source-key:assistant_usage_events:observed-db-generation:17", "turn_index": 1, "agent_id": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "parent_tool_call_id": "call_qx4FH5DADTeT1qVLb37HNpBk", "finish_reason": "stop"}, - {"row_id": 18, "logical_call_id": "copilot:usage:observed-source-key:assistant_usage_events:observed-db-generation:18", "turn_index": 1, "agent_id": null, "parent_tool_call_id": null, "finish_reason": "stop"} + { + "row_id": 13, + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13", + "usage_source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "turn_index": 0, + "agent_id": null, + "parent_tool_call_id": null, + "initiator": "user", + "model": "gpt-5.6-luna", + "finish_reason": "tool_calls", + "expected_call_id": "copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "expected_stored_turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42", + "interaction_id": "6d2b89fd-a653-430c-b532-b0936d72eb42" + }, + { + "row_id": 14, + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:14", + "usage_source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:14:sha256:20cb0f98d6130c8d0e139391cfc8e245d1d0ac96d41469c4da60f26e6d9e0a26", + "turn_index": 0, + "agent_id": null, + "parent_tool_call_id": null, + "initiator": "agent", + "model": "gpt-5.6-luna", + "finish_reason": "stop", + "expected_call_id": "copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/33cc6465-29e1-4a04-8bdb-00241474b4d2", + "expected_stored_turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42", + "interaction_id": "6d2b89fd-a653-430c-b532-b0936d72eb42" + }, + { + "row_id": 15, + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:15", + "usage_source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:15:sha256:c78b2b2ed28b8641c996bf9c84a05812d5a108776ffe091a54378f9ff79162d8", + "turn_index": 1, + "agent_id": null, + "parent_tool_call_id": null, + "initiator": "user", + "model": "gpt-5.6-luna", + "finish_reason": "tool_calls", + "expected_call_id": "copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/2c2e4ab8-f283-4837-957d-da992ba55e65", + "expected_stored_turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:793d3703-6f4a-4814-8877-34a7325848ce", + "interaction_id": "793d3703-6f4a-4814-8877-34a7325848ce" + }, + { + "row_id": 16, + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:16", + "usage_source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:16:sha256:be2e28cbe7e6071667598eb2156481465762532a6652196b7735a46ef412c67c", + "turn_index": 1, + "agent_id": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", + "parent_tool_call_id": "call_qx4FH5DADTeT1qVLb37HNpBk", + "initiator": "sub-agent", + "model": "gpt-5.6-luna", + "finish_reason": "tool_calls", + "expected_call_id": "copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/df600539-fdd4-4de2-bd42-7f7ff2c952ab", + "expected_stored_turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:793d3703-6f4a-4814-8877-34a7325848ce", + "interaction_id": "7c0fa097-c0e2-48da-b2b6-fcfc1ad83a6b" + }, + { + "row_id": 17, + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:17", + "usage_source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:17:sha256:92ca44e9db1e93336e94649492c5c59945ef4268ca1188f710b77e6e4b148435", + "turn_index": 1, + "agent_id": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", + "parent_tool_call_id": "call_qx4FH5DADTeT1qVLb37HNpBk", + "initiator": "sub-agent", + "model": "gpt-5.6-luna", + "finish_reason": "stop", + "expected_call_id": "copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/450a1f4c-de11-4d37-bf84-08f63d29588f", + "expected_stored_turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:793d3703-6f4a-4814-8877-34a7325848ce", + "interaction_id": "7c0fa097-c0e2-48da-b2b6-fcfc1ad83a6b" + }, + { + "row_id": 18, + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:18", + "usage_source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:18:sha256:5fb5f685b5495f2fcc1874ad3b563a6750fa35c863af4647ea91225dd3ec907a", + "turn_index": 1, + "agent_id": null, + "parent_tool_call_id": null, + "initiator": "agent", + "model": "gpt-5.6-luna", + "finish_reason": "stop", + "expected_call_id": "copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/10001566-2704-4f75-add3-2c044373eaba", + "expected_stored_turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:793d3703-6f4a-4814-8877-34a7325848ce", + "interaction_id": "793d3703-6f4a-4814-8877-34a7325848ce" + } ], - "invariants": {"call_count": 6, "main_agent_call_count": 4, "child_agent_call_count": 2, "shutdown_usage_is_validation_only": true, "checkpoint_usage_is_additive": false, "auxiliary_model_calls_are_main_usage": false} + "expected_attributions": [ + { + "usage_source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13", + "stored_turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42", + "agent_id": null, + "call_id": "copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "status": "matched", + "join_kind": "inferred", + "evidence": [ + "join_kind:inferred", + "interaction_id:6d2b89fd-a653-430c-b532-b0936d72eb42", + "turn_index:0", + "agent_id:None", + "parent_tool_call_id:None", + "model:gpt-5.6-luna", + "finish:tool_calls", + "initiator:user", + "order:0", + "call_id:copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328" + ] + }, + { + "usage_source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:14:sha256:20cb0f98d6130c8d0e139391cfc8e245d1d0ac96d41469c4da60f26e6d9e0a26", + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:14", + "stored_turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42", + "agent_id": null, + "call_id": "copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/33cc6465-29e1-4a04-8bdb-00241474b4d2", + "status": "matched", + "join_kind": "inferred", + "evidence": [ + "join_kind:inferred", + "interaction_id:6d2b89fd-a653-430c-b532-b0936d72eb42", + "turn_index:0", + "agent_id:None", + "parent_tool_call_id:None", + "model:gpt-5.6-luna", + "finish:stop", + "initiator:agent", + "order:1", + "call_id:copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/33cc6465-29e1-4a04-8bdb-00241474b4d2" + ] + }, + { + "usage_source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:15:sha256:c78b2b2ed28b8641c996bf9c84a05812d5a108776ffe091a54378f9ff79162d8", + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:15", + "stored_turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:793d3703-6f4a-4814-8877-34a7325848ce", + "agent_id": null, + "call_id": "copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/2c2e4ab8-f283-4837-957d-da992ba55e65", + "status": "matched", + "join_kind": "inferred", + "evidence": [ + "join_kind:inferred", + "interaction_id:793d3703-6f4a-4814-8877-34a7325848ce", + "turn_index:1", + "agent_id:None", + "parent_tool_call_id:None", + "model:gpt-5.6-luna", + "finish:tool_calls", + "initiator:user", + "order:2", + "call_id:copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/2c2e4ab8-f283-4837-957d-da992ba55e65" + ] + }, + { + "usage_source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:16:sha256:be2e28cbe7e6071667598eb2156481465762532a6652196b7735a46ef412c67c", + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:16", + "stored_turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:793d3703-6f4a-4814-8877-34a7325848ce", + "agent_id": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", + "call_id": "copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/df600539-fdd4-4de2-bd42-7f7ff2c952ab", + "status": "matched", + "join_kind": "inferred", + "evidence": [ + "join_kind:inferred", + "interaction_id:7c0fa097-c0e2-48da-b2b6-fcfc1ad83a6b", + "turn_index:1", + "agent_id:bf8cb9f3-2097-4db0-a3c8-78a2653b2106", + "parent_tool_call_id:call_qx4FH5DADTeT1qVLb37HNpBk", + "model:gpt-5.6-luna", + "finish:tool_calls", + "initiator:sub-agent", + "order:3", + "call_id:copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/df600539-fdd4-4de2-bd42-7f7ff2c952ab" + ] + }, + { + "usage_source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:17:sha256:92ca44e9db1e93336e94649492c5c59945ef4268ca1188f710b77e6e4b148435", + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:17", + "stored_turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:793d3703-6f4a-4814-8877-34a7325848ce", + "agent_id": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", + "call_id": "copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/450a1f4c-de11-4d37-bf84-08f63d29588f", + "status": "matched", + "join_kind": "inferred", + "evidence": [ + "join_kind:inferred", + "interaction_id:7c0fa097-c0e2-48da-b2b6-fcfc1ad83a6b", + "turn_index:1", + "agent_id:bf8cb9f3-2097-4db0-a3c8-78a2653b2106", + "parent_tool_call_id:call_qx4FH5DADTeT1qVLb37HNpBk", + "model:gpt-5.6-luna", + "finish:stop", + "initiator:sub-agent", + "order:4", + "call_id:copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/450a1f4c-de11-4d37-bf84-08f63d29588f" + ] + }, + { + "usage_source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:18:sha256:5fb5f685b5495f2fcc1874ad3b563a6750fa35c863af4647ea91225dd3ec907a", + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:18", + "stored_turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:793d3703-6f4a-4814-8877-34a7325848ce", + "agent_id": null, + "call_id": "copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/10001566-2704-4f75-add3-2c044373eaba", + "status": "matched", + "join_kind": "inferred", + "evidence": [ + "join_kind:inferred", + "interaction_id:793d3703-6f4a-4814-8877-34a7325848ce", + "turn_index:1", + "agent_id:None", + "parent_tool_call_id:None", + "model:gpt-5.6-luna", + "finish:stop", + "initiator:agent", + "order:5", + "call_id:copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/10001566-2704-4f75-add3-2c044373eaba" + ] + } + ], + "invariants": { + "call_count": 6, + "main_agent_call_count": 4, + "child_agent_call_count": 2, + "shutdown_usage_is_validation_only": true, + "checkpoint_usage_is_additive": false, + "auxiliary_model_calls_are_main_usage": false + } } diff --git a/tests/platforms/copilot/fixtures/reconciliation-cases/semantic-projection.json b/tests/platforms/copilot/fixtures/reconciliation-cases/semantic-projection.json index 0e6eefc..ced17b0 100644 --- a/tests/platforms/copilot/fixtures/reconciliation-cases/semantic-projection.json +++ b/tests/platforms/copilot/fixtures/reconciliation-cases/semantic-projection.json @@ -1,40 +1,276 @@ { + "note": "input_records are V1 SourceRecords. Pure-function tests may pass them directly to build_semantics. They are not a generated archive.", "input_records": [ { - "source_id": "copilot:transcript:source-a:session-a:event-user-1", + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/f07404d0-af52-4260-89fb-358a10e86034", "source_kind": "transcript", - "native_session_id": "session-a", - "ts": "2026-09-10T17:08:20.000Z", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:22.203Z", "observed_at": "2026-09-10T17:09:00.000Z", - "payload": {"event": "user.message", "data": {"interactionId": "interaction-1", "agentId": null}}, - "locator": {"event_id": "event-user-1", "generation": "file-a", "offset": 0} + "payload": { + "type": "user.message", + "data": { + "content": "Read alpha.txt and beta.txt with separate view calls in parallel and report their sum. Only read those two files.", + "messageId": "a856cb38-7609-45ab-8a55-645553155db3", + "supportedNativeDocumentMimeTypes": [], + "delivery": "idle", + "interactionId": "6d2b89fd-a653-430c-b532-b0936d72eb42", + "turnId": "0", + "parentAgentTaskId": "fd800f7a-8163-4304-9681-efde4731e879" + }, + "id": "f07404d0-af52-4260-89fb-358a10e86034", + "timestamp": "2026-09-10T17:08:22.203Z", + "parentId": "3942810f-1caf-4251-82fb-3bf72698147a", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 2122, + "byte_length": 548, + "native_event_id": "f07404d0-af52-4260-89fb-358a10e86034" + } }, { - "source_id": "copilot:transcript:source-a:session-a:event-assistant-1", + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", "source_kind": "transcript", - "native_session_id": "session-a", - "ts": "2026-09-10T17:08:24.000Z", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.503Z", "observed_at": "2026-09-10T17:09:00.000Z", - "payload": {"event": "assistant.message", "data": {"interactionId": "interaction-1", "agentId": null, "model": "gpt-5.6-luna", "toolCallIds": ["tool-read-a", "tool-read-b"]}}, - "locator": {"event_id": "event-assistant-1", "generation": "file-a", "offset": 101} + "payload": { + "type": "assistant.message", + "data": { + "messageId": "5a29b09d-5e7d-4603-975e-7801ce54232b", + "model": "gpt-5.6-luna", + "content": "", + "toolRequests": [ + { + "toolCallId": "call_YSSva4HCniiETlxdGGjcrHbh", + "name": "view", + "arguments": { + "path": "/fixture/workspace/alpha.txt" + }, + "type": "function", + "intentionSummary": "view the file at /fixture/workspace/alpha.txt." + }, + { + "toolCallId": "call_ayHplfzxjRFMTCpmTKEFhCSJ", + "name": "view", + "arguments": { + "path": "/fixture/workspace/beta.txt" + }, + "type": "function", + "intentionSummary": "view the file at /fixture/workspace/beta.txt." + } + ], + "interactionId": "6d2b89fd-a653-430c-b532-b0936d72eb42", + "turnId": "0", + "rte": true + }, + "id": "a4a17e63-7ba5-422f-8ee9-b495be417328", + "timestamp": "2026-09-10T17:08:24.503Z", + "parentId": "64f9e436-651f-4a9d-919c-a4d7abad2652", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 3731, + "byte_length": 793, + "native_event_id": "a4a17e63-7ba5-422f-8ee9-b495be417328" + } }, { - "source_id": "copilot:transcript:source-a:session-a:event-finish-1", + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/0080e44c-ad62-4288-b2b2-061ec2b73d80", "source_kind": "transcript", - "native_session_id": "session-a", - "ts": "2026-09-10T17:08:26.000Z", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.593Z", "observed_at": "2026-09-10T17:09:00.000Z", - "payload": {"event": "agent.stop", "data": {"interactionId": "interaction-1", "agentId": null, "reason": "tool_calls"}}, - "locator": {"event_id": "event-finish-1", "generation": "file-a", "offset": 202} + "payload": { + "type": "assistant.turn_end", + "data": { + "turnId": "0" + }, + "id": "0080e44c-ad62-4288-b2b2-061ec2b73d80", + "timestamp": "2026-09-10T17:08:24.593Z", + "parentId": "032cf87e-05b7-4e5f-96b9-1fda1a8242f5", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 9311, + "byte_length": 195, + "native_event_id": "0080e44c-ad62-4288-b2b2-061ec2b73d80" + } } ], "expected": { "events": [ - {"id": "copilot:event:copilot:transcript:source-a:session-a:event-user-1", "kind": "user_prompt", "ts": "2026-09-10T17:08:20.000Z", "source_ids": ["copilot:transcript:source-a:session-a:event-user-1"], "source_references": [{"source_id": "copilot:transcript:source-a:session-a:event-user-1", "source_kind": "transcript", "role": "user_prompt"}], "attributes": {"interaction_id": "interaction-1", "agent_id": null}}, - {"id": "copilot:event:copilot:transcript:source-a:session-a:event-assistant-1", "kind": "assistant_message", "ts": "2026-09-10T17:08:24.000Z", "source_ids": ["copilot:transcript:source-a:session-a:event-assistant-1"], "source_references": [{"source_id": "copilot:transcript:source-a:session-a:event-assistant-1", "source_kind": "transcript", "role": "assistant_message"}], "attributes": {"interaction_id": "interaction-1", "agent_id": null}} + { + "id": "copilot:event:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/f07404d0-af52-4260-89fb-358a10e86034", + "kind": "user_prompt", + "classification": "main", + "ts": "2026-09-10T17:08:22.203Z", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/f07404d0-af52-4260-89fb-358a10e86034" + ], + "source_references": [ + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/f07404d0-af52-4260-89fb-358a10e86034", + "source_kind": "transcript", + "role": "user_prompt" + } + ], + "attributes": { + "interaction_id": "6d2b89fd-a653-430c-b532-b0936d72eb42", + "agent_id": null + } + }, + { + "id": "copilot:event:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "kind": "assistant_message", + "classification": "main", + "ts": "2026-09-10T17:08:24.503Z", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328" + ], + "source_references": [ + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "source_kind": "transcript", + "role": "assistant_message" + } + ], + "attributes": { + "interaction_id": "6d2b89fd-a653-430c-b532-b0936d72eb42", + "agent_id": null + } + }, + { + "id": "copilot:event:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328:tool:call_YSSva4HCniiETlxdGGjcrHbh", + "kind": "tool_request", + "classification": "main", + "ts": "2026-09-10T17:08:24.503Z", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328" + ], + "source_references": [ + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "source_kind": "transcript", + "role": "tool_request" + } + ], + "attributes": { + "tool_call_id": "call_YSSva4HCniiETlxdGGjcrHbh", + "name": "view" + } + }, + { + "id": "copilot:event:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328:tool:call_ayHplfzxjRFMTCpmTKEFhCSJ", + "kind": "tool_request", + "classification": "main", + "ts": "2026-09-10T17:08:24.503Z", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328" + ], + "source_references": [ + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "source_kind": "transcript", + "role": "tool_request" + } + ], + "attributes": { + "tool_call_id": "call_ayHplfzxjRFMTCpmTKEFhCSJ", + "name": "view" + } + } + ], + "turns": [ + { + "turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42", + "start_ts": "2026-09-10T17:08:22.203Z", + "end_ts": "2026-09-10T17:08:24.593Z", + "input_message": "Read alpha.txt and beta.txt with separate view calls in parallel and report their sum. Only read those two files.", + "output_message": "", + "status": "completed", + "llm_calls": [ + { + "call_id": "copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "provider": "unknown", + "model": "gpt-5.6-luna", + "start_ts": "2026-09-10T17:08:24.503Z", + "end_ts": "2026-09-10T17:08:24.593Z", + "input_messages": [], + "output_messages": [], + "usage": {}, + "tool_calls": [ + { + "tool_call_id": "call_YSSva4HCniiETlxdGGjcrHbh", + "name": "view", + "start_ts": "2026-09-10T17:08:24.503Z", + "end_ts": "2026-09-10T17:08:24.593Z", + "attributes": {} + }, + { + "tool_call_id": "call_ayHplfzxjRFMTCpmTKEFhCSJ", + "name": "view", + "start_ts": "2026-09-10T17:08:24.503Z", + "end_ts": "2026-09-10T17:08:24.593Z", + "attributes": {} + } + ] + } + ], + "permission_requests": [], + "subagents": [], + "attributes": { + "interaction_id": "6d2b89fd-a653-430c-b532-b0936d72eb42", + "agent_id": null + }, + "accounting_calls": [] + } ], "call_candidates": [ - {"call_id": "copilot:call:copilot:transcript:source-a:session-a:event-assistant-1", "stored_turn_id": "copilot:turn:source-a:session-a:interaction-1", "interaction_id": "interaction-1", "agent_id": null, "parent_tool_call_id": null, "model": "gpt-5.6-luna", "source_ids": ["copilot:transcript:source-a:session-a:event-assistant-1", "copilot:transcript:source-a:session-a:event-finish-1"], "source_references": [{"source_id": "copilot:transcript:source-a:session-a:event-assistant-1", "source_kind": "transcript", "role": "assistant_message"}, {"source_id": "copilot:transcript:source-a:session-a:event-finish-1", "source_kind": "transcript", "role": "finish"}], "start_ts": "2026-09-10T17:08:24.000Z", "end_ts": "2026-09-10T17:08:26.000Z", "tool_call_ids": ["tool-read-a", "tool-read-b"], "finish_evidence": [{"source_id": "copilot:transcript:source-a:session-a:event-finish-1", "kind": "agent_stop", "value": "tool_calls"}]} + { + "call_id": "copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "stored_turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42", + "interaction_id": "6d2b89fd-a653-430c-b532-b0936d72eb42", + "agent_id": null, + "parent_tool_call_id": null, + "model": "gpt-5.6-luna", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/0080e44c-ad62-4288-b2b2-061ec2b73d80" + ], + "source_references": [ + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "source_kind": "transcript", + "role": "assistant_message" + }, + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/0080e44c-ad62-4288-b2b2-061ec2b73d80", + "source_kind": "transcript", + "role": "finish" + } + ], + "start_ts": "2026-09-10T17:08:24.503Z", + "end_ts": "2026-09-10T17:08:24.593Z", + "tool_call_ids": [ + "call_YSSva4HCniiETlxdGGjcrHbh", + "call_ayHplfzxjRFMTCpmTKEFhCSJ" + ], + "finish_evidence": [ + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/0080e44c-ad62-4288-b2b2-061ec2b73d80", + "kind": "assistant_turn_end", + "value": null + } + ] + } ], "pending": [], "diagnostics": [] diff --git a/tests/platforms/copilot/fixtures/reconciliation-cases/storage.json b/tests/platforms/copilot/fixtures/reconciliation-cases/storage.json index e4ab336..e898dbc 100644 --- a/tests/platforms/copilot/fixtures/reconciliation-cases/storage.json +++ b/tests/platforms/copilot/fixtures/reconciliation-cases/storage.json @@ -1,27 +1,158 @@ { "projection_state": { - "schema_version": 2, - "archive_source_ids": ["copilot:transcript:source-a:session-a:event-user-1", "copilot:database:source-a:session-a:assistant_usage_events:db-a:13:sha256:usage-13"], - "semantic_state": {"open_interactions": {}}, - "accounting_state": {"logical_calls": {"copilot:usage:source-a:assistant_usage_events:db-a:13": "sha256:usage-13"}}, + "projection_schema_version": 2, + "archive_source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/f07404d0-af52-4260-89fb-358a10e86034", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47" + ], + "semantic_state": { + "open_interactions": {} + }, + "accounting_state": { + "logical_calls": { + "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13": { + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13", + "generation": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "content_revision": "sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "metrics_digest": "sha256:feee488caa281c39aa67f248f5e9d45220c1db1ee47b1f733b2eed66de686a53", + "usage_source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47" + } + } + }, "projection_revision": "sha256:derived-state-example" }, - "commit_result": {"events": 2, "turns": 1, "usage": 1, "attributions": 1, "pending": 0, "diagnostics": 0}, + "open_interactions_key": "|", + "open_interactions_example": { + "6d2b89fd-a653-430c-b532-b0936d72eb42|main": { + "interaction_id": "6d2b89fd-a653-430c-b532-b0936d72eb42", + "agent_id": null, + "stored_turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/f07404d0-af52-4260-89fb-358a10e86034" + ], + "last_event_source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/f07404d0-af52-4260-89fb-358a10e86034", + "start_ts": "2026-09-10T17:08:22.203Z", + "pending_tool_call_ids": [] + } + }, + "commit_result": { + "events": 2, + "turns": 1, + "usage": 1, + "attributions": 1, + "pending": 0, + "diagnostics": 0 + }, "read_projected_turns": [ { - "id": "copilot-source-a-session-a:copilot:turn:source-a:session-a:interaction-1", - "turn_id": "copilot:turn:source-a:session-a:interaction-1", - "session_id": "copilot-source-a-session-a", + "id": "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42", + "turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42", + "session_id": "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", "platform": "copilot", - "cwd": "/workspace/example", - "start_seq": 12, - "end_seq": 24, - "start_ts": "2026-09-10T17:08:20.000Z", - "end_ts": "2026-09-10T17:08:26.000Z", + "cwd": "/fixture/workspace", + "start_seq": 0, + "end_seq": 1, + "start_ts": "2026-09-10T17:08:22.203Z", + "end_ts": "2026-09-10T17:08:24.503Z", "events": [ - {"id": "copilot:event:copilot:transcript:source-a:session-a:event-user-1", "kind": "user_prompt", "source_ids": ["copilot:transcript:source-a:session-a:event-user-1"]}, - {"id": "copilot:event:copilot:transcript:source-a:session-a:event-assistant-1", "kind": "assistant_message", "source_ids": ["copilot:transcript:source-a:session-a:event-assistant-1"]} + { + "t": "copilot_transcript", + "ts": "2026-09-10T17:08:22.203Z", + "seq": 0, + "data": { + "schema_version": 1, + "source_record": { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/f07404d0-af52-4260-89fb-358a10e86034", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:22.203Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "user.message", + "data": { + "content": "Read alpha.txt and beta.txt with separate view calls in parallel and report their sum. Only read those two files.", + "messageId": "a856cb38-7609-45ab-8a55-645553155db3", + "supportedNativeDocumentMimeTypes": [], + "delivery": "idle", + "interactionId": "6d2b89fd-a653-430c-b532-b0936d72eb42", + "turnId": "0", + "parentAgentTaskId": "fd800f7a-8163-4304-9681-efde4731e879" + }, + "id": "f07404d0-af52-4260-89fb-358a10e86034", + "timestamp": "2026-09-10T17:08:22.203Z", + "parentId": "3942810f-1caf-4251-82fb-3bf72698147a", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 2122, + "byte_length": 548, + "native_event_id": "f07404d0-af52-4260-89fb-358a10e86034" + } + } + } + }, + { + "t": "copilot_transcript", + "ts": "2026-09-10T17:08:24.503Z", + "seq": 1, + "data": { + "schema_version": 1, + "source_record": { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.503Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "assistant.message", + "data": { + "messageId": "5a29b09d-5e7d-4603-975e-7801ce54232b", + "model": "gpt-5.6-luna", + "content": "", + "toolRequests": [ + { + "toolCallId": "call_YSSva4HCniiETlxdGGjcrHbh", + "name": "view", + "arguments": { + "path": "/fixture/workspace/alpha.txt" + }, + "type": "function", + "intentionSummary": "view the file at /fixture/workspace/alpha.txt." + }, + { + "toolCallId": "call_ayHplfzxjRFMTCpmTKEFhCSJ", + "name": "view", + "arguments": { + "path": "/fixture/workspace/beta.txt" + }, + "type": "function", + "intentionSummary": "view the file at /fixture/workspace/beta.txt." + } + ], + "interactionId": "6d2b89fd-a653-430c-b532-b0936d72eb42", + "turnId": "0", + "rte": true + }, + "id": "a4a17e63-7ba5-422f-8ee9-b495be417328", + "timestamp": "2026-09-10T17:08:24.503Z", + "parentId": "64f9e436-651f-4a9d-919c-a4d7abad2652", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 3731, + "byte_length": 793, + "native_event_id": "a4a17e63-7ba5-422f-8ee9-b495be417328" + } + } + } + } ] } - ] + ], + "events_note": "events are Store records with seq/t/ts/data. data is the V1 envelope {schema_version, source_record}." } diff --git a/tests/platforms/copilot/fixtures/reconciliation-cases/transport.json b/tests/platforms/copilot/fixtures/reconciliation-cases/transport.json index 27fe33a..eac2aea 100644 --- a/tests/platforms/copilot/fixtures/reconciliation-cases/transport.json +++ b/tests/platforms/copilot/fixtures/reconciliation-cases/transport.json @@ -1,23 +1,186 @@ { + "rules": { + "call_id_set": "export tokens on the matching LlmCallSpanDict chat span", + "call_id_null": "export on a user-turn or agent accounting span, never a fabricated turn", + "llm_call_usage": "LlmCallSpanDict.usage stays empty whenever accounting_calls is present", + "never_both": "matched chat span and fallback accounting span are mutually exclusive", + "span_id_chat": "existing chat span id for call_id; no extra span", + "span_id_turn": "accounting:{session_id}:{turn_id}:{accounting_id}", + "span_id_session": "accounting:{session_id}:{accounting_id}" + }, "turn_accounting_calls": [ { - "accounting_id": "copilot:usage:source-a:assistant_usage_events:db-a:13", - "usage": {"session_id": "copilot-source-a-session-a", "seq": 13, "call_id": "copilot:usage:source-a:assistant_usage_events:db-a:13", "ts": "2026-09-10T17:08:24.498Z", "platform": "copilot", "gen_ai.conversation.id": "copilot-source-a-session-a", "gen_ai.provider.name": "unknown", "gen_ai.operation.name": "chat", "gen_ai.response.model": "gpt-5.6-luna", "gen_ai.usage.input_tokens": 6452, "gen_ai.usage.output_tokens": 107}, + "accounting_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13", + "usage": { + "session_id": "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "seq": 0, + "call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13", + "ts": "2026-09-10T17:08:24.498Z", + "platform": "copilot", + "gen_ai.conversation.id": "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "gen_ai.provider.name": "unknown", + "gen_ai.operation.name": "chat", + "gen_ai.response.model": "gpt-5.6-luna", + "gen_ai.usage.input_tokens": 6452, + "gen_ai.usage.output_tokens": 107, + "gen_ai.usage.cache_read.input_tokens": 0, + "gen_ai.usage.cache_creation.input_tokens": 6449, + "gen_ai.usage.reasoning.output_tokens": 29 + }, "attribution_status": "matched", "agent_id": null, - "attributes": {"copilot.logical_call_id": "copilot:usage:source-a:assistant_usage_events:db-a:13", "copilot.evidence": ["interaction_id:interaction-1", "model:gpt-5.6-luna", "finish:tool_calls"]} + "call_id": "copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "attributes": { + "accounting.destination": "chat-span", + "copilot.logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13", + "copilot.evidence": [ + "join_kind:inferred", + "interaction_id:6d2b89fd-a653-430c-b532-b0936d72eb42", + "model:gpt-5.6-luna", + "turn_index:0", + "finish:tool_calls", + "initiator:user", + "order:0" + ] + } } ], + "turn_owned_unmatched_job": { + "job_id": "accounting:copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42:copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:14", + "kind": "turn_accounting", + "session_id": "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42", + "accounting_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:14", + "destination": "turn-accounting-span", + "attempt": 0, + "state": "queued", + "usage": { + "session_id": "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "seq": 0, + "call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:14", + "ts": "2026-09-10T17:08:25.618Z", + "platform": "copilot", + "gen_ai.conversation.id": "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "gen_ai.provider.name": "unknown", + "gen_ai.operation.name": "chat", + "gen_ai.response.model": "gpt-5.6-luna", + "gen_ai.usage.input_tokens": 6587, + "gen_ai.usage.output_tokens": 5, + "gen_ai.usage.cache_read.input_tokens": 6449, + "gen_ai.usage.cache_creation.input_tokens": 135, + "gen_ai.usage.reasoning.output_tokens": 0 + }, + "attribution_status": "pending", + "agent_id": null, + "span_id": "accounting:copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42:copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:14" + }, "session_accounting_job": { - "job_id": "accounting:copilot-source-a-session-a:copilot:usage:source-a:assistant_usage_events:db-a:14", + "job_id": "accounting:copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:14", "kind": "session_accounting", - "session_id": "copilot-source-a-session-a", - "accounting_id": "copilot:usage:source-a:assistant_usage_events:db-a:14", + "session_id": "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "accounting_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:14", "destination": "session-accounting-span", "attempt": 0, "state": "queued", - "usage": {"session_id": "copilot-source-a-session-a", "seq": 14, "call_id": "copilot:usage:source-a:assistant_usage_events:db-a:14", "ts": "2026-09-10T17:08:25.618Z", "platform": "copilot", "gen_ai.conversation.id": "copilot-source-a-session-a", "gen_ai.provider.name": "unknown", "gen_ai.operation.name": "chat", "gen_ai.response.model": "gpt-5.6-luna", "gen_ai.usage.input_tokens": 6587, "gen_ai.usage.output_tokens": 5}, - "attribution_status": "pending" + "usage": { + "session_id": "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "seq": 0, + "call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:14", + "ts": "2026-09-10T17:08:25.618Z", + "platform": "copilot", + "gen_ai.conversation.id": "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "gen_ai.provider.name": "unknown", + "gen_ai.operation.name": "chat", + "gen_ai.response.model": "gpt-5.6-luna", + "gen_ai.usage.input_tokens": 6587, + "gen_ai.usage.output_tokens": 5, + "gen_ai.usage.cache_read.input_tokens": 6449, + "gen_ai.usage.cache_creation.input_tokens": 135, + "gen_ai.usage.reasoning.output_tokens": 0 + }, + "attribution_status": "pending", + "span_id": "accounting:copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:14" + }, + "ledger_entry": { + "accounting_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:14", + "destination": "session-accounting-span", + "span_id": "accounting:copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:14", + "emitted": false, + "last_error": null }, - "ledger_entry": {"accounting_id": "copilot:usage:source-a:assistant_usage_events:db-a:14", "destination": "session-accounting-span", "emitted": false, "last_error": null} + "turn_span_with_accounting_calls": { + "turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42", + "start_ts": "2026-09-10T17:08:22.203Z", + "end_ts": "2026-09-10T17:08:25.626Z", + "input_message": "Read alpha.txt and beta.txt with separate view calls in parallel and report their sum. Only read those two files.", + "output_message": "42", + "status": "completed", + "llm_calls": [ + { + "call_id": "copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "provider": "unknown", + "model": "gpt-5.6-luna", + "start_ts": "2026-09-10T17:08:24.503Z", + "end_ts": "2026-09-10T17:08:24.593Z", + "input_messages": [], + "output_messages": [], + "usage": {}, + "tool_calls": [] + } + ], + "permission_requests": [], + "subagents": [], + "attributes": { + "interaction_id": "6d2b89fd-a653-430c-b532-b0936d72eb42", + "agent_id": null + }, + "accounting_calls": [ + { + "accounting_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13", + "usage": { + "session_id": "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "seq": 0, + "call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13", + "ts": "2026-09-10T17:08:24.498Z", + "platform": "copilot", + "gen_ai.conversation.id": "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "gen_ai.provider.name": "unknown", + "gen_ai.operation.name": "chat", + "gen_ai.response.model": "gpt-5.6-luna", + "gen_ai.usage.input_tokens": 6452, + "gen_ai.usage.output_tokens": 107, + "gen_ai.usage.cache_read.input_tokens": 0, + "gen_ai.usage.cache_creation.input_tokens": 6449, + "gen_ai.usage.reasoning.output_tokens": 29 + }, + "attribution_status": "matched", + "agent_id": null, + "call_id": "copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "attributes": {"accounting.destination": "chat-span"} + }, + { + "accounting_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:14", + "usage": { + "session_id": "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "seq": 0, + "call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:14", + "ts": "2026-09-10T17:08:25.618Z", + "platform": "copilot", + "gen_ai.conversation.id": "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "gen_ai.provider.name": "unknown", + "gen_ai.operation.name": "chat", + "gen_ai.response.model": "gpt-5.6-luna", + "gen_ai.usage.input_tokens": 6587, + "gen_ai.usage.output_tokens": 5, + "gen_ai.usage.cache_read.input_tokens": 6449, + "gen_ai.usage.cache_creation.input_tokens": 135, + "gen_ai.usage.reasoning.output_tokens": 0 + }, + "attribution_status": "pending", + "agent_id": null, + "call_id": null, + "attributes": {"accounting.destination": "turn-accounting-span"} + } + ] + } } From db061db46b0cee593757f91a51afd3c3b7b48f48 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Fri, 11 Sep 2026 15:27:17 -0700 Subject: [PATCH 47/88] Implement Copilot semantic reconstruction --- src/thirdeye/platforms/copilot/events.py | 187 ++++++++++++++++++++++ src/thirdeye/platforms/copilot/tracing.py | 30 ++++ src/thirdeye/platforms/copilot/turns.py | 186 +++++++++++++++++++++ 3 files changed, 403 insertions(+) create mode 100644 src/thirdeye/platforms/copilot/events.py create mode 100644 src/thirdeye/platforms/copilot/tracing.py create mode 100644 src/thirdeye/platforms/copilot/turns.py diff --git a/src/thirdeye/platforms/copilot/events.py b/src/thirdeye/platforms/copilot/events.py new file mode 100644 index 0000000..5794071 --- /dev/null +++ b/src/thirdeye/platforms/copilot/events.py @@ -0,0 +1,187 @@ +"""Lossless semantic normalization for archived Copilot source records. + +This module deliberately does not correlate records across sources. In +particular, a hook observation has no invocation ID in the observed corpus, +so it remains an observation instead of becoming a guessed tool span. +""" + +from __future__ import annotations + +from copy import deepcopy +from typing import Any + +from .types import NormalizedEvent, SourceRecord, SourceReference, SourceReferenceRole + + +def record_type(record: SourceRecord) -> str | None: + """Return the native event name carried by an archived record.""" + payload = record.get("payload", {}) + if not isinstance(payload, dict): + return None + value = payload.get("type") if record.get("source_kind") != "hook" else payload.get("event") + return value if isinstance(value, str) and value else None + + +def record_data(record: SourceRecord) -> dict[str, Any]: + """Return the native data mapping without rewriting the raw record.""" + payload = record.get("payload", {}) + if not isinstance(payload, dict): + return {} + if record.get("source_kind") == "hook": + data = payload.get("hook_payload") + else: + data = payload.get("data") + return data if isinstance(data, dict) else {} + + +def agent_id(record: SourceRecord) -> str | None: + payload = record.get("payload", {}) + if isinstance(payload, dict) and isinstance(payload.get("agentId"), str): + return payload["agentId"] + data = record_data(record) + for key in ("agentId", "agent_id"): + value = data.get(key) + if value is not None and str(value): + return str(value) + return None + + +def interaction_id(record: SourceRecord) -> str | None: + value = record_data(record).get("interactionId") + return str(value) if value is not None and str(value) else None + + +def tool_call_id(record: SourceRecord) -> str | None: + data = record_data(record) + for key in ("toolCallId", "tool_call_id"): + value = data.get(key) + if value is not None and str(value): + return str(value) + return None + + +def _reference(record: SourceRecord, role: SourceReferenceRole) -> SourceReference: + return {"source_id": record["source_id"], "source_kind": record["source_kind"], "role": role} + + +def _event( + record: SourceRecord, + kind: str, + role: SourceReferenceRole, + *, + classification: str = "main", + suffix: str = "", + attributes: dict[str, Any] | None = None, +) -> NormalizedEvent: + return { + "id": f"copilot:event:{record['source_id']}{suffix}", + "kind": kind, # type: ignore[typeddict-item] + "classification": classification, # type: ignore[typeddict-item] + "ts": record.get("ts"), + "source_ids": [record["source_id"]], + "source_references": [_reference(record, role)], + "attributes": attributes or {}, + } + + +def _identity_attributes(record: SourceRecord) -> dict[str, Any]: + data = record_data(record) + return { + "interaction_id": interaction_id(record), + "agent_id": agent_id(record), + "parent_tool_call_id": data.get("parentToolCallId"), + "turn_id": data.get("turnId"), + } + + +def normalize_record(record: SourceRecord) -> list[NormalizedEvent]: + """Normalize one record while retaining a direct pointer to its evidence.""" + native_type = record_type(record) + data = record_data(record) + identity = _identity_attributes(record) + if native_type is None: + return [_event(record, "unknown", "hook", attributes={"raw_payload": deepcopy(record.get("payload"))})] + + if native_type == "user.message": + attrs = {**identity, "delivery": data.get("delivery"), "source": data.get("source")} + return [_event(record, "user_prompt", "user_prompt", attributes=attrs)] + if native_type == "assistant.message": + attrs = {**identity, "model": data.get("model"), "phase": data.get("phase")} + events = [_event(record, "assistant_message", "assistant_message", attributes=attrs)] + requests = data.get("toolRequests") + if isinstance(requests, list): + for request in requests: + if not isinstance(request, dict) or not request.get("toolCallId"): + continue + call_id = str(request["toolCallId"]) + events.append( + _event( + record, + "tool_request", + "tool_request", + suffix=f":tool:{call_id}", + attributes={ + **identity, + "tool_call_id": call_id, + "name": request.get("name"), + "arguments": deepcopy(request.get("arguments")), + "intention_summary": request.get("intentionSummary"), + }, + ) + ) + return events + if native_type == "tool.execution_start": + return [_event(record, "tool_execution_start", "tool_execution", attributes={**identity, "tool_call_id": tool_call_id(record), "name": data.get("toolName"), "arguments": deepcopy(data.get("arguments"))})] + if native_type == "tool.execution_complete": + kind = "tool_execution_complete" if data.get("success") is not False else "tool_execution_failure" + return [_event(record, kind, "tool_result", attributes={**identity, "tool_call_id": tool_call_id(record), "success": data.get("success"), "result": deepcopy(data.get("result"))})] + if native_type in {"permission.request", "permissionRequest"}: + return [_event(record, "permission_request", "permission_request", attributes={**identity, "tool_name": data.get("toolName"), "arguments": deepcopy(data.get("toolArgs"))})] + if native_type in {"permission.decision", "permissionDecision"}: + return [_event(record, "permission_decision", "permission_decision", attributes={**identity, "decision": data.get("decision"), "tool_name": data.get("toolName")})] + if native_type in {"notification", "session.notification", "hook.start", "hook.end"}: + return [_event(record, "notification", "hook", attributes={**identity, "native_type": native_type, **deepcopy(data)})] + if native_type in {"prompt.transformation", "prompt.transformed"}: + return [_event(record, "prompt_transformation", "hook", attributes={**identity, **deepcopy(data)})] + if native_type in {"assistant.abort", "session.abort", "abort"}: + return [_event(record, "abort", "finish", attributes=identity)] + if "error" in native_type.lower(): + return [_event(record, "error", "finish", attributes={**identity, "message": data.get("message") or data.get("error")})] + if native_type in {"session.start", "session.resume"}: + return [_event(record, "session_start", "hook", attributes=deepcopy(data))] + if native_type in {"session.end", "session.shutdown", "session.close"}: + kind = "session_shutdown" if native_type == "session.shutdown" else "session_end" + return [_event(record, kind, "shutdown", classification="shutdown_validation" if kind == "session_shutdown" else "main", attributes=deepcopy(data))] + if native_type in {"session.usage_checkpoint", "preCompact", "session.compaction", "context.compaction"}: + return [_event(record, "compaction", "checkpoint" if native_type == "session.usage_checkpoint" else "compaction", classification="checkpoint" if native_type == "session.usage_checkpoint" else "main", attributes=deepcopy(data))] + if native_type.startswith("model."): + return [_event(record, "auxiliary_model_call", "auxiliary_model", classification="title_generation", attributes={"native_type": native_type, **deepcopy(data)})] + if native_type == "subagent.started": + return [_event(record, "subagent_started", "nested_child", attributes={**identity, "tool_call_id": tool_call_id(record), **deepcopy(data)})] + if native_type == "subagent.completed": + return [_event(record, "subagent_completed", "nested_child", attributes={**identity, "tool_call_id": tool_call_id(record), **deepcopy(data)})] + + hook_kinds = { + "permissionRequest": ("permission_request", "permission_request"), + "preToolUse": ("tool_execution_start", "hook"), + "postToolUse": ("tool_execution_complete", "hook"), + "postToolUseFailure": ("tool_execution_failure", "hook"), + "notification": ("notification", "hook"), + "userPromptSubmitted": ("prompt_transformation", "hook"), + "agentStop": ("session_end", "finish"), + "subagentStart": ("subagent_started", "nested_child"), + "subagentStop": ("subagent_completed", "nested_child"), + "sessionStart": ("session_start", "hook"), + "sessionEnd": ("session_end", "hook"), + } + mapped = hook_kinds.get(native_type) + if mapped is not None: + kind, role = mapped + attrs = {**identity, **deepcopy(data)} + return [_event(record, kind, role, attributes=attrs)] + return [_event(record, "unknown", "hook", attributes={"native_type": native_type, "raw_payload": deepcopy(record.get("payload"))})] + + +def normalize_records(records: list[SourceRecord]) -> list[NormalizedEvent]: + """Normalize archive records in replay order without deduplicating evidence.""" + return [event for record in records for event in normalize_record(record)] diff --git a/src/thirdeye/platforms/copilot/tracing.py b/src/thirdeye/platforms/copilot/tracing.py new file mode 100644 index 0000000..ba52588 --- /dev/null +++ b/src/thirdeye/platforms/copilot/tracing.py @@ -0,0 +1,30 @@ +"""Pure semantic projection for V1 Copilot archives.""" + +from __future__ import annotations + +from typing import Any + +from .events import normalize_records +from .turns import build_turns +from .types import SemanticProjection, SourceRecord + + +def build_semantics( + records: list[SourceRecord], prior_state: dict[str, Any] +) -> tuple[SemanticProjection, dict[str, Any]]: + """Replay immutable source records into semantic events and main turns. + + ``prior_state`` is intentionally not treated as evidence. It is only a + retained description of still-open interactions for incremental callers; + replaying the complete archive always produces the authoritative result. + """ + events = normalize_records(records) + turns, call_candidates, pending, semantic_state = build_turns(records) + old_open = prior_state.get("open_interactions") if isinstance(prior_state, dict) else None + if isinstance(old_open, dict): + for key, item in old_open.items(): + if key not in semantic_state["open_interactions"] and isinstance(item, dict): + # State cannot prove a new association, but retaining an + # unfinished prior partition avoids silently claiming closure. + semantic_state["open_interactions"][key] = item + return ({"events": events, "turns": turns, "call_candidates": call_candidates, "pending": pending, "diagnostics": []}, semantic_state) diff --git a/src/thirdeye/platforms/copilot/turns.py b/src/thirdeye/platforms/copilot/turns.py new file mode 100644 index 0000000..e7122ed --- /dev/null +++ b/src/thirdeye/platforms/copilot/turns.py @@ -0,0 +1,186 @@ +"""Explicit-identity Copilot interaction and recursive child-tree assembly.""" + +from __future__ import annotations + +from copy import deepcopy +from typing import Any + +from thirdeye.tracing.model import LlmCallSpanDict, ToolCallSpanDict, TurnSpanDict + +from .events import agent_id, interaction_id, record_data, record_type, tool_call_id +from .types import CallCandidate, PendingItem, SourceRecord, SourceReference + + +def _source_key(record: SourceRecord) -> str | None: + source_id = record["source_id"] + if source_id.startswith("hook/") or source_id.startswith("copilot-db:"): + return None + return source_id.split("/", 1)[0] or None + + +def _turn_id(record: SourceRecord, interaction: str) -> str: + source_key = _source_key(record) + if source_key is None: + return f"copilot:turn:unknown:{record['native_session_id']}:{interaction}" + return f"copilot:turn:{source_key}:{record['native_session_id']}:{interaction}" + + +def _reference(record: SourceRecord, role: str) -> SourceReference: + return {"source_id": record["source_id"], "source_kind": record["source_kind"], "role": role} # type: ignore[typeddict-item] + + +def _key(interaction: str, agent: str | None) -> str: + return f"{interaction}|{agent or 'main'}" + + +def build_turns(records: list[SourceRecord]) -> tuple[list[TurnSpanDict], list[CallCandidate], list[PendingItem], dict[str, Any]]: + """Build completed main turns and recursive child spans from transcript IDs only.""" + interactions: dict[str, dict[str, Any]] = {} + active_native_turns: dict[tuple[str | None, str], str] = {} + child_parent: dict[str, tuple[str, str]] = {} + tool_owner: dict[str, str] = {} + calls: dict[str, dict[str, Any]] = {} + tool_spans: dict[str, ToolCallSpanDict] = {} + pending: list[PendingItem] = [] + + def ensure(record: SourceRecord, interaction: str, agent: str | None) -> dict[str, Any]: + key = _key(interaction, agent) + if key not in interactions: + interactions[key] = { + "key": key, "interaction": interaction, "agent": agent, "turn_id": _turn_id(record, interaction), + "source_ids": [], "start_ts": None, "end_ts": None, "input": "", "output": "", "calls": [], + "permission_requests": [], "status": "completed", "complete": False, "parent": None, + } + item = interactions[key] + item["source_ids"].append(record["source_id"]) + item["start_ts"] = item["start_ts"] or record.get("ts") + return item + + for record in records: + if record.get("source_kind") != "transcript": + continue + native_type = record_type(record) + data = record_data(record) + agent = agent_id(record) + interaction = interaction_id(record) + + if native_type == "subagent.started": + child = agent_id(record) + parent_call = tool_call_id(record) + if child and parent_call and parent_call in tool_owner: + child_parent[child] = (tool_owner[parent_call], parent_call) + continue + if native_type == "user.message": + if interaction is None: + pending.append({"id": f"pending:identity:{record['source_id']}", "kind": "missing_identity", "reason": "user message has no interactionId", "source_ids": [record["source_id"]], "evidence": ["native_type:user.message"]}) + continue + item = ensure(record, interaction, agent) + item["input"] = str(data.get("content") or item["input"]) + continue + if native_type == "assistant.turn_start": + if interaction is None: + pending.append({"id": f"pending:identity:{record['source_id']}", "kind": "missing_identity", "reason": "assistant turn has no interactionId", "source_ids": [record["source_id"]], "evidence": ["native_type:assistant.turn_start"]}) + continue + item = ensure(record, interaction, agent) + native_turn = data.get("turnId") + if native_turn is not None: + active_native_turns[(agent, str(native_turn))] = item["key"] + continue + if native_type == "assistant.message": + if interaction is None: + pending.append({"id": f"pending:identity:{record['source_id']}", "kind": "missing_identity", "reason": "assistant message has no interactionId", "source_ids": [record["source_id"]], "evidence": ["native_type:assistant.message"]}) + continue + item = ensure(record, interaction, agent) + call_id = f"copilot:call:{record['source_id']}" + requests = data.get("toolRequests") if isinstance(data.get("toolRequests"), list) else [] + requested_ids = [str(request["toolCallId"]) for request in requests if isinstance(request, dict) and request.get("toolCallId")] + candidate = {"call_id": call_id, "stored_turn_id": item["turn_id"], "interaction_id": interaction, "agent_id": agent, "parent_tool_call_id": data.get("parentToolCallId"), "model": data.get("model"), "source_ids": [record["source_id"]], "source_references": [_reference(record, "assistant_message")], "start_ts": record.get("ts"), "end_ts": None, "tool_call_ids": requested_ids, "finish_evidence": []} + calls[call_id] = candidate + item["calls"].append(call_id) + content = data.get("content") + if isinstance(content, str) and content: + item["output"] = content + for request in requests: + if not isinstance(request, dict) or not request.get("toolCallId"): + continue + call = str(request["toolCallId"]) + tool_owner[call] = item["key"] + tool_spans[call] = {"tool_call_id": call, "name": str(request.get("name") or ""), "start_ts": str(record.get("ts") or ""), "end_ts": "", "attributes": {"arguments": deepcopy(request.get("arguments")), "intention_summary": request.get("intentionSummary"), "request_source_id": record["source_id"]}} + continue + if native_type == "tool.execution_start": + call = tool_call_id(record) + if call and call in tool_spans: + span = tool_spans[call] + span["start_ts"] = str(record.get("ts") or span["start_ts"]) + span["attributes"].update({"arguments": deepcopy(data.get("arguments")), "execution_start_source_id": record["source_id"]}) + elif call: + pending.append({"id": f"pending:tool:{call}", "kind": "incomplete_tool_pair", "reason": "tool execution start has no requesting assistant message", "source_ids": [record["source_id"]], "evidence": [f"tool_call_id:{call}"]}) + continue + if native_type == "tool.execution_complete": + call = tool_call_id(record) + if call and call in tool_spans: + span = tool_spans[call] + span["end_ts"] = str(record.get("ts") or "") + span["attributes"].update({"result": deepcopy(data.get("result")), "success": data.get("success"), "execution_result_source_id": record["source_id"]}) + elif call: + pending.append({"id": f"pending:tool:{call}", "kind": "incomplete_tool_pair", "reason": "tool result has no requesting assistant message", "source_ids": [record["source_id"]], "evidence": [f"tool_call_id:{call}"]}) + continue + if native_type in {"assistant.abort", "session.abort", "abort"} or (native_type and "error" in native_type.lower()): + native_turn = data.get("turnId") + owner = active_native_turns.get((agent, str(native_turn))) if native_turn is not None else None + if owner and owner in interactions: + interactions[owner]["status"] = "interrupted" if native_type in {"assistant.abort", "session.abort", "abort"} else "errored" + interactions[owner]["end_ts"] = record.get("ts") + continue + if native_type == "assistant.turn_end": + native_turn = data.get("turnId") + owner = active_native_turns.pop((agent, str(native_turn)), None) if native_turn is not None else None + if owner is None: + pending.append({"id": f"pending:identity:{record['source_id']}", "kind": "missing_identity", "reason": "assistant.turn_end cannot be assigned without agent and active turn identity", "source_ids": [record["source_id"]], "evidence": [f"turn_id:{native_turn}"]}) + continue + item = interactions[owner] + item["source_ids"].append(record["source_id"]) + item["end_ts"] = record.get("ts") + # A turn end completes a model cycle. It only completes the user + # interaction when the cycle contains the final answer (or was + # explicitly aborted/errored); tool cycles remain open. + last_call = calls.get(item["calls"][-1]) if item["calls"] else None + if last_call is not None: + last_call["end_ts"] = record.get("ts") + last_call["source_ids"].append(record["source_id"]) + last_call["source_references"].append(_reference(record, "finish")) + last_call["finish_evidence"].append({"source_id": record["source_id"], "kind": "assistant_turn_end", "value": None}) + if not last_call["tool_call_ids"] and item["output"]: + item["complete"] = True + continue + + def call_span(candidate: dict[str, Any]) -> LlmCallSpanDict: + attached = [tool_spans[tool] for tool in candidate["tool_call_ids"] if tool in tool_spans] + return {"call_id": candidate["call_id"], "provider": "unknown", "model": str(candidate.get("model") or "unknown"), "start_ts": str(candidate.get("start_ts") or ""), "end_ts": str(candidate.get("end_ts") or candidate.get("start_ts") or ""), "input_messages": [], "output_messages": [], "usage": {}, "tool_calls": attached} + + spans: dict[str, TurnSpanDict] = {} + for key, item in interactions.items(): + if not item["complete"]: + continue + spans[key] = {"turn_id": item["turn_id"], "start_ts": str(item["start_ts"] or ""), "end_ts": str(item["end_ts"] or item["start_ts"] or ""), "input_message": item["input"], "output_message": item["output"], "status": item["status"], "llm_calls": [call_span(calls[call]) for call in item["calls"]], "permission_requests": item["permission_requests"], "subagents": [], "attributes": {"interaction_id": item["interaction"], "agent_id": item["agent"]}, "accounting_calls": []} + + for child, (parent_key, parent_call) in child_parent.items(): + child_items = [item for item in interactions.values() if item["agent"] == child and item["complete"]] + for item in child_items: + child_span = spans.get(item["key"]) + parent_span = spans.get(parent_key) + if child_span is not None and parent_span is not None: + child_span["attributes"]["parent_tool_call_id"] = parent_call + parent_span["subagents"].append(child_span) + + open_state: dict[str, Any] = {} + for item in interactions.values(): + if item["complete"]: + continue + open_state[item["key"]] = {"interaction_id": item["interaction"], "agent_id": item["agent"], "stored_turn_id": item["turn_id"], "source_ids": item["source_ids"], "last_event_source_id": item["source_ids"][-1] if item["source_ids"] else None, "start_ts": item["start_ts"], "pending_tool_call_ids": [tool for call in item["calls"] for tool in calls[call]["tool_call_ids"] if not tool_spans.get(tool, {}).get("end_ts")]} + pending.append({"id": f"pending:{item['turn_id']}", "kind": "open_interaction", "reason": "user interaction has no completed final assistant turn", "source_ids": item["source_ids"], "evidence": [f"interaction_id:{item['interaction']}"]}) + + main_turns = [span for key, span in spans.items() if interactions[key]["agent"] is None] + main_turns.sort(key=lambda turn: turn["start_ts"]) + call_candidates: list[CallCandidate] = list(calls.values()) # type: ignore[assignment] + return main_turns, call_candidates, pending, {"open_interactions": open_state} From 60a0e547a5506e9647731cadc2ca53307e29b3c5 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Fri, 11 Sep 2026 15:27:56 -0700 Subject: [PATCH 48/88] Add Copilot usage normalization --- src/thirdeye/platforms/copilot/usage.py | 399 ++++++++++++++++++++++++ 1 file changed, 399 insertions(+) create mode 100644 src/thirdeye/platforms/copilot/usage.py diff --git a/src/thirdeye/platforms/copilot/usage.py b/src/thirdeye/platforms/copilot/usage.py new file mode 100644 index 0000000..b38a1fb --- /dev/null +++ b/src/thirdeye/platforms/copilot/usage.py @@ -0,0 +1,399 @@ +"""Normalize archived Copilot database usage without assigning it to messages. + +The SQLite rows are the accounting authority. Transcript checkpoints and +shutdown records can validate the result, but never create another charge. +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Iterable +from datetime import datetime +from typing import Any +from urllib.parse import quote + +from thirdeye.usage.types import UsageRow + +from .types import ( + AccountingCandidate, + AccountingProjection, + AccountingProjectionState, + DatabaseRevision, + ProjectionDiagnostic, + SourceRecord, +) + +_USAGE_TABLE = "assistant_usage_events" +_METRIC_FIELDS = ( + "input_tokens", + "output_tokens", + "cache_read_tokens", + "cache_write_tokens", + "reasoning_tokens", + "total_nano_aiu", +) +_SUPPLEMENTAL_FIELDS = ( + "total_nano_aiu", + "request_multiplier", + "duration_ms", + "time_to_first_token_ms", + "output_ttft_ms", + "inter_token_latency_ms", + "initiator", + "api_endpoint", + "reasoning_effort", + "content_filter_triggered", + "token_details_json", +) +_TOKEN_FIELDS = _METRIC_FIELDS[:-1] + + +def _canonical_json(value: Any) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False, default=str) + + +def _valid_timestamp(value: object) -> str | None: + if not isinstance(value, str) or not value: + return None + try: + datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + return value + + +def _string(value: object) -> str | None: + return value if isinstance(value, str) and value else None + + +def _integer(value: object) -> int | None: + """Return a non-negative integer without turning missing values into zero.""" + + if isinstance(value, bool): + return None + if isinstance(value, int) and value >= 0: + return value + return None + + +def _source_key(record: SourceRecord) -> str | None: + prefix = "copilot-db:" + source_id = record["source_id"] + if not source_id.startswith(prefix): + return None + key = source_id[len(prefix) :].split(":", 1)[0] + return key if len(key) == 64 else None + + +def _revision(record: SourceRecord) -> DatabaseRevision | None: + locator = record.get("locator") + if not isinstance(locator, dict) or locator.get("table") != _USAGE_TABLE: + return None + generation = _string(locator.get("generation")) + content_revision = _string(locator.get("content_revision")) + if generation is None or content_revision is None or "primary_key" not in locator: + return None + return { + "table": _USAGE_TABLE, + "primary_key": quote(_canonical_json(locator["primary_key"]), safe=""), + "generation": generation, + "content_revision": content_revision, + } + + +def _logical_call_id(record: SourceRecord, revision: DatabaseRevision) -> str | None: + source_key = _source_key(record) + if source_key is None: + return None + return ( + f"copilot:usage:{source_key}:{revision['table']}:" + f"{quote(revision['generation'], safe='')}:{revision['primary_key']}" + ) + + +def _metrics_digest(row: dict[str, Any]) -> str: + metrics = {field: row.get(field) for field in _METRIC_FIELDS} + return "sha256:" + hashlib.sha256(_canonical_json(metrics).encode("utf-8")).hexdigest() + + +def _row_is_incompatible(row: dict[str, Any]) -> bool: + """Reject revisions that cannot describe a single completed request.""" + + input_tokens = _integer(row.get("input_tokens")) + cache_read = _integer(row.get("cache_read_tokens")) + cache_write = _integer(row.get("cache_write_tokens")) + if input_tokens is None: + return False + return (cache_read is not None and cache_read > input_tokens) or ( + cache_write is not None and cache_write > input_tokens + ) + + +def _candidate( + record: SourceRecord, revision: DatabaseRevision, logical_id: str +) -> AccountingCandidate: + payload = record.get("payload") + row = payload.get("row") if isinstance(payload, dict) else None + assert isinstance(row, dict) + timestamp = _valid_timestamp(record.get("ts")) or _valid_timestamp(row.get("created_at")) + supplemental = { + field: row[field] + for field in _SUPPLEMENTAL_FIELDS + if field in row and row[field] is not None + } + # Fully specified token usage belongs in UsageRow. Retain token values in + # the candidate only when a partial row cannot produce a UsageRow. + if _integer(row.get("input_tokens")) is None or _integer(row.get("output_tokens")) is None: + supplemental.update( + { + field: row[field] + for field in _TOKEN_FIELDS + if field in row and row[field] is not None + } + ) + turn_index = _integer(row.get("turn_index")) + return { + "usage_source_id": record["source_id"], + "logical_call_id": logical_id, + "turn_index": turn_index, + "agent_id": _string(row.get("agent_id")), + "parent_tool_call_id": _string(row.get("parent_tool_call_id")), + "model": _string(row.get("model")), + "provider": _string(row.get("provider")) or _string(row.get("provider_name")), + "source_ids": [record["source_id"]], + "source_references": [ + {"source_id": record["source_id"], "source_kind": "database", "role": "usage_row"} + ], + "revision": revision, + "timestamp": timestamp, + "finish_reason": _string(row.get("finish_reason")), + "supplemental_metrics": supplemental, + } + + +def _missing_fields(candidate: AccountingCandidate, row: dict[str, Any]) -> list[str]: + required = { + "timestamp": candidate["timestamp"], + "model": candidate["model"], + "input_tokens": _integer(row.get("input_tokens")), + "output_tokens": _integer(row.get("output_tokens")), + } + return [name for name, value in required.items() if value is None] + + +def _usage_row( + candidate: AccountingCandidate, session_id: str, row: dict[str, Any] +) -> UsageRow | None: + missing = _missing_fields(candidate, row) + if missing: + return None + input_tokens = _integer(row["input_tokens"]) + output_tokens = _integer(row["output_tokens"]) + assert input_tokens is not None and output_tokens is not None + return UsageRow( + session_id=session_id, + seq=0, + call_id=candidate["logical_call_id"], + ts=candidate["timestamp"], # guarded by _missing_fields + platform="copilot", + provider_name=candidate["provider"] or "unknown", + response_model=candidate["model"], # guarded by _missing_fields + input_tokens=input_tokens, + output_tokens=output_tokens, + cache_read_input_tokens=_integer(row.get("cache_read_tokens")), + cache_creation_input_tokens=_integer(row.get("cache_write_tokens")), + reasoning_output_tokens=_integer(row.get("reasoning_tokens")), + ) + + +def _diagnostic( + code: str, severity: str, message: str, source_ids: list[str], **details: Any +) -> ProjectionDiagnostic: + return { + "code": code, # type: ignore[typeddict-item] + "severity": severity, # type: ignore[typeddict-item] + "message": message, + "source_ids": source_ids, + "details": details, + } + + +def _shutdown_totals(records: Iterable[SourceRecord]) -> list[tuple[str, dict[str, int]]]: + totals: list[tuple[str, dict[str, int]]] = [] + for record in records: + payload = record.get("payload") + if not isinstance(payload, dict) or payload.get("type") != "session.shutdown": + continue + data = payload.get("data") + if not isinstance(data, dict): + continue + model_metrics = data.get("modelMetrics") + if not isinstance(model_metrics, dict): + continue + aggregate = {name: 0 for name in _METRIC_FIELDS} + found = False + for model in model_metrics.values(): + usage = model.get("usage") if isinstance(model, dict) else None + if not isinstance(usage, dict): + continue + values = { + "input_tokens": usage.get("inputTokens"), + "output_tokens": usage.get("outputTokens"), + "cache_read_tokens": usage.get("cacheReadTokens"), + "cache_write_tokens": usage.get("cacheWriteTokens"), + "reasoning_tokens": usage.get("reasoningTokens"), + "total_nano_aiu": model.get("totalNanoAiu"), + } + if all(_integer(value) is not None for value in values.values()): + found = True + for name, value in values.items(): + aggregate[name] += _integer(value) or 0 + if found: + totals.append((record["source_id"], aggregate)) + return totals + + +def build_accounting( + records: list[SourceRecord], prior_state: dict[str, Any] +) -> tuple[AccountingProjection, dict[str, Any]]: + """Build independent database accounting from immutable V1 source records. + + Revisions are applied in archive order. A missing row is intentionally not + a deletion: only archived observations can replace an accounting result. + """ + + # The V1 archive preserves observation order, so a later archived revision + # is authoritative. Keep every source ID as conflict evidence even though + # only the newest revision supplies the candidate. + inherited_calls = prior_state.get("logical_calls") + if not isinstance(inherited_calls, dict): + accounting_state = prior_state.get("accounting_state") + inherited_calls = ( + accounting_state.get("logical_calls") if isinstance(accounting_state, dict) else {} + ) + selected: dict[str, tuple[SourceRecord, DatabaseRevision, AccountingCandidate]] = {} + revision_sources: dict[str, list[str]] = {} + diagnostics: list[ProjectionDiagnostic] = [] + primary_generations: dict[tuple[str, str], set[str]] = {} + + for record in records: + payload = record.get("payload") + if record.get("source_kind") != "database" or not isinstance(payload, dict): + continue + if payload.get("table") != _USAGE_TABLE or not isinstance(payload.get("row"), dict): + continue + revision = _revision(record) + logical_id = _logical_call_id(record, revision) if revision is not None else None + if revision is None or logical_id is None: + continue + candidate = _candidate(record, revision, logical_id) + primary_generations.setdefault((revision["table"], revision["primary_key"]), set()).add( + revision["generation"] + ) + selected[logical_id] = (record, revision, candidate) + revision_sources.setdefault(logical_id, []).append(record["source_id"]) + + for (table, primary_key), generations in sorted(primary_generations.items()): + if len(generations) > 1: + diagnostics.append( + _diagnostic( + "usage_row_id_reuse", + "warning", + "database row ID was reused by a different database generation", + [], + table=table, + primary_key=primary_key, + generations=sorted(generations), + ) + ) + + candidates: list[AccountingCandidate] = [] + usage_rows: list[UsageRow] = [] + accounted = {field: 0 for field in _METRIC_FIELDS} + logical_calls = { + key: value.copy() + for key, value in inherited_calls.items() + if isinstance(key, str) and isinstance(value, dict) + } + for logical_id, (record, revision, candidate) in selected.items(): + payload = record["payload"] + row = payload["row"] + assert isinstance(row, dict) + if _row_is_incompatible(row): + logical_calls.pop(logical_id, None) + diagnostics.append( + _diagnostic( + "usage_revision_conflict", + "error", + "incompatible metrics for one logical call; quarantined", + revision_sources[logical_id], + logical_call_id=logical_id, + ) + ) + continue + candidates.append(candidate) + missing = _missing_fields(candidate, row) + if missing: + diagnostics.append( + _diagnostic( + "missing_usage_fields", + "warning", + "database usage row is incomplete", + [record["source_id"]], + logical_call_id=logical_id, + missing_fields=missing, + ) + ) + source_key = _source_key(record) + assert source_key is not None + session_id = f"copilot-{source_key[:16]}-{record['native_session_id']}" + usage_row = _usage_row(candidate, session_id, row) + if usage_row is not None: + usage_rows.append(usage_row) + for field in _METRIC_FIELDS: + value = _integer(row.get(field)) + if value is not None: + accounted[field] += value + logical_calls[logical_id] = { + "logical_call_id": logical_id, + "generation": revision["generation"], + "content_revision": revision["content_revision"], + "metrics_digest": _metrics_digest(row), + "usage_source_id": record["source_id"], + } + + for record in records: + payload = record.get("payload") + if not isinstance(payload, dict) or payload.get("type") != "session.usage_checkpoint": + continue + diagnostics.append( + _diagnostic( + "checkpoint_not_additive", + "info", + "checkpoint snapshot validates totals and is not a seventh call", + [record["source_id"]], + ) + ) + + for source_id, expected in _shutdown_totals(records): + if any(accounted[field] != expected[field] for field in _METRIC_FIELDS): + diagnostics.append( + _diagnostic( + "shutdown_total_mismatch", + "warning", + "per-call totals do not equal session.shutdown usage", + [source_id], + **{ + f"expected_{field}": expected[field] + for field in _METRIC_FIELDS + }, + **{ + f"accounted_{field}": accounted[field] + for field in _METRIC_FIELDS + }, + ) + ) + + next_state: AccountingProjectionState = {"logical_calls": logical_calls} + return {"usage_rows": usage_rows, "candidates": candidates, "diagnostics": diagnostics}, next_state From d4b7afed78e5c055a0eb24cb0c98452e75f14297 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Fri, 11 Sep 2026 15:28:18 -0700 Subject: [PATCH 49/88] Add durable Copilot projection storage --- .../platforms/copilot/projection_state.py | 170 +++++++++ .../platforms/copilot/projection_store.py | 338 ++++++++++++++++++ 2 files changed, 508 insertions(+) create mode 100644 src/thirdeye/platforms/copilot/projection_state.py create mode 100644 src/thirdeye/platforms/copilot/projection_store.py diff --git a/src/thirdeye/platforms/copilot/projection_state.py b/src/thirdeye/platforms/copilot/projection_state.py new file mode 100644 index 0000000..307ff57 --- /dev/null +++ b/src/thirdeye/platforms/copilot/projection_state.py @@ -0,0 +1,170 @@ +"""Recoverable, versioned storage for Copilot's derived V2 projection. + +This state deliberately lives beside, rather than inside, the V1 capture +checkpoint. The capture archive is immutable evidence; deleting this file is +therefore a safe way to request a local projection rebuild. It must never +touch the export ledger (which has its own lock and lifecycle). +""" + +from __future__ import annotations + +import json +import os +import tempfile +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from thirdeye._compat import fsops + +from .types import PROJECTION_SCHEMA_VERSION + +PROJECTION_STATE_FILENAME = "copilot.projection.state.json" +PROJECTION_JOURNAL_FILENAME = "copilot.projection.journal.json" +PROJECTION_LOCK_FILENAME = "copilot.projection.lock" +PROJECTION_JOURNAL_SCHEMA_VERSION = 1 + +# Kept private, just like the V1 archive fault hook. It gives durability tests +# precise crash boundaries without making fault injection a runtime feature. +_fault_injector: Callable[[str], None] | None = None + + +def _fault(point: str) -> None: + if _fault_injector is not None: + _fault_injector(point) + + +def projection_state_path(session_dir: Path) -> Path: + return session_dir / PROJECTION_STATE_FILENAME + + +def projection_journal_path(session_dir: Path) -> Path: + return session_dir / PROJECTION_JOURNAL_FILENAME + + +def projection_lock_path(session_dir: Path) -> Path: + return session_dir / PROJECTION_LOCK_FILENAME + + +def empty_projection_state() -> dict[str, Any]: + """Return the public state shape used as pure-builder input.""" + return { + "projection_schema_version": PROJECTION_SCHEMA_VERSION, + "archive_source_ids": [], + "semantic_state": {"open_interactions": {}}, + "accounting_state": {"logical_calls": {}}, + "projection_revision": "", + } + + +def empty_projection_document() -> dict[str, Any]: + """Return the private envelope around public state and replay indexes.""" + return { + "schema_version": PROJECTION_SCHEMA_VERSION, + "state": empty_projection_state(), + "indexes": { + "events": {}, + "turns": {}, + "usage": {}, + "attributions": {}, + "pending": {}, + "diagnostics": {}, + }, + } + + +def _atomic_json(path: Path, value: dict[str, Any], *, fault_point: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + fd, temp_name = tempfile.mkstemp(dir=path.parent, prefix=f"{path.name}.", suffix=".tmp") + try: + with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as stream: + json.dump(value, stream, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + fsops.replace(temp_name, path) + fsops.sync_directory(path.parent) + _fault(fault_point) + except BaseException: + fsops.unlink(Path(temp_name), missing_ok=True) + raise + + +def _read_json(path: Path) -> dict[str, Any] | None: + try: + value = json.loads(fsops.read_text(path, encoding="utf-8")) + except FileNotFoundError: + return None + except (OSError, json.JSONDecodeError): + raise ValueError(f"invalid Copilot projection state: {path}") from None + if not isinstance(value, dict): + raise ValueError(f"invalid Copilot projection state: {path}") + return value + + +def _validate_document(document: dict[str, Any]) -> dict[str, Any]: + if document.get("schema_version") != PROJECTION_SCHEMA_VERSION: + raise ValueError("unsupported Copilot projection state schema") + state = document.get("state") + indexes = document.get("indexes") + if not isinstance(state, dict) or not isinstance(indexes, dict): + raise ValueError("invalid Copilot projection state") + if state.get("projection_schema_version") != PROJECTION_SCHEMA_VERSION: + raise ValueError("unsupported Copilot projection schema") + for name in ("events", "turns", "usage", "attributions", "pending", "diagnostics"): + if not isinstance(indexes.get(name), dict): + raise ValueError("invalid Copilot projection indexes") + return document + + +def read_projection_document(session_dir: Path) -> dict[str, Any]: + """Read the durable document, returning an empty one before first use. + + Callers holding :func:`projection_lock_path` may rely on this to finish a + previously published journal before reading. Keeping recovery here makes + it impossible for a later commit to merge against an incomplete snapshot. + """ + journal = _read_json(projection_journal_path(session_dir)) + if journal is not None: + if journal.get("schema_version") != PROJECTION_JOURNAL_SCHEMA_VERSION: + raise ValueError("unsupported Copilot projection journal schema") + document = journal.get("document") + if not isinstance(document, dict): + raise ValueError("invalid Copilot projection journal") + _validate_document(document) + _atomic_json( + projection_state_path(session_dir), document, fault_point="after_projection_recovery" + ) + fsops.unlink(projection_journal_path(session_dir), missing_ok=True) + fsops.sync_directory(session_dir) + _fault("after_projection_recovery_clear") + return document + + document = _read_json(projection_state_path(session_dir)) + return empty_projection_document() if document is None else _validate_document(document) + + +def publish_projection_document(session_dir: Path, document: dict[str, Any]) -> None: + """Publish ``document`` with write-ahead journaling. + + The caller owns the projection lock. If interrupted after either replace, + the next reader replays the complete immutable document from the journal. + """ + _validate_document(document) + journal = {"schema_version": PROJECTION_JOURNAL_SCHEMA_VERSION, "document": document} + _atomic_json( + projection_journal_path(session_dir), journal, fault_point="after_projection_journal" + ) + _atomic_json( + projection_state_path(session_dir), document, fault_point="after_projection_state" + ) + fsops.unlink(projection_journal_path(session_dir), missing_ok=True) + fsops.sync_directory(session_dir) + _fault("after_projection_journal_clear") + + +def remove_projection_state(session_dir: Path) -> None: + """Remove only rebuildable local projection files, never V1 or export data.""" + fsops.unlink(projection_state_path(session_dir), missing_ok=True) + fsops.unlink(projection_journal_path(session_dir), missing_ok=True) + fsops.sync_directory(session_dir) diff --git a/src/thirdeye/platforms/copilot/projection_store.py b/src/thirdeye/platforms/copilot/projection_store.py new file mode 100644 index 0000000..72da04c --- /dev/null +++ b/src/thirdeye/platforms/copilot/projection_store.py @@ -0,0 +1,338 @@ +"""Durable local storage for completed Copilot V2 projections. + +This module is intentionally a persistence boundary. It accepts already +constructed :class:`Projection` DTOs and archived Store events; it does not +read Copilot's live transcript/database files and does not perform semantic, +accounting, attribution, or export work. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import tempfile +from collections.abc import Iterable +from pathlib import Path +from typing import Any + +from thirdeye._compat import fsops +from thirdeye._compat.locking import LockMode, locked +from thirdeye.config import Config +from thirdeye.meta import read_meta +from thirdeye.paths import meta_path, session_dir, usage_jsonl_path +from thirdeye.reader import SessionReader +from thirdeye.usage.types import UsageRow + +from .constants import PLATFORM_NAME, SOURCE_SCHEMA_VERSION +from .projection_state import ( + empty_projection_state, + projection_lock_path, + publish_projection_document, + read_projection_document, + remove_projection_state, +) +from .types import PROJECTION_SCHEMA_VERSION, Projection + +_RAW_EVENT_TYPES = frozenset( + {"copilot_transcript", "copilot_database", "copilot_hook", "copilot_metadata"} +) + + +def _directory(config: Config, stored_session_id: str) -> Path: + return session_dir(config.root, PLATFORM_NAME, stored_session_id) + + +def _canonical(value: Any) -> str: + return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), default=str) + + +def _digest(value: Any) -> str: + return "sha256:" + hashlib.sha256(_canonical(value).encode("utf-8")).hexdigest() + + +def _mapping(value: object) -> dict[str, Any]: + return value if isinstance(value, dict) else {} + + +def _index_key(item: dict[str, Any], field: str, *, prefix: str) -> str: + value = item.get(field) + return value if isinstance(value, str) and value else f"{prefix}:{_digest(item)}" + + +def _source_id(event: dict[str, Any]) -> str | None: + if event.get("t") not in _RAW_EVENT_TYPES: + return None + data = _mapping(event.get("data")) + if data.get("schema_version") != SOURCE_SCHEMA_VERSION: + return None + record = _mapping(data.get("source_record")) + source_id = record.get("source_id") + return source_id if isinstance(source_id, str) else None + + +def _read_archived_events(directory: Path) -> dict[str, dict[str, Any]]: + if not directory.exists(): + return {} + indexed: dict[str, dict[str, Any]] = {} + for event in SessionReader(directory).iter_events(types=_RAW_EVENT_TYPES): + source_id = _source_id(event) + if source_id is not None: + indexed[source_id] = event + return indexed + + +def _add_source_ids(value: object, result: set[str]) -> None: + if not isinstance(value, dict): + return + for key in ("source_ids",): + ids = value.get(key) + if isinstance(ids, list): + result.update(source_id for source_id in ids if isinstance(source_id, str)) + references = value.get("source_references") + if isinstance(references, list): + for reference in references: + source_id = _mapping(reference).get("source_id") + if isinstance(source_id, str): + result.add(source_id) + attributes = value.get("attributes") + if isinstance(attributes, dict): + _add_source_ids(attributes, result) + + +def _turn_source_ids(turn: dict[str, Any], events: Iterable[dict[str, Any]]) -> set[str]: + """Find archived evidence owned by a top-level reconstructed interaction. + + Producers can provide explicit source references on a turn/call. The + interaction-id fallback is evidence-preserving (not a guessed parent): it + includes all main and nested-child events bearing that exact native + interaction identity, which keeps child evidence inside its parent view. + """ + source_ids: set[str] = set() + _add_source_ids(turn, source_ids) + for call in turn.get("llm_calls", []): + _add_source_ids(call, source_ids) + for child in turn.get("subagents", []): + source_ids.update(_turn_source_ids(_mapping(child), ())) + + attributes = _mapping(turn.get("attributes")) + interaction_id = attributes.get("interaction_id") + turn_id = turn.get("turn_id") + for semantic_event in events: + event_attrs = _mapping(semantic_event.get("attributes")) + if ( + isinstance(interaction_id, str) + and event_attrs.get("interaction_id") == interaction_id + ) or (isinstance(turn_id, str) and event_attrs.get("stored_turn_id") == turn_id): + ids = semantic_event.get("source_ids") + if isinstance(ids, list): + source_ids.update(source_id for source_id in ids if isinstance(source_id, str)) + return source_ids + + +def _projected_turn_record( + stored_session_id: str, + directory: Path, + turn: dict[str, Any], + semantic_events: Iterable[dict[str, Any]], + archived_events: dict[str, dict[str, Any]], +) -> dict[str, Any]: + turn_id = _index_key(turn, "turn_id", prefix="turn") + source_ids = _turn_source_ids(turn, semantic_events) + events = [archived_events[source_id] for source_id in source_ids if source_id in archived_events] + events.sort(key=lambda event: int(event.get("seq", -1))) + meta = read_meta(meta_path(directory)) if directory.exists() else None + start_ts = turn.get("start_ts") if isinstance(turn.get("start_ts"), str) else None + end_ts = turn.get("end_ts") if isinstance(turn.get("end_ts"), str) else None + return { + "id": f"{stored_session_id}:{turn_id}", + "turn_id": turn_id, + "session_id": stored_session_id, + "platform": PLATFORM_NAME, + "cwd": meta.cwd if meta is not None else "", + "start_seq": events[0].get("seq") if events else None, + "end_seq": events[-1].get("seq") if events else None, + "start_ts": events[0].get("ts") if events else start_ts, + "end_ts": events[-1].get("ts") if events else end_ts, + "events": events, + } + + +def _merge_state(current: dict[str, Any], next_state: dict[str, Any]) -> dict[str, Any]: + """Merge independent incremental builders without losing another commit.""" + merged = empty_projection_state() + merged.update({key: value for key, value in current.items() if key in merged}) + merged.update({key: value for key, value in next_state.items() if key in merged}) + current_ids = current.get("archive_source_ids") + next_ids = next_state.get("archive_source_ids") + current_source_ids = ( + {source_id for source_id in current_ids if isinstance(source_id, str)} + if isinstance(current_ids, list) + else set() + ) + next_source_ids = ( + {source_id for source_id in next_ids if isinstance(source_id, str)} + if isinstance(next_ids, list) + else set() + ) + merged["archive_source_ids"] = sorted(current_source_ids | next_source_ids) + for section, key in (("semantic_state", "open_interactions"), ("accounting_state", "logical_calls")): + old = _mapping(_mapping(current.get(section)).get(key)) + new = _mapping(_mapping(next_state.get(section)).get(key)) + merged[section] = {key: {**old, **new}} + merged["projection_schema_version"] = PROJECTION_SCHEMA_VERSION + return merged + + +def _usage_identity(row: UsageRow, attributions: dict[str, dict[str, Any]]) -> str: + for logical_id, attribution in attributions.items(): + if attribution.get("usage_source_id") == row.call_id: + return logical_id + # Usage normalization makes call_id the durable logical ID. The fallback + # keeps hand-built DTOs valid without ever keying on an import sequence. + return row.call_id + + +def _write_usage_index(directory: Path, usage_index: dict[str, Any]) -> None: + """Materialize one latest serialized row per logical call atomically. + + UsageStore is append-only and its generic reader is last-wins, which is + insufficient for correction/rebuild guarantees. This derived-only + materialization preserves its exact UsageRow JSON serialization while the + projection index supplies replacement semantics. + """ + path = usage_jsonl_path(directory) + path.parent.mkdir(parents=True, exist_ok=True) + fd, temporary = tempfile.mkstemp(dir=path.parent, prefix=f"{path.name}.", suffix=".tmp") + try: + with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as stream: + for logical_id in sorted(usage_index): + row = usage_index[logical_id] + if isinstance(row, dict): + stream.write(_canonical(row)) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + fsops.replace(temporary, path) + fsops.sync_directory(path.parent) + except BaseException: + fsops.unlink(Path(temporary), missing_ok=True) + raise + + +def _commit_counts(indexes: dict[str, Any]) -> dict[str, int]: + return { + "events": len(_mapping(indexes.get("events"))), + "turns": len(_mapping(indexes.get("turns"))), + "usage": len(_mapping(indexes.get("usage"))), + "attributions": len(_mapping(indexes.get("attributions"))), + "pending": len(_mapping(indexes.get("pending"))), + "diagnostics": len(_mapping(indexes.get("diagnostics"))), + } + + +def commit_projection( + config: Config, + stored_session_id: str, + projection: Projection, + next_state: dict[str, Any], +) -> dict[str, int]: + """Atomically merge a DTO projection into local, replayable derived state. + + V1 events are read only from the captured Store archive to form the generic + turn view. No source reader is invoked, so a captured session remains + projectable after Copilot removes its original files. + """ + directory = _directory(config, stored_session_id) + with locked(projection_lock_path(directory), LockMode.EXCLUSIVE): + document = read_projection_document(directory) + indexes = _mapping(document.get("indexes")) + merged_indexes = {name: dict(_mapping(indexes.get(name))) for name in indexes} + for name in ("events", "turns", "usage", "attributions", "pending", "diagnostics"): + merged_indexes.setdefault(name, {}) + + event_items = [item for item in projection.get("normalized_events", []) if isinstance(item, dict)] + for item in event_items: + merged_indexes["events"][_index_key(item, "id", prefix="event")] = item + + attribution_items = [item for item in projection.get("attributions", []) if isinstance(item, dict)] + for item in attribution_items: + merged_indexes["attributions"][_index_key(item, "logical_call_id", prefix="attribution")] = item + + for item in projection.get("usage_rows", []): + if not isinstance(item, UsageRow): + raise TypeError("projection usage_rows must contain UsageRow instances") + row = item.to_dict() + logical_id = _usage_identity(item, merged_indexes["attributions"]) + merged_indexes["usage"][logical_id] = row + + archived_events = _read_archived_events(directory) + for item in projection.get("turns", []): + if not isinstance(item, dict): + continue + # Child-agent spans are represented recursively by their owning + # main interaction and never become separate generic user turns. + if _mapping(item.get("attributes")).get("agent_id") is not None: + continue + turn_id = _index_key(item, "turn_id", prefix="turn") + merged_indexes["turns"][turn_id] = _projected_turn_record( + stored_session_id, directory, item, event_items, archived_events + ) + + for item in projection.get("pending", []): + if isinstance(item, dict): + merged_indexes["pending"][_index_key(item, "id", prefix="pending")] = item + for item in projection.get("diagnostics", []): + if isinstance(item, dict): + merged_indexes["diagnostics"][_index_key(item, "id", prefix="diagnostic")] = item + + state = _merge_state(_mapping(document.get("state")), next_state) + counts = _commit_counts(merged_indexes) + state["commit_result"] = counts + next_document = { + "schema_version": PROJECTION_SCHEMA_VERSION, + "state": state, + "indexes": merged_indexes, + } + publish_projection_document(directory, next_document) + # A crash after publication is repaired on the next load/commit from + # the durable usage index; this file contains no raw V1 evidence. + _write_usage_index(directory, merged_indexes["usage"]) + return counts + + +def load_projection_state(config: Config, stored_session_id: str) -> dict[str, Any]: + """Return a copy of derived builder state, completing journal recovery.""" + directory = _directory(config, stored_session_id) + with locked(projection_lock_path(directory), LockMode.EXCLUSIVE): + document = read_projection_document(directory) + # Recover a sidecar lost after durable state publication without + # changing V1 evidence or export bookkeeping. + usage_index = _mapping(_mapping(document.get("indexes")).get("usage")) + if usage_index: + _write_usage_index(directory, usage_index) + return json.loads(_canonical(_mapping(document.get("state")))) + + +def read_projected_turns(config: Config, stored_session_id: str) -> list[dict[str, Any]]: + """Read completed main interaction records without semantic duplicates.""" + directory = _directory(config, stored_session_id) + with locked(projection_lock_path(directory), LockMode.EXCLUSIVE): + document = read_projection_document(directory) + turns = _mapping(_mapping(document.get("indexes")).get("turns")) + values = [value for value in turns.values() if isinstance(value, dict)] + values.sort(key=lambda turn: (str(turn.get("start_ts") or ""), str(turn.get("turn_id") or ""))) + return json.loads(_canonical(values)) + + +def reset_projection_state(config: Config, stored_session_id: str) -> None: + """Delete only reproducible projection state for a local rebuild. + + The caller must subsequently commit a full replay. This intentionally + does not import, reset, or otherwise interact with export-state files. + """ + directory = _directory(config, stored_session_id) + with locked(projection_lock_path(directory), LockMode.EXCLUSIVE): + remove_projection_state(directory) + fsops.unlink(usage_jsonl_path(directory), missing_ok=True) + fsops.sync_directory(directory) From 242b4b28502dda955ae731d2fd42ede3122a01d6 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Fri, 11 Sep 2026 15:29:06 -0700 Subject: [PATCH 50/88] Add behavioral tests for Copilot database usage normalization. Cover six-call corpus totals, revisions, provider handling, checkpoint/shutdown validation, and incremental replay through build_accounting. Co-authored-by: Cursor --- tests/platforms/copilot/test_usage.py | 497 ++++++++++++++++++++++++++ 1 file changed, 497 insertions(+) create mode 100644 tests/platforms/copilot/test_usage.py diff --git a/tests/platforms/copilot/test_usage.py b/tests/platforms/copilot/test_usage.py new file mode 100644 index 0000000..d2cbe62 --- /dev/null +++ b/tests/platforms/copilot/test_usage.py @@ -0,0 +1,497 @@ +"""Behavioral tests for Copilot database usage normalization.""" + +from __future__ import annotations + +import copy +import json +from pathlib import Path +from typing import Any + +import pytest + +from thirdeye.platforms.copilot.database import read_database +from thirdeye.platforms.copilot.identity import resolve_sources +from thirdeye.platforms.copilot.types import SourceRecord +from thirdeye.platforms.copilot.usage import build_accounting +from thirdeye.usage.types import UsageRow + +FIXTURES = Path(__file__).parent / "fixtures" +RECONCILIATION = FIXTURES / "reconciliation-cases" +CLI_FIXTURE = FIXTURES + +NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" +SOURCE_KEY = "a" * 64 +GENERATION = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" +CHILD_AGENT_ID = "bf8cb9f3-2097-4db0-a3c8-78a2653b2106" +OBSERVED_AT = "2026-09-10T17:09:00.000Z" + +SHUTDOWN_TOTALS = { + "input_tokens": 35396, + "output_tokens": 328, + "reasoning_tokens": 55, + "cache_read_tokens": 23948, + "cache_write_tokens": 11430, + "total_nano_aiu": 373366000, +} + + +def _load_json(path: Path) -> Any: + return json.loads(path.read_text(encoding="utf-8")) + + +def _diagnostic_codes(projection: dict[str, Any]) -> set[str]: + return {item["code"] for item in projection["diagnostics"]} + + +def _usage_records(records: list[SourceRecord]) -> list[SourceRecord]: + return [ + record + for record in records + if record.get("source_kind") == "database" + and isinstance(record.get("payload"), dict) + and record["payload"].get("table") == "assistant_usage_events" + ] + + +def _usage_record( + row: dict[str, Any], + *, + content_revision: str, + generation: str = GENERATION, + source_key: str = SOURCE_KEY, + observed_at: str = OBSERVED_AT, +) -> SourceRecord: + primary_key = row["id"] + source_id = ( + f"copilot-db:{source_key}:{NATIVE_SESSION_ID}:assistant_usage_events:" + f"{primary_key}:{content_revision}" + ) + return { + "source_id": source_id, + "source_kind": "database", + "native_session_id": NATIVE_SESSION_ID, + "ts": row.get("created_at"), + "observed_at": observed_at, + "payload": {"table": "assistant_usage_events", "row": row}, + "locator": { + "database": "/example/.copilot/session-store.db", + "table": "assistant_usage_events", + "primary_key": primary_key, + "content_revision": content_revision, + "generation": generation, + }, + } + + +def _shutdown_record(*, source_id: str = "transcript/shutdown-1") -> SourceRecord: + usage = _load_json(CLI_FIXTURE / "usage.json") + return { + "source_id": source_id, + "source_kind": "transcript", + "native_session_id": NATIVE_SESSION_ID, + "ts": "2026-09-10T17:08:50.000Z", + "observed_at": OBSERVED_AT, + "payload": {"type": "session.shutdown", "data": usage}, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 0, + "byte_length": 100, + "native_event_id": "shutdown-1", + }, + } + + +def _checkpoint_record(*, source_id: str = "transcript/checkpoint-1") -> SourceRecord: + return { + "source_id": source_id, + "source_kind": "transcript", + "native_session_id": NATIVE_SESSION_ID, + "ts": "2026-09-10T17:08:25.700Z", + "observed_at": OBSERVED_AT, + "payload": { + "type": "session.usage_checkpoint", + "data": { + "inputTokens": 26416, + "outputTokens": 229, + "cacheReadTokens": 19522, + "cacheWriteTokens": 6882, + "reasoningTokens": 39, + "totalNanoAiu": 238814000, + }, + }, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 0, + "byte_length": 100, + "native_event_id": "checkpoint-1", + }, + } + + +def _six_call_records() -> list[SourceRecord]: + rows = _load_json(CLI_FIXTURE / "assistant-usage-events.json") + revisions = { + call["row_id"]: call["usage_source_id"].rsplit(":", 1)[-1] + for call in _load_json(RECONCILIATION / "observed-six-calls.json")["calls"] + } + return [_usage_record(row, content_revision=revisions[row["id"]]) for row in rows] + + +def _metric_totals(usage_rows: list[UsageRow]) -> dict[str, int]: + totals = { + "input_tokens": 0, + "output_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "reasoning_tokens": 0, + } + for row in usage_rows: + totals["input_tokens"] += row.input_tokens + totals["output_tokens"] += row.output_tokens + totals["cache_read_tokens"] += row.cache_read_input_tokens or 0 + totals["cache_write_tokens"] += row.cache_creation_input_tokens or 0 + totals["reasoning_tokens"] += row.reasoning_output_tokens or 0 + return totals + + +def _nano_aiu_total(candidates: list[dict[str, Any]]) -> int: + return sum( + candidate["supplemental_metrics"]["total_nano_aiu"] + for candidate in candidates + if "total_nano_aiu" in candidate["supplemental_metrics"] + ) + + +# --- module boundaries --- + + +def test_usage_module_has_no_forbidden_imports(): + import thirdeye.platforms.copilot.usage as usage + + source = Path(usage.__file__).read_text(encoding="utf-8") + forbidden = ( + "tracing", + "attribution", + "projection_store", + "export_state", + "build_semantics", + "join_usage", + "UsageStore", + "usage_store", + ) + for token in forbidden: + assert token not in source + + +# --- contract fixture --- + + +def test_accounting_projection_fixture_matches_build_accounting(): + case = _load_json(RECONCILIATION / "accounting-projection.json") + projection, state = build_accounting(case["input_records"], {}) + expected = case["expected"] + + assert [row.to_dict() for row in projection["usage_rows"]] == expected["usage_rows"] + assert projection["candidates"] == expected["candidates"] + assert projection["diagnostics"] == expected["diagnostics"] + assert set(state["logical_calls"]) == {expected["candidates"][0]["logical_call_id"]} + + +# --- observed six-call corpus --- + + +def test_six_calls_are_accounted_once_from_cli_rows(): + records = _six_call_records() + projection, _state = build_accounting(records, {}) + + assert len(projection["usage_rows"]) == 6 + assert len(projection["candidates"]) == 6 + assert len({row.call_id for row in projection["usage_rows"]}) == 6 + assert len({candidate["logical_call_id"] for candidate in projection["candidates"]}) == 6 + + +def test_six_call_totals_match_observed_shutdown_fixture(): + records = _six_call_records() + projection, _state = build_accounting(records, {}) + totals = _metric_totals(projection["usage_rows"]) + + assert totals["input_tokens"] == SHUTDOWN_TOTALS["input_tokens"] + assert totals["output_tokens"] == SHUTDOWN_TOTALS["output_tokens"] + assert totals["cache_read_tokens"] == SHUTDOWN_TOTALS["cache_read_tokens"] + assert totals["cache_write_tokens"] == SHUTDOWN_TOTALS["cache_write_tokens"] + assert totals["reasoning_tokens"] == SHUTDOWN_TOTALS["reasoning_tokens"] + assert _nano_aiu_total(projection["candidates"]) == SHUTDOWN_TOTALS["total_nano_aiu"] + + +def test_candidates_preserve_agent_parent_and_turn_index_evidence(): + records = _six_call_records() + projection, _state = build_accounting(records, {}) + by_row_id = { + candidate["revision"]["primary_key"]: candidate for candidate in projection["candidates"] + } + + main_turn_zero = [by_row_id[str(row_id)] for row_id in (13, 14)] + assert all(item["turn_index"] == 0 for item in main_turn_zero) + assert all(item["agent_id"] is None for item in main_turn_zero) + + child_calls = [by_row_id[str(row_id)] for row_id in (16, 17)] + assert all(item["agent_id"] == CHILD_AGENT_ID for item in child_calls) + assert all(item["parent_tool_call_id"] == "call_qx4FH5DADTeT1qVLb37HNpBk" for item in child_calls) + + +def test_shutdown_validation_passes_when_totals_match(): + records = _six_call_records() + [_shutdown_record()] + projection, _state = build_accounting(records, {}) + assert "shutdown_total_mismatch" not in _diagnostic_codes(projection) + + +def test_database_reader_records_normalize_to_same_six_calls(tmp_path: Path): + from tests.platforms.copilot.test_database import _collect_all, _write_database + + usage_rows = _load_json(CLI_FIXTURE / "assistant-usage-events.json") + home = tmp_path / "copilot" + _write_database( + home, + session_id=NATIVE_SESSION_ID, + cwd="/tmp/probe", + usage_rows=usage_rows, + ) + db_records = _usage_records(_collect_all(resolve_sources(home), NATIVE_SESSION_ID)) + synthetic_records = _six_call_records() + + db_projection, _ = build_accounting(db_records, {}) + synthetic_projection, _ = build_accounting(synthetic_records, {}) + + assert len(db_projection["usage_rows"]) == len(synthetic_projection["usage_rows"]) == 6 + assert _metric_totals(db_projection["usage_rows"]) == _metric_totals( + synthetic_projection["usage_rows"] + ) + + +# --- provider handling --- + + +def test_unknown_provider_maps_to_unknown_in_usage_row(): + row = copy.deepcopy(_load_json(CLI_FIXTURE / "assistant-usage-events.json")[0]) + record = _usage_record(row, content_revision="sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47") + projection, _state = build_accounting([record], {}) + + assert projection["candidates"][0]["provider"] is None + assert projection["usage_rows"][0].provider_name == "unknown" + + +def test_explicit_provider_is_preserved(): + row = copy.deepcopy(_load_json(CLI_FIXTURE / "assistant-usage-events.json")[0]) + row["provider"] = "openai" + record = _usage_record(row, content_revision="sha256:provider-rev") + projection, _state = build_accounting([record], {}) + + assert projection["candidates"][0]["provider"] == "openai" + assert projection["usage_rows"][0].provider_name == "openai" + + +def test_provider_name_column_is_accepted(): + row = copy.deepcopy(_load_json(CLI_FIXTURE / "assistant-usage-events.json")[0]) + row["provider_name"] = "anthropic" + record = _usage_record(row, content_revision="sha256:provider-name-rev") + projection, _state = build_accounting([record], {}) + + assert projection["candidates"][0]["provider"] == "anthropic" + assert projection["usage_rows"][0].provider_name == "anthropic" + + +# --- absent vs zero --- + + +def test_absent_cache_and_reasoning_tokens_stay_none_in_usage_row(): + row = copy.deepcopy(_load_json(CLI_FIXTURE / "assistant-usage-events.json")[0]) + row.pop("cache_read_tokens", None) + row.pop("cache_write_tokens", None) + row.pop("reasoning_tokens", None) + record = _usage_record(row, content_revision="sha256:absent-cache-rev") + projection, _state = build_accounting([record], {}) + + usage_row = projection["usage_rows"][0] + assert usage_row.cache_read_input_tokens is None + assert usage_row.cache_creation_input_tokens is None + assert usage_row.reasoning_output_tokens is None + serialized = usage_row.to_dict() + assert "gen_ai.usage.cache_read.input_tokens" not in serialized + assert "gen_ai.usage.cache_creation.input_tokens" not in serialized + assert "gen_ai.usage.reasoning.output_tokens" not in serialized + + +# --- missing / partial rows --- + + +def test_missing_required_fields_keep_candidate_without_usage_row(): + row = copy.deepcopy(_load_json(CLI_FIXTURE / "assistant-usage-events.json")[0]) + row.pop("output_tokens") + record = _usage_record(row, content_revision="sha256:missing-output-rev") + projection, _state = build_accounting([record], {}) + + assert len(projection["candidates"]) == 1 + assert projection["usage_rows"] == [] + missing = [ + item + for item in projection["diagnostics"] + if item["code"] == "missing_usage_fields" + ] + assert missing + assert "output_tokens" in missing[0]["details"]["missing_fields"] + supplemental = projection["candidates"][0]["supplemental_metrics"] + assert "output_tokens" not in supplemental + assert supplemental["input_tokens"] == row["input_tokens"] + + +def test_missing_timestamp_diagnostic_and_no_usage_row(): + row = copy.deepcopy(_load_json(CLI_FIXTURE / "assistant-usage-events.json")[0]) + row.pop("created_at") + record = _usage_record(row, content_revision="sha256:missing-ts-rev") + record["ts"] = None + projection, _state = build_accounting([record], {}) + + assert projection["usage_rows"] == [] + missing = projection["diagnostics"][0] + assert missing["code"] == "missing_usage_fields" + assert "timestamp" in missing["details"]["missing_fields"] + + +# --- revisions and conflicts --- + + +def test_later_revision_replaces_earlier_for_same_logical_call(): + row = copy.deepcopy(_load_json(CLI_FIXTURE / "assistant-usage-events.json")[0]) + first = _usage_record(row, content_revision="sha256:first-revision") + updated = copy.deepcopy(row) + updated["output_tokens"] = 999 + second = _usage_record(updated, content_revision="sha256:second-revision") + projection, state = build_accounting([first, second], {}) + + assert len(projection["usage_rows"]) == 1 + assert projection["usage_rows"][0].output_tokens == 999 + assert projection["candidates"][0]["usage_source_id"] == second["source_id"] + assert len(state["logical_calls"]) == 1 + + +def test_reused_row_id_across_generations_emits_warning(): + row = copy.deepcopy(_load_json(CLI_FIXTURE / "assistant-usage-events.json")[0]) + first = _usage_record( + row, + content_revision="sha256:gen-a-rev", + generation="sha256:generation-a", + ) + second = _usage_record( + row, + content_revision="sha256:gen-b-rev", + generation="sha256:generation-b", + ) + projection, state = build_accounting([first, second], {}) + + assert "usage_row_id_reuse" in _diagnostic_codes(projection) + assert len(projection["usage_rows"]) == 2 + assert len(state["logical_calls"]) == 2 + + +def test_incompatible_metrics_quarantine_logical_call(): + row = copy.deepcopy(_load_json(CLI_FIXTURE / "assistant-usage-events.json")[0]) + row["cache_read_tokens"] = row["input_tokens"] + 1 + record = _usage_record(row, content_revision="sha256:incompatible-rev") + projection, state = build_accounting([record], {}) + + assert projection["usage_rows"] == [] + assert projection["candidates"] == [] + assert "usage_revision_conflict" in _diagnostic_codes(projection) + assert state["logical_calls"] == {} + + +# --- checkpoint / shutdown --- + + +def test_checkpoint_snapshot_is_not_additive(): + records = _six_call_records() + [_checkpoint_record()] + projection, _state = build_accounting(records, {}) + + assert len(projection["usage_rows"]) == 6 + assert "checkpoint_not_additive" in _diagnostic_codes(projection) + + +def test_shutdown_mismatch_emits_diagnostic(): + usage = _load_json(CLI_FIXTURE / "usage.json") + usage["modelMetrics"]["gpt-5.6-luna"]["usage"]["inputTokens"] = 1 + shutdown = _shutdown_record() + shutdown["payload"]["data"] = usage + projection, _state = build_accounting(_six_call_records() + [shutdown], {}) + + assert "shutdown_total_mismatch" in _diagnostic_codes(projection) + mismatch = next( + item for item in projection["diagnostics"] if item["code"] == "shutdown_total_mismatch" + ) + assert mismatch["details"]["expected_input_tokens"] == 1 + assert mismatch["details"]["accounted_input_tokens"] == SHUTDOWN_TOTALS["input_tokens"] + + +# --- supplemental metrics --- + + +def test_supplemental_metrics_preserve_nano_aiu_separately(): + records = _six_call_records() + projection, _state = build_accounting(records, {}) + candidate = projection["candidates"][0] + + assert "total_nano_aiu" in candidate["supplemental_metrics"] + assert candidate["supplemental_metrics"]["total_nano_aiu"] == 174125000 + usage_row = projection["usage_rows"][0].to_dict() + assert "total_nano_aiu" not in usage_row + + +# --- incremental replay --- + + +def test_incremental_replay_matches_full_archive(): + records = _six_call_records() + full_projection, full_state = build_accounting(records, {}) + + state: dict[str, Any] = {} + incremental_projection = None + for index in range(1, len(records) + 1): + incremental_projection, state = build_accounting(records[:index], state) + + assert incremental_projection is not None + assert [row.to_dict() for row in incremental_projection["usage_rows"]] == [ + row.to_dict() for row in full_projection["usage_rows"] + ] + assert incremental_projection["candidates"] == full_projection["candidates"] + assert state["logical_calls"] == full_state["logical_calls"] + + +def test_late_arrival_adds_new_call_without_rerunning_agent(): + first_batch = _six_call_records()[:3] + late_row = copy.deepcopy(_load_json(CLI_FIXTURE / "assistant-usage-events.json")[3]) + late_record = _usage_record( + late_row, + content_revision="sha256:late-arrival-rev", + ) + + first_projection, state = build_accounting(first_batch, {}) + second_projection, _state = build_accounting(first_batch + [late_record], state) + + assert len(first_projection["usage_rows"]) == 3 + assert len(second_projection["usage_rows"]) == 4 + assert second_projection["candidates"][-1]["agent_id"] == CHILD_AGENT_ID + + +# --- prior state --- + + +def test_prior_state_logical_calls_are_preserved_for_unseen_ids(): + records = _six_call_records()[:1] + _, state = build_accounting(records, {}) + inherited = copy.deepcopy(state) + + projection, next_state = build_accounting([], inherited) + assert projection["usage_rows"] == [] + assert projection["candidates"] == [] + assert next_state["logical_calls"] == inherited["logical_calls"] From 63e06571a17c8c95a020373f0326390c3a448927 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Fri, 11 Sep 2026 15:29:06 -0700 Subject: [PATCH 51/88] Add behavioral tests for Copilot semantic reconstruction. Cover event normalization, turn grouping, nested child ownership, and the observed CLI fixture acceptance criteria. Co-authored-by: Cursor --- tests/platforms/copilot/test_events.py | 338 ++++++++++++++++++++++++ tests/platforms/copilot/test_tracing.py | 333 +++++++++++++++++++++++ 2 files changed, 671 insertions(+) create mode 100644 tests/platforms/copilot/test_events.py create mode 100644 tests/platforms/copilot/test_tracing.py diff --git a/tests/platforms/copilot/test_events.py b/tests/platforms/copilot/test_events.py new file mode 100644 index 0000000..52524dc --- /dev/null +++ b/tests/platforms/copilot/test_events.py @@ -0,0 +1,338 @@ +"""Behavioral tests for Copilot semantic event normalization.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from thirdeye.platforms.copilot.events import ( + agent_id, + interaction_id, + normalize_record, + normalize_records, + record_data, + record_type, + tool_call_id, +) +from thirdeye.platforms.copilot.types import SourceRecord + +FIXTURES = Path(__file__).parent / "fixtures" +RECON_CASES = FIXTURES / "reconciliation-cases" +NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" +SOURCE_KEY = "a" * 64 + + +def _load_json(path: Path) -> Any: + return json.loads(path.read_text(encoding="utf-8")) + + +def _transcript_record( + *, + source_id: str, + native_type: str, + data: dict[str, Any], + ts: str = "2026-09-10T17:08:24.503Z", + agent: str | None = None, +) -> SourceRecord: + payload: dict[str, Any] = { + "type": native_type, + "data": data, + "id": source_id.rsplit("/", 1)[-1], + "timestamp": ts, + "schema_version": 1, + } + if agent is not None: + payload["agentId"] = agent + return { + "source_id": source_id, + "source_kind": "transcript", + "native_session_id": NATIVE_SESSION_ID, + "ts": ts, + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": payload, + "locator": {"file": "events.jsonl", "native_event_id": payload["id"]}, + } + + +def _hook_record(*, source_id: str, event: str, hook_payload: dict[str, Any]) -> SourceRecord: + return { + "source_id": source_id, + "source_kind": "hook", + "native_session_id": NATIVE_SESSION_ID, + "ts": "2026-09-10T17:08:24.500Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "schema_version": 1, + "event": event, + "hook_payload": hook_payload, + "context": {}, + }, + "locator": {"observation_id": source_id.rsplit("/", 1)[-1], "event": event}, + } + + +def _event_kinds(record: SourceRecord) -> list[str]: + return [event["kind"] for event in normalize_record(record)] + + +# --- record accessors --- + + +def test_record_type_uses_event_field_for_hooks(): + record = _hook_record( + source_id=f"hook/{NATIVE_SESSION_ID}/obs-1", + event="preToolUse", + hook_payload={"toolName": "view"}, + ) + assert record_type(record) == "preToolUse" + + +def test_record_data_reads_hook_payload(): + record = _hook_record( + source_id=f"hook/{NATIVE_SESSION_ID}/obs-1", + event="permissionRequest", + hook_payload={"toolName": "view", "toolArgs": {"path": "/tmp/a.txt"}}, + ) + assert record_data(record)["toolName"] == "view" + + +def test_identity_helpers_read_transcript_fields(): + record = _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/msg-1", + native_type="assistant.message", + data={ + "interactionId": "interaction-a", + "turnId": "0", + "toolRequests": [{"toolCallId": "call_abc", "name": "view", "arguments": {}}], + }, + agent="agent-child", + ) + execution = _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/tool-1", + native_type="tool.execution_start", + data={"toolCallId": "call_abc", "toolName": "view", "arguments": {}}, + ) + assert interaction_id(record) == "interaction-a" + assert agent_id(record) == "agent-child" + assert tool_call_id(execution) == "call_abc" + + +# --- user and assistant messages --- + + +def test_user_message_normalizes_delivery_and_source_metadata(): + record = _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/user-1", + native_type="user.message", + data={ + "content": "hello", + "interactionId": "ix-1", + "turnId": "0", + "delivery": "idle", + "source": "agent-parent", + }, + ) + events = normalize_record(record) + assert len(events) == 1 + event = events[0] + assert event["kind"] == "user_prompt" + assert event["source_references"][0]["role"] == "user_prompt" + assert event["attributes"]["delivery"] == "idle" + assert event["attributes"]["source"] == "agent-parent" + assert event["attributes"]["interaction_id"] == "ix-1" + + +def test_assistant_message_emits_tool_requests_with_stable_ids(): + source_id = f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/assistant-1" + record = _transcript_record( + source_id=source_id, + native_type="assistant.message", + data={ + "content": "", + "model": "gpt-5.6-luna", + "interactionId": "ix-1", + "turnId": "0", + "toolRequests": [ + { + "toolCallId": "call_alpha", + "name": "view", + "arguments": {"path": "/fixture/workspace/alpha.txt"}, + "intentionSummary": "view alpha", + }, + { + "toolCallId": "call_beta", + "name": "view", + "arguments": {"path": "/fixture/workspace/beta.txt"}, + }, + ], + }, + ) + events = normalize_record(record) + assert [event["kind"] for event in events] == [ + "assistant_message", + "tool_request", + "tool_request", + ] + tool_events = events[1:] + assert tool_events[0]["id"] == f"copilot:event:{source_id}:tool:call_alpha" + assert tool_events[1]["id"] == f"copilot:event:{source_id}:tool:call_beta" + assert tool_events[0]["attributes"]["arguments"] == {"path": "/fixture/workspace/alpha.txt"} + assert tool_events[0]["attributes"]["intention_summary"] == "view alpha" + + +# --- tool execution --- + + +def test_tool_execution_complete_and_failure_kinds(): + success = _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/tool-ok", + native_type="tool.execution_complete", + data={"toolCallId": "call_ok", "success": True, "result": "17"}, + ) + failure = _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/tool-bad", + native_type="tool.execution_complete", + data={"toolCallId": "call_bad", "success": False, "result": "denied"}, + ) + assert _event_kinds(success) == ["tool_execution_complete"] + assert _event_kinds(failure) == ["tool_execution_failure"] + assert normalize_record(failure)[0]["attributes"]["result"] == "denied" + + +def test_tool_execution_start_preserves_arguments(): + record = _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/tool-start", + native_type="tool.execution_start", + data={ + "toolCallId": "call_start", + "toolName": "view", + "arguments": {"path": "/fixture/workspace/alpha.txt"}, + }, + ) + event = normalize_record(record)[0] + assert event["kind"] == "tool_execution_start" + assert event["attributes"]["name"] == "view" + assert event["attributes"]["arguments"] == {"path": "/fixture/workspace/alpha.txt"} + + +# --- permission, compaction, lifecycle --- + + +@pytest.mark.parametrize( + ("case_name", "expected_kind", "expected_classification"), + [ + ("permission", "permission_request", "main"), + ("compaction", "compaction", "checkpoint"), + ("auxiliary_title_generation", "auxiliary_model_call", "title_generation"), + ], +) +def test_reconciliation_case_event_normalization( + case_name: str, + expected_kind: str, + expected_classification: str, +) -> None: + case = _load_json(RECON_CASES / "cases.json")[case_name] + projection, _ = _build_semantics_from_case_records(case["input_records"]) + matching = [event for event in projection["events"] if event["kind"] == expected_kind] + assert matching, f"expected {expected_kind} in {case_name}" + assert matching[0]["classification"] == expected_classification + + +def _build_semantics_from_case_records(records: list[SourceRecord]) -> tuple[dict[str, Any], dict[str, Any]]: + from thirdeye.platforms.copilot.tracing import build_semantics + + return build_semantics(records, {}) + + +def test_permission_hook_maps_without_becoming_tool_execution_role(): + case = _load_json(RECON_CASES / "cases.json")["permission"] + event = normalize_record(case["input_records"][0])[0] + assert event["kind"] == "permission_request" + assert event["source_references"][0]["role"] == "permission_request" + + +def test_compaction_checkpoint_classification(): + case = _load_json(RECON_CASES / "cases.json")["compaction"] + transcript = case["input_records"][0] + event = normalize_record(transcript)[0] + assert event["kind"] == "compaction" + assert event["classification"] == "checkpoint" + assert event["source_references"][0]["role"] == "checkpoint" + + +def test_session_shutdown_is_shutdown_validation(): + record = _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/shutdown", + native_type="session.shutdown", + data={"totalNanoAiu": 123}, + ) + event = normalize_record(record)[0] + assert event["kind"] == "session_shutdown" + assert event["classification"] == "shutdown_validation" + + +def test_subagent_events_use_nested_child_role(): + record = _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/sub-start", + native_type="subagent.started", + data={"toolCallId": "call_parent_task", "agentName": "explore"}, + agent="bf8cb9f3-2097-4db0-a3c8-78a2653b2106", + ) + event = normalize_record(record)[0] + assert event["kind"] == "subagent_started" + assert event["source_references"][0]["role"] == "nested_child" + + +# --- unknown and hook observations --- + + +def test_unknown_native_type_is_preserved(): + record = _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/future", + native_type="future.event.v2", + data={"feature": "beta"}, + ) + event = normalize_record(record)[0] + assert event["kind"] == "unknown" + assert event["attributes"]["native_type"] == "future.event.v2" + assert event["attributes"]["raw_payload"]["type"] == "future.event.v2" + + +def test_hook_pretooluse_stays_hook_observation_not_transcript_execution(): + record = _hook_record( + source_id=f"hook/{NATIVE_SESSION_ID}/obs-pre", + event="preToolUse", + hook_payload={"toolName": "view", "toolArgs": {"path": "/tmp/a.txt"}}, + ) + event = normalize_record(record)[0] + assert event["kind"] == "tool_execution_start" + assert event["source_references"][0]["role"] == "hook" + + +def test_normalize_records_replays_in_order_without_deduplication(): + records = [ + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/a", + native_type="user.message", + data={"content": "one", "interactionId": "ix", "turnId": "0"}, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/b", + native_type="user.message", + data={"content": "two", "interactionId": "ix", "turnId": "0"}, + ), + ] + events = normalize_records(records) + assert len(events) == 2 + assert events[0]["attributes"]["interaction_id"] == "ix" + assert events[1]["source_ids"] == [records[1]["source_id"]] + + +def test_events_module_has_no_usage_or_export_imports() -> None: + source = Path(__import__("thirdeye.platforms.copilot.events", fromlist=["__file__"]).__file__) + text = source.read_text(encoding="utf-8") + for forbidden in ("usage.py", "attribution", "projection_store", "export_state"): + assert forbidden not in text diff --git a/tests/platforms/copilot/test_tracing.py b/tests/platforms/copilot/test_tracing.py new file mode 100644 index 0000000..72a2f59 --- /dev/null +++ b/tests/platforms/copilot/test_tracing.py @@ -0,0 +1,333 @@ +"""Behavioral tests for Copilot semantic projection (build_semantics).""" + +from __future__ import annotations + +import json +import shutil +from collections import Counter +from pathlib import Path +from typing import Any + +import pytest + +from thirdeye.platforms.copilot.identity import resolve_sources +from thirdeye.platforms.copilot.tracing import build_semantics +from thirdeye.platforms.copilot.transcript import read_transcript +from thirdeye.platforms.copilot.turns import build_turns +from thirdeye.platforms.copilot.types import SourceRecord + +FIXTURES = Path(__file__).parent / "fixtures" +RECON_CASES = FIXTURES / "reconciliation-cases" +NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" +CHILD_AGENT_ID = "bf8cb9f3-2097-4db0-a3c8-78a2653b2106" +SOURCE_KEY = "a" * 64 + + +def _load_json(path: Path) -> Any: + return json.loads(path.read_text(encoding="utf-8")) + + +def _drain_cli_transcript(home: Path) -> list[SourceRecord]: + session_dir = home / "session-state" / NATIVE_SESSION_ID + session_dir.mkdir(parents=True, exist_ok=True) + shutil.copy(FIXTURES / "events.jsonl", session_dir / "events.jsonl") + (session_dir / "workspace.yaml").write_text("cwd: /sanitized/workspace\n", encoding="utf-8") + paths = resolve_sources(home) + cursor: dict[str, Any] = {} + records: list[SourceRecord] = [] + while True: + slice_ = read_transcript(paths, NATIVE_SESSION_ID, cursor) + records.extend(slice_["records"]) + cursor = slice_["next_cursor"] + if slice_["exhausted"]: + break + return [record for record in records if record["source_kind"] == "transcript"] + + +@pytest.fixture +def cli_transcript_records(tmp_path: Path) -> list[SourceRecord]: + return _drain_cli_transcript(tmp_path / "copilot-home") + + +def _transcript_record( + *, + source_id: str, + native_type: str, + data: dict[str, Any], + ts: str = "2026-09-10T17:08:24.503Z", + agent: str | None = None, +) -> SourceRecord: + payload: dict[str, Any] = { + "type": native_type, + "data": data, + "id": source_id.rsplit("/", 1)[-1], + "timestamp": ts, + "schema_version": 1, + } + if agent is not None: + payload["agentId"] = agent + return { + "source_id": source_id, + "source_kind": "transcript", + "native_session_id": NATIVE_SESSION_ID, + "ts": ts, + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": payload, + "locator": {"file": "events.jsonl", "native_event_id": payload["id"]}, + } + + +# --- observed CLI fixture --- + + +def test_cli_fixture_reconstructs_two_main_interactions_and_explore_child( + cli_transcript_records: list[SourceRecord], +) -> None: + projection, state = build_semantics(cli_transcript_records, {}) + turns = projection["turns"] + + assert len(turns) == 2 + assert [turn["output_message"] for turn in turns] == ["42", "42"] + assert [turn["attributes"]["interaction_id"] for turn in turns] == [ + "6d2b89fd-a653-430c-b532-b0936d72eb42", + "793d3703-6f4a-4814-8877-34a7325848ce", + ] + + child_turn = turns[1]["subagents"][0] + assert child_turn["output_message"] == "42" + assert child_turn["attributes"]["agent_id"] == CHILD_AGENT_ID + assert child_turn["attributes"]["parent_tool_call_id"] == "call_qx4FH5DADTeT1qVLb37HNpBk" + assert child_turn["attributes"]["interaction_id"] == "7c0fa097-c0e2-48da-b2b6-fcfc1ad83a6b" + + assert projection["pending"] == [] + assert state["open_interactions"] == {} + + +def test_cli_fixture_counts_five_tools_and_six_call_candidates( + cli_transcript_records: list[SourceRecord], +) -> None: + projection, _ = build_semantics(cli_transcript_records, {}) + tool_requests = [ + event for event in projection["events"] if event["kind"] == "tool_request" + ] + tool_starts = [ + event for event in projection["events"] if event["kind"] == "tool_execution_start" + ] + tool_completes = [ + event for event in projection["events"] if event["kind"] == "tool_execution_complete" + ] + + assert len(tool_requests) == 5 + assert len(tool_starts) == 5 + assert len(tool_completes) == 5 + assert len(projection["call_candidates"]) == 6 + + tool_ids = sorted(event["attributes"]["tool_call_id"] for event in tool_requests) + assert tool_ids == [ + "call_Jeh7IbrUHaq4jVdxtyQCVrns", + "call_YSSva4HCniiETlxdGGjcrHbh", + "call_ayHplfzxjRFMTCpmTKEFhCSJ", + "call_qx4FH5DADTeT1qVLb37HNpBk", + "call_zZncCGtp1twgcL2eoFUNwInh", + ] + + +def test_cli_fixture_replay_is_deterministic(cli_transcript_records: list[SourceRecord]) -> None: + first, first_state = build_semantics(cli_transcript_records, {}) + second, _ = build_semantics(cli_transcript_records, first_state) + third, _ = build_semantics(cli_transcript_records, {}) + + assert first["events"] == third["events"] + assert first["turns"] == third["turns"] + assert first["call_candidates"] == third["call_candidates"] + assert second["events"] == third["events"] + + +def test_build_semantics_does_not_import_usage_or_attribution_modules() -> None: + for module_name in ("tracing", "turns", "events"): + source = Path( + __import__(f"thirdeye.platforms.copilot.{module_name}", fromlist=["__file__"]).__file__ + ) + text = source.read_text(encoding="utf-8") + for forbidden in ("usage.py", "attribution", "projection_store", "export_state"): + assert forbidden not in text + + +# --- interaction grouping and turn completion --- + + +def test_tool_cycle_does_not_complete_user_turn_without_final_answer() -> None: + case = _load_json(RECON_CASES / "semantic-projection.json") + projection, state = build_semantics(case["input_records"], {}) + + assert projection["turns"] == [] + pending_kinds = Counter(item["kind"] for item in projection["pending"]) + assert pending_kinds["open_interaction"] == 1 + assert pending_kinds["missing_identity"] == 1 + + open_key = "6d2b89fd-a653-430c-b532-b0936d72eb42|main" + assert open_key in state["open_interactions"] + assert state["open_interactions"][open_key]["pending_tool_call_ids"] == [ + "call_YSSva4HCniiETlxdGGjcrHbh", + "call_ayHplfzxjRFMTCpmTKEFhCSJ", + ] + + +def test_partial_user_prompt_stays_open_with_pending_item() -> None: + case = _load_json(RECON_CASES / "cases.json")["abort"] + projection, state = build_semantics(case["input_records"], {}) + + assert projection["turns"] == [] + assert any(item["kind"] == "open_interaction" for item in projection["pending"]) + assert "6d2b89fd-a653-430c-b532-b0936d72eb42|main" in state["open_interactions"] + + +def test_missing_interaction_id_creates_pending_not_fabricated_turn() -> None: + record = _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/user-no-ix", + native_type="user.message", + data={"content": "orphan prompt", "turnId": "0"}, + ) + _, _, pending, _ = build_turns([record]) + + assert any(item["kind"] == "missing_identity" for item in pending) + assert all(item["kind"] != "open_interaction" for item in pending) + + +def test_incomplete_tool_pair_when_execution_lacks_request() -> None: + execution = _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/exec-orphan", + native_type="tool.execution_start", + data={"toolCallId": "call_orphan", "toolName": "view", "arguments": {}}, + ) + _, _, pending, _ = build_turns([execution]) + + assert pending == [ + { + "id": "pending:tool:call_orphan", + "kind": "incomplete_tool_pair", + "reason": "tool execution start has no requesting assistant message", + "source_ids": [execution["source_id"]], + "evidence": ["tool_call_id:call_orphan"], + } + ] + + +def test_identical_concurrent_tools_pair_by_tool_call_id() -> None: + case = _load_json(RECON_CASES / "cases.json")["identical_concurrent_tools"] + projection, _ = build_semantics(case["input_records"], {}) + expected = case["expected"] + + start_events = [ + event for event in projection["events"] if event["kind"] == "tool_execution_start" + ] + assert sorted(event["id"] for event in start_events) == sorted(expected["event_ids"]) + assert sorted(event["attributes"]["tool_call_id"] for event in start_events) == sorted( + expected["tool_call_ids"] + ) + + candidate = projection["call_candidates"][0] + assert candidate["tool_call_ids"] == expected["tool_call_ids"] + assert len({event["attributes"]["tool_call_id"] for event in start_events}) == 2 + + +# --- nested child ownership --- + + +def test_nested_child_partial_records_do_not_create_main_turn_from_hook_session_id() -> None: + case = _load_json(RECON_CASES / "cases.json")["nested_child"] + projection, _ = build_semantics(case["input_records"], {}) + + assert projection["turns"] == [] + assert {event["kind"] for event in projection["events"]} == { + "subagent_started", + "user_prompt", + "prompt_transformation", + } + + hook_only = [case["input_records"][2]] + hook_projection, _ = build_semantics(hook_only, {}) + assert hook_projection["turns"] == [] + + +def test_child_turn_nests_under_parent_when_full_fixture_replayed( + cli_transcript_records: list[SourceRecord], +) -> None: + projection, _ = build_semantics(cli_transcript_records, {}) + parent = next( + turn + for turn in projection["turns"] + if turn["attributes"]["interaction_id"] == "793d3703-6f4a-4814-8877-34a7325848ce" + ) + assert len(parent["subagents"]) == 1 + child = parent["subagents"][0] + assert child["attributes"]["agent_id"] == CHILD_AGENT_ID + assert child["attributes"]["parent_tool_call_id"] == "call_qx4FH5DADTeT1qVLb37HNpBk" + assert all(turn["attributes"].get("agent_id") is None for turn in projection["turns"]) + + +# --- prior state retention --- + + +def test_prior_open_interaction_state_is_retained_when_replay_still_open() -> None: + case = _load_json(RECON_CASES / "cases.json")["abort"] + _, state = build_semantics(case["input_records"], {}) + prior_item = dict(state["open_interactions"]["6d2b89fd-a653-430c-b532-b0936d72eb42|main"]) + prior_item["note"] = "retained-from-incremental-caller" + + _, merged_state = build_semantics(case["input_records"], {"open_interactions": state["open_interactions"]}) + retained = merged_state["open_interactions"]["6d2b89fd-a653-430c-b532-b0936d72eb42|main"] + assert retained["interaction_id"] == prior_item["interaction_id"] + assert retained["stored_turn_id"] == prior_item["stored_turn_id"] + + # Simulate a key only present in prior state (unfinished partition from earlier chunk). + orphan_key = "orphan|main" + prior = { + "open_interactions": { + orphan_key: { + "interaction_id": "orphan-interaction", + "agent_id": None, + "stored_turn_id": f"copilot:turn:{SOURCE_KEY}:{NATIVE_SESSION_ID}:orphan-interaction", + "source_ids": ["kept/source"], + "last_event_source_id": "kept/source", + "start_ts": "2026-09-10T17:08:00.000Z", + "pending_tool_call_ids": [], + } + } + } + _, merged = build_semantics(case["input_records"], prior) + assert orphan_key in merged["open_interactions"] + + +# --- semantic projection contract slice --- + + +def test_semantic_projection_fixture_normalizes_expected_events() -> None: + case = _load_json(RECON_CASES / "semantic-projection.json") + projection, _ = build_semantics(case["input_records"], {}) + expected_events = case["expected"]["events"] + + by_id = {event["id"]: event for event in projection["events"]} + for expected in expected_events: + actual = by_id[expected["id"]] + assert actual["kind"] == expected["kind"] + assert actual["classification"] == expected["classification"] + assert actual["ts"] == expected["ts"] + assert actual["source_ids"] == expected["source_ids"] + for key, value in expected["attributes"].items(): + assert actual["attributes"].get(key) == value + + turn_end_id = ( + "copilot:event:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/" + "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/0080e44c-ad62-4288-b2b2-061ec2b73d80" + ) + assert by_id[turn_end_id]["kind"] == "unknown" + + +def test_auxiliary_model_events_do_not_create_call_candidates() -> None: + case = _load_json(RECON_CASES / "cases.json")["auxiliary_title_generation"] + projection, _ = build_semantics(case["input_records"], {}) + + assert projection["call_candidates"] == [] + assert projection["turns"] == [] + assert projection["events"][0]["classification"] == "title_generation" From dc6b0d6cdb8ed1d606a919eef0d0afce6d08745c Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Fri, 11 Sep 2026 15:30:04 -0700 Subject: [PATCH 52/88] Add projection storage behavioral and migration tests. Cover commit/load/read paths, journal recovery, usage revision identity, rebuild equivalence, and V1 archive upgrade without live Copilot sources. Co-authored-by: Cursor --- tests/platforms/copilot/test_migration.py | 411 ++++++++++++++ .../copilot/test_projection_store.py | 521 ++++++++++++++++++ 2 files changed, 932 insertions(+) create mode 100644 tests/platforms/copilot/test_migration.py create mode 100644 tests/platforms/copilot/test_projection_store.py diff --git a/tests/platforms/copilot/test_migration.py b/tests/platforms/copilot/test_migration.py new file mode 100644 index 0000000..ceeb541 --- /dev/null +++ b/tests/platforms/copilot/test_migration.py @@ -0,0 +1,411 @@ +"""Migration, journal recovery, rebuild equivalence, and competing projection commits.""" + +from __future__ import annotations + +import json +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path +from typing import Any + +import pytest + +import thirdeye.platforms.copilot.projection_state as projection_state_mod +from thirdeye._compat import fsops +from thirdeye.config import Config +from thirdeye.paths import session_dir, usage_jsonl_path +from thirdeye.platforms.copilot.archive import commit_batch +from thirdeye.platforms.copilot.constants import PLATFORM_NAME +from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id +from thirdeye.platforms.copilot.projection_state import ( + empty_projection_state, + projection_journal_path, + projection_state_path, + publish_projection_document, + read_projection_document, +) +from thirdeye.platforms.copilot.projection_store import ( + commit_projection, + load_projection_state, + read_projected_turns, + reset_projection_state, +) +from thirdeye.platforms.copilot.types import Projection, SourceBatch, SourcePaths, SourceRecord +from thirdeye.usage.read import iter_calls +from thirdeye.usage.types import UsageRow + +NATIVE_ID = "session-migrate" +INTERACTION_ID = "interaction-migrate-1" + + +def _record(source_id: str, *, ts: str = "2026-09-10T17:08:24.000Z") -> SourceRecord: + return { + "source_id": source_id, + "source_kind": "transcript", + "native_session_id": NATIVE_ID, + "ts": ts, + "observed_at": "2026-09-10T17:08:25.000Z", + "payload": {"schema_version": 1, "type": "user.message"}, + "locator": {"file": "events.jsonl", "offset": 0}, + } + + +def _batch(paths: SourcePaths, records: list[SourceRecord]) -> SourceBatch: + return { + "source_key": paths["source_key"], + "native_session_id": NATIVE_ID, + "cwd": "/proj", + "records": records, + "next_cursor": {"generation": 1}, + "diagnostics": [], + } + + +def _usage_row(*, call_id: str, input_tokens: int, session_id: str) -> UsageRow: + return UsageRow( + session_id=session_id, + seq=0, + call_id=call_id, + ts="2026-09-10T17:08:30.000Z", + platform=PLATFORM_NAME, + provider_name="openai", + response_model="gpt-5.6-luna", + input_tokens=input_tokens, + output_tokens=10, + ) + + +def _main_turn(*, source_ids: list[str]) -> dict[str, Any]: + return { + "turn_id": "copilot:turn:key:session:interaction-migrate-1", + "start_ts": "2026-09-10T17:08:24.000Z", + "end_ts": "2026-09-10T17:08:28.000Z", + "input_message": "hello", + "output_message": "done", + "status": "completed", + "llm_calls": [], + "permission_requests": [], + "subagents": [], + "attributes": {"interaction_id": INTERACTION_ID}, + "source_ids": source_ids, + } + + +def _normalized_event(event_id: str, *, source_ids: list[str]) -> dict[str, Any]: + return { + "id": event_id, + "kind": "user_prompt", + "classification": "main", + "initiator": "user", + "source_ids": source_ids, + "attributes": {"interaction_id": INTERACTION_ID}, + } + + +def _projection(**overrides: Any) -> Projection: + base: Projection = { + "normalized_events": [], + "turns": [], + "usage_rows": [], + "attributions": [], + "pending": [], + "diagnostics": [], + } + base.update(overrides) + return base + + +@contextmanager +def fault_at(point: str) -> Iterator[list[str]]: + seen: list[str] = [] + + def injector(name: str) -> None: + seen.append(name) + if name == point: + raise RuntimeError(f"injected fault at {point}") + + projection_state_mod._fault_injector = injector + try: + yield seen + finally: + projection_state_mod._fault_injector = None + + +@pytest.fixture +def config(tmp_path: Path) -> Config: + return Config(root=tmp_path / "thirdeye") + + +@pytest.fixture +def paths(tmp_path: Path) -> SourcePaths: + home = tmp_path / "copilot-home" + home.mkdir() + return resolve_sources(home) + + +def _stored(config: Config, paths: SourcePaths) -> str: + return stored_session_id(paths, NATIVE_ID) + + +def _directory(config: Config, stored: str) -> Path: + return session_dir(config.root, PLATFORM_NAME, stored) + + +def _seed_v1_archive(config: Config, paths: SourcePaths) -> str: + commit_batch( + config, + paths, + _batch(paths, [_record("key/a/v1-one"), _record("key/a/v1-two", ts="2026-09-10T17:08:26.000Z")]), + ) + return _stored(config, paths) + + +def _sample_projection(stored: str) -> Projection: + return _projection( + normalized_events=[ + _normalized_event("evt-one", source_ids=["key/a/v1-one"]), + _normalized_event("evt-two", source_ids=["key/a/v1-two"]), + ], + turns=[_main_turn(source_ids=["key/a/v1-one", "key/a/v1-two"])], + usage_rows=[_usage_row(call_id="usage-src-1", input_tokens=111, session_id=stored)], + attributions=[ + { + "usage_source_id": "usage-src-1", + "logical_call_id": "logical-usage-1", + "stored_turn_id": "copilot:turn:key:session:interaction-migrate-1", + "agent_id": None, + "call_id": None, + "status": "matched", + "join_kind": "direct", + "evidence": [], + } + ], + ) + + +def test_v1_archive_first_projection_commit_is_upgrade_safe( + config: Config, paths: SourcePaths +) -> None: + stored = _seed_v1_archive(config, paths) + directory = _directory(config, stored) + + assert not projection_state_path(directory).exists() + counts = commit_projection(config, stored, _sample_projection(stored), empty_projection_state()) + + assert counts["events"] == 2 + assert counts["turns"] == 1 + assert counts["usage"] == 1 + assert projection_state_path(directory).is_file() + assert len(read_projected_turns(config, stored)) == 1 + + +@pytest.mark.parametrize( + "fault_point", + [ + "after_projection_journal", + "after_projection_state", + "after_projection_journal_clear", + ], +) +def test_projection_journal_crash_recovers_on_next_read( + config: Config, paths: SourcePaths, fault_point: str +) -> None: + stored = _seed_v1_archive(config, paths) + directory = _directory(config, stored) + projection = _sample_projection(stored) + + with fault_at(fault_point): + with pytest.raises(RuntimeError, match="injected fault"): + commit_projection(config, stored, projection, empty_projection_state()) + + if fault_point == "after_projection_journal": + assert projection_journal_path(directory).is_file() + + state = load_projection_state(config, stored) + assert state["commit_result"]["events"] == 2 + assert not projection_journal_path(directory).exists() + turns = read_projected_turns(config, stored) + assert len(turns) == 1 + assert len(turns[0]["events"]) == 2 + + +def test_orphan_journal_recovers_without_new_commit(config: Config, paths: SourcePaths) -> None: + stored = _seed_v1_archive(config, paths) + directory = _directory(config, stored) + document = read_projection_document(directory) + document["indexes"]["events"] = { + "evt-orphan": _normalized_event("evt-orphan", source_ids=["key/a/v1-one"]) + } + publish_projection_document(directory, document) + + with fault_at("after_projection_journal"): + with pytest.raises(RuntimeError): + publish_projection_document(directory, document) + + assert projection_journal_path(directory).is_file() + recovered = read_projection_document(directory) + assert "evt-orphan" in recovered["indexes"]["events"] + assert not projection_journal_path(directory).exists() + + +def test_load_projection_state_recovers_missing_usage_sidecar( + config: Config, paths: SourcePaths +) -> None: + stored = _seed_v1_archive(config, paths) + commit_projection(config, stored, _sample_projection(stored), empty_projection_state()) + directory = _directory(config, stored) + fsops.unlink(usage_jsonl_path(directory), missing_ok=True) + assert not usage_jsonl_path(directory).exists() + + load_projection_state(config, stored) + + rows = list(iter_calls(directory)) + assert len(rows) == 1 + assert rows[0].input_tokens == 111 + + +def test_rebuild_after_reset_matches_original_projection( + config: Config, paths: SourcePaths +) -> None: + stored = _seed_v1_archive(config, paths) + projection = _sample_projection(stored) + commit_projection(config, stored, projection, empty_projection_state()) + + before_turns = read_projected_turns(config, stored) + before_state = load_projection_state(config, stored) + + reset_projection_state(config, stored) + commit_projection(config, stored, projection, empty_projection_state()) + + after_turns = read_projected_turns(config, stored) + after_state = load_projection_state(config, stored) + + assert after_turns == before_turns + assert after_state["commit_result"] == before_state["commit_result"] + + +def test_competing_projection_commits_merge_indexes(config: Config, paths: SourcePaths) -> None: + stored = _seed_v1_archive(config, paths) + + first = _projection( + normalized_events=[_normalized_event("evt-a", source_ids=["key/a/v1-one"])], + usage_rows=[_usage_row(call_id="usage-a", input_tokens=50, session_id=stored)], + attributions=[ + { + "usage_source_id": "usage-a", + "logical_call_id": "logical-a", + "stored_turn_id": None, + "agent_id": None, + "call_id": None, + "status": "pending", + "join_kind": None, + "evidence": [], + } + ], + ) + second = _projection( + normalized_events=[_normalized_event("evt-b", source_ids=["key/a/v1-two"])], + usage_rows=[_usage_row(call_id="usage-b", input_tokens=75, session_id=stored)], + attributions=[ + { + "usage_source_id": "usage-b", + "logical_call_id": "logical-b", + "stored_turn_id": None, + "agent_id": None, + "call_id": None, + "status": "pending", + "join_kind": None, + "evidence": [], + } + ], + ) + + commit_projection(config, stored, first, empty_projection_state()) + commit_projection(config, stored, second, empty_projection_state()) + + state = load_projection_state(config, stored) + assert state["commit_result"]["events"] == 2 + assert state["commit_result"]["usage"] == 2 + + document = json.loads(projection_state_path(_directory(config, stored)).read_text()) + assert set(document["indexes"]["events"]) == {"evt-a", "evt-b"} + assert set(document["indexes"]["usage"]) == {"logical-a", "logical-b"} + + +def test_replay_same_event_id_replaces_without_duplicating( + config: Config, paths: SourcePaths +) -> None: + stored = _seed_v1_archive(config, paths) + first = _projection( + normalized_events=[ + { + **_normalized_event("evt-stable", source_ids=["key/a/v1-one"]), + "attributes": {"interaction_id": INTERACTION_ID, "note": "first"}, + } + ], + ) + second = _projection( + normalized_events=[ + { + **_normalized_event("evt-stable", source_ids=["key/a/v1-one"]), + "attributes": {"interaction_id": INTERACTION_ID, "note": "second"}, + } + ], + ) + + commit_projection(config, stored, first, empty_projection_state()) + commit_projection(config, stored, second, empty_projection_state()) + + document = json.loads(projection_state_path(_directory(config, stored)).read_text()) + events = document["indexes"]["events"] + assert len(events) == 1 + assert events["evt-stable"]["attributes"]["note"] == "second" + + +def test_usage_sidecar_rewrite_after_state_publication_crash( + config: Config, paths: SourcePaths, monkeypatch: pytest.MonkeyPatch +) -> None: + stored = _seed_v1_archive(config, paths) + directory = _directory(config, stored) + writes: list[int] = [] + + import thirdeye.platforms.copilot.projection_store as store_mod + + original_write_usage = store_mod._write_usage_index + + def counting_write_usage(path: Path, usage_index: dict[str, Any]) -> None: + writes.append(len(usage_index)) + if len(writes) == 2: + raise RuntimeError("crash while rewriting usage sidecar") + original_write_usage(path, usage_index) + + monkeypatch.setattr(store_mod, "_write_usage_index", counting_write_usage) + + commit_projection(config, stored, _sample_projection(stored), empty_projection_state()) + + with pytest.raises(RuntimeError, match="crash while rewriting usage sidecar"): + commit_projection( + config, + stored, + _projection( + usage_rows=[_usage_row(call_id="usage-src-2", input_tokens=222, session_id=stored)], + attributions=[ + { + "usage_source_id": "usage-src-2", + "logical_call_id": "logical-usage-1", + "stored_turn_id": None, + "agent_id": None, + "call_id": None, + "status": "matched", + "join_kind": "direct", + "evidence": [], + } + ], + ), + empty_projection_state(), + ) + + load_projection_state(config, stored) + rows = list(iter_calls(directory)) + assert len(rows) == 1 + assert rows[0].input_tokens == 222 diff --git a/tests/platforms/copilot/test_projection_store.py b/tests/platforms/copilot/test_projection_store.py new file mode 100644 index 0000000..69537bb --- /dev/null +++ b/tests/platforms/copilot/test_projection_store.py @@ -0,0 +1,521 @@ +"""Behavioral tests for Copilot V2 projection commit, load, and turn reads.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from thirdeye.config import Config +from thirdeye.paths import session_dir, usage_jsonl_path +from thirdeye.platforms.copilot.archive import commit_batch +from thirdeye.platforms.copilot.constants import PLATFORM_NAME, SOURCE_SCHEMA_VERSION +from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id +from thirdeye.platforms.copilot.projection_state import ( + empty_projection_state, + projection_journal_path, + projection_state_path, +) +from thirdeye.platforms.copilot.projection_store import ( + commit_projection, + load_projection_state, + read_projected_turns, + reset_projection_state, +) +from thirdeye.platforms.copilot.types import Projection, SourceBatch, SourcePaths, SourceRecord +from thirdeye.reader import SessionReader +from thirdeye.usage.read import iter_calls +from thirdeye.usage.types import UsageRow + +NATIVE_ID = "session-proj" +INTERACTION_ID = "interaction-main-1" + + +def _record( + source_id: str, + *, + native_session_id: str = NATIVE_ID, + source_kind: str = "transcript", + ts: str = "2026-09-10T17:08:24.000Z", +) -> SourceRecord: + return { + "source_id": source_id, + "source_kind": source_kind, + "native_session_id": native_session_id, + "ts": ts, + "observed_at": "2026-09-10T17:08:25.000Z", + "payload": {"schema_version": 1, "type": "user.message"}, + "locator": {"file": "events.jsonl", "offset": 0}, + } + + +def _batch( + paths: SourcePaths, + records: list[SourceRecord], + *, + next_cursor: dict[str, Any] | None = None, +) -> SourceBatch: + return { + "source_key": paths["source_key"], + "native_session_id": NATIVE_ID, + "cwd": "/proj", + "records": records, + "next_cursor": dict(next_cursor or {"generation": 1}), + "diagnostics": [], + } + + +def _usage_row( + *, + call_id: str, + input_tokens: int = 100, + output_tokens: int = 10, + session_id: str = "stored-session", + seq: int = 0, +) -> UsageRow: + return UsageRow( + session_id=session_id, + seq=seq, + call_id=call_id, + ts="2026-09-10T17:08:30.000Z", + platform=PLATFORM_NAME, + provider_name="openai", + response_model="gpt-5.6-luna", + input_tokens=input_tokens, + output_tokens=output_tokens, + ) + + +def _main_turn( + *, + turn_id: str = "copilot:turn:key:session:interaction-main-1", + interaction_id: str = INTERACTION_ID, + start_ts: str = "2026-09-10T17:08:24.000Z", + end_ts: str = "2026-09-10T17:08:28.000Z", + source_ids: list[str] | None = None, + agent_id: str | None = None, +) -> dict[str, Any]: + attributes: dict[str, Any] = {"interaction_id": interaction_id} + if agent_id is not None: + attributes["agent_id"] = agent_id + turn: dict[str, Any] = { + "turn_id": turn_id, + "start_ts": start_ts, + "end_ts": end_ts, + "input_message": "hello", + "output_message": "done", + "status": "completed", + "llm_calls": [], + "permission_requests": [], + "subagents": [], + "attributes": attributes, + } + if source_ids: + turn["source_ids"] = source_ids + return turn + + +def _normalized_event( + event_id: str, + *, + interaction_id: str = INTERACTION_ID, + source_ids: list[str], + stored_turn_id: str | None = None, +) -> dict[str, Any]: + attributes: dict[str, Any] = {"interaction_id": interaction_id} + if stored_turn_id is not None: + attributes["stored_turn_id"] = stored_turn_id + return { + "id": event_id, + "kind": "user_prompt", + "classification": "main", + "initiator": "user", + "source_ids": source_ids, + "attributes": attributes, + } + + +def _attribution( + *, + logical_call_id: str, + usage_source_id: str, + status: str = "matched", +) -> dict[str, Any]: + return { + "usage_source_id": usage_source_id, + "logical_call_id": logical_call_id, + "stored_turn_id": "copilot:turn:key:session:interaction-main-1", + "agent_id": None, + "call_id": None, + "status": status, + "join_kind": "direct", + "evidence": ["direct:usage_source_id"], + } + + +def _projection(**overrides: Any) -> Projection: + base: Projection = { + "normalized_events": [], + "turns": [], + "usage_rows": [], + "attributions": [], + "pending": [], + "diagnostics": [], + } + base.update(overrides) + return base + + +@pytest.fixture +def config(tmp_path: Path) -> Config: + return Config(root=tmp_path / "thirdeye") + + +@pytest.fixture +def paths(tmp_path: Path) -> SourcePaths: + home = tmp_path / "copilot-home" + home.mkdir() + return resolve_sources(home) + + +def _stored(config: Config, paths: SourcePaths) -> str: + return stored_session_id(paths, NATIVE_ID) + + +def _directory(config: Config, stored: str) -> Path: + return session_dir(config.root, PLATFORM_NAME, stored) + + +def _seed_archive(config: Config, paths: SourcePaths, records: list[SourceRecord]) -> str: + commit_batch(config, paths, _batch(paths, records)) + return _stored(config, paths) + + +def test_commit_projection_persists_indexes_and_usage_sidecar( + config: Config, paths: SourcePaths +) -> None: + stored = _seed_archive( + config, + paths, + [_record("key/a/event-1"), _record("key/a/event-2", ts="2026-09-10T17:08:26.000Z")], + ) + directory = _directory(config, stored) + + projection = _projection( + normalized_events=[ + _normalized_event("copilot:event:key/a/event-1", source_ids=["key/a/event-1"]), + _normalized_event("copilot:event:key/a/event-2", source_ids=["key/a/event-2"]), + ], + turns=[_main_turn(source_ids=["key/a/event-1", "key/a/event-2"])], + usage_rows=[_usage_row(call_id="src-usage-1", session_id=stored)], + attributions=[_attribution(logical_call_id="logical-1", usage_source_id="src-usage-1")], + pending=[{"id": "pending-1", "kind": "open_interaction", "message": "unfinished"}], + diagnostics=[ + { + "code": "missing_capability", + "severity": "info", + "message": "no direct join id", + "source_ids": [], + "details": {}, + } + ], + ) + next_state = empty_projection_state() + next_state["archive_source_ids"] = ["key/a/event-1", "key/a/event-2"] + + counts = commit_projection(config, stored, projection, next_state) + + assert counts == { + "events": 2, + "turns": 1, + "usage": 1, + "attributions": 1, + "pending": 1, + "diagnostics": 1, + } + assert projection_state_path(directory).is_file() + assert not projection_journal_path(directory).exists() + + usage_rows = list(iter_calls(directory)) + assert len(usage_rows) == 1 + assert usage_rows[0].call_id == "src-usage-1" + + state = load_projection_state(config, stored) + assert state["archive_source_ids"] == ["key/a/event-1", "key/a/event-2"] + assert state["commit_result"]["events"] == 2 + + +def test_read_projected_turns_join_archived_store_events( + config: Config, paths: SourcePaths +) -> None: + stored = _seed_archive( + config, + paths, + [_record("key/a/user"), _record("key/a/assistant", ts="2026-09-10T17:08:26.000Z")], + ) + turn_id = "copilot:turn:key:session:interaction-main-1" + + commit_projection( + config, + stored, + _projection( + normalized_events=[ + _normalized_event("evt-user", source_ids=["key/a/user"]), + _normalized_event("evt-assistant", source_ids=["key/a/assistant"]), + ], + turns=[_main_turn(turn_id=turn_id)], + ), + empty_projection_state(), + ) + + turns = read_projected_turns(config, stored) + assert len(turns) == 1 + turn = turns[0] + assert turn["turn_id"] == turn_id + assert turn["session_id"] == stored + assert turn["platform"] == PLATFORM_NAME + assert turn["cwd"] == "/proj" + assert [event["data"]["source_record"]["source_id"] for event in turn["events"]] == [ + "key/a/user", + "key/a/assistant", + ] + assert turn["start_seq"] == 0 + assert turn["end_seq"] == 1 + + +def test_child_agent_top_level_turns_are_excluded(config: Config, paths: SourcePaths) -> None: + stored = _seed_archive(config, paths, [_record("key/a/main")]) + main_turn = _main_turn() + child_turn = _main_turn( + turn_id="copilot:turn:key:session:child", + interaction_id="child-interaction", + agent_id="subagent-123", + ) + + commit_projection( + config, + stored, + _projection( + normalized_events=[_normalized_event("evt-main", source_ids=["key/a/main"])], + turns=[main_turn, child_turn], + ), + empty_projection_state(), + ) + + turns = read_projected_turns(config, stored) + assert [turn["turn_id"] for turn in turns] == [main_turn["turn_id"]] + + +def test_usage_revision_replaces_under_stable_logical_identity( + config: Config, paths: SourcePaths +) -> None: + stored = _seed_archive(config, paths, [_record("key/a/usage")]) + + first = _projection( + usage_rows=[_usage_row(call_id="src-rev-1", input_tokens=100, session_id=stored)], + attributions=[_attribution(logical_call_id="logical-1", usage_source_id="src-rev-1")], + ) + commit_projection(config, stored, first, empty_projection_state()) + + second = _projection( + usage_rows=[_usage_row(call_id="src-rev-2", input_tokens=150, session_id=stored, seq=1)], + attributions=[_attribution(logical_call_id="logical-1", usage_source_id="src-rev-2")], + ) + counts = commit_projection(config, stored, second, empty_projection_state()) + + assert counts["usage"] == 1 + directory = _directory(config, stored) + rows = list(iter_calls(directory)) + assert len(rows) == 1 + assert rows[0].input_tokens == 150 + assert rows[0].call_id == "src-rev-2" + + sidecar_lines = usage_jsonl_path(directory).read_text(encoding="utf-8").splitlines() + assert len(sidecar_lines) == 1 + + +def test_incremental_commits_merge_builder_state(config: Config, paths: SourcePaths) -> None: + stored = _seed_archive(config, paths, [_record("key/a/one"), _record("key/a/two")]) + + first_state = empty_projection_state() + first_state["archive_source_ids"] = ["key/a/one"] + first_state["semantic_state"] = { + "open_interactions": { + INTERACTION_ID: { + "interaction_id": INTERACTION_ID, + "agent_id": None, + "stored_turn_id": "turn-open", + "source_ids": ["key/a/one"], + "last_event_source_id": "key/a/one", + "start_ts": "2026-09-10T17:08:24.000Z", + "pending_tool_call_ids": [], + } + } + } + commit_projection( + config, + stored, + _projection( + normalized_events=[_normalized_event("evt-1", source_ids=["key/a/one"])], + ), + first_state, + ) + + second_state = empty_projection_state() + second_state["archive_source_ids"] = ["key/a/two"] + second_state["accounting_state"] = { + "logical_calls": { + "logical-1": { + "logical_call_id": "logical-1", + "generation": "gen-a", + "content_revision": "rev-a", + "metrics_digest": "sha256:abc", + "usage_source_id": "src-rev-1", + } + } + } + commit_projection( + config, + stored, + _projection( + normalized_events=[_normalized_event("evt-2", source_ids=["key/a/two"])], + ), + second_state, + ) + + merged = load_projection_state(config, stored) + assert merged["archive_source_ids"] == ["key/a/one", "key/a/two"] + assert INTERACTION_ID in merged["semantic_state"]["open_interactions"] + assert "logical-1" in merged["accounting_state"]["logical_calls"] + + document_events = json.loads(projection_state_path(_directory(config, stored)).read_text())[ + "indexes" + ]["events"] + assert set(document_events) == {"evt-1", "evt-2"} + + +def test_load_projection_state_returns_defensive_copy(config: Config, paths: SourcePaths) -> None: + stored = _seed_archive(config, paths, [_record("key/a/event")]) + state = empty_projection_state() + state["archive_source_ids"] = ["key/a/event"] + commit_projection(config, stored, _projection(), state) + + loaded = load_projection_state(config, stored) + loaded["archive_source_ids"].append("mutated") + again = load_projection_state(config, stored) + assert again["archive_source_ids"] == ["key/a/event"] + + +def test_commit_projection_requires_usage_row_instances(config: Config, paths: SourcePaths) -> None: + stored = _seed_archive(config, paths, [_record("key/a/event")]) + bad: Projection = _projection(usage_rows=[{"call_id": "not-a-row"}]) # type: ignore[list-item] + with pytest.raises(TypeError, match="UsageRow"): + commit_projection(config, stored, bad, empty_projection_state()) + + +def test_projection_reads_do_not_require_live_copilot_sources( + config: Config, paths: SourcePaths, tmp_path: Path +) -> None: + stored = _seed_archive( + config, + paths, + [_record("key/a/user"), _record("key/a/assistant", ts="2026-09-10T17:08:26.000Z")], + ) + commit_projection( + config, + stored, + _projection( + normalized_events=[ + _normalized_event("evt-user", source_ids=["key/a/user"]), + _normalized_event("evt-assistant", source_ids=["key/a/assistant"]), + ], + turns=[_main_turn()], + ), + empty_projection_state(), + ) + + copilot_home = tmp_path / "copilot-home" + assert copilot_home.exists() + for child in copilot_home.iterdir(): + if child.is_file(): + child.unlink() + else: + import shutil + + shutil.rmtree(child) + + turns = read_projected_turns(config, stored) + assert len(turns) == 1 + assert len(turns[0]["events"]) == 2 + archived = list(SessionReader(_directory(config, stored)).iter_events()) + assert len(archived) == 2 + + +def test_unfinished_projection_pending_does_not_block_later_capture( + config: Config, paths: SourcePaths +) -> None: + stored = _seed_archive(config, paths, [_record("key/a/first")]) + commit_projection( + config, + stored, + _projection( + pending=[{"id": "pending-open", "kind": "open_interaction", "message": "still open"}], + diagnostics=[ + { + "code": "missing_capability", + "severity": "warning", + "message": "partial semantics", + "source_ids": ["key/a/first"], + "details": {}, + } + ], + ), + empty_projection_state(), + ) + + commit_batch(config, paths, _batch(paths, [_record("key/a/second")], next_cursor={"generation": 2})) + + commit_projection( + config, + stored, + _projection( + normalized_events=[_normalized_event("evt-second", source_ids=["key/a/second"])], + ), + empty_projection_state(), + ) + + state = load_projection_state(config, stored) + assert state["commit_result"]["events"] == 1 + assert state["commit_result"]["pending"] == 1 + captured = { + event["data"]["source_record"]["source_id"] + for event in SessionReader(_directory(config, stored)).iter_events( + types={"copilot_transcript"} + ) + } + assert captured == {"key/a/first", "key/a/second"} + + +def test_reset_projection_state_removes_only_derived_files(config: Config, paths: SourcePaths) -> None: + stored = _seed_archive(config, paths, [_record("key/a/persist")]) + commit_projection( + config, + stored, + _projection( + usage_rows=[_usage_row(call_id="usage-1", session_id=stored)], + normalized_events=[_normalized_event("evt", source_ids=["key/a/persist"])], + turns=[_main_turn(source_ids=["key/a/persist"])], + ), + empty_projection_state(), + ) + directory = _directory(config, stored) + + reset_projection_state(config, stored) + + assert not projection_state_path(directory).exists() + assert not usage_jsonl_path(directory).exists() + archived = list(SessionReader(directory).iter_events()) + assert len(archived) == 1 + assert archived[0]["data"]["schema_version"] == SOURCE_SCHEMA_VERSION From b3f077da2f53ddfb185ed759adaf1f5f55b694bd Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Fri, 11 Sep 2026 15:30:17 -0700 Subject: [PATCH 53/88] Add recoverable accounting export transport --- src/thirdeye/otel_export.py | 226 ++++++++++++++++++++++++++++++++-- src/thirdeye/otel_worker.py | 101 ++++++++++++++- src/thirdeye/tracing/model.py | 10 ++ 3 files changed, 327 insertions(+), 10 deletions(-) diff --git a/src/thirdeye/otel_export.py b/src/thirdeye/otel_export.py index b3885e5..7435b37 100644 --- a/src/thirdeye/otel_export.py +++ b/src/thirdeye/otel_export.py @@ -532,6 +532,19 @@ def _write_job(thirdeye_home: Path, payload: dict[str, Any]) -> Path: return job_path +def _write_accounting_job(thirdeye_home: Path, payload: dict[str, Any]) -> Path: + """Persist one deterministic accounting job without creating duplicates.""" + jobs_dir = otel_jobs_dir(thirdeye_home) + jobs_dir.mkdir(parents=True, exist_ok=True) + job_id = str(payload["job_id"]) + digest = hashlib.sha256(job_id.encode("utf-8")).hexdigest() + job_path = jobs_dir / f"accounting-{digest}.json" + queued = {**payload, "state": "queued", "attempt": int(payload.get("attempt", 0))} + if not _atomic_create(job_path, json.dumps(queued, default=str)): + return job_path + return job_path + + def _spawn(job_path: Path) -> None: """Hand a job file to a detached ``thirdeye.otel_worker``. @@ -693,6 +706,54 @@ def export_spans( return False +def export_session_accounting( + config: Config, + session_dir_: Path, + session_id: str, + platform: str, + cwd: str, + accounting: dict[str, Any], + *, + captured_env: dict[str, str] | None = None, +) -> bool: + """Queue accounting that has no owning user turn.""" + if not config.logfire.enabled or not config.logfire.token: + return False + try: + accounting_id = str(accounting["accounting_id"]) + logical_span_id = f"accounting:{session_id}:{accounting_id}" + job_path = _write_accounting_job( + config.root, + { + "job_id": logical_span_id, + "kind": "session_accounting", + "session_dir": str(session_dir_), + "session_id": session_id, + "platform": platform, + "cwd": cwd, + "captured_attributes": _resolve_captured_attributes(config, captured_env), + "accounting_id": accounting_id, + "destination": "session-accounting-span", + "usage": dict(accounting["usage"]), + "attribution_status": str(accounting["attribution_status"]), + "agent_id": accounting.get("agent_id"), + "attributes": dict(accounting.get("attributes") or {}), + "span_id": logical_span_id, + }, + ) + _spawn(job_path) + return True + except Exception as exc: + log_capture_error( + thirdeye_home=config.root, + phase="logfire_session_accounting_export_spawn", + error=exc, + platform=platform, + session_id=session_id, + ) + return False + + def export_subagent_turn( config: Config, session_dir_: Path, @@ -886,6 +947,66 @@ def _export_subagent_turn_inner( claim_path.write_text("sent", encoding="utf-8", newline="\n") +def _export_session_accounting_inner( + *, + config: Config, + session_dir_: Path, + session_id: str, + platform: str, + cwd: str, + accounting: dict[str, Any], +) -> None: + """Emit a session-owned accounting span under the durable session root.""" + instance = _get_instance(config, platform) + if instance is None: + return + usage = dict(accounting.get("usage") or {}) + fallback_ts = _accounting_timestamp(usage, "1970-01-01T00:00:00.000Z") + tracer = instance.config.get_tracer_provider().get_tracer("thirdeye") + root_path = otel_state_path(session_dir_) + parent, root_lock = _root_or_ownership(root_path) + try: + if parent is None and root_lock is None: + raise RuntimeError("could not resolve or create session root") + if parent is None: + root_ns = _ts_to_ns(fallback_ts) + derived = ( + trace_id_for_session(platform, session_id), + root_span_id_for_session(platform, session_id), + ) + parent, created_root = _create_root_atomic(root_path, *derived) + if created_root: + root_span = _start_span_with_id( + tracer, + "session", + derived[1], + trace_id=derived[0], + start_time=root_ns, + attributes=_flatten_attrs( + { + **(_captured_attributes.get() or {}), + **_identity_attributes( + session_id=session_id, platform=platform, cwd=cwd + ), + } + ), + ) + root_span.end(end_time=root_ns) + finally: + if root_lock is not None: + fsops.unlink(root_lock, missing_ok=True) + _export_accounting_span( + tracer, + _parent_context(*parent), + accounting, + platform=platform, + session_id=session_id, + fallback_ts=fallback_ts, + ) + if instance.force_flush(timeout_millis=_FLUSH_TIMEOUT_MS) is False: + raise RuntimeError("session accounting export was not flushed") + + @lru_cache(maxsize=128) def _repo_name(cwd: str) -> str | None: """The name of the git repository `cwd` sits in, or None outside one. @@ -1110,6 +1231,67 @@ def _chat_attributes( ) +def _accounting_attributes(accounting: dict[str, Any]) -> dict[str, Any]: + """Project an immutable usage row onto its one allowed export span.""" + usage = dict(accounting.get("usage") or {}) + attributes = _merge_raw( + usage, + accounting.get("attributes"), + { + "thirdeye.accounting.id": str(accounting["accounting_id"]), + "thirdeye.accounting.attribution_status": str(accounting["attribution_status"]), + "thirdeye.accounting.agent_id": accounting.get("agent_id"), + "thirdeye.accounting.usage": usage, + }, + ) + # Native Copilot billing units are not an estimated USD model price. + if any(key.startswith("copilot.billing") or "nano_aiu" in key.lower() for key in attributes): + attributes["thirdeye.accounting.billing.kind"] = "copilot-native-unit" + return _flatten_attrs(attributes) + + +def _accounting_span_id( + platform: str, session_id: str, accounting_id: str, turn_id: str | None = None +) -> int: + """Return an OTel-safe deterministic ID for an accounting fallback span.""" + return chat_span_id(platform, session_id, f"accounting:{turn_id or 'session'}:{accounting_id}") + + +def _accounting_timestamp(usage: dict[str, Any], fallback: str) -> str: + value = usage.get("ts") + if isinstance(value, str): + try: + _ts_to_ns(value) + except ValueError: + pass + else: + return value + return fallback + + +def _export_accounting_span( + tracer: Any, + parent_ctx: Any, + accounting: dict[str, Any], + *, + platform: str, + session_id: str, + fallback_ts: str, + turn_id: str | None = None, +) -> None: + usage = dict(accounting.get("usage") or {}) + ts = _accounting_timestamp(usage, fallback_ts) + span = _start_span_with_id( + tracer, + "accounting", + _accounting_span_id(platform, session_id, str(accounting["accounting_id"]), turn_id), + parent_ctx=parent_ctx, + start_time=_ts_to_ns(ts), + attributes=_accounting_attributes(accounting), + ) + span.end(end_time=_ts_to_ns(ts)) + + def _tool_attributes( attributes: dict[str, Any], *, @@ -1335,6 +1517,11 @@ def _export_turn_subtree( turn_span.end(end_time=_ts_to_ns(turn["end_ts"])) turn_ctx = turn_span.get_span_context() turn_parent_ctx = _parent_context(turn_ctx.trace_id, turn_ctx.span_id) + accounting_by_call = { + str(accounting["call_id"]): accounting + for accounting in turn.get("accounting_calls") or [] + if accounting.get("call_id") is not None + } for interaction in turn.get("interactions") or []: kind = interaction["kind"] @@ -1362,20 +1549,26 @@ def _export_turn_subtree( for llm_call in turn["llm_calls"]: model = llm_call.get("model") or "" + call_attrs = _chat_attributes( + llm_call, + session_id=session_id, + platform=platform, + cwd=cwd, + turn_id=turn["turn_id"], + turn_span_id=turn.get("turn_span_id"), + ) + accounting = accounting_by_call.get(str(llm_call["call_id"])) + if accounting is not None: + # Actual accounting, rather than the semantic LLM record, owns + # the token fields for this chat span. + call_attrs = _flatten_attrs(_merge_raw(call_attrs, _accounting_attributes(accounting))) call_span = _start_span_with_id( tracer, f"chat {model}" if model else "chat", chat_span_id(platform, session_id, llm_call["call_id"]), parent_ctx=turn_parent_ctx, start_time=_ts_to_ns(llm_call["start_ts"]), - attributes=_chat_attributes( - llm_call, - session_id=session_id, - platform=platform, - cwd=cwd, - turn_id=turn["turn_id"], - turn_span_id=turn.get("turn_span_id"), - ), + attributes=call_attrs, ) call_span.end(end_time=_ts_to_ns(llm_call["end_ts"])) call_ctx = call_span.get_span_context() @@ -1402,6 +1595,23 @@ def _export_turn_subtree( ) tool_span.end(end_time=_ts_to_ns(tool_call["end_ts"])) + known_call_ids = {str(call["call_id"]) for call in turn["llm_calls"]} + for accounting in turn.get("accounting_calls") or []: + call_id = accounting.get("call_id") + if call_id is not None and str(call_id) in known_call_ids: + continue + # Unknown call ids do not prove chat-span ownership. Keep the usage on + # a concrete user-turn accounting span instead of inventing a chat. + _export_accounting_span( + tracer, + turn_parent_ctx, + accounting, + platform=platform, + session_id=session_id, + fallback_ts=turn["end_ts"], + turn_id=turn["turn_id"], + ) + for orphan in turn.get("orphan_tool_calls") or []: parent_call_id = orphan["parent_call_id"] tool_call = orphan["tool_call"] diff --git a/src/thirdeye/otel_worker.py b/src/thirdeye/otel_worker.py index 2733310..7503584 100644 --- a/src/thirdeye/otel_worker.py +++ b/src/thirdeye/otel_worker.py @@ -19,12 +19,77 @@ from __future__ import annotations import json +import os import sys +import time from pathlib import Path from typing import Any from thirdeye._compat import fsops +_JOB_CLAIM_STALE_S = 30.0 + + +def _write_job_state(job_path: Path, payload: dict[str, Any]) -> None: + """Atomically publish a claim/retry state without partial JSON.""" + temporary = job_path.with_name(f".{job_path.name}.{os.getpid()}.tmp") + temporary.write_text(json.dumps(payload, default=str), encoding="utf-8", newline="\n") + fsops.replace(temporary, job_path) + fsops.sync_directory(job_path.parent) + + +def _job_claim_path(job_path: Path) -> Path: + return job_path.with_suffix(f"{job_path.suffix}.claim") + + +def _create_job_claim(path: Path) -> bool: + try: + with path.open("x", encoding="utf-8") as handle: + handle.write(str(os.getpid())) + except FileExistsError: + return False + return True + + +def _claim_job(job_path: Path, payload: dict[str, Any]) -> dict[str, Any] | None: + """Recover a claimed-but-unsent job and claim it for this worker. + + Remote flush and local deletion cannot be a single transaction. A crash in + between can retry a deterministic span, reducing but not eliminating + remote duplicates. + """ + if payload.get("state") == "emitted": + fsops.unlink(job_path, missing_ok=True) + _release_job_claim(job_path) + return None + claim_path = _job_claim_path(job_path) + if not _create_job_claim(claim_path): + try: + stale = time.time() - claim_path.stat().st_mtime > _JOB_CLAIM_STALE_S + except OSError: + stale = True + if not stale: + return None + fsops.unlink(claim_path, missing_ok=True) + if not _create_job_claim(claim_path): + return None + claimed = dict(payload) + claimed["state"] = "claimed" + claimed["attempt"] = int(payload.get("attempt", 0)) + _write_job_state(job_path, claimed) + return claimed + + +def _release_job_claim(job_path: Path) -> None: + fsops.unlink(_job_claim_path(job_path), missing_ok=True) + + +def _retry_job(job_path: Path, payload: dict[str, Any]) -> None: + retry = dict(payload) + retry["state"] = "queued" + retry["attempt"] = int(payload.get("attempt", 0)) + 1 + _write_job_state(job_path, retry) + def main(argv: list[str] | None = None) -> None: argv = sys.argv[1:] if argv is None else argv @@ -35,14 +100,21 @@ def main(argv: list[str] | None = None) -> None: payload = json.loads(job_path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError) as exc: _log_worker_failure(kind="job_read", payload={}, error=exc) - return - finally: fsops.unlink(job_path, missing_ok=True) + return + try: + payload = _claim_job(job_path, payload) + except Exception as exc: + _log_worker_failure(kind="job_claim", payload=payload, error=exc) + return + if payload is None: + return from thirdeye.otel_export import _captured_attributes token = _captured_attributes.set(payload.get("captured_attributes") or {}) kind = payload.get("kind") + delivered = False try: from thirdeye.config import Config @@ -83,10 +155,35 @@ def main(argv: list[str] | None = None) -> None: parent_span_id=payload["parent_span_id"], turn=payload["turn"], ) + elif kind == "session_accounting": + from thirdeye.otel_export import _export_session_accounting_inner + + _export_session_accounting_inner( + config=config, + session_dir_=Path(payload["session_dir"]), + session_id=payload["session_id"], + platform=payload["platform"], + cwd=payload["cwd"], + accounting={ + "accounting_id": payload["accounting_id"], + "usage": payload["usage"], + "attribution_status": payload["attribution_status"], + "agent_id": payload.get("agent_id"), + "attributes": payload.get("attributes") or {}, + }, + ) + delivered = True except Exception as exc: + try: + _retry_job(job_path, payload) + except Exception: + pass _log_worker_failure(kind=str(kind or ""), payload=payload, error=exc) finally: _captured_attributes.reset(token) + _release_job_claim(job_path) + if delivered: + fsops.unlink(job_path, missing_ok=True) def _log_worker_failure(*, kind: str, payload: dict[str, Any], error: Exception) -> None: diff --git a/src/thirdeye/tracing/model.py b/src/thirdeye/tracing/model.py index e5f257f..a2f2c9e 100644 --- a/src/thirdeye/tracing/model.py +++ b/src/thirdeye/tracing/model.py @@ -130,6 +130,16 @@ class SessionAccountingJobDict(TypedDict): usage: dict[str, Any] attribution_status: str span_id: str + # The durable generic job can retain an agent owner even though there is + # intentionally no fabricated user-turn owner. + agent_id: NotRequired[str | None] + attributes: NotRequired[dict[str, Any]] + # Worker envelope fields remain optional so the serializable public job + # shape above is usable by placement ledgers without filesystem context. + session_dir: NotRequired[str] + platform: NotRequired[str] + cwd: NotRequired[str] + captured_attributes: NotRequired[dict[str, Any]] class TurnAccountingJobDict(TypedDict): From 434260bcb970b8664e5f3b03d0c7ec8e00f95b29 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Fri, 11 Sep 2026 15:32:10 -0700 Subject: [PATCH 54/88] Add comprehensive tests for accounting export transport. Cover session and turn accounting spans, deterministic job deduplication, worker claim recovery, and retry-on-failure semantics for the generic export path. Co-authored-by: Cursor --- tests/shared/test_accounting_export.py | 515 +++++++++++++++++++++++++ 1 file changed, 515 insertions(+) create mode 100644 tests/shared/test_accounting_export.py diff --git a/tests/shared/test_accounting_export.py b/tests/shared/test_accounting_export.py new file mode 100644 index 0000000..f6ed541 --- /dev/null +++ b/tests/shared/test_accounting_export.py @@ -0,0 +1,515 @@ +from __future__ import annotations + +import hashlib +import json +import time +from pathlib import Path +from typing import Any + +import pytest + +from thirdeye import otel_export, otel_worker +from thirdeye.config import Config, LogfireSettings +from thirdeye.paths import otel_jobs_dir +from thirdeye.span_ids import chat_span_id +from thirdeye.usage.types import UsageRow + +pytest.importorskip("logfire") + +from logfire.testing import TestExporter # noqa: E402 +from opentelemetry.sdk.trace.export import SimpleSpanProcessor # noqa: E402 + +_FIXTURES = ( + Path(__file__).resolve().parents[1] + / "platforms" + / "copilot" + / "fixtures" + / "reconciliation-cases" +) +_TRANSPORT = json.loads((_FIXTURES / "transport.json").read_text(encoding="utf-8")) + + +@pytest.fixture(autouse=True) +def _reset_state(): + otel_export._state["attempted"] = False + otel_export._state["instance"] = None + otel_export._state["id_generator"] = None + yield + otel_export._state["attempted"] = False + otel_export._state["instance"] = None + otel_export._state["id_generator"] = None + + +@pytest.fixture +def exporter(): + return TestExporter() + + +@pytest.fixture +def wired_instance(exporter, monkeypatch: pytest.MonkeyPatch): + import logfire + + instance = logfire.configure( + send_to_logfire=False, + console=False, + additional_span_processors=[SimpleSpanProcessor(exporter)], + advanced=logfire.AdvancedOptions(id_generator=otel_export._id_generator()), + ) + monkeypatch.setattr(otel_export, "_get_instance", lambda config, platform: instance) + return instance + + +@pytest.fixture +def enabled_config(tmp_path: Path) -> Config: + return Config( + root=tmp_path, + logfire=LogfireSettings(enabled=True, token="fake-token"), + ) + + +def _turn(**overrides: Any) -> dict[str, Any]: + defaults: dict[str, Any] = dict( + turn_id="turn_1", + start_ts="2026-01-01T00:00:00.000Z", + end_ts="2026-01-01T00:00:05.000Z", + input_message="hi", + output_message="hello", + status="completed", + llm_calls=[], + permission_requests=[], + subagents=[], + attributes={}, + ) + defaults.update(overrides) + return defaults + + +def _llm_call(**overrides: Any) -> dict[str, Any]: + defaults: dict[str, Any] = dict( + call_id="call_1", + provider="anthropic", + model="claude-sonnet-5", + start_ts="2026-01-01T00:00:01.000Z", + end_ts="2026-01-01T00:00:02.000Z", + input_messages=[{"role": "user", "parts": [{"type": "text", "content": "hi"}]}], + output_messages=[{"role": "assistant", "parts": [{"type": "text", "content": "hello"}]}], + usage={"input_tokens": 100, "output_tokens": 50}, + tool_calls=[], + ) + defaults.update(overrides) + return defaults + + +def _usage_row(**overrides: Any) -> dict[str, Any]: + fields = dict( + session_id="s1", + seq=0, + call_id="usage-1", + ts="2026-01-01T00:00:01.500Z", + platform="copilot", + provider_name="unknown", + response_model="gpt-test", + input_tokens=6452, + output_tokens=107, + cache_creation_input_tokens=6449, + ) + fields.update(overrides) + return UsageRow(**fields).to_dict() + + +def _accounting_call(**overrides: Any) -> dict[str, Any]: + defaults: dict[str, Any] = dict( + accounting_id="acct-1", + usage=_usage_row(), + attribution_status="matched", + agent_id=None, + call_id="call_1", + attributes={"accounting.destination": "chat-span"}, + ) + defaults.update(overrides) + return defaults + + +def _error_log_entries(home: Path) -> list[dict]: + log = home / "logs" / "usage-errors.jsonl" + if not log.exists(): + return [] + return [json.loads(line) for line in log.read_text().splitlines() if line] + + +class TestAccountingAttributes: + def test_projects_usage_and_metadata(self): + accounting = _accounting_call( + accounting_id="acct-42", + attribution_status="pending", + agent_id="agent-9", + attributes={"copilot.logical_call_id": "logical-1"}, + ) + attrs = otel_export._accounting_attributes(accounting) + + assert attrs["thirdeye.accounting.id"] == "acct-42" + assert attrs["thirdeye.accounting.attribution_status"] == "pending" + assert attrs["thirdeye.accounting.agent_id"] == "agent-9" + assert attrs["gen_ai.usage.input_tokens"] == 6452 + assert attrs["copilot.logical_call_id"] == "logical-1" + assert json.loads(attrs["thirdeye.accounting.usage"])["call_id"] == "usage-1" + + def test_labels_copilot_native_billing_distinct_from_usd(self): + accounting = _accounting_call( + attributes={"copilot.billing.nano_aiu": 12, "operation.cost": 0.05}, + ) + attrs = otel_export._accounting_attributes(accounting) + + assert attrs["thirdeye.accounting.billing.kind"] == "copilot-native-unit" + assert attrs["copilot.billing.nano_aiu"] == 12 + + +class TestOrdinaryTurnCompatibility: + def test_turn_without_accounting_calls_is_unchanged( + self, tmp_path: Path, enabled_config: Config, wired_instance, exporter + ): + call = _llm_call() + turn = _turn(llm_calls=[call]) + session_dir = tmp_path / "traces" / "claude" / "s1" + + otel_export._export_turn_inner( + config=enabled_config, + session_dir_=session_dir, + session_id="s1", + platform="claude", + cwd="/proj", + turn=turn, + ) + + spans = exporter.exported_spans_as_dict() + chat_spans = [span for span in spans if span["name"].startswith("chat")] + accounting_spans = [span for span in spans if span["name"] == "accounting"] + + assert len(chat_spans) == 1 + assert accounting_spans == [] + assert chat_spans[0]["attributes"]["gen_ai.usage.input_tokens"] == 100 + assert "thirdeye.accounting.id" not in chat_spans[0]["attributes"] + + +class TestNestedTurnAccounting: + def test_matched_accounting_merges_onto_chat_span_only( + self, tmp_path: Path, enabled_config: Config, wired_instance, exporter + ): + session_id = "copilot-session" + platform = "copilot" + call_id = "copilot:call:matched" + call = _llm_call(call_id=call_id, usage={}) + accounting = _accounting_call( + call_id=call_id, + usage=_usage_row( + call_id="usage-matched", + input_tokens=6452, + output_tokens=107, + ), + ) + turn = _turn(llm_calls=[call], accounting_calls=[accounting]) + + otel_export._export_turn_inner( + config=enabled_config, + session_dir_=tmp_path / "traces" / platform / session_id, + session_id=session_id, + platform=platform, + cwd="/proj", + turn=turn, + ) + + spans = exporter.exported_spans_as_dict() + chat_span = next(span for span in spans if span["name"].startswith("chat")) + accounting_spans = [span for span in spans if span["name"] == "accounting"] + + assert accounting_spans == [] + assert chat_span["context"]["span_id"] == chat_span_id(platform, session_id, call_id) + assert chat_span["attributes"]["gen_ai.usage.input_tokens"] == 6452 + assert chat_span["attributes"]["thirdeye.accounting.id"] == "acct-1" + + def test_unmatched_accounting_emits_turn_owned_span_with_deterministic_id( + self, tmp_path: Path, enabled_config: Config, wired_instance, exporter + ): + session_id = "copilot-session" + platform = "copilot" + turn_id = "turn-unmatched" + call = _llm_call(call_id="known-call", usage={}) + unmatched = _accounting_call( + accounting_id="acct-unmatched", + call_id=None, + attribution_status="pending", + usage=_usage_row(call_id="usage-unmatched", input_tokens=6587, output_tokens=5), + attributes={"accounting.destination": "turn-accounting-span"}, + ) + turn = _turn(turn_id=turn_id, llm_calls=[call], accounting_calls=[unmatched]) + + otel_export._export_turn_inner( + config=enabled_config, + session_dir_=tmp_path / "traces" / platform / session_id, + session_id=session_id, + platform=platform, + cwd="/proj", + turn=turn, + ) + + spans = exporter.exported_spans_as_dict() + chat_span = next(span for span in spans if span["name"].startswith("chat")) + accounting_span = next(span for span in spans if span["name"] == "accounting") + turn_span = next(span for span in spans if span["name"] == "invoke_agent") + + assert chat_span["attributes"].get("thirdeye.accounting.id") is None + assert accounting_span["parent"]["span_id"] == turn_span["context"]["span_id"] + assert accounting_span["context"]["span_id"] == otel_export._accounting_span_id( + platform, session_id, "acct-unmatched", turn_id + ) + assert accounting_span["attributes"]["gen_ai.usage.input_tokens"] == 6587 + assert accounting_span["attributes"]["thirdeye.accounting.attribution_status"] == "pending" + + def test_fixture_turn_with_matched_and_unmatched_accounting( + self, tmp_path: Path, enabled_config: Config, wired_instance, exporter + ): + turn = dict(_TRANSPORT["turn_span_with_accounting_calls"]) + session_id = ( + turn["llm_calls"][0]["usage"]["session_id"] + if turn["llm_calls"][0].get("usage") + else "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" + ) + session_id = turn["accounting_calls"][0]["usage"]["session_id"] + platform = "copilot" + + otel_export._export_turn_inner( + config=enabled_config, + session_dir_=tmp_path / "traces" / platform / session_id, + session_id=session_id, + platform=platform, + cwd="/proj", + turn=turn, + ) + + spans = exporter.exported_spans_as_dict() + chat_spans = [span for span in spans if span["name"].startswith("chat")] + accounting_spans = [span for span in spans if span["name"] == "accounting"] + + assert len(chat_spans) == 1 + assert len(accounting_spans) == 1 + matched_id = turn["accounting_calls"][0]["accounting_id"] + unmatched_id = turn["accounting_calls"][1]["accounting_id"] + assert chat_spans[0]["attributes"]["thirdeye.accounting.id"] == matched_id + assert accounting_spans[0]["attributes"]["thirdeye.accounting.id"] == unmatched_id + + +class TestSessionAccountingExport: + def test_queues_deterministic_job_without_duplicates( + self, enabled_config: Config, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ): + spawned: list[Path] = [] + monkeypatch.setattr(otel_export, "_spawn", spawned.append) + session_dir = tmp_path / "traces" / "copilot" / "s1" + accounting = { + "accounting_id": "acct-session", + "usage": _usage_row(session_id="s1"), + "attribution_status": "pending", + "agent_id": None, + "attributes": {"accounting.destination": "session-accounting-span"}, + } + + first = otel_export.export_session_accounting( + enabled_config, session_dir, "s1", "copilot", "/proj", accounting + ) + second = otel_export.export_session_accounting( + enabled_config, session_dir, "s1", "copilot", "/proj", accounting + ) + + assert first is True + assert second is True + jobs = list(otel_jobs_dir(enabled_config.root).glob("accounting-*.json")) + assert len(jobs) == 1 + payload = json.loads(jobs[0].read_text(encoding="utf-8")) + assert payload["kind"] == "session_accounting" + assert payload["destination"] == "session-accounting-span" + assert payload["state"] == "queued" + assert payload["job_id"] == "accounting:s1:acct-session" + assert payload["span_id"] == payload["job_id"] + assert len(spawned) == 2 + + def test_disabled_config_does_not_queue(self, tmp_path: Path): + config = Config(root=tmp_path, logfire=LogfireSettings(enabled=False, token="")) + session_dir = tmp_path / "traces" / "copilot" / "s1" + accounting = { + "accounting_id": "acct-session", + "usage": _usage_row(), + "attribution_status": "pending", + } + assert ( + otel_export.export_session_accounting( + config, session_dir, "s1", "copilot", "/proj", accounting + ) + is False + ) + assert list(otel_jobs_dir(config.root).glob("accounting-*.json")) == [] + + def test_worker_round_trips_session_accounting_job( + self, + enabled_config: Config, + wired_instance, + exporter, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ): + monkeypatch.setattr(Config, "load", lambda: enabled_config) + session_id = "s1" + platform = "copilot" + session_dir = tmp_path / "traces" / platform / session_id + accounting_id = "copilot-usage-7f3a" + job_id = f"accounting:{session_id}:{accounting_id}" + digest = hashlib.sha256(job_id.encode("utf-8")).hexdigest() + job_path = otel_jobs_dir(enabled_config.root) / f"accounting-{digest}.json" + job_path.parent.mkdir(parents=True, exist_ok=True) + job_path.write_text( + json.dumps( + { + "job_id": job_id, + "kind": "session_accounting", + "session_dir": str(session_dir), + "session_id": session_id, + "platform": platform, + "cwd": "/proj", + "accounting_id": accounting_id, + "destination": "session-accounting-span", + "usage": _usage_row(session_id=session_id), + "attribution_status": "pending", + "agent_id": None, + "attributes": {}, + "span_id": job_id, + "state": "queued", + "attempt": 0, + } + ), + encoding="utf-8", + ) + + otel_worker.main([str(job_path)]) + + assert not job_path.exists() + spans = exporter.exported_spans_as_dict() + accounting_span = next(span for span in spans if span["name"] == "accounting") + session_span = next(span for span in spans if span["name"] == "session") + + assert accounting_span["parent"]["span_id"] == session_span["context"]["span_id"] + assert accounting_span["attributes"]["thirdeye.accounting.id"] == accounting_id + assert accounting_span["context"]["span_id"] == otel_export._accounting_span_id( + platform, session_id, accounting_id + ) + + +class TestWorkerClaimRecovery: + def test_stale_claim_is_recovered_and_export_retried( + self, + enabled_config: Config, + wired_instance, + exporter, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ): + monkeypatch.setattr(Config, "load", lambda: enabled_config) + session_id = "s1" + platform = "copilot" + session_dir = tmp_path / "traces" / platform / session_id + job_id = "accounting:s1:acct-retry" + digest = hashlib.sha256(job_id.encode("utf-8")).hexdigest() + job_path = otel_jobs_dir(enabled_config.root) / f"accounting-{digest}.json" + job_path.parent.mkdir(parents=True, exist_ok=True) + job_path.write_text( + json.dumps( + { + "job_id": job_id, + "kind": "session_accounting", + "session_dir": str(session_dir), + "session_id": session_id, + "platform": platform, + "cwd": "/proj", + "accounting_id": "acct-retry", + "destination": "session-accounting-span", + "usage": _usage_row(session_id=session_id), + "attribution_status": "pending", + "state": "claimed", + "attempt": 0, + } + ), + encoding="utf-8", + ) + claim_path = otel_worker._job_claim_path(job_path) + claim_path.write_text("99999", encoding="utf-8") + stale_at = time.time() - otel_worker._JOB_CLAIM_STALE_S - 5 + import os + + os.utime(claim_path, (stale_at, stale_at)) + + otel_worker.main([str(job_path)]) + + assert not job_path.exists() + assert not claim_path.exists() + assert any(span["name"] == "accounting" for span in exporter.exported_spans_as_dict()) + + def test_export_failure_retains_job_and_increments_attempt( + self, enabled_config: Config, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ): + monkeypatch.setattr(Config, "load", lambda: enabled_config) + session_id = "s1" + platform = "copilot" + session_dir = tmp_path / "traces" / platform / session_id + job_id = "accounting:s1:acct-fail" + digest = hashlib.sha256(job_id.encode("utf-8")).hexdigest() + job_path = otel_jobs_dir(enabled_config.root) / f"accounting-{digest}.json" + job_path.parent.mkdir(parents=True, exist_ok=True) + job_path.write_text( + json.dumps( + { + "job_id": job_id, + "kind": "session_accounting", + "session_dir": str(session_dir), + "session_id": session_id, + "platform": platform, + "cwd": "/proj", + "accounting_id": "acct-fail", + "destination": "session-accounting-span", + "usage": _usage_row(session_id=session_id), + "attribution_status": "pending", + "state": "queued", + "attempt": 0, + } + ), + encoding="utf-8", + ) + + def _boom(**kwargs): + raise RuntimeError("flush failed") + + monkeypatch.setattr(otel_export, "_export_session_accounting_inner", _boom) + otel_worker.main([str(job_path)]) + + assert job_path.exists() + payload = json.loads(job_path.read_text(encoding="utf-8")) + assert payload["state"] == "queued" + assert payload["attempt"] == 1 + assert not otel_worker._job_claim_path(job_path).exists() + entries = _error_log_entries(enabled_config.root) + assert any("kind=session_accounting" in entry["message"] for entry in entries) + + def test_fresh_claim_blocks_concurrent_worker( + self, enabled_config: Config, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ): + job_path = tmp_path / "accounting-job.json" + payload = {"state": "queued", "attempt": 0} + job_path.write_text(json.dumps(payload), encoding="utf-8") + claim_path = otel_worker._job_claim_path(job_path) + claim_path.write_text("42", encoding="utf-8") + monkeypatch.setattr( + time, + "time", + lambda: claim_path.stat().st_mtime + 1, + ) + + claimed = otel_worker._claim_job(job_path, payload) + + assert claimed is None From 00eccb290344e09e7ee8823a137861299eeba7ce Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Fri, 11 Sep 2026 15:45:58 -0700 Subject: [PATCH 55/88] Expose Copilot usage accounting gaps and keep partitioned replay consistent. Incomplete identities, unknown providers, and revision conflicts are now diagnostics instead of silent drops, and cumulative state prevents false shutdown mismatches across archive partitions. Co-authored-by: Cursor --- src/thirdeye/platforms/copilot/usage.py | 483 +++++++++++++++--- .../accounting-projection.json | 12 +- tests/platforms/copilot/test_usage.py | 295 +++++++++-- 3 files changed, 653 insertions(+), 137 deletions(-) diff --git a/src/thirdeye/platforms/copilot/usage.py b/src/thirdeye/platforms/copilot/usage.py index b38a1fb..c16426f 100644 --- a/src/thirdeye/platforms/copilot/usage.py +++ b/src/thirdeye/platforms/copilot/usage.py @@ -8,23 +8,25 @@ import hashlib import json -from collections.abc import Iterable from datetime import datetime from typing import Any from urllib.parse import quote from thirdeye.usage.types import UsageRow +from .identity import SOURCE_KEY_DIGEST_LEN, SOURCE_KEY_PREFIX_LEN from .types import ( AccountingCandidate, AccountingProjection, - AccountingProjectionState, DatabaseRevision, + DiagnosticCode, + DiagnosticSeverity, ProjectionDiagnostic, SourceRecord, ) _USAGE_TABLE = "assistant_usage_events" +_MAIN_AGENT_KEY = "main" _METRIC_FIELDS = ( "input_tokens", "output_tokens", @@ -46,7 +48,13 @@ "content_filter_triggered", "token_details_json", ) -_TOKEN_FIELDS = _METRIC_FIELDS[:-1] +_TOKEN_FIELDS = ( + "input_tokens", + "output_tokens", + "cache_read_tokens", + "cache_write_tokens", + "reasoning_tokens", +) def _canonical_json(value: Any) -> str: @@ -77,13 +85,25 @@ def _integer(value: object) -> int | None: return None +def _metric_number(value: object) -> int | None: + """Accept whole-number floats so shutdown ``totalNanoAiu`` can validate.""" + + if isinstance(value, bool): + return None + if isinstance(value, int) and value >= 0: + return value + if isinstance(value, float) and value >= 0 and value.is_integer(): + return int(value) + return None + + def _source_key(record: SourceRecord) -> str | None: prefix = "copilot-db:" source_id = record["source_id"] if not source_id.startswith(prefix): return None key = source_id[len(prefix) :].split(":", 1)[0] - return key if len(key) == 64 else None + return key if len(key) == SOURCE_KEY_DIGEST_LEN else None def _revision(record: SourceRecord) -> DatabaseRevision | None: @@ -117,6 +137,15 @@ def _metrics_digest(row: dict[str, Any]) -> str: return "sha256:" + hashlib.sha256(_canonical_json(metrics).encode("utf-8")).hexdigest() +def _row_metrics(row: dict[str, Any]) -> dict[str, int]: + metrics: dict[str, int] = {} + for field in _METRIC_FIELDS: + value = _integer(row.get(field)) + if value is not None: + metrics[field] = value + return metrics + + def _row_is_incompatible(row: dict[str, Any]) -> bool: """Reject revisions that cannot describe a single completed request.""" @@ -130,6 +159,14 @@ def _row_is_incompatible(row: dict[str, Any]) -> bool: ) +def _row_key(table: str, primary_key: str) -> str: + return f"{table}\x1f{primary_key}" + + +def _agent_key(agent_id: str | None) -> str: + return agent_id or _MAIN_AGENT_KEY + + def _candidate( record: SourceRecord, revision: DatabaseRevision, logical_id: str ) -> AccountingCandidate: @@ -172,6 +209,14 @@ def _candidate( } +def _attach_revision_sources(candidate: AccountingCandidate, source_ids: list[str]) -> None: + candidate["source_ids"] = list(source_ids) + candidate["source_references"] = [ + {"source_id": source_id, "source_kind": "database", "role": "usage_row"} + for source_id in source_ids + ] + + def _missing_fields(candidate: AccountingCandidate, row: dict[str, Any]) -> list[str]: required = { "timestamp": candidate["timestamp"], @@ -208,50 +253,203 @@ def _usage_row( def _diagnostic( - code: str, severity: str, message: str, source_ids: list[str], **details: Any + code: DiagnosticCode, + severity: DiagnosticSeverity, + message: str, + source_ids: list[str], + **details: Any, ) -> ProjectionDiagnostic: return { - "code": code, # type: ignore[typeddict-item] - "severity": severity, # type: ignore[typeddict-item] + "code": code, + "severity": severity, "message": message, "source_ids": source_ids, "details": details, } -def _shutdown_totals(records: Iterable[SourceRecord]) -> list[tuple[str, dict[str, int]]]: - totals: list[tuple[str, dict[str, int]]] = [] - for record in records: - payload = record.get("payload") - if not isinstance(payload, dict) or payload.get("type") != "session.shutdown": +def _add_metrics(target: dict[str, int], metrics: dict[str, int], *, sign: int = 1) -> None: + for field, value in metrics.items(): + target[field] = target.get(field, 0) + sign * value + + +def _zero_metrics() -> dict[str, int]: + return {field: 0 for field in _METRIC_FIELDS} + + +def _metrics_from_call(call: dict[str, Any]) -> dict[str, int]: + if call.get("quarantined"): + return {} + metrics = call.get("metrics") + if not isinstance(metrics, dict): + return {} + parsed: dict[str, int] = {} + for field in _METRIC_FIELDS: + value = _integer(metrics.get(field)) + if value is not None: + parsed[field] = value + return parsed + + +def _parse_usage_block(usage: dict[str, Any], total_nano_aiu: object) -> dict[str, int] | None: + values = { + "input_tokens": usage.get("inputTokens"), + "output_tokens": usage.get("outputTokens"), + "cache_read_tokens": usage.get("cacheReadTokens"), + "cache_write_tokens": usage.get("cacheWriteTokens"), + "reasoning_tokens": usage.get("reasoningTokens"), + "total_nano_aiu": total_nano_aiu, + } + parsed = {name: _metric_number(value) for name, value in values.items()} + if any(value is None for value in parsed.values()): + return None + return {name: value for name, value in parsed.items() if value is not None} + + +def _aggregate_model_metrics(model_metrics: object) -> dict[str, int] | None: + if not isinstance(model_metrics, dict) or not model_metrics: + return None + aggregate = _zero_metrics() + for model in model_metrics.values(): + if not isinstance(model, dict): + return None + usage = model.get("usage") + if not isinstance(usage, dict): + return None + parsed = _parse_usage_block(usage, model.get("totalNanoAiu")) + if parsed is None: + return None + _add_metrics(aggregate, parsed) + return aggregate + + +def _agent_shutdown_totals(agent_metrics: object) -> dict[str, dict[str, int]] | None: + if not isinstance(agent_metrics, dict) or not agent_metrics: + return None + totals: dict[str, dict[str, int]] = {} + for agent_id, payload in agent_metrics.items(): + if not isinstance(agent_id, str) or not isinstance(payload, dict): + return None + parsed = _aggregate_model_metrics(payload.get("modelMetrics")) + if parsed is None: + return None + totals[agent_id] = parsed + return totals + + +def _unwrap_prior_state(prior_state: dict[str, Any]) -> dict[str, Any]: + """Accept a flat accounting state or a nested ProjectionState envelope. + + ``prior_state`` may be ``{"logical_calls": ...}`` (and optional cumulative + keys) or ``{"accounting_state": {"logical_calls": ...}}``. + """ + + inherited_calls = prior_state.get("logical_calls") + if isinstance(inherited_calls, dict): + return prior_state + nested = prior_state.get("accounting_state") + if isinstance(nested, dict): + return nested + return {} + + +def _hydrate_row_identity( + source: dict[str, Any], inherited_calls: dict[str, Any] +) -> tuple[dict[str, set[str]], dict[str, list[str]]]: + generations: dict[str, set[str]] = {} + sources: dict[str, list[str]] = {} + prior_generations = source.get("row_generations") + if isinstance(prior_generations, dict): + for key, values in prior_generations.items(): + if isinstance(key, str) and isinstance(values, list): + generations[key] = {item for item in values if isinstance(item, str)} + prior_sources = source.get("row_sources") + if isinstance(prior_sources, dict): + for key, values in prior_sources.items(): + if isinstance(key, str) and isinstance(values, list): + sources[key] = [item for item in values if isinstance(item, str)] + for call in inherited_calls.values(): + if not isinstance(call, dict): continue - data = payload.get("data") - if not isinstance(data, dict): + table = call.get("table") if isinstance(call.get("table"), str) else _USAGE_TABLE + primary_key = call.get("primary_key") + generation = call.get("generation") + if not isinstance(primary_key, str) or not isinstance(generation, str): continue - model_metrics = data.get("modelMetrics") - if not isinstance(model_metrics, dict): + key = _row_key(table, primary_key) + generations.setdefault(key, set()).add(generation) + usage_source_id = call.get("usage_source_id") + if isinstance(usage_source_id, str) and usage_source_id not in sources.setdefault(key, []): + sources[key].append(usage_source_id) + return generations, sources + + +def _hydrate_accounted( + source: dict[str, Any], inherited_calls: dict[str, Any] +) -> tuple[dict[str, int], dict[str, dict[str, int]]]: + accounted = _zero_metrics() + accounted_by_agent: dict[str, dict[str, int]] = {} + prior_accounted = source.get("accounted_metrics") + if isinstance(prior_accounted, dict) and any( + _integer(prior_accounted.get(field)) is not None for field in _METRIC_FIELDS + ): + for field in _METRIC_FIELDS: + value = _integer(prior_accounted.get(field)) + if value is not None: + accounted[field] = value + prior_agents = source.get("accounted_by_agent") + if isinstance(prior_agents, dict): + for agent_id, metrics in prior_agents.items(): + if isinstance(agent_id, str) and isinstance(metrics, dict): + bucket = _zero_metrics() + _add_metrics(bucket, _metrics_from_call({"metrics": metrics})) + accounted_by_agent[agent_id] = bucket + return accounted, accounted_by_agent + for call in inherited_calls.values(): + if not isinstance(call, dict): continue - aggregate = {name: 0 for name in _METRIC_FIELDS} - found = False - for model in model_metrics.values(): - usage = model.get("usage") if isinstance(model, dict) else None - if not isinstance(usage, dict): - continue - values = { - "input_tokens": usage.get("inputTokens"), - "output_tokens": usage.get("outputTokens"), - "cache_read_tokens": usage.get("cacheReadTokens"), - "cache_write_tokens": usage.get("cacheWriteTokens"), - "reasoning_tokens": usage.get("reasoningTokens"), - "total_nano_aiu": model.get("totalNanoAiu"), - } - if all(_integer(value) is not None for value in values.values()): - found = True - for name, value in values.items(): - aggregate[name] += _integer(value) or 0 - if found: - totals.append((record["source_id"], aggregate)) - return totals + metrics = _metrics_from_call(call) + _add_metrics(accounted, metrics) + agent_id = call.get("agent_id") + key = _agent_key(agent_id if isinstance(agent_id, str) else None) + accounted_by_agent.setdefault(key, _zero_metrics()) + _add_metrics(accounted_by_agent[key], metrics) + return accounted, accounted_by_agent + + +def _logical_call_entry( + logical_id: str, + revision: DatabaseRevision, + record: SourceRecord, + row: dict[str, Any], + candidate: AccountingCandidate, + *, + quarantined: bool, +) -> dict[str, Any]: + return { + "logical_call_id": logical_id, + "generation": revision["generation"], + "content_revision": revision["content_revision"], + "metrics_digest": _metrics_digest(row), + "usage_source_id": record["source_id"], + "table": revision["table"], + "primary_key": revision["primary_key"], + "agent_id": candidate["agent_id"], + "metrics": {} if quarantined else _row_metrics(row), + "quarantined": quarantined, + } + + +def _mismatch_details(expected: dict[str, int], accounted: dict[str, int]) -> dict[str, int]: + details: dict[str, int] = {} + for field in _METRIC_FIELDS: + details[f"expected_{field}"] = expected[field] + details[f"accounted_{field}"] = accounted[field] + return details + + +def _metrics_match(expected: dict[str, int], accounted: dict[str, int]) -> bool: + return all(accounted.get(field, 0) == expected[field] for field in _METRIC_FIELDS) def build_accounting( @@ -261,21 +459,29 @@ def build_accounting( Revisions are applied in archive order. A missing row is intentionally not a deletion: only archived observations can replace an accounting result. + + ``prior_state`` may be a flat accounting document (``logical_calls`` plus + optional cumulative keys) or a full projection state with nested + ``accounting_state``. Cumulative ``accounted_metrics``, + ``accounted_by_agent``, ``row_generations``, and ``row_sources`` support + disjoint archive partitions; prefix replay remains equivalent to a full + pass. """ - # The V1 archive preserves observation order, so a later archived revision - # is authoritative. Keep every source ID as conflict evidence even though - # only the newest revision supplies the candidate. - inherited_calls = prior_state.get("logical_calls") - if not isinstance(inherited_calls, dict): - accounting_state = prior_state.get("accounting_state") - inherited_calls = ( - accounting_state.get("logical_calls") if isinstance(accounting_state, dict) else {} - ) - selected: dict[str, tuple[SourceRecord, DatabaseRevision, AccountingCandidate]] = {} + source = _unwrap_prior_state(prior_state) + inherited_raw = source.get("logical_calls") + inherited_calls = inherited_raw if isinstance(inherited_raw, dict) else {} + selected: dict[str, tuple[SourceRecord, DatabaseRevision, AccountingCandidate, str | None]] = {} revision_sources: dict[str, list[str]] = {} diagnostics: list[ProjectionDiagnostic] = [] - primary_generations: dict[tuple[str, str], set[str]] = {} + row_generations, row_sources = _hydrate_row_identity(source, inherited_calls) + prior_digests = { + logical_id: call["metrics_digest"] + for logical_id, call in inherited_calls.items() + if isinstance(logical_id, str) + and isinstance(call, dict) + and isinstance(call.get("metrics_digest"), str) + } for record in records: payload = record.get("payload") @@ -286,22 +492,43 @@ def build_accounting( revision = _revision(record) logical_id = _logical_call_id(record, revision) if revision is not None else None if revision is None or logical_id is None: + reason = ( + "source_id is not a 64-character copilot-db identity" + if revision is not None + else "locator is missing generation, content_revision, or primary_key" + ) + diagnostics.append( + _diagnostic( + "capability_gap", + "warning", + "usage row cannot be accounted because archive identity is incomplete", + [record["source_id"]], + reason=reason, + ) + ) continue candidate = _candidate(record, revision, logical_id) - primary_generations.setdefault((revision["table"], revision["primary_key"]), set()).add( - revision["generation"] - ) - selected[logical_id] = (record, revision, candidate) + key = _row_key(revision["table"], revision["primary_key"]) + row_generations.setdefault(key, set()).add(revision["generation"]) + if record["source_id"] not in row_sources.setdefault(key, []): + row_sources[key].append(record["source_id"]) + previous_digest = prior_digests.get(logical_id) + if logical_id in selected: + previous_digest = _metrics_digest(selected[logical_id][0]["payload"]["row"]) + selected[logical_id] = (record, revision, candidate, previous_digest) revision_sources.setdefault(logical_id, []).append(record["source_id"]) + prior_digests[logical_id] = _metrics_digest(payload["row"]) - for (table, primary_key), generations in sorted(primary_generations.items()): + for key in sorted(row_generations): + generations = row_generations[key] if len(generations) > 1: + table, primary_key = key.split("\x1f", 1) diagnostics.append( _diagnostic( "usage_row_id_reuse", "warning", "database row ID was reused by a different database generation", - [], + list(row_sources.get(key, [])), table=table, primary_key=primary_key, generations=sorted(generations), @@ -310,27 +537,47 @@ def build_accounting( candidates: list[AccountingCandidate] = [] usage_rows: list[UsageRow] = [] - accounted = {field: 0 for field in _METRIC_FIELDS} + accounted, accounted_by_agent = _hydrate_accounted(source, inherited_calls) logical_calls = { key: value.copy() for key, value in inherited_calls.items() if isinstance(key, str) and isinstance(value, dict) } - for logical_id, (record, revision, candidate) in selected.items(): + unknown_provider_ids: list[str] = [] + for logical_id, (record, revision, candidate, previous_digest) in selected.items(): payload = record["payload"] row = payload["row"] assert isinstance(row, dict) + _attach_revision_sources(candidate, revision_sources[logical_id]) + previous = logical_calls.get(logical_id) + if isinstance(previous, dict): + old_metrics = _metrics_from_call(previous) + _add_metrics(accounted, old_metrics, sign=-1) + previous_agent = previous.get("agent_id") + old_agent = _agent_key(previous_agent if isinstance(previous_agent, str) else None) + if old_agent in accounted_by_agent: + _add_metrics(accounted_by_agent[old_agent], old_metrics, sign=-1) + digest = _metrics_digest(row) if _row_is_incompatible(row): - logical_calls.pop(logical_id, None) + details: dict[str, Any] = { + "logical_call_id": logical_id, + "metrics_digest": digest, + } + if previous_digest is not None: + details["prior_metrics_digest"] = previous_digest diagnostics.append( _diagnostic( "usage_revision_conflict", "error", "incompatible metrics for one logical call; quarantined", revision_sources[logical_id], - logical_call_id=logical_id, + **details, ) ) + candidates.append(candidate) + logical_calls[logical_id] = _logical_call_entry( + logical_id, revision, record, row, candidate, quarantined=True + ) continue candidates.append(candidate) missing = _missing_fields(candidate, row) @@ -347,21 +594,33 @@ def build_accounting( ) source_key = _source_key(record) assert source_key is not None - session_id = f"copilot-{source_key[:16]}-{record['native_session_id']}" + session_id = f"copilot-{source_key[:SOURCE_KEY_PREFIX_LEN]}-{record['native_session_id']}" usage_row = _usage_row(candidate, session_id, row) if usage_row is not None: usage_rows.append(usage_row) - for field in _METRIC_FIELDS: - value = _integer(row.get(field)) - if value is not None: - accounted[field] += value - logical_calls[logical_id] = { - "logical_call_id": logical_id, - "generation": revision["generation"], - "content_revision": revision["content_revision"], - "metrics_digest": _metrics_digest(row), - "usage_source_id": record["source_id"], - } + if usage_row.provider_name == "unknown": + unknown_provider_ids.append(record["source_id"]) + # Account every present metric field, including partial rows that cannot + # emit a UsageRow. Mismatch diagnostics therefore describe archived + # observations, not only exported totals. + metrics = _row_metrics(row) + _add_metrics(accounted, metrics) + agent_key = _agent_key(candidate["agent_id"]) + accounted_by_agent.setdefault(agent_key, _zero_metrics()) + _add_metrics(accounted_by_agent[agent_key], metrics) + logical_calls[logical_id] = _logical_call_entry( + logical_id, revision, record, row, candidate, quarantined=False + ) + + if unknown_provider_ids: + diagnostics.append( + _diagnostic( + "unknown_provider", + "info", + "database usage rows do not name a provider; UsageRow uses the unknown sentinel", + unknown_provider_ids, + ) + ) for record in records: payload = record.get("payload") @@ -376,24 +635,82 @@ def build_accounting( ) ) - for source_id, expected in _shutdown_totals(records): - if any(accounted[field] != expected[field] for field in _METRIC_FIELDS): + for record in records: + payload = record.get("payload") + if not isinstance(payload, dict) or payload.get("type") != "session.shutdown": + continue + data = payload.get("data") + if not isinstance(data, dict): + diagnostics.append( + _diagnostic( + "capability_gap", + "warning", + "session.shutdown usage cannot be used to validate accounting", + [record["source_id"]], + reason="missing_shutdown_data", + ) + ) + continue + session_totals = _aggregate_model_metrics(data.get("modelMetrics")) + if session_totals is None: + diagnostics.append( + _diagnostic( + "capability_gap", + "warning", + "session.shutdown usage cannot be used to validate accounting", + [record["source_id"]], + reason="unparseable_model_metrics", + ) + ) + continue + if not _metrics_match(session_totals, accounted): diagnostics.append( _diagnostic( "shutdown_total_mismatch", "warning", "per-call totals do not equal session.shutdown usage", - [source_id], - **{ - f"expected_{field}": expected[field] - for field in _METRIC_FIELDS - }, - **{ - f"accounted_{field}": accounted[field] - for field in _METRIC_FIELDS - }, + [record["source_id"]], + **_mismatch_details(session_totals, accounted), + ) + ) + agent_metrics = data.get("agentMetrics") + if agent_metrics is None: + continue + agent_totals = _agent_shutdown_totals(agent_metrics) + if agent_totals is None: + diagnostics.append( + _diagnostic( + "capability_gap", + "warning", + "session.shutdown agentMetrics cannot be used to validate per-agent accounting", + [record["source_id"]], + reason="unparseable_agent_metrics", ) ) + continue + for agent_id, expected in agent_totals.items(): + actual = accounted_by_agent.get(agent_id, _zero_metrics()) + if not _metrics_match(expected, actual): + diagnostics.append( + _diagnostic( + "shutdown_total_mismatch", + "warning", + "per-agent totals do not equal session.shutdown agentMetrics", + [record["source_id"]], + agent_id=agent_id, + **_mismatch_details(expected, actual), + ) + ) - next_state: AccountingProjectionState = {"logical_calls": logical_calls} - return {"usage_rows": usage_rows, "candidates": candidates, "diagnostics": diagnostics}, next_state + next_state: dict[str, Any] = { + "logical_calls": logical_calls, + "accounted_metrics": accounted, + "accounted_by_agent": accounted_by_agent, + "row_generations": {key: sorted(values) for key, values in row_generations.items()}, + "row_sources": row_sources, + } + return { + "usage_rows": usage_rows, + "candidates": candidates, + "diagnostics": diagnostics, + }, next_state diff --git a/tests/platforms/copilot/fixtures/reconciliation-cases/accounting-projection.json b/tests/platforms/copilot/fixtures/reconciliation-cases/accounting-projection.json index 2de915c..a11851c 100644 --- a/tests/platforms/copilot/fixtures/reconciliation-cases/accounting-projection.json +++ b/tests/platforms/copilot/fixtures/reconciliation-cases/accounting-projection.json @@ -106,6 +106,16 @@ } } ], - "diagnostics": [] + "diagnostics": [ + { + "code": "unknown_provider", + "severity": "info", + "message": "database usage rows do not name a provider; UsageRow uses the unknown sentinel", + "source_ids": [ + "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47" + ], + "details": {} + } + ] } } diff --git a/tests/platforms/copilot/test_usage.py b/tests/platforms/copilot/test_usage.py index d2cbe62..609000f 100644 --- a/tests/platforms/copilot/test_usage.py +++ b/tests/platforms/copilot/test_usage.py @@ -7,9 +7,6 @@ from pathlib import Path from typing import Any -import pytest - -from thirdeye.platforms.copilot.database import read_database from thirdeye.platforms.copilot.identity import resolve_sources from thirdeye.platforms.copilot.types import SourceRecord from thirdeye.platforms.copilot.usage import build_accounting @@ -17,7 +14,6 @@ FIXTURES = Path(__file__).parent / "fixtures" RECONCILIATION = FIXTURES / "reconciliation-cases" -CLI_FIXTURE = FIXTURES NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" SOURCE_KEY = "a" * 64 @@ -34,6 +30,22 @@ "total_nano_aiu": 373366000, } +MAIN_AGENT_TOTALS = { + "input_tokens": 26416, + "output_tokens": 229, + "cache_read_tokens": 19522, + "cache_write_tokens": 6882, + "reasoning_tokens": 39, +} + +CHILD_AGENT_TOTALS = { + "input_tokens": 8980, + "output_tokens": 99, + "cache_read_tokens": 4426, + "cache_write_tokens": 4548, + "reasoning_tokens": 16, +} + def _load_json(path: Path) -> Any: return json.loads(path.read_text(encoding="utf-8")) @@ -84,7 +96,7 @@ def _usage_record( def _shutdown_record(*, source_id: str = "transcript/shutdown-1") -> SourceRecord: - usage = _load_json(CLI_FIXTURE / "usage.json") + usage = _load_json(FIXTURES / "usage.json") return { "source_id": source_id, "source_kind": "transcript", @@ -131,7 +143,7 @@ def _checkpoint_record(*, source_id: str = "transcript/checkpoint-1") -> SourceR def _six_call_records() -> list[SourceRecord]: - rows = _load_json(CLI_FIXTURE / "assistant-usage-events.json") + rows = _load_json(FIXTURES / "assistant-usage-events.json") revisions = { call["row_id"]: call["usage_source_id"].rsplit(":", 1)[-1] for call in _load_json(RECONCILIATION / "observed-six-calls.json")["calls"] @@ -164,27 +176,6 @@ def _nano_aiu_total(candidates: list[dict[str, Any]]) -> int: ) -# --- module boundaries --- - - -def test_usage_module_has_no_forbidden_imports(): - import thirdeye.platforms.copilot.usage as usage - - source = Path(usage.__file__).read_text(encoding="utf-8") - forbidden = ( - "tracing", - "attribution", - "projection_store", - "export_state", - "build_semantics", - "join_usage", - "UsageStore", - "usage_store", - ) - for token in forbidden: - assert token not in source - - # --- contract fixture --- @@ -225,6 +216,18 @@ def test_six_call_totals_match_observed_shutdown_fixture(): assert _nano_aiu_total(projection["candidates"]) == SHUTDOWN_TOTALS["total_nano_aiu"] +def test_per_agent_totals_match_shutdown_agent_metrics(): + records = _six_call_records() + projection, _state = build_accounting(records, {}) + by_call = {candidate["logical_call_id"]: candidate for candidate in projection["candidates"]} + grouped: dict[str | None, list[UsageRow]] = {} + for usage_row in projection["usage_rows"]: + grouped.setdefault(by_call[usage_row.call_id]["agent_id"], []).append(usage_row) + + assert _metric_totals(grouped[None]) == MAIN_AGENT_TOTALS + assert _metric_totals(grouped[CHILD_AGENT_ID]) == CHILD_AGENT_TOTALS + + def test_candidates_preserve_agent_parent_and_turn_index_evidence(): records = _six_call_records() projection, _state = build_accounting(records, {}) @@ -238,7 +241,9 @@ def test_candidates_preserve_agent_parent_and_turn_index_evidence(): child_calls = [by_row_id[str(row_id)] for row_id in (16, 17)] assert all(item["agent_id"] == CHILD_AGENT_ID for item in child_calls) - assert all(item["parent_tool_call_id"] == "call_qx4FH5DADTeT1qVLb37HNpBk" for item in child_calls) + assert all( + item["parent_tool_call_id"] == "call_qx4FH5DADTeT1qVLb37HNpBk" for item in child_calls + ) def test_shutdown_validation_passes_when_totals_match(): @@ -250,7 +255,7 @@ def test_shutdown_validation_passes_when_totals_match(): def test_database_reader_records_normalize_to_same_six_calls(tmp_path: Path): from tests.platforms.copilot.test_database import _collect_all, _write_database - usage_rows = _load_json(CLI_FIXTURE / "assistant-usage-events.json") + usage_rows = _load_json(FIXTURES / "assistant-usage-events.json") home = tmp_path / "copilot" _write_database( home, @@ -261,10 +266,28 @@ def test_database_reader_records_normalize_to_same_six_calls(tmp_path: Path): db_records = _usage_records(_collect_all(resolve_sources(home), NATIVE_SESSION_ID)) synthetic_records = _six_call_records() + sample = db_records[0] + source_key = sample["source_id"].split(":")[1] + generation = sample["locator"]["generation"] + aligned = [ + _usage_record( + record["payload"]["row"], + content_revision=record["locator"]["content_revision"], + generation=generation, + source_key=source_key, + ) + for record in synthetic_records + ] + db_projection, _ = build_accounting(db_records, {}) - synthetic_projection, _ = build_accounting(synthetic_records, {}) + synthetic_projection, _ = build_accounting(aligned, {}) - assert len(db_projection["usage_rows"]) == len(synthetic_projection["usage_rows"]) == 6 + assert [row.call_id for row in db_projection["usage_rows"]] == [ + row.call_id for row in synthetic_projection["usage_rows"] + ] + assert [candidate["logical_call_id"] for candidate in db_projection["candidates"]] == [ + candidate["logical_call_id"] for candidate in synthetic_projection["candidates"] + ] assert _metric_totals(db_projection["usage_rows"]) == _metric_totals( synthetic_projection["usage_rows"] ) @@ -274,26 +297,33 @@ def test_database_reader_records_normalize_to_same_six_calls(tmp_path: Path): def test_unknown_provider_maps_to_unknown_in_usage_row(): - row = copy.deepcopy(_load_json(CLI_FIXTURE / "assistant-usage-events.json")[0]) - record = _usage_record(row, content_revision="sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47") + row = copy.deepcopy(_load_json(FIXTURES / "assistant-usage-events.json")[0]) + record = _usage_record( + row, + content_revision="sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + ) projection, _state = build_accounting([record], {}) assert projection["candidates"][0]["provider"] is None assert projection["usage_rows"][0].provider_name == "unknown" + unknown = [item for item in projection["diagnostics"] if item["code"] == "unknown_provider"] + assert len(unknown) == 1 + assert record["source_id"] in unknown[0]["source_ids"] def test_explicit_provider_is_preserved(): - row = copy.deepcopy(_load_json(CLI_FIXTURE / "assistant-usage-events.json")[0]) + row = copy.deepcopy(_load_json(FIXTURES / "assistant-usage-events.json")[0]) row["provider"] = "openai" record = _usage_record(row, content_revision="sha256:provider-rev") projection, _state = build_accounting([record], {}) assert projection["candidates"][0]["provider"] == "openai" assert projection["usage_rows"][0].provider_name == "openai" + assert "unknown_provider" not in _diagnostic_codes(projection) def test_provider_name_column_is_accepted(): - row = copy.deepcopy(_load_json(CLI_FIXTURE / "assistant-usage-events.json")[0]) + row = copy.deepcopy(_load_json(FIXTURES / "assistant-usage-events.json")[0]) row["provider_name"] = "anthropic" record = _usage_record(row, content_revision="sha256:provider-name-rev") projection, _state = build_accounting([record], {}) @@ -306,7 +336,7 @@ def test_provider_name_column_is_accepted(): def test_absent_cache_and_reasoning_tokens_stay_none_in_usage_row(): - row = copy.deepcopy(_load_json(CLI_FIXTURE / "assistant-usage-events.json")[0]) + row = copy.deepcopy(_load_json(FIXTURES / "assistant-usage-events.json")[0]) row.pop("cache_read_tokens", None) row.pop("cache_write_tokens", None) row.pop("reasoning_tokens", None) @@ -327,18 +357,14 @@ def test_absent_cache_and_reasoning_tokens_stay_none_in_usage_row(): def test_missing_required_fields_keep_candidate_without_usage_row(): - row = copy.deepcopy(_load_json(CLI_FIXTURE / "assistant-usage-events.json")[0]) + row = copy.deepcopy(_load_json(FIXTURES / "assistant-usage-events.json")[0]) row.pop("output_tokens") record = _usage_record(row, content_revision="sha256:missing-output-rev") projection, _state = build_accounting([record], {}) assert len(projection["candidates"]) == 1 assert projection["usage_rows"] == [] - missing = [ - item - for item in projection["diagnostics"] - if item["code"] == "missing_usage_fields" - ] + missing = [item for item in projection["diagnostics"] if item["code"] == "missing_usage_fields"] assert missing assert "output_tokens" in missing[0]["details"]["missing_fields"] supplemental = projection["candidates"][0]["supplemental_metrics"] @@ -347,7 +373,7 @@ def test_missing_required_fields_keep_candidate_without_usage_row(): def test_missing_timestamp_diagnostic_and_no_usage_row(): - row = copy.deepcopy(_load_json(CLI_FIXTURE / "assistant-usage-events.json")[0]) + row = copy.deepcopy(_load_json(FIXTURES / "assistant-usage-events.json")[0]) row.pop("created_at") record = _usage_record(row, content_revision="sha256:missing-ts-rev") record["ts"] = None @@ -363,7 +389,7 @@ def test_missing_timestamp_diagnostic_and_no_usage_row(): def test_later_revision_replaces_earlier_for_same_logical_call(): - row = copy.deepcopy(_load_json(CLI_FIXTURE / "assistant-usage-events.json")[0]) + row = copy.deepcopy(_load_json(FIXTURES / "assistant-usage-events.json")[0]) first = _usage_record(row, content_revision="sha256:first-revision") updated = copy.deepcopy(row) updated["output_tokens"] = 999 @@ -373,38 +399,57 @@ def test_later_revision_replaces_earlier_for_same_logical_call(): assert len(projection["usage_rows"]) == 1 assert projection["usage_rows"][0].output_tokens == 999 assert projection["candidates"][0]["usage_source_id"] == second["source_id"] + assert projection["candidates"][0]["source_ids"] == [first["source_id"], second["source_id"]] assert len(state["logical_calls"]) == 1 + stored = state["logical_calls"][projection["candidates"][0]["logical_call_id"]] + assert stored["metrics_digest"].startswith("sha256:") + assert stored["metrics_digest"] != "" def test_reused_row_id_across_generations_emits_warning(): - row = copy.deepcopy(_load_json(CLI_FIXTURE / "assistant-usage-events.json")[0]) + row_a = copy.deepcopy(_load_json(FIXTURES / "assistant-usage-events.json")[0]) + row_b = copy.deepcopy(row_a) + row_b["output_tokens"] = 50 first = _usage_record( - row, + row_a, content_revision="sha256:gen-a-rev", generation="sha256:generation-a", ) second = _usage_record( - row, + row_b, content_revision="sha256:gen-b-rev", generation="sha256:generation-b", ) projection, state = build_accounting([first, second], {}) assert "usage_row_id_reuse" in _diagnostic_codes(projection) - assert len(projection["usage_rows"]) == 2 + reuse = next(item for item in projection["diagnostics"] if item["code"] == "usage_row_id_reuse") + assert first["source_id"] in reuse["source_ids"] + assert second["source_id"] in reuse["source_ids"] + call_ids = [row.call_id for row in projection["usage_rows"]] + assert len(call_ids) == 2 + assert call_ids[0] != call_ids[1] + assert projection["usage_rows"][0].output_tokens == row_a["output_tokens"] + assert projection["usage_rows"][1].output_tokens == 50 + assert projection["candidates"][0]["usage_source_id"] == first["source_id"] assert len(state["logical_calls"]) == 2 def test_incompatible_metrics_quarantine_logical_call(): - row = copy.deepcopy(_load_json(CLI_FIXTURE / "assistant-usage-events.json")[0]) + row = copy.deepcopy(_load_json(FIXTURES / "assistant-usage-events.json")[0]) row["cache_read_tokens"] = row["input_tokens"] + 1 record = _usage_record(row, content_revision="sha256:incompatible-rev") projection, state = build_accounting([record], {}) assert projection["usage_rows"] == [] - assert projection["candidates"] == [] + assert len(projection["candidates"]) == 1 + assert projection["candidates"][0]["usage_source_id"] == record["source_id"] assert "usage_revision_conflict" in _diagnostic_codes(projection) - assert state["logical_calls"] == {} + conflict = next( + item for item in projection["diagnostics"] if item["code"] == "usage_revision_conflict" + ) + assert record["source_id"] in conflict["source_ids"] + assert state["logical_calls"] # --- checkpoint / shutdown --- @@ -419,7 +464,7 @@ def test_checkpoint_snapshot_is_not_additive(): def test_shutdown_mismatch_emits_diagnostic(): - usage = _load_json(CLI_FIXTURE / "usage.json") + usage = _load_json(FIXTURES / "usage.json") usage["modelMetrics"]["gpt-5.6-luna"]["usage"]["inputTokens"] = 1 shutdown = _shutdown_record() shutdown["payload"]["data"] = usage @@ -433,6 +478,50 @@ def test_shutdown_mismatch_emits_diagnostic(): assert mismatch["details"]["accounted_input_tokens"] == SHUTDOWN_TOTALS["input_tokens"] +def test_shutdown_float_nano_aiu_is_used_for_validation(): + usage = _load_json(FIXTURES / "usage.json") + usage["modelMetrics"]["gpt-5.6-luna"]["totalNanoAiu"] = 1.0 + shutdown = _shutdown_record() + shutdown["payload"]["data"] = usage + projection, _state = build_accounting(_six_call_records() + [shutdown], {}) + + mismatch = next( + item for item in projection["diagnostics"] if item["code"] == "shutdown_total_mismatch" + ) + assert mismatch["details"]["expected_total_nano_aiu"] == 1 + assert "capability_gap" not in _diagnostic_codes(projection) + + +def test_shutdown_agent_metrics_mismatch_is_reported(): + usage = _load_json(FIXTURES / "usage.json") + usage["agentMetrics"]["main"]["modelMetrics"]["gpt-5.6-luna"]["usage"]["inputTokens"] = 1 + shutdown = _shutdown_record() + shutdown["payload"]["data"] = usage + projection, _state = build_accounting(_six_call_records() + [shutdown], {}) + + mismatches = [ + item for item in projection["diagnostics"] if item["code"] == "shutdown_total_mismatch" + ] + assert any( + item["details"].get("agent_id") == "main" + and item["details"].get("expected_input_tokens") == 1 + for item in mismatches + ) + + +def test_unusable_shutdown_emits_capability_gap_instead_of_silent_skip(): + shutdown = _shutdown_record() + shutdown["payload"]["data"] = { + "modelMetrics": {"gpt-5.6-luna": {"usage": {"inputTokens": "not-a-number"}}} + } + projection, _state = build_accounting(_six_call_records() + [shutdown], {}) + + gaps = [item for item in projection["diagnostics"] if item["code"] == "capability_gap"] + assert gaps + assert shutdown["source_id"] in gaps[0]["source_ids"] + assert "shutdown_total_mismatch" not in _diagnostic_codes(projection) + + # --- supplemental metrics --- @@ -469,7 +558,7 @@ def test_incremental_replay_matches_full_archive(): def test_late_arrival_adds_new_call_without_rerunning_agent(): first_batch = _six_call_records()[:3] - late_row = copy.deepcopy(_load_json(CLI_FIXTURE / "assistant-usage-events.json")[3]) + late_row = copy.deepcopy(_load_json(FIXTURES / "assistant-usage-events.json")[3]) late_record = _usage_record( late_row, content_revision="sha256:late-arrival-rev", @@ -495,3 +584,103 @@ def test_prior_state_logical_calls_are_preserved_for_unseen_ids(): assert projection["usage_rows"] == [] assert projection["candidates"] == [] assert next_state["logical_calls"] == inherited["logical_calls"] + + +def test_nested_accounting_state_preserves_unseen_logical_calls(): + records = _six_call_records()[:1] + _, state = build_accounting(records, {}) + nested = {"accounting_state": copy.deepcopy(state)} + + projection, next_state = build_accounting([], nested) + assert projection["usage_rows"] == [] + assert next_state["logical_calls"] == state["logical_calls"] + + +def test_incomplete_locator_emits_capability_gap_and_keeps_the_source_id(): + row = copy.deepcopy(_load_json(FIXTURES / "assistant-usage-events.json")[0]) + record = _usage_record(row, content_revision="sha256:missing-generation-rev") + del record["locator"]["generation"] + projection, _state = build_accounting([record], {}) + + assert projection["usage_rows"] == [] + assert projection["candidates"] == [] + gap = next(item for item in projection["diagnostics"] if item["code"] == "capability_gap") + assert record["source_id"] in gap["source_ids"] + + +def test_invalid_source_identity_emits_capability_gap(): + row = copy.deepcopy(_load_json(FIXTURES / "assistant-usage-events.json")[0]) + record = _usage_record(row, content_revision="sha256:bad-source-rev") + record["source_id"] = "not-a-copilot-db-identity" + projection, _state = build_accounting([record], {}) + + assert projection["usage_rows"] == [] + assert projection["candidates"] == [] + gap = next(item for item in projection["diagnostics"] if item["code"] == "capability_gap") + assert record["source_id"] in gap["source_ids"] + + +def test_truncated_source_key_emits_capability_gap(): + row = copy.deepcopy(_load_json(FIXTURES / "assistant-usage-events.json")[0]) + record = _usage_record(row, content_revision="sha256:short-key-rev", source_key="abc") + projection, _state = build_accounting([record], {}) + + assert projection["usage_rows"] == [] + gap = next(item for item in projection["diagnostics"] if item["code"] == "capability_gap") + assert record["source_id"] in gap["source_ids"] + assert gap["details"]["reason"] == "source_id is not a 64-character copilot-db identity" + + +def test_later_incompatible_revision_conflicts_using_prior_metrics_digest(): + row = copy.deepcopy(_load_json(FIXTURES / "assistant-usage-events.json")[0]) + first = _usage_record(row, content_revision="sha256:first-digest-rev") + _, state = build_accounting([first], {}) + logical_id = next(iter(state["logical_calls"])) + prior_digest = state["logical_calls"][logical_id]["metrics_digest"] + + bad = copy.deepcopy(row) + bad["cache_read_tokens"] = bad["input_tokens"] + 1 + second = _usage_record(bad, content_revision="sha256:second-digest-rev") + projection, _next_state = build_accounting([second], state) + + conflict = next( + item for item in projection["diagnostics"] if item["code"] == "usage_revision_conflict" + ) + assert conflict["details"]["prior_metrics_digest"] == prior_digest + assert conflict["details"]["metrics_digest"] != prior_digest + assert projection["usage_rows"] == [] + assert len(projection["candidates"]) == 1 + + +def test_disjoint_partition_does_not_false_mismatch_shutdown(): + records = _six_call_records() + _first, state = build_accounting(records[:3], {}) + second, _next_state = build_accounting(records[3:] + [_shutdown_record()], state) + + assert "shutdown_total_mismatch" not in _diagnostic_codes(second) + assert len(second["usage_rows"]) == 3 + + +def test_row_id_reuse_is_detected_across_partitions(): + row_a = copy.deepcopy(_load_json(FIXTURES / "assistant-usage-events.json")[0]) + row_b = copy.deepcopy(row_a) + row_b["output_tokens"] = 50 + first = _usage_record( + row_a, + content_revision="sha256:gen-a-rev", + generation="sha256:generation-a", + ) + second = _usage_record( + row_b, + content_revision="sha256:gen-b-rev", + generation="sha256:generation-b", + ) + _first_projection, state = build_accounting([first], {}) + projection, _next_state = build_accounting([second], state) + + assert "usage_row_id_reuse" in _diagnostic_codes(projection) + reuse = next(item for item in projection["diagnostics"] if item["code"] == "usage_row_id_reuse") + assert first["source_id"] in reuse["source_ids"] + assert second["source_id"] in reuse["source_ids"] + assert len(projection["usage_rows"]) == 1 + assert projection["usage_rows"][0].output_tokens == 50 From 6e0177a0eec322566196f5f3139e1016a04bcc67 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Fri, 11 Sep 2026 15:49:31 -0700 Subject: [PATCH 56/88] Fix Copilot semantic reconstruction so derived turns stay honest. Keep tool cycles open while attaching finish evidence, surface dropped children and auxiliary diagnostics, and stop prior-state from resurrecting completed interactions. Co-authored-by: Cursor --- src/thirdeye/platforms/copilot/events.py | 286 +++++++-- src/thirdeye/platforms/copilot/tracing.py | 53 +- src/thirdeye/platforms/copilot/turns.py | 567 +++++++++++++++--- src/thirdeye/platforms/copilot/types.py | 6 + .../fixtures/reconciliation-cases/README.md | 2 + .../semantic-projection.json | 89 +-- tests/platforms/copilot/test_events.py | 130 +++- tests/platforms/copilot/test_tracing.py | 427 ++++++++++++- 8 files changed, 1342 insertions(+), 218 deletions(-) diff --git a/src/thirdeye/platforms/copilot/events.py b/src/thirdeye/platforms/copilot/events.py index 5794071..6cd6c0e 100644 --- a/src/thirdeye/platforms/copilot/events.py +++ b/src/thirdeye/platforms/copilot/events.py @@ -3,6 +3,10 @@ This module deliberately does not correlate records across sources. In particular, a hook observation has no invocation ID in the observed corpus, so it remains an observation instead of becoming a guessed tool span. + +Database and metadata records are accounting/identity evidence owned by +other V2 modules. They are skipped here so usage rows cannot appear as +``unknown`` semantic events in default turn views. """ from __future__ import annotations @@ -10,7 +14,38 @@ from copy import deepcopy from typing import Any -from .types import NormalizedEvent, SourceRecord, SourceReference, SourceReferenceRole +from .types import ( + EventClassification, + NormalizedEvent, + NormalizedEventKind, + SourceRecord, + SourceReference, + SourceReferenceRole, +) + +_TRANSCRIPT_OR_HOOK = frozenset({"transcript", "hook"}) +_SESSION_LIFECYCLE_NOTIFICATIONS = frozenset( + { + "notification", + "session.notification", + "session.model_change", + "session.auto_mode_resolved", + "hook.start", + "hook.end", + } +) +_ABORT_TYPES = frozenset({"assistant.abort", "session.abort", "abort"}) +_HOOK_KINDS: dict[str, tuple[NormalizedEventKind, SourceReferenceRole]] = { + "preToolUse": ("tool_execution_start", "hook"), + "postToolUse": ("tool_execution_complete", "hook"), + "postToolUseFailure": ("tool_execution_failure", "hook"), + "userPromptSubmitted": ("prompt_transformation", "hook"), + "agentStop": ("agent_stop", "finish"), + "subagentStart": ("subagent_started", "nested_child"), + "subagentStop": ("subagent_completed", "nested_child"), + "sessionStart": ("session_start", "hook"), + "sessionEnd": ("session_end", "hook"), +} def record_type(record: SourceRecord) -> str | None: @@ -60,26 +95,31 @@ def tool_call_id(record: SourceRecord) -> str | None: return None -def _reference(record: SourceRecord, role: SourceReferenceRole) -> SourceReference: - return {"source_id": record["source_id"], "source_kind": record["source_kind"], "role": role} +def source_reference(record: SourceRecord, role: SourceReferenceRole) -> SourceReference: + """Role-labelled pointer back to one immutable source record.""" + return { + "source_id": record["source_id"], + "source_kind": record["source_kind"], + "role": role, + } def _event( record: SourceRecord, - kind: str, + kind: NormalizedEventKind, role: SourceReferenceRole, *, - classification: str = "main", + classification: EventClassification = "main", suffix: str = "", attributes: dict[str, Any] | None = None, ) -> NormalizedEvent: return { "id": f"copilot:event:{record['source_id']}{suffix}", - "kind": kind, # type: ignore[typeddict-item] - "classification": classification, # type: ignore[typeddict-item] + "kind": kind, + "classification": classification, "ts": record.get("ts"), "source_ids": [record["source_id"]], - "source_references": [_reference(record, role)], + "source_references": [source_reference(record, role)], "attributes": attributes or {}, } @@ -94,17 +134,32 @@ def _identity_attributes(record: SourceRecord) -> dict[str, Any]: } +def _unknown(record: SourceRecord, native_type: str | None) -> list[NormalizedEvent]: + attrs: dict[str, Any] = {"raw_payload": deepcopy(record.get("payload"))} + if native_type is not None: + attrs["native_type"] = native_type + return [_event(record, "unknown", "hook", attributes=attrs)] + + def normalize_record(record: SourceRecord) -> list[NormalizedEvent]: """Normalize one record while retaining a direct pointer to its evidence.""" + if record.get("source_kind") not in _TRANSCRIPT_OR_HOOK: + return [] + native_type = record_type(record) data = record_data(record) identity = _identity_attributes(record) if native_type is None: - return [_event(record, "unknown", "hook", attributes={"raw_payload": deepcopy(record.get("payload"))})] + return _unknown(record, None) if native_type == "user.message": - attrs = {**identity, "delivery": data.get("delivery"), "source": data.get("source")} + attrs = { + **identity, + "delivery": data.get("delivery"), + "source": data.get("source"), + } return [_event(record, "user_prompt", "user_prompt", attributes=attrs)] + if native_type == "assistant.message": attrs = {**identity, "model": data.get("model"), "phase": data.get("phase")} events = [_event(record, "assistant_message", "assistant_message", attributes=attrs)] @@ -130,56 +185,189 @@ def normalize_record(record: SourceRecord) -> list[NormalizedEvent]: ) ) return events + + if native_type == "assistant.turn_start": + return [_event(record, "assistant_turn_start", "assistant_message", attributes=identity)] + if native_type == "assistant.turn_end": + return [_event(record, "assistant_turn_end", "finish", attributes=identity)] + if native_type == "tool.execution_start": - return [_event(record, "tool_execution_start", "tool_execution", attributes={**identity, "tool_call_id": tool_call_id(record), "name": data.get("toolName"), "arguments": deepcopy(data.get("arguments"))})] - if native_type == "tool.execution_complete": - kind = "tool_execution_complete" if data.get("success") is not False else "tool_execution_failure" - return [_event(record, kind, "tool_result", attributes={**identity, "tool_call_id": tool_call_id(record), "success": data.get("success"), "result": deepcopy(data.get("result"))})] + return [ + _event( + record, + "tool_execution_start", + "tool_execution", + attributes={ + **identity, + "tool_call_id": tool_call_id(record), + "name": data.get("toolName"), + "arguments": deepcopy(data.get("arguments")), + }, + ) + ] + if native_type in {"tool.execution_complete", "tool.execution_error", "tool.execution_failure"}: + failed = native_type != "tool.execution_complete" or data.get("success") is False + kind: NormalizedEventKind = ( + "tool_execution_failure" if failed else "tool_execution_complete" + ) + return [ + _event( + record, + kind, + "tool_result", + attributes={ + **identity, + "tool_call_id": tool_call_id(record), + "success": data.get("success"), + "result": deepcopy(data.get("result")), + }, + ) + ] + if native_type in {"permission.request", "permissionRequest"}: - return [_event(record, "permission_request", "permission_request", attributes={**identity, "tool_name": data.get("toolName"), "arguments": deepcopy(data.get("toolArgs"))})] + return [ + _event( + record, + "permission_request", + "permission_request", + attributes={ + **identity, + "tool_name": data.get("toolName"), + "arguments": deepcopy(data.get("toolArgs")), + }, + ) + ] if native_type in {"permission.decision", "permissionDecision"}: - return [_event(record, "permission_decision", "permission_decision", attributes={**identity, "decision": data.get("decision"), "tool_name": data.get("toolName")})] - if native_type in {"notification", "session.notification", "hook.start", "hook.end"}: - return [_event(record, "notification", "hook", attributes={**identity, "native_type": native_type, **deepcopy(data)})] + return [ + _event( + record, + "permission_decision", + "permission_decision", + attributes={ + **identity, + "decision": data.get("decision"), + "tool_name": data.get("toolName"), + }, + ) + ] + if native_type in {"prompt.transformation", "prompt.transformed"}: - return [_event(record, "prompt_transformation", "hook", attributes={**identity, **deepcopy(data)})] - if native_type in {"assistant.abort", "session.abort", "abort"}: + return [ + _event( + record, + "prompt_transformation", + "hook", + attributes={**identity, **deepcopy(data)}, + ) + ] + + if native_type in _ABORT_TYPES: return [_event(record, "abort", "finish", attributes=identity)] - if "error" in native_type.lower(): - return [_event(record, "error", "finish", attributes={**identity, "message": data.get("message") or data.get("error")})] + + if native_type.startswith("model."): + return [ + _event( + record, + "auxiliary_model_call", + "auxiliary_model", + classification="title_generation", + attributes={"native_type": native_type, **deepcopy(data)}, + ) + ] + if native_type in {"session.start", "session.resume"}: return [_event(record, "session_start", "hook", attributes=deepcopy(data))] - if native_type in {"session.end", "session.shutdown", "session.close"}: - kind = "session_shutdown" if native_type == "session.shutdown" else "session_end" - return [_event(record, kind, "shutdown", classification="shutdown_validation" if kind == "session_shutdown" else "main", attributes=deepcopy(data))] - if native_type in {"session.usage_checkpoint", "preCompact", "session.compaction", "context.compaction"}: - return [_event(record, "compaction", "checkpoint" if native_type == "session.usage_checkpoint" else "compaction", classification="checkpoint" if native_type == "session.usage_checkpoint" else "main", attributes=deepcopy(data))] - if native_type.startswith("model."): - return [_event(record, "auxiliary_model_call", "auxiliary_model", classification="title_generation", attributes={"native_type": native_type, **deepcopy(data)})] + if native_type in {"session.end", "session.close"}: + return [_event(record, "session_end", "shutdown", attributes=deepcopy(data))] + if native_type == "session.shutdown": + return [ + _event( + record, + "session_shutdown", + "shutdown", + classification="shutdown_validation", + attributes=deepcopy(data), + ) + ] + if native_type in { + "session.usage_checkpoint", + "preCompact", + "session.compaction", + "context.compaction", + }: + checkpoint = native_type == "session.usage_checkpoint" + return [ + _event( + record, + "compaction", + "checkpoint" if checkpoint else "compaction", + classification="checkpoint" if checkpoint else "main", + attributes=deepcopy(data), + ) + ] + if native_type == "subagent.started": - return [_event(record, "subagent_started", "nested_child", attributes={**identity, "tool_call_id": tool_call_id(record), **deepcopy(data)})] + return [ + _event( + record, + "subagent_started", + "nested_child", + attributes={ + **identity, + "tool_call_id": tool_call_id(record), + **deepcopy(data), + }, + ) + ] if native_type == "subagent.completed": - return [_event(record, "subagent_completed", "nested_child", attributes={**identity, "tool_call_id": tool_call_id(record), **deepcopy(data)})] - - hook_kinds = { - "permissionRequest": ("permission_request", "permission_request"), - "preToolUse": ("tool_execution_start", "hook"), - "postToolUse": ("tool_execution_complete", "hook"), - "postToolUseFailure": ("tool_execution_failure", "hook"), - "notification": ("notification", "hook"), - "userPromptSubmitted": ("prompt_transformation", "hook"), - "agentStop": ("session_end", "finish"), - "subagentStart": ("subagent_started", "nested_child"), - "subagentStop": ("subagent_completed", "nested_child"), - "sessionStart": ("session_start", "hook"), - "sessionEnd": ("session_end", "hook"), - } - mapped = hook_kinds.get(native_type) + return [ + _event( + record, + "subagent_completed", + "nested_child", + attributes={ + **identity, + "tool_call_id": tool_call_id(record), + **deepcopy(data), + }, + ) + ] + if native_type == "subagent.configured": + return [ + _event( + record, + "subagent_started", + "nested_child", + attributes={**identity, "native_type": native_type, **deepcopy(data)}, + ) + ] + + if native_type in _SESSION_LIFECYCLE_NOTIFICATIONS: + return [ + _event( + record, + "notification", + "hook", + attributes={**identity, "native_type": native_type, **deepcopy(data)}, + ) + ] + + mapped = _HOOK_KINDS.get(native_type) if mapped is not None: kind, role = mapped - attrs = {**identity, **deepcopy(data)} - return [_event(record, kind, role, attributes=attrs)] - return [_event(record, "unknown", "hook", attributes={"native_type": native_type, "raw_payload": deepcopy(record.get("payload"))})] + return [_event(record, kind, role, attributes={**identity, **deepcopy(data)})] + + if "error" in native_type.lower(): + return [ + _event( + record, + "error", + "finish", + attributes={**identity, "message": data.get("message") or data.get("error")}, + ) + ] + + return _unknown(record, native_type) def normalize_records(records: list[SourceRecord]) -> list[NormalizedEvent]: diff --git a/src/thirdeye/platforms/copilot/tracing.py b/src/thirdeye/platforms/copilot/tracing.py index ba52588..12caed6 100644 --- a/src/thirdeye/platforms/copilot/tracing.py +++ b/src/thirdeye/platforms/copilot/tracing.py @@ -2,11 +2,12 @@ from __future__ import annotations +from copy import deepcopy from typing import Any from .events import normalize_records from .turns import build_turns -from .types import SemanticProjection, SourceRecord +from .types import ProjectionDiagnostic, SemanticProjection, SourceRecord def build_semantics( @@ -14,17 +15,43 @@ def build_semantics( ) -> tuple[SemanticProjection, dict[str, Any]]: """Replay immutable source records into semantic events and main turns. - ``prior_state`` is intentionally not treated as evidence. It is only a - retained description of still-open interactions for incremental callers; - replaying the complete archive always produces the authoritative result. + ``prior_state`` is not evidence. It only retains still-open interactions + whose ``source_ids`` are absent from this partition. Replaying the + complete archive with an empty prior state is always authoritative. """ events = normalize_records(records) - turns, call_candidates, pending, semantic_state = build_turns(records) - old_open = prior_state.get("open_interactions") if isinstance(prior_state, dict) else None - if isinstance(old_open, dict): - for key, item in old_open.items(): - if key not in semantic_state["open_interactions"] and isinstance(item, dict): - # State cannot prove a new association, but retaining an - # unfinished prior partition avoids silently claiming closure. - semantic_state["open_interactions"][key] = item - return ({"events": events, "turns": turns, "call_candidates": call_candidates, "pending": pending, "diagnostics": []}, semantic_state) + turns, call_candidates, pending, diagnostics, semantic_state = build_turns( + records, prior_state if isinstance(prior_state, dict) else {} + ) + diagnostics = [*_auxiliary_diagnostics(events), *diagnostics] + return ( + { + "events": events, + "turns": turns, + "call_candidates": call_candidates, + "pending": pending, + "diagnostics": diagnostics, + }, + deepcopy(semantic_state), + ) + + +def _auxiliary_diagnostics(events: list[dict[str, Any]]) -> list[ProjectionDiagnostic]: + found: list[ProjectionDiagnostic] = [] + for event in events: + if event.get("kind") != "auxiliary_model_call": + continue + attributes = event.get("attributes") or {} + found.append( + { + "code": "auxiliary_excluded_from_main", + "severity": "info", + "message": "title-generation model.* record classified auxiliary", + "source_ids": list(event.get("source_ids") or []), + "details": { + "classification": event.get("classification"), + "model": attributes.get("model"), + }, + } + ) + return found diff --git a/src/thirdeye/platforms/copilot/turns.py b/src/thirdeye/platforms/copilot/turns.py index e7122ed..9887460 100644 --- a/src/thirdeye/platforms/copilot/turns.py +++ b/src/thirdeye/platforms/copilot/turns.py @@ -1,4 +1,10 @@ -"""Explicit-identity Copilot interaction and recursive child-tree assembly.""" +"""Explicit-identity Copilot interaction and recursive child-tree assembly. + +Callers that partition an archive must include every record belonging to +still-open interactions. ``prior_state`` only retains open keys whose +``source_ids`` are absent from the current partition; a complete archive +replay with ``prior_state={}`` is always the authoritative result. +""" from __future__ import annotations @@ -7,34 +13,108 @@ from thirdeye.tracing.model import LlmCallSpanDict, ToolCallSpanDict, TurnSpanDict -from .events import agent_id, interaction_id, record_data, record_type, tool_call_id -from .types import CallCandidate, PendingItem, SourceRecord, SourceReference - +from .events import ( + agent_id, + interaction_id, + record_data, + record_type, + source_reference, + tool_call_id, +) +from .types import CallCandidate, PendingItem, ProjectionDiagnostic, SourceRecord -def _source_key(record: SourceRecord) -> str | None: - source_id = record["source_id"] - if source_id.startswith("hook/") or source_id.startswith("copilot-db:"): - return None - return source_id.split("/", 1)[0] or None +_ABORT_TYPES = frozenset({"assistant.abort", "session.abort", "abort"}) +_PUBLIC_CANDIDATE_KEYS = ( + "call_id", + "stored_turn_id", + "interaction_id", + "agent_id", + "parent_tool_call_id", + "model", + "source_ids", + "source_references", + "start_ts", + "end_ts", + "tool_call_ids", + "finish_evidence", +) -def _turn_id(record: SourceRecord, interaction: str) -> str: - source_key = _source_key(record) - if source_key is None: - return f"copilot:turn:unknown:{record['native_session_id']}:{interaction}" - return f"copilot:turn:{source_key}:{record['native_session_id']}:{interaction}" +def _source_key(record: SourceRecord) -> str: + return record["source_id"].split("/", 1)[0] -def _reference(record: SourceRecord, role: str) -> SourceReference: - return {"source_id": record["source_id"], "source_kind": record["source_kind"], "role": role} # type: ignore[typeddict-item] +def _turn_id(record: SourceRecord, interaction: str, agent: str | None) -> str: + base = f"copilot:turn:{_source_key(record)}:{record['native_session_id']}:{interaction}" + if agent: + return f"{base}:{agent}" + return base def _key(interaction: str, agent: str | None) -> str: return f"{interaction}|{agent or 'main'}" -def build_turns(records: list[SourceRecord]) -> tuple[list[TurnSpanDict], list[CallCandidate], list[PendingItem], dict[str, Any]]: - """Build completed main turns and recursive child spans from transcript IDs only.""" +def _public_candidate(raw: dict[str, Any]) -> CallCandidate: + return {key: raw[key] for key in _PUBLIC_CANDIDATE_KEYS} # type: ignore[return-value] + + +def _text_parts(text: str) -> list[dict[str, Any]]: + return [{"type": "text", "content": text}] if text else [] + + +def _reasoning_parts(summary: str | None) -> list[dict[str, Any]]: + if not isinstance(summary, str) or not summary: + return [] + return [{"type": "reasoning", "content": summary}] + + +def _missing_identity(record: SourceRecord, reason: str, evidence: str) -> PendingItem: + return { + "id": f"pending:identity:{record['source_id']}", + "kind": "missing_identity", + "reason": reason, + "source_ids": [record["source_id"]], + "evidence": [evidence], + } + + +def _incomplete_tool(record: SourceRecord, call: str, reason: str) -> PendingItem: + return { + "id": f"pending:tool:{call}", + "kind": "incomplete_tool_pair", + "reason": reason, + "source_ids": [record["source_id"]], + "evidence": [f"tool_call_id:{call}"], + } + + +def _collect_nested_turn_ids(turns: list[TurnSpanDict]) -> set[str]: + found: set[str] = set() + stack = list(turns) + while stack: + turn = stack.pop() + found.add(turn["turn_id"]) + stack.extend(turn.get("subagents") or []) + return found + + +def build_turns( + records: list[SourceRecord], + prior_state: dict[str, Any] | None = None, +) -> tuple[ + list[TurnSpanDict], + list[CallCandidate], + list[PendingItem], + list[ProjectionDiagnostic], + dict[str, Any], +]: + """Build completed main turns and recursive child spans from transcript IDs. + + ``prior_state`` is not evidence. Unfinished interactions keep the + ``OpenInteractionState`` shape; reconstructing a partition still requires + the records that belong to those interactions. + """ interactions: dict[str, dict[str, Any]] = {} active_native_turns: dict[tuple[str | None, str], str] = {} child_parent: dict[str, tuple[str, str]] = {} @@ -42,20 +122,88 @@ def build_turns(records: list[SourceRecord]) -> tuple[list[TurnSpanDict], list[C calls: dict[str, dict[str, Any]] = {} tool_spans: dict[str, ToolCallSpanDict] = {} pending: list[PendingItem] = [] + diagnostics: list[ProjectionDiagnostic] = [] def ensure(record: SourceRecord, interaction: str, agent: str | None) -> dict[str, Any]: key = _key(interaction, agent) if key not in interactions: interactions[key] = { - "key": key, "interaction": interaction, "agent": agent, "turn_id": _turn_id(record, interaction), - "source_ids": [], "start_ts": None, "end_ts": None, "input": "", "output": "", "calls": [], - "permission_requests": [], "status": "completed", "complete": False, "parent": None, + "key": key, + "interaction": interaction, + "agent": agent, + "turn_id": _turn_id(record, interaction, agent), + "source_ids": [], + "start_ts": record.get("ts"), + "end_ts": None, + "input": "", + "output": "", + "calls": [], + "permission_requests": [], + "status": "completed", + "complete": False, + "saw_final_answer": False, } item = interactions[key] item["source_ids"].append(record["source_id"]) - item["start_ts"] = item["start_ts"] or record.get("ts") + if item["start_ts"] is None: + item["start_ts"] = record.get("ts") return item + def remember_parent(child: str | None, parent_call: str | None) -> None: + if child and parent_call and parent_call in tool_owner: + child_parent[child] = (tool_owner[parent_call], parent_call) + + def close_previous_for_agent(agent: str | None, new_interaction: str, ts: str | None) -> None: + for item in interactions.values(): + if item["agent"] != agent or item["interaction"] == new_interaction: + continue + if item["complete"]: + continue + item["complete"] = True + item["end_ts"] = item["end_ts"] or ts + + def complete_item(item: dict[str, Any], ts: str | None) -> None: + item["complete"] = True + item["end_ts"] = ts or item["end_ts"] + + def attach_finish( + item: dict[str, Any], record: SourceRecord, kind: str + ) -> dict[str, Any] | None: + last_id = item["calls"][-1] if item["calls"] else None + last_call = calls.get(last_id) if last_id else None + if last_call is None: + return None + last_call["end_ts"] = record.get("ts") + if record["source_id"] not in last_call["source_ids"]: + last_call["source_ids"].append(record["source_id"]) + last_call["source_references"].append(source_reference(record, "finish")) + last_call["finish_evidence"].append( + {"source_id": record["source_id"], "kind": kind, "value": None} + ) + return last_call + + def owner_for_turn_end(record: SourceRecord, native_turn: Any, agent: str | None) -> str | None: + if native_turn is None: + return None + turn_key = str(native_turn) + bound = active_native_turns.get((agent, turn_key)) + if bound is not None: + return bound + matches: list[str] = [] + for candidate in calls.values(): + if candidate.get("native_turn_id") != turn_key: + continue + if candidate["finish_evidence"]: + continue + if candidate["agent_id"] != agent: + continue + key = candidate["interaction_key"] + if key not in matches: + matches.append(key) + if len(matches) == 1: + return matches[0] + return None + for record in records: if record.get("source_kind") != "transcript": continue @@ -65,107 +213,306 @@ def ensure(record: SourceRecord, interaction: str, agent: str | None) -> dict[st interaction = interaction_id(record) if native_type == "subagent.started": - child = agent_id(record) + remember_parent(agent, tool_call_id(record)) + continue + + if native_type == "subagent.completed": + child = agent parent_call = tool_call_id(record) - if child and parent_call and parent_call in tool_owner: - child_parent[child] = (tool_owner[parent_call], parent_call) + remember_parent(child, parent_call) + if child: + for item in interactions.values(): + if item["agent"] == child and not item["complete"]: + complete_item(item, record.get("ts")) continue + if native_type == "user.message": if interaction is None: - pending.append({"id": f"pending:identity:{record['source_id']}", "kind": "missing_identity", "reason": "user message has no interactionId", "source_ids": [record["source_id"]], "evidence": ["native_type:user.message"]}) + pending.append( + _missing_identity( + record, + "user message has no interactionId", + "native_type:user.message", + ) + ) continue + close_previous_for_agent(agent, interaction, record.get("ts")) item = ensure(record, interaction, agent) item["input"] = str(data.get("content") or item["input"]) continue + if native_type == "assistant.turn_start": if interaction is None: - pending.append({"id": f"pending:identity:{record['source_id']}", "kind": "missing_identity", "reason": "assistant turn has no interactionId", "source_ids": [record["source_id"]], "evidence": ["native_type:assistant.turn_start"]}) + pending.append( + _missing_identity( + record, + "assistant turn has no interactionId", + "native_type:assistant.turn_start", + ) + ) continue item = ensure(record, interaction, agent) native_turn = data.get("turnId") if native_turn is not None: active_native_turns[(agent, str(native_turn))] = item["key"] continue + if native_type == "assistant.message": if interaction is None: - pending.append({"id": f"pending:identity:{record['source_id']}", "kind": "missing_identity", "reason": "assistant message has no interactionId", "source_ids": [record["source_id"]], "evidence": ["native_type:assistant.message"]}) + pending.append( + _missing_identity( + record, + "assistant message has no interactionId", + "native_type:assistant.message", + ) + ) continue item = ensure(record, interaction, agent) + remember_parent(agent, data.get("parentToolCallId")) call_id = f"copilot:call:{record['source_id']}" - requests = data.get("toolRequests") if isinstance(data.get("toolRequests"), list) else [] - requested_ids = [str(request["toolCallId"]) for request in requests if isinstance(request, dict) and request.get("toolCallId")] - candidate = {"call_id": call_id, "stored_turn_id": item["turn_id"], "interaction_id": interaction, "agent_id": agent, "parent_tool_call_id": data.get("parentToolCallId"), "model": data.get("model"), "source_ids": [record["source_id"]], "source_references": [_reference(record, "assistant_message")], "start_ts": record.get("ts"), "end_ts": None, "tool_call_ids": requested_ids, "finish_evidence": []} + requests = ( + data.get("toolRequests") if isinstance(data.get("toolRequests"), list) else [] + ) + requested_ids = [ + str(request["toolCallId"]) + for request in requests + if isinstance(request, dict) and request.get("toolCallId") + ] + content = data.get("content") if isinstance(data.get("content"), str) else "" + reasoning = data.get("reasoningSummary") or data.get("intentionSummary") + native_turn = data.get("turnId") + candidate = { + "call_id": call_id, + "stored_turn_id": item["turn_id"], + "interaction_id": interaction, + "agent_id": agent, + "parent_tool_call_id": data.get("parentToolCallId"), + "model": data.get("model"), + "source_ids": [record["source_id"]], + "source_references": [source_reference(record, "assistant_message")], + "start_ts": record.get("ts"), + "end_ts": None, + "tool_call_ids": requested_ids, + "finish_evidence": [], + "interaction_key": item["key"], + "native_turn_id": str(native_turn) if native_turn is not None else None, + "content": content, + "reasoning_summary": reasoning if isinstance(reasoning, str) else None, + "final_answer": data.get("phase") == "final_answer", + } calls[call_id] = candidate item["calls"].append(call_id) - content = data.get("content") - if isinstance(content, str) and content: + if content: item["output"] = content + if candidate["final_answer"]: + item["saw_final_answer"] = True + item["status"] = "completed" for request in requests: if not isinstance(request, dict) or not request.get("toolCallId"): continue call = str(request["toolCallId"]) tool_owner[call] = item["key"] - tool_spans[call] = {"tool_call_id": call, "name": str(request.get("name") or ""), "start_ts": str(record.get("ts") or ""), "end_ts": "", "attributes": {"arguments": deepcopy(request.get("arguments")), "intention_summary": request.get("intentionSummary"), "request_source_id": record["source_id"]}} + tool_spans[call] = { + "tool_call_id": call, + "name": str(request.get("name") or ""), + "start_ts": str(record.get("ts") or ""), + "end_ts": "", + "attributes": { + "arguments": deepcopy(request.get("arguments")), + "intention_summary": request.get("intentionSummary"), + "request_source_id": record["source_id"], + }, + } continue + if native_type == "tool.execution_start": call = tool_call_id(record) if call and call in tool_spans: span = tool_spans[call] span["start_ts"] = str(record.get("ts") or span["start_ts"]) - span["attributes"].update({"arguments": deepcopy(data.get("arguments")), "execution_start_source_id": record["source_id"]}) + span["attributes"].update( + { + "arguments": deepcopy(data.get("arguments")), + "execution_start_source_id": record["source_id"], + } + ) elif call: - pending.append({"id": f"pending:tool:{call}", "kind": "incomplete_tool_pair", "reason": "tool execution start has no requesting assistant message", "source_ids": [record["source_id"]], "evidence": [f"tool_call_id:{call}"]}) + pending.append( + _incomplete_tool( + record, + call, + "tool execution start has no requesting assistant message", + ) + ) continue - if native_type == "tool.execution_complete": + + if native_type in { + "tool.execution_complete", + "tool.execution_error", + "tool.execution_failure", + }: call = tool_call_id(record) if call and call in tool_spans: span = tool_spans[call] span["end_ts"] = str(record.get("ts") or "") - span["attributes"].update({"result": deepcopy(data.get("result")), "success": data.get("success"), "execution_result_source_id": record["source_id"]}) + span["attributes"].update( + { + "result": deepcopy(data.get("result")), + "success": data.get("success"), + "execution_result_source_id": record["source_id"], + } + ) elif call: - pending.append({"id": f"pending:tool:{call}", "kind": "incomplete_tool_pair", "reason": "tool result has no requesting assistant message", "source_ids": [record["source_id"]], "evidence": [f"tool_call_id:{call}"]}) + pending.append( + _incomplete_tool( + record, + call, + "tool result has no requesting assistant message", + ) + ) continue - if native_type in {"assistant.abort", "session.abort", "abort"} or (native_type and "error" in native_type.lower()): + + if native_type in {"permission.request", "permissionRequest"}: + if interaction is None: + continue + item = ensure(record, interaction, agent) + item["permission_requests"].append( + { + "ts": str(record.get("ts") or ""), + "tool_name": str(data.get("toolName") or ""), + "attributes": { + "arguments": deepcopy(data.get("toolArgs")), + "source_id": record["source_id"], + }, + } + ) + continue + + if native_type in {"permission.decision", "permissionDecision"}: + if interaction is None: + continue + item = ensure(record, interaction, agent) + decision = data.get("decision") + tool_name = data.get("toolName") + for request in reversed(item["permission_requests"]): + if tool_name and request["tool_name"] != tool_name: + continue + request["attributes"]["decision"] = decision + request["attributes"]["decision_source_id"] = record["source_id"] + break + continue + + is_abort = native_type in _ABORT_TYPES + is_error = ( + bool(native_type) + and "error" in native_type.lower() + and not native_type.startswith("model.") + and not native_type.startswith("tool.") + ) + if is_abort or is_error: native_turn = data.get("turnId") - owner = active_native_turns.get((agent, str(native_turn))) if native_turn is not None else None + owner = ( + active_native_turns.pop((agent, str(native_turn)), None) + if native_turn is not None + else None + ) + if owner is None and interaction is not None: + owner = ( + _key(interaction, agent) if _key(interaction, agent) in interactions else None + ) if owner and owner in interactions: - interactions[owner]["status"] = "interrupted" if native_type in {"assistant.abort", "session.abort", "abort"} else "errored" - interactions[owner]["end_ts"] = record.get("ts") + item = interactions[owner] + item["status"] = "interrupted" if is_abort else "errored" + item["source_ids"].append(record["source_id"]) + attach_finish(item, record, "abort" if is_abort else "error") + # Abort closes the user turn. An error leaves it open so a + # later model cycle in the same interaction can retry. + if is_abort: + complete_item(item, record.get("ts")) continue + if native_type == "assistant.turn_end": native_turn = data.get("turnId") - owner = active_native_turns.pop((agent, str(native_turn)), None) if native_turn is not None else None + owner = owner_for_turn_end(record, native_turn, agent) + if native_turn is not None: + active_native_turns.pop((agent, str(native_turn)), None) if owner is None: - pending.append({"id": f"pending:identity:{record['source_id']}", "kind": "missing_identity", "reason": "assistant.turn_end cannot be assigned without agent and active turn identity", "source_ids": [record["source_id"]], "evidence": [f"turn_id:{native_turn}"]}) + pending.append( + { + "id": f"pending:identity:{record['source_id']}", + "kind": "incomplete_tool_pair", + "reason": "assistant.turn_end has no matching open model cycle", + "source_ids": [record["source_id"]], + "evidence": [f"turn_id:{native_turn}"], + } + ) continue item = interactions[owner] item["source_ids"].append(record["source_id"]) item["end_ts"] = record.get("ts") - # A turn end completes a model cycle. It only completes the user - # interaction when the cycle contains the final answer (or was - # explicitly aborted/errored); tool cycles remain open. - last_call = calls.get(item["calls"][-1]) if item["calls"] else None - if last_call is not None: - last_call["end_ts"] = record.get("ts") - last_call["source_ids"].append(record["source_id"]) - last_call["source_references"].append(_reference(record, "finish")) - last_call["finish_evidence"].append({"source_id": record["source_id"], "kind": "assistant_turn_end", "value": None}) - if not last_call["tool_call_ids"] and item["output"]: - item["complete"] = True - continue - - def call_span(candidate: dict[str, Any]) -> LlmCallSpanDict: + last_call = attach_finish(item, record, "assistant_turn_end") + if ( + last_call is not None + and not last_call["tool_call_ids"] + and last_call.get("final_answer") + ): + complete_item(item, record.get("ts")) + continue + + if native_type in {"session.shutdown", "session.end", "session.close"}: + for item in interactions.values(): + if item["complete"]: + continue + if item["saw_final_answer"] or item["status"] != "completed": + complete_item(item, record.get("ts")) + continue + + def call_span(candidate: dict[str, Any], user_input: str) -> LlmCallSpanDict: attached = [tool_spans[tool] for tool in candidate["tool_call_ids"] if tool in tool_spans] - return {"call_id": candidate["call_id"], "provider": "unknown", "model": str(candidate.get("model") or "unknown"), "start_ts": str(candidate.get("start_ts") or ""), "end_ts": str(candidate.get("end_ts") or candidate.get("start_ts") or ""), "input_messages": [], "output_messages": [], "usage": {}, "tool_calls": attached} + output_parts = _text_parts(str(candidate.get("content") or "")) + _reasoning_parts( + candidate.get("reasoning_summary") + ) + return { + "call_id": candidate["call_id"], + "provider": "unknown", + "model": str(candidate.get("model") or "unknown"), + "start_ts": str(candidate.get("start_ts") or ""), + "end_ts": str(candidate.get("end_ts") or candidate.get("start_ts") or ""), + "input_messages": ( + [{"role": "user", "parts": _text_parts(user_input)}] if user_input else [] + ), + "output_messages": ( + [{"role": "assistant", "parts": output_parts}] if output_parts else [] + ), + "usage": {}, + "tool_calls": attached, + } spans: dict[str, TurnSpanDict] = {} for key, item in interactions.items(): if not item["complete"]: continue - spans[key] = {"turn_id": item["turn_id"], "start_ts": str(item["start_ts"] or ""), "end_ts": str(item["end_ts"] or item["start_ts"] or ""), "input_message": item["input"], "output_message": item["output"], "status": item["status"], "llm_calls": [call_span(calls[call]) for call in item["calls"]], "permission_requests": item["permission_requests"], "subagents": [], "attributes": {"interaction_id": item["interaction"], "agent_id": item["agent"]}, "accounting_calls": []} + spans[key] = { + "turn_id": item["turn_id"], + "start_ts": str(item["start_ts"] or ""), + "end_ts": str(item["end_ts"] or item["start_ts"] or ""), + "input_message": item["input"], + "output_message": item["output"], + "status": item["status"], + "llm_calls": [call_span(calls[call], item["input"]) for call in item["calls"]], + "permission_requests": item["permission_requests"], + "subagents": [], + "attributes": { + "interaction_id": item["interaction"], + "agent_id": item["agent"], + }, + "accounting_calls": [], + } for child, (parent_key, parent_call) in child_parent.items(): - child_items = [item for item in interactions.values() if item["agent"] == child and item["complete"]] + child_items = [ + item for item in interactions.values() if item["agent"] == child and item["complete"] + ] for item in child_items: child_span = spans.get(item["key"]) parent_span = spans.get(parent_key) @@ -173,14 +520,100 @@ def call_span(candidate: dict[str, Any]) -> LlmCallSpanDict: child_span["attributes"]["parent_tool_call_id"] = parent_call parent_span["subagents"].append(child_span) + emitted_ids = _collect_nested_turn_ids( + [span for key, span in spans.items() if interactions[key]["agent"] is None] + ) + for item in interactions.values(): + if not item["complete"] or item["agent"] is None: + continue + if item["turn_id"] in emitted_ids: + continue + pending.append( + { + "id": f"pending:identity:{item['turn_id']}", + "kind": "missing_identity", + "reason": "completed child interaction has no resolved parent tool call", + "source_ids": item["source_ids"], + "evidence": [ + f"interaction_id:{item['interaction']}", + f"agent_id:{item['agent']}", + ], + } + ) + diagnostics.append( + { + "code": "capability_gap", + "severity": "warning", + "message": "completed child interaction has no resolved parent tool call", + "source_ids": item["source_ids"], + "details": { + "interaction_id": item["interaction"], + "agent_id": item["agent"], + "stored_turn_id": item["turn_id"], + }, + } + ) + for call_id in item["calls"]: + calls[call_id]["stored_turn_id"] = None + open_state: dict[str, Any] = {} + current_source_ids = {record["source_id"] for record in records} for item in interactions.values(): if item["complete"]: continue - open_state[item["key"]] = {"interaction_id": item["interaction"], "agent_id": item["agent"], "stored_turn_id": item["turn_id"], "source_ids": item["source_ids"], "last_event_source_id": item["source_ids"][-1] if item["source_ids"] else None, "start_ts": item["start_ts"], "pending_tool_call_ids": [tool for call in item["calls"] for tool in calls[call]["tool_call_ids"] if not tool_spans.get(tool, {}).get("end_ts")]} - pending.append({"id": f"pending:{item['turn_id']}", "kind": "open_interaction", "reason": "user interaction has no completed final assistant turn", "source_ids": item["source_ids"], "evidence": [f"interaction_id:{item['interaction']}"]}) + pending_tools = [ + tool + for call in item["calls"] + for tool in calls[call]["tool_call_ids"] + if not tool_spans.get(tool, {}).get("end_ts") + ] + has_finish = any(calls[call]["finish_evidence"] for call in item["calls"]) + reason = ( + "user interaction has no completed final assistant turn" + if has_finish + else "user interaction has no assistant.turn_end" + ) + open_state[item["key"]] = { + "interaction_id": item["interaction"], + "agent_id": item["agent"], + "stored_turn_id": item["turn_id"], + "source_ids": item["source_ids"], + "last_event_source_id": item["source_ids"][-1] if item["source_ids"] else None, + "start_ts": item["start_ts"], + "pending_tool_call_ids": pending_tools, + } + pending.append( + { + "id": f"pending:{item['turn_id']}", + "kind": "open_interaction", + "reason": reason, + "source_ids": item["source_ids"], + "evidence": [f"interaction_id:{item['interaction']}"], + } + ) + diagnostics.append( + { + "code": "open_interaction", + "severity": "info", + "message": reason, + "source_ids": item["source_ids"], + "details": {"interaction_id": item["interaction"]}, + } + ) + + prior = prior_state.get("open_interactions") if isinstance(prior_state, dict) else None + if isinstance(prior, dict): + for key, prior_item in prior.items(): + if key in open_state or not isinstance(prior_item, dict): + continue + prior_sources = prior_item.get("source_ids") or [] + if prior_sources and any( + source_id in current_source_ids for source_id in prior_sources + ): + continue + open_state[key] = deepcopy(prior_item) main_turns = [span for key, span in spans.items() if interactions[key]["agent"] is None] - main_turns.sort(key=lambda turn: turn["start_ts"]) - call_candidates: list[CallCandidate] = list(calls.values()) # type: ignore[assignment] - return main_turns, call_candidates, pending, {"open_interactions": open_state} + main_turns.sort(key=lambda turn: (turn["start_ts"] == "", turn["start_ts"])) + call_candidates = [_public_candidate(raw) for raw in calls.values()] + return main_turns, call_candidates, pending, diagnostics, {"open_interactions": open_state} diff --git a/src/thirdeye/platforms/copilot/types.py b/src/thirdeye/platforms/copilot/types.py index c599c5c..b01dd85 100644 --- a/src/thirdeye/platforms/copilot/types.py +++ b/src/thirdeye/platforms/copilot/types.py @@ -101,6 +101,9 @@ class SourceSlice(TypedDict): # # Semantic call: ``copilot:call:``. # Main turn: ``copilot:turn:::``. +# Child turn: append ``:`` so two agents that share an +# ``interactionId`` cannot collide. Main interactions omit the suffix +# because ``agent_id`` is null. # # Logical database call / accounting span: # ``copilot:usage::
::``. @@ -127,6 +130,8 @@ class SourceSlice(TypedDict): NormalizedEventKind = Literal[ "user_prompt", "assistant_message", + "assistant_turn_start", + "assistant_turn_end", "tool_request", "tool_execution_start", "tool_execution_complete", @@ -138,6 +143,7 @@ class SourceSlice(TypedDict): "compaction", "abort", "error", + "agent_stop", "session_start", "session_end", "session_shutdown", diff --git a/tests/platforms/copilot/fixtures/reconciliation-cases/README.md b/tests/platforms/copilot/fixtures/reconciliation-cases/README.md index a9ca254..1cdbd47 100644 --- a/tests/platforms/copilot/fixtures/reconciliation-cases/README.md +++ b/tests/platforms/copilot/fixtures/reconciliation-cases/README.md @@ -50,6 +50,8 @@ Hook IDs are `hook/{native_id}/{observation_id}`. (or `:` when no native suffix exists) - semantic call: `copilot:call:` - main turn: `copilot:turn:::` +- child turn: the main form plus `:` so agents that share an + `interactionId` cannot collide. Main interactions omit the suffix. - logical call / accounting span: `copilot:usage::
::` diff --git a/tests/platforms/copilot/fixtures/reconciliation-cases/semantic-projection.json b/tests/platforms/copilot/fixtures/reconciliation-cases/semantic-projection.json index ced17b0..9211820 100644 --- a/tests/platforms/copilot/fixtures/reconciliation-cases/semantic-projection.json +++ b/tests/platforms/copilot/fixtures/reconciliation-cases/semantic-projection.json @@ -1,5 +1,5 @@ { - "note": "input_records are V1 SourceRecords. Pure-function tests may pass them directly to build_semantics. They are not a generated archive.", + "note": "input_records are V1 SourceRecords. Pure-function tests may pass them directly to build_semantics. They are not a generated archive. This slice is a tool cycle: assistant.turn_end attaches finish_evidence without requiring assistant.turn_start, but outstanding tools keep the user interaction open. A completed TurnSpanDict is not emitted.", "input_records": [ { "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/f07404d0-af52-4260-89fb-358a10e86034", @@ -186,53 +186,28 @@ "tool_call_id": "call_ayHplfzxjRFMTCpmTKEFhCSJ", "name": "view" } - } - ], - "turns": [ + }, { - "turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42", - "start_ts": "2026-09-10T17:08:22.203Z", - "end_ts": "2026-09-10T17:08:24.593Z", - "input_message": "Read alpha.txt and beta.txt with separate view calls in parallel and report their sum. Only read those two files.", - "output_message": "", - "status": "completed", - "llm_calls": [ + "id": "copilot:event:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/0080e44c-ad62-4288-b2b2-061ec2b73d80", + "kind": "assistant_turn_end", + "classification": "main", + "ts": "2026-09-10T17:08:24.593Z", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/0080e44c-ad62-4288-b2b2-061ec2b73d80" + ], + "source_references": [ { - "call_id": "copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", - "provider": "unknown", - "model": "gpt-5.6-luna", - "start_ts": "2026-09-10T17:08:24.503Z", - "end_ts": "2026-09-10T17:08:24.593Z", - "input_messages": [], - "output_messages": [], - "usage": {}, - "tool_calls": [ - { - "tool_call_id": "call_YSSva4HCniiETlxdGGjcrHbh", - "name": "view", - "start_ts": "2026-09-10T17:08:24.503Z", - "end_ts": "2026-09-10T17:08:24.593Z", - "attributes": {} - }, - { - "tool_call_id": "call_ayHplfzxjRFMTCpmTKEFhCSJ", - "name": "view", - "start_ts": "2026-09-10T17:08:24.503Z", - "end_ts": "2026-09-10T17:08:24.593Z", - "attributes": {} - } - ] + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/0080e44c-ad62-4288-b2b2-061ec2b73d80", + "source_kind": "transcript", + "role": "finish" } ], - "permission_requests": [], - "subagents": [], "attributes": { - "interaction_id": "6d2b89fd-a653-430c-b532-b0936d72eb42", - "agent_id": null - }, - "accounting_calls": [] + "turn_id": "0" + } } ], + "turns": [], "call_candidates": [ { "call_id": "copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", @@ -272,7 +247,35 @@ ] } ], - "pending": [], - "diagnostics": [] + "pending": [ + { + "id": "pending:copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42", + "kind": "open_interaction", + "reason": "user interaction has no completed final assistant turn", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/f07404d0-af52-4260-89fb-358a10e86034", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/0080e44c-ad62-4288-b2b2-061ec2b73d80" + ], + "evidence": [ + "interaction_id:6d2b89fd-a653-430c-b532-b0936d72eb42" + ] + } + ], + "diagnostics": [ + { + "code": "open_interaction", + "severity": "info", + "message": "user interaction has no completed final assistant turn", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/f07404d0-af52-4260-89fb-358a10e86034", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/0080e44c-ad62-4288-b2b2-061ec2b73d80" + ], + "details": { + "interaction_id": "6d2b89fd-a653-430c-b532-b0936d72eb42" + } + } + ] } } diff --git a/tests/platforms/copilot/test_events.py b/tests/platforms/copilot/test_events.py index 52524dc..6d7c38f 100644 --- a/tests/platforms/copilot/test_events.py +++ b/tests/platforms/copilot/test_events.py @@ -2,6 +2,7 @@ from __future__ import annotations +import ast import json from pathlib import Path from typing import Any @@ -222,29 +223,32 @@ def test_tool_execution_start_preserves_arguments(): @pytest.mark.parametrize( - ("case_name", "expected_kind", "expected_classification"), + ("case_name", "expected_kind", "expected_classification", "observed"), [ - ("permission", "permission_request", "main"), - ("compaction", "compaction", "checkpoint"), - ("auxiliary_title_generation", "auxiliary_model_call", "title_generation"), + ("permission", "permission_request", "main", False), + ("compaction", "compaction", "checkpoint", False), + ("auxiliary_title_generation", "auxiliary_model_call", "title_generation", False), ], ) -def test_reconciliation_case_event_normalization( +def test_synthetic_reconciliation_case_event_normalization( case_name: str, expected_kind: str, expected_classification: str, + observed: bool, ) -> None: case = _load_json(RECON_CASES / "cases.json")[case_name] - projection, _ = _build_semantics_from_case_records(case["input_records"]) - matching = [event for event in projection["events"] if event["kind"] == expected_kind] - assert matching, f"expected {expected_kind} in {case_name}" + assert case["observed"] is observed + events = normalize_records(case["input_records"]) + matching = [event for event in events if event["kind"] == expected_kind] + assert matching, f"expected {expected_kind} in synthetic case {case_name}" assert matching[0]["classification"] == expected_classification - - -def _build_semantics_from_case_records(records: list[SourceRecord]) -> tuple[dict[str, Any], dict[str, Any]]: - from thirdeye.platforms.copilot.tracing import build_semantics - - return build_semantics(records, {}) + expected_events = case["expected"].get("events") + if expected_events: + by_id = {event["id"]: event for event in events} + for expected in expected_events: + actual = by_id[expected["id"]] + assert actual["kind"] == expected["kind"] + assert actual["classification"] == expected["classification"] def test_permission_hook_maps_without_becoming_tool_execution_role(): @@ -331,8 +335,98 @@ def test_normalize_records_replays_in_order_without_deduplication(): assert events[1]["source_ids"] == [records[1]["source_id"]] -def test_events_module_has_no_usage_or_export_imports() -> None: +def test_events_module_does_not_import_usage_or_export() -> None: source = Path(__import__("thirdeye.platforms.copilot.events", fromlist=["__file__"]).__file__) - text = source.read_text(encoding="utf-8") - for forbidden in ("usage.py", "attribution", "projection_store", "export_state"): - assert forbidden not in text + tree = ast.parse(source.read_text(encoding="utf-8")) + imported: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imported.update(alias.name.split(".")[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module: + imported.add(node.module.split(".")[0]) + imported.add(node.module) + forbidden = {"usage", "attribution", "projection_store", "export_state"} + assert not (imported & forbidden) + assert "thirdeye.platforms.copilot.usage" not in imported + + +def test_database_records_are_skipped_not_unknown() -> None: + record: SourceRecord = { + "source_id": f"copilot-db:{SOURCE_KEY}:{NATIVE_SESSION_ID}:assistant_usage_events:13:sha256:abc", + "source_kind": "database", + "native_session_id": NATIVE_SESSION_ID, + "ts": "2026-09-10T17:08:24.498Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": {"table": "assistant_usage_events", "row": {"id": 13, "model": "gpt-5.6-luna"}}, + "locator": {"table": "assistant_usage_events", "primary_key": 13}, + } + assert normalize_record(record) == [] + case = _load_json(RECON_CASES / "cases.json")["compaction"] + events = normalize_records(case["input_records"]) + assert [event["kind"] for event in events] == ["compaction"] + + +def test_assistant_turn_markers_are_cycle_events_not_unknown() -> None: + start = _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/turn-start", + native_type="assistant.turn_start", + data={"turnId": "0", "interactionId": "ix-1"}, + ) + end = _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/turn-end", + native_type="assistant.turn_end", + data={"turnId": "0"}, + ) + assert _event_kinds(start) == ["assistant_turn_start"] + assert _event_kinds(end) == ["assistant_turn_end"] + assert normalize_record(end)[0]["source_references"][0]["role"] == "finish" + + +def test_session_lifecycle_and_subagent_configured_are_not_unknown() -> None: + model_change = _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/model-change", + native_type="session.model_change", + data={"newModel": "auto"}, + ) + auto_mode = _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/auto-mode", + native_type="session.auto_mode_resolved", + data={"chosenModel": "gpt-5.6-luna"}, + ) + configured = _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/sub-configured", + native_type="subagent.configured", + data={"model": "gpt-5.6-luna"}, + agent="child-agent", + ) + assert _event_kinds(model_change) == ["notification"] + assert _event_kinds(auto_mode) == ["notification"] + assert _event_kinds(configured) == ["subagent_started"] + + +def test_agent_stop_hook_is_not_session_end() -> None: + record = _hook_record( + source_id=f"hook/{NATIVE_SESSION_ID}/obs-stop", + event="agentStop", + hook_payload={"sessionId": NATIVE_SESSION_ID, "stopReason": "end_turn"}, + ) + event = normalize_record(record)[0] + assert event["kind"] == "agent_stop" + assert event["source_references"][0]["role"] == "finish" + + +def test_model_error_stays_auxiliary_and_tool_error_keeps_call_id() -> None: + model_error = _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/model-error", + native_type="model.error", + data={"model": "gpt-4o-mini", "purpose": "session_title"}, + ) + tool_error = _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/tool-error", + native_type="tool.execution_error", + data={"toolCallId": "call_bad", "result": "denied"}, + ) + assert _event_kinds(model_error) == ["auxiliary_model_call"] + assert normalize_record(model_error)[0]["classification"] == "title_generation" + assert _event_kinds(tool_error) == ["tool_execution_failure"] + assert normalize_record(tool_error)[0]["attributes"]["tool_call_id"] == "call_bad" diff --git a/tests/platforms/copilot/test_tracing.py b/tests/platforms/copilot/test_tracing.py index 72a2f59..a86f89f 100644 --- a/tests/platforms/copilot/test_tracing.py +++ b/tests/platforms/copilot/test_tracing.py @@ -2,6 +2,7 @@ from __future__ import annotations +import ast import json import shutil from collections import Counter @@ -107,9 +108,7 @@ def test_cli_fixture_counts_five_tools_and_six_call_candidates( cli_transcript_records: list[SourceRecord], ) -> None: projection, _ = build_semantics(cli_transcript_records, {}) - tool_requests = [ - event for event in projection["events"] if event["kind"] == "tool_request" - ] + tool_requests = [event for event in projection["events"] if event["kind"] == "tool_request"] tool_starts = [ event for event in projection["events"] if event["kind"] == "tool_execution_start" ] @@ -143,14 +142,48 @@ def test_cli_fixture_replay_is_deterministic(cli_transcript_records: list[Source assert second["events"] == third["events"] +def _assert_expected_projection(projection: dict[str, Any], expected: dict[str, Any]) -> None: + for key in ("events", "turns", "call_candidates", "pending", "diagnostics"): + if key not in expected: + continue + actual = projection[key] + wanted = expected[key] + if key == "events": + by_id = {event["id"]: event for event in actual} + for item in wanted: + got = by_id[item["id"]] + assert got["kind"] == item["kind"] + assert got["classification"] == item["classification"] + assert got["ts"] == item["ts"] + assert got["source_ids"] == item["source_ids"] + for attr_key, attr_value in item.get("attributes", {}).items(): + assert got["attributes"].get(attr_key) == attr_value + continue + if key == "call_candidates": + by_id = {item["call_id"]: item for item in actual} + for item in wanted: + got = by_id[item["call_id"]] + for field, value in item.items(): + assert got[field] == value, field + continue + assert actual == wanted + + def test_build_semantics_does_not_import_usage_or_attribution_modules() -> None: + forbidden = {"usage", "attribution", "projection_store", "export_state"} for module_name in ("tracing", "turns", "events"): source = Path( __import__(f"thirdeye.platforms.copilot.{module_name}", fromlist=["__file__"]).__file__ ) - text = source.read_text(encoding="utf-8") - for forbidden in ("usage.py", "attribution", "projection_store", "export_state"): - assert forbidden not in text + tree = ast.parse(source.read_text(encoding="utf-8")) + imported: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imported.update(alias.name.split(".", 1)[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module: + imported.add(node.module.split(".", 1)[0]) + imported.add(node.module) + assert not (imported & forbidden), module_name # --- interaction grouping and turn completion --- @@ -159,11 +192,11 @@ def test_build_semantics_does_not_import_usage_or_attribution_modules() -> None: def test_tool_cycle_does_not_complete_user_turn_without_final_answer() -> None: case = _load_json(RECON_CASES / "semantic-projection.json") projection, state = build_semantics(case["input_records"], {}) + _assert_expected_projection(projection, case["expected"]) - assert projection["turns"] == [] pending_kinds = Counter(item["kind"] for item in projection["pending"]) assert pending_kinds["open_interaction"] == 1 - assert pending_kinds["missing_identity"] == 1 + assert "missing_identity" not in pending_kinds open_key = "6d2b89fd-a653-430c-b532-b0936d72eb42|main" assert open_key in state["open_interactions"] @@ -171,6 +204,9 @@ def test_tool_cycle_does_not_complete_user_turn_without_final_answer() -> None: "call_YSSva4HCniiETlxdGGjcrHbh", "call_ayHplfzxjRFMTCpmTKEFhCSJ", ] + candidate = projection["call_candidates"][0] + assert candidate["end_ts"] == "2026-09-10T17:08:24.593Z" + assert candidate["finish_evidence"][0]["kind"] == "assistant_turn_end" def test_partial_user_prompt_stays_open_with_pending_item() -> None: @@ -188,7 +224,7 @@ def test_missing_interaction_id_creates_pending_not_fabricated_turn() -> None: native_type="user.message", data={"content": "orphan prompt", "turnId": "0"}, ) - _, _, pending, _ = build_turns([record]) + _, _, pending, _, _ = build_turns([record]) assert any(item["kind"] == "missing_identity" for item in pending) assert all(item["kind"] != "open_interaction" for item in pending) @@ -200,7 +236,7 @@ def test_incomplete_tool_pair_when_execution_lacks_request() -> None: native_type="tool.execution_start", data={"toolCallId": "call_orphan", "toolName": "view", "arguments": {}}, ) - _, _, pending, _ = build_turns([execution]) + _, _, pending, _, _ = build_turns([execution]) assert pending == [ { @@ -266,16 +302,15 @@ def test_child_turn_nests_under_parent_when_full_fixture_replayed( assert all(turn["attributes"].get("agent_id") is None for turn in projection["turns"]) -# --- prior state retention --- - - def test_prior_open_interaction_state_is_retained_when_replay_still_open() -> None: case = _load_json(RECON_CASES / "cases.json")["abort"] _, state = build_semantics(case["input_records"], {}) prior_item = dict(state["open_interactions"]["6d2b89fd-a653-430c-b532-b0936d72eb42|main"]) prior_item["note"] = "retained-from-incremental-caller" - _, merged_state = build_semantics(case["input_records"], {"open_interactions": state["open_interactions"]}) + _, merged_state = build_semantics( + case["input_records"], {"open_interactions": state["open_interactions"]} + ) retained = merged_state["open_interactions"]["6d2b89fd-a653-430c-b532-b0936d72eb42|main"] assert retained["interaction_id"] == prior_item["interaction_id"] assert retained["stored_turn_id"] == prior_item["stored_turn_id"] @@ -299,35 +334,371 @@ def test_prior_open_interaction_state_is_retained_when_replay_still_open() -> No assert orphan_key in merged["open_interactions"] +def test_prior_state_does_not_resurrect_completed_interaction( + cli_transcript_records: list[SourceRecord], +) -> None: + completed_key = "6d2b89fd-a653-430c-b532-b0936d72eb42|main" + interaction_id = completed_key.split("|", 1)[0] + user = next( + record + for record in cli_transcript_records + if record["payload"].get("type") == "user.message" + and record["payload"]["data"].get("interactionId") == interaction_id + ) + prior = { + "open_interactions": { + completed_key: { + "interaction_id": interaction_id, + "agent_id": None, + "stored_turn_id": ( + f"copilot:turn:{SOURCE_KEY}:{NATIVE_SESSION_ID}:{interaction_id}" + ), + "source_ids": [user["source_id"]], + "last_event_source_id": user["source_id"], + "start_ts": user.get("ts"), + "pending_tool_call_ids": [], + } + } + } + full, merged = build_semantics(cli_transcript_records, prior) + assert completed_key not in merged["open_interactions"] + assert any(turn["attributes"]["interaction_id"] == interaction_id for turn in full["turns"]) + + # --- semantic projection contract slice --- def test_semantic_projection_fixture_normalizes_expected_events() -> None: case = _load_json(RECON_CASES / "semantic-projection.json") projection, _ = build_semantics(case["input_records"], {}) - expected_events = case["expected"]["events"] - - by_id = {event["id"]: event for event in projection["events"]} - for expected in expected_events: - actual = by_id[expected["id"]] - assert actual["kind"] == expected["kind"] - assert actual["classification"] == expected["classification"] - assert actual["ts"] == expected["ts"] - assert actual["source_ids"] == expected["source_ids"] - for key, value in expected["attributes"].items(): - assert actual["attributes"].get(key) == value - + _assert_expected_projection(projection, case["expected"]) turn_end_id = ( "copilot:event:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/" "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/0080e44c-ad62-4288-b2b2-061ec2b73d80" ) - assert by_id[turn_end_id]["kind"] == "unknown" + by_id = {event["id"]: event for event in projection["events"]} + assert by_id[turn_end_id]["kind"] == "assistant_turn_end" def test_auxiliary_model_events_do_not_create_call_candidates() -> None: case = _load_json(RECON_CASES / "cases.json")["auxiliary_title_generation"] + assert case["observed"] is False projection, _ = build_semantics(case["input_records"], {}) + _assert_expected_projection(projection, case["expected"]) + assert projection["turns"] == [] + +def test_retry_case_is_database_only_and_emits_no_semantic_events() -> None: + """The shared retry fixture is accounting-only; semantics has nothing to reconstruct.""" + case = _load_json(RECON_CASES / "cases.json")["retry"] + assert case["observed"] is False + assert all(record["source_kind"] == "database" for record in case["input_records"]) + projection, _ = build_semantics(case["input_records"], {}) + assert projection["events"] == [] + assert projection["turns"] == [] assert projection["call_candidates"] == [] + assert projection["pending"] == [] + + +def test_observed_versus_synthetic_cases_run_through_build_semantics() -> None: + cases = _load_json(RECON_CASES / "cases.json") + observed = {name for name, case in cases.items() if case["observed"]} + synthetic = {name for name, case in cases.items() if not case["observed"]} + assert observed == {"identical_concurrent_tools", "nested_child"} + assert "retry" in synthetic + assert "permission" in synthetic + for name in ("identical_concurrent_tools", "permission", "abort"): + projection, _ = build_semantics(cases[name]["input_records"], {}) + assert "events" in projection + _assert_expected_projection(projection, cases[name]["expected"]) + nested = build_semantics(cases["nested_child"]["input_records"], {})[0] + assert nested["turns"] == [] + + +def test_completed_child_without_parent_link_is_pending_not_dropped() -> None: + child = "child-agent-1" + records = [ + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/child-user", + native_type="user.message", + data={"content": "explore", "interactionId": "child-ix", "turnId": "0"}, + agent=child, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/child-final", + native_type="assistant.message", + data={ + "content": "42", + "model": "gpt-5.6-luna", + "interactionId": "child-ix", + "turnId": "0", + "phase": "final_answer", + "toolRequests": [], + }, + agent=child, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/child-end", + native_type="assistant.turn_end", + data={"turnId": "0"}, + agent=child, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/child-stop", + native_type="subagent.completed", + data={"agentName": "explore"}, + agent=child, + ), + ] + projection, _ = build_semantics(records, {}) assert projection["turns"] == [] - assert projection["events"][0]["classification"] == "title_generation" + assert any(item["kind"] == "missing_identity" for item in projection["pending"]) + assert any(item["code"] == "capability_gap" for item in projection["diagnostics"]) + assert projection["call_candidates"][0]["stored_turn_id"] is None + + +def test_abort_emits_interrupted_turn() -> None: + records = [ + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/abort-user", + native_type="user.message", + data={"content": "hello", "interactionId": "ix-abort", "turnId": "0"}, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/abort-start", + native_type="assistant.turn_start", + data={"turnId": "0", "interactionId": "ix-abort"}, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/abort-msg", + native_type="assistant.message", + data={ + "content": "partial", + "model": "gpt-5.6-luna", + "interactionId": "ix-abort", + "turnId": "0", + "toolRequests": [], + }, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/abort-event", + native_type="assistant.abort", + data={"turnId": "0", "interactionId": "ix-abort"}, + ), + ] + projection, state = build_semantics(records, {}) + assert len(projection["turns"]) == 1 + assert projection["turns"][0]["status"] == "interrupted" + assert projection["turns"][0]["output_message"] == "partial" + assert state["open_interactions"] == {} + + +def test_intermediate_assistant_output_does_not_close_user_turn() -> None: + records = [ + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/mid-user", + native_type="user.message", + data={"content": "keep going", "interactionId": "ix-mid", "turnId": "0"}, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/mid-msg", + native_type="assistant.message", + data={ + "content": "working...", + "model": "gpt-5.6-luna", + "interactionId": "ix-mid", + "turnId": "0", + "toolRequests": [], + }, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/mid-end", + native_type="assistant.turn_end", + data={"turnId": "0"}, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/mid-tool-msg", + native_type="assistant.message", + data={ + "content": "", + "model": "gpt-5.6-luna", + "interactionId": "ix-mid", + "turnId": "1", + "toolRequests": [{"toolCallId": "call_later", "name": "view", "arguments": {}}], + }, + ), + ] + projection, state = build_semantics(records, {}) + assert projection["turns"] == [] + assert "ix-mid|main" in state["open_interactions"] + + +def test_turn_id_includes_agent_to_avoid_collision() -> None: + shared = "shared-interaction" + records = [ + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/main-user", + native_type="user.message", + data={"content": "parent", "interactionId": shared, "turnId": "0"}, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/child-user", + native_type="user.message", + data={"content": "child", "interactionId": shared, "turnId": "0"}, + agent="agent-child", + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/main-final", + native_type="assistant.message", + data={ + "content": "done", + "model": "gpt-5.6-luna", + "interactionId": shared, + "turnId": "0", + "phase": "final_answer", + "toolRequests": [], + }, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/child-final", + native_type="assistant.message", + data={ + "content": "done", + "model": "gpt-5.6-luna", + "interactionId": shared, + "turnId": "0", + "phase": "final_answer", + "toolRequests": [], + "parentToolCallId": "call_parent", + }, + agent="agent-child", + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/main-end", + native_type="assistant.turn_end", + data={"turnId": "0"}, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/child-end", + native_type="assistant.turn_end", + data={"turnId": "0"}, + agent="agent-child", + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/shutdown", + native_type="session.shutdown", + data={}, + ), + ] + main_id = f"copilot:turn:{SOURCE_KEY}:{NATIVE_SESSION_ID}:{shared}" + child_id = f"{main_id}:agent-child" + assert main_id != child_id + projection, _ = build_semantics(records, {}) + assert projection["turns"][0]["turn_id"] == main_id + stored = {candidate["stored_turn_id"] for candidate in projection["call_candidates"]} + assert main_id in stored + assert child_id not in stored + + +def test_permission_request_attaches_to_completed_turn() -> None: + records = [ + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/perm-user", + native_type="user.message", + data={"content": "read it", "interactionId": "ix-perm", "turnId": "0"}, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/perm-req", + native_type="permission.request", + data={ + "toolName": "view", + "toolArgs": {"path": "/tmp/a.txt"}, + "interactionId": "ix-perm", + "turnId": "0", + }, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/perm-dec", + native_type="permission.decision", + data={"toolName": "view", "decision": "allow", "interactionId": "ix-perm"}, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/perm-final", + native_type="assistant.message", + data={ + "content": "ok", + "model": "gpt-5.6-luna", + "interactionId": "ix-perm", + "turnId": "0", + "phase": "final_answer", + "toolRequests": [], + }, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/perm-end", + native_type="assistant.turn_end", + data={"turnId": "0"}, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/perm-shutdown", + native_type="session.shutdown", + data={}, + ), + ] + projection, _ = build_semantics(records, {}) + assert len(projection["turns"]) == 1 + requests = projection["turns"][0]["permission_requests"] + assert len(requests) == 1 + assert requests[0]["tool_name"] == "view" + assert requests[0]["attributes"]["decision"] == "allow" + + +def test_semantic_retry_after_error_stays_same_interaction() -> None: + records = [ + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/retry-user", + native_type="user.message", + data={"content": "try again", "interactionId": "ix-retry", "turnId": "0"}, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/retry-err-msg", + native_type="assistant.message", + data={ + "content": "", + "model": "gpt-5.6-luna", + "interactionId": "ix-retry", + "turnId": "0", + "toolRequests": [], + }, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/retry-error", + native_type="assistant.error", + data={"turnId": "0", "interactionId": "ix-retry", "message": "timeout"}, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/retry-final", + native_type="assistant.message", + data={ + "content": "42", + "model": "gpt-5.6-luna", + "interactionId": "ix-retry", + "turnId": "1", + "phase": "final_answer", + "toolRequests": [], + }, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/retry-end", + native_type="assistant.turn_end", + data={"turnId": "1"}, + ), + ] + projection, _ = build_semantics(records, {}) + assert len(projection["turns"]) == 1 + assert projection["turns"][0]["status"] == "completed" + assert projection["turns"][0]["output_message"] == "42" + assert len(projection["call_candidates"]) == 2 + assert {candidate["interaction_id"] for candidate in projection["call_candidates"]} == { + "ix-retry" + } From 9f1fb2d516e31aa889442e759d554b333d2a436a Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Fri, 11 Sep 2026 15:51:36 -0700 Subject: [PATCH 57/88] Fix projection storage durability, identity, and contract gaps. Keep derived indexes free of raw archive copies, re-key usage corrections under a stable logical call id, and treat next_state as authoritative so incremental commits no longer drop evidence or resurrect closed interactions. Co-authored-by: Cursor --- src/thirdeye/platforms/copilot/jsonio.py | 61 +++ .../platforms/copilot/projection_state.py | 92 ++-- .../platforms/copilot/projection_store.py | 400 ++++++++++++------ src/thirdeye/platforms/copilot/state.py | 51 +-- src/thirdeye/platforms/copilot/types.py | 70 ++- .../reconciliation-cases/storage.json | 2 +- tests/platforms/copilot/test_migration.py | 166 ++++++-- .../copilot/test_projection_store.py | 314 +++++++++++++- 8 files changed, 903 insertions(+), 253 deletions(-) create mode 100644 src/thirdeye/platforms/copilot/jsonio.py diff --git a/src/thirdeye/platforms/copilot/jsonio.py b/src/thirdeye/platforms/copilot/jsonio.py new file mode 100644 index 0000000..1558840 --- /dev/null +++ b/src/thirdeye/platforms/copilot/jsonio.py @@ -0,0 +1,61 @@ +"""Atomic JSON publication shared by Copilot archive and projection state.""" + +from __future__ import annotations + +import json +import os +import tempfile +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from thirdeye._compat import fsops + + +def atomic_write_json( + path: Path, + value: dict[str, Any], + *, + on_replaced: Callable[[], None] | None = None, + on_synced: Callable[[], None] | None = None, +) -> None: + """Replace ``path`` with canonical JSON, fsyncing the file and directory. + + ``on_replaced`` runs after the durable rename and before the directory + sync so callers can inject crash boundaries at the same points as before. + """ + path.parent.mkdir(parents=True, exist_ok=True) + fd, temp_name = tempfile.mkstemp(dir=path.parent, prefix=f"{path.name}.", suffix=".tmp") + try: + with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as stream: + json.dump(value, stream, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + fsops.replace(temp_name, path) + if on_replaced is not None: + on_replaced() + fsops.sync_directory(path.parent) + if on_synced is not None: + on_synced() + except BaseException: + fsops.unlink(Path(temp_name), missing_ok=True) + raise + + +def read_json_object(path: Path, *, invalid_message: str) -> dict[str, Any] | None: + """Read a JSON object, distinguishing absence, corruption, and I/O errors. + + ``FileNotFoundError`` returns ``None``. Malformed JSON or a non-object + becomes ``ValueError``. Transient ``OSError`` (sharing violations, EIO) + propagates unchanged so callers do not treat a busy file as corruption. + """ + try: + raw = json.loads(fsops.read_text(path, encoding="utf-8")) + except FileNotFoundError: + return None + except json.JSONDecodeError: + raise ValueError(invalid_message) from None + if not isinstance(raw, dict): + raise ValueError(invalid_message) + return raw diff --git a/src/thirdeye/platforms/copilot/projection_state.py b/src/thirdeye/platforms/copilot/projection_state.py index 307ff57..9e01ab6 100644 --- a/src/thirdeye/platforms/copilot/projection_state.py +++ b/src/thirdeye/platforms/copilot/projection_state.py @@ -8,15 +8,13 @@ from __future__ import annotations -import json -import os -import tempfile from collections.abc import Callable from pathlib import Path from typing import Any from thirdeye._compat import fsops +from .jsonio import atomic_write_json, read_json_object from .types import PROJECTION_SCHEMA_VERSION PROJECTION_STATE_FILENAME = "copilot.projection.state.json" @@ -66,6 +64,7 @@ def empty_projection_document() -> dict[str, Any]: "events": {}, "turns": {}, "usage": {}, + "usage_identities": {}, "attributions": {}, "pending": {}, "diagnostics": {}, @@ -74,46 +73,32 @@ def empty_projection_document() -> dict[str, Any]: def _atomic_json(path: Path, value: dict[str, Any], *, fault_point: str) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - fd, temp_name = tempfile.mkstemp(dir=path.parent, prefix=f"{path.name}.", suffix=".tmp") - try: - with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as stream: - json.dump(value, stream, ensure_ascii=False, sort_keys=True, separators=(",", ":")) - stream.write("\n") - stream.flush() - os.fsync(stream.fileno()) - fsops.replace(temp_name, path) - fsops.sync_directory(path.parent) - _fault(fault_point) - except BaseException: - fsops.unlink(Path(temp_name), missing_ok=True) - raise + atomic_write_json(path, value, on_synced=lambda: _fault(fault_point)) def _read_json(path: Path) -> dict[str, Any] | None: - try: - value = json.loads(fsops.read_text(path, encoding="utf-8")) - except FileNotFoundError: - return None - except (OSError, json.JSONDecodeError): - raise ValueError(f"invalid Copilot projection state: {path}") from None - if not isinstance(value, dict): - raise ValueError(f"invalid Copilot projection state: {path}") - return value - - -def _validate_document(document: dict[str, Any]) -> dict[str, Any]: + return read_json_object(path, invalid_message=f"invalid Copilot projection state: {path}") + + +def _schema_current(document: dict[str, Any]) -> bool: if document.get("schema_version") != PROJECTION_SCHEMA_VERSION: - raise ValueError("unsupported Copilot projection state schema") + return False state = document.get("state") + return isinstance(state, dict) and state.get("projection_schema_version") == ( + PROJECTION_SCHEMA_VERSION + ) + + +def _validate_current_document(document: dict[str, Any]) -> dict[str, Any]: indexes = document.get("indexes") - if not isinstance(state, dict) or not isinstance(indexes, dict): + if not isinstance(document.get("state"), dict) or not isinstance(indexes, dict): raise ValueError("invalid Copilot projection state") - if state.get("projection_schema_version") != PROJECTION_SCHEMA_VERSION: - raise ValueError("unsupported Copilot projection schema") for name in ("events", "turns", "usage", "attributions", "pending", "diagnostics"): if not isinstance(indexes.get(name), dict): raise ValueError("invalid Copilot projection indexes") + if not isinstance(indexes.get("usage_identities", {}), dict): + raise ValueError("invalid Copilot projection indexes") + indexes.setdefault("usage_identities", {}) return document @@ -123,25 +108,36 @@ def read_projection_document(session_dir: Path) -> dict[str, Any]: Callers holding :func:`projection_lock_path` may rely on this to finish a previously published journal before reading. Keeping recovery here makes it impossible for a later commit to merge against an incomplete snapshot. + + A stale schema version is disposable derived state: the files are removed + and the caller rebuilds from the immutable V1 archive. """ journal = _read_json(projection_journal_path(session_dir)) if journal is not None: - if journal.get("schema_version") != PROJECTION_JOURNAL_SCHEMA_VERSION: - raise ValueError("unsupported Copilot projection journal schema") document = journal.get("document") - if not isinstance(document, dict): - raise ValueError("invalid Copilot projection journal") - _validate_document(document) - _atomic_json( - projection_state_path(session_dir), document, fault_point="after_projection_recovery" - ) + journal_ok = journal.get("schema_version") == PROJECTION_JOURNAL_SCHEMA_VERSION + if journal_ok and isinstance(document, dict) and _schema_current(document): + _validate_current_document(document) + _atomic_json( + projection_state_path(session_dir), + document, + fault_point="after_projection_recovery", + ) + fsops.unlink(projection_journal_path(session_dir), missing_ok=True) + fsops.sync_directory(session_dir) + _fault("after_projection_recovery_clear") + return document + # Stale or unreadable journal: drop it and fall through to the snapshot. fsops.unlink(projection_journal_path(session_dir), missing_ok=True) fsops.sync_directory(session_dir) - _fault("after_projection_recovery_clear") - return document document = _read_json(projection_state_path(session_dir)) - return empty_projection_document() if document is None else _validate_document(document) + if document is None: + return empty_projection_document() + if not _schema_current(document): + remove_projection_state(session_dir) + return empty_projection_document() + return _validate_current_document(document) def publish_projection_document(session_dir: Path, document: dict[str, Any]) -> None: @@ -150,14 +146,14 @@ def publish_projection_document(session_dir: Path, document: dict[str, Any]) -> The caller owns the projection lock. If interrupted after either replace, the next reader replays the complete immutable document from the journal. """ - _validate_document(document) + if not _schema_current(document): + raise ValueError("unsupported Copilot projection state schema") + _validate_current_document(document) journal = {"schema_version": PROJECTION_JOURNAL_SCHEMA_VERSION, "document": document} _atomic_json( projection_journal_path(session_dir), journal, fault_point="after_projection_journal" ) - _atomic_json( - projection_state_path(session_dir), document, fault_point="after_projection_state" - ) + _atomic_json(projection_state_path(session_dir), document, fault_point="after_projection_state") fsops.unlink(projection_journal_path(session_dir), missing_ok=True) fsops.sync_directory(session_dir) _fault("after_projection_journal_clear") diff --git a/src/thirdeye/platforms/copilot/projection_store.py b/src/thirdeye/platforms/copilot/projection_store.py index 72da04c..327543d 100644 --- a/src/thirdeye/platforms/copilot/projection_store.py +++ b/src/thirdeye/platforms/copilot/projection_store.py @@ -22,11 +22,13 @@ from thirdeye.meta import read_meta from thirdeye.paths import meta_path, session_dir, usage_jsonl_path from thirdeye.reader import SessionReader +from thirdeye.usage.index import UsageIndex from thirdeye.usage.types import UsageRow from .constants import PLATFORM_NAME, SOURCE_SCHEMA_VERSION from .projection_state import ( empty_projection_state, + projection_journal_path, projection_lock_path, publish_projection_document, read_projection_document, @@ -37,6 +39,7 @@ _RAW_EVENT_TYPES = frozenset( {"copilot_transcript", "copilot_database", "copilot_hook", "copilot_metadata"} ) +_INDEX_NAMES = ("events", "turns", "usage", "attributions", "pending", "diagnostics") def _directory(config: Config, stored_session_id: str) -> Path: @@ -55,11 +58,20 @@ def _mapping(value: object) -> dict[str, Any]: return value if isinstance(value, dict) else {} +def _items(value: object) -> list[Any]: + return value if isinstance(value, list) else [] + + def _index_key(item: dict[str, Any], field: str, *, prefix: str) -> str: value = item.get(field) return value if isinstance(value, str) and value else f"{prefix}:{_digest(item)}" +def _require_session(directory: Path, stored_session_id: str) -> None: + if read_meta(meta_path(directory)) is None: + raise ValueError(f"unknown Copilot session: {stored_session_id}") + + def _source_id(event: dict[str, Any]) -> str | None: if event.get("t") not in _RAW_EVENT_TYPES: return None @@ -85,10 +97,9 @@ def _read_archived_events(directory: Path) -> dict[str, dict[str, Any]]: def _add_source_ids(value: object, result: set[str]) -> None: if not isinstance(value, dict): return - for key in ("source_ids",): - ids = value.get(key) - if isinstance(ids, list): - result.update(source_id for source_id in ids if isinstance(source_id, str)) + ids = value.get("source_ids") + if isinstance(ids, list): + result.update(source_id for source_id in ids if isinstance(source_id, str)) references = value.get("source_references") if isinstance(references, list): for reference in references: @@ -110,9 +121,11 @@ def _turn_source_ids(turn: dict[str, Any], events: Iterable[dict[str, Any]]) -> """ source_ids: set[str] = set() _add_source_ids(turn, source_ids) - for call in turn.get("llm_calls", []): - _add_source_ids(call, source_ids) - for child in turn.get("subagents", []): + for call in _items(turn.get("llm_calls")): + _add_source_ids(_mapping(call), source_ids) + for call in _items(turn.get("accounting_calls")): + _add_source_ids(_mapping(call), source_ids) + for child in _items(turn.get("subagents")): source_ids.update(_turn_source_ids(_mapping(child), ())) attributes = _mapping(turn.get("attributes")) @@ -120,97 +133,138 @@ def _turn_source_ids(turn: dict[str, Any], events: Iterable[dict[str, Any]]) -> turn_id = turn.get("turn_id") for semantic_event in events: event_attrs = _mapping(semantic_event.get("attributes")) - if ( - isinstance(interaction_id, str) - and event_attrs.get("interaction_id") == interaction_id - ) or (isinstance(turn_id, str) and event_attrs.get("stored_turn_id") == turn_id): - ids = semantic_event.get("source_ids") - if isinstance(ids, list): - source_ids.update(source_id for source_id in ids if isinstance(source_id, str)) + matched = ( + isinstance(interaction_id, str) and event_attrs.get("interaction_id") == interaction_id + ) or (isinstance(turn_id, str) and event_attrs.get("stored_turn_id") == turn_id) + if matched: + _add_source_ids(semantic_event, source_ids) return source_ids -def _projected_turn_record( - stored_session_id: str, - directory: Path, - turn: dict[str, Any], - semantic_events: Iterable[dict[str, Any]], - archived_events: dict[str, dict[str, Any]], -) -> dict[str, Any]: - turn_id = _index_key(turn, "turn_id", prefix="turn") - source_ids = _turn_source_ids(turn, semantic_events) - events = [archived_events[source_id] for source_id in source_ids if source_id in archived_events] - events.sort(key=lambda event: int(event.get("seq", -1))) - meta = read_meta(meta_path(directory)) if directory.exists() else None - start_ts = turn.get("start_ts") if isinstance(turn.get("start_ts"), str) else None - end_ts = turn.get("end_ts") if isinstance(turn.get("end_ts"), str) else None - return { - "id": f"{stored_session_id}:{turn_id}", - "turn_id": turn_id, - "session_id": stored_session_id, - "platform": PLATFORM_NAME, - "cwd": meta.cwd if meta is not None else "", - "start_seq": events[0].get("seq") if events else None, - "end_seq": events[-1].get("seq") if events else None, - "start_ts": events[0].get("ts") if events else start_ts, - "end_ts": events[-1].get("ts") if events else end_ts, - "events": events, +def _is_main_turn(turn: dict[str, Any]) -> bool: + return _mapping(turn.get("attributes")).get("agent_id") is None + + +def _replace_state(next_state: dict[str, Any]) -> dict[str, Any]: + """Treat ``next_state`` as an authoritative ProjectionState snapshot.""" + state = empty_projection_state() + archive_ids = next_state.get("archive_source_ids") + if isinstance(archive_ids, list): + state["archive_source_ids"] = [ + source_id for source_id in archive_ids if isinstance(source_id, str) + ] + semantic = _mapping(next_state.get("semantic_state")) + state["semantic_state"] = { + "open_interactions": dict(_mapping(semantic.get("open_interactions"))) + } + accounting = _mapping(next_state.get("accounting_state")) + state["accounting_state"] = {"logical_calls": dict(_mapping(accounting.get("logical_calls")))} + revision = next_state.get("projection_revision") + if isinstance(revision, str) and revision: + state["projection_revision"] = revision + state["projection_schema_version"] = PROJECTION_SCHEMA_VERSION + return state + + +def _collect_identities( + current: dict[str, Any], + attributions: dict[str, dict[str, Any]], + accounting_calls: dict[str, Any], +) -> dict[str, str]: + identities = { + key: value + for key, value in _mapping(current).items() + if isinstance(key, str) and isinstance(value, str) } + for attribution in attributions.values(): + source_id = attribution.get("usage_source_id") + logical_id = attribution.get("logical_call_id") + if isinstance(source_id, str) and isinstance(logical_id, str) and logical_id: + identities[source_id] = logical_id + identities[logical_id] = logical_id + for call in accounting_calls.values(): + mapped = _mapping(call) + source_id = mapped.get("usage_source_id") + logical_id = mapped.get("logical_call_id") + if isinstance(logical_id, str) and logical_id: + identities[logical_id] = logical_id + if isinstance(source_id, str): + identities[source_id] = logical_id + return identities + + +def _usage_identity(row: UsageRow, identities: dict[str, str]) -> str: + return identities.get(row.call_id, row.call_id) + + +def _drop_aliased_usage_keys(usage_index: dict[str, Any], identities: dict[str, str]) -> None: + for source_id, logical_id in identities.items(): + if source_id != logical_id: + usage_index.pop(source_id, None) + + +def _usage_seq( + logical_id: str, + row: UsageRow, + identities: dict[str, str], + archived_events: dict[str, dict[str, Any]], +) -> int: + candidates = [ + source_id + for source_id, mapped in identities.items() + if mapped == logical_id and source_id in archived_events + ] + if row.call_id in archived_events: + candidates.append(row.call_id) + if not candidates: + return 0 + return max(int(archived_events[source_id].get("seq", 0)) for source_id in candidates) -def _merge_state(current: dict[str, Any], next_state: dict[str, Any]) -> dict[str, Any]: - """Merge independent incremental builders without losing another commit.""" - merged = empty_projection_state() - merged.update({key: value for key, value in current.items() if key in merged}) - merged.update({key: value for key, value in next_state.items() if key in merged}) - current_ids = current.get("archive_source_ids") - next_ids = next_state.get("archive_source_ids") - current_source_ids = ( - {source_id for source_id in current_ids if isinstance(source_id, str)} - if isinstance(current_ids, list) - else set() - ) - next_source_ids = ( - {source_id for source_id in next_ids if isinstance(source_id, str)} - if isinstance(next_ids, list) - else set() - ) - merged["archive_source_ids"] = sorted(current_source_ids | next_source_ids) - for section, key in (("semantic_state", "open_interactions"), ("accounting_state", "logical_calls")): - old = _mapping(_mapping(current.get(section)).get(key)) - new = _mapping(_mapping(next_state.get(section)).get(key)) - merged[section] = {key: {**old, **new}} - merged["projection_schema_version"] = PROJECTION_SCHEMA_VERSION - return merged - - -def _usage_identity(row: UsageRow, attributions: dict[str, dict[str, Any]]) -> str: - for logical_id, attribution in attributions.items(): - if attribution.get("usage_source_id") == row.call_id: - return logical_id - # Usage normalization makes call_id the durable logical ID. The fallback - # keeps hand-built DTOs valid without ever keying on an import sequence. - return row.call_id +def _usage_line(row: dict[str, Any]) -> str: + return json.dumps(row, ensure_ascii=False, separators=(",", ":")) + + +def _sidecar_rows(usage_index: dict[str, Any]) -> list[dict[str, Any]]: + return [ + usage_index[logical_id] + for logical_id in sorted(usage_index) + if isinstance(usage_index[logical_id], dict) + ] + + +def _sidecar_payload(usage_index: dict[str, Any]) -> str: + return "".join(_usage_line(row) + "\n" for row in _sidecar_rows(usage_index)) + + +def _sidecar_matches(path: Path, usage_index: dict[str, Any]) -> bool: + if not path.exists(): + return not usage_index + try: + lines = [ + json.loads(line) + for line in path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + except (OSError, json.JSONDecodeError): + return False + return lines == _sidecar_rows(usage_index) def _write_usage_index(directory: Path, usage_index: dict[str, Any]) -> None: """Materialize one latest serialized row per logical call atomically. UsageStore is append-only and its generic reader is last-wins, which is - insufficient for correction/rebuild guarantees. This derived-only - materialization preserves its exact UsageRow JSON serialization while the - projection index supplies replacement semantics. + insufficient when a correction must replace a row under a re-keyed + identity. This derived-only rewrite preserves UsageRow.to_dict JSON + while the projection index supplies replacement semantics. """ path = usage_jsonl_path(directory) path.parent.mkdir(parents=True, exist_ok=True) fd, temporary = tempfile.mkstemp(dir=path.parent, prefix=f"{path.name}.", suffix=".tmp") try: with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as stream: - for logical_id in sorted(usage_index): - row = usage_index[logical_id] - if isinstance(row, dict): - stream.write(_canonical(row)) - stream.write("\n") + stream.write(_sidecar_payload(usage_index)) stream.flush() os.fsync(stream.fileno()) fsops.replace(temporary, path) @@ -220,14 +274,89 @@ def _write_usage_index(directory: Path, usage_index: dict[str, Any]) -> None: raise +def _invalidate_usage_index(config: Config, stored_session_id: str, directory: Path) -> None: + """Force UsageIndex to re-read a rewritten sidecar of unchanged size.""" + index = UsageIndex(config.root) + connection = index.connect() + try: + connection.execute("DELETE FROM usage WHERE session_id = ?", (stored_session_id,)) + connection.execute("DELETE FROM usage_sync WHERE session_id = ?", (stored_session_id,)) + index.refresh_session(connection, stored_session_id, directory) + finally: + connection.close() + + +def _publish_usage( + config: Config, + stored_session_id: str, + directory: Path, + usage_index: dict[str, Any], + *, + force: bool = False, +) -> None: + path = usage_jsonl_path(directory) + if not usage_index: + if path.exists(): + fsops.unlink(path, missing_ok=True) + fsops.sync_directory(directory) + _invalidate_usage_index(config, stored_session_id, directory) + return + if not force and _sidecar_matches(path, usage_index): + return + _write_usage_index(directory, usage_index) + _invalidate_usage_index(config, stored_session_id, directory) + + def _commit_counts(indexes: dict[str, Any]) -> dict[str, int]: + return {name: len(_mapping(indexes.get(name))) for name in _INDEX_NAMES} + + +def _stored_turn( + turn: dict[str, Any], + source_ids: set[str], + prior: dict[str, Any] | None, +) -> dict[str, Any]: + turn_id = _index_key(turn, "turn_id", prefix="turn") + merged_ids = set(source_ids) + if prior is not None: + merged_ids.update( + source_id for source_id in _items(prior.get("source_ids")) if isinstance(source_id, str) + ) + status = turn.get("status") + return { + "turn_id": turn_id, + "status": status if isinstance(status, str) else "", + "source_ids": sorted(merged_ids), + "span": turn, + } + + +def _hydrate_turn( + stored_session_id: str, + record: dict[str, Any], + archived_events: dict[str, dict[str, Any]], + cwd: str, +) -> dict[str, Any]: + turn_id = record.get("turn_id") if isinstance(record.get("turn_id"), str) else "" + source_ids = [item for item in _items(record.get("source_ids")) if isinstance(item, str)] + events = [ + archived_events[source_id] for source_id in source_ids if source_id in archived_events + ] + events.sort(key=lambda event: int(event.get("seq", -1))) + span = _mapping(record.get("span")) + start_ts = span.get("start_ts") if isinstance(span.get("start_ts"), str) else None + end_ts = span.get("end_ts") if isinstance(span.get("end_ts"), str) else None return { - "events": len(_mapping(indexes.get("events"))), - "turns": len(_mapping(indexes.get("turns"))), - "usage": len(_mapping(indexes.get("usage"))), - "attributions": len(_mapping(indexes.get("attributions"))), - "pending": len(_mapping(indexes.get("pending"))), - "diagnostics": len(_mapping(indexes.get("diagnostics"))), + "id": f"{stored_session_id}:{turn_id}", + "turn_id": turn_id, + "session_id": stored_session_id, + "platform": PLATFORM_NAME, + "cwd": cwd, + "start_seq": events[0].get("seq") if events else None, + "end_seq": events[-1].get("seq") if events else None, + "start_ts": events[0].get("ts") if events else start_ts, + "end_ts": events[-1].get("ts") if events else end_ts, + "events": events, } @@ -244,60 +373,84 @@ def commit_projection( projectable after Copilot removes its original files. """ directory = _directory(config, stored_session_id) + _require_session(directory, stored_session_id) with locked(projection_lock_path(directory), LockMode.EXCLUSIVE): document = read_projection_document(directory) indexes = _mapping(document.get("indexes")) merged_indexes = {name: dict(_mapping(indexes.get(name))) for name in indexes} - for name in ("events", "turns", "usage", "attributions", "pending", "diagnostics"): + for name in (*_INDEX_NAMES, "usage_identities"): merged_indexes.setdefault(name, {}) - event_items = [item for item in projection.get("normalized_events", []) if isinstance(item, dict)] + event_items = [ + item for item in projection.get("normalized_events", []) if isinstance(item, dict) + ] for item in event_items: merged_indexes["events"][_index_key(item, "id", prefix="event")] = item - attribution_items = [item for item in projection.get("attributions", []) if isinstance(item, dict)] + attribution_items = [ + item for item in projection.get("attributions", []) if isinstance(item, dict) + ] for item in attribution_items: - merged_indexes["attributions"][_index_key(item, "logical_call_id", prefix="attribution")] = item + merged_indexes["attributions"][ + _index_key(item, "logical_call_id", prefix="attribution") + ] = item + + state = _replace_state(next_state) + identities = _collect_identities( + merged_indexes["usage_identities"], + merged_indexes["attributions"], + _mapping(_mapping(state.get("accounting_state")).get("logical_calls")), + ) + archived_events = _read_archived_events(directory) for item in projection.get("usage_rows", []): if not isinstance(item, UsageRow): raise TypeError("projection usage_rows must contain UsageRow instances") + if item.session_id != stored_session_id: + raise ValueError("usage row session_id does not match stored session") + logical_id = _usage_identity(item, identities) + identities[item.call_id] = logical_id + identities[logical_id] = logical_id row = item.to_dict() - logical_id = _usage_identity(item, merged_indexes["attributions"]) + row["call_id"] = logical_id + row["seq"] = _usage_seq(logical_id, item, identities, archived_events) merged_indexes["usage"][logical_id] = row + _drop_aliased_usage_keys(merged_indexes["usage"], identities) + merged_indexes["usage_identities"] = identities - archived_events = _read_archived_events(directory) for item in projection.get("turns", []): - if not isinstance(item, dict): - continue - # Child-agent spans are represented recursively by their owning - # main interaction and never become separate generic user turns. - if _mapping(item.get("attributes")).get("agent_id") is not None: + if not isinstance(item, dict) or not _is_main_turn(item): continue turn_id = _index_key(item, "turn_id", prefix="turn") - merged_indexes["turns"][turn_id] = _projected_turn_record( - stored_session_id, directory, item, event_items, archived_events + source_ids = _turn_source_ids(item, merged_indexes["events"].values()) + merged_indexes["turns"][turn_id] = _stored_turn( + item, source_ids, _mapping(merged_indexes["turns"].get(turn_id)) or None ) - for item in projection.get("pending", []): - if isinstance(item, dict): - merged_indexes["pending"][_index_key(item, "id", prefix="pending")] = item - for item in projection.get("diagnostics", []): - if isinstance(item, dict): - merged_indexes["diagnostics"][_index_key(item, "id", prefix="diagnostic")] = item + merged_indexes["pending"] = { + _index_key(item, "id", prefix="pending"): item + for item in projection.get("pending", []) + if isinstance(item, dict) + } + merged_indexes["diagnostics"] = { + _index_key(item, "id", prefix="diagnostic"): item + for item in projection.get("diagnostics", []) + if isinstance(item, dict) + } - state = _merge_state(_mapping(document.get("state")), next_state) counts = _commit_counts(merged_indexes) - state["commit_result"] = counts + if not state["projection_revision"]: + state["projection_revision"] = _digest( + {key: merged_indexes.get(key) for key in (*_INDEX_NAMES, "usage_identities")} + ) + state["index_totals"] = counts next_document = { "schema_version": PROJECTION_SCHEMA_VERSION, "state": state, "indexes": merged_indexes, } publish_projection_document(directory, next_document) - # A crash after publication is repaired on the next load/commit from - # the durable usage index; this file contains no raw V1 evidence. - _write_usage_index(directory, merged_indexes["usage"]) + _publish_usage(config, stored_session_id, directory, merged_indexes["usage"]) return counts @@ -306,22 +459,31 @@ def load_projection_state(config: Config, stored_session_id: str) -> dict[str, A directory = _directory(config, stored_session_id) with locked(projection_lock_path(directory), LockMode.EXCLUSIVE): document = read_projection_document(directory) - # Recover a sidecar lost after durable state publication without - # changing V1 evidence or export bookkeeping. usage_index = _mapping(_mapping(document.get("indexes")).get("usage")) - if usage_index: - _write_usage_index(directory, usage_index) + _publish_usage(config, stored_session_id, directory, usage_index) return json.loads(_canonical(_mapping(document.get("state")))) def read_projected_turns(config: Config, stored_session_id: str) -> list[dict[str, Any]]: """Read completed main interaction records without semantic duplicates.""" directory = _directory(config, stored_session_id) - with locked(projection_lock_path(directory), LockMode.EXCLUSIVE): + writer = projection_journal_path(directory).exists() + with locked(projection_lock_path(directory), LockMode.EXCLUSIVE if writer else LockMode.SHARED): document = read_projection_document(directory) + meta = read_meta(meta_path(directory)) + cwd = meta.cwd if meta is not None else "" + archived_events = _read_archived_events(directory) turns = _mapping(_mapping(document.get("indexes")).get("turns")) - values = [value for value in turns.values() if isinstance(value, dict)] - values.sort(key=lambda turn: (str(turn.get("start_ts") or ""), str(turn.get("turn_id") or ""))) + values = [] + for record in turns.values(): + if not isinstance(record, dict): + continue + if record.get("status") != "completed": + continue + values.append(_hydrate_turn(stored_session_id, record, archived_events, cwd)) + values.sort( + key=lambda turn: (str(turn.get("start_ts") or ""), str(turn.get("turn_id") or "")) + ) return json.loads(_canonical(values)) @@ -335,4 +497,6 @@ def reset_projection_state(config: Config, stored_session_id: str) -> None: with locked(projection_lock_path(directory), LockMode.EXCLUSIVE): remove_projection_state(directory) fsops.unlink(usage_jsonl_path(directory), missing_ok=True) - fsops.sync_directory(directory) + if directory.exists(): + _invalidate_usage_index(config, stored_session_id, directory) + fsops.sync_directory(directory) diff --git a/src/thirdeye/platforms/copilot/state.py b/src/thirdeye/platforms/copilot/state.py index 99a26c0..f908711 100644 --- a/src/thirdeye/platforms/copilot/state.py +++ b/src/thirdeye/platforms/copilot/state.py @@ -2,15 +2,14 @@ from __future__ import annotations -import json -import os -import tempfile from collections.abc import Callable from pathlib import Path from typing import Any from thirdeye._compat import fsops +from .jsonio import atomic_write_json, read_json_object + STATE_SCHEMA_VERSION = 1 STATE_FILENAME = "copilot.state.json" JOURNAL_FILENAME = "copilot.journal.json" @@ -39,39 +38,27 @@ def lock_path(session_dir: Path) -> Path: def _atomic_json(path: Path, value: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - fd, temp_name = tempfile.mkstemp(dir=path.parent, prefix=f"{path.name}.", suffix=".tmp") - try: - with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as stream: - json.dump(value, stream, ensure_ascii=False, sort_keys=True, separators=(",", ":")) - stream.write("\n") - stream.flush() - os.fsync(stream.fileno()) - fsops.replace(temp_name, path) - if path.name == STATE_FILENAME: - _fault("after_state_replace") - elif path.name == JOURNAL_FILENAME: - _fault("after_journal_replace") - fsops.sync_directory(path.parent) - _fault("after_dirsync") - except BaseException: - fsops.unlink(Path(temp_name), missing_ok=True) - raise + after_replace = None + if path.name == STATE_FILENAME: + after_replace = "after_state_replace" + elif path.name == JOURNAL_FILENAME: + after_replace = "after_journal_replace" + atomic_write_json( + path, + value, + on_replaced=(lambda: _fault(after_replace)) if after_replace else None, + on_synced=lambda: _fault("after_dirsync"), + ) def read_json(path: Path) -> dict[str, Any] | None: try: - raw = json.loads(fsops.read_text(path, encoding="utf-8")) - except FileNotFoundError: - return None - except (OSError, json.JSONDecodeError): - # A malformed state file is never silently used as a fresh cursor. The - # archive caller turns this into a diagnostic and leaves the evidence - # log itself readable. - raise ValueError(f"invalid Copilot archive state: {path}") from None - if not isinstance(raw, dict): - raise ValueError(f"invalid Copilot archive state: {path}") - return raw + return read_json_object(path, invalid_message=f"invalid Copilot archive state: {path}") + except OSError as exc: + # A malformed or unreadable state file is never silently used as a + # fresh cursor. The archive caller turns this into a diagnostic and + # leaves the evidence log itself readable. + raise ValueError(f"invalid Copilot archive state: {path}") from exc def write_state(session_dir: Path, value: dict[str, Any]) -> None: diff --git a/src/thirdeye/platforms/copilot/types.py b/src/thirdeye/platforms/copilot/types.py index c599c5c..7770b6c 100644 --- a/src/thirdeye/platforms/copilot/types.py +++ b/src/thirdeye/platforms/copilot/types.py @@ -117,6 +117,38 @@ class SourceSlice(TypedDict): # # ``Attribution.usage_source_id`` is the newest revision's source ID. # Durable attribution is keyed by ``logical_call_id``. +# +# ``UsageRow.call_id`` is that same durable logical call identity +# (``copilot:usage:...``), never a revision source ID, import sequence, or +# Store seq. Persistence keys the usage index and ``usage.jsonl`` by this +# value and stamps ``UsageRow.seq`` with the Store seq of the newest archived +# revision. A content correction replaces the existing row under that +# identity. Storage also keeps a ``usage_source_id -> logical_call_id`` map +# so a row committed before its attribution/accounting identity is known is +# re-keyed rather than counted twice. +# +# Copilot ``TurnSpanDict.attributes`` carries ``interaction_id`` (native user +# request) and ``agent_id`` (null/absent on a main interaction; non-null on a +# nested child that must not become a generic user turn). Copilot +# ``NormalizedEvent.attributes`` carries ``interaction_id``, optional +# ``stored_turn_id``, and ``agent_id``. Producers may also set auxiliary +# ``source_ids`` / ``source_references`` on a turn, llm call, subagent, or +# accounting call; storage joins those plus matching normalized events onto +# the archived Store records for ``read_projected_turns``. Missing evidence +# yields ``events: []`` and null seq/ts rather than guessing. +# +# ``commit_projection``'s ``next_state`` is an authoritative +# :class:`ProjectionState` snapshot: omitted keys stay at their empty +# defaults, and a now-closed interaction is absent from +# ``semantic_state.open_interactions``. Storage does not resurrect keys the +# builder removed. ``pending`` and ``diagnostics`` on a Projection replace +# those indexes in full (status is current, not cumulative). Semantic +# events, turns, usage, and attributions merge by stable id. +# +# Storage entry points: ``commit_projection``, ``load_projection_state``, +# ``read_projected_turns``, and ``reset_projection_state`` (deletes only +# rebuildable local projection files and the derived usage sidecar; never +# V1 evidence or the export ledger). AttributionStatus = Literal["matched", "pending", "ambiguous", "conflicting"] AttributionJoinKind = Literal["direct", "inferred"] @@ -260,6 +292,26 @@ class DatabaseRevision(TypedDict): content_revision: str +class CopilotEventAttributes(TypedDict, total=False): + """``NormalizedEvent.attributes`` keys Copilot storage uses for joins.""" + + interaction_id: str + stored_turn_id: str + agent_id: str | None + + +class CopilotTurnAttributes(TypedDict, total=False): + """``TurnSpanDict.attributes`` keys Copilot storage uses. + + ``agent_id`` is null or omitted on a main user interaction. A non-null + value marks a nested child span; storage never emits those as standalone + ``session_turns`` records. + """ + + interaction_id: str + agent_id: str | None + + class NormalizedEvent(TypedDict): """One semantic event with stable identity and source provenance. @@ -267,6 +319,8 @@ class NormalizedEvent(TypedDict): ``kind="auxiliary_model_call"`` and ``classification="title_generation"``. They must not appear as main ``call_candidates`` and must not inflate conversation token totals. + + Copilot ``attributes`` follow :class:`CopilotEventAttributes`. """ id: str @@ -425,7 +479,12 @@ class AccountingProjection(TypedDict): class Projection(TypedDict): - """Combined local-only V2 projection, serializable via UsageRow.to_dict.""" + """Combined local-only V2 projection, serializable via UsageRow.to_dict. + + ``pending`` and ``diagnostics`` are the complete current sets; a later + commit that omits an item retracts it. ``usage_rows`` use + ``UsageRow.call_id`` as the logical call identity. + """ normalized_events: list[NormalizedEvent] turns: list[TurnSpanDict] @@ -506,6 +565,13 @@ class ProjectionState(TypedDict): ``projection_schema_version`` is :data:`PROJECTION_SCHEMA_VERSION`. It is not the V1 ``schema_version`` field on source envelopes. + + Passed as ``next_state`` to ``commit_projection``, this snapshot replaces + the previously loaded builder state. ``projection_revision`` is a digest + of the committed indexes when the builder leaves it empty. + ``index_totals`` is the size of each derived index after the commit, not + a per-commit delta. A stale schema version is discarded so a rebuild + can reconstruct derived indexes from the V1 archive. """ projection_schema_version: int @@ -513,4 +579,4 @@ class ProjectionState(TypedDict): semantic_state: SemanticProjectionState accounting_state: AccountingProjectionState projection_revision: str - commit_result: NotRequired[dict[str, int]] + index_totals: NotRequired[dict[str, int]] diff --git a/tests/platforms/copilot/fixtures/reconciliation-cases/storage.json b/tests/platforms/copilot/fixtures/reconciliation-cases/storage.json index e898dbc..63bd31e 100644 --- a/tests/platforms/copilot/fixtures/reconciliation-cases/storage.json +++ b/tests/platforms/copilot/fixtures/reconciliation-cases/storage.json @@ -36,7 +36,7 @@ "pending_tool_call_ids": [] } }, - "commit_result": { + "index_totals": { "events": 2, "turns": 1, "usage": 1, diff --git a/tests/platforms/copilot/test_migration.py b/tests/platforms/copilot/test_migration.py index ceeb541..a4d21c0 100644 --- a/tests/platforms/copilot/test_migration.py +++ b/tests/platforms/copilot/test_migration.py @@ -3,6 +3,9 @@ from __future__ import annotations import json +import subprocess +import sys +import time from collections.abc import Iterator from contextlib import contextmanager from pathlib import Path @@ -30,7 +33,13 @@ read_projected_turns, reset_projection_state, ) -from thirdeye.platforms.copilot.types import Projection, SourceBatch, SourcePaths, SourceRecord +from thirdeye.platforms.copilot.types import ( + PROJECTION_SCHEMA_VERSION, + Projection, + SourceBatch, + SourcePaths, + SourceRecord, +) from thirdeye.usage.read import iter_calls from thirdeye.usage.types import UsageRow @@ -143,7 +152,7 @@ def paths(tmp_path: Path) -> SourcePaths: return resolve_sources(home) -def _stored(config: Config, paths: SourcePaths) -> str: +def _stored(_config: Config, paths: SourcePaths) -> str: return stored_session_id(paths, NATIVE_ID) @@ -155,7 +164,9 @@ def _seed_v1_archive(config: Config, paths: SourcePaths) -> str: commit_batch( config, paths, - _batch(paths, [_record("key/a/v1-one"), _record("key/a/v1-two", ts="2026-09-10T17:08:26.000Z")]), + _batch( + paths, [_record("key/a/v1-one"), _record("key/a/v1-two", ts="2026-09-10T17:08:26.000Z")] + ), ) return _stored(config, paths) @@ -222,7 +233,7 @@ def test_projection_journal_crash_recovers_on_next_read( assert projection_journal_path(directory).is_file() state = load_projection_state(config, stored) - assert state["commit_result"]["events"] == 2 + assert state["index_totals"]["events"] == 2 assert not projection_journal_path(directory).exists() turns = read_projected_turns(config, stored) assert len(turns) == 1 @@ -281,35 +292,65 @@ def test_rebuild_after_reset_matches_original_projection( after_state = load_projection_state(config, stored) assert after_turns == before_turns - assert after_state["commit_result"] == before_state["commit_result"] + assert after_state["index_totals"] == before_state["index_totals"] def test_competing_projection_commits_merge_indexes(config: Config, paths: SourcePaths) -> None: stored = _seed_v1_archive(config, paths) + start_signal = config.root.parent / "start-projection-writers" + script = r""" +from pathlib import Path +import sys +import time - first = _projection( - normalized_events=[_normalized_event("evt-a", source_ids=["key/a/v1-one"])], - usage_rows=[_usage_row(call_id="usage-a", input_tokens=50, session_id=stored)], - attributions=[ +from thirdeye.config import Config +from thirdeye.platforms.copilot.projection_state import empty_projection_state +from thirdeye.platforms.copilot.projection_store import commit_projection +from thirdeye.usage.types import UsageRow + +root = Path(sys.argv[1]) +stored = sys.argv[2] +writer_id = sys.argv[3] +start_signal = Path(sys.argv[4]) +while not start_signal.exists(): + time.sleep(0.001) + +source_id = f"key/a/v1-{'one' if writer_id == 'a' else 'two'}" +event_id = f"evt-{writer_id}" +logical_id = f"logical-{writer_id}" +usage_source = f"usage-{writer_id}" +config = Config(root=root) +row = UsageRow( + session_id=stored, + seq=0, + call_id=usage_source, + ts="2026-09-10T17:08:30.000Z", + platform="copilot", + provider_name="openai", + response_model="gpt-5.6-luna", + input_tokens=50 if writer_id == "a" else 75, + output_tokens=10, +) +commit_projection( + config, + stored, + { + "normalized_events": [ { - "usage_source_id": "usage-a", - "logical_call_id": "logical-a", - "stored_turn_id": None, - "agent_id": None, - "call_id": None, - "status": "pending", - "join_kind": None, - "evidence": [], + "id": event_id, + "kind": "user_prompt", + "classification": "main", + "initiator": "user", + "source_ids": [source_id], + "attributes": {"interaction_id": "interaction-migrate-1"}, } ], - ) - second = _projection( - normalized_events=[_normalized_event("evt-b", source_ids=["key/a/v1-two"])], - usage_rows=[_usage_row(call_id="usage-b", input_tokens=75, session_id=stored)], - attributions=[ + "turns": [], + "usage_rows": [row], + "attributions": [ { - "usage_source_id": "usage-b", - "logical_call_id": "logical-b", + "usage_source_id": usage_source, + "logical_call_id": logical_id, "stored_turn_id": None, "agent_id": None, "call_id": None, @@ -318,14 +359,34 @@ def test_competing_projection_commits_merge_indexes(config: Config, paths: Sourc "evidence": [], } ], - ) - - commit_projection(config, stored, first, empty_projection_state()) - commit_projection(config, stored, second, empty_projection_state()) + "pending": [], + "diagnostics": [], + }, + empty_projection_state(), +) +""" + processes = [ + subprocess.Popen( + [sys.executable, "-c", script, str(config.root), stored, writer_id, str(start_signal)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + encoding="utf-8", + ) + for writer_id in ("a", "b") + ] + start_signal.touch() + deadline = time.monotonic() + 30 + failures: list[str] = [] + for process in processes: + stdout, stderr = process.communicate(timeout=max(0.1, deadline - time.monotonic())) + if process.returncode != 0: + failures.append(f"exit={process.returncode} stdout={stdout!r} stderr={stderr!r}") + assert not failures, failures state = load_projection_state(config, stored) - assert state["commit_result"]["events"] == 2 - assert state["commit_result"]["usage"] == 2 + assert state["index_totals"]["events"] == 2 + assert state["index_totals"]["usage"] == 2 document = json.loads(projection_state_path(_directory(config, stored)).read_text()) assert set(document["indexes"]["events"]) == {"evt-a", "evt-b"} @@ -409,3 +470,48 @@ def counting_write_usage(path: Path, usage_index: dict[str, Any]) -> None: rows = list(iter_calls(directory)) assert len(rows) == 1 assert rows[0].input_tokens == 222 + + +def test_stale_projection_schema_is_discarded_for_rebuild( + config: Config, paths: SourcePaths +) -> None: + stored = _seed_v1_archive(config, paths) + directory = _directory(config, stored) + stale = { + "schema_version": PROJECTION_SCHEMA_VERSION - 1, + "state": { + "projection_schema_version": PROJECTION_SCHEMA_VERSION - 1, + "archive_source_ids": ["stale"], + "semantic_state": {"open_interactions": {"old": {}}}, + "accounting_state": {"logical_calls": {}}, + "projection_revision": "stale", + }, + "indexes": {"events": {"old": {"id": "old"}}}, + } + projection_state_path(directory).write_text(json.dumps(stale), encoding="utf-8") + + state = load_projection_state(config, stored) + assert state["archive_source_ids"] == [] + assert state["semantic_state"]["open_interactions"] == {} + assert not projection_state_path(directory).exists() + assert read_projected_turns(config, stored) == [] + + +def test_stale_journal_schema_is_discarded_without_raising( + config: Config, paths: SourcePaths +) -> None: + stored = _seed_v1_archive(config, paths) + directory = _directory(config, stored) + journal = { + "schema_version": 0, + "document": { + "schema_version": 1, + "state": {"projection_schema_version": 1}, + "indexes": {}, + }, + } + projection_journal_path(directory).write_text(json.dumps(journal) + "\n", encoding="utf-8") + + state = load_projection_state(config, stored) + assert state["projection_schema_version"] == PROJECTION_SCHEMA_VERSION + assert not projection_journal_path(directory).exists() diff --git a/tests/platforms/copilot/test_projection_store.py b/tests/platforms/copilot/test_projection_store.py index 69537bb..2e976fe 100644 --- a/tests/platforms/copilot/test_projection_store.py +++ b/tests/platforms/copilot/test_projection_store.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import shutil from pathlib import Path from typing import Any @@ -26,6 +27,7 @@ ) from thirdeye.platforms.copilot.types import Projection, SourceBatch, SourcePaths, SourceRecord from thirdeye.reader import SessionReader +from thirdeye.usage.index import UsageIndex from thirdeye.usage.read import iter_calls from thirdeye.usage.types import UsageRow @@ -133,6 +135,10 @@ def _normalized_event( "classification": "main", "initiator": "user", "source_ids": source_ids, + "source_references": [ + {"source_id": source_id, "source_kind": "transcript", "role": "user_prompt"} + for source_id in source_ids + ], "attributes": attributes, } @@ -180,7 +186,7 @@ def paths(tmp_path: Path) -> SourcePaths: return resolve_sources(home) -def _stored(config: Config, paths: SourcePaths) -> str: +def _stored(_config: Config, paths: SourcePaths) -> str: return stored_session_id(paths, NATIVE_ID) @@ -240,11 +246,12 @@ def test_commit_projection_persists_indexes_and_usage_sidecar( usage_rows = list(iter_calls(directory)) assert len(usage_rows) == 1 - assert usage_rows[0].call_id == "src-usage-1" + assert usage_rows[0].call_id == "logical-1" state = load_projection_state(config, stored) assert state["archive_source_ids"] == ["key/a/event-1", "key/a/event-2"] - assert state["commit_result"]["events"] == 2 + assert state["index_totals"]["events"] == 2 + assert state["projection_revision"].startswith("sha256:") def test_read_projected_turns_join_archived_store_events( @@ -330,13 +337,15 @@ def test_usage_revision_replaces_under_stable_logical_identity( rows = list(iter_calls(directory)) assert len(rows) == 1 assert rows[0].input_tokens == 150 - assert rows[0].call_id == "src-rev-2" + assert rows[0].call_id == "logical-1" sidecar_lines = usage_jsonl_path(directory).read_text(encoding="utf-8").splitlines() assert len(sidecar_lines) == 1 -def test_incremental_commits_merge_builder_state(config: Config, paths: SourcePaths) -> None: +def test_next_state_is_authoritative_and_closes_interactions( + config: Config, paths: SourcePaths +) -> None: stored = _seed_archive(config, paths, [_record("key/a/one"), _record("key/a/two")]) first_state = empty_projection_state() @@ -364,7 +373,8 @@ def test_incremental_commits_merge_builder_state(config: Config, paths: SourcePa ) second_state = empty_projection_state() - second_state["archive_source_ids"] = ["key/a/two"] + second_state["archive_source_ids"] = ["key/a/one", "key/a/two"] + second_state["semantic_state"] = {"open_interactions": {}} second_state["accounting_state"] = { "logical_calls": { "logical-1": { @@ -385,10 +395,10 @@ def test_incremental_commits_merge_builder_state(config: Config, paths: SourcePa second_state, ) - merged = load_projection_state(config, stored) - assert merged["archive_source_ids"] == ["key/a/one", "key/a/two"] - assert INTERACTION_ID in merged["semantic_state"]["open_interactions"] - assert "logical-1" in merged["accounting_state"]["logical_calls"] + replaced = load_projection_state(config, stored) + assert replaced["archive_source_ids"] == ["key/a/one", "key/a/two"] + assert replaced["semantic_state"]["open_interactions"] == {} + assert "logical-1" in replaced["accounting_state"]["logical_calls"] document_events = json.loads(projection_state_path(_directory(config, stored)).read_text())[ "indexes" @@ -437,14 +447,15 @@ def test_projection_reads_do_not_require_live_copilot_sources( ) copilot_home = tmp_path / "copilot-home" - assert copilot_home.exists() - for child in copilot_home.iterdir(): - if child.is_file(): - child.unlink() - else: - import shutil - - shutil.rmtree(child) + session_root = copilot_home / "session-state" / NATIVE_ID + session_root.mkdir(parents=True) + (session_root / "events.jsonl").write_text("{}\n", encoding="utf-8") + (copilot_home / "session-store.db").write_bytes(b"sqlite") + assert any(copilot_home.iterdir()) + shutil.rmtree(session_root) + (copilot_home / "session-store.db").unlink() + assert not session_root.exists() + assert not (copilot_home / "session-store.db").exists() turns = read_projected_turns(config, stored) assert len(turns) == 1 @@ -475,7 +486,9 @@ def test_unfinished_projection_pending_does_not_block_later_capture( empty_projection_state(), ) - commit_batch(config, paths, _batch(paths, [_record("key/a/second")], next_cursor={"generation": 2})) + commit_batch( + config, paths, _batch(paths, [_record("key/a/second")], next_cursor={"generation": 2}) + ) commit_projection( config, @@ -487,8 +500,9 @@ def test_unfinished_projection_pending_does_not_block_later_capture( ) state = load_projection_state(config, stored) - assert state["commit_result"]["events"] == 1 - assert state["commit_result"]["pending"] == 1 + assert state["index_totals"]["events"] == 1 + assert state["index_totals"]["pending"] == 0 + assert state["index_totals"]["diagnostics"] == 0 captured = { event["data"]["source_record"]["source_id"] for event in SessionReader(_directory(config, stored)).iter_events( @@ -498,7 +512,9 @@ def test_unfinished_projection_pending_does_not_block_later_capture( assert captured == {"key/a/first", "key/a/second"} -def test_reset_projection_state_removes_only_derived_files(config: Config, paths: SourcePaths) -> None: +def test_reset_projection_state_removes_only_derived_files( + config: Config, paths: SourcePaths +) -> None: stored = _seed_archive(config, paths, [_record("key/a/persist")]) commit_projection( config, @@ -519,3 +535,257 @@ def test_reset_projection_state_removes_only_derived_files(config: Config, paths archived = list(SessionReader(directory).iter_events()) assert len(archived) == 1 assert archived[0]["data"]["schema_version"] == SOURCE_SCHEMA_VERSION + + +def _indexed_usage(config: Config, stored: str) -> list[tuple[str, int, int]]: + index = UsageIndex(config.root) + connection = index.connect() + try: + index.refresh(connection) + rows = connection.execute( + "SELECT call_id, gen_ai_usage_input_tokens, seq FROM usage WHERE session_id = ? " + "ORDER BY call_id", + (stored,), + ).fetchall() + finally: + connection.close() + return [(str(call_id), int(tokens), int(seq)) for call_id, tokens, seq in rows] + + +def test_incremental_turn_commit_retains_prior_partition_evidence( + config: Config, paths: SourcePaths +) -> None: + stored = _seed_archive( + config, + paths, + [_record("key/a/e1"), _record("key/a/e2", ts="2026-09-10T17:08:26.000Z")], + ) + turn = _main_turn(source_ids=["key/a/e1"]) + commit_projection( + config, + stored, + _projection( + normalized_events=[_normalized_event("evt-1", source_ids=["key/a/e1"])], + turns=[turn], + ), + empty_projection_state(), + ) + first = read_projected_turns(config, stored) + assert [event["data"]["source_record"]["source_id"] for event in first[0]["events"]] == [ + "key/a/e1" + ] + + later = _main_turn(source_ids=["key/a/e2"]) + commit_projection( + config, + stored, + _projection( + normalized_events=[_normalized_event("evt-2", source_ids=["key/a/e2"])], + turns=[later], + ), + empty_projection_state(), + ) + turns = read_projected_turns(config, stored) + assert len(turns) == 1 + assert [event["data"]["source_record"]["source_id"] for event in turns[0]["events"]] == [ + "key/a/e1", + "key/a/e2", + ] + assert turns[0]["start_seq"] == 0 + assert turns[0]["end_seq"] == 1 + + +def test_derived_state_does_not_duplicate_raw_archive_payloads( + config: Config, paths: SourcePaths +) -> None: + stored = _seed_archive( + config, + paths, + [_record("key/a/user"), _record("key/a/assistant", ts="2026-09-10T17:08:26.000Z")], + ) + commit_projection( + config, + stored, + _projection( + normalized_events=[ + _normalized_event("evt-user", source_ids=["key/a/user"]), + _normalized_event("evt-assistant", source_ids=["key/a/assistant"]), + ], + turns=[_main_turn(source_ids=["key/a/user", "key/a/assistant"])], + ), + empty_projection_state(), + ) + document = json.loads(projection_state_path(_directory(config, stored)).read_text()) + dumped = json.dumps(document) + assert "user.message" not in dumped + assert document["indexes"]["turns"] + turn_record = next(iter(document["indexes"]["turns"].values())) + assert "events" not in turn_record + assert turn_record["span"]["input_message"] == "hello" + assert set(turn_record["source_ids"]) == {"key/a/user", "key/a/assistant"} + + +def test_unresolved_turn_evidence_does_not_invent_store_events( + config: Config, paths: SourcePaths +) -> None: + stored = _seed_archive(config, paths, [_record("key/a/other")]) + commit_projection( + config, + stored, + _projection(turns=[_main_turn()]), + empty_projection_state(), + ) + turns = read_projected_turns(config, stored) + assert len(turns) == 1 + assert turns[0]["events"] == [] + assert turns[0]["start_seq"] is None + assert turns[0]["end_seq"] is None + + +def test_in_progress_turns_are_omitted_from_projected_reads( + config: Config, paths: SourcePaths +) -> None: + stored = _seed_archive(config, paths, [_record("key/a/open")]) + open_turn = _main_turn(source_ids=["key/a/open"]) + open_turn["status"] = "in_progress" + commit_projection( + config, + stored, + _projection( + normalized_events=[_normalized_event("evt-open", source_ids=["key/a/open"])], + turns=[open_turn], + ), + empty_projection_state(), + ) + assert read_projected_turns(config, stored) == [] + + +def test_commit_projection_rejects_unknown_session(config: Config) -> None: + with pytest.raises(ValueError, match="unknown Copilot session"): + commit_projection(config, "ghost-session", _projection(), empty_projection_state()) + assert not (config.root / "traces" / PLATFORM_NAME / "ghost-session").exists() + + +def test_commit_projection_rejects_usage_row_for_another_session( + config: Config, paths: SourcePaths +) -> None: + stored = _seed_archive(config, paths, [_record("key/a/event")]) + with pytest.raises(ValueError, match="session_id"): + commit_projection( + config, + stored, + _projection(usage_rows=[_usage_row(call_id="logical-1", session_id="other")]), + empty_projection_state(), + ) + + +def test_usage_seq_is_stamped_from_archived_revision(config: Config, paths: SourcePaths) -> None: + stored = _seed_archive( + config, + paths, + [ + _record("key/a/prompt"), + _record("db-src-1", source_kind="database", ts="2026-09-10T17:08:30.000Z"), + ], + ) + next_state = empty_projection_state() + next_state["accounting_state"] = { + "logical_calls": { + "logical-1": { + "logical_call_id": "logical-1", + "generation": "gen-a", + "content_revision": "rev-a", + "metrics_digest": "sha256:abc", + "usage_source_id": "db-src-1", + } + } + } + commit_projection( + config, + stored, + _projection( + usage_rows=[_usage_row(call_id="db-src-1", session_id=stored)], + attributions=[_attribution(logical_call_id="logical-1", usage_source_id="db-src-1")], + ), + next_state, + ) + rows = list(iter_calls(_directory(config, stored))) + assert len(rows) == 1 + assert rows[0].seq == 1 + assert rows[0].call_id == "logical-1" + assert _indexed_usage(config, stored) == [("logical-1", 100, 1)] + + +def test_usage_correction_is_visible_through_usage_index( + config: Config, paths: SourcePaths +) -> None: + stored = _seed_archive(config, paths, [_record("key/a/usage")]) + first = _projection( + usage_rows=[_usage_row(call_id="logical-1", input_tokens=100, session_id=stored)], + ) + commit_projection(config, stored, first, empty_projection_state()) + assert _indexed_usage(config, stored) == [("logical-1", 100, 0)] + + second = _projection( + usage_rows=[_usage_row(call_id="logical-1", input_tokens=150, session_id=stored, seq=1)], + ) + commit_projection(config, stored, second, empty_projection_state()) + assert _indexed_usage(config, stored) == [("logical-1", 150, 0)] + + +def test_delayed_attribution_rekeys_source_id_without_double_counting( + config: Config, paths: SourcePaths +) -> None: + stored = _seed_archive(config, paths, [_record("key/a/usage")]) + first_state = empty_projection_state() + first_state["accounting_state"] = { + "logical_calls": { + "copilot:usage:logical-1": { + "logical_call_id": "copilot:usage:logical-1", + "generation": "gen-a", + "content_revision": "rev-1", + "metrics_digest": "sha256:abc", + "usage_source_id": "db-src-rev1", + } + } + } + commit_projection( + config, + stored, + _projection( + usage_rows=[_usage_row(call_id="db-src-rev1", input_tokens=100, session_id=stored)] + ), + first_state, + ) + + second_state = empty_projection_state() + second_state["accounting_state"] = { + "logical_calls": { + "copilot:usage:logical-1": { + "logical_call_id": "copilot:usage:logical-1", + "generation": "gen-a", + "content_revision": "rev-2", + "metrics_digest": "sha256:def", + "usage_source_id": "db-src-rev2", + } + } + } + commit_projection( + config, + stored, + _projection( + usage_rows=[_usage_row(call_id="db-src-rev2", input_tokens=150, session_id=stored)], + attributions=[ + _attribution( + logical_call_id="copilot:usage:logical-1", usage_source_id="db-src-rev2" + ) + ], + ), + second_state, + ) + + rows = list(iter_calls(_directory(config, stored))) + assert [(row.call_id, row.input_tokens) for row in rows] == [("copilot:usage:logical-1", 150)] + assert _indexed_usage(config, stored) == [("copilot:usage:logical-1", 150, 0)] + document = json.loads(projection_state_path(_directory(config, stored)).read_text()) + assert set(document["indexes"]["usage"]) == {"copilot:usage:logical-1"} From 698beb767855f5c4149c1dc6630aa55f571281ff Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Fri, 11 Sep 2026 16:07:03 -0700 Subject: [PATCH 58/88] Fix delayed usage rekeying and projection recovery under lock. Identity-only commits now relocate existing usage rows, load repairs a stale UsageIndex even when the sidecar already matches, and journal recovery always runs under an exclusive lock without creating unknown session paths. Co-authored-by: Cursor --- .../platforms/copilot/projection_state.py | 12 +- .../platforms/copilot/projection_store.py | 68 +++++-- tests/platforms/copilot/test_migration.py | 51 +++++ .../copilot/test_projection_store.py | 175 +++++++++++++++++- 4 files changed, 282 insertions(+), 24 deletions(-) diff --git a/src/thirdeye/platforms/copilot/projection_state.py b/src/thirdeye/platforms/copilot/projection_state.py index 9e01ab6..e8d7a24 100644 --- a/src/thirdeye/platforms/copilot/projection_state.py +++ b/src/thirdeye/platforms/copilot/projection_state.py @@ -112,7 +112,13 @@ def read_projection_document(session_dir: Path) -> dict[str, Any]: A stale schema version is disposable derived state: the files are removed and the caller rebuilds from the immutable V1 archive. """ - journal = _read_json(projection_journal_path(session_dir)) + journal_path = projection_journal_path(session_dir) + try: + journal = _read_json(journal_path) + except ValueError: + # Derived journals are disposable: corrupt JSON is treated like a stale + # schema so recovery can fall through to the last snapshot. + journal = {} if journal is not None: document = journal.get("document") journal_ok = journal.get("schema_version") == PROJECTION_JOURNAL_SCHEMA_VERSION @@ -123,12 +129,12 @@ def read_projection_document(session_dir: Path) -> dict[str, Any]: document, fault_point="after_projection_recovery", ) - fsops.unlink(projection_journal_path(session_dir), missing_ok=True) + fsops.unlink(journal_path, missing_ok=True) fsops.sync_directory(session_dir) _fault("after_projection_recovery_clear") return document # Stale or unreadable journal: drop it and fall through to the snapshot. - fsops.unlink(projection_journal_path(session_dir), missing_ok=True) + fsops.unlink(journal_path, missing_ok=True) fsops.sync_directory(session_dir) document = _read_json(projection_state_path(session_dir)) diff --git a/src/thirdeye/platforms/copilot/projection_store.py b/src/thirdeye/platforms/copilot/projection_store.py index 327543d..f138ccc 100644 --- a/src/thirdeye/platforms/copilot/projection_store.py +++ b/src/thirdeye/platforms/copilot/projection_store.py @@ -28,7 +28,6 @@ from .constants import PLATFORM_NAME, SOURCE_SCHEMA_VERSION from .projection_state import ( empty_projection_state, - projection_journal_path, projection_lock_path, publish_projection_document, read_projection_document, @@ -72,6 +71,18 @@ def _require_session(directory: Path, stored_session_id: str) -> None: raise ValueError(f"unknown Copilot session: {stored_session_id}") +def _existing_session_dir(config: Config, stored_session_id: str) -> Path | None: + """Return the session directory only when V1 meta is already on disk. + + ``locked()`` creates parent directories, so unknown ids must be rejected + before acquiring the projection lock. + """ + directory = _directory(config, stored_session_id) + if read_meta(meta_path(directory)) is None: + return None + return directory + + def _source_id(event: dict[str, Any]) -> str | None: if event.get("t") not in _RAW_EVENT_TYPES: return None @@ -198,9 +209,21 @@ def _usage_identity(row: UsageRow, identities: dict[str, str]) -> str: def _drop_aliased_usage_keys(usage_index: dict[str, Any], identities: dict[str, str]) -> None: + """Move a source-id row onto its logical id instead of deleting the call. + + An identity-only later commit may learn ``usage_source_id -> logical_call_id`` + without sending the ``UsageRow`` again. Popping the alias without copying + would drop the only persisted call. + """ for source_id, logical_id in identities.items(): - if source_id != logical_id: - usage_index.pop(source_id, None) + if source_id == logical_id: + continue + existing = usage_index.pop(source_id, None) + if existing is None or logical_id in usage_index or not isinstance(existing, dict): + continue + relocated = dict(existing) + relocated["call_id"] = logical_id + usage_index[logical_id] = relocated def _usage_seq( @@ -295,15 +318,16 @@ def _publish_usage( force: bool = False, ) -> None: path = usage_jsonl_path(directory) - if not usage_index: - if path.exists(): - fsops.unlink(path, missing_ok=True) - fsops.sync_directory(directory) - _invalidate_usage_index(config, stored_session_id, directory) - return - if not force and _sidecar_matches(path, usage_index): + matches = _sidecar_matches(path, usage_index) + if not matches: + if not usage_index: + if path.exists(): + fsops.unlink(path, missing_ok=True) + fsops.sync_directory(directory) + else: + _write_usage_index(directory, usage_index) + elif not force: return - _write_usage_index(directory, usage_index) _invalidate_usage_index(config, stored_session_id, directory) @@ -456,19 +480,22 @@ def commit_projection( def load_projection_state(config: Config, stored_session_id: str) -> dict[str, Any]: """Return a copy of derived builder state, completing journal recovery.""" - directory = _directory(config, stored_session_id) + directory = _existing_session_dir(config, stored_session_id) + if directory is None: + return empty_projection_state() with locked(projection_lock_path(directory), LockMode.EXCLUSIVE): document = read_projection_document(directory) usage_index = _mapping(_mapping(document.get("indexes")).get("usage")) - _publish_usage(config, stored_session_id, directory, usage_index) + _publish_usage(config, stored_session_id, directory, usage_index, force=True) return json.loads(_canonical(_mapping(document.get("state")))) def read_projected_turns(config: Config, stored_session_id: str) -> list[dict[str, Any]]: """Read completed main interaction records without semantic duplicates.""" - directory = _directory(config, stored_session_id) - writer = projection_journal_path(directory).exists() - with locked(projection_lock_path(directory), LockMode.EXCLUSIVE if writer else LockMode.SHARED): + directory = _existing_session_dir(config, stored_session_id) + if directory is None: + return [] + with locked(projection_lock_path(directory), LockMode.EXCLUSIVE): document = read_projection_document(directory) meta = read_meta(meta_path(directory)) cwd = meta.cwd if meta is not None else "" @@ -493,10 +520,11 @@ def reset_projection_state(config: Config, stored_session_id: str) -> None: The caller must subsequently commit a full replay. This intentionally does not import, reset, or otherwise interact with export-state files. """ - directory = _directory(config, stored_session_id) + directory = _existing_session_dir(config, stored_session_id) + if directory is None: + return with locked(projection_lock_path(directory), LockMode.EXCLUSIVE): remove_projection_state(directory) fsops.unlink(usage_jsonl_path(directory), missing_ok=True) - if directory.exists(): - _invalidate_usage_index(config, stored_session_id, directory) - fsops.sync_directory(directory) + _invalidate_usage_index(config, stored_session_id, directory) + fsops.sync_directory(directory) diff --git a/tests/platforms/copilot/test_migration.py b/tests/platforms/copilot/test_migration.py index a4d21c0..6cbc7ec 100644 --- a/tests/platforms/copilot/test_migration.py +++ b/tests/platforms/copilot/test_migration.py @@ -40,6 +40,7 @@ SourcePaths, SourceRecord, ) +from thirdeye.usage.index import UsageIndex from thirdeye.usage.read import iter_calls from thirdeye.usage.types import UsageRow @@ -275,6 +276,42 @@ def test_load_projection_state_recovers_missing_usage_sidecar( assert rows[0].input_tokens == 111 +def _indexed_usage(config: Config, stored: str) -> list[tuple[str, int]]: + index = UsageIndex(config.root) + connection = index.connect() + try: + index.refresh(connection) + rows = connection.execute( + "SELECT call_id, gen_ai_usage_input_tokens FROM usage WHERE session_id = ? " + "ORDER BY call_id", + (stored,), + ).fetchall() + finally: + connection.close() + return [(str(call_id), int(tokens)) for call_id, tokens in rows] + + +def test_load_clears_stale_usage_index_after_sidecar_unlink_crash( + config: Config, paths: SourcePaths, monkeypatch: pytest.MonkeyPatch +) -> None: + stored = _seed_v1_archive(config, paths) + commit_projection(config, stored, _sample_projection(stored), empty_projection_state()) + assert _indexed_usage(config, stored) == [("logical-usage-1", 111)] + + import thirdeye.platforms.copilot.projection_store as store_mod + + monkeypatch.setattr(store_mod, "_invalidate_usage_index", lambda *_args, **_kwargs: None) + reset_projection_state(config, stored) + directory = _directory(config, stored) + assert not usage_jsonl_path(directory).exists() + assert _indexed_usage(config, stored) == [("logical-usage-1", 111)] + + monkeypatch.undo() + load_projection_state(config, stored) + assert _indexed_usage(config, stored) == [] + assert list(iter_calls(directory)) == [] + + def test_rebuild_after_reset_matches_original_projection( config: Config, paths: SourcePaths ) -> None: @@ -515,3 +552,17 @@ def test_stale_journal_schema_is_discarded_without_raising( state = load_projection_state(config, stored) assert state["projection_schema_version"] == PROJECTION_SCHEMA_VERSION assert not projection_journal_path(directory).exists() + + +def test_corrupt_journal_is_discarded_and_snapshot_used( + config: Config, paths: SourcePaths +) -> None: + stored = _seed_v1_archive(config, paths) + commit_projection(config, stored, _sample_projection(stored), empty_projection_state()) + directory = _directory(config, stored) + projection_journal_path(directory).write_text("{not-json", encoding="utf-8") + + state = load_projection_state(config, stored) + assert state["index_totals"]["events"] == 2 + assert not projection_journal_path(directory).exists() + assert len(read_projected_turns(config, stored)) == 1 diff --git a/tests/platforms/copilot/test_projection_store.py b/tests/platforms/copilot/test_projection_store.py index 2e976fe..6eb7fac 100644 --- a/tests/platforms/copilot/test_projection_store.py +++ b/tests/platforms/copilot/test_projection_store.py @@ -10,7 +10,7 @@ import pytest from thirdeye.config import Config -from thirdeye.paths import session_dir, usage_jsonl_path +from thirdeye.paths import session_dir, usage_db_path, usage_jsonl_path from thirdeye.platforms.copilot.archive import commit_batch from thirdeye.platforms.copilot.constants import PLATFORM_NAME, SOURCE_SCHEMA_VERSION from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id @@ -315,6 +315,54 @@ def test_child_agent_top_level_turns_are_excluded(config: Config, paths: SourceP assert [turn["turn_id"] for turn in turns] == [main_turn["turn_id"]] +def test_nested_child_archive_events_stay_on_main_turn( + config: Config, paths: SourcePaths +) -> None: + stored = _seed_archive( + config, + paths, + [_record("key/a/main"), _record("key/a/child", ts="2026-09-10T17:08:26.000Z")], + ) + main_turn = _main_turn(source_ids=["key/a/main"]) + main_turn["subagents"] = [ + { + "turn_id": "copilot:turn:key:session:child", + "start_ts": "2026-09-10T17:08:26.000Z", + "end_ts": "2026-09-10T17:08:27.000Z", + "input_message": "explore", + "output_message": "found", + "status": "completed", + "llm_calls": [], + "permission_requests": [], + "subagents": [], + "attributes": {"interaction_id": INTERACTION_ID, "agent_id": "subagent-123"}, + "source_ids": ["key/a/child"], + } + ] + child_event = _normalized_event("evt-child", source_ids=["key/a/child"]) + child_event["attributes"]["agent_id"] = "subagent-123" + + commit_projection( + config, + stored, + _projection( + normalized_events=[ + _normalized_event("evt-main", source_ids=["key/a/main"]), + child_event, + ], + turns=[main_turn], + ), + empty_projection_state(), + ) + + turns = read_projected_turns(config, stored) + assert len(turns) == 1 + assert [event["data"]["source_record"]["source_id"] for event in turns[0]["events"]] == [ + "key/a/main", + "key/a/child", + ] + + def test_usage_revision_replaces_under_stable_logical_identity( config: Config, paths: SourcePaths ) -> None: @@ -537,6 +585,32 @@ def test_reset_projection_state_removes_only_derived_files( assert archived[0]["data"]["schema_version"] == SOURCE_SCHEMA_VERSION +def test_reset_projection_state_leaves_export_ledger_untouched( + config: Config, paths: SourcePaths +) -> None: + stored = _seed_archive(config, paths, [_record("key/a/persist")]) + commit_projection( + config, + stored, + _projection( + usage_rows=[_usage_row(call_id="usage-1", session_id=stored)], + turns=[_main_turn(source_ids=["key/a/persist"])], + ), + empty_projection_state(), + ) + directory = _directory(config, stored) + ledger = directory / "copilot.export.ledger.json" + payload = '{"schema_version":1,"jobs":[]}\n' + ledger.write_text(payload, encoding="utf-8") + + reset_projection_state(config, stored) + + assert ledger.is_file() + assert ledger.read_text(encoding="utf-8") == payload + assert not projection_state_path(directory).exists() + assert not usage_jsonl_path(directory).exists() + + def _indexed_usage(config: Config, stored: str) -> list[tuple[str, int, int]]: index = UsageIndex(config.root) connection = index.connect() @@ -666,6 +740,17 @@ def test_commit_projection_rejects_unknown_session(config: Config) -> None: assert not (config.root / "traces" / PLATFORM_NAME / "ghost-session").exists() +def test_read_and_reset_unknown_session_do_not_create_paths(config: Config) -> None: + ghost = "ghost-session" + state = load_projection_state(config, ghost) + assert state["archive_source_ids"] == [] + assert state["semantic_state"]["open_interactions"] == {} + assert read_projected_turns(config, ghost) == [] + reset_projection_state(config, ghost) + assert not (config.root / "traces" / PLATFORM_NAME / ghost).exists() + assert not usage_db_path(config.root).exists() + + def test_commit_projection_rejects_usage_row_for_another_session( config: Config, paths: SourcePaths ) -> None: @@ -789,3 +874,91 @@ def test_delayed_attribution_rekeys_source_id_without_double_counting( assert _indexed_usage(config, stored) == [("copilot:usage:logical-1", 150, 0)] document = json.loads(projection_state_path(_directory(config, stored)).read_text()) assert set(document["indexes"]["usage"]) == {"copilot:usage:logical-1"} + + +def test_identity_only_commit_rekeys_existing_usage_without_a_row( + config: Config, paths: SourcePaths +) -> None: + stored = _seed_archive(config, paths, [_record("key/a/usage")]) + commit_projection( + config, + stored, + _projection( + usage_rows=[_usage_row(call_id="db-src-rev1", input_tokens=100, session_id=stored)] + ), + empty_projection_state(), + ) + directory = _directory(config, stored) + assert [(row.call_id, row.input_tokens) for row in iter_calls(directory)] == [ + ("db-src-rev1", 100) + ] + + later_state = empty_projection_state() + later_state["accounting_state"] = { + "logical_calls": { + "copilot:usage:logical-1": { + "logical_call_id": "copilot:usage:logical-1", + "generation": "gen-a", + "content_revision": "rev-1", + "metrics_digest": "sha256:abc", + "usage_source_id": "db-src-rev1", + } + } + } + counts = commit_projection( + config, + stored, + _projection( + attributions=[ + _attribution( + logical_call_id="copilot:usage:logical-1", usage_source_id="db-src-rev1" + ) + ] + ), + later_state, + ) + + assert counts["usage"] == 1 + rows = list(iter_calls(directory)) + assert [(row.call_id, row.input_tokens) for row in rows] == [("copilot:usage:logical-1", 100)] + assert _indexed_usage(config, stored) == [("copilot:usage:logical-1", 100, 0)] + document = json.loads(projection_state_path(directory).read_text()) + assert set(document["indexes"]["usage"]) == {"copilot:usage:logical-1"} + + +def test_load_repairs_stale_usage_index_when_sidecar_already_matches( + config: Config, paths: SourcePaths, monkeypatch: pytest.MonkeyPatch +) -> None: + stored = _seed_archive(config, paths, [_record("key/a/usage")]) + commit_projection( + config, + stored, + _projection( + usage_rows=[_usage_row(call_id="logical-1", input_tokens=100, session_id=stored)] + ), + empty_projection_state(), + ) + assert _indexed_usage(config, stored) == [("logical-1", 100, 0)] + + import thirdeye.platforms.copilot.projection_store as store_mod + + monkeypatch.setattr(store_mod, "_invalidate_usage_index", lambda *_args, **_kwargs: None) + commit_projection( + config, + stored, + _projection( + usage_rows=[_usage_row(call_id="logical-1", input_tokens=150, session_id=stored)] + ), + empty_projection_state(), + ) + + directory = _directory(config, stored) + sidecar_rows = [ + json.loads(line) for line in usage_jsonl_path(directory).read_text().splitlines() if line + ] + assert sidecar_rows[0]["gen_ai.usage.input_tokens"] == 150 + assert _indexed_usage(config, stored) == [("logical-1", 100, 0)] + + monkeypatch.undo() + load_projection_state(config, stored) + assert _indexed_usage(config, stored) == [("logical-1", 150, 0)] From 2b5afe7c8d7767dac68bc0b9cd394f81225b3c5c Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Fri, 11 Sep 2026 16:09:16 -0700 Subject: [PATCH 59/88] Preserve partitioned usage provenance and quarantine cache-inclusive impossibilities. Watch/disjoint replay now keeps prior source IDs on logical calls, and row metrics accept integer-valued floats so SQLite REAL values still reconcile with shutdown totals. Co-authored-by: Cursor --- src/thirdeye/platforms/copilot/usage.py | 86 +++++++++++---- tests/platforms/copilot/test_usage.py | 140 ++++++++++++++++++++++++ 2 files changed, 207 insertions(+), 19 deletions(-) diff --git a/src/thirdeye/platforms/copilot/usage.py b/src/thirdeye/platforms/copilot/usage.py index c16426f..9f93657 100644 --- a/src/thirdeye/platforms/copilot/usage.py +++ b/src/thirdeye/platforms/copilot/usage.py @@ -140,7 +140,7 @@ def _metrics_digest(row: dict[str, Any]) -> str: def _row_metrics(row: dict[str, Any]) -> dict[str, int]: metrics: dict[str, int] = {} for field in _METRIC_FIELDS: - value = _integer(row.get(field)) + value = _metric_number(row.get(field)) if value is not None: metrics[field] = value return metrics @@ -149,13 +149,19 @@ def _row_metrics(row: dict[str, Any]) -> dict[str, int]: def _row_is_incompatible(row: dict[str, Any]) -> bool: """Reject revisions that cannot describe a single completed request.""" - input_tokens = _integer(row.get("input_tokens")) - cache_read = _integer(row.get("cache_read_tokens")) - cache_write = _integer(row.get("cache_write_tokens")) + input_tokens = _metric_number(row.get("input_tokens")) + cache_read = _metric_number(row.get("cache_read_tokens")) + cache_write = _metric_number(row.get("cache_write_tokens")) if input_tokens is None: return False - return (cache_read is not None and cache_read > input_tokens) or ( + if (cache_read is not None and cache_read > input_tokens) or ( cache_write is not None and cache_write > input_tokens + ): + return True + return ( + cache_read is not None + and cache_write is not None + and cache_read + cache_write > input_tokens ) @@ -181,7 +187,10 @@ def _candidate( } # Fully specified token usage belongs in UsageRow. Retain token values in # the candidate only when a partial row cannot produce a UsageRow. - if _integer(row.get("input_tokens")) is None or _integer(row.get("output_tokens")) is None: + if ( + _metric_number(row.get("input_tokens")) is None + or _metric_number(row.get("output_tokens")) is None + ): supplemental.update( { field: row[field] @@ -221,8 +230,8 @@ def _missing_fields(candidate: AccountingCandidate, row: dict[str, Any]) -> list required = { "timestamp": candidate["timestamp"], "model": candidate["model"], - "input_tokens": _integer(row.get("input_tokens")), - "output_tokens": _integer(row.get("output_tokens")), + "input_tokens": _metric_number(row.get("input_tokens")), + "output_tokens": _metric_number(row.get("output_tokens")), } return [name for name, value in required.items() if value is None] @@ -233,8 +242,8 @@ def _usage_row( missing = _missing_fields(candidate, row) if missing: return None - input_tokens = _integer(row["input_tokens"]) - output_tokens = _integer(row["output_tokens"]) + input_tokens = _metric_number(row["input_tokens"]) + output_tokens = _metric_number(row["output_tokens"]) assert input_tokens is not None and output_tokens is not None return UsageRow( session_id=session_id, @@ -246,9 +255,9 @@ def _usage_row( response_model=candidate["model"], # guarded by _missing_fields input_tokens=input_tokens, output_tokens=output_tokens, - cache_read_input_tokens=_integer(row.get("cache_read_tokens")), - cache_creation_input_tokens=_integer(row.get("cache_write_tokens")), - reasoning_output_tokens=_integer(row.get("reasoning_tokens")), + cache_read_input_tokens=_metric_number(row.get("cache_read_tokens")), + cache_creation_input_tokens=_metric_number(row.get("cache_write_tokens")), + reasoning_output_tokens=_metric_number(row.get("reasoning_tokens")), ) @@ -285,7 +294,7 @@ def _metrics_from_call(call: dict[str, Any]) -> dict[str, int]: return {} parsed: dict[str, int] = {} for field in _METRIC_FIELDS: - value = _integer(metrics.get(field)) + value = _metric_number(metrics.get(field)) if value is not None: parsed[field] = value return parsed @@ -391,10 +400,10 @@ def _hydrate_accounted( accounted_by_agent: dict[str, dict[str, int]] = {} prior_accounted = source.get("accounted_metrics") if isinstance(prior_accounted, dict) and any( - _integer(prior_accounted.get(field)) is not None for field in _METRIC_FIELDS + _metric_number(prior_accounted.get(field)) is not None for field in _METRIC_FIELDS ): for field in _METRIC_FIELDS: - value = _integer(prior_accounted.get(field)) + value = _metric_number(prior_accounted.get(field)) if value is not None: accounted[field] = value prior_agents = source.get("accounted_by_agent") @@ -417,12 +426,36 @@ def _hydrate_accounted( return accounted, accounted_by_agent +def _prior_source_ids(call: dict[str, Any]) -> list[str]: + prior_ids = call.get("source_ids") + if isinstance(prior_ids, list): + seeded = [item for item in prior_ids if isinstance(item, str)] + if seeded: + return seeded + usage_source_id = call.get("usage_source_id") + return [usage_source_id] if isinstance(usage_source_id, str) else [] + + +def _seed_revision_sources( + revision_sources: dict[str, list[str]], + logical_id: str, + inherited_calls: dict[str, Any], +) -> None: + if logical_id in revision_sources: + return + previous = inherited_calls.get(logical_id) + revision_sources[logical_id] = ( + _prior_source_ids(previous) if isinstance(previous, dict) else [] + ) + + def _logical_call_entry( logical_id: str, revision: DatabaseRevision, record: SourceRecord, row: dict[str, Any], candidate: AccountingCandidate, + source_ids: list[str], *, quarantined: bool, ) -> dict[str, Any]: @@ -432,6 +465,7 @@ def _logical_call_entry( "content_revision": revision["content_revision"], "metrics_digest": _metrics_digest(row), "usage_source_id": record["source_id"], + "source_ids": list(source_ids), "table": revision["table"], "primary_key": revision["primary_key"], "agent_id": candidate["agent_id"], @@ -516,7 +550,9 @@ def build_accounting( if logical_id in selected: previous_digest = _metrics_digest(selected[logical_id][0]["payload"]["row"]) selected[logical_id] = (record, revision, candidate, previous_digest) - revision_sources.setdefault(logical_id, []).append(record["source_id"]) + _seed_revision_sources(revision_sources, logical_id, inherited_calls) + if record["source_id"] not in revision_sources[logical_id]: + revision_sources[logical_id].append(record["source_id"]) prior_digests[logical_id] = _metrics_digest(payload["row"]) for key in sorted(row_generations): @@ -576,7 +612,13 @@ def build_accounting( ) candidates.append(candidate) logical_calls[logical_id] = _logical_call_entry( - logical_id, revision, record, row, candidate, quarantined=True + logical_id, + revision, + record, + row, + candidate, + revision_sources[logical_id], + quarantined=True, ) continue candidates.append(candidate) @@ -609,7 +651,13 @@ def build_accounting( accounted_by_agent.setdefault(agent_key, _zero_metrics()) _add_metrics(accounted_by_agent[agent_key], metrics) logical_calls[logical_id] = _logical_call_entry( - logical_id, revision, record, row, candidate, quarantined=False + logical_id, + revision, + record, + row, + candidate, + revision_sources[logical_id], + quarantined=False, ) if unknown_provider_ids: diff --git a/tests/platforms/copilot/test_usage.py b/tests/platforms/copilot/test_usage.py index 609000f..1a11441 100644 --- a/tests/platforms/copilot/test_usage.py +++ b/tests/platforms/copilot/test_usage.py @@ -406,6 +406,35 @@ def test_later_revision_replaces_earlier_for_same_logical_call(): assert stored["metrics_digest"] != "" +def test_later_revision_in_new_partition_keeps_prior_source_ids(): + row = copy.deepcopy(_load_json(FIXTURES / "assistant-usage-events.json")[0]) + first = _usage_record(row, content_revision="sha256:first-revision") + updated = copy.deepcopy(row) + updated["output_tokens"] = 999 + second = _usage_record(updated, content_revision="sha256:second-revision") + _, state = build_accounting([first], {}) + projection, next_state = build_accounting([second], state) + + assert projection["candidates"][0]["source_ids"] == [first["source_id"], second["source_id"]] + stored = next_state["logical_calls"][projection["candidates"][0]["logical_call_id"]] + assert stored["source_ids"] == [first["source_id"], second["source_id"]] + assert projection["usage_rows"][0].output_tokens == 999 + + +def test_prior_usage_source_id_seeds_when_source_ids_absent(): + row = copy.deepcopy(_load_json(FIXTURES / "assistant-usage-events.json")[0]) + first = _usage_record(row, content_revision="sha256:first-revision") + _, state = build_accounting([first], {}) + logical_id = next(iter(state["logical_calls"])) + del state["logical_calls"][logical_id]["source_ids"] + updated = copy.deepcopy(row) + updated["output_tokens"] = 999 + second = _usage_record(updated, content_revision="sha256:second-revision") + projection, _next_state = build_accounting([second], state) + + assert projection["candidates"][0]["source_ids"] == [first["source_id"], second["source_id"]] + + def test_reused_row_id_across_generations_emits_warning(): row_a = copy.deepcopy(_load_json(FIXTURES / "assistant-usage-events.json")[0]) row_b = copy.deepcopy(row_a) @@ -452,6 +481,21 @@ def test_incompatible_metrics_quarantine_logical_call(): assert state["logical_calls"] +def test_cache_read_plus_write_exceeding_input_quarantines(): + row = copy.deepcopy(_load_json(FIXTURES / "assistant-usage-events.json")[0]) + row["input_tokens"] = 100 + row["cache_read_tokens"] = 80 + row["cache_write_tokens"] = 80 + record = _usage_record(row, content_revision="sha256:cache-sum-rev") + projection, state = build_accounting([record], {}) + + assert projection["usage_rows"] == [] + assert len(projection["candidates"]) == 1 + assert "usage_revision_conflict" in _diagnostic_codes(projection) + logical_id = projection["candidates"][0]["logical_call_id"] + assert state["logical_calls"][logical_id]["quarantined"] is True + + # --- checkpoint / shutdown --- @@ -522,6 +566,71 @@ def test_unusable_shutdown_emits_capability_gap_instead_of_silent_skip(): assert "shutdown_total_mismatch" not in _diagnostic_codes(projection) +def test_shutdown_data_not_a_dict_emits_capability_gap(): + shutdown = _shutdown_record() + shutdown["payload"]["data"] = "not-a-dict" + projection, _state = build_accounting(_six_call_records() + [shutdown], {}) + + gaps = [item for item in projection["diagnostics"] if item["code"] == "capability_gap"] + assert any(item["details"].get("reason") == "missing_shutdown_data" for item in gaps) + assert shutdown["source_id"] in gaps[0]["source_ids"] + assert "shutdown_total_mismatch" not in _diagnostic_codes(projection) + + +def test_unparseable_agent_metrics_emits_capability_gap(): + usage = _load_json(FIXTURES / "usage.json") + usage["agentMetrics"] = {"main": "not-a-dict"} + shutdown = _shutdown_record() + shutdown["payload"]["data"] = usage + projection, _state = build_accounting(_six_call_records() + [shutdown], {}) + + gaps = [item for item in projection["diagnostics"] if item["code"] == "capability_gap"] + assert any( + item["details"].get("reason") == "unparseable_agent_metrics" + and shutdown["source_id"] in item["source_ids"] + for item in gaps + ) + + +def test_mixed_archive_does_not_charge_sessions_turns_or_title_calls(): + title = _load_json(RECONCILIATION / "cases.json")["auxiliary_title_generation"][ + "input_records" + ][0] + sessions_record: SourceRecord = { + "source_id": ( + f"copilot-db:{SOURCE_KEY}:{NATIVE_SESSION_ID}:sessions:1:sha256:sessions-rev" + ), + "source_kind": "database", + "native_session_id": NATIVE_SESSION_ID, + "ts": None, + "observed_at": OBSERVED_AT, + "payload": { + "schema_version": 1, + "table": "sessions", + "row": {"id": NATIVE_SESSION_ID}, + }, + "locator": { + "database": "/example/.copilot/session-store.db", + "table": "sessions", + "primary_key": NATIVE_SESSION_ID, + "content_revision": "sha256:sessions-rev", + "generation": GENERATION, + }, + } + records = _six_call_records() + [sessions_record, title] + projection, _state = build_accounting(records, {}) + totals = _metric_totals(projection["usage_rows"]) + + assert len(projection["usage_rows"]) == 6 + assert len(projection["candidates"]) == 6 + assert totals["input_tokens"] == SHUTDOWN_TOTALS["input_tokens"] + assert totals["output_tokens"] == SHUTDOWN_TOTALS["output_tokens"] + assert totals["cache_read_tokens"] == SHUTDOWN_TOTALS["cache_read_tokens"] + assert totals["cache_write_tokens"] == SHUTDOWN_TOTALS["cache_write_tokens"] + assert totals["reasoning_tokens"] == SHUTDOWN_TOTALS["reasoning_tokens"] + assert _nano_aiu_total(projection["candidates"]) == SHUTDOWN_TOTALS["total_nano_aiu"] + + # --- supplemental metrics --- @@ -650,6 +759,9 @@ def test_later_incompatible_revision_conflicts_using_prior_metrics_digest(): assert conflict["details"]["metrics_digest"] != prior_digest assert projection["usage_rows"] == [] assert len(projection["candidates"]) == 1 + assert projection["candidates"][0]["source_ids"] == [first["source_id"], second["source_id"]] + assert first["source_id"] in conflict["source_ids"] + assert second["source_id"] in conflict["source_ids"] def test_disjoint_partition_does_not_false_mismatch_shutdown(): @@ -684,3 +796,31 @@ def test_row_id_reuse_is_detected_across_partitions(): assert second["source_id"] in reuse["source_ids"] assert len(projection["usage_rows"]) == 1 assert projection["usage_rows"][0].output_tokens == 50 + + +def test_integer_valued_float_row_metrics_do_not_false_mismatch_shutdown(): + records = _six_call_records() + for record in records: + row = record["payload"]["row"] + assert isinstance(row, dict) + for field in ( + "input_tokens", + "output_tokens", + "cache_read_tokens", + "cache_write_tokens", + "reasoning_tokens", + "total_nano_aiu", + ): + if isinstance(row.get(field), int): + row[field] = float(row[field]) + projection, state = build_accounting(records + [_shutdown_record()], {}) + totals = _metric_totals(projection["usage_rows"]) + + assert len(projection["usage_rows"]) == 6 + assert totals["input_tokens"] == SHUTDOWN_TOTALS["input_tokens"] + assert totals["output_tokens"] == SHUTDOWN_TOTALS["output_tokens"] + assert totals["cache_read_tokens"] == SHUTDOWN_TOTALS["cache_read_tokens"] + assert totals["cache_write_tokens"] == SHUTDOWN_TOTALS["cache_write_tokens"] + assert totals["reasoning_tokens"] == SHUTDOWN_TOTALS["reasoning_tokens"] + assert state["accounted_metrics"]["total_nano_aiu"] == SHUTDOWN_TOTALS["total_nano_aiu"] + assert "shutdown_total_mismatch" not in _diagnostic_codes(projection) From 9c5f0a17fe387f7282edcd286d9cb7e5e3a493a9 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Fri, 11 Sep 2026 16:10:52 -0700 Subject: [PATCH 60/88] Fix accounting export retries without dropping live platform jobs. Claim/retry, emitted-before-unlink, and turn_accounting transport now apply only to accounting jobs so delayed usage stays retryable and existing turn/spans workers keep delete-after-read behavior. Co-authored-by: Cursor --- src/thirdeye/otel_export.py | 245 +++++++++++++++---- src/thirdeye/otel_worker.py | 132 +++++++--- src/thirdeye/tracing/model.py | 8 + tests/shared/test_accounting_export.py | 324 ++++++++++++++++++++++++- 4 files changed, 624 insertions(+), 85 deletions(-) diff --git a/src/thirdeye/otel_export.py b/src/thirdeye/otel_export.py index 7435b37..bbb202d 100644 --- a/src/thirdeye/otel_export.py +++ b/src/thirdeye/otel_export.py @@ -754,6 +754,57 @@ def export_session_accounting( return False +def export_turn_accounting( + config: Config, + session_dir_: Path, + session_id: str, + platform: str, + cwd: str, + turn_id: str, + accounting: dict[str, Any], + *, + turn_span_id: str | int | None = None, + captured_env: dict[str, str] | None = None, +) -> bool: + """Queue unmatched accounting owned by an already-exported user turn.""" + if not config.logfire.enabled or not config.logfire.token: + return False + try: + accounting_id = str(accounting["accounting_id"]) + logical_span_id = f"accounting:{session_id}:{turn_id}:{accounting_id}" + payload: dict[str, Any] = { + "job_id": logical_span_id, + "kind": "turn_accounting", + "session_dir": str(session_dir_), + "session_id": session_id, + "platform": platform, + "cwd": cwd, + "captured_attributes": _resolve_captured_attributes(config, captured_env), + "turn_id": turn_id, + "accounting_id": accounting_id, + "destination": "turn-accounting-span", + "usage": dict(accounting["usage"]), + "attribution_status": str(accounting["attribution_status"]), + "agent_id": accounting.get("agent_id"), + "attributes": dict(accounting.get("attributes") or {}), + "span_id": logical_span_id, + } + if turn_span_id is not None: + payload["turn_span_id"] = str(turn_span_id) + job_path = _write_accounting_job(config.root, payload) + _spawn(job_path) + return True + except Exception as exc: + log_capture_error( + thirdeye_home=config.root, + phase="logfire_turn_accounting_export_spawn", + error=exc, + platform=platform, + session_id=session_id, + ) + return False + + def export_subagent_turn( config: Config, session_dir_: Path, @@ -947,21 +998,23 @@ def _export_subagent_turn_inner( claim_path.write_text("sent", encoding="utf-8", newline="\n") -def _export_session_accounting_inner( +def _require_export_instance(config: Config, platform: str): + instance = _get_instance(config, platform) + if instance is None: + raise RuntimeError("logfire instance unavailable for accounting export") + return instance + + +def _resolve_session_parent( *, - config: Config, + instance: Any, session_dir_: Path, session_id: str, platform: str, cwd: str, - accounting: dict[str, Any], -) -> None: - """Emit a session-owned accounting span under the durable session root.""" - instance = _get_instance(config, platform) - if instance is None: - return - usage = dict(accounting.get("usage") or {}) - fallback_ts = _accounting_timestamp(usage, "1970-01-01T00:00:00.000Z") + fallback_ts: str, +) -> tuple[Any, tuple[int, int]]: + """Return (tracer, (trace_id, root_span_id)), creating the session root if needed.""" tracer = instance.config.get_tracer_provider().get_tracer("thirdeye") root_path = otel_state_path(session_dir_) parent, root_lock = _root_or_ownership(root_path) @@ -995,18 +1048,86 @@ def _export_session_accounting_inner( finally: if root_lock is not None: fsops.unlink(root_lock, missing_ok=True) + if parent is None: + raise RuntimeError("could not resolve or create session root") + return tracer, parent + + +def _export_session_accounting_inner( + *, + config: Config, + session_dir_: Path, + session_id: str, + platform: str, + cwd: str, + accounting: dict[str, Any], +) -> None: + """Emit a session-owned accounting span under the durable session root.""" + instance = _require_export_instance(config, platform) + usage = dict(accounting.get("usage") or {}) + fallback_ts = _accounting_timestamp(usage, "1970-01-01T00:00:00.000Z") + tracer, parent = _resolve_session_parent( + instance=instance, + session_dir_=session_dir_, + session_id=session_id, + platform=platform, + cwd=cwd, + fallback_ts=fallback_ts, + ) _export_accounting_span( tracer, _parent_context(*parent), accounting, platform=platform, session_id=session_id, + cwd=cwd, fallback_ts=fallback_ts, ) if instance.force_flush(timeout_millis=_FLUSH_TIMEOUT_MS) is False: raise RuntimeError("session accounting export was not flushed") +def _export_turn_accounting_inner( + *, + config: Config, + session_dir_: Path, + session_id: str, + platform: str, + cwd: str, + turn_id: str, + accounting: dict[str, Any], + turn_span_id: str | int | None = None, +) -> None: + """Emit a turn-owned accounting span under existing turn/session context.""" + instance = _require_export_instance(config, platform) + usage = dict(accounting.get("usage") or {}) + fallback_ts = _accounting_timestamp(usage, "1970-01-01T00:00:00.000Z") + tracer, parent = _resolve_session_parent( + instance=instance, + session_dir_=session_dir_, + session_id=session_id, + platform=platform, + cwd=cwd, + fallback_ts=fallback_ts, + ) + if turn_span_id is not None and str(turn_span_id) != "": + parent_ctx = _parent_context(parent[0], int(turn_span_id)) + else: + parent_ctx = _parent_context(*parent) + _export_accounting_span( + tracer, + parent_ctx, + accounting, + platform=platform, + session_id=session_id, + cwd=cwd, + fallback_ts=fallback_ts, + turn_id=turn_id, + ) + if instance.force_flush(timeout_millis=_FLUSH_TIMEOUT_MS) is False: + raise RuntimeError("turn accounting export was not flushed") + + @lru_cache(maxsize=128) def _repo_name(cwd: str) -> str | None: """The name of the git repository `cwd` sits in, or None outside one. @@ -1167,21 +1288,8 @@ def _cost_attributes(attributes: dict[str, Any]) -> dict[str, Any]: return {} -def _chat_attributes( - call_or_attributes: dict[str, Any], - *, - session_id: str, - platform: str, - cwd: str, - turn_id: Any = None, - turn_span_id: Any = None, -) -> dict[str, Any]: - """Build flattened attributes for a chat span. - - Completed-turn export passes an LLM-call record, while live batch export - passes the already-built semantic attributes from its job. Accepting both - forms keeps the vocabulary and JSON handling in one place. - """ +def _chat_core_attributes(call_or_attributes: dict[str, Any]) -> tuple[dict[str, Any], str | None]: + """Return unflattened chat attributes and the provider name they imply.""" if all( key in call_or_attributes for key in ("input_messages", "output_messages", "provider", "usage") @@ -1211,10 +1319,27 @@ def _chat_attributes( for source, target in _USAGE_KEYS.items(): if source in usage: attributes[target] = usage[source] - else: - attributes = call_or_attributes - # Already-built attributes carry the provider under its final name. - provider = attributes.get("gen_ai.provider.name") + return attributes, provider + attributes = dict(call_or_attributes) + return attributes, attributes.get("gen_ai.provider.name") + + +def _chat_attributes( + call_or_attributes: dict[str, Any], + *, + session_id: str, + platform: str, + cwd: str, + turn_id: Any = None, + turn_span_id: Any = None, +) -> dict[str, Any]: + """Build flattened attributes for a chat span. + + Completed-turn export passes an LLM-call record, while live batch export + passes the already-built semantic attributes from its job. Accepting both + forms keeps the vocabulary and JSON handling in one place. + """ + attributes, provider = _chat_core_attributes(call_or_attributes) return _flatten_attrs( _merge_raw( attributes, @@ -1231,8 +1356,14 @@ def _chat_attributes( ) -def _accounting_attributes(accounting: dict[str, Any]) -> dict[str, Any]: - """Project an immutable usage row onto its one allowed export span.""" +def _has_copilot_native_billing(attributes: dict[str, Any]) -> bool: + return any( + key.startswith("copilot.billing") or "nano_aiu" in str(key).lower() for key in attributes + ) + + +def _accounting_attributes_raw(accounting: dict[str, Any]) -> dict[str, Any]: + """Unflattened projection of one usage row onto its export span.""" usage = dict(accounting.get("usage") or {}) attributes = _merge_raw( usage, @@ -1245,9 +1376,14 @@ def _accounting_attributes(accounting: dict[str, Any]) -> dict[str, Any]: }, ) # Native Copilot billing units are not an estimated USD model price. - if any(key.startswith("copilot.billing") or "nano_aiu" in key.lower() for key in attributes): + if _has_copilot_native_billing(attributes): attributes["thirdeye.accounting.billing.kind"] = "copilot-native-unit" - return _flatten_attrs(attributes) + return attributes + + +def _accounting_attributes(accounting: dict[str, Any]) -> dict[str, Any]: + """Project an immutable usage row onto its one allowed export span.""" + return _flatten_attrs(_accounting_attributes_raw(accounting)) def _accounting_span_id( @@ -1276,6 +1412,7 @@ def _export_accounting_span( *, platform: str, session_id: str, + cwd: str, fallback_ts: str, turn_id: str | None = None, ) -> None: @@ -1287,7 +1424,17 @@ def _export_accounting_span( _accounting_span_id(platform, session_id, str(accounting["accounting_id"]), turn_id), parent_ctx=parent_ctx, start_time=_ts_to_ns(ts), - attributes=_accounting_attributes(accounting), + attributes=_flatten_attrs( + _merge_raw( + _identity_attributes( + session_id=session_id, + platform=platform, + cwd=cwd, + turn_id=turn_id, + ), + _accounting_attributes_raw(accounting), + ) + ), ) span.end(end_time=_ts_to_ns(ts)) @@ -1549,26 +1696,35 @@ def _export_turn_subtree( for llm_call in turn["llm_calls"]: model = llm_call.get("model") or "" - call_attrs = _chat_attributes( - llm_call, - session_id=session_id, - platform=platform, - cwd=cwd, - turn_id=turn["turn_id"], - turn_span_id=turn.get("turn_span_id"), + core_attrs, provider = _chat_core_attributes(llm_call) + raw_call_attrs = _merge_raw( + core_attrs, + _identity_attributes( + session_id=session_id, + platform=platform, + cwd=cwd, + turn_id=turn["turn_id"], + turn_span_id=turn.get("turn_span_id"), + provider=provider, + ), ) accounting = accounting_by_call.get(str(llm_call["call_id"])) if accounting is not None: # Actual accounting, rather than the semantic LLM record, owns - # the token fields for this chat span. - call_attrs = _flatten_attrs(_merge_raw(call_attrs, _accounting_attributes(accounting))) + # the token fields for this chat span. Merge raw then flatten once + # so one logfire.json_schema covers chat messages and usage JSON. + raw_call_attrs = _merge_raw(raw_call_attrs, _accounting_attributes_raw(accounting)) + if _has_copilot_native_billing(raw_call_attrs): + raw_call_attrs["thirdeye.accounting.billing.kind"] = "copilot-native-unit" + else: + raw_call_attrs = _merge_raw(raw_call_attrs, _cost_attributes(raw_call_attrs)) call_span = _start_span_with_id( tracer, f"chat {model}" if model else "chat", chat_span_id(platform, session_id, llm_call["call_id"]), parent_ctx=turn_parent_ctx, start_time=_ts_to_ns(llm_call["start_ts"]), - attributes=call_attrs, + attributes=_flatten_attrs(raw_call_attrs), ) call_span.end(end_time=_ts_to_ns(llm_call["end_ts"])) call_ctx = call_span.get_span_context() @@ -1608,6 +1764,7 @@ def _export_turn_subtree( accounting, platform=platform, session_id=session_id, + cwd=cwd, fallback_ts=turn["end_ts"], turn_id=turn["turn_id"], ) diff --git a/src/thirdeye/otel_worker.py b/src/thirdeye/otel_worker.py index 7503584..2323dac 100644 --- a/src/thirdeye/otel_worker.py +++ b/src/thirdeye/otel_worker.py @@ -28,6 +28,8 @@ from thirdeye._compat import fsops _JOB_CLAIM_STALE_S = 30.0 +_JOB_MAX_ATTEMPTS = 5 +_ACCOUNTING_KINDS = frozenset({"session_accounting", "turn_accounting"}) def _write_job_state(job_path: Path, payload: dict[str, Any]) -> None: @@ -58,10 +60,14 @@ def _claim_job(job_path: Path, payload: dict[str, Any]) -> dict[str, Any] | None between can retry a deterministic span, reducing but not eliminating remote duplicates. """ - if payload.get("state") == "emitted": + state = payload.get("state") + if state == "emitted": fsops.unlink(job_path, missing_ok=True) _release_job_claim(job_path) return None + if state == "failed": + _release_job_claim(job_path) + return None claim_path = _job_claim_path(job_path) if not _create_job_claim(claim_path): try: @@ -85,12 +91,90 @@ def _release_job_claim(job_path: Path) -> None: def _retry_job(job_path: Path, payload: dict[str, Any]) -> None: + next_attempt = int(payload.get("attempt", 0)) + 1 retry = dict(payload) - retry["state"] = "queued" - retry["attempt"] = int(payload.get("attempt", 0)) + 1 + retry["attempt"] = next_attempt + retry["state"] = "failed" if next_attempt >= _JOB_MAX_ATTEMPTS else "queued" _write_job_state(job_path, retry) +def _mark_emitted(job_path: Path, payload: dict[str, Any]) -> None: + emitted = dict(payload) + emitted["state"] = "emitted" + _write_job_state(job_path, emitted) + + +def _accounting_from_payload(payload: dict[str, Any]) -> dict[str, Any]: + return { + "accounting_id": payload["accounting_id"], + "usage": payload["usage"], + "attribution_status": payload["attribution_status"], + "agent_id": payload.get("agent_id"), + "attributes": payload.get("attributes") or {}, + "call_id": payload.get("call_id"), + } + + +def _run_accounting_job(job_path: Path, payload: dict[str, Any]) -> None: + """Claim, export, and ack one accounting job. Retryable on failure.""" + try: + claimed = _claim_job(job_path, payload) + except Exception as exc: + _log_worker_failure(kind="job_claim", payload=payload, error=exc) + return + if claimed is None: + return + + from thirdeye.otel_export import _captured_attributes + + token = _captured_attributes.set(claimed.get("captured_attributes") or {}) + kind = claimed.get("kind") + delivered = False + try: + from thirdeye.config import Config + from thirdeye.otel_export import ( + _export_session_accounting_inner, + _export_turn_accounting_inner, + ) + + config = Config.load() + if kind == "session_accounting": + _export_session_accounting_inner( + config=config, + session_dir_=Path(claimed["session_dir"]), + session_id=claimed["session_id"], + platform=claimed["platform"], + cwd=claimed["cwd"], + accounting=_accounting_from_payload(claimed), + ) + elif kind == "turn_accounting": + _export_turn_accounting_inner( + config=config, + session_dir_=Path(claimed["session_dir"]), + session_id=claimed["session_id"], + platform=claimed["platform"], + cwd=claimed["cwd"], + turn_id=str(claimed["turn_id"]), + accounting=_accounting_from_payload(claimed), + turn_span_id=claimed.get("turn_span_id"), + ) + else: + raise RuntimeError(f"unhandled accounting job kind {kind!r}") + _mark_emitted(job_path, claimed) + delivered = True + except Exception as exc: + try: + _retry_job(job_path, claimed) + except Exception: + pass + _log_worker_failure(kind=str(kind or ""), payload=claimed, error=exc) + finally: + _captured_attributes.reset(token) + _release_job_claim(job_path) + if delivered: + fsops.unlink(job_path, missing_ok=True) + + def main(argv: list[str] | None = None) -> None: argv = sys.argv[1:] if argv is None else argv if not argv: @@ -102,19 +186,20 @@ def main(argv: list[str] | None = None) -> None: _log_worker_failure(kind="job_read", payload={}, error=exc) fsops.unlink(job_path, missing_ok=True) return - try: - payload = _claim_job(job_path, payload) - except Exception as exc: - _log_worker_failure(kind="job_claim", payload=payload, error=exc) - return - if payload is None: + + kind = payload.get("kind") + if kind in _ACCOUNTING_KINDS: + _run_accounting_job(job_path, payload) return + # Existing platforms delete the ULID job as soon as it is readable. There + # is no scanner to respawn retained turn/spans/subagent jobs, so a crash + # or export failure must not leave poison pills on disk. + fsops.unlink(job_path, missing_ok=True) + from thirdeye.otel_export import _captured_attributes token = _captured_attributes.set(payload.get("captured_attributes") or {}) - kind = payload.get("kind") - delivered = False try: from thirdeye.config import Config @@ -155,35 +240,10 @@ def main(argv: list[str] | None = None) -> None: parent_span_id=payload["parent_span_id"], turn=payload["turn"], ) - elif kind == "session_accounting": - from thirdeye.otel_export import _export_session_accounting_inner - - _export_session_accounting_inner( - config=config, - session_dir_=Path(payload["session_dir"]), - session_id=payload["session_id"], - platform=payload["platform"], - cwd=payload["cwd"], - accounting={ - "accounting_id": payload["accounting_id"], - "usage": payload["usage"], - "attribution_status": payload["attribution_status"], - "agent_id": payload.get("agent_id"), - "attributes": payload.get("attributes") or {}, - }, - ) - delivered = True except Exception as exc: - try: - _retry_job(job_path, payload) - except Exception: - pass _log_worker_failure(kind=str(kind or ""), payload=payload, error=exc) finally: _captured_attributes.reset(token) - _release_job_claim(job_path) - if delivered: - fsops.unlink(job_path, missing_ok=True) def _log_worker_failure(*, kind: str, payload: dict[str, Any], error: Exception) -> None: diff --git a/src/thirdeye/tracing/model.py b/src/thirdeye/tracing/model.py index a2f2c9e..3a8bddd 100644 --- a/src/thirdeye/tracing/model.py +++ b/src/thirdeye/tracing/model.py @@ -157,6 +157,14 @@ class TurnAccountingJobDict(TypedDict): attribution_status: str agent_id: str | None span_id: str + attributes: NotRequired[dict[str, Any]] + # Worker envelope fields remain optional so the serializable public job + # shape above is usable by placement ledgers without filesystem context. + session_dir: NotRequired[str] + platform: NotRequired[str] + cwd: NotRequired[str] + captured_attributes: NotRequired[dict[str, Any]] + turn_span_id: NotRequired[str] class AccountingLedgerEntryDict(TypedDict): diff --git a/tests/shared/test_accounting_export.py b/tests/shared/test_accounting_export.py index f6ed541..2482dd7 100644 --- a/tests/shared/test_accounting_export.py +++ b/tests/shared/test_accounting_export.py @@ -137,6 +137,20 @@ def _error_log_entries(home: Path) -> list[dict]: return [json.loads(line) for line in log.read_text().splitlines() if line] +def _accounting_job_path(root: Path, job_id: str) -> Path: + digest = hashlib.sha256(job_id.encode("utf-8")).hexdigest() + return otel_jobs_dir(root) / f"accounting-{digest}.json" + + +def _write_queued_accounting_job(root: Path, **fields: Any) -> Path: + job_id = str(fields["job_id"]) + job_path = _accounting_job_path(root, job_id) + job_path.parent.mkdir(parents=True, exist_ok=True) + payload = {"state": "queued", "attempt": 0, **fields} + job_path.write_text(json.dumps(payload), encoding="utf-8") + return job_path + + class TestAccountingAttributes: def test_projects_usage_and_metadata(self): accounting = _accounting_call( @@ -163,6 +177,13 @@ def test_labels_copilot_native_billing_distinct_from_usd(self): assert attrs["thirdeye.accounting.billing.kind"] == "copilot-native-unit" assert attrs["copilot.billing.nano_aiu"] == 12 + def test_usd_only_cost_does_not_set_native_billing_kind(self): + accounting = _accounting_call(attributes={"operation.cost": 0.05}) + attrs = otel_export._accounting_attributes(accounting) + + assert attrs["operation.cost"] == 0.05 + assert "thirdeye.accounting.billing.kind" not in attrs + class TestOrdinaryTurnCompatibility: def test_turn_without_accounting_calls_is_unchanged( @@ -226,6 +247,75 @@ def test_matched_accounting_merges_onto_chat_span_only( assert chat_span["context"]["span_id"] == chat_span_id(platform, session_id, call_id) assert chat_span["attributes"]["gen_ai.usage.input_tokens"] == 6452 assert chat_span["attributes"]["thirdeye.accounting.id"] == "acct-1" + schema = json.loads(chat_span["attributes"]["logfire.json_schema"]) + assert "gen_ai.input.messages" in schema["properties"] + assert "gen_ai.output.messages" in schema["properties"] + + def test_matched_copilot_billing_labels_chat_span( + self, tmp_path: Path, enabled_config: Config, wired_instance, exporter + ): + session_id = "copilot-session" + platform = "copilot" + call_id = "copilot:call:billed" + call = _llm_call(call_id=call_id, usage={}) + accounting = _accounting_call( + call_id=call_id, + usage=_usage_row(call_id="usage-billed", input_tokens=6452, output_tokens=107), + attributes={"copilot.billing.nano_aiu": 12}, + ) + turn = _turn(llm_calls=[call], accounting_calls=[accounting]) + + otel_export._export_turn_inner( + config=enabled_config, + session_dir_=tmp_path / "traces" / platform / session_id, + session_id=session_id, + platform=platform, + cwd="/proj", + turn=turn, + ) + + chat_span = next( + span for span in exporter.exported_spans_as_dict() if span["name"].startswith("chat") + ) + assert chat_span["attributes"]["thirdeye.accounting.billing.kind"] == "copilot-native-unit" + assert chat_span["attributes"]["copilot.billing.nano_aiu"] == 12 + assert chat_span["attributes"]["gen_ai.usage.input_tokens"] == 6452 + + def test_unknown_call_id_emits_turn_owned_span_not_chat( + self, tmp_path: Path, enabled_config: Config, wired_instance, exporter + ): + session_id = "copilot-session" + platform = "copilot" + turn_id = "turn-unknown-call" + call = _llm_call(call_id="known-call") + accounting = _accounting_call( + accounting_id="acct-unknown-call", + call_id="not-in-llm-calls", + attribution_status="pending", + usage=_usage_row(call_id="usage-unknown", input_tokens=10, output_tokens=2), + attributes={"accounting.destination": "turn-accounting-span"}, + ) + turn = _turn(turn_id=turn_id, llm_calls=[call], accounting_calls=[accounting]) + + otel_export._export_turn_inner( + config=enabled_config, + session_dir_=tmp_path / "traces" / platform / session_id, + session_id=session_id, + platform=platform, + cwd="/proj", + turn=turn, + ) + + spans = exporter.exported_spans_as_dict() + chat_spans = [span for span in spans if span["name"].startswith("chat")] + accounting_spans = [span for span in spans if span["name"] == "accounting"] + turn_span = next(span for span in spans if span["name"] == "invoke_agent") + + assert len(chat_spans) == 1 + assert chat_spans[0]["attributes"].get("thirdeye.accounting.id") is None + assert len(accounting_spans) == 1 + assert accounting_spans[0]["parent"]["span_id"] == turn_span["context"]["span_id"] + assert accounting_spans[0]["attributes"]["thirdeye.accounting.id"] == "acct-unknown-call" def test_unmatched_accounting_emits_turn_owned_span_with_deterministic_id( self, tmp_path: Path, enabled_config: Config, wired_instance, exporter @@ -269,11 +359,6 @@ def test_fixture_turn_with_matched_and_unmatched_accounting( self, tmp_path: Path, enabled_config: Config, wired_instance, exporter ): turn = dict(_TRANSPORT["turn_span_with_accounting_calls"]) - session_id = ( - turn["llm_calls"][0]["usage"]["session_id"] - if turn["llm_calls"][0].get("usage") - else "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" - ) session_id = turn["accounting_calls"][0]["usage"]["session_id"] platform = "copilot" @@ -397,11 +482,102 @@ def test_worker_round_trips_session_accounting_job( assert accounting_span["parent"]["span_id"] == session_span["context"]["span_id"] assert accounting_span["attributes"]["thirdeye.accounting.id"] == accounting_id + assert accounting_span["attributes"]["thirdeye.platform"] == platform + assert accounting_span["attributes"]["thirdeye.cwd"] == "/proj" assert accounting_span["context"]["span_id"] == otel_export._accounting_span_id( platform, session_id, accounting_id ) +class TestTurnAccountingExport: + def test_queues_deterministic_job_without_duplicates( + self, enabled_config: Config, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ): + spawned: list[Path] = [] + monkeypatch.setattr(otel_export, "_spawn", spawned.append) + session_dir = tmp_path / "traces" / "copilot" / "s1" + accounting = { + "accounting_id": "acct-turn", + "usage": _usage_row(session_id="s1"), + "attribution_status": "pending", + "agent_id": None, + "attributes": {"accounting.destination": "turn-accounting-span"}, + } + + first = otel_export.export_turn_accounting( + enabled_config, session_dir, "s1", "copilot", "/proj", "turn_1", accounting + ) + second = otel_export.export_turn_accounting( + enabled_config, session_dir, "s1", "copilot", "/proj", "turn_1", accounting + ) + + assert first is True + assert second is True + jobs = list(otel_jobs_dir(enabled_config.root).glob("accounting-*.json")) + assert len(jobs) == 1 + payload = json.loads(jobs[0].read_text(encoding="utf-8")) + assert payload["kind"] == "turn_accounting" + assert payload["destination"] == "turn-accounting-span" + assert payload["state"] == "queued" + assert payload["job_id"] == "accounting:s1:turn_1:acct-turn" + assert payload["span_id"] == payload["job_id"] + assert payload["turn_id"] == "turn_1" + assert len(spawned) == 2 + + def test_worker_round_trips_turn_accounting_job( + self, + enabled_config: Config, + wired_instance, + exporter, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ): + monkeypatch.setattr(Config, "load", lambda: enabled_config) + session_id = "s1" + platform = "copilot" + turn_id = "turn_1" + session_dir = tmp_path / "traces" / platform / session_id + accounting_id = "acct-delayed" + job_id = f"accounting:{session_id}:{turn_id}:{accounting_id}" + turn_span_id = 4242 + job_path = _write_queued_accounting_job( + enabled_config.root, + job_id=job_id, + kind="turn_accounting", + session_dir=str(session_dir), + session_id=session_id, + platform=platform, + cwd="/proj", + turn_id=turn_id, + turn_span_id=str(turn_span_id), + accounting_id=accounting_id, + destination="turn-accounting-span", + usage=_usage_row(session_id=session_id), + attribution_status="pending", + agent_id=None, + attributes={"accounting.destination": "turn-accounting-span"}, + span_id=job_id, + ) + + otel_worker.main([str(job_path)]) + + assert not job_path.exists() + spans = exporter.exported_spans_as_dict() + accounting_span = next(span for span in spans if span["name"] == "accounting") + invented_turns = [span for span in spans if span["name"] == "invoke_agent"] + invented_chats = [span for span in spans if span["name"].startswith("chat")] + + assert invented_turns == [] + assert invented_chats == [] + assert accounting_span["parent"]["span_id"] == turn_span_id + assert accounting_span["attributes"]["thirdeye.accounting.id"] == accounting_id + assert accounting_span["attributes"]["thirdeye.turn.id"] == turn_id + assert accounting_span["attributes"]["thirdeye.platform"] == platform + assert accounting_span["context"]["span_id"] == otel_export._accounting_span_id( + platform, session_id, accounting_id, turn_id + ) + + class TestWorkerClaimRecovery: def test_stale_claim_is_recovered_and_export_retried( self, @@ -513,3 +689,141 @@ def test_fresh_claim_blocks_concurrent_worker( claimed = otel_worker._claim_job(job_path, payload) assert claimed is None + + def test_success_writes_emitted_before_unlinking( + self, + enabled_config: Config, + wired_instance, + exporter, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ): + monkeypatch.setattr(Config, "load", lambda: enabled_config) + states: list[str] = [] + original = otel_worker._write_job_state + + def _capture(job_path: Path, payload: dict[str, Any]) -> None: + states.append(str(payload.get("state"))) + original(job_path, payload) + + monkeypatch.setattr(otel_worker, "_write_job_state", _capture) + job_path = _write_queued_accounting_job( + enabled_config.root, + job_id="accounting:s1:acct-emitted", + kind="session_accounting", + session_dir=str(tmp_path / "traces" / "copilot" / "s1"), + session_id="s1", + platform="copilot", + cwd="/proj", + accounting_id="acct-emitted", + destination="session-accounting-span", + usage=_usage_row(session_id="s1"), + attribution_status="pending", + span_id="accounting:s1:acct-emitted", + ) + + otel_worker.main([str(job_path)]) + + assert "emitted" in states + assert states[-1] == "emitted" + assert not job_path.exists() + assert any(span["name"] == "accounting" for span in exporter.exported_spans_as_dict()) + + def test_missing_logfire_instance_retains_accounting_job( + self, enabled_config: Config, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ): + monkeypatch.setattr(Config, "load", lambda: enabled_config) + monkeypatch.setattr(otel_export, "_get_instance", lambda config, platform: None) + job_path = _write_queued_accounting_job( + enabled_config.root, + job_id="accounting:s1:acct-noop", + kind="session_accounting", + session_dir=str(tmp_path / "traces" / "copilot" / "s1"), + session_id="s1", + platform="copilot", + cwd="/proj", + accounting_id="acct-noop", + destination="session-accounting-span", + usage=_usage_row(session_id="s1"), + attribution_status="pending", + span_id="accounting:s1:acct-noop", + ) + + otel_worker.main([str(job_path)]) + + assert job_path.exists() + payload = json.loads(job_path.read_text(encoding="utf-8")) + assert payload["state"] == "queued" + assert payload["attempt"] == 1 + assert not otel_worker._job_claim_path(job_path).exists() + + def test_worker_dispatches_turn_accounting_kind( + self, enabled_config: Config, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ): + monkeypatch.setattr(Config, "load", lambda: enabled_config) + calls: list[dict[str, Any]] = [] + monkeypatch.setattr( + otel_export, + "_export_turn_accounting_inner", + lambda **kwargs: calls.append(kwargs), + raising=False, + ) + job_path = _write_queued_accounting_job( + enabled_config.root, + job_id="accounting:s1:turn_1:acct-dispatch", + kind="turn_accounting", + session_dir=str(tmp_path / "traces" / "copilot" / "s1"), + session_id="s1", + platform="copilot", + cwd="/proj", + turn_id="turn_1", + accounting_id="acct-dispatch", + destination="turn-accounting-span", + usage=_usage_row(session_id="s1"), + attribution_status="pending", + span_id="accounting:s1:turn_1:acct-dispatch", + ) + + otel_worker.main([str(job_path)]) + + assert len(calls) == 1 + assert calls[0]["session_id"] == "s1" + assert calls[0]["turn_id"] == "turn_1" + assert calls[0]["accounting"]["accounting_id"] == "acct-dispatch" + assert not job_path.exists() + + def test_exhausted_retries_mark_job_failed( + self, enabled_config: Config, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ): + monkeypatch.setattr(Config, "load", lambda: enabled_config) + job_path = _write_queued_accounting_job( + enabled_config.root, + job_id="accounting:s1:acct-exhausted", + kind="session_accounting", + session_dir=str(tmp_path / "traces" / "copilot" / "s1"), + session_id="s1", + platform="copilot", + cwd="/proj", + accounting_id="acct-exhausted", + destination="session-accounting-span", + usage=_usage_row(session_id="s1"), + attribution_status="pending", + span_id="accounting:s1:acct-exhausted", + attempt=otel_worker._JOB_MAX_ATTEMPTS - 1, + ) + + def _boom(**kwargs): + raise RuntimeError("flush failed") + + monkeypatch.setattr(otel_export, "_export_session_accounting_inner", _boom) + otel_worker.main([str(job_path)]) + + assert job_path.exists() + payload = json.loads(job_path.read_text(encoding="utf-8")) + assert payload["state"] == "failed" + assert payload["attempt"] == otel_worker._JOB_MAX_ATTEMPTS + + otel_worker.main([str(job_path)]) + still = json.loads(job_path.read_text(encoding="utf-8")) + assert still["state"] == "failed" + assert still["attempt"] == otel_worker._JOB_MAX_ATTEMPTS From 8f4b754ca208fc09a5d49fd18b7e0fa9fb370b1c Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Fri, 11 Sep 2026 16:14:14 -0700 Subject: [PATCH 61/88] Fix Copilot reconstruction so finish, tools, hooks, and nested children stay honest. Attach turn_end to the matching model cycle, keep incomplete tools and prior nested children, and stop treating external hooks as tool executions. Co-authored-by: Cursor --- src/thirdeye/platforms/copilot/events.py | 38 +- src/thirdeye/platforms/copilot/tracing.py | 7 +- src/thirdeye/platforms/copilot/turns.py | 192 +++++--- src/thirdeye/platforms/copilot/types.py | 1 + .../fixtures/reconciliation-cases/cases.json | 199 +++++++- .../semantic-projection.json | 22 + tests/platforms/copilot/test_events.py | 65 ++- tests/platforms/copilot/test_tracing.py | 462 +++++++++++++++++- 8 files changed, 904 insertions(+), 82 deletions(-) diff --git a/src/thirdeye/platforms/copilot/events.py b/src/thirdeye/platforms/copilot/events.py index 6cd6c0e..88b6ed9 100644 --- a/src/thirdeye/platforms/copilot/events.py +++ b/src/thirdeye/platforms/copilot/events.py @@ -32,13 +32,14 @@ "session.auto_mode_resolved", "hook.start", "hook.end", + "preToolUse", + "postToolUse", + "postToolUseFailure", + "subagent.configured", } ) _ABORT_TYPES = frozenset({"assistant.abort", "session.abort", "abort"}) _HOOK_KINDS: dict[str, tuple[NormalizedEventKind, SourceReferenceRole]] = { - "preToolUse": ("tool_execution_start", "hook"), - "postToolUse": ("tool_execution_complete", "hook"), - "postToolUseFailure": ("tool_execution_failure", "hook"), "userPromptSubmitted": ("prompt_transformation", "hook"), "agentStop": ("agent_stop", "finish"), "subagentStart": ("subagent_started", "nested_child"), @@ -138,7 +139,10 @@ def _unknown(record: SourceRecord, native_type: str | None) -> list[NormalizedEv attrs: dict[str, Any] = {"raw_payload": deepcopy(record.get("payload"))} if native_type is not None: attrs["native_type"] = native_type - return [_event(record, "unknown", "hook", attributes=attrs)] + role: SourceReferenceRole = ( + "hook" if record.get("source_kind") == "hook" else "assistant_message" + ) + return [_event(record, "unknown", role, attributes=attrs)] def normalize_record(record: SourceRecord) -> list[NormalizedEvent]: @@ -307,6 +311,7 @@ def normalize_record(record: SourceRecord) -> list[NormalizedEvent]: ] if native_type == "subagent.started": + call = tool_call_id(record) return [ _event( record, @@ -314,12 +319,14 @@ def normalize_record(record: SourceRecord) -> list[NormalizedEvent]: "nested_child", attributes={ **identity, - "tool_call_id": tool_call_id(record), **deepcopy(data), + "tool_call_id": call, + "parent_tool_call_id": call, }, ) ] if native_type == "subagent.completed": + call = tool_call_id(record) return [ _event( record, @@ -327,20 +334,12 @@ def normalize_record(record: SourceRecord) -> list[NormalizedEvent]: "nested_child", attributes={ **identity, - "tool_call_id": tool_call_id(record), **deepcopy(data), + "tool_call_id": call, + "parent_tool_call_id": call, }, ) ] - if native_type == "subagent.configured": - return [ - _event( - record, - "subagent_started", - "nested_child", - attributes={**identity, "native_type": native_type, **deepcopy(data)}, - ) - ] if native_type in _SESSION_LIFECYCLE_NOTIFICATIONS: return [ @@ -355,7 +354,14 @@ def normalize_record(record: SourceRecord) -> list[NormalizedEvent]: mapped = _HOOK_KINDS.get(native_type) if mapped is not None: kind, role = mapped - return [_event(record, kind, role, attributes={**identity, **deepcopy(data)})] + return [ + _event( + record, + kind, + role, + attributes={**identity, "native_type": native_type, **deepcopy(data)}, + ) + ] if "error" in native_type.lower(): return [ diff --git a/src/thirdeye/platforms/copilot/tracing.py b/src/thirdeye/platforms/copilot/tracing.py index 12caed6..16fdb7c 100644 --- a/src/thirdeye/platforms/copilot/tracing.py +++ b/src/thirdeye/platforms/copilot/tracing.py @@ -15,9 +15,10 @@ def build_semantics( ) -> tuple[SemanticProjection, dict[str, Any]]: """Replay immutable source records into semantic events and main turns. - ``prior_state`` is not evidence. It only retains still-open interactions - whose ``source_ids`` are absent from this partition. Replaying the - complete archive with an empty prior state is always authoritative. + ``prior_state`` is not evidence. It retains still-open interactions + (including completed nested children) whose ``source_ids`` are absent + from this partition. Replaying the complete archive with an empty + prior state is always authoritative. """ events = normalize_records(records) turns, call_candidates, pending, diagnostics, semantic_state = build_turns( diff --git a/src/thirdeye/platforms/copilot/turns.py b/src/thirdeye/platforms/copilot/turns.py index 9887460..18705ec 100644 --- a/src/thirdeye/platforms/copilot/turns.py +++ b/src/thirdeye/platforms/copilot/turns.py @@ -1,9 +1,10 @@ """Explicit-identity Copilot interaction and recursive child-tree assembly. Callers that partition an archive must include every record belonging to -still-open interactions. ``prior_state`` only retains open keys whose -``source_ids`` are absent from the current partition; a complete archive -replay with ``prior_state={}`` is always the authoritative result. +still-open interactions. ``prior_state`` retains those open keys, including +completed nested child spans, when their ``source_ids`` are absent from the +current partition. A complete archive replay with ``prior_state={}`` is +always the authoritative result. """ from __future__ import annotations @@ -116,13 +117,16 @@ def build_turns( the records that belong to those interactions. """ interactions: dict[str, dict[str, Any]] = {} - active_native_turns: dict[tuple[str | None, str], str] = {} + active_native_turns: dict[tuple[str | None, str, str], str] = {} child_parent: dict[str, tuple[str, str]] = {} tool_owner: dict[str, str] = {} calls: dict[str, dict[str, Any]] = {} tool_spans: dict[str, ToolCallSpanDict] = {} pending: list[PendingItem] = [] diagnostics: list[ProjectionDiagnostic] = [] + prior = prior_state if isinstance(prior_state, dict) else {} + raw_open = prior.get("open_interactions") + prior_open = raw_open if isinstance(raw_open, dict) else {} def ensure(record: SourceRecord, interaction: str, agent: str | None) -> dict[str, Any]: key = _key(interaction, agent) @@ -153,26 +157,41 @@ def remember_parent(child: str | None, parent_call: str | None) -> None: if child and parent_call and parent_call in tool_owner: child_parent[child] = (tool_owner[parent_call], parent_call) - def close_previous_for_agent(agent: str | None, new_interaction: str, ts: str | None) -> None: - for item in interactions.values(): - if item["agent"] != agent or item["interaction"] == new_interaction: - continue - if item["complete"]: - continue - item["complete"] = True - item["end_ts"] = item["end_ts"] or ts + def bind_native_turn( + item: dict[str, Any], agent: str | None, interaction: str | None, native_turn: Any + ) -> None: + if interaction is None or native_turn is None: + return + active_native_turns[(agent, interaction, str(native_turn))] = item["key"] def complete_item(item: dict[str, Any], ts: str | None) -> None: item["complete"] = True item["end_ts"] = ts or item["end_ts"] def attach_finish( - item: dict[str, Any], record: SourceRecord, kind: str + item: dict[str, Any], record: SourceRecord, kind: str, native_turn: Any ) -> dict[str, Any] | None: - last_id = item["calls"][-1] if item["calls"] else None - last_call = calls.get(last_id) if last_id else None - if last_call is None: + turn_key = str(native_turn) if native_turn is not None else None + matches: list[dict[str, Any]] = [] + if turn_key is not None: + for call_id in item["calls"]: + candidate = calls[call_id] + if candidate["finish_evidence"]: + continue + if candidate.get("native_turn_id") == turn_key: + matches.append(candidate) + if turn_key is None or len(matches) != 1: + pending.append( + { + "id": f"pending:identity:{record['source_id']}", + "kind": "missing_identity", + "reason": f"{kind} has no uniquely matching open model cycle", + "source_ids": [record["source_id"]], + "evidence": [f"turn_id:{native_turn}"], + } + ) return None + last_call = matches[0] last_call["end_ts"] = record.get("ts") if record["source_id"] not in last_call["source_ids"]: last_call["source_ids"].append(record["source_id"]) @@ -182,28 +201,32 @@ def attach_finish( ) return last_call - def owner_for_turn_end(record: SourceRecord, native_turn: Any, agent: str | None) -> str | None: + def owner_for_native_turn( + agent: str | None, native_turn: Any, interaction: str | None + ) -> str | None: if native_turn is None: return None turn_key = str(native_turn) - bound = active_native_turns.get((agent, turn_key)) - if bound is not None: - return bound - matches: list[str] = [] - for candidate in calls.values(): - if candidate.get("native_turn_id") != turn_key: - continue - if candidate["finish_evidence"]: - continue - if candidate["agent_id"] != agent: - continue - key = candidate["interaction_key"] - if key not in matches: - matches.append(key) - if len(matches) == 1: - return matches[0] + if interaction is not None: + return active_native_turns.get((agent, interaction, turn_key)) + matches = [ + key + for (bound_agent, _bound_ix, bound_turn), key in active_native_turns.items() + if bound_agent == agent and bound_turn == turn_key + ] + unique = list(dict.fromkeys(matches)) + if len(unique) == 1: + return unique[0] return None + def drop_native_turn(agent: str | None, native_turn: Any, owner: str | None) -> None: + if native_turn is None or owner is None: + return + turn_key = str(native_turn) + for bind_key, bind_owner in list(active_native_turns.items()): + if bind_owner == owner and bind_key[0] == agent and bind_key[2] == turn_key: + active_native_turns.pop(bind_key, None) + for record in records: if record.get("source_kind") != "transcript": continue @@ -236,7 +259,6 @@ def owner_for_turn_end(record: SourceRecord, native_turn: Any, agent: str | None ) ) continue - close_previous_for_agent(agent, interaction, record.get("ts")) item = ensure(record, interaction, agent) item["input"] = str(data.get("content") or item["input"]) continue @@ -252,9 +274,7 @@ def owner_for_turn_end(record: SourceRecord, native_turn: Any, agent: str | None ) continue item = ensure(record, interaction, agent) - native_turn = data.get("turnId") - if native_turn is not None: - active_native_turns[(agent, str(native_turn))] = item["key"] + bind_native_turn(item, agent, interaction, data.get("turnId")) continue if native_type == "assistant.message": @@ -279,8 +299,9 @@ def owner_for_turn_end(record: SourceRecord, native_turn: Any, agent: str | None if isinstance(request, dict) and request.get("toolCallId") ] content = data.get("content") if isinstance(data.get("content"), str) else "" - reasoning = data.get("reasoningSummary") or data.get("intentionSummary") + reasoning = data.get("reasoningSummary") native_turn = data.get("turnId") + bind_native_turn(item, agent, interaction, native_turn) candidate = { "call_id": call_id, "stored_turn_id": item["turn_id"], @@ -411,20 +432,17 @@ def owner_for_turn_end(record: SourceRecord, native_turn: Any, agent: str | None ) if is_abort or is_error: native_turn = data.get("turnId") - owner = ( - active_native_turns.pop((agent, str(native_turn)), None) - if native_turn is not None - else None - ) + owner = owner_for_native_turn(agent, native_turn, interaction) if owner is None and interaction is not None: owner = ( _key(interaction, agent) if _key(interaction, agent) in interactions else None ) + drop_native_turn(agent, native_turn, owner) if owner and owner in interactions: item = interactions[owner] item["status"] = "interrupted" if is_abort else "errored" item["source_ids"].append(record["source_id"]) - attach_finish(item, record, "abort" if is_abort else "error") + attach_finish(item, record, "abort" if is_abort else "error", native_turn) # Abort closes the user turn. An error leaves it open so a # later model cycle in the same interaction can retry. if is_abort: @@ -433,14 +451,13 @@ def owner_for_turn_end(record: SourceRecord, native_turn: Any, agent: str | None if native_type == "assistant.turn_end": native_turn = data.get("turnId") - owner = owner_for_turn_end(record, native_turn, agent) - if native_turn is not None: - active_native_turns.pop((agent, str(native_turn)), None) + owner = owner_for_native_turn(agent, native_turn, interaction) + drop_native_turn(agent, native_turn, owner) if owner is None: pending.append( { "id": f"pending:identity:{record['source_id']}", - "kind": "incomplete_tool_pair", + "kind": "missing_identity", "reason": "assistant.turn_end has no matching open model cycle", "source_ids": [record["source_id"]], "evidence": [f"turn_id:{native_turn}"], @@ -450,7 +467,7 @@ def owner_for_turn_end(record: SourceRecord, native_turn: Any, agent: str | None item = interactions[owner] item["source_ids"].append(record["source_id"]) item["end_ts"] = record.get("ts") - last_call = attach_finish(item, record, "assistant_turn_end") + last_call = attach_finish(item, record, "assistant_turn_end", native_turn) if ( last_call is not None and not last_call["tool_call_ids"] @@ -515,11 +532,47 @@ def call_span(candidate: dict[str, Any], user_input: str) -> LlmCallSpanDict: ] for item in child_items: child_span = spans.get(item["key"]) + if child_span is None: + continue + child_span["attributes"]["parent_tool_call_id"] = parent_call parent_span = spans.get(parent_key) - if child_span is not None and parent_span is not None: - child_span["attributes"]["parent_tool_call_id"] = parent_call + if parent_span is not None: parent_span["subagents"].append(child_span) + def nested_children_for(parent_key: str) -> list[TurnSpanDict]: + found: list[TurnSpanDict] = [] + seen: set[str] = set() + for child, (pkey, _parent_call) in child_parent.items(): + if pkey != parent_key: + continue + for item in interactions.values(): + if item["agent"] != child or not item["complete"]: + continue + child_span = spans.get(item["key"]) + if child_span is None or child_span["turn_id"] in seen: + continue + found.append(deepcopy(child_span)) + seen.add(child_span["turn_id"]) + prior_item = prior_open.get(parent_key) if isinstance(prior_open, dict) else None + if isinstance(prior_item, dict): + for child in prior_item.get("nested_children") or []: + if not isinstance(child, dict): + continue + turn_id = child.get("turn_id") + if not isinstance(turn_id, str) or turn_id in seen: + continue + found.append(deepcopy(child)) + seen.add(turn_id) + return found + + for key, span in spans.items(): + existing = {child["turn_id"] for child in span.get("subagents") or []} + for child in nested_children_for(key): + if child["turn_id"] in existing: + continue + span.setdefault("subagents", []).append(child) + existing.add(child["turn_id"]) + emitted_ids = _collect_nested_turn_ids( [span for key, span in spans.items() if interactions[key]["agent"] is None] ) @@ -573,7 +626,7 @@ def call_span(candidate: dict[str, Any], user_input: str) -> LlmCallSpanDict: if has_finish else "user interaction has no assistant.turn_end" ) - open_state[item["key"]] = { + retained: dict[str, Any] = { "interaction_id": item["interaction"], "agent_id": item["agent"], "stored_turn_id": item["turn_id"], @@ -582,6 +635,10 @@ def call_span(candidate: dict[str, Any], user_input: str) -> LlmCallSpanDict: "start_ts": item["start_ts"], "pending_tool_call_ids": pending_tools, } + nested = nested_children_for(item["key"]) + if nested: + retained["nested_children"] = nested + open_state[item["key"]] = retained pending.append( { "id": f"pending:{item['turn_id']}", @@ -601,9 +658,34 @@ def call_span(candidate: dict[str, Any], user_input: str) -> LlmCallSpanDict: } ) - prior = prior_state.get("open_interactions") if isinstance(prior_state, dict) else None - if isinstance(prior, dict): - for key, prior_item in prior.items(): + seen_incomplete_tools = { + item["id"] for item in pending if item["kind"] == "incomplete_tool_pair" + } + for candidate in calls.values(): + for tool in candidate["tool_call_ids"]: + span = tool_spans.get(tool) + if span and span.get("end_ts"): + continue + pending_id = f"pending:tool:{tool}" + if pending_id in seen_incomplete_tools: + continue + source_ids = list(candidate["source_ids"][:1]) + request_source = (span or {}).get("attributes", {}).get("request_source_id") + if request_source: + source_ids = [request_source] + pending.append( + { + "id": pending_id, + "kind": "incomplete_tool_pair", + "reason": "tool request has no execution result", + "source_ids": source_ids, + "evidence": [f"tool_call_id:{tool}"], + } + ) + seen_incomplete_tools.add(pending_id) + + if isinstance(prior_open, dict): + for key, prior_item in prior_open.items(): if key in open_state or not isinstance(prior_item, dict): continue prior_sources = prior_item.get("source_ids") or [] diff --git a/src/thirdeye/platforms/copilot/types.py b/src/thirdeye/platforms/copilot/types.py index b01dd85..0a573e8 100644 --- a/src/thirdeye/platforms/copilot/types.py +++ b/src/thirdeye/platforms/copilot/types.py @@ -477,6 +477,7 @@ class OpenInteractionState(TypedDict): last_event_source_id: str | None start_ts: str | None pending_tool_call_ids: list[str] + nested_children: NotRequired[list[TurnSpanDict]] class LogicalCallState(TypedDict): diff --git a/tests/platforms/copilot/fixtures/reconciliation-cases/cases.json b/tests/platforms/copilot/fixtures/reconciliation-cases/cases.json index 9a18de6..2055b1e 100644 --- a/tests/platforms/copilot/fixtures/reconciliation-cases/cases.json +++ b/tests/platforms/copilot/fixtures/reconciliation-cases/cases.json @@ -184,9 +184,9 @@ ] } }, - "abort": { + "partial_turn": { "observed": false, - "required": "keep it pending and retain open state; missing usage is absent, not zero", + "required": "keep a prompt-only interaction pending; missing usage is absent, not zero", "input_records": [ { "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/f07404d0-af52-4260-89fb-358a10e86034", @@ -252,6 +252,199 @@ } } }, + "abort": { + "observed": false, + "required": "assistant.abort closes the user turn as interrupted; do not treat a prompt-only slice as abort", + "input_records": [ + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-abort-user", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.400Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "user.message", + "data": { + "content": "hello", + "interactionId": "ix-abort", + "turnId": "0" + }, + "id": "synthetic-abort-user", + "timestamp": "2026-09-10T17:08:24.400Z", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "native_event_id": "synthetic-abort-user" + } + }, + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-abort-start", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.450Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "assistant.turn_start", + "data": { + "turnId": "0", + "interactionId": "ix-abort" + }, + "id": "synthetic-abort-start", + "timestamp": "2026-09-10T17:08:24.450Z", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "native_event_id": "synthetic-abort-start" + } + }, + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-abort-msg", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.503Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "assistant.message", + "data": { + "content": "partial", + "model": "gpt-5.6-luna", + "interactionId": "ix-abort", + "turnId": "0", + "reasoningSummary": "started answering", + "toolRequests": [] + }, + "id": "synthetic-abort-msg", + "timestamp": "2026-09-10T17:08:24.503Z", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "native_event_id": "synthetic-abort-msg" + } + }, + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-abort-event", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.800Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "assistant.abort", + "data": { + "turnId": "0", + "interactionId": "ix-abort" + }, + "id": "synthetic-abort-event", + "timestamp": "2026-09-10T17:08:24.800Z", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "native_event_id": "synthetic-abort-event" + } + } + ], + "expected": { + "events": [ + { + "id": "copilot:event:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-abort-user", + "kind": "user_prompt", + "classification": "main", + "ts": "2026-09-10T17:08:24.400Z", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-abort-user" + ], + "attributes": { + "interaction_id": "ix-abort" + } + }, + { + "id": "copilot:event:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-abort-event", + "kind": "abort", + "classification": "main", + "ts": "2026-09-10T17:08:24.800Z", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-abort-event" + ] + } + ], + "turns": [ + { + "turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:ix-abort", + "start_ts": "2026-09-10T17:08:24.400Z", + "end_ts": "2026-09-10T17:08:24.800Z", + "input_message": "hello", + "output_message": "partial", + "status": "interrupted", + "llm_calls": [ + { + "call_id": "copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-abort-msg", + "provider": "unknown", + "model": "gpt-5.6-luna", + "start_ts": "2026-09-10T17:08:24.503Z", + "end_ts": "2026-09-10T17:08:24.800Z", + "input_messages": [ + { + "role": "user", + "parts": [ + { + "type": "text", + "content": "hello" + } + ] + } + ], + "output_messages": [ + { + "role": "assistant", + "parts": [ + { + "type": "text", + "content": "partial" + }, + { + "type": "reasoning", + "content": "started answering" + } + ] + } + ], + "usage": {}, + "tool_calls": [] + } + ], + "permission_requests": [], + "subagents": [], + "attributes": { + "interaction_id": "ix-abort", + "agent_id": null + }, + "accounting_calls": [] + } + ], + "call_candidates": [ + { + "call_id": "copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-abort-msg", + "stored_turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:ix-abort", + "interaction_id": "ix-abort", + "agent_id": null, + "parent_tool_call_id": null, + "model": "gpt-5.6-luna", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-abort-msg", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-abort-event" + ], + "start_ts": "2026-09-10T17:08:24.503Z", + "end_ts": "2026-09-10T17:08:24.800Z", + "tool_call_ids": [] + } + ], + "pending": [], + "usage_rows": [] + } + }, "retry": { "observed": false, "required": "stable source-derived IDs make projection replay equivalent without duplicate accounting", @@ -515,7 +708,7 @@ "permission_requests": [], "subagents": [ { - "turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:7c0fa097-c0e2-48da-b2b6-fcfc1ad83a6b", + "turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:7c0fa097-c0e2-48da-b2b6-fcfc1ad83a6b:bf8cb9f3-2097-4db0-a3c8-78a2653b2106", "start_ts": "2026-09-10T17:08:44.557Z", "end_ts": "2026-09-10T17:08:47.463Z", "input_message": "Read only /fixture/workspace/alpha.txt and /fixture/workspace/beta.txt. Do not modify files. Do not access any other files or services. Report the sum of the numeric values in those two files.", diff --git a/tests/platforms/copilot/fixtures/reconciliation-cases/semantic-projection.json b/tests/platforms/copilot/fixtures/reconciliation-cases/semantic-projection.json index 9211820..9dd8ce5 100644 --- a/tests/platforms/copilot/fixtures/reconciliation-cases/semantic-projection.json +++ b/tests/platforms/copilot/fixtures/reconciliation-cases/semantic-projection.json @@ -260,6 +260,28 @@ "evidence": [ "interaction_id:6d2b89fd-a653-430c-b532-b0936d72eb42" ] + }, + { + "id": "pending:tool:call_YSSva4HCniiETlxdGGjcrHbh", + "kind": "incomplete_tool_pair", + "reason": "tool request has no execution result", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328" + ], + "evidence": [ + "tool_call_id:call_YSSva4HCniiETlxdGGjcrHbh" + ] + }, + { + "id": "pending:tool:call_ayHplfzxjRFMTCpmTKEFhCSJ", + "kind": "incomplete_tool_pair", + "reason": "tool request has no execution result", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328" + ], + "evidence": [ + "tool_call_id:call_ayHplfzxjRFMTCpmTKEFhCSJ" + ] } ], "diagnostics": [ diff --git a/tests/platforms/copilot/test_events.py b/tests/platforms/copilot/test_events.py index 6d7c38f..d2a94c7 100644 --- a/tests/platforms/copilot/test_events.py +++ b/tests/platforms/copilot/test_events.py @@ -288,6 +288,32 @@ def test_subagent_events_use_nested_child_role(): event = normalize_record(record)[0] assert event["kind"] == "subagent_started" assert event["source_references"][0]["role"] == "nested_child" + assert event["attributes"]["parent_tool_call_id"] == "call_parent_task" + + +def test_unknown_record_without_native_type_is_preserved() -> None: + record = _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/typeless", + native_type="user.message", + data={"content": "hello"}, + ) + record["payload"].pop("type") + event = normalize_record(record)[0] + assert event["kind"] == "unknown" + assert "native_type" not in event["attributes"] + assert event["attributes"]["raw_payload"]["data"]["content"] == "hello" + assert event["source_references"][0]["source_kind"] == "transcript" + + +def test_prompt_transformation_transcript_event() -> None: + record = _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/prompt-xf", + native_type="prompt.transformation", + data={"interactionId": "ix-1", "prompt": "expanded"}, + ) + event = normalize_record(record)[0] + assert event["kind"] == "prompt_transformation" + assert event["attributes"]["prompt"] == "expanded" # --- unknown and hook observations --- @@ -312,8 +338,40 @@ def test_hook_pretooluse_stays_hook_observation_not_transcript_execution(): hook_payload={"toolName": "view", "toolArgs": {"path": "/tmp/a.txt"}}, ) event = normalize_record(record)[0] - assert event["kind"] == "tool_execution_start" + assert event["kind"] in {"notification", "unknown"} + assert event["kind"] != "tool_execution_start" assert event["source_references"][0]["role"] == "hook" + assert event["attributes"]["native_type"] == "preToolUse" + + +def test_external_tool_hooks_are_not_tool_execution_events() -> None: + records = [ + _hook_record( + source_id=f"hook/{NATIVE_SESSION_ID}/obs-pre", + event="preToolUse", + hook_payload={"toolName": "view"}, + ), + _hook_record( + source_id=f"hook/{NATIVE_SESSION_ID}/obs-post", + event="postToolUse", + hook_payload={"toolName": "view"}, + ), + _hook_record( + source_id=f"hook/{NATIVE_SESSION_ID}/obs-fail", + event="postToolUseFailure", + hook_payload={"toolName": "view"}, + ), + ] + kinds = [normalize_record(record)[0]["kind"] for record in records] + assert kinds == ["notification", "notification", "notification"] or all( + kind == "unknown" for kind in kinds + ) + for kind in kinds: + assert kind not in { + "tool_execution_start", + "tool_execution_complete", + "tool_execution_failure", + } def test_normalize_records_replays_in_order_without_deduplication(): @@ -401,7 +459,10 @@ def test_session_lifecycle_and_subagent_configured_are_not_unknown() -> None: ) assert _event_kinds(model_change) == ["notification"] assert _event_kinds(auto_mode) == ["notification"] - assert _event_kinds(configured) == ["subagent_started"] + configured_event = normalize_record(configured)[0] + assert configured_event["kind"] in {"notification", "unknown"} + assert configured_event["kind"] != "subagent_started" + assert configured_event["attributes"]["native_type"] == "subagent.configured" def test_agent_stop_hook_is_not_session_end() -> None: diff --git a/tests/platforms/copilot/test_tracing.py b/tests/platforms/copilot/test_tracing.py index a86f89f..1f6e6f6 100644 --- a/tests/platforms/copilot/test_tracing.py +++ b/tests/platforms/copilot/test_tracing.py @@ -196,6 +196,7 @@ def test_tool_cycle_does_not_complete_user_turn_without_final_answer() -> None: pending_kinds = Counter(item["kind"] for item in projection["pending"]) assert pending_kinds["open_interaction"] == 1 + assert pending_kinds["incomplete_tool_pair"] == 2 assert "missing_identity" not in pending_kinds open_key = "6d2b89fd-a653-430c-b532-b0936d72eb42|main" @@ -210,7 +211,7 @@ def test_tool_cycle_does_not_complete_user_turn_without_final_answer() -> None: def test_partial_user_prompt_stays_open_with_pending_item() -> None: - case = _load_json(RECON_CASES / "cases.json")["abort"] + case = _load_json(RECON_CASES / "cases.json")["partial_turn"] projection, state = build_semantics(case["input_records"], {}) assert projection["turns"] == [] @@ -303,7 +304,7 @@ def test_child_turn_nests_under_parent_when_full_fixture_replayed( def test_prior_open_interaction_state_is_retained_when_replay_still_open() -> None: - case = _load_json(RECON_CASES / "cases.json")["abort"] + case = _load_json(RECON_CASES / "cases.json")["partial_turn"] _, state = build_semantics(case["input_records"], {}) prior_item = dict(state["open_interactions"]["6d2b89fd-a653-430c-b532-b0936d72eb42|main"]) prior_item["note"] = "retained-from-incremental-caller" @@ -407,12 +408,16 @@ def test_observed_versus_synthetic_cases_run_through_build_semantics() -> None: assert observed == {"identical_concurrent_tools", "nested_child"} assert "retry" in synthetic assert "permission" in synthetic - for name in ("identical_concurrent_tools", "permission", "abort"): + assert "partial_turn" in synthetic + assert "abort" in synthetic + for name in ("identical_concurrent_tools", "permission", "partial_turn", "abort"): projection, _ = build_semantics(cases[name]["input_records"], {}) assert "events" in projection _assert_expected_projection(projection, cases[name]["expected"]) nested = build_semantics(cases["nested_child"]["input_records"], {})[0] assert nested["turns"] == [] + started = [event for event in nested["events"] if event["kind"] == "subagent_started"] + assert started[0]["attributes"]["parent_tool_call_id"] == "call_qx4FH5DADTeT1qVLb37HNpBk" def test_completed_child_without_parent_link_is_pending_not_dropped() -> None: @@ -702,3 +707,454 @@ def test_semantic_retry_after_error_stays_same_interaction() -> None: assert {candidate["interaction_id"] for candidate in projection["call_candidates"]} == { "ix-retry" } + + +def test_turn_end_attaches_finish_to_matching_native_turn_not_last_call() -> None: + records = [ + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/ov-user", + native_type="user.message", + data={"content": "two cycles", "interactionId": "ix-overlap", "turnId": "0"}, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/ov-call-a", + native_type="assistant.message", + data={ + "content": "first", + "model": "gpt-5.6-luna", + "interactionId": "ix-overlap", + "turnId": "0", + "toolRequests": [], + }, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/ov-call-b", + native_type="assistant.message", + data={ + "content": "second", + "model": "gpt-5.6-luna", + "interactionId": "ix-overlap", + "turnId": "1", + "phase": "final_answer", + "toolRequests": [], + }, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/ov-end-0", + native_type="assistant.turn_end", + data={"turnId": "0"}, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/ov-end-1", + native_type="assistant.turn_end", + data={"turnId": "1"}, + ), + ] + projection, _ = build_semantics(records, {}) + by_id = {candidate["call_id"]: candidate for candidate in projection["call_candidates"]} + first = by_id[f"copilot:call:{SOURCE_KEY}/{NATIVE_SESSION_ID}/ov-call-a"] + second = by_id[f"copilot:call:{SOURCE_KEY}/{NATIVE_SESSION_ID}/ov-call-b"] + assert first["finish_evidence"][0]["source_id"].endswith("/ov-end-0") + assert second["finish_evidence"][0]["source_id"].endswith("/ov-end-1") + assert ( + first["end_ts"] != second["end_ts"] or first["finish_evidence"] != second["finish_evidence"] + ) + + +def test_unmatched_turn_end_is_missing_identity_not_incomplete_tool() -> None: + record = _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/orphan-end", + native_type="assistant.turn_end", + data={"turnId": "99"}, + ) + _, _, pending, _, _ = build_turns([record]) + assert pending + assert pending[0]["kind"] == "missing_identity" + assert all(item["kind"] != "incomplete_tool_pair" for item in pending) + + +def test_incomplete_tools_stay_pending_on_completed_turn() -> None: + records = [ + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/inc-user", + native_type="user.message", + data={"content": "read then answer", "interactionId": "ix-inc", "turnId": "0"}, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/inc-tools", + native_type="assistant.message", + data={ + "content": "", + "model": "gpt-5.6-luna", + "interactionId": "ix-inc", + "turnId": "0", + "toolRequests": [{"toolCallId": "call_missing", "name": "view", "arguments": {}}], + }, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/inc-final", + native_type="assistant.message", + data={ + "content": "42", + "model": "gpt-5.6-luna", + "interactionId": "ix-inc", + "turnId": "1", + "phase": "final_answer", + "toolRequests": [], + }, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/inc-end", + native_type="assistant.turn_end", + data={"turnId": "1"}, + ), + ] + projection, _ = build_semantics(records, {}) + assert len(projection["turns"]) == 1 + assert any( + item["kind"] == "incomplete_tool_pair" and "call_missing" in item["id"] + for item in projection["pending"] + ) + + +def test_followup_user_message_does_not_mark_previous_turn_completed() -> None: + records = [ + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/prev-user", + native_type="user.message", + data={"content": "first", "interactionId": "ix-old", "turnId": "0"}, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/prev-tools", + native_type="assistant.message", + data={ + "content": "", + "model": "gpt-5.6-luna", + "interactionId": "ix-old", + "turnId": "0", + "toolRequests": [{"toolCallId": "call_out", "name": "view", "arguments": {}}], + }, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/next-user", + native_type="user.message", + data={"content": "second", "interactionId": "ix-new", "turnId": "0"}, + ), + ] + projection, state = build_semantics(records, {}) + old_turns = [ + turn for turn in projection["turns"] if turn["attributes"]["interaction_id"] == "ix-old" + ] + assert old_turns == [] + assert "ix-old|main" in state["open_interactions"] + assert any( + item["kind"] == "open_interaction" and "ix-old" in item["evidence"][0] + for item in projection["pending"] + ) + + +def test_completed_child_is_retained_when_later_partition_only_has_parent() -> None: + child = "agent-child" + parent_tool = "call_parent_task" + parent_open = [ + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/p-user", + native_type="user.message", + data={"content": "delegate", "interactionId": "ix-parent", "turnId": "0"}, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/p-task", + native_type="assistant.message", + data={ + "content": "", + "model": "gpt-5.6-luna", + "interactionId": "ix-parent", + "turnId": "0", + "toolRequests": [{"toolCallId": parent_tool, "name": "task", "arguments": {}}], + }, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/p-task-end", + native_type="assistant.turn_end", + data={"turnId": "0"}, + ), + ] + child_complete = [ + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/c-start", + native_type="subagent.started", + data={"toolCallId": parent_tool, "agentName": "explore"}, + agent=child, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/c-user", + native_type="user.message", + data={"content": "explore", "interactionId": "ix-child", "turnId": "0"}, + agent=child, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/c-final", + native_type="assistant.message", + data={ + "content": "42", + "model": "gpt-5.6-luna", + "interactionId": "ix-child", + "turnId": "0", + "phase": "final_answer", + "toolRequests": [], + "parentToolCallId": parent_tool, + }, + agent=child, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/c-end", + native_type="assistant.turn_end", + data={"turnId": "0"}, + agent=child, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/c-stop", + native_type="subagent.completed", + data={"toolCallId": parent_tool, "agentName": "explore"}, + agent=child, + ), + ] + parent_close = [ + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/p-final", + native_type="assistant.message", + data={ + "content": "42", + "model": "gpt-5.6-luna", + "interactionId": "ix-parent", + "turnId": "1", + "phase": "final_answer", + "toolRequests": [], + }, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/p-final-end", + native_type="assistant.turn_end", + data={"turnId": "1"}, + ), + ] + _, state = build_semantics([*parent_open, *child_complete], {}) + open_parent = state["open_interactions"]["ix-parent|main"] + assert open_parent.get("nested_children") + + projection, _ = build_semantics([*parent_open, *parent_close], state) + assert len(projection["turns"]) == 1 + assert len(projection["turns"][0]["subagents"]) == 1 + assert projection["turns"][0]["subagents"][0]["output_message"] == "42" + + full, _ = build_semantics([*parent_open, *child_complete, *parent_close], {}) + assert len(full["turns"][0]["subagents"]) == 1 + + +def test_reasoning_summary_is_readable_span_content() -> None: + records = [ + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/rs-user", + native_type="user.message", + data={"content": "why", "interactionId": "ix-rs", "turnId": "0"}, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/rs-msg", + native_type="assistant.message", + data={ + "content": "42", + "model": "gpt-5.6-luna", + "interactionId": "ix-rs", + "turnId": "0", + "phase": "final_answer", + "reasoningSummary": "added the two file values", + "intentionSummary": "tool intent must not become reasoning", + "toolRequests": [], + }, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/rs-end", + native_type="assistant.turn_end", + data={"turnId": "0"}, + ), + ] + projection, _ = build_semantics(records, {}) + parts = projection["turns"][0]["llm_calls"][0]["output_messages"][0]["parts"] + reasoning = [part for part in parts if part["type"] == "reasoning"] + assert reasoning == [{"type": "reasoning", "content": "added the two file values"}] + assert all(part["content"] != "tool intent must not become reasoning" for part in parts) + + +def test_failed_tool_execution_pairs_through_build_turns() -> None: + records = [ + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/fail-user", + native_type="user.message", + data={"content": "read it", "interactionId": "ix-fail", "turnId": "0"}, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/fail-req", + native_type="assistant.message", + data={ + "content": "", + "model": "gpt-5.6-luna", + "interactionId": "ix-fail", + "turnId": "0", + "toolRequests": [ + {"toolCallId": "call_denied", "name": "view", "arguments": {"path": "/tmp/x"}} + ], + }, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/fail-result", + native_type="tool.execution_complete", + data={"toolCallId": "call_denied", "success": False, "result": "denied"}, + ts="2026-09-10T17:08:24.700Z", + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/fail-final", + native_type="assistant.message", + data={ + "content": "could not read", + "model": "gpt-5.6-luna", + "interactionId": "ix-fail", + "turnId": "1", + "phase": "final_answer", + "toolRequests": [], + }, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/fail-end", + native_type="assistant.turn_end", + data={"turnId": "1"}, + ), + ] + projection, _ = build_semantics(records, {}) + tools = projection["turns"][0]["llm_calls"][0]["tool_calls"] + assert len(tools) == 1 + assert tools[0]["tool_call_id"] == "call_denied" + assert tools[0]["attributes"]["success"] is False + assert tools[0]["attributes"]["result"] == "denied" + assert tools[0]["end_ts"] == "2026-09-10T17:08:24.700Z" + assert not any(item["kind"] == "incomplete_tool_pair" for item in projection["pending"]) + + +def test_assistant_markers_without_interaction_id_are_pending() -> None: + start = _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/noix-start", + native_type="assistant.turn_start", + data={"turnId": "0"}, + ) + message = _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/noix-msg", + native_type="assistant.message", + data={"content": "hello", "model": "gpt-5.6-luna", "turnId": "0", "toolRequests": []}, + ) + _, _, pending, _, _ = build_turns([start, message]) + kinds = [item["kind"] for item in pending] + assert kinds.count("missing_identity") >= 2 + assert all(item["kind"] != "open_interaction" for item in pending) + + +def test_mixed_hook_and_transcript_does_not_double_count_tools() -> None: + records: list[SourceRecord] = [ + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/mix-user", + native_type="user.message", + data={"content": "read", "interactionId": "ix-mix", "turnId": "0"}, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/mix-req", + native_type="assistant.message", + data={ + "content": "", + "model": "gpt-5.6-luna", + "interactionId": "ix-mix", + "turnId": "0", + "toolRequests": [{"toolCallId": "call_mix", "name": "view", "arguments": {}}], + }, + ), + { + "source_id": f"hook/{NATIVE_SESSION_ID}/obs-pre-mix", + "source_kind": "hook", + "native_session_id": NATIVE_SESSION_ID, + "ts": "2026-09-10T17:08:24.500Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "schema_version": 1, + "event": "preToolUse", + "hook_payload": {"toolName": "view", "toolArgs": {}}, + "context": {}, + }, + "locator": {"observation_id": "obs-pre-mix", "event": "preToolUse"}, + }, + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/mix-start", + native_type="tool.execution_start", + data={"toolCallId": "call_mix", "toolName": "view", "arguments": {}}, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/mix-done", + native_type="tool.execution_complete", + data={"toolCallId": "call_mix", "success": True, "result": "17"}, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/mix-final", + native_type="assistant.message", + data={ + "content": "17", + "model": "gpt-5.6-luna", + "interactionId": "ix-mix", + "turnId": "1", + "phase": "final_answer", + "toolRequests": [], + }, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/mix-end", + native_type="assistant.turn_end", + data={"turnId": "1"}, + ), + ] + projection, _ = build_semantics(records, {}) + starts = [event for event in projection["events"] if event["kind"] == "tool_execution_start"] + assert len(starts) == 1 + hook_obs = [ + event + for event in projection["events"] + if event["source_ids"] == [f"hook/{NATIVE_SESSION_ID}/obs-pre-mix"] + ] + assert hook_obs[0]["kind"] in {"notification", "unknown"} + + +def test_cli_fixture_has_one_subagent_started( + cli_transcript_records: list[SourceRecord], +) -> None: + projection, _ = build_semantics(cli_transcript_records, {}) + started = [event for event in projection["events"] if event["kind"] == "subagent_started"] + assert len(started) == 1 + configured = [ + event + for event in projection["events"] + if (event.get("attributes") or {}).get("native_type") == "subagent.configured" + ] + assert configured + assert configured[0]["kind"] != "subagent_started" + + +def test_prompt_transformation_flows_through_build_semantics() -> None: + records = [ + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/xf-user", + native_type="user.message", + data={"content": "hi", "interactionId": "ix-xf", "turnId": "0"}, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/xf-prompt", + native_type="prompt.transformation", + data={"interactionId": "ix-xf", "prompt": "expanded hi"}, + ), + ] + projection, _ = build_semantics(records, {}) + kinds = [event["kind"] for event in projection["events"]] + assert "prompt_transformation" in kinds From 85233aa86c48ffb524b87878310fe3dbfeadc545 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Fri, 11 Sep 2026 16:19:49 -0700 Subject: [PATCH 62/88] Add Copilot turn and usage views --- src/thirdeye/turns.py | 11 +++++++++++ src/thirdeye/web/templates/usage/global.html | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/thirdeye/turns.py b/src/thirdeye/turns.py index 55e825b..ea8c2e8 100644 --- a/src/thirdeye/turns.py +++ b/src/thirdeye/turns.py @@ -29,6 +29,17 @@ def _record(meta: SessionMeta, events: list[dict[str, Any]]) -> dict[str, Any]: def session_turns(meta: SessionMeta, store: Store) -> list[dict[str, Any]]: """Return durable top-level turn slices reconstructed from stored events.""" + if meta.platform == "copilot": + # Copilot's transcript interleaves child-agent messages with the main + # interaction, and its native turn IDs reset for each model cycle. + # The V2 projection has already reconstructed completed main user + # interactions from explicit identities. Reading it here preserves + # the normal turn-record shape without treating an intermediate or + # child assistant message as the end of a user turn. + from thirdeye.platforms.copilot.projection_store import read_projected_turns + + return read_projected_turns(store.config, meta.session_id) + events = list(store.reader(meta.session_id).iter_events()) if meta.platform == "codex": starts = [i for i, event in enumerate(events) if event.get("t") == "agent_turn"] diff --git a/src/thirdeye/web/templates/usage/global.html b/src/thirdeye/web/templates/usage/global.html index 4b5ee49..e028f4a 100644 --- a/src/thirdeye/web/templates/usage/global.html +++ b/src/thirdeye/web/templates/usage/global.html @@ -11,7 +11,7 @@ ") >= 3 + + +def test_copilot_logfire_turn_export_uses_projected_turns( + client, app, web_config, tmp_path: Path, monkeypatch +) -> None: + stored_id = seed_two_main_interaction_projection(web_config, tmp_path) + app.state.config = app.state.config.write_logfire_settings( + LogfireSettings(api_key="dataset-key") + ) + captured: dict = {} + + def fake_export_sessions(**kwargs): + captured.update(kwargs) + return len(kwargs.get("sessions", [])) + + monkeypatch.setattr( + "thirdeye.web.routes.sessions.export_sessions", fake_export_sessions + ) + response = client.post( + "/sessions/logfire-dataset", + data={ + "dataset_name": "copilot-turns", + "dataset_scope": "turn", + "platform": "copilot", + "since": "2020-01-01", + }, + ) + + assert response.status_code == 200 + assert captured["scope"] == "turn" + turn_ids = [turn["id"] for turn in filter_turns(captured["sessions"], captured["store"])] + assert f"{stored_id}:{TURN_ONE_ID}" in turn_ids + assert f"{stored_id}:{TURN_TWO_ID}" in turn_ids + + +def test_copilot_session_usage_page_has_no_token_rows_without_projection( + client, web_config, tmp_path: Path +) -> None: paths = resolve_sources(tmp_path / "copilot-home") records: list[SourceRecord] = [ { diff --git a/tests/web/test_routes_usage.py b/tests/web/test_routes_usage.py index 7d2e755..848bbe8 100644 --- a/tests/web/test_routes_usage.py +++ b/tests/web/test_routes_usage.py @@ -79,17 +79,36 @@ def test_session_usage_404(client): assert r.status_code == 404 -def test_global_platform_filter_only_claude_and_codex(client): - """The platform filter offers only claude and codex — gemini is gone.""" +def test_global_platform_filter_includes_copilot(client): + """The platform filter lists supported agents, including copilot.""" r = client.get("/usage") assert r.status_code == 200 body = r.text assert 'value="claude"' in body assert 'value="codex"' in body + assert 'value="cursor"' in body + assert 'value="copilot"' in body assert 'value="gemini"' not in body assert ">gemini<" not in body +def test_global_usage_platform_filter_shows_copilot_rows(client, web_config, tmp_path): + from tests.shared.copilot_projection_fixtures import ( + USAGE_TOKENS_ONE, + USAGE_TOKENS_TWO, + seed_two_main_interaction_projection, + ) + + seed_two_main_interaction_projection(web_config, tmp_path) + r = client.get( + "/usage?platform=copilot&since=2026-09-01&until=2026-09-30" + ) + assert r.status_code == 200 + assert 'value="copilot"' in r.text + assert "selected>copilot<" in r.text.replace("\n", "") + assert str(USAGE_TOKENS_ONE + USAGE_TOKENS_TWO) in r.text + + def test_session_usage_renders_per_call_rows_with_model(client, web_config): """Per-call rows show the response_model in a model column.""" sd = _make_session( From 31bc0313523d255b1c751a99a5c806888b550357 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Fri, 11 Sep 2026 16:24:40 -0700 Subject: [PATCH 65/88] Add behavioral tests for Copilot usage attribution joins. Cover inferred, direct, pending, ambiguous, and conflicting assignment paths plus build_projection composition. Co-authored-by: Cursor --- tests/platforms/copilot/test_attribution.py | 619 ++++++++++++++++++++ 1 file changed, 619 insertions(+) create mode 100644 tests/platforms/copilot/test_attribution.py diff --git a/tests/platforms/copilot/test_attribution.py b/tests/platforms/copilot/test_attribution.py new file mode 100644 index 0000000..452f653 --- /dev/null +++ b/tests/platforms/copilot/test_attribution.py @@ -0,0 +1,619 @@ +"""Behavioral tests for Copilot usage attribution and projection composition.""" + +from __future__ import annotations + +import copy +import json +import shutil +from pathlib import Path +from typing import Any + +import pytest + +from thirdeye.platforms.copilot.attribution import join_usage +from thirdeye.platforms.copilot.identity import resolve_sources +from thirdeye.platforms.copilot.projection import build_projection +from thirdeye.platforms.copilot.tracing import build_semantics +from thirdeye.platforms.copilot.transcript import read_transcript +from thirdeye.platforms.copilot.types import ( + AccountingCandidate, + AccountingProjection, + CallCandidate, + SemanticProjection, + SourceRecord, +) +from thirdeye.platforms.copilot.usage import build_accounting +from thirdeye.usage.types import UsageRow + +FIXTURES = Path(__file__).parent / "fixtures" +RECON = FIXTURES / "reconciliation-cases" +NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" +CHILD_AGENT_ID = "bf8cb9f3-2097-4db0-a3c8-78a2653b2106" +SOURCE_KEY = "a" * 64 +GENERATION = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" +OBSERVED_AT = "2026-09-10T17:09:00.000Z" +INTERACTION_ONE = "6d2b89fd-a653-430c-b532-b0936d72eb42" +TURN_ONE = ( + f"copilot:turn:{SOURCE_KEY}:{NATIVE_SESSION_ID}:{INTERACTION_ONE}" +) +CALL_A = ( + f"copilot:call:{SOURCE_KEY}/{NATIVE_SESSION_ID}/a4a17e63-7ba5-422f-8ee9-b495be417328" +) +CALL_B = ( + f"copilot:call:{SOURCE_KEY}/{NATIVE_SESSION_ID}/33cc6465-29e1-4a04-8bdb-00241474b4d2" +) + + +def _load_json(path: Path) -> Any: + return json.loads(path.read_text(encoding="utf-8")) + + +def _drain_cli_transcript(home: Path) -> list[SourceRecord]: + session_dir = home / "session-state" / NATIVE_SESSION_ID + session_dir.mkdir(parents=True, exist_ok=True) + shutil.copy(FIXTURES / "events.jsonl", session_dir / "events.jsonl") + (session_dir / "workspace.yaml").write_text("cwd: /sanitized/workspace\n", encoding="utf-8") + paths = resolve_sources(home) + cursor: dict[str, Any] = {} + records: list[SourceRecord] = [] + while True: + slice_ = read_transcript(paths, NATIVE_SESSION_ID, cursor) + records.extend(slice_["records"]) + cursor = slice_["next_cursor"] + if slice_["exhausted"]: + break + return [record for record in records if record["source_kind"] == "transcript"] + + +def _usage_record( + row: dict[str, Any], + *, + content_revision: str, + generation: str = GENERATION, + source_key: str = SOURCE_KEY, + observed_at: str = OBSERVED_AT, +) -> SourceRecord: + primary_key = row["id"] + source_id = ( + f"copilot-db:{source_key}:{NATIVE_SESSION_ID}:assistant_usage_events:" + f"{primary_key}:{content_revision}" + ) + return { + "source_id": source_id, + "source_kind": "database", + "native_session_id": NATIVE_SESSION_ID, + "ts": row.get("created_at"), + "observed_at": observed_at, + "payload": {"table": "assistant_usage_events", "row": row}, + "locator": { + "database": "/example/.copilot/session-store.db", + "table": "assistant_usage_events", + "primary_key": primary_key, + "content_revision": content_revision, + "generation": generation, + }, + } + + +def _source_key(records: list[SourceRecord]) -> str: + for record in records: + if record["source_kind"] == "transcript": + return record["source_id"].split("/", 1)[0] + pytest.fail("expected at least one transcript record") + + +def _substitute_source_key(value: str, source_key: str) -> str: + return value.replace(SOURCE_KEY, source_key) + + +def _align_attribution(expected: dict[str, Any], source_key: str) -> dict[str, Any]: + aligned = copy.deepcopy(expected) + for field in ("call_id", "stored_turn_id", "logical_call_id", "usage_source_id"): + if isinstance(aligned.get(field), str): + aligned[field] = _substitute_source_key(aligned[field], source_key) + aligned["evidence"] = [ + _substitute_source_key(item, source_key) for item in aligned["evidence"] + ] + return aligned + + +def _six_call_records(*, source_key: str = SOURCE_KEY) -> list[SourceRecord]: + rows = _load_json(FIXTURES / "assistant-usage-events.json") + revisions = { + call["row_id"]: call["usage_source_id"].rsplit(":", 1)[-1] + for call in _load_json(RECON / "observed-six-calls.json")["calls"] + } + return [ + _usage_record(row, content_revision=revisions[row["id"]], source_key=source_key) + for row in rows + ] + + +def _metric_totals(usage_rows: list[UsageRow]) -> dict[str, int]: + totals = { + "input_tokens": 0, + "output_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "reasoning_tokens": 0, + } + for row in usage_rows: + totals["input_tokens"] += row.input_tokens + totals["output_tokens"] += row.output_tokens + totals["cache_read_tokens"] += row.cache_read_input_tokens or 0 + totals["cache_write_tokens"] += row.cache_creation_input_tokens or 0 + totals["reasoning_tokens"] += row.reasoning_output_tokens or 0 + return totals + + +def _call_candidate( + *, + call_id: str, + stored_turn_id: str | None = TURN_ONE, + interaction_id: str = INTERACTION_ONE, + agent_id: str | None = None, + parent_tool_call_id: str | None = None, + model: str = "gpt-5.6-luna", + tool_call_ids: list[str] | None = None, + finish_evidence: list[dict[str, Any]] | None = None, + assistant_message_id: str | None = None, +) -> CallCandidate: + candidate: CallCandidate = { + "call_id": call_id, + "stored_turn_id": stored_turn_id, + "interaction_id": interaction_id, + "agent_id": agent_id, + "parent_tool_call_id": parent_tool_call_id, + "model": model, + "source_ids": [call_id.rsplit("/", 1)[-1]], + "source_references": [], + "start_ts": "2026-09-10T17:08:24.503Z", + "end_ts": "2026-09-10T17:08:24.593Z", + "tool_call_ids": tool_call_ids or [], + "finish_evidence": finish_evidence or [], + } + if assistant_message_id is not None: + candidate["assistant_message_id"] = assistant_message_id # type: ignore[typeddict-unknown-key] + return candidate + + +def _accounting_candidate( + *, + row_id: int, + turn_index: int | None = 0, + agent_id: str | None = None, + parent_tool_call_id: str | None = None, + model: str = "gpt-5.6-luna", + finish_reason: str | None = "tool_calls", + initiator: str | None = "user", + revision: str = "sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + assistant_message_id: str | None = None, +) -> AccountingCandidate: + logical_call_id = ( + f"copilot:usage:{SOURCE_KEY}:assistant_usage_events:" + f"sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:{row_id}" + ) + usage_source_id = ( + f"copilot-db:{SOURCE_KEY}:{NATIVE_SESSION_ID}:assistant_usage_events:" + f"{row_id}:{revision}" + ) + supplemental: dict[str, Any] = {} + if initiator is not None: + supplemental["initiator"] = initiator + candidate: AccountingCandidate = { + "usage_source_id": usage_source_id, + "logical_call_id": logical_call_id, + "turn_index": turn_index, + "agent_id": agent_id, + "parent_tool_call_id": parent_tool_call_id, + "model": model, + "provider": None, + "source_ids": [usage_source_id], + "source_references": [], + "revision": { + "primary_key": row_id, + "content_revision": revision, + "generation": GENERATION, + }, + "timestamp": "2026-09-10T17:08:24.498Z", + "finish_reason": finish_reason, + "supplemental_metrics": supplemental, + } + if assistant_message_id is not None: + candidate["assistant_message_id"] = assistant_message_id # type: ignore[typeddict-unknown-key] + return candidate + + +def _semantic(*, calls: list[CallCandidate]) -> SemanticProjection: + return { + "events": [], + "turns": [ + { + "turn_id": TURN_ONE, + "stored_turn_id": TURN_ONE, + "status": "completed", + "output_message": "42", + "subagents": [], + "attributes": {"interaction_id": INTERACTION_ONE}, + } + ], + "call_candidates": calls, + "pending": [], + "diagnostics": [], + } + + +def _accounting(*, candidates: list[AccountingCandidate]) -> AccountingProjection: + return {"usage_rows": [], "candidates": candidates, "diagnostics": []} + + +def _attribution_by_logical(attributions: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + return {item["logical_call_id"]: item for item in attributions} + + +# --- observed six-call corpus --- + + +def test_six_call_corpus_joins_all_usage_rows_inferred(cli_transcript_records: list[SourceRecord]): + source_key = _source_key(cli_transcript_records) + records = cli_transcript_records + _six_call_records(source_key=source_key) + projection, _state = build_projection(records, {}) + expected = _load_json(RECON / "observed-six-calls.json")["expected_attributions"] + by_logical = _attribution_by_logical(projection["attributions"]) + + assert len(projection["attributions"]) == 6 + assert all(item["status"] == "matched" for item in projection["attributions"]) + assert all(item["join_kind"] == "inferred" for item in projection["attributions"]) + for wanted in expected: + aligned = _align_attribution(wanted, source_key) + actual = by_logical[aligned["logical_call_id"]] + assert actual["call_id"] == aligned["call_id"] + assert actual["stored_turn_id"] == aligned["stored_turn_id"] + assert actual["agent_id"] == aligned["agent_id"] + assert actual["evidence"][0] == "join_kind:inferred" + assert actual["evidence"][-1] == aligned["evidence"][-1] + assert f"turn_index:{aligned['evidence'][2].split(':', 1)[1]}" in actual["evidence"] + + +def test_six_call_usage_totals_unchanged_by_attribution(cli_transcript_records: list[SourceRecord]): + source_key = _source_key(cli_transcript_records) + records = cli_transcript_records + _six_call_records(source_key=source_key) + projection, _state = build_projection(records, {}) + assert len(projection["usage_rows"]) == 6 + assert _metric_totals(projection["usage_rows"])["input_tokens"] == 35396 + + +def test_child_usage_maps_to_main_stored_turn(cli_transcript_records: list[SourceRecord]): + source_key = _source_key(cli_transcript_records) + records = cli_transcript_records + _six_call_records(source_key=source_key) + projection, _state = build_projection(records, {}) + child = next( + item for item in projection["attributions"] if item["agent_id"] == CHILD_AGENT_ID + ) + main_turn_two = ( + f"copilot:turn:{source_key}:{NATIVE_SESSION_ID}:793d3703-6f4a-4814-8877-34a7325848ce" + ) + assert child["stored_turn_id"] == main_turn_two + assert child["status"] == "matched" + + +# --- contract fixtures via join_usage --- + + +def test_attribution_fixture_matched_inferred(): + examples = _load_json(RECON / "attributions.json") + semantic, _ = build_semantics(_load_json(RECON / "semantic-projection.json")["input_records"], {}) + accounting, _ = build_accounting( + [ + _usage_record( + _load_json(FIXTURES / "assistant-usage-events.json")[0], + content_revision="sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + ) + ], + {}, + ) + attributions = join_usage(semantic, accounting) + assert attributions == [examples["matched_inferred"]] + + +def test_attribution_fixture_pending_without_matching_call(): + examples = _load_json(RECON / "attributions.json") + semantic = _semantic( + calls=[ + _call_candidate( + call_id=CALL_A, + tool_call_ids=["call_YSSva4HCniiETlxdGGjcrHbh", "call_ayHplfzxjRFMTCpmTKEFhCSJ"], + ) + ] + ) + accounting = _accounting( + candidates=[ + _accounting_candidate( + row_id=14, + finish_reason="stop", + initiator="agent", + ) + ] + ) + attributions = join_usage(semantic, accounting) + assert attributions[0]["status"] == "pending" + assert attributions[0]["call_id"] is None + assert attributions[0]["stored_turn_id"] == examples["pending"]["stored_turn_id"] + assert "delayed_row:true" in attributions[0]["evidence"] + + +# --- direct / native joins --- + + +def test_direct_join_wins_over_inferred_evidence(): + shared_message_id = "native-msg-abc" + semantic = _semantic( + calls=[ + _call_candidate( + call_id=CALL_A, + tool_call_ids=["tool-a"], + assistant_message_id=shared_message_id, + ), + _call_candidate( + call_id=CALL_B, + tool_call_ids=[], + finish_evidence=[{"source_id": "finish"}], + ), + ] + ) + accounting = _accounting( + candidates=[_accounting_candidate(row_id=13, assistant_message_id=shared_message_id)] + ) + attributions = join_usage(semantic, accounting) + assert attributions[0]["status"] == "matched" + assert attributions[0]["join_kind"] == "direct" + assert attributions[0]["call_id"] == CALL_A + assert "join_kind:direct" in attributions[0]["evidence"] + + +def test_competing_direct_ids_are_conflicting(): + shared_message_id = "shared-native-id" + semantic = _semantic( + calls=[ + _call_candidate(call_id=CALL_A, assistant_message_id=shared_message_id), + _call_candidate(call_id=CALL_B, assistant_message_id=shared_message_id), + ] + ) + accounting = _accounting( + candidates=[_accounting_candidate(row_id=13, assistant_message_id=shared_message_id)] + ) + attributions = join_usage(semantic, accounting) + assert attributions[0]["status"] == "conflicting" + assert attributions[0]["call_id"] is None + assert "join_conflict:competing_direct_ids" in attributions[0]["evidence"] + + +# --- inferred / ambiguous / pending --- + + +def test_inferred_join_requires_unique_consistent_evidence(): + semantic = _semantic( + calls=[ + _call_candidate( + call_id=CALL_A, + tool_call_ids=["tool-a"], + ), + _call_candidate( + call_id=CALL_B, + tool_call_ids=[], + finish_evidence=[{"source_id": "finish-b"}], + ), + ] + ) + accounting = _accounting( + candidates=[ + _accounting_candidate(row_id=13, finish_reason="tool_calls", initiator="user"), + _accounting_candidate( + row_id=14, + finish_reason="stop", + initiator="agent", + ), + ] + ) + attributions = join_usage(semantic, accounting) + by_row = {item["logical_call_id"].rsplit(":", 1)[-1]: item for item in attributions} + assert by_row["13"]["status"] == "matched" + assert by_row["13"]["join_kind"] == "inferred" + assert by_row["13"]["call_id"] == CALL_A + assert by_row["14"]["status"] == "matched" + assert by_row["14"]["call_id"] == CALL_B + + +def test_ambiguous_when_multiple_calls_fit_same_evidence(): + semantic = _semantic( + calls=[ + _call_candidate(call_id=CALL_A, tool_call_ids=["tool-a"]), + _call_candidate(call_id=CALL_B, tool_call_ids=["tool-b"]), + ] + ) + accounting = _accounting( + candidates=[ + _accounting_candidate(row_id=13, finish_reason=None, initiator=None), + ] + ) + attributions = join_usage(semantic, accounting) + assert attributions[0]["status"] == "ambiguous" + assert attributions[0]["call_id"] is None + assert f"call_id:{CALL_A}" in attributions[0]["evidence"] + assert f"call_id:{CALL_B}" in attributions[0]["evidence"] + + +def test_pending_when_turn_index_has_no_main_interaction(): + semantic = _semantic(calls=[_call_candidate(call_id=CALL_A, tool_call_ids=["tool-a"])]) + accounting = _accounting( + candidates=[_accounting_candidate(row_id=13, turn_index=99)] + ) + attributions = join_usage(semantic, accounting) + assert attributions[0]["status"] == "pending" + assert "delayed_row:true" in attributions[0]["evidence"] + + +def test_late_row_case_stays_pending_until_semantics_catch_up(): + case = _load_json(RECON / "cases.json")["late_row"] + projection, _state = build_projection(case["input_records"], {}) + attribution = projection["attributions"][0] + assert attribution["status"] == "pending" + assert attribution["call_id"] is None + assert attribution["logical_call_id"] == case["expected"]["attributions"][0]["logical_call_id"] + assert "delayed_row:true" in attribution["evidence"] + assert len(projection["usage_rows"]) == 1 + + +def test_delayed_row_resolves_after_transcript_replay(): + case = _load_json(RECON / "cases.json")["late_row"] + partial = case["input_records"] + final_answer = _load_json(RECON / "cases.json")["ambiguous"]["input_records"][1] + turn_end = { + "source_id": ( + f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/4a386a37-ca7e-4ebc-a746-cdda20f2a4bb" + ), + "source_kind": "transcript", + "native_session_id": NATIVE_SESSION_ID, + "ts": "2026-09-10T17:08:25.626Z", + "observed_at": OBSERVED_AT, + "payload": { + "type": "assistant.turn_end", + "data": {"turnId": "1"}, + "id": "4a386a37-ca7e-4ebc-a746-cdda20f2a4bb", + "timestamp": "2026-09-10T17:08:25.626Z", + "parentId": "33cc6465-29e1-4a04-8bdb-00241474b4d2", + "schema_version": 1, + }, + "locator": { + "file": "events.jsonl", + "native_event_id": "4a386a37-ca7e-4ebc-a746-cdda20f2a4bb", + }, + } + full = partial + [final_answer, turn_end] + pending_projection, _ = build_projection(partial, {}) + resolved_projection, _ = build_projection(full, {}) + assert pending_projection["attributions"][0]["status"] == "pending" + assert resolved_projection["attributions"][0]["status"] == "matched" + assert resolved_projection["attributions"][0]["call_id"].endswith( + "33cc6465-29e1-4a04-8bdb-00241474b4d2" + ) + + +# --- conflicting joins --- + + +def test_two_usage_rows_claiming_one_call_become_conflicting(): + semantic = _semantic( + calls=[ + _call_candidate( + call_id=CALL_A, + tool_call_ids=["tool-a"], + ), + _call_candidate( + call_id=CALL_B, + tool_call_ids=[], + finish_evidence=[{"source_id": "finish-b"}], + ), + ] + ) + accounting = _accounting( + candidates=[ + _accounting_candidate(row_id=13, finish_reason="tool_calls", initiator="user"), + _accounting_candidate(row_id=14, finish_reason="tool_calls", initiator="user"), + ] + ) + attributions = join_usage(semantic, accounting) + assert all(item["status"] == "conflicting" for item in attributions) + assert all(item["call_id"] is None for item in attributions) + assert all("join_conflict:multiple_usage_rows" in item["evidence"] for item in attributions) + + +# --- build_projection composition --- + + +def test_build_projection_attaches_accounting_calls_without_mutating_semantics( + cli_transcript_records: list[SourceRecord], +): + source_key = _source_key(cli_transcript_records) + records = cli_transcript_records + _six_call_records(source_key=source_key) + semantic_before, _ = build_semantics(records, {}) + projection, _state = build_projection(records, {}) + semantic_after, _ = build_semantics(records, {}) + + assert semantic_before["turns"] == semantic_after["turns"] + assert all(not turn.get("accounting_calls") for turn in semantic_before["turns"]) + + attached = sum( + len(turn.get("accounting_calls") or []) + for turn in projection["turns"] + ) + assert attached == 6 + matched = [item for item in projection["attributions"] if item["status"] == "matched"] + for attribution in matched: + owner = next( + turn + for turn in projection["turns"] + if turn["turn_id"] == attribution["stored_turn_id"] + ) + call_ids = [ + item["call_id"] + for item in owner.get("accounting_calls") or [] + if item["attribution_status"] == "matched" + ] + assert attribution["call_id"] in call_ids + + +def test_accounting_rows_persist_when_attribution_is_ambiguous(): + semantic = _semantic( + calls=[ + _call_candidate(call_id=CALL_A, tool_call_ids=["tool-a"]), + _call_candidate(call_id=CALL_B, tool_call_ids=["tool-b"]), + ] + ) + accounting = _accounting( + candidates=[_accounting_candidate(row_id=13, finish_reason=None, initiator=None)] + ) + attributions = join_usage(semantic, accounting) + assert attributions[0]["status"] == "ambiguous" + assert len(accounting["candidates"]) == 1 + + +def test_late_row_projection_keeps_usage_rows_while_attribution_pending(): + case = _load_json(RECON / "cases.json")["late_row"] + projection, _ = build_projection(case["input_records"], {}) + assert projection["attributions"][0]["status"] == "pending" + assert len(projection["usage_rows"]) == 1 + assert projection["usage_rows"][0].input_tokens == 6587 + + +def test_matched_inferred_usage_emits_diagnostic(cli_transcript_records: list[SourceRecord]): + source_key = _source_key(cli_transcript_records) + records = cli_transcript_records + _six_call_records(source_key=source_key) + projection, _ = build_projection(records, {}) + inferred = [item for item in projection["diagnostics"] if item["code"] == "inferred_join"] + assert len(inferred) == 6 + + +def test_unmatched_usage_emits_pending_item(): + case = _load_json(RECON / "cases.json")["late_row"] + projection, _ = build_projection(case["input_records"], {}) + pending = [item for item in projection["pending"] if item["kind"] == "unmatched_usage"] + assert len(pending) == 1 + assert pending[0]["reason"] == "usage attribution is pending" + assert pending[0]["id"].startswith("pending:usage:") + + +def test_source_correction_replaces_attribution_target_after_replay( + cli_transcript_records: list[SourceRecord], +): + source_key = _source_key(cli_transcript_records) + records = cli_transcript_records + _six_call_records(source_key=source_key) + first, state = build_projection(records, {}) + second, _ = build_projection(records, state) + assert first["attributions"] == second["attributions"] + assert len(first["usage_rows"]) == len(second["usage_rows"]) == 6 + + +@pytest.fixture +def cli_transcript_records(tmp_path: Path) -> list[SourceRecord]: + return _drain_cli_transcript(tmp_path / "copilot-home") From 98407e3584b959d34b69903fa3132c406015b6e7 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Fri, 11 Sep 2026 16:36:29 -0700 Subject: [PATCH 66/88] Cover Copilot projected turns through session, search, tag, and eval-input routes. The previous views work sliced Copilot correctly, but review coverage stopped at session_turns and usage pages. Drive those remaining HTTP surfaces from stored projections, keep incomplete mains out of turn records, and assemble real Logfire turn cases instead of stubbing export_sessions. Co-authored-by: Cursor --- src/thirdeye/turns.py | 3 +- tests/shared/copilot_projection_fixtures.py | 55 ++++++--- tests/shared/test_turns.py | 33 +++++- tests/web/test_copilot_capture_views.py | 120 +++++++++++++++----- tests/web/test_routes_usage.py | 5 +- 5 files changed, 161 insertions(+), 55 deletions(-) diff --git a/src/thirdeye/turns.py b/src/thirdeye/turns.py index ea8c2e8..6a74930 100644 --- a/src/thirdeye/turns.py +++ b/src/thirdeye/turns.py @@ -4,6 +4,7 @@ from typing import Any from thirdeye.meta import SessionMeta +from thirdeye.platforms.copilot.projection_store import read_projected_turns from thirdeye.store import Store @@ -36,8 +37,6 @@ def session_turns(meta: SessionMeta, store: Store) -> list[dict[str, Any]]: # interactions from explicit identities. Reading it here preserves # the normal turn-record shape without treating an intermediate or # child assistant message as the end of a user turn. - from thirdeye.platforms.copilot.projection_store import read_projected_turns - return read_projected_turns(store.config, meta.session_id) events = list(store.reader(meta.session_id).iter_events()) diff --git a/tests/shared/copilot_projection_fixtures.py b/tests/shared/copilot_projection_fixtures.py index 3f4bbee..44cde08 100644 --- a/tests/shared/copilot_projection_fixtures.py +++ b/tests/shared/copilot_projection_fixtures.py @@ -17,18 +17,26 @@ NATIVE_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" INTERACTION_ONE = "6d2b89fd-a653-430c-b532-b0936d72eb42" INTERACTION_TWO = "7e3c90ae-b764-541d-c643-c1047e83fc53" -TURN_ONE_ID = ( - "copilot:turn:fixture:session:" - "6d2b89fd-a653-430c-b532-b0936d72eb42" -) -TURN_TWO_ID = ( - "copilot:turn:fixture:session:" - "7e3c90ae-b764-541d-c643-c1047e83fc53" -) +INTERACTION_THREE = "8f4d01bf-c875-652e-d754-d2158f94ad64" +TURN_ONE_ID = "copilot:turn:fixture:session:6d2b89fd-a653-430c-b532-b0936d72eb42" +TURN_TWO_ID = "copilot:turn:fixture:session:7e3c90ae-b764-541d-c643-c1047e83fc53" +TURN_THREE_ID = "copilot:turn:fixture:session:8f4d01bf-c875-652e-d754-d2158f94ad64" USAGE_MODEL_ONE = "gpt-5.6-luna" USAGE_MODEL_TWO = "gpt-4.1-copilot-sentinel" USAGE_TOKENS_ONE = 1111 USAGE_TOKENS_TWO = 2222 +TURN_RECORD_KEYS = ( + "id", + "turn_id", + "session_id", + "platform", + "cwd", + "start_seq", + "end_seq", + "start_ts", + "end_ts", + "events", +) def _transcript_record( @@ -71,6 +79,7 @@ def _main_turn( source_ids: list[str], start_ts: str, end_ts: str, + status: str = "completed", subagents: list[dict[str, Any]] | None = None, ) -> dict[str, Any]: return { @@ -78,8 +87,8 @@ def _main_turn( "start_ts": start_ts, "end_ts": end_ts, "input_message": "prompt", - "output_message": "done", - "status": "completed", + "output_message": "done" if status == "completed" else "", + "status": status, "llm_calls": [], "permission_requests": [], "subagents": subagents or [], @@ -102,14 +111,16 @@ def _usage_row( call_id=call_id, ts=ts, platform=PLATFORM_NAME, - provider_name="openai", + provider_name="unknown", response_model=response_model, input_tokens=input_tokens, output_tokens=10, ) -def _attribution(*, logical_call_id: str, usage_source_id: str, stored_turn_id: str) -> dict[str, Any]: +def _attribution( + *, logical_call_id: str, usage_source_id: str, stored_turn_id: str +) -> dict[str, Any]: return { "usage_source_id": usage_source_id, "logical_call_id": logical_call_id, @@ -131,11 +142,13 @@ def seed_two_main_interaction_projection(config: Config, tmp_path: Path) -> str: ts3 = "2026-09-10T17:08:25.503Z" ts4 = "2026-09-10T17:08:40.000Z" ts5 = "2026-09-10T17:08:42.000Z" + ts6 = "2026-09-10T17:08:50.000Z" user1 = "user-one" asst1 = "assistant-one" child1 = "child-one" user2 = "user-two" asst2 = "assistant-two" + user3 = "user-three" records = [ _transcript_record( source_key, @@ -176,6 +189,13 @@ def seed_two_main_interaction_projection(config: Config, tmp_path: Path) -> str: interaction_id=INTERACTION_TWO, message_type="assistant.message", ), + _transcript_record( + source_key, + user3, + content="Keep going on the still-open follow-up.", + ts=ts6, + interaction_id=INTERACTION_THREE, + ), ] batch: SourceBatch = { "source_key": source_key, @@ -197,6 +217,7 @@ def seed_two_main_interaction_projection(config: Config, tmp_path: Path) -> str: f"{source_key}/{NATIVE_ID}/{user2}", f"{source_key}/{NATIVE_ID}/{asst2}", ] + turn_three_sources = [f"{source_key}/{NATIVE_ID}/{user3}"] turn_one = _main_turn( turn_id=TURN_ONE_ID, interaction_id=INTERACTION_ONE, @@ -229,6 +250,14 @@ def seed_two_main_interaction_projection(config: Config, tmp_path: Path) -> str: start_ts=ts4, end_ts=ts5, ) + turn_three = _main_turn( + turn_id=TURN_THREE_ID, + interaction_id=INTERACTION_THREE, + source_ids=turn_three_sources, + start_ts=ts6, + end_ts=ts6, + status="in_progress", + ) usage_one = _usage_row( stored_id=stored_id, call_id="usage-source-one", @@ -245,7 +274,7 @@ def seed_two_main_interaction_projection(config: Config, tmp_path: Path) -> str: ) projection: Projection = { "normalized_events": [], - "turns": [turn_one, turn_two], + "turns": [turn_one, turn_two, turn_three], "usage_rows": [usage_one, usage_two], "attributions": [ _attribution( diff --git a/tests/shared/test_turns.py b/tests/shared/test_turns.py index ce1f90b..b61cdaa 100644 --- a/tests/shared/test_turns.py +++ b/tests/shared/test_turns.py @@ -1,14 +1,15 @@ from __future__ import annotations -from thirdeye.config import Config -from thirdeye.store import Store -from thirdeye.turns import filter_turns, session_turns - from tests.shared.copilot_projection_fixtures import ( TURN_ONE_ID, + TURN_RECORD_KEYS, + TURN_THREE_ID, TURN_TWO_ID, seed_two_main_interaction_projection, ) +from thirdeye.config import Config +from thirdeye.store import Store +from thirdeye.turns import filter_turns, session_turns def test_claude_turns_are_bounded_by_user_and_assistant_messages(tmp_path): @@ -75,10 +76,30 @@ def test_copilot_session_turns_use_projected_main_interactions_not_child_slices( turns = session_turns(meta, store) assert [turn["turn_id"] for turn in turns] == [TURN_ONE_ID, TURN_TWO_ID] + assert TURN_THREE_ID not in [turn["turn_id"] for turn in turns] + for turn in turns: + assert [key for key in TURN_RECORD_KEYS if key not in turn] == [] + assert turn["id"] == f"{stored_id}:{turn['turn_id']}" + assert turn["session_id"] == stored_id + assert turn["platform"] == "copilot" + assert turn["cwd"] == "/fixture/workspace" + assert turn["start_seq"] is not None + assert turn["end_seq"] is not None + assert turn["start_ts"] + assert turn["end_ts"] + assert isinstance(turn["events"], list) + assert turns[0]["start_seq"] == 0 + assert turns[0]["end_seq"] == 2 + assert turns[1]["start_seq"] == 3 + assert turns[1]["end_seq"] == 4 assert len(turns[0]["events"]) == 3 assert len(turns[1]["events"]) == 2 - assert "alpha.txt" in turns[0]["events"][0]["data"]["source_record"]["payload"]["data"]["content"] - assert "final sum" in turns[1]["events"][0]["data"]["source_record"]["payload"]["data"]["content"] + assert ( + "alpha.txt" in turns[0]["events"][0]["data"]["source_record"]["payload"]["data"]["content"] + ) + assert ( + "final sum" in turns[1]["events"][0]["data"]["source_record"]["payload"]["data"]["content"] + ) assert filter_turns([meta], store, query="alpha.txt") == [turns[0]] assert filter_turns([meta], store, query="final sum") == [turns[1]] assert filter_turns([meta], store, query="alpha.txt,final sum") == [] diff --git a/tests/web/test_copilot_capture_views.py b/tests/web/test_copilot_capture_views.py index f079004..8fe2184 100644 --- a/tests/web/test_copilot_capture_views.py +++ b/tests/web/test_copilot_capture_views.py @@ -2,18 +2,15 @@ from __future__ import annotations +import sys from pathlib import Path +from types import ModuleType import pytest -from thirdeye.config import LogfireSettings -from thirdeye.platforms.copilot.archive import commit_batch -from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id -from thirdeye.platforms.copilot.types import SourceBatch, SourceRecord -from thirdeye.turns import filter_turns, session_turns - from tests.shared.copilot_projection_fixtures import ( TURN_ONE_ID, + TURN_THREE_ID, TURN_TWO_ID, USAGE_MODEL_ONE, USAGE_MODEL_TWO, @@ -21,6 +18,11 @@ USAGE_TOKENS_TWO, seed_two_main_interaction_projection, ) +from thirdeye.config import LogfireSettings +from thirdeye.platforms.copilot.archive import commit_batch +from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id +from thirdeye.platforms.copilot.types import SourceBatch, SourceRecord +from thirdeye.turns import session_turns pytest.importorskip("starlette") @@ -192,12 +194,8 @@ def test_copilot_projected_turns_participate_in_index_turn_query( assert [turn["turn_id"] for turn in turns] == [TURN_ONE_ID, TURN_TWO_ID] - first_turn_query = client.get( - "/?platform=copilot&since=2020-01-01&turn_query=alpha.txt" - ) - second_turn_query = client.get( - "/?platform=copilot&since=2020-01-01&turn_query=final%20sum" - ) + first_turn_query = client.get("/?platform=copilot&since=2020-01-01&turn_query=alpha.txt") + second_turn_query = client.get("/?platform=copilot&since=2020-01-01&turn_query=final%20sum") cross_turn_query = client.get( "/?platform=copilot&since=2020-01-01&turn_query=alpha.txt,final%20sum" ) @@ -217,30 +215,56 @@ def test_copilot_session_usage_page_shows_projected_usage_rows( assert usage.status_code == 200 body = usage.text - assert USAGE_MODEL_ONE in body - assert USAGE_MODEL_TWO in body - assert str(USAGE_TOKENS_ONE) in body - assert str(USAGE_TOKENS_TWO) in body - assert body.count("") >= 3 + tbody = body.split("", 1)[1].split("", 1)[0] + assert tbody.count("") == 2 + assert USAGE_MODEL_ONE in tbody + assert USAGE_MODEL_TWO in tbody + assert str(USAGE_TOKENS_ONE) in tbody + assert str(USAGE_TOKENS_TWO) in tbody + assert str(USAGE_TOKENS_TWO * 2) not in body -def test_copilot_logfire_turn_export_uses_projected_turns( +def test_copilot_projected_session_search_tag_and_eval_routes( client, app, web_config, tmp_path: Path, monkeypatch ) -> None: stored_id = seed_two_main_interaction_projection(web_config, tmp_path) app.state.config = app.state.config.write_logfire_settings( LogfireSettings(api_key="dataset-key") ) - captured: dict = {} + added: list[dict] = [] - def fake_export_sessions(**kwargs): - captured.update(kwargs) - return len(kwargs.get("sessions", [])) + class Client: + def __init__(self, api_key): + pass - monkeypatch.setattr( - "thirdeye.web.routes.sessions.export_sessions", fake_export_sessions - ) - response = client.post( + def __enter__(self): + return self + + def __exit__(self, *args): + return None + + def create_dataset(self, **kwargs): + pass + + def add_cases(self, name, *, cases): + added.extend(cases) + + package = ModuleType("logfire") + experimental = ModuleType("logfire.experimental") + api_client = ModuleType("logfire.experimental.api_client") + api_client.LogfireAPIClient = Client + monkeypatch.setitem(sys.modules, "logfire", package) + monkeypatch.setitem(sys.modules, "logfire.experimental", experimental) + monkeypatch.setitem(sys.modules, "logfire.experimental.api_client", api_client) + + session = client.get(f"/sessions/{stored_id}") + tree = client.get(f"/sessions/{stored_id}/tree") + detail = client.get(f"/sessions/{stored_id}/events/0") + search = client.get("/search?q=alpha.txt&platform=copilot") + tagged = client.post(f"/sessions/{stored_id}/events/0/tags", data={"tag": "review"}) + tagged_index = client.get("/?tag=review&since=2020-01-01") + untagged_index = client.get("/?tag=missing-tag&since=2020-01-01") + export = client.post( "/sessions/logfire-dataset", data={ "dataset_name": "copilot-turns", @@ -250,11 +274,45 @@ def fake_export_sessions(**kwargs): }, ) - assert response.status_code == 200 - assert captured["scope"] == "turn" - turn_ids = [turn["id"] for turn in filter_turns(captured["sessions"], captured["store"])] - assert f"{stored_id}:{TURN_ONE_ID}" in turn_ids - assert f"{stored_id}:{TURN_TWO_ID}" in turn_ids + assert session.status_code == tree.status_code == search.status_code == 200 + assert detail.status_code == 200 + assert b"copilot" in session.content + assert b"/fixture/workspace" in session.content + assert b"copilot_transcript" in tree.content + assert b"user_message" not in tree.content + assert b"tool_call" not in tree.content + assert b"alpha.txt" in detail.content + assert b'"schema_version": 1' in detail.content + assert stored_id.encode() in search.content + assert b"alpha.txt" in search.content + assert tagged.status_code == 200 + assert b"review" in tagged.content + assert tagged_index.status_code == untagged_index.status_code == 200 + assert stored_id.encode() in tagged_index.content + assert stored_id.encode() not in untagged_index.content + + assert export.status_code == 200 + assert "Sent 2 turns" in export.text + names = [case["name"] for case in added] + assert names == [ + f"{stored_id}:{TURN_ONE_ID}", + f"{stored_id}:{TURN_TWO_ID}", + ] + assert TURN_THREE_ID not in "".join(names) + assert not any(case["name"].endswith(":child") for case in added) + first, second = added + assert "turn" in first["inputs"] and "turn" in second["inputs"] + assert len(first["inputs"]["turn"]["events"]) == 3 + assert len(second["inputs"]["turn"]["events"]) == 2 + event_types = {event.get("t") for case in added for event in case["inputs"]["turn"]["events"]} + assert event_types <= { + "copilot_transcript", + "copilot_database", + "copilot_hook", + "copilot_metadata", + } + assert "user_message" not in event_types + assert "tool_call" not in event_types def test_copilot_session_usage_page_has_no_token_rows_without_projection( diff --git a/tests/web/test_routes_usage.py b/tests/web/test_routes_usage.py index 848bbe8..f4eed35 100644 --- a/tests/web/test_routes_usage.py +++ b/tests/web/test_routes_usage.py @@ -100,13 +100,12 @@ def test_global_usage_platform_filter_shows_copilot_rows(client, web_config, tmp ) seed_two_main_interaction_projection(web_config, tmp_path) - r = client.get( - "/usage?platform=copilot&since=2026-09-01&until=2026-09-30" - ) + r = client.get("/usage?platform=copilot&since=2026-09-01&until=2026-09-30") assert r.status_code == 200 assert 'value="copilot"' in r.text assert "selected>copilot<" in r.text.replace("\n", "") assert str(USAGE_TOKENS_ONE + USAGE_TOKENS_TWO) in r.text + assert str(USAGE_TOKENS_TWO * 2) not in r.text def test_session_usage_renders_per_call_rows_with_model(client, web_config): From 3d5ff426b52c2e950936d02710fc94a2f5fb990d Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Fri, 11 Sep 2026 16:43:03 -0700 Subject: [PATCH 67/88] Fix Copilot attribution so joins stay stable across partitions, children, and retries. Unwrap nested semantic state, attach matched child usage to nested chat spans, and order model cycles by source start_ts instead of list index so accounting is never lost when a join is uncertain. Co-authored-by: Cursor --- src/thirdeye/platforms/copilot/attribution.py | 91 ++-- src/thirdeye/platforms/copilot/projection.py | 59 ++- tests/platforms/copilot/test_attribution.py | 426 ++++++++++++++++-- 3 files changed, 480 insertions(+), 96 deletions(-) diff --git a/src/thirdeye/platforms/copilot/attribution.py b/src/thirdeye/platforms/copilot/attribution.py index 1a342d9..30772b8 100644 --- a/src/thirdeye/platforms/copilot/attribution.py +++ b/src/thirdeye/platforms/copilot/attribution.py @@ -49,6 +49,15 @@ def _direct_ids(candidate: dict[str, Any], *, semantic: bool) -> set[str]: return result +def _tool_index(calls: list[CallCandidate]) -> dict[str, list[CallCandidate]]: + by_tool: dict[str, list[CallCandidate]] = defaultdict(list) + for call in calls: + for tool_id in call.get("tool_call_ids") or []: + if isinstance(tool_id, str) and tool_id: + by_tool[tool_id].append(call) + return by_tool + + def _root_candidate( candidate: CallCandidate, by_tool: dict[str, list[CallCandidate]] ) -> CallCandidate | None: @@ -69,17 +78,34 @@ def _root_candidate( return current +def _cycle_order(calls: list[CallCandidate]) -> list[CallCandidate]: + """Order model cycles inside one identity group by source start_ts.""" + + return sorted( + calls, + key=lambda call: (str(call.get("start_ts") or ""), call["call_id"]), + ) + + +def _identity_group( + call: CallCandidate, calls: list[CallCandidate] +) -> list[CallCandidate]: + group = [ + item + for item in calls + if item.get("interaction_id") == call.get("interaction_id") + and item.get("agent_id") == call.get("agent_id") + and item.get("parent_tool_call_id") == call.get("parent_tool_call_id") + ] + return _cycle_order(group) + + def _interaction_indexes( calls: list[CallCandidate], + by_tool: dict[str, list[CallCandidate]], ) -> tuple[dict[str, int], dict[str, CallCandidate]]: """Build only the verified main interaction order used by DB turn_index.""" - by_tool: dict[str, list[CallCandidate]] = defaultdict(list) - for call in calls: - for tool_id in call.get("tool_call_ids", []): - if isinstance(tool_id, str) and tool_id: - by_tool[tool_id].append(call) - indexes: dict[str, int] = {} roots: dict[str, CallCandidate] = {} for call in calls: @@ -91,21 +117,13 @@ def _interaction_indexes( continue # A main interaction appears in archive order. Repeated model cycles # intentionally retain its first position; this is not a bare turnId. + # Failed child walks never contribute a slot. if interaction not in indexes: indexes[interaction] = len(indexes) roots[interaction] = root return indexes, roots -def _root_for(call: CallCandidate, calls: list[CallCandidate]) -> CallCandidate | None: - by_tool: dict[str, list[CallCandidate]] = defaultdict(list) - for item in calls: - for tool_id in item.get("tool_call_ids", []): - if isinstance(tool_id, str) and tool_id: - by_tool[tool_id].append(item) - return _root_candidate(call, by_tool) - - def _finish_matches(usage: AccountingCandidate, call: CallCandidate) -> bool: reason = _string(usage.get("finish_reason")) if reason is None: @@ -127,14 +145,8 @@ def _initiator_matches( return True if not isinstance(initiator, str): return False - same_interaction = [ - item - for item in candidates - if item.get("interaction_id") == call.get("interaction_id") - and item.get("agent_id") == call.get("agent_id") - and item.get("parent_tool_call_id") == call.get("parent_tool_call_id") - ] - call_order = same_interaction.index(call) + group = _identity_group(call, candidates) + call_order = group.index(call) if initiator == "user": return call_order == 0 if initiator == "sub-agent": @@ -184,7 +196,8 @@ def join_usage(semantic: SemanticProjection, accounting: AccountingProjection) - """ calls = list(semantic.get("call_candidates") or []) - indexes, roots = _interaction_indexes(calls) + by_tool = _tool_index(calls) + indexes, roots = _interaction_indexes(calls, by_tool) preliminary: list[tuple[Attribution, CallCandidate | None]] = [] for usage in accounting.get("candidates") or []: @@ -197,15 +210,16 @@ def join_usage(semantic: SemanticProjection, accounting: AccountingProjection) - result.update( status="conflicting", evidence=[ + f"logical_call_id:{usage['logical_call_id']}", *(f"call_id:{call['call_id']}" for call in direct_matches), - "join_conflict:competing_direct_ids", ], ) preliminary.append((result, None)) continue call = direct_matches[0] + root = _root_candidate(call, by_tool) or call result.update( - stored_turn_id=_string((_root_for(call, calls) or call).get("stored_turn_id")), + stored_turn_id=_string(root.get("stored_turn_id")), agent_id=call.get("agent_id"), call_id=call["call_id"], status="matched", @@ -216,20 +230,19 @@ def join_usage(semantic: SemanticProjection, accounting: AccountingProjection) - continue if interaction is None: - result["evidence"].extend( - [ - f"turn_index:{usage.get('turn_index')}", - "delayed_row:true", - ] - ) + result["evidence"].append(f"turn_index:{usage.get('turn_index')}") preliminary.append((result, None)) continue - possible = [ + same_turn = [ call for call in calls - if (_root_for(call, calls) or call).get("interaction_id") == interaction - and call.get("agent_id") == usage.get("agent_id") + if (_root_candidate(call, by_tool) or call).get("interaction_id") == interaction + ] + possible = [ + call + for call in same_turn + if call.get("agent_id") == usage.get("agent_id") and call.get("parent_tool_call_id") == usage.get("parent_tool_call_id") and call.get("model") == usage.get("model") and _finish_matches(usage, call) @@ -251,14 +264,13 @@ def join_usage(semantic: SemanticProjection, accounting: AccountingProjection) - [ f"interaction_id:{interaction}", f"turn_index:{usage.get('turn_index')}", - "delayed_row:true", ] ) preliminary.append((result, None)) continue call = possible[0] - order = calls.index(call) + order = _identity_group(call, calls).index(call) initiator = (usage.get("supplemental_metrics") or {}).get("initiator") result.update( call_id=call["call_id"], @@ -291,6 +303,9 @@ def join_usage(semantic: SemanticProjection, accounting: AccountingProjection) - call_id=None, status="conflicting", join_kind=None, - evidence=[f"call_id:{call_id}", "join_conflict:multiple_usage_rows"], + evidence=[ + f"logical_call_id:{entry['logical_call_id']}", + f"call_id:{call_id}", + ], ) return [attribution for attribution, _ in preliminary] diff --git a/src/thirdeye/platforms/copilot/projection.py b/src/thirdeye/platforms/copilot/projection.py index e0273bd..8726e23 100644 --- a/src/thirdeye/platforms/copilot/projection.py +++ b/src/thirdeye/platforms/copilot/projection.py @@ -11,11 +11,24 @@ from .usage import build_accounting -def _find_turn(turns: list[dict[str, Any]], turn_id: str) -> dict[str, Any] | None: +def _unwrap_semantic_state(prior_state: dict[str, Any]) -> dict[str, Any]: + """Accept flat tracing state or a nested ProjectionState envelope.""" + + if isinstance(prior_state.get("open_interactions"), dict): + return prior_state + nested = prior_state.get("semantic_state") + if isinstance(nested, dict): + return nested + return {} + + +def _find_turn(turns: list[dict[str, Any]], identity: str) -> dict[str, Any] | None: for turn in turns: - if turn.get("turn_id") == turn_id: + if turn.get("turn_id") == identity: + return turn + if any(call.get("call_id") == identity for call in turn.get("llm_calls") or []): return turn - found = _find_turn(turn.get("subagents") or [], turn_id) + found = _find_turn(turn.get("subagents") or [], identity) if found is not None: return found return None @@ -44,7 +57,7 @@ def build_projection( ) -> tuple[Projection, dict[str, Any]]: """Build a local projection without mutating either source projection.""" - semantic, semantic_state = build_semantics(records, prior_state) + semantic, semantic_state = build_semantics(records, _unwrap_semantic_state(prior_state)) accounting, accounting_state = build_accounting(records, prior_state) attributions = join_usage(semantic, accounting) turns = deepcopy(semantic["turns"]) @@ -56,23 +69,19 @@ def build_projection( row = rows.get(attribution["logical_call_id"]) accounting_call = _accounting_call(attribution, row) if attribution["status"] == "matched": - inferred = attribution["join_kind"] == "inferred" - diagnostics.append( - { - "code": "inferred_join" if inferred else "capability_gap", - "severity": "info", - "message": ( - "usage joined by uniquely consistent evidence" - if inferred - else "usage joined to an assistant call" - ), - "source_ids": [attribution["usage_source_id"]], - "details": { - "logical_call_id": attribution["logical_call_id"], - "call_id": attribution["call_id"], - }, - } - ) + if attribution["join_kind"] == "inferred": + diagnostics.append( + { + "code": "inferred_join", + "severity": "info", + "message": "usage joined by uniquely consistent evidence", + "source_ids": [attribution["usage_source_id"]], + "details": { + "logical_call_id": attribution["logical_call_id"], + "call_id": attribution["call_id"], + }, + } + ) else: pending.append( { @@ -83,9 +92,13 @@ def build_projection( "evidence": list(attribution["evidence"]), } ) - if accounting_call is None or attribution["stored_turn_id"] is None: + if accounting_call is None: continue - owner = _find_turn(turns, attribution["stored_turn_id"]) + owner = None + if attribution["status"] == "matched" and attribution["call_id"] is not None: + owner = _find_turn(turns, attribution["call_id"]) + elif attribution["stored_turn_id"] is not None: + owner = _find_turn(turns, attribution["stored_turn_id"]) if owner is not None: owner.setdefault("accounting_calls", []).append(accounting_call) diff --git a/tests/platforms/copilot/test_attribution.py b/tests/platforms/copilot/test_attribution.py index 452f653..50a9919 100644 --- a/tests/platforms/copilot/test_attribution.py +++ b/tests/platforms/copilot/test_attribution.py @@ -42,6 +42,9 @@ CALL_B = ( f"copilot:call:{SOURCE_KEY}/{NATIVE_SESSION_ID}/33cc6465-29e1-4a04-8bdb-00241474b4d2" ) +TS_CYCLE_0 = "2026-09-10T17:08:24.503Z" +TS_CYCLE_1 = "2026-09-10T17:08:25.624Z" +INTERACTION_TWO = "793d3703-6f4a-4814-8877-34a7325848ce" def _load_json(path: Path) -> Any: @@ -106,6 +109,31 @@ def _substitute_source_key(value: str, source_key: str) -> str: return value.replace(SOURCE_KEY, source_key) +def _rewrite_source_key(value: Any, source_key: str) -> Any: + if isinstance(value, str): + return _substitute_source_key(value, source_key) + if isinstance(value, list): + return [_rewrite_source_key(item, source_key) for item in value] + if isinstance(value, dict): + return {key: _rewrite_source_key(item, source_key) for key, item in value.items()} + return value + + +def _iter_turns(turns: list[dict[str, Any]]) -> list[dict[str, Any]]: + found: list[dict[str, Any]] = [] + for turn in turns: + found.append(turn) + found.extend(_iter_turns(turn.get("subagents") or [])) + return found + + +def _accounting_calls(turns: list[dict[str, Any]]) -> list[dict[str, Any]]: + found: list[dict[str, Any]] = [] + for turn in _iter_turns(turns): + found.extend(turn.get("accounting_calls") or []) + return found + + def _align_attribution(expected: dict[str, Any], source_key: str) -> dict[str, Any]: aligned = copy.deepcopy(expected) for field in ("call_id", "stored_turn_id", "logical_call_id", "usage_source_id"): @@ -157,6 +185,8 @@ def _call_candidate( tool_call_ids: list[str] | None = None, finish_evidence: list[dict[str, Any]] | None = None, assistant_message_id: str | None = None, + start_ts: str = TS_CYCLE_0, + end_ts: str = "2026-09-10T17:08:24.593Z", ) -> CallCandidate: candidate: CallCandidate = { "call_id": call_id, @@ -167,8 +197,8 @@ def _call_candidate( "model": model, "source_ids": [call_id.rsplit("/", 1)[-1]], "source_references": [], - "start_ts": "2026-09-10T17:08:24.503Z", - "end_ts": "2026-09-10T17:08:24.593Z", + "start_ts": start_ts, + "end_ts": end_ts, "tool_call_ids": tool_call_ids or [], "finish_evidence": finish_evidence or [], } @@ -339,7 +369,9 @@ def test_attribution_fixture_pending_without_matching_call(): assert attributions[0]["status"] == "pending" assert attributions[0]["call_id"] is None assert attributions[0]["stored_turn_id"] == examples["pending"]["stored_turn_id"] - assert "delayed_row:true" in attributions[0]["evidence"] + assert "delayed_row:true" not in attributions[0]["evidence"] + assert "turn_index:0" in attributions[0]["evidence"] + assert f"interaction_id:{INTERACTION_ONE}" in attributions[0]["evidence"] # --- direct / native joins --- @@ -358,6 +390,8 @@ def test_direct_join_wins_over_inferred_evidence(): call_id=CALL_B, tool_call_ids=[], finish_evidence=[{"source_id": "finish"}], + start_ts=TS_CYCLE_1, + end_ts=TS_CYCLE_1, ), ] ) @@ -385,7 +419,9 @@ def test_competing_direct_ids_are_conflicting(): attributions = join_usage(semantic, accounting) assert attributions[0]["status"] == "conflicting" assert attributions[0]["call_id"] is None - assert "join_conflict:competing_direct_ids" in attributions[0]["evidence"] + assert f"call_id:{CALL_A}" in attributions[0]["evidence"] + assert f"call_id:{CALL_B}" in attributions[0]["evidence"] + assert not any(item.startswith("join_conflict:") for item in attributions[0]["evidence"]) # --- inferred / ambiguous / pending --- @@ -402,6 +438,8 @@ def test_inferred_join_requires_unique_consistent_evidence(): call_id=CALL_B, tool_call_ids=[], finish_evidence=[{"source_id": "finish-b"}], + start_ts=TS_CYCLE_1, + end_ts=TS_CYCLE_1, ), ] ) @@ -428,7 +466,7 @@ def test_ambiguous_when_multiple_calls_fit_same_evidence(): semantic = _semantic( calls=[ _call_candidate(call_id=CALL_A, tool_call_ids=["tool-a"]), - _call_candidate(call_id=CALL_B, tool_call_ids=["tool-b"]), + _call_candidate(call_id=CALL_B, tool_call_ids=["tool-b"], start_ts=TS_CYCLE_1), ] ) accounting = _accounting( @@ -450,7 +488,10 @@ def test_pending_when_turn_index_has_no_main_interaction(): ) attributions = join_usage(semantic, accounting) assert attributions[0]["status"] == "pending" - assert "delayed_row:true" in attributions[0]["evidence"] + assert attributions[0]["stored_turn_id"] is None + assert attributions[0]["call_id"] is None + assert "turn_index:99" in attributions[0]["evidence"] + assert "delayed_row:true" not in attributions[0]["evidence"] def test_late_row_case_stays_pending_until_semantics_catch_up(): @@ -460,7 +501,8 @@ def test_late_row_case_stays_pending_until_semantics_catch_up(): assert attribution["status"] == "pending" assert attribution["call_id"] is None assert attribution["logical_call_id"] == case["expected"]["attributions"][0]["logical_call_id"] - assert "delayed_row:true" in attribution["evidence"] + assert "delayed_row:true" not in attribution["evidence"] + assert "turn_index:0" in attribution["evidence"] assert len(projection["usage_rows"]) == 1 @@ -513,6 +555,8 @@ def test_two_usage_rows_claiming_one_call_become_conflicting(): call_id=CALL_B, tool_call_ids=[], finish_evidence=[{"source_id": "finish-b"}], + start_ts=TS_CYCLE_1, + end_ts=TS_CYCLE_1, ), ] ) @@ -525,7 +569,11 @@ def test_two_usage_rows_claiming_one_call_become_conflicting(): attributions = join_usage(semantic, accounting) assert all(item["status"] == "conflicting" for item in attributions) assert all(item["call_id"] is None for item in attributions) - assert all("join_conflict:multiple_usage_rows" in item["evidence"] for item in attributions) + assert all(f"call_id:{CALL_A}" in item["evidence"] for item in attributions) + assert all( + not any(token.startswith("join_conflict:") for token in item["evidence"]) + for item in attributions + ) # --- build_projection composition --- @@ -543,39 +591,294 @@ def test_build_projection_attaches_accounting_calls_without_mutating_semantics( assert semantic_before["turns"] == semantic_after["turns"] assert all(not turn.get("accounting_calls") for turn in semantic_before["turns"]) - attached = sum( - len(turn.get("accounting_calls") or []) - for turn in projection["turns"] - ) - assert attached == 6 + attached = _accounting_calls(projection["turns"]) + assert len(attached) == 6 matched = [item for item in projection["attributions"] if item["status"] == "matched"] + by_turn = {turn["turn_id"]: turn for turn in _iter_turns(projection["turns"])} for attribution in matched: owner = next( turn - for turn in projection["turns"] - if turn["turn_id"] == attribution["stored_turn_id"] + for turn in _iter_turns(projection["turns"]) + if any( + item.get("call_id") == attribution["call_id"] + for item in turn.get("accounting_calls") or [] + ) ) - call_ids = [ - item["call_id"] - for item in owner.get("accounting_calls") or [] - if item["attribution_status"] == "matched" + if attribution["agent_id"] is None: + assert owner["turn_id"] == attribution["stored_turn_id"] + else: + assert owner["turn_id"] != attribution["stored_turn_id"] + assert owner in (by_turn[attribution["stored_turn_id"]].get("subagents") or []) + assert attribution["call_id"] in [ + item["call_id"] for item in owner.get("accounting_calls") or [] ] - assert attribution["call_id"] in call_ids + + +def test_matched_child_usage_attaches_to_nested_turn( + cli_transcript_records: list[SourceRecord], +): + source_key = _source_key(cli_transcript_records) + records = cli_transcript_records + _six_call_records(source_key=source_key) + projection, _ = build_projection(records, {}) + child = next( + item for item in projection["attributions"] if item["agent_id"] == CHILD_AGENT_ID + ) + main = next( + turn for turn in projection["turns"] if turn["turn_id"] == child["stored_turn_id"] + ) + nested = main["subagents"][0] + nested_ids = [ + item["call_id"] + for item in nested.get("accounting_calls") or [] + if item["attribution_status"] == "matched" + ] + main_ids = [ + item["call_id"] + for item in main.get("accounting_calls") or [] + if item.get("call_id") + ] + assert child["call_id"] in nested_ids + assert child["call_id"] not in main_ids + assert any( + llm.get("call_id") == child["call_id"] for llm in nested.get("llm_calls") or [] + ) + + +def test_retry_reordering_same_interaction_keeps_call_assignments(): + first = _call_candidate(call_id=CALL_A, tool_call_ids=["tool-a"], start_ts=TS_CYCLE_0) + second = _call_candidate( + call_id=CALL_B, + tool_call_ids=[], + finish_evidence=[{"source_id": "finish-b"}], + start_ts=TS_CYCLE_1, + end_ts=TS_CYCLE_1, + ) + usage_user = _accounting_candidate(row_id=13, finish_reason="tool_calls", initiator="user") + usage_agent = _accounting_candidate(row_id=14, finish_reason="stop", initiator="agent") + forward = join_usage( + _semantic(calls=[first, second]), + _accounting(candidates=[usage_user, usage_agent]), + ) + reversed_calls = join_usage( + _semantic(calls=[second, first]), + _accounting(candidates=[usage_agent, usage_user]), + ) + by_forward = {item["logical_call_id"].rsplit(":", 1)[-1]: item for item in forward} + by_reversed = {item["logical_call_id"].rsplit(":", 1)[-1]: item for item in reversed_calls} + assert by_forward["13"]["call_id"] == by_reversed["13"]["call_id"] == CALL_A + assert by_forward["14"]["call_id"] == by_reversed["14"]["call_id"] == CALL_B + assert "order:0" in by_forward["13"]["evidence"] + assert "order:1" in by_forward["14"]["evidence"] + assert "order:0" in by_reversed["13"]["evidence"] + assert "order:1" in by_reversed["14"]["evidence"] + + +def test_cyclic_child_does_not_consume_turn_index(): + child_a = _call_candidate( + call_id=f"copilot:call:{SOURCE_KEY}/{NATIVE_SESSION_ID}/cycle-a", + stored_turn_id=f"{TURN_ONE}:agent-a", + interaction_id="child-cycle", + agent_id="agent-a", + parent_tool_call_id="tool-b", + tool_call_ids=["tool-a"], + ) + child_b = _call_candidate( + call_id=f"copilot:call:{SOURCE_KEY}/{NATIVE_SESSION_ID}/cycle-b", + stored_turn_id=f"{TURN_ONE}:agent-b", + interaction_id="child-cycle", + agent_id="agent-b", + parent_tool_call_id="tool-a", + tool_call_ids=["tool-b"], + start_ts=TS_CYCLE_1, + ) + main = _call_candidate(call_id=CALL_A, tool_call_ids=["tool-main"]) + attributions = join_usage( + _semantic(calls=[child_a, child_b, main]), + _accounting(candidates=[_accounting_candidate(row_id=13, turn_index=0)]), + ) + assert attributions[0]["stored_turn_id"] == TURN_ONE + assert attributions[0]["call_id"] == CALL_A + assert attributions[0]["status"] == "matched" + + +def test_non_unique_parent_tool_does_not_consume_turn_index(): + child = _call_candidate( + call_id=f"copilot:call:{SOURCE_KEY}/{NATIVE_SESSION_ID}/orphan-child", + stored_turn_id=f"{TURN_ONE}:agent-a", + interaction_id="child-orphan", + agent_id="agent-a", + parent_tool_call_id="shared-tool", + tool_call_ids=["child-tool"], + ) + main = _call_candidate(call_id=CALL_A, tool_call_ids=["shared-tool"]) + other = _call_candidate( + call_id=CALL_B, + stored_turn_id=f"copilot:turn:{SOURCE_KEY}:{NATIVE_SESSION_ID}:{INTERACTION_TWO}", + interaction_id=INTERACTION_TWO, + tool_call_ids=["shared-tool"], + start_ts=TS_CYCLE_1, + ) + attributions = join_usage( + _semantic(calls=[child, main, other]), + _accounting(candidates=[_accounting_candidate(row_id=13, turn_index=0)]), + ) + assert attributions[0]["stored_turn_id"] == TURN_ONE + assert attributions[0]["call_id"] == CALL_A + assert attributions[0]["status"] == "matched" + + +def test_partial_turn_incremental_state_matches_full_archive(): + cases = _load_json(RECON / "cases.json") + partial = cases["partial_turn"]["input_records"] + later = cases["abort"]["input_records"] + full = partial + later + first, state = build_projection(partial, {}) + assert "6d2b89fd-a653-430c-b532-b0936d72eb42|main" in state["semantic_state"][ + "open_interactions" + ] + incremental, inc_state = build_projection(later, state) + from_flat, flat_state = build_projection(later, state["semantic_state"]) + complete, full_state = build_projection(full, {}) + assert incremental["turns"] == complete["turns"] == from_flat["turns"] + assert ( + inc_state["semantic_state"]["open_interactions"].keys() + == full_state["semantic_state"]["open_interactions"].keys() + == flat_state["semantic_state"]["open_interactions"].keys() + ) + assert "6d2b89fd-a653-430c-b532-b0936d72eb42|main" in inc_state["semantic_state"][ + "open_interactions" + ] + assert first["turns"] == [] + + +def test_direct_match_does_not_emit_join_capability_gap( + cli_transcript_records: list[SourceRecord], +): + source_key = _source_key(cli_transcript_records) + records = cli_transcript_records + _six_call_records(source_key=source_key) + projection, _ = build_projection(records, {}) + join_gaps = [ + item + for item in projection["diagnostics"] + if item["code"] == "capability_gap" and item.get("details", {}).get("logical_call_id") + ] + assert join_gaps == [] def test_accounting_rows_persist_when_attribution_is_ambiguous(): - semantic = _semantic( - calls=[ - _call_candidate(call_id=CALL_A, tool_call_ids=["tool-a"]), - _call_candidate(call_id=CALL_B, tool_call_ids=["tool-b"]), - ] + user = { + "source_id": f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/amb-user", + "source_kind": "transcript", + "native_session_id": NATIVE_SESSION_ID, + "ts": "2026-09-10T17:08:22.203Z", + "observed_at": OBSERVED_AT, + "payload": { + "type": "user.message", + "data": { + "content": "two similar cycles", + "interactionId": INTERACTION_ONE, + "turnId": "0", + }, + "id": "amb-user", + "timestamp": "2026-09-10T17:08:22.203Z", + "schema_version": 1, + }, + "locator": {"file": "events.jsonl", "native_event_id": "amb-user"}, + } + first_msg = { + "source_id": f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/a4a17e63-7ba5-422f-8ee9-b495be417328", + "source_kind": "transcript", + "native_session_id": NATIVE_SESSION_ID, + "ts": TS_CYCLE_0, + "observed_at": OBSERVED_AT, + "payload": { + "type": "assistant.message", + "data": { + "content": "", + "model": "gpt-5.6-luna", + "interactionId": INTERACTION_ONE, + "turnId": "0", + "toolRequests": [{"toolCallId": "tool-a", "name": "view", "arguments": {}}], + }, + "id": "a4a17e63-7ba5-422f-8ee9-b495be417328", + "timestamp": TS_CYCLE_0, + "schema_version": 1, + }, + "locator": { + "file": "events.jsonl", + "native_event_id": "a4a17e63-7ba5-422f-8ee9-b495be417328", + }, + } + first_end = { + "source_id": f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/amb-end-0", + "source_kind": "transcript", + "native_session_id": NATIVE_SESSION_ID, + "ts": "2026-09-10T17:08:24.593Z", + "observed_at": OBSERVED_AT, + "payload": { + "type": "assistant.turn_end", + "data": {"turnId": "0"}, + "id": "amb-end-0", + "timestamp": "2026-09-10T17:08:24.593Z", + "schema_version": 1, + }, + "locator": {"file": "events.jsonl", "native_event_id": "amb-end-0"}, + } + second_msg = { + "source_id": f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/33cc6465-29e1-4a04-8bdb-00241474b4d2", + "source_kind": "transcript", + "native_session_id": NATIVE_SESSION_ID, + "ts": TS_CYCLE_1, + "observed_at": OBSERVED_AT, + "payload": { + "type": "assistant.message", + "data": { + "content": "", + "model": "gpt-5.6-luna", + "interactionId": INTERACTION_ONE, + "turnId": "1", + "toolRequests": [{"toolCallId": "tool-b", "name": "view", "arguments": {}}], + }, + "id": "33cc6465-29e1-4a04-8bdb-00241474b4d2", + "timestamp": TS_CYCLE_1, + "schema_version": 1, + }, + "locator": { + "file": "events.jsonl", + "native_event_id": "33cc6465-29e1-4a04-8bdb-00241474b4d2", + }, + } + second_end = { + "source_id": f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/amb-end-1", + "source_kind": "transcript", + "native_session_id": NATIVE_SESSION_ID, + "ts": "2026-09-10T17:08:25.700Z", + "observed_at": OBSERVED_AT, + "payload": { + "type": "assistant.turn_end", + "data": {"turnId": "1"}, + "id": "amb-end-1", + "timestamp": "2026-09-10T17:08:25.700Z", + "schema_version": 1, + }, + "locator": {"file": "events.jsonl", "native_event_id": "amb-end-1"}, + } + row = copy.deepcopy(_load_json(FIXTURES / "assistant-usage-events.json")[0]) + row["finish_reason"] = None + row["initiator"] = None + usage = _usage_record( + row, + content_revision="sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", ) - accounting = _accounting( - candidates=[_accounting_candidate(row_id=13, finish_reason=None, initiator=None)] + ambiguous, _ = build_projection( + [user, first_msg, first_end, second_msg, second_end, usage], {} ) - attributions = join_usage(semantic, accounting) - assert attributions[0]["status"] == "ambiguous" - assert len(accounting["candidates"]) == 1 + matched, _ = build_projection([user, first_msg, first_end, usage], {}) + assert ambiguous["attributions"][0]["status"] == "ambiguous" + assert matched["attributions"][0]["status"] == "matched" + assert len(ambiguous["usage_rows"]) == len(matched["usage_rows"]) == 1 + assert _metric_totals(ambiguous["usage_rows"]) == _metric_totals(matched["usage_rows"]) + assert ambiguous["usage_rows"][0].input_tokens == 6452 def test_late_row_projection_keeps_usage_rows_while_attribution_pending(): @@ -586,6 +889,38 @@ def test_late_row_projection_keeps_usage_rows_while_attribution_pending(): assert projection["usage_rows"][0].input_tokens == 6587 +def test_unmatched_usage_with_known_turn_stays_on_main_without_call_id(): + abort = _load_json(RECON / "cases.json")["abort"]["input_records"] + row = copy.deepcopy(_load_json(FIXTURES / "assistant-usage-events.json")[0]) + row["model"] = "not-the-transcript-model" + usage = _usage_record( + row, + content_revision="sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + ) + projection, _ = build_projection(abort + [usage], {}) + assert projection["attributions"][0]["status"] == "pending" + assert projection["attributions"][0]["stored_turn_id"] is not None + assert projection["attributions"][0]["call_id"] is None + attached = _accounting_calls(projection["turns"]) + assert len(attached) == 1 + assert attached[0]["call_id"] is None + assert attached[0]["attribution_status"] == "pending" + owner = next(turn for turn in projection["turns"] if turn.get("accounting_calls")) + assert owner["turn_id"] == projection["attributions"][0]["stored_turn_id"] + + +def test_pending_usage_without_stored_turn_skips_accounting_calls(): + records = _six_call_records()[:1] + projection, _ = build_projection(records, {}) + assert projection["attributions"][0]["stored_turn_id"] is None + assert projection["attributions"][0]["status"] == "pending" + assert len(projection["usage_rows"]) == 1 + assert projection["turns"] == [] + assert _accounting_calls(projection["turns"]) == [] + pending = [item for item in projection["pending"] if item["kind"] == "unmatched_usage"] + assert len(pending) == 1 + + def test_matched_inferred_usage_emits_diagnostic(cli_transcript_records: list[SourceRecord]): source_key = _source_key(cli_transcript_records) records = cli_transcript_records + _six_call_records(source_key=source_key) @@ -607,11 +942,32 @@ def test_source_correction_replaces_attribution_target_after_replay( cli_transcript_records: list[SourceRecord], ): source_key = _source_key(cli_transcript_records) - records = cli_transcript_records + _six_call_records(source_key=source_key) - first, state = build_projection(records, {}) - second, _ = build_projection(records, state) - assert first["attributions"] == second["attributions"] - assert len(first["usage_rows"]) == len(second["usage_rows"]) == 6 + original = cli_transcript_records + _six_call_records(source_key=source_key) + first, state = build_projection(original, {}) + original_row = next(row for row in first["usage_rows"] if row.call_id.endswith(":13")) + original_join = next( + item for item in first["attributions"] if item["logical_call_id"] == original_row.call_id + ) + corrected = _rewrite_source_key( + copy.deepcopy(_load_json(RECON / "cases.json")["revision"]["input_records"][1]), + source_key, + ) + second, _ = build_projection(original + [corrected], state) + assert len(second["usage_rows"]) == 6 + replaced = next(row for row in second["usage_rows"] if row.call_id == original_row.call_id) + joined = next( + item for item in second["attributions"] if item["logical_call_id"] == original_row.call_id + ) + assert replaced.output_tokens == 999 + assert original_row.output_tokens != 999 + assert joined["call_id"] == original_join["call_id"] + assert joined["usage_source_id"] == corrected["source_id"] + assert joined["logical_call_id"] == original_join["logical_call_id"] + assert _metric_totals(second["usage_rows"])["output_tokens"] == ( + _metric_totals(first["usage_rows"])["output_tokens"] + - original_row.output_tokens + + 999 + ) @pytest.fixture From e6781e59786386db2f663ea05f1cf28e059fb5aa Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Fri, 11 Sep 2026 20:56:51 -0700 Subject: [PATCH 68/88] Add Copilot export assembly ledger --- src/thirdeye/platforms/copilot/export.py | 339 ++++++++++++++++++ .../platforms/copilot/export_state.py | 250 +++++++++++++ 2 files changed, 589 insertions(+) create mode 100644 src/thirdeye/platforms/copilot/export.py create mode 100644 src/thirdeye/platforms/copilot/export_state.py diff --git a/src/thirdeye/platforms/copilot/export.py b/src/thirdeye/platforms/copilot/export.py new file mode 100644 index 0000000..8ed874b --- /dev/null +++ b/src/thirdeye/platforms/copilot/export.py @@ -0,0 +1,339 @@ +"""Assemble eligible Copilot projections into generic detached OTel jobs. + +No network operation happens here. This module only writes/dispatches the +generic transport's local jobs; the detached worker performs remote delivery. +The split cannot provide transactional exactly-once delivery: a crash after a +remote flush and before acknowledgement can retry a deterministic span. +""" + +from __future__ import annotations + +from copy import deepcopy +from pathlib import Path +from typing import Any + +from thirdeye import otel_export +from thirdeye.config import Config +from thirdeye.meta import read_meta +from thirdeye.paths import meta_path, session_dir +from thirdeye.span_ids import chat_span_id + +from .constants import PLATFORM_NAME +from .export_state import ( + initialize_eligibility, + is_accounting_eligible, + is_turn_eligible, + mark_placement_error, + record_placement, + update_export_state, +) +from .types import Projection + +_TERMINAL = frozenset({"completed", "interrupted", "errored"}) +_EXPORTABLE_ATTRIBUTIONS = frozenset({"matched", "ambiguous"}) + + +def _directory(config: Config, stored_session_id: str) -> Path: + return session_dir(config.root, PLATFORM_NAME, stored_session_id) + + +def _terminal(turn: dict[str, Any]) -> bool: + return turn.get("status") in _TERMINAL + + +def _walk_turns(turns: list[dict[str, Any]]) -> list[dict[str, Any]]: + found: list[dict[str, Any]] = [] + for turn in turns: + found.append(turn) + children = turn.get("subagents") + if isinstance(children, list): + found.extend(_walk_turns([child for child in children if isinstance(child, dict)])) + return found + + +def _main_terminal_turns(projection: Projection) -> list[dict[str, Any]]: + return [ + turn + for turn in projection["turns"] + if isinstance(turn, dict) + and _terminal(turn) + and not isinstance((turn.get("attributes") or {}).get("agent_id"), str) + ] + + +def _accounting_calls(projection: Projection) -> dict[str, tuple[dict[str, Any], dict[str, Any]]]: + """Map an identity to its owner turn and serialized generic accounting.""" + found: dict[str, tuple[dict[str, Any], dict[str, Any]]] = {} + for turn in _walk_turns([turn for turn in projection["turns"] if isinstance(turn, dict)]): + for item in turn.get("accounting_calls") or []: + if not isinstance(item, dict): + continue + accounting_id = item.get("accounting_id") + usage = item.get("usage") + if isinstance(accounting_id, str) and isinstance(usage, dict): + found[accounting_id] = (turn, item) + return found + + +def _usage_ids(projection: Projection) -> list[str]: + ids = [row.call_id for row in projection["usage_rows"]] + ids.extend(item["logical_call_id"] for item in projection["attributions"]) + return sorted({item for item in ids if isinstance(item, str) and item}) + + +def _span_id(session_id: str, turn_id: str | None, accounting_id: str) -> str: + if turn_id is None: + return f"accounting:{session_id}:{accounting_id}" + return f"accounting:{session_id}:{turn_id}:{accounting_id}" + + +def _turn_span_id(stored_session_id: str, turn_id: str) -> str: + """Return the OTel-safe deterministic span id for a source-derived turn.""" + return str(chat_span_id(PLATFORM_NAME, stored_session_id, f"turn:{turn_id}")) + + +def _configured(config: Config) -> bool: + return config.logfire.enabled and bool(config.logfire.token) + + +def _error_state(config: Config, stored_session_id: str, accounting_id: str, message: str) -> None: + def update(state: dict[str, Any]) -> dict[str, Any]: + return mark_placement_error(state, accounting_id, message) + + update_export_state(config, stored_session_id, update) + + +def _eligible_state( + config: Config, + stored_session_id: str, + projection: Projection, + *, + include_history: bool, +) -> dict[str, Any]: + terminal_ids = [str(turn["turn_id"]) for turn in _main_terminal_turns(projection)] + + def update(state: dict[str, Any]) -> dict[str, Any]: + return initialize_eligibility( + state, + terminal_turn_ids=terminal_ids, + accounting_ids=_usage_ids(projection), + include_history=include_history, + ) + + return update_export_state(config, stored_session_id, update) + + +def _place( + config: Config, + stored_session_id: str, + *, + accounting_id: str, + destination: str, + span_id: str, + usage: dict[str, Any], +) -> tuple[dict[str, Any] | None, bool]: + captured: dict[str, Any] = {} + + def update(state: dict[str, Any]) -> dict[str, Any]: + next_state, entry, accepted = record_placement( + state, + accounting_id=accounting_id, + destination=destination, + span_id=span_id, + usage=usage, + ) + captured["entry"] = entry + captured["accepted"] = accepted + return next_state + + update_export_state(config, stored_session_id, update) + return captured.get("entry"), bool(captured.get("accepted")) + + +def _turn_with_placed_accounting( + turn: dict[str, Any], placements: dict[str, dict[str, Any]] +) -> dict[str, Any]: + """Return a recursive turn copy containing only its chat-placed tokens.""" + result = deepcopy(turn) + calls = [] + for accounting in result.get("accounting_calls") or []: + entry = placements.get(str(accounting.get("accounting_id"))) + if entry and entry.get("destination") == "chat-span": + calls.append(accounting) + result["accounting_calls"] = calls + result["subagents"] = [ + _turn_with_placed_accounting(child, placements) + for child in result.get("subagents") or [] + if isinstance(child, dict) + ] + return result + + +def _with_deterministic_turn_ids( + turn: dict[str, Any], stored_session_id: str +) -> dict[str, Any]: + """Finalize a complete turn tree with ids stable across archive replays. + + Copilot's stable turn identity is a source-derived string rather than the + integer sequence used by the older live adapters. ``chat_span_id`` is a + domain-separated deterministic 64-bit id function, and the ``turn:`` + prefix keeps this turn namespace distinct from model-call ids. + """ + result = deepcopy(turn) + turn_id = result.get("turn_id") + if isinstance(turn_id, str) and turn_id: + result["turn_span_id"] = _turn_span_id(stored_session_id, turn_id) + result["subagents"] = [ + _with_deterministic_turn_ids(child, stored_session_id) + for child in result.get("subagents") or [] + if isinstance(child, dict) + ] + return result + + +def queue_exports( + config: Config, + stored_session_id: str, + projection: Projection, + *, + include_history: bool = False, +) -> int: + """Queue eligible terminal projection evidence without reading live Copilot data. + + ``include_history`` is the explicit ``--export`` opt-in supplied by + runtime composition. The default first activation records all currently + terminal evidence as local-only. Subsequent terminal interactions and + later accounting rows are eligible. The returned count is local jobs + accepted for dispatch, not confirmed remote delivery. + """ + directory = _directory(config, stored_session_id) + meta = read_meta(meta_path(directory)) + if meta is None: + raise ValueError(f"unknown Copilot session: {stored_session_id}") + state = _eligible_state( + config, stored_session_id, projection, include_history=include_history + ) + if not _configured(config): + return 0 + + accounting = _accounting_calls(projection) + placements: dict[str, dict[str, Any]] = {} + queued = 0 + + # Place terminal owned accounting first. This decision happens before a + # whole-turn job is assembled, so a future local match cannot move a + # fallback charge onto a chat span. + for accounting_id, (owner, item) in accounting.items(): + owner_id = owner.get("turn_id") + usage = item["usage"] + if ( + not isinstance(owner_id, str) + or not _terminal(owner) + or item.get("attribution_status") not in _EXPORTABLE_ATTRIBUTIONS + ): + continue + if not is_turn_eligible(state, owner_id) or not is_accounting_eligible(state, accounting_id): + continue + call_id = item.get("call_id") + if item.get("attribution_status") == "matched" and isinstance(call_id, str) and call_id: + destination = "chat-span" + span_id = str(call_id) + else: + destination = "turn-accounting-span" + span_id = _span_id(stored_session_id, owner_id, accounting_id) + entry, accepted = _place( + config, + stored_session_id, + accounting_id=accounting_id, + destination=destination, + span_id=span_id, + usage=usage, + ) + if not accepted: + continue + if entry is not None: + placements[accounting_id] = entry + if entry is not None and entry.get("emitted"): + # ``emitted`` means a delivery acknowledgement, unlike a queued + # job. Never recreate an accounting job after that point. + continue + if destination != "turn-accounting-span" or entry is None: + continue + sent = otel_export.export_turn_accounting( + config, + directory, + stored_session_id, + PLATFORM_NAME, + meta.cwd, + owner_id, + item, + turn_span_id=_turn_span_id(stored_session_id, owner_id), + ) + if sent: + queued += 1 + else: + _error_state(config, stored_session_id, accounting_id, "accounting job was not queued") + + # Usage with no known user-turn owner is intentionally a session accounting + # job. It never manufactures a prompt/turn merely to satisfy tracing. + rows = {row.call_id: row.to_dict() for row in projection["usage_rows"]} + attached_ids = set(accounting) + for attribution in projection["attributions"]: + accounting_id = attribution["logical_call_id"] + if attribution["status"] not in _EXPORTABLE_ATTRIBUTIONS: + # Pending attribution is intentionally local-only until later + # archive evidence resolves it. A conflicting join is likewise + # quarantined rather than guessed into an accounting span. + continue + if accounting_id in attached_ids or not is_accounting_eligible(state, accounting_id): + continue + usage = rows.get(accounting_id) + if usage is None: + continue + span_id = _span_id(stored_session_id, None, accounting_id) + entry, accepted = _place( + config, + stored_session_id, + accounting_id=accounting_id, + destination="session-accounting-span", + span_id=span_id, + usage=usage, + ) + if not accepted or entry is None: + continue + if entry.get("emitted"): + continue + item = { + "accounting_id": accounting_id, + "usage": usage, + "attribution_status": attribution["status"], + "agent_id": attribution["agent_id"], + "call_id": None, + "attributes": { + "logical_call_id": accounting_id, + "usage_source_id": attribution["usage_source_id"], + "evidence": list(attribution["evidence"]), + }, + } + sent = otel_export.export_session_accounting( + config, directory, stored_session_id, PLATFORM_NAME, meta.cwd, item + ) + if sent: + queued += 1 + else: + _error_state(config, stored_session_id, accounting_id, "accounting job was not queued") + + # Generic transport provides a persistent completed-turn claim. Sending + # a duplicate local job is harmless there; no network I/O occurs here. + for turn in _main_terminal_turns(projection): + turn_id = str(turn["turn_id"]) + if not is_turn_eligible(state, turn_id): + continue + assembled = _with_deterministic_turn_ids( + _turn_with_placed_accounting(turn, placements), stored_session_id + ) + otel_export.export_turn( + config, directory, stored_session_id, PLATFORM_NAME, meta.cwd, assembled + ) + queued += 1 + return queued diff --git a/src/thirdeye/platforms/copilot/export_state.py b/src/thirdeye/platforms/copilot/export_state.py new file mode 100644 index 0000000..b7211d9 --- /dev/null +++ b/src/thirdeye/platforms/copilot/export_state.py @@ -0,0 +1,250 @@ +"""Durable Copilot export eligibility and accounting-placement state. + +This state intentionally has no relationship to projection state. Projection +state is disposable and rebuilt from the V1 archive; export placement is an +accounting decision and survives rebuilds. The generic OTel worker cannot +atomically acknowledge a remote collector and this file, so an absent job is +never treated as proof of delivery. A crash after a remote flush can still +lead to a deterministic retry and therefore a duplicate remote span. +""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +from thirdeye._compat import fsops +from thirdeye._compat.locking import LockMode, locked +from thirdeye.config import Config +from thirdeye.paths import session_dir + +from .constants import PLATFORM_NAME +from .jsonio import atomic_write_json, read_json_object + +EXPORT_STATE_FILENAME = "copilot.export.ledger.json" +EXPORT_LOCK_FILENAME = "copilot.export.lock" +EXPORT_STATE_SCHEMA_VERSION = 1 + + +def export_state_path(directory: Path) -> Path: + return directory / EXPORT_STATE_FILENAME + + +def export_lock_path(directory: Path) -> Path: + return directory / EXPORT_LOCK_FILENAME + + +def _directory(config: Config, stored_session_id: str) -> Path: + return session_dir(config.root, PLATFORM_NAME, stored_session_id) + + +def empty_export_state() -> dict[str, Any]: + """Return the versioned state used before the first V2 export activation.""" + return { + "schema_version": EXPORT_STATE_SCHEMA_VERSION, + "activated": False, + "excluded_turn_ids": [], + "excluded_accounting_ids": [], + "placements": {}, + "conflicts": {}, + } + + +def _ids(values: object) -> list[str]: + if not isinstance(values, list): + return [] + return sorted({value for value in values if isinstance(value, str) and value}) + + +def _mapping(value: object) -> dict[str, Any]: + return value if isinstance(value, dict) else {} + + +def _state(value: dict[str, Any] | None) -> dict[str, Any]: + if value is None or value.get("schema_version") != EXPORT_STATE_SCHEMA_VERSION: + return empty_export_state() + placements = _mapping(value.get("placements")) + conflicts = _mapping(value.get("conflicts")) + return { + "schema_version": EXPORT_STATE_SCHEMA_VERSION, + "activated": bool(value.get("activated", False)), + "excluded_turn_ids": _ids(value.get("excluded_turn_ids")), + "excluded_accounting_ids": _ids(value.get("excluded_accounting_ids")), + "placements": {key: item for key, item in placements.items() if isinstance(key, str) and isinstance(item, dict)}, + "conflicts": {key: item for key, item in conflicts.items() if isinstance(key, str) and isinstance(item, dict)}, + } + + +def _read(directory: Path) -> dict[str, Any]: + try: + return _state( + read_json_object( + export_state_path(directory), invalid_message="invalid Copilot export ledger" + ) + ) + except ValueError: + # Export accounting is not disposable. Do not overwrite a corrupt + # ledger and risk emitting tokens at another location. + raise ValueError("invalid Copilot export ledger") from None + + +def _write(directory: Path, state: dict[str, Any]) -> None: + atomic_write_json(export_state_path(directory), state) + + +def load_export_state(config: Config, stored_session_id: str) -> dict[str, Any]: + """Read a snapshot of the independent export ledger. + + Callers that mutate the result must use :func:`update_export_state`; this + helper intentionally returns a detached JSON-safe dictionary. + """ + directory = _directory(config, stored_session_id) + with locked(export_lock_path(directory), LockMode.SHARED): + return json.loads(json.dumps(_read(directory))) + + +def update_export_state( + config: Config, stored_session_id: str, update: Any +) -> dict[str, Any]: + """Atomically apply ``update(state)`` and return the published state.""" + directory = _directory(config, stored_session_id) + with locked(export_lock_path(directory), LockMode.EXCLUSIVE): + state = _read(directory) + updated = update(state) + if not isinstance(updated, dict): + raise TypeError("Copilot export state update must return a dictionary") + state = _state(updated) + _write(directory, state) + return json.loads(json.dumps(state)) + + +def usage_digest(usage: dict[str, Any]) -> str: + """Stable correction detector for a serialized ``UsageRow``.""" + encoded = json.dumps(usage, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + return "sha256:" + hashlib.sha256(encoded.encode("utf-8")).hexdigest() + + +def is_turn_eligible(state: dict[str, Any], turn_id: str) -> bool: + return turn_id not in set(_ids(state.get("excluded_turn_ids"))) + + +def is_accounting_eligible(state: dict[str, Any], accounting_id: str) -> bool: + return accounting_id not in set(_ids(state.get("excluded_accounting_ids"))) + + +def initialize_eligibility( + state: dict[str, Any], + *, + terminal_turn_ids: list[str], + accounting_ids: list[str], + include_history: bool, +) -> dict[str, Any]: + """Set the first-activation boundary without changing any placement. + + Normal activation excludes terminal history. An explicit historical + export opts the supplied completed turns and accounting identities in by + removing them from that boundary. New evidence is absent from both lists + and is therefore eligible after restart. + """ + result = _state(state) + turn_ids = set(_ids(terminal_turn_ids)) + usage_ids = set(_ids(accounting_ids)) + if not result["activated"]: + result["activated"] = True + if not include_history: + result["excluded_turn_ids"] = sorted(turn_ids) + result["excluded_accounting_ids"] = sorted(usage_ids) + return result + if include_history: + result["excluded_turn_ids"] = sorted(set(result["excluded_turn_ids"]) - turn_ids) + result["excluded_accounting_ids"] = sorted( + set(result["excluded_accounting_ids"]) - usage_ids + ) + return result + + +def record_placement( + state: dict[str, Any], + *, + accounting_id: str, + destination: str, + span_id: str, + usage: dict[str, Any], +) -> tuple[dict[str, Any], dict[str, Any] | None, bool]: + """Persist the first token location for an accounting identity. + + Returns ``(state, entry, accepted)``. A source correction or a changed + destination after placement is a durable conflict: silently replacing a + queued job could produce two different token totals at the same span. + """ + result = _state(state) + digest = usage_digest(usage) + placements = result["placements"] + existing = _mapping(placements.get(accounting_id)) + if existing: + same = ( + existing.get("destination") == destination + and existing.get("span_id") == span_id + and existing.get("usage_digest") == digest + ) + if same: + return result, existing, True + result["conflicts"][accounting_id] = { + "accounting_id": accounting_id, + "reason": "accounting placement or usage changed after queueing", + "existing": existing, + "candidate": { + "destination": destination, + "span_id": span_id, + "usage_digest": digest, + }, + } + return result, existing, False + entry = { + "accounting_id": accounting_id, + "destination": destination, + "span_id": span_id, + "usage_digest": digest, + "emitted": False, + "last_error": None, + } + placements[accounting_id] = entry + return result, entry, True + + +def mark_placement_error(state: dict[str, Any], accounting_id: str, error: str) -> dict[str, Any]: + result = _state(state) + entry = _mapping(result["placements"].get(accounting_id)) + if entry: + entry["last_error"] = error + result["placements"][accounting_id] = entry + return result + + +def mark_placement_delivered(state: dict[str, Any], accounting_id: str) -> dict[str, Any]: + """Record an externally confirmed delivery, never inferred from a job file. + + The current detached worker has no transactional callback into this + ledger. This hook is deliberately available for a future confirmed + transport, while ordinary queueing leaves ``emitted`` false. + """ + result = _state(state) + entry = _mapping(result["placements"].get(accounting_id)) + if entry: + entry["emitted"] = True + entry["last_error"] = None + result["placements"][accounting_id] = entry + return result + + +def remove_export_state(config: Config, stored_session_id: str) -> None: + """Remove export state only for an explicit export-ledger reset. + + Projection rebuilds must not call this function. + """ + directory = _directory(config, stored_session_id) + with locked(export_lock_path(directory), LockMode.EXCLUSIVE): + fsops.unlink(export_state_path(directory), missing_ok=True) + fsops.sync_directory(directory) From 85f5085c10c63e13252241b04e2b4d04707a7984 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Fri, 11 Sep 2026 20:57:56 -0700 Subject: [PATCH 69/88] Add Copilot archive reconciliation --- src/thirdeye/platforms/copilot/reconcile.py | 150 ++++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 src/thirdeye/platforms/copilot/reconcile.py diff --git a/src/thirdeye/platforms/copilot/reconcile.py b/src/thirdeye/platforms/copilot/reconcile.py new file mode 100644 index 0000000..c465fb9 --- /dev/null +++ b/src/thirdeye/platforms/copilot/reconcile.py @@ -0,0 +1,150 @@ +"""Archive-only composition for Copilot's local V2 projection. + +Reconciliation deliberately reads the immutable V1 archive rather than the +live Copilot home. That makes it safe to run after a session has ended (or +after Copilot has removed its transient transcript files), and keeps capture +failures independent from derived-state failures. +""" + +from __future__ import annotations + +from importlib import import_module +from typing import Any, Callable + +from thirdeye.config import Config + +from .archive import iter_captured_records +from .projection import build_projection +from .projection_store import ( + commit_projection, + load_projection_state, + reset_projection_state, +) +from .types import Projection + +_COUNT_KEYS = ("events", "usage", "turns", "exports", "pending", "ambiguous", "conflicting") + + +def _empty_result(*, errors: int = 0) -> dict[str, int]: + result = {key: 0 for key in _COUNT_KEYS} + result["errors"] = errors + return result + + +def _stored_counts(state: dict[str, Any]) -> dict[str, int]: + """Return safe status counts when a projection attempt cannot proceed.""" + + result = _empty_result() + totals = state.get("index_totals") + if not isinstance(totals, dict): + return result + for source, target in ( + ("events", "events"), + ("usage", "usage"), + ("turns", "turns"), + ("pending", "pending"), + ): + value = totals.get(source) + if isinstance(value, int) and value >= 0: + result[target] = value + return result + + +def _attribution_counts(projection: Projection) -> tuple[int, int]: + ambiguous = 0 + conflicting = 0 + for attribution in projection["attributions"]: + status = attribution.get("status") + if status == "ambiguous": + ambiguous += 1 + elif status == "conflicting": + conflicting += 1 + return ambiguous, conflicting + + +def _diagnostic_errors(projection: Projection) -> int: + return sum( + 1 + for diagnostic in projection["diagnostics"] + if diagnostic.get("severity") == "error" + ) + + +def queue_exports(config: Config, stored_session_id: str, projection: Projection) -> int: + """Queue export work only when an explicit caller requests it. + + Export assembly is intentionally not an import-time dependency of local + reconciliation. Runtime integration can call this boundary after the + export implementation is installed; archive-only callers never load it. + """ + + module = import_module(".export", package=__package__) + enqueue = getattr(module, "queue_exports") + if not callable(enqueue): + raise TypeError("Copilot export assembly does not provide queue_exports") + exporter: Callable[[Config, str, Projection], int] = enqueue + return exporter(config, stored_session_id, projection) + + +def reconcile_archive( + config: Config, + stored_session_id: str, + *, + rebuild: bool = False, + export: bool = False, +) -> dict[str, int]: + """Rebuild local Copilot projections from the immutable V1 archive. + + A normal reconciliation is a complete archive replay. Stable derived + identities make committing that replay idempotent, while a full replay + keeps late usage rows eligible to join semantic evidence captured in an + earlier pass. ``rebuild`` deletes only reproducible projection state; + raw archive records and the separate export ledger are untouched. + + Errors are reported as counts instead of escaping so source capture can + remain operational when a derived projection is malformed. Export is a + second, opt-in phase performed only after the local commit succeeds. + """ + + try: + # Build a replacement before deleting a previous projection. This is + # especially important for an operator-triggered rebuild: malformed + # archived evidence must not turn a recoverable projection error into + # a loss of the last readable local view. Rebuild input is empty so + # no unfinished incremental state can leak into the full replay. + prior_state = {} if rebuild else load_projection_state(config, stored_session_id) + records = list(iter_captured_records(config, stored_session_id)) + projection, next_state = build_projection(records, prior_state) + next_state["archive_source_ids"] = [record["source_id"] for record in records] + if rebuild: + reset_projection_state(config, stored_session_id) + counts = commit_projection(config, stored_session_id, projection, next_state) + except Exception: + # Do not reset or mutate V1 capture state when derivation fails. + try: + result = _stored_counts(load_projection_state(config, stored_session_id)) + except Exception: + result = _empty_result() + result["errors"] += 1 + return result + + ambiguous, conflicting = _attribution_counts(projection) + result = { + "events": counts["events"], + "usage": counts["usage"], + "turns": counts["turns"], + "exports": 0, + "pending": counts["pending"], + "ambiguous": ambiguous, + "conflicting": conflicting, + "errors": _diagnostic_errors(projection), + } + if not export: + return result + + try: + result["exports"] = queue_exports(config, stored_session_id, projection) + except Exception: + # Export delivery must never roll back a successful local projection. + result["errors"] += 1 + return result From 71067404e64a132918aa8f158a3b68ba2e051928 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Fri, 11 Sep 2026 20:58:54 -0700 Subject: [PATCH 70/88] Add behavioral tests for Copilot export assembly. Cover eligibility boundaries, placement ledger permanence, and queue_exports job dispatch without network I/O. Co-authored-by: Cursor --- tests/platforms/copilot/test_export.py | 854 +++++++++++++++++++++++++ 1 file changed, 854 insertions(+) create mode 100644 tests/platforms/copilot/test_export.py diff --git a/tests/platforms/copilot/test_export.py b/tests/platforms/copilot/test_export.py new file mode 100644 index 0000000..35be70e --- /dev/null +++ b/tests/platforms/copilot/test_export.py @@ -0,0 +1,854 @@ +"""Behavioral tests for Copilot export eligibility, placement ledger, and queue_exports.""" + +from __future__ import annotations + +import json +import shutil +from pathlib import Path +from typing import Any + +import pytest + +from thirdeye import otel_export +from thirdeye.config import Config, LogfireSettings +from thirdeye.paths import session_dir +from thirdeye.platforms.copilot.archive import commit_batch +from thirdeye.platforms.copilot.constants import PLATFORM_NAME +from thirdeye.platforms.copilot.export import queue_exports +from thirdeye.platforms.copilot.export_state import ( + empty_export_state, + export_state_path, + initialize_eligibility, + load_export_state, + mark_placement_delivered, + record_placement, + update_export_state, + usage_digest, +) +from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id +from thirdeye.platforms.copilot.projection import build_projection +from thirdeye.platforms.copilot.transcript import read_transcript +from thirdeye.platforms.copilot.types import Projection, SourceBatch, SourcePaths, SourceRecord +from thirdeye.usage.types import UsageRow + +FIXTURES = Path(__file__).parent / "fixtures" +RECON = FIXTURES / "reconciliation-cases" +NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" +SOURCE_KEY = "a" * 64 +GENERATION = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" +OBSERVED_AT = "2026-09-10T17:09:00.000Z" +TURN_ONE = f"copilot:turn:{SOURCE_KEY}:{NATIVE_SESSION_ID}:interaction-main-1" +CALL_MATCHED = ( + f"copilot:call:{SOURCE_KEY}/{NATIVE_SESSION_ID}/a4a17e63-7ba5-422f-8ee9-b495be417328" +) +ACCOUNTING_MATCHED = ( + f"copilot:usage:{SOURCE_KEY}:assistant_usage_events:" + f"sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13" +) +ACCOUNTING_UNMATCHED = ( + f"copilot:usage:{SOURCE_KEY}:assistant_usage_events:" + f"sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:14" +) + + +def _load_json(path: Path) -> Any: + return json.loads(path.read_text(encoding="utf-8")) + + +def _record(source_id: str, *, native_session_id: str = NATIVE_SESSION_ID) -> SourceRecord: + return { + "source_id": source_id, + "source_kind": "transcript", + "native_session_id": native_session_id, + "ts": "2026-09-10T17:08:24.000Z", + "observed_at": "2026-09-10T17:08:25.000Z", + "payload": {"schema_version": 1, "type": "user.message"}, + "locator": {"file": "events.jsonl", "offset": 0}, + } + + +def _batch(paths: SourcePaths, records: list[SourceRecord]) -> SourceBatch: + return { + "source_key": paths["source_key"], + "native_session_id": NATIVE_SESSION_ID, + "cwd": "/proj", + "records": records, + "next_cursor": {"generation": 1}, + "diagnostics": [], + } + + +def _usage_row(**overrides: Any) -> UsageRow: + fields = dict( + session_id="stored-session", + seq=0, + call_id=ACCOUNTING_MATCHED, + ts="2026-09-10T17:08:24.498Z", + platform=PLATFORM_NAME, + provider_name="unknown", + response_model="gpt-5.6-luna", + input_tokens=6452, + output_tokens=107, + cache_creation_input_tokens=6449, + ) + fields.update(overrides) + return UsageRow(**fields) + + +def _main_turn( + *, + turn_id: str = TURN_ONE, + status: str = "completed", + accounting_calls: list[dict[str, Any]] | None = None, + subagents: list[dict[str, Any]] | None = None, +) -> dict[str, Any]: + return { + "turn_id": turn_id, + "start_ts": "2026-09-10T17:08:24.000Z", + "end_ts": "2026-09-10T17:08:28.000Z", + "input_message": "hello", + "output_message": "done", + "status": status, + "llm_calls": [ + { + "call_id": CALL_MATCHED, + "provider": "unknown", + "model": "gpt-5.6-luna", + "start_ts": "2026-09-10T17:08:24.503Z", + "end_ts": "2026-09-10T17:08:24.593Z", + "input_messages": [], + "output_messages": [], + "usage": {}, + "tool_calls": [], + } + ], + "permission_requests": [], + "subagents": subagents or [], + "attributes": {"interaction_id": "interaction-main-1"}, + "accounting_calls": accounting_calls or [], + } + + +def _accounting_call( + *, + accounting_id: str = ACCOUNTING_MATCHED, + attribution_status: str = "matched", + call_id: str | None = CALL_MATCHED, + usage: dict[str, Any] | None = None, +) -> dict[str, Any]: + return { + "accounting_id": accounting_id, + "usage": usage or _usage_row(call_id=accounting_id).to_dict(), + "attribution_status": attribution_status, + "agent_id": None, + "call_id": call_id, + "attributes": {"copilot.logical_call_id": accounting_id}, + } + + +def _attribution( + *, + logical_call_id: str, + status: str = "matched", + stored_turn_id: str | None = TURN_ONE, + call_id: str | None = None, + usage_source_id: str = "db-row-1", +) -> dict[str, Any]: + return { + "usage_source_id": usage_source_id, + "logical_call_id": logical_call_id, + "stored_turn_id": stored_turn_id, + "agent_id": None, + "call_id": call_id, + "status": status, + "join_kind": "inferred", + "evidence": ["join_kind:inferred"], + } + + +def _projection(**overrides: Any) -> Projection: + base: Projection = { + "normalized_events": [], + "turns": [], + "usage_rows": [], + "attributions": [], + "pending": [], + "diagnostics": [], + } + base.update(overrides) + return base + + +@pytest.fixture +def config(tmp_path: Path) -> Config: + return Config(root=tmp_path / "thirdeye") + + +@pytest.fixture +def enabled_config(tmp_path: Path) -> Config: + return Config( + root=tmp_path / "thirdeye", + logfire=LogfireSettings(enabled=True, token="fake-token"), + ) + + +@pytest.fixture +def paths(tmp_path: Path) -> SourcePaths: + home = tmp_path / "copilot-home" + home.mkdir() + return resolve_sources(home) + + +def _seed_session(config: Config, paths: SourcePaths) -> str: + commit_batch(config, paths, _batch(paths, [_record("key/a/event-1")])) + return stored_session_id(paths, NATIVE_SESSION_ID) + + +def _directory(config: Config, stored: str) -> Path: + return session_dir(config.root, PLATFORM_NAME, stored) + + +@pytest.fixture +def export_calls(monkeypatch: pytest.MonkeyPatch) -> dict[str, list[Any]]: + calls: dict[str, list[Any]] = { + "turn": [], + "turn_accounting": [], + "session_accounting": [], + } + + def _turn(*args: Any, **kwargs: Any) -> None: + calls["turn"].append(args[5]) + + def _turn_accounting(*args: Any, **kwargs: Any) -> bool: + calls["turn_accounting"].append(args[6]) + return True + + def _session_accounting(*args: Any, **kwargs: Any) -> bool: + calls["session_accounting"].append(args[5]) + return True + + monkeypatch.setattr(otel_export, "export_turn", _turn) + monkeypatch.setattr(otel_export, "export_turn_accounting", _turn_accounting) + monkeypatch.setattr(otel_export, "export_session_accounting", _session_accounting) + return calls + + +class TestExportState: + def test_initialize_eligibility_first_activation_excludes_terminal_history(self) -> None: + state = initialize_eligibility( + empty_export_state(), + terminal_turn_ids=["turn-a", "turn-b"], + accounting_ids=["acct-1"], + include_history=False, + ) + assert state["activated"] is True + assert state["excluded_turn_ids"] == ["turn-a", "turn-b"] + assert state["excluded_accounting_ids"] == ["acct-1"] + + def test_initialize_eligibility_include_history_keeps_terminal_eligible(self) -> None: + state = initialize_eligibility( + empty_export_state(), + terminal_turn_ids=["turn-a"], + accounting_ids=["acct-1"], + include_history=True, + ) + assert state["activated"] is True + assert state["excluded_turn_ids"] == [] + assert state["excluded_accounting_ids"] == [] + + def test_initialize_eligibility_later_include_history_opts_in(self) -> None: + first = initialize_eligibility( + empty_export_state(), + terminal_turn_ids=["turn-a"], + accounting_ids=["acct-1"], + include_history=False, + ) + second = initialize_eligibility( + first, + terminal_turn_ids=["turn-a"], + accounting_ids=["acct-1"], + include_history=True, + ) + assert second["excluded_turn_ids"] == [] + assert second["excluded_accounting_ids"] == [] + + def test_record_placement_is_idempotent_for_same_decision(self) -> None: + usage = _usage_row().to_dict() + state, entry, accepted = record_placement( + empty_export_state(), + accounting_id="acct-1", + destination="chat-span", + span_id="call-1", + usage=usage, + ) + assert accepted is True + assert entry is not None + again, same_entry, same_accepted = record_placement( + state, + accounting_id="acct-1", + destination="chat-span", + span_id="call-1", + usage=usage, + ) + assert same_accepted is True + assert same_entry == entry + + def test_record_placement_conflicts_on_changed_destination(self) -> None: + usage = _usage_row().to_dict() + state, _, accepted = record_placement( + empty_export_state(), + accounting_id="acct-1", + destination="turn-accounting-span", + span_id="span-a", + usage=usage, + ) + assert accepted is True + conflicted, existing, rejected = record_placement( + state, + accounting_id="acct-1", + destination="chat-span", + span_id="call-1", + usage=usage, + ) + assert rejected is False + assert existing is not None + assert existing["destination"] == "turn-accounting-span" + assert "acct-1" in conflicted["conflicts"] + + def test_record_placement_conflicts_on_usage_correction(self) -> None: + usage = _usage_row().to_dict() + state, _, accepted = record_placement( + empty_export_state(), + accounting_id="acct-1", + destination="chat-span", + span_id="call-1", + usage=usage, + ) + assert accepted is True + corrected = dict(usage) + corrected["output_tokens"] = 999 + conflicted, _, rejected = record_placement( + state, + accounting_id="acct-1", + destination="chat-span", + span_id="call-1", + usage=corrected, + ) + assert rejected is False + assert "acct-1" in conflicted["conflicts"] + assert conflicted["conflicts"]["acct-1"]["candidate"]["usage_digest"] == usage_digest( + corrected + ) + + +class TestQueueExports: + def test_unknown_session_raises(self, enabled_config: Config) -> None: + projection = _projection() + with pytest.raises(ValueError, match="unknown Copilot session"): + queue_exports(enabled_config, "missing-session", projection) + + def test_unconfigured_returns_zero_but_activates( + self, config: Config, paths: SourcePaths + ) -> None: + stored = _seed_session(config, paths) + projection = _projection( + turns=[_main_turn(accounting_calls=[_accounting_call()])], + usage_rows=[_usage_row()], + attributions=[_attribution(logical_call_id=ACCOUNTING_MATCHED)], + ) + queued = queue_exports(config, stored, projection) + assert queued == 0 + state = load_export_state(config, stored) + assert state["activated"] is True + assert TURN_ONE in state["excluded_turn_ids"] + assert ACCOUNTING_MATCHED in state["excluded_accounting_ids"] + + def test_first_activation_without_history_queues_nothing( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + ) -> None: + stored = _seed_session(enabled_config, paths) + projection = _projection( + turns=[ + _main_turn( + accounting_calls=[ + _accounting_call(), + _accounting_call( + accounting_id=ACCOUNTING_UNMATCHED, + attribution_status="ambiguous", + call_id=None, + usage=_usage_row(call_id=ACCOUNTING_UNMATCHED).to_dict(), + ), + ] + ) + ], + usage_rows=[ + _usage_row(), + _usage_row(call_id=ACCOUNTING_UNMATCHED, input_tokens=6587, output_tokens=5), + ], + attributions=[ + _attribution(logical_call_id=ACCOUNTING_MATCHED), + _attribution( + logical_call_id=ACCOUNTING_UNMATCHED, + status="ambiguous", + call_id=None, + ), + ], + ) + queued = queue_exports(enabled_config, stored, projection) + assert queued == 0 + assert export_calls["turn"] == [] + assert export_calls["turn_accounting"] == [] + assert export_calls["session_accounting"] == [] + + def test_include_history_queues_terminal_turn_and_matched_accounting( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + ) -> None: + stored = _seed_session(enabled_config, paths) + projection = _projection( + turns=[_main_turn(accounting_calls=[_accounting_call()])], + usage_rows=[_usage_row()], + attributions=[_attribution(logical_call_id=ACCOUNTING_MATCHED)], + ) + queued = queue_exports(enabled_config, stored, projection, include_history=True) + assert queued == 1 + assert len(export_calls["turn"]) == 1 + turn = export_calls["turn"][0] + assert turn["turn_span_id"] + assert len(turn["accounting_calls"]) == 1 + assert turn["accounting_calls"][0]["accounting_id"] == ACCOUNTING_MATCHED + assert export_calls["turn_accounting"] == [] + state = load_export_state(enabled_config, stored) + placement = state["placements"][ACCOUNTING_MATCHED] + assert placement["destination"] == "chat-span" + assert placement["span_id"] == CALL_MATCHED + + def test_ambiguous_terminal_usage_queues_turn_accounting_job( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + ) -> None: + stored = _seed_session(enabled_config, paths) + projection = _projection( + turns=[ + _main_turn( + accounting_calls=[ + _accounting_call( + accounting_id=ACCOUNTING_UNMATCHED, + attribution_status="ambiguous", + call_id=None, + usage=_usage_row(call_id=ACCOUNTING_UNMATCHED).to_dict(), + ) + ] + ) + ], + usage_rows=[_usage_row(call_id=ACCOUNTING_UNMATCHED, input_tokens=6587, output_tokens=5)], + attributions=[ + _attribution( + logical_call_id=ACCOUNTING_UNMATCHED, + status="ambiguous", + call_id=None, + ) + ], + ) + queued = queue_exports(enabled_config, stored, projection, include_history=True) + assert queued == 2 + assert len(export_calls["turn_accounting"]) == 1 + assert export_calls["turn_accounting"][0]["accounting_id"] == ACCOUNTING_UNMATCHED + assert len(export_calls["turn"]) == 1 + assert export_calls["turn"][0]["accounting_calls"] == [] + state = load_export_state(enabled_config, stored) + assert state["placements"][ACCOUNTING_UNMATCHED]["destination"] == "turn-accounting-span" + + def test_session_accounting_for_unattached_ambiguous_usage( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + ) -> None: + stored = _seed_session(enabled_config, paths) + usage = _usage_row(call_id=ACCOUNTING_UNMATCHED, input_tokens=6587, output_tokens=5) + projection = _projection( + turns=[_main_turn()], + usage_rows=[usage], + attributions=[ + _attribution( + logical_call_id=ACCOUNTING_UNMATCHED, + status="ambiguous", + stored_turn_id=None, + call_id=None, + ) + ], + ) + queued = queue_exports(enabled_config, stored, projection, include_history=True) + assert queued == 2 + assert len(export_calls["session_accounting"]) == 1 + assert export_calls["session_accounting"][0]["accounting_id"] == ACCOUNTING_UNMATCHED + state = load_export_state(enabled_config, stored) + assert state["placements"][ACCOUNTING_UNMATCHED]["destination"] == "session-accounting-span" + + def test_pending_and_conflicting_attributions_are_not_exported( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + ) -> None: + stored = _seed_session(enabled_config, paths) + pending_usage = _usage_row(call_id="pending-call", input_tokens=100, output_tokens=1) + conflicting_usage = _usage_row(call_id="conflict-call", input_tokens=200, output_tokens=2) + projection = _projection( + turns=[ + _main_turn( + accounting_calls=[ + _accounting_call( + accounting_id="pending-call", + attribution_status="pending", + call_id=None, + usage=pending_usage.to_dict(), + ) + ] + ) + ], + usage_rows=[pending_usage, conflicting_usage], + attributions=[ + _attribution(logical_call_id="pending-call", status="pending", call_id=None), + _attribution( + logical_call_id="conflict-call", + status="conflicting", + stored_turn_id=None, + call_id=None, + ), + ], + ) + queued = queue_exports(enabled_config, stored, projection, include_history=True) + assert queued == 1 + assert export_calls["turn_accounting"] == [] + assert export_calls["session_accounting"] == [] + state = load_export_state(enabled_config, stored) + assert "pending-call" not in state["placements"] + assert "conflict-call" not in state["placements"] + + def test_fallback_placement_prevents_later_chat_relocation( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + ) -> None: + stored = _seed_session(enabled_config, paths) + ambiguous_projection = _projection( + turns=[ + _main_turn( + accounting_calls=[ + _accounting_call( + accounting_id=ACCOUNTING_UNMATCHED, + attribution_status="ambiguous", + call_id=None, + usage=_usage_row(call_id=ACCOUNTING_UNMATCHED).to_dict(), + ) + ] + ) + ], + usage_rows=[_usage_row(call_id=ACCOUNTING_UNMATCHED, input_tokens=6587, output_tokens=5)], + attributions=[ + _attribution( + logical_call_id=ACCOUNTING_UNMATCHED, + status="ambiguous", + call_id=None, + ) + ], + ) + queue_exports(enabled_config, stored, ambiguous_projection, include_history=True) + export_calls["turn"].clear() + export_calls["turn_accounting"].clear() + + matched_projection = _projection( + turns=[ + _main_turn( + accounting_calls=[ + _accounting_call( + accounting_id=ACCOUNTING_UNMATCHED, + attribution_status="matched", + call_id=CALL_MATCHED, + usage=_usage_row(call_id=ACCOUNTING_UNMATCHED).to_dict(), + ) + ] + ) + ], + usage_rows=[_usage_row(call_id=ACCOUNTING_UNMATCHED, input_tokens=6587, output_tokens=5)], + attributions=[_attribution(logical_call_id=ACCOUNTING_UNMATCHED, call_id=CALL_MATCHED)], + ) + queue_exports(enabled_config, stored, matched_projection, include_history=True) + + state = load_export_state(enabled_config, stored) + placement = state["placements"][ACCOUNTING_UNMATCHED] + assert placement["destination"] == "turn-accounting-span" + assert ACCOUNTING_UNMATCHED in state["conflicts"] + assert export_calls["turn_accounting"] == [] + turn = export_calls["turn"][0] + assert turn["accounting_calls"] == [] + + def test_emitted_placement_skips_requeue( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + ) -> None: + stored = _seed_session(enabled_config, paths) + + def _mark_delivered(state: dict[str, Any]) -> dict[str, Any]: + placed, _, _ = record_placement( + state, + accounting_id=ACCOUNTING_UNMATCHED, + destination="turn-accounting-span", + span_id=f"accounting:{stored}:{TURN_ONE}:{ACCOUNTING_UNMATCHED}", + usage=_usage_row(call_id=ACCOUNTING_UNMATCHED).to_dict(), + ) + return mark_placement_delivered(placed, ACCOUNTING_UNMATCHED) + + update_export_state(enabled_config, stored, _mark_delivered) + update_export_state( + enabled_config, + stored, + lambda state: initialize_eligibility( + state, + terminal_turn_ids=[TURN_ONE], + accounting_ids=[ACCOUNTING_UNMATCHED], + include_history=True, + ), + ) + + projection = _projection( + turns=[ + _main_turn( + accounting_calls=[ + _accounting_call( + accounting_id=ACCOUNTING_UNMATCHED, + attribution_status="ambiguous", + call_id=None, + usage=_usage_row(call_id=ACCOUNTING_UNMATCHED).to_dict(), + ) + ] + ) + ], + usage_rows=[_usage_row(call_id=ACCOUNTING_UNMATCHED, input_tokens=6587, output_tokens=5)], + attributions=[ + _attribution( + logical_call_id=ACCOUNTING_UNMATCHED, + status="ambiguous", + call_id=None, + ) + ], + ) + queued = queue_exports(enabled_config, stored, projection, include_history=True) + assert queued == 1 + assert export_calls["turn_accounting"] == [] + assert len(export_calls["turn"]) == 1 + + def test_late_terminal_turn_becomes_eligible_without_history_opt_in( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + ) -> None: + stored = _seed_session(enabled_config, paths) + first_turn = _main_turn(turn_id="turn-first") + queue_exports( + enabled_config, + stored, + _projection(turns=[first_turn], usage_rows=[], attributions=[]), + ) + export_calls["turn"].clear() + + second_turn = _main_turn(turn_id="turn-second") + queued = queue_exports( + enabled_config, + stored, + _projection(turns=[first_turn, second_turn], usage_rows=[], attributions=[]), + ) + assert queued == 1 + assert export_calls["turn"][0]["turn_id"] == "turn-second" + + def test_accounting_job_failure_records_error( + self, + enabled_config: Config, + paths: SourcePaths, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + stored = _seed_session(enabled_config, paths) + monkeypatch.setattr(otel_export, "export_turn", lambda *args, **kwargs: None) + monkeypatch.setattr(otel_export, "export_turn_accounting", lambda *args, **kwargs: False) + monkeypatch.setattr(otel_export, "export_session_accounting", lambda *args, **kwargs: False) + + projection = _projection( + turns=[ + _main_turn( + accounting_calls=[ + _accounting_call( + accounting_id=ACCOUNTING_UNMATCHED, + attribution_status="ambiguous", + call_id=None, + usage=_usage_row(call_id=ACCOUNTING_UNMATCHED).to_dict(), + ) + ] + ) + ], + usage_rows=[_usage_row(call_id=ACCOUNTING_UNMATCHED, input_tokens=6587, output_tokens=5)], + attributions=[ + _attribution( + logical_call_id=ACCOUNTING_UNMATCHED, + status="ambiguous", + call_id=None, + ) + ], + ) + queued = queue_exports(enabled_config, stored, projection, include_history=True) + assert queued == 1 + state = load_export_state(enabled_config, stored) + assert state["placements"][ACCOUNTING_UNMATCHED]["last_error"] == ( + "accounting job was not queued" + ) + + def test_restart_preserves_ledger_and_boundary( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + ) -> None: + stored = _seed_session(enabled_config, paths) + projection = _projection( + turns=[_main_turn(accounting_calls=[_accounting_call()])], + usage_rows=[_usage_row()], + attributions=[_attribution(logical_call_id=ACCOUNTING_MATCHED)], + ) + queue_exports(enabled_config, stored, projection, include_history=True) + ledger_path = export_state_path(_directory(enabled_config, stored)) + saved = json.loads(ledger_path.read_text(encoding="utf-8")) + + export_calls["turn"].clear() + reloaded = Config( + root=enabled_config.root, + logfire=LogfireSettings(enabled=True, token="fake-token"), + ) + queued = queue_exports(reloaded, stored, projection, include_history=True) + assert queued == 1 + assert json.loads(ledger_path.read_text(encoding="utf-8"))["placements"] == saved["placements"] + + def test_non_terminal_turns_are_not_exported( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + ) -> None: + stored = _seed_session(enabled_config, paths) + projection = _projection( + turns=[_main_turn(status="in_progress", accounting_calls=[_accounting_call()])], + usage_rows=[_usage_row()], + attributions=[_attribution(logical_call_id=ACCOUNTING_MATCHED)], + ) + queued = queue_exports(enabled_config, stored, projection, include_history=True) + assert queued == 0 + assert export_calls["turn"] == [] + + +def _drain_cli_transcript(home: Path) -> list[SourceRecord]: + session_dir = home / "session-state" / NATIVE_SESSION_ID + session_dir.mkdir(parents=True, exist_ok=True) + shutil.copy(FIXTURES / "events.jsonl", session_dir / "events.jsonl") + (session_dir / "workspace.yaml").write_text("cwd: /sanitized/workspace\n", encoding="utf-8") + paths = resolve_sources(home) + cursor: dict[str, Any] = {} + records: list[SourceRecord] = [] + while True: + slice_ = read_transcript(paths, NATIVE_SESSION_ID, cursor) + records.extend(slice_["records"]) + cursor = slice_["next_cursor"] + if slice_["exhausted"]: + break + return [record for record in records if record["source_kind"] == "transcript"] + + +def _source_key(records: list[SourceRecord]) -> str: + for record in records: + if record["source_kind"] == "transcript": + return record["source_id"].split("/", 1)[0] + pytest.fail("expected transcript record") + + +def _usage_record( + row: dict[str, Any], + *, + content_revision: str, + source_key: str = SOURCE_KEY, +) -> SourceRecord: + primary_key = row["id"] + source_id = ( + f"copilot-db:{source_key}:{NATIVE_SESSION_ID}:assistant_usage_events:" + f"{primary_key}:{content_revision}" + ) + return { + "source_id": source_id, + "source_kind": "database", + "native_session_id": NATIVE_SESSION_ID, + "ts": row.get("created_at"), + "observed_at": OBSERVED_AT, + "payload": {"table": "assistant_usage_events", "row": row}, + "locator": { + "database": "/example/.copilot/session-store.db", + "table": "assistant_usage_events", + "primary_key": primary_key, + "content_revision": content_revision, + "generation": GENERATION, + }, + } + + +def _six_call_records(*, source_key: str = SOURCE_KEY) -> list[SourceRecord]: + rows = _load_json(FIXTURES / "assistant-usage-events.json") + revisions = { + call["row_id"]: call["usage_source_id"].rsplit(":", 1)[-1] + for call in _load_json(RECON / "observed-six-calls.json")["calls"] + } + return [ + _usage_record(row, content_revision=revisions[row["id"]], source_key=source_key) + for row in rows + ] + + +def test_build_projection_fixture_export_with_history( + tmp_path: Path, + export_calls: dict[str, list[Any]], +) -> None: + home = tmp_path / "copilot-home" + home.mkdir() + paths = resolve_sources(home) + config = Config( + root=tmp_path / "thirdeye", + logfire=LogfireSettings(enabled=True, token="fake-token"), + ) + commit_batch(config, paths, _batch(paths, _drain_cli_transcript(home))) + stored = stored_session_id(paths, NATIVE_SESSION_ID) + + source_key = _source_key(_drain_cli_transcript(home)) + records = _drain_cli_transcript(home) + _six_call_records(source_key=source_key) + projection, _ = build_projection(records, {}) + + queued = queue_exports(config, stored, projection, include_history=True) + assert queued >= len(projection["turns"]) + assert export_calls["turn"] + matched = [ + item + for turn in projection["turns"] + for item in turn.get("accounting_calls") or [] + if item.get("attribution_status") == "matched" + ] + assert matched + state = load_export_state(config, stored) + for item in matched: + assert item["accounting_id"] in state["placements"] + assert state["placements"][item["accounting_id"]]["destination"] == "chat-span" From 4cbe1f5c37ee1ab8d543186e955964902bf6df59 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Fri, 11 Sep 2026 20:59:19 -0700 Subject: [PATCH 71/88] Add behavioral tests for Copilot archive reconciliation. Cover incremental replay, rebuild idempotence, failure isolation, export opt-in, and archive-only operation without live source files. Co-authored-by: Cursor --- tests/platforms/copilot/test_reconcile.py | 459 ++++++++++++++++++++++ 1 file changed, 459 insertions(+) create mode 100644 tests/platforms/copilot/test_reconcile.py diff --git a/tests/platforms/copilot/test_reconcile.py b/tests/platforms/copilot/test_reconcile.py new file mode 100644 index 0000000..5ea4fe1 --- /dev/null +++ b/tests/platforms/copilot/test_reconcile.py @@ -0,0 +1,459 @@ +"""Behavioral tests for archive-only Copilot reconciliation.""" + +from __future__ import annotations + +import copy +import json +import shutil +from pathlib import Path +from typing import Any + +import pytest + +from thirdeye.config import Config +from thirdeye.platforms.copilot.archive import commit_batch, iter_captured_records +from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id +from thirdeye.platforms.copilot.projection_store import ( + load_projection_state, + read_projected_turns, +) +from thirdeye.platforms.copilot.reconcile import reconcile_archive +from thirdeye.platforms.copilot.transcript import read_transcript +from thirdeye.platforms.copilot.types import SourceBatch, SourcePaths, SourceRecord +from thirdeye.reader import SessionReader + +FIXTURES = Path(__file__).parent / "fixtures" +RECON = FIXTURES / "reconciliation-cases" +NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" +SOURCE_KEY = "a" * 64 +GENERATION = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" +OBSERVED_AT = "2026-09-10T17:09:00.000Z" +RESULT_KEYS = ( + "events", + "usage", + "turns", + "exports", + "pending", + "ambiguous", + "conflicting", + "errors", +) + + +def _load_json(path: Path) -> Any: + return json.loads(path.read_text(encoding="utf-8")) + + +def _drain_cli_transcript(home: Path) -> list[SourceRecord]: + session_dir = home / "session-state" / NATIVE_SESSION_ID + session_dir.mkdir(parents=True, exist_ok=True) + shutil.copy(FIXTURES / "events.jsonl", session_dir / "events.jsonl") + (session_dir / "workspace.yaml").write_text("cwd: /sanitized/workspace\n", encoding="utf-8") + paths = resolve_sources(home) + cursor: dict[str, Any] = {} + records: list[SourceRecord] = [] + while True: + slice_ = read_transcript(paths, NATIVE_SESSION_ID, cursor) + records.extend(slice_["records"]) + cursor = slice_["next_cursor"] + if slice_["exhausted"]: + break + return [record for record in records if record["source_kind"] == "transcript"] + + +def _source_key(records: list[SourceRecord]) -> str: + for record in records: + if record["source_kind"] == "transcript": + return record["source_id"].split("/", 1)[0] + pytest.fail("expected at least one transcript record") + + +def _usage_record( + row: dict[str, Any], + *, + content_revision: str, + generation: str = GENERATION, + source_key: str = SOURCE_KEY, + observed_at: str = OBSERVED_AT, +) -> SourceRecord: + primary_key = row["id"] + source_id = ( + f"copilot-db:{source_key}:{NATIVE_SESSION_ID}:assistant_usage_events:" + f"{primary_key}:{content_revision}" + ) + return { + "source_id": source_id, + "source_kind": "database", + "native_session_id": NATIVE_SESSION_ID, + "ts": row.get("created_at"), + "observed_at": observed_at, + "payload": {"table": "assistant_usage_events", "row": row}, + "locator": { + "database": "/example/.copilot/session-store.db", + "table": "assistant_usage_events", + "primary_key": primary_key, + "content_revision": content_revision, + "generation": generation, + }, + } + + +def _six_call_records(*, source_key: str = SOURCE_KEY) -> list[SourceRecord]: + rows = _load_json(FIXTURES / "assistant-usage-events.json") + revisions = { + call["row_id"]: call["usage_source_id"].rsplit(":", 1)[-1] + for call in _load_json(RECON / "observed-six-calls.json")["calls"] + } + return [ + _usage_record(row, content_revision=revisions[row["id"]], source_key=source_key) + for row in rows + ] + + +def _batch(paths: SourcePaths, records: list[SourceRecord]) -> SourceBatch: + return { + "source_key": paths["source_key"], + "native_session_id": NATIVE_SESSION_ID, + "cwd": "/proj", + "records": records, + "next_cursor": {"generation": 1}, + "diagnostics": [], + } + + +def _seed_archive(config: Config, paths: SourcePaths, records: list[SourceRecord]) -> str: + commit_batch(config, paths, _batch(paths, records)) + return stored_session_id(paths, NATIVE_SESSION_ID) + + +def _substitute_source_key(value: str, source_key: str) -> str: + return value.replace(SOURCE_KEY, source_key) + + +def _rewrite_source_key(value: Any, source_key: str) -> Any: + if isinstance(value, str): + return _substitute_source_key(value, source_key) + if isinstance(value, list): + return [_rewrite_source_key(item, source_key) for item in value] + if isinstance(value, dict): + return {key: _rewrite_source_key(item, source_key) for key, item in value.items()} + return value + + +def _append_archive( + config: Config, paths: SourcePaths, records: list[SourceRecord], *, generation: int +) -> None: + commit_batch( + config, + paths, + { + **_batch(paths, records), + "next_cursor": {"generation": generation}, + }, + ) + + +@pytest.fixture +def config(tmp_path: Path) -> Config: + return Config(root=tmp_path / "thirdeye") + + +@pytest.fixture +def copilot_home(tmp_path: Path) -> Path: + home = tmp_path / "copilot-home" + home.mkdir() + return home + + +@pytest.fixture +def paths(copilot_home: Path) -> SourcePaths: + return resolve_sources(copilot_home) + + +@pytest.fixture +def cli_transcript_records(copilot_home: Path) -> list[SourceRecord]: + return _drain_cli_transcript(copilot_home) + + +def _full_corpus_records(cli_transcript_records: list[SourceRecord]) -> list[SourceRecord]: + source_key = _source_key(cli_transcript_records) + return cli_transcript_records + _six_call_records(source_key=source_key) + + +def _seed_full_corpus( + config: Config, paths: SourcePaths, cli_transcript_records: list[SourceRecord] +) -> str: + return _seed_archive(config, paths, _full_corpus_records(cli_transcript_records)) + + +def test_reconcile_archive_projects_six_call_corpus( + config: Config, + paths: SourcePaths, + cli_transcript_records: list[SourceRecord], +) -> None: + stored = _seed_full_corpus(config, paths, cli_transcript_records) + + result = reconcile_archive(config, stored) + + assert set(result.keys()) == set(RESULT_KEYS) + assert result["exports"] == 0 + assert result["errors"] == 0 + assert result["usage"] == 6 + assert result["turns"] == 2 + assert result["events"] > 0 + assert result["ambiguous"] == 0 + assert result["conflicting"] == 0 + turns = read_projected_turns(config, stored) + assert len(turns) == 2 + state = load_projection_state(config, stored) + assert len(state["archive_source_ids"]) == len(list(iter_captured_records(config, stored))) + + +def test_reconcile_archive_is_idempotent( + config: Config, + paths: SourcePaths, + cli_transcript_records: list[SourceRecord], +) -> None: + stored = _seed_full_corpus(config, paths, cli_transcript_records) + + first = reconcile_archive(config, stored) + second = reconcile_archive(config, stored) + + assert first == second + assert read_projected_turns(config, stored) == read_projected_turns(config, stored) + + +def test_rebuild_is_idempotent_and_matches_initial_reconcile( + config: Config, + paths: SourcePaths, + cli_transcript_records: list[SourceRecord], +) -> None: + stored = _seed_full_corpus(config, paths, cli_transcript_records) + initial = reconcile_archive(config, stored) + first_rebuild = reconcile_archive(config, stored, rebuild=True) + second_rebuild = reconcile_archive(config, stored, rebuild=True) + + assert first_rebuild == second_rebuild + assert first_rebuild["events"] == initial["events"] + assert first_rebuild["usage"] == initial["usage"] + assert first_rebuild["turns"] == initial["turns"] + assert first_rebuild["exports"] == 0 + + +def test_incremental_reconcile_matches_full_replay( + config: Config, + paths: SourcePaths, + cli_transcript_records: list[SourceRecord], +) -> None: + source_key = _source_key(cli_transcript_records) + usage = _six_call_records(source_key=source_key) + partial = cli_transcript_records[: len(cli_transcript_records) // 2] + remainder = cli_transcript_records[len(cli_transcript_records) // 2 :] + + stored_incremental = _seed_archive(config, paths, partial) + reconcile_archive(config, stored_incremental) + _append_archive(config, paths, remainder + usage, generation=2) + incremental = reconcile_archive(config, stored_incremental) + incremental_turns = read_projected_turns(config, stored_incremental) + + stored_full = _seed_full_corpus(config, paths, cli_transcript_records) + full = reconcile_archive(config, stored_full) + full_turns = read_projected_turns(config, stored_full) + + assert incremental["turns"] == full["turns"] + assert incremental["usage"] == full["usage"] + assert incremental["events"] == full["events"] + assert [turn["turn_id"] for turn in incremental_turns] == [ + turn["turn_id"] for turn in full_turns + ] + + +def test_reconcile_default_does_not_export( + config: Config, + paths: SourcePaths, + cli_transcript_records: list[SourceRecord], + monkeypatch: pytest.MonkeyPatch, +) -> None: + stored = _seed_full_corpus(config, paths, cli_transcript_records) + imports: list[str] = [] + original = __import__ + + def tracking_import(name: str, *args: Any, **kwargs: Any) -> Any: + imports.append(name) + return original(name, *args, **kwargs) + + monkeypatch.setattr("importlib.import_module", tracking_import) + + result = reconcile_archive(config, stored) + + assert result["exports"] == 0 + assert not any("export" in item for item in imports) + + +def test_export_failure_does_not_roll_back_projection( + config: Config, + paths: SourcePaths, + cli_transcript_records: list[SourceRecord], + monkeypatch: pytest.MonkeyPatch, +) -> None: + stored = _seed_full_corpus(config, paths, cli_transcript_records) + + def boom(*_args: Any, **_kwargs: Any) -> int: + raise RuntimeError("export unavailable") + + monkeypatch.setattr( + "thirdeye.platforms.copilot.reconcile.queue_exports", + boom, + ) + + result = reconcile_archive(config, stored, export=True) + + assert result["turns"] == 2 + assert result["usage"] == 6 + assert result["exports"] == 0 + assert result["errors"] == 1 + assert len(read_projected_turns(config, stored)) == 2 + + +def test_projection_failure_preserves_prior_derived_state( + config: Config, + paths: SourcePaths, + cli_transcript_records: list[SourceRecord], + monkeypatch: pytest.MonkeyPatch, +) -> None: + stored = _seed_full_corpus(config, paths, cli_transcript_records) + baseline = reconcile_archive(config, stored) + prior_turns = read_projected_turns(config, stored) + + def explode(*_args: Any, **_kwargs: Any) -> tuple[Any, dict[str, Any]]: + raise RuntimeError("projection failed") + + monkeypatch.setattr("thirdeye.platforms.copilot.reconcile.build_projection", explode) + + result = reconcile_archive(config, stored) + + assert result["errors"] == 1 + assert result["events"] == baseline["events"] + assert result["usage"] == baseline["usage"] + assert result["turns"] == baseline["turns"] + assert read_projected_turns(config, stored) == prior_turns + + +def test_build_failure_during_rebuild_preserves_prior_projection( + config: Config, + paths: SourcePaths, + cli_transcript_records: list[SourceRecord], + monkeypatch: pytest.MonkeyPatch, +) -> None: + stored = _seed_full_corpus(config, paths, cli_transcript_records) + reconcile_archive(config, stored) + prior_turns = read_projected_turns(config, stored) + + def explode(*_args: Any, **_kwargs: Any) -> tuple[Any, dict[str, Any]]: + raise RuntimeError("projection failed") + + monkeypatch.setattr("thirdeye.platforms.copilot.reconcile.build_projection", explode) + + result = reconcile_archive(config, stored, rebuild=True) + + assert result["errors"] == 1 + assert read_projected_turns(config, stored) == prior_turns + + +def test_reconcile_unknown_session_reports_error_without_creating_paths( + config: Config, +) -> None: + ghost = "copilot-ghost-session" + + result = reconcile_archive(config, ghost) + + assert result["errors"] == 1 + assert result["exports"] == 0 + assert all(result[key] == 0 for key in RESULT_KEYS if key not in {"errors", "exports"}) + assert not (config.root / "traces" / "copilot" / ghost).exists() + + +def test_reconcile_works_without_live_copilot_source_files( + config: Config, + copilot_home: Path, + cli_transcript_records: list[SourceRecord], +) -> None: + paths = resolve_sources(copilot_home) + stored = _seed_full_corpus(config, paths, cli_transcript_records) + shutil.rmtree(copilot_home) + + result = reconcile_archive(config, stored, rebuild=True) + + assert result["errors"] == 0 + assert result["turns"] == 2 + assert result["usage"] == 6 + assert len(list(iter_captured_records(config, stored))) > 0 + + +def test_rebuild_preserves_immutable_archive_records( + config: Config, + paths: SourcePaths, + cli_transcript_records: list[SourceRecord], +) -> None: + stored = _seed_full_corpus(config, paths, cli_transcript_records) + before = { + event["data"]["source_record"]["source_id"] + for event in SessionReader( + config.root / "traces" / "copilot" / stored + ).iter_events() + } + + reconcile_archive(config, stored, rebuild=True) + + after = { + event["data"]["source_record"]["source_id"] + for event in SessionReader( + config.root / "traces" / "copilot" / stored + ).iter_events() + } + assert before == after + + +def test_late_row_archive_reconcile_resolves_after_full_replay( + config: Config, + paths: SourcePaths, +) -> None: + source_key = paths["source_key"] + case = _load_json(RECON / "cases.json")["late_row"] + partial = _rewrite_source_key(case["input_records"], source_key) + final_answer = _rewrite_source_key( + _load_json(RECON / "cases.json")["ambiguous"]["input_records"][1], + source_key, + ) + turn_end = copy.deepcopy(final_answer) + turn_end.update( + { + "source_id": ( + f"{source_key}/{NATIVE_SESSION_ID}/4a386a37-ca7e-4ebc-a746-cdda20f2a4bb" + ), + "payload": { + "type": "assistant.turn_end", + "data": {"turnId": "1"}, + "id": "4a386a37-ca7e-4ebc-a746-cdda20f2a4bb", + "timestamp": "2026-09-10T17:08:25.626Z", + "parentId": "33cc6465-29e1-4a04-8bdb-00241474b4d2", + "schema_version": 1, + }, + } + ) + full = partial + [final_answer, turn_end] + + stored = _seed_archive(config, paths, partial) + pending = reconcile_archive(config, stored) + assert pending["errors"] == 0 + assert pending["usage"] == 1 + assert pending["pending"] >= 1 + + _append_archive(config, paths, [final_answer, turn_end], generation=2) + resolved = reconcile_archive(config, stored) + + assert resolved["errors"] == 0 + assert resolved["usage"] == 1 + assert resolved["pending"] < pending["pending"] + state = load_projection_state(config, stored) + assert len(state["archive_source_ids"]) == len(full) From 8f1b5d9fa419882f5e15b4ee75d477a7e58f9410 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Fri, 11 Sep 2026 23:32:12 -0700 Subject: [PATCH 72/88] Make Copilot archive rebuild atomic and tighten reconcile test coverage. Rebuild previously deleted derived projection state (reset_projection_state) in a separate lock acquisition before committing the replacement, so a mid-commit failure (bad usage row, IO error) could lose the last valid projection instead of just failing the current attempt. commit_projection now takes a replace flag and discards prior indexes inside the same locked, single-writer commit, so nothing is deleted until the replacement is fully validated and ready to publish. Failure-path status reporting now reads the last successfully published projection via a new read_projection_status helper instead of always zeroing out ambiguous/conflicting/error counts. Test fixes: the incremental-vs-full-replay test compared an archive against itself (same stored session ID), the idempotence test compared the same read against itself, and the "no export import" test patched a name reconcile.py doesn't reference. Added a test that forces a failure inside commit_projection during rebuild (not just during build) to directly exercise the atomicity fix. Co-Authored-By: Claude Sonnet 5 --- .../platforms/copilot/projection_store.py | 54 ++++- src/thirdeye/platforms/copilot/reconcile.py | 65 +++--- tests/platforms/copilot/test_reconcile.py | 186 ++++++++++++++---- 3 files changed, 229 insertions(+), 76 deletions(-) diff --git a/src/thirdeye/platforms/copilot/projection_store.py b/src/thirdeye/platforms/copilot/projection_store.py index f138ccc..6ee2c2e 100644 --- a/src/thirdeye/platforms/copilot/projection_store.py +++ b/src/thirdeye/platforms/copilot/projection_store.py @@ -389,18 +389,27 @@ def commit_projection( stored_session_id: str, projection: Projection, next_state: dict[str, Any], + *, + replace: bool = False, ) -> dict[str, int]: - """Atomically merge a DTO projection into local, replayable derived state. + """Atomically merge or replace a DTO projection into replayable derived state. V1 events are read only from the captured Store archive to form the generic turn view. No source reader is invoked, so a captured session remains projectable after Copilot removes its original files. + + ``replace=True`` discards prior derived indexes as part of this same + locked operation instead of requiring a separate reset call. Nothing is + written to disk until every validation below succeeds, so a rebuild that + fails partway (a malformed usage row, an IO error) leaves the previous + projection completely untouched rather than losing it to a non-atomic + delete-then-commit sequence. """ directory = _directory(config, stored_session_id) _require_session(directory, stored_session_id) with locked(projection_lock_path(directory), LockMode.EXCLUSIVE): document = read_projection_document(directory) - indexes = _mapping(document.get("indexes")) + indexes = {} if replace else _mapping(document.get("indexes")) merged_indexes = {name: dict(_mapping(indexes.get(name))) for name in indexes} for name in (*_INDEX_NAMES, "usage_identities"): merged_indexes.setdefault(name, {}) @@ -514,6 +523,47 @@ def read_projected_turns(config: Config, stored_session_id: str) -> list[dict[st return json.loads(_canonical(values)) +_STATUS_KEYS = ("events", "usage", "turns", "pending", "ambiguous", "conflicting", "errors") + + +def read_projection_status(config: Config, stored_session_id: str) -> dict[str, int]: + """Return persisted projection counts without attempting a new derive. + + A caller whose current reconciliation attempt failed can use this to + report the status of the last successfully published projection (which + remains on disk and readable) instead of fabricating zeroed-out counts. + """ + directory = _existing_session_dir(config, stored_session_id) + if directory is None: + return dict.fromkeys(_STATUS_KEYS, 0) + with locked(projection_lock_path(directory), LockMode.EXCLUSIVE): + document = read_projection_document(directory) + indexes = _mapping(document.get("indexes")) + attributions = _mapping(indexes.get("attributions")).values() + diagnostics = _mapping(indexes.get("diagnostics")).values() + return { + "events": len(_mapping(indexes.get("events"))), + "usage": len(_mapping(indexes.get("usage"))), + "turns": len(_mapping(indexes.get("turns"))), + "pending": len(_mapping(indexes.get("pending"))), + "ambiguous": sum( + 1 + for item in attributions + if isinstance(item, dict) and item.get("status") == "ambiguous" + ), + "conflicting": sum( + 1 + for item in attributions + if isinstance(item, dict) and item.get("status") == "conflicting" + ), + "errors": sum( + 1 + for item in diagnostics + if isinstance(item, dict) and item.get("severity") == "error" + ), + } + + def reset_projection_state(config: Config, stored_session_id: str) -> None: """Delete only reproducible projection state for a local rebuild. diff --git a/src/thirdeye/platforms/copilot/reconcile.py b/src/thirdeye/platforms/copilot/reconcile.py index c465fb9..eb44315 100644 --- a/src/thirdeye/platforms/copilot/reconcile.py +++ b/src/thirdeye/platforms/copilot/reconcile.py @@ -8,18 +8,14 @@ from __future__ import annotations +from collections.abc import Callable from importlib import import_module -from typing import Any, Callable from thirdeye.config import Config from .archive import iter_captured_records from .projection import build_projection -from .projection_store import ( - commit_projection, - load_projection_state, - reset_projection_state, -) +from .projection_store import commit_projection, load_projection_state, read_projection_status from .types import Projection _COUNT_KEYS = ("events", "usage", "turns", "exports", "pending", "ambiguous", "conflicting") @@ -31,22 +27,18 @@ def _empty_result(*, errors: int = 0) -> dict[str, int]: return result -def _stored_counts(state: dict[str, Any]) -> dict[str, int]: - """Return safe status counts when a projection attempt cannot proceed.""" +def _failure_result(config: Config, stored_session_id: str) -> dict[str, int]: + """Report the last successfully published projection's status on failure. - result = _empty_result() - totals = state.get("index_totals") - if not isinstance(totals, dict): - return result - for source, target in ( - ("events", "events"), - ("usage", "usage"), - ("turns", "turns"), - ("pending", "pending"), - ): - value = totals.get(source) - if isinstance(value, int) and value >= 0: - result[target] = value + The local projection on disk (if any) is untouched by a failed attempt, + so its counts -- not zeros -- describe what a reader will actually see. + """ + try: + status = read_projection_status(config, stored_session_id) + except Exception: + return _empty_result(errors=1) + result = {**_empty_result(), **status} + result["errors"] = status.get("errors", 0) + 1 return result @@ -79,7 +71,7 @@ def queue_exports(config: Config, stored_session_id: str, projection: Projection """ module = import_module(".export", package=__package__) - enqueue = getattr(module, "queue_exports") + enqueue = module.queue_exports if not callable(enqueue): raise TypeError("Copilot export assembly does not provide queue_exports") exporter: Callable[[Config, str, Projection], int] = enqueue @@ -96,10 +88,12 @@ def reconcile_archive( """Rebuild local Copilot projections from the immutable V1 archive. A normal reconciliation is a complete archive replay. Stable derived - identities make committing that replay idempotent, while a full replay - keeps late usage rows eligible to join semantic evidence captured in an - earlier pass. ``rebuild`` deletes only reproducible projection state; - raw archive records and the separate export ledger are untouched. + identities make committing that replay idempotent. ``rebuild`` discards + reproducible projection state as part of the same locked commit used for + an incremental merge (see ``commit_projection(..., replace=True)``): raw + archive records and the separate export ledger are untouched, and nothing + is written to disk until the replacement projection is fully validated, + so a rebuild that fails partway cannot erase the last readable local view. Errors are reported as counts instead of escaping so source capture can remain operational when a derived projection is malformed. Export is a @@ -107,26 +101,15 @@ def reconcile_archive( """ try: - # Build a replacement before deleting a previous projection. This is - # especially important for an operator-triggered rebuild: malformed - # archived evidence must not turn a recoverable projection error into - # a loss of the last readable local view. Rebuild input is empty so - # no unfinished incremental state can leak into the full replay. prior_state = {} if rebuild else load_projection_state(config, stored_session_id) records = list(iter_captured_records(config, stored_session_id)) projection, next_state = build_projection(records, prior_state) next_state["archive_source_ids"] = [record["source_id"] for record in records] - if rebuild: - reset_projection_state(config, stored_session_id) - counts = commit_projection(config, stored_session_id, projection, next_state) + counts = commit_projection( + config, stored_session_id, projection, next_state, replace=rebuild + ) except Exception: - # Do not reset or mutate V1 capture state when derivation fails. - try: - result = _stored_counts(load_projection_state(config, stored_session_id)) - except Exception: - result = _empty_result() - result["errors"] += 1 - return result + return _failure_result(config, stored_session_id) ambiguous, conflicting = _attribution_counts(projection) result = { diff --git a/tests/platforms/copilot/test_reconcile.py b/tests/platforms/copilot/test_reconcile.py index 5ea4fe1..8858d71 100644 --- a/tests/platforms/copilot/test_reconcile.py +++ b/tests/platforms/copilot/test_reconcile.py @@ -13,6 +13,7 @@ from thirdeye.config import Config from thirdeye.platforms.copilot.archive import commit_batch, iter_captured_records from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id +from thirdeye.platforms.copilot.projection import build_projection as _real_build_projection from thirdeye.platforms.copilot.projection_store import ( load_projection_state, read_projected_turns, @@ -25,6 +26,7 @@ FIXTURES = Path(__file__).parent / "fixtures" RECON = FIXTURES / "reconciliation-cases" NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" +FULL_NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd7" SOURCE_KEY = "a" * 64 GENERATION = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" OBSERVED_AT = "2026-09-10T17:09:00.000Z" @@ -44,8 +46,10 @@ def _load_json(path: Path) -> Any: return json.loads(path.read_text(encoding="utf-8")) -def _drain_cli_transcript(home: Path) -> list[SourceRecord]: - session_dir = home / "session-state" / NATIVE_SESSION_ID +def _drain_cli_transcript( + home: Path, *, native_session_id: str = NATIVE_SESSION_ID +) -> list[SourceRecord]: + session_dir = home / "session-state" / native_session_id session_dir.mkdir(parents=True, exist_ok=True) shutil.copy(FIXTURES / "events.jsonl", session_dir / "events.jsonl") (session_dir / "workspace.yaml").write_text("cwd: /sanitized/workspace\n", encoding="utf-8") @@ -53,7 +57,7 @@ def _drain_cli_transcript(home: Path) -> list[SourceRecord]: cursor: dict[str, Any] = {} records: list[SourceRecord] = [] while True: - slice_ = read_transcript(paths, NATIVE_SESSION_ID, cursor) + slice_ = read_transcript(paths, native_session_id, cursor) records.extend(slice_["records"]) cursor = slice_["next_cursor"] if slice_["exhausted"]: @@ -74,17 +78,18 @@ def _usage_record( content_revision: str, generation: str = GENERATION, source_key: str = SOURCE_KEY, + native_session_id: str = NATIVE_SESSION_ID, observed_at: str = OBSERVED_AT, ) -> SourceRecord: primary_key = row["id"] source_id = ( - f"copilot-db:{source_key}:{NATIVE_SESSION_ID}:assistant_usage_events:" + f"copilot-db:{source_key}:{native_session_id}:assistant_usage_events:" f"{primary_key}:{content_revision}" ) return { "source_id": source_id, "source_kind": "database", - "native_session_id": NATIVE_SESSION_ID, + "native_session_id": native_session_id, "ts": row.get("created_at"), "observed_at": observed_at, "payload": {"table": "assistant_usage_events", "row": row}, @@ -98,22 +103,34 @@ def _usage_record( } -def _six_call_records(*, source_key: str = SOURCE_KEY) -> list[SourceRecord]: +def _six_call_records( + *, source_key: str = SOURCE_KEY, native_session_id: str = NATIVE_SESSION_ID +) -> list[SourceRecord]: rows = _load_json(FIXTURES / "assistant-usage-events.json") revisions = { call["row_id"]: call["usage_source_id"].rsplit(":", 1)[-1] for call in _load_json(RECON / "observed-six-calls.json")["calls"] } return [ - _usage_record(row, content_revision=revisions[row["id"]], source_key=source_key) + _usage_record( + row, + content_revision=revisions[row["id"]], + source_key=source_key, + native_session_id=native_session_id, + ) for row in rows ] -def _batch(paths: SourcePaths, records: list[SourceRecord]) -> SourceBatch: +def _batch( + paths: SourcePaths, + records: list[SourceRecord], + *, + native_session_id: str = NATIVE_SESSION_ID, +) -> SourceBatch: return { "source_key": paths["source_key"], - "native_session_id": NATIVE_SESSION_ID, + "native_session_id": native_session_id, "cwd": "/proj", "records": records, "next_cursor": {"generation": 1}, @@ -121,9 +138,15 @@ def _batch(paths: SourcePaths, records: list[SourceRecord]) -> SourceBatch: } -def _seed_archive(config: Config, paths: SourcePaths, records: list[SourceRecord]) -> str: - commit_batch(config, paths, _batch(paths, records)) - return stored_session_id(paths, NATIVE_SESSION_ID) +def _seed_archive( + config: Config, + paths: SourcePaths, + records: list[SourceRecord], + *, + native_session_id: str = NATIVE_SESSION_ID, +) -> str: + commit_batch(config, paths, _batch(paths, records, native_session_id=native_session_id)) + return stored_session_id(paths, native_session_id) def _substitute_source_key(value: str, source_key: str) -> str: @@ -140,14 +163,43 @@ def _rewrite_source_key(value: Any, source_key: str) -> Any: return value +_VOLATILE_KEYS = frozenset({"observed_at", "locator"}) + + +def _rewrite_native_id(value: Any, native_session_id: str, placeholder: str = "NATIVE") -> Any: + """Normalize identity fields that legitimately differ between two + + independently seeded stored sessions built from the same fixture content: + the native session ID baked into IDs/paths, each session's own wall-clock + capture timestamp, and each archived copy's own filesystem locator (byte + offsets/inode generation are per-file-instance, not semantic content). + """ + if isinstance(value, str): + return value.replace(native_session_id, placeholder) + if isinstance(value, list): + return [_rewrite_native_id(item, native_session_id, placeholder) for item in value] + if isinstance(value, dict): + return { + key: _rewrite_native_id(item, native_session_id, placeholder) + for key, item in value.items() + if key not in _VOLATILE_KEYS + } + return value + + def _append_archive( - config: Config, paths: SourcePaths, records: list[SourceRecord], *, generation: int + config: Config, + paths: SourcePaths, + records: list[SourceRecord], + *, + generation: int, + native_session_id: str = NATIVE_SESSION_ID, ) -> None: commit_batch( config, paths, { - **_batch(paths, records), + **_batch(paths, records, native_session_id=native_session_id), "next_cursor": {"generation": generation}, }, ) @@ -175,15 +227,28 @@ def cli_transcript_records(copilot_home: Path) -> list[SourceRecord]: return _drain_cli_transcript(copilot_home) -def _full_corpus_records(cli_transcript_records: list[SourceRecord]) -> list[SourceRecord]: +def _full_corpus_records( + cli_transcript_records: list[SourceRecord], *, native_session_id: str = NATIVE_SESSION_ID +) -> list[SourceRecord]: source_key = _source_key(cli_transcript_records) - return cli_transcript_records + _six_call_records(source_key=source_key) + return cli_transcript_records + _six_call_records( + source_key=source_key, native_session_id=native_session_id + ) def _seed_full_corpus( - config: Config, paths: SourcePaths, cli_transcript_records: list[SourceRecord] + config: Config, + paths: SourcePaths, + cli_transcript_records: list[SourceRecord], + *, + native_session_id: str = NATIVE_SESSION_ID, ) -> str: - return _seed_archive(config, paths, _full_corpus_records(cli_transcript_records)) + return _seed_archive( + config, + paths, + _full_corpus_records(cli_transcript_records, native_session_id=native_session_id), + native_session_id=native_session_id, + ) def test_reconcile_archive_projects_six_call_corpus( @@ -217,10 +282,19 @@ def test_reconcile_archive_is_idempotent( stored = _seed_full_corpus(config, paths, cli_transcript_records) first = reconcile_archive(config, stored) + first_turns = read_projected_turns(config, stored) + first_state = load_projection_state(config, stored) + second = reconcile_archive(config, stored) + second_turns = read_projected_turns(config, stored) + second_state = load_projection_state(config, stored) assert first == second - assert read_projected_turns(config, stored) == read_projected_turns(config, stored) + assert first_turns == second_turns + assert first_state == second_state + for turn in second_turns: + seqs = [event.get("seq") for event in turn.get("events") or []] + assert len(seqs) == len(set(seqs)), "re-reconciling must not duplicate turn events" def test_rebuild_is_idempotent_and_matches_initial_reconcile( @@ -243,6 +317,7 @@ def test_rebuild_is_idempotent_and_matches_initial_reconcile( def test_incremental_reconcile_matches_full_replay( config: Config, paths: SourcePaths, + copilot_home: Path, cli_transcript_records: list[SourceRecord], ) -> None: source_key = _source_key(cli_transcript_records) @@ -256,16 +331,22 @@ def test_incremental_reconcile_matches_full_replay( incremental = reconcile_archive(config, stored_incremental) incremental_turns = read_projected_turns(config, stored_incremental) - stored_full = _seed_full_corpus(config, paths, cli_transcript_records) + # A genuinely independent stored session -- same source home, but a + # different native session ID -- committed in one shot from the same + # fixture content. Reusing the incremental session's own ID here would + # make "full" just another reconcile of the archive the incremental case + # already fully populated, which proves nothing about replay equivalence. + full_transcript = _drain_cli_transcript(copilot_home, native_session_id=FULL_NATIVE_SESSION_ID) + stored_full = _seed_full_corpus( + config, paths, full_transcript, native_session_id=FULL_NATIVE_SESSION_ID + ) full = reconcile_archive(config, stored_full) full_turns = read_projected_turns(config, stored_full) - assert incremental["turns"] == full["turns"] - assert incremental["usage"] == full["usage"] - assert incremental["events"] == full["events"] - assert [turn["turn_id"] for turn in incremental_turns] == [ - turn["turn_id"] for turn in full_turns - ] + assert incremental == full + normalized_incremental = _rewrite_native_id(incremental_turns, NATIVE_SESSION_ID) + normalized_full = _rewrite_native_id(full_turns, FULL_NATIVE_SESSION_ID) + assert normalized_incremental == normalized_full def test_reconcile_default_does_not_export( @@ -275,19 +356,19 @@ def test_reconcile_default_does_not_export( monkeypatch: pytest.MonkeyPatch, ) -> None: stored = _seed_full_corpus(config, paths, cli_transcript_records) - imports: list[str] = [] - original = __import__ - def tracking_import(name: str, *args: Any, **kwargs: Any) -> Any: - imports.append(name) - return original(name, *args, **kwargs) + def explode(*_args: Any, **_kwargs: Any) -> Any: + raise AssertionError("reconcile_archive must not import export assembly by default") - monkeypatch.setattr("importlib.import_module", tracking_import) + # Patch the name bound inside reconcile.py itself: `import_module` there + # is `from importlib import import_module`, a local reference that a + # patch on `importlib.import_module` would not intercept. + monkeypatch.setattr("thirdeye.platforms.copilot.reconcile.import_module", explode) result = reconcile_archive(config, stored) assert result["exports"] == 0 - assert not any("export" in item for item in imports) + assert result["errors"] == 0 def test_export_failure_does_not_roll_back_projection( @@ -360,6 +441,45 @@ def explode(*_args: Any, **_kwargs: Any) -> tuple[Any, dict[str, Any]]: assert read_projected_turns(config, stored) == prior_turns +def test_commit_failure_during_rebuild_preserves_prior_projection( + config: Config, + paths: SourcePaths, + cli_transcript_records: list[SourceRecord], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A rebuild that fails inside commit_projection (after build succeeds) + + must not lose the previous projection. This is the atomicity gap a + delete-then-commit rebuild would have: this test forces the failure to + happen after a valid projection has already been built, exercising the + commit step itself rather than the build step. + """ + stored = _seed_full_corpus(config, paths, cli_transcript_records) + baseline = reconcile_archive(config, stored) + prior_turns = read_projected_turns(config, stored) + prior_state = load_projection_state(config, stored) + + def build_with_invalid_usage_row( + records: list[SourceRecord], prior_state: dict[str, Any] + ) -> tuple[Any, dict[str, Any]]: + projection, next_state = _real_build_projection(records, prior_state) + broken = dict(projection) + broken["usage_rows"] = [*projection["usage_rows"], {"not": "a UsageRow instance"}] + return broken, next_state + + monkeypatch.setattr( + "thirdeye.platforms.copilot.reconcile.build_projection", build_with_invalid_usage_row + ) + + result = reconcile_archive(config, stored, rebuild=True) + + assert result["errors"] == baseline["errors"] + 1 + assert result["turns"] == baseline["turns"] + assert result["usage"] == baseline["usage"] + assert read_projected_turns(config, stored) == prior_turns + assert load_projection_state(config, stored) == prior_state + + def test_reconcile_unknown_session_reports_error_without_creating_paths( config: Config, ) -> None: From 3a8e94151754a47b7e77f09d2828dc93de5b06ca Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Sat, 12 Sep 2026 03:45:54 -0700 Subject: [PATCH 73/88] Fix Copilot export assembly delivery/eligibility bugs from review - Add a durable accounting/turn "sent" claim in otel_export so a successful worker delivery survives the job file being deleted; Copilot's ledger now self-heals its emitted flag from that claim instead of losing track of confirmed deliveries. - Only escalate a placement correction to a quarantined conflict once delivery is actually confirmed; pre-delivery corrections replace the queued placement instead of getting stuck. - Gate export eligibility on whether the owning turn's chat span was already flushed, so late-arriving matched usage falls back to an accounting span instead of being silently dropped or double-counted. - Compute the first-activation history boundary from actual terminal status instead of every currently-known id, so accounting owned by a still-open interaction stays eligible once it completes. - Resolve eligibility through the owning main interaction (not a nested child's own id), since children are never exported as independent jobs. - Make export_turn report durable queue acceptance (like export_spans) so a turn-job failure is recorded rather than silently counted. - Fail closed on an unrecognized export ledger schema_version instead of silently treating it as empty state. Co-Authored-By: Claude Sonnet 5 --- src/thirdeye/otel_export.py | 46 +- src/thirdeye/platforms/copilot/export.py | 144 ++++- .../platforms/copilot/export_state.py | 177 ++++-- tests/platforms/copilot/test_export.py | 512 +++++++++++++++++- 4 files changed, 801 insertions(+), 78 deletions(-) diff --git a/src/thirdeye/otel_export.py b/src/thirdeye/otel_export.py index bbb202d..1720898 100644 --- a/src/thirdeye/otel_export.py +++ b/src/thirdeye/otel_export.py @@ -580,6 +580,39 @@ def turn_export_sent(session_dir_: Path, turn_id: str) -> bool: return False +def _accounting_claim_path(session_dir_: Path, accounting_id: str) -> Path: + # Same hashed-filename rationale as `_turn_claim_path`: an accounting id + # is caller-derived and not guaranteed filesystem-safe. + digest = hashlib.sha256(accounting_id.encode()).hexdigest() + return session_dir_ / "otel-accounting-sent" / f"{digest}.json" + + +def accounting_export_sent(session_dir_: Path, accounting_id: str) -> bool: + """Whether one accounting identity's fallback/chat tokens were ever + confirmed flushed to Logfire. + + The deterministic accounting job file the worker writes is deleted once + delivery succeeds (see `otel_worker._run_accounting_job`), so it cannot + be reused as a durable "already delivered" signal — a later caller would + see no job file and wrongly conclude nothing was ever sent, and requeue + a duplicate. This claim persists independently of that job file, mirroring + `_turn_claim_path`, so a caller like Copilot's export ledger can tell a + confirmed delivery apart from one that is merely queued or still retrying. + """ + try: + return _accounting_claim_path(session_dir_, accounting_id).read_text( + encoding="utf-8" + ) == "sent" + except OSError: + return False + + +def _mark_accounting_sent(session_dir_: Path, accounting_id: str) -> None: + path = _accounting_claim_path(session_dir_, accounting_id) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("sent", encoding="utf-8", newline="\n") + + def _claim_turn_export(session_dir_: Path, turn_id: str) -> bool: """First-wins claim on exporting this turn's span tree, ever, for this session. A replayed/duplicate hook invocation for the same turn (e.g. the @@ -617,16 +650,21 @@ def export_turn( turn: TurnSpanDict, *, captured_env: dict[str, str] | None = None, -) -> None: +) -> bool: """Hand a completed turn off for background export. Never raises, never blocks on network I/O — the actual Logfire call happens in a detached child process this spawns and does not wait for. See module docstring. ``captured_env`` lets a caller whose own ``os.environ`` is unreliable supply the raw opted-in env dict; when omitted it is read here. + + Returns whether the local job was durably written and dispatched, same + contract as :func:`export_spans` — never whether Logfire ever received it. + No existing caller inspects the return value, so this is backward + compatible with every platform still calling this positionally. """ if not config.logfire.enabled or not config.logfire.token: - return + return False try: job_path = _write_job( config.root, @@ -641,6 +679,7 @@ def export_turn( }, ) _spawn(job_path) + return True except Exception as exc: log_capture_error( thirdeye_home=config.root, @@ -649,6 +688,7 @@ def export_turn( platform=platform, session_id=session_id, ) + return False def export_spans( @@ -1085,6 +1125,7 @@ def _export_session_accounting_inner( ) if instance.force_flush(timeout_millis=_FLUSH_TIMEOUT_MS) is False: raise RuntimeError("session accounting export was not flushed") + _mark_accounting_sent(session_dir_, str(accounting["accounting_id"])) def _export_turn_accounting_inner( @@ -1126,6 +1167,7 @@ def _export_turn_accounting_inner( ) if instance.force_flush(timeout_millis=_FLUSH_TIMEOUT_MS) is False: raise RuntimeError("turn accounting export was not flushed") + _mark_accounting_sent(session_dir_, str(accounting["accounting_id"])) @lru_cache(maxsize=128) diff --git a/src/thirdeye/platforms/copilot/export.py b/src/thirdeye/platforms/copilot/export.py index 8ed874b..c0e996d 100644 --- a/src/thirdeye/platforms/copilot/export.py +++ b/src/thirdeye/platforms/copilot/export.py @@ -3,7 +3,11 @@ No network operation happens here. This module only writes/dispatches the generic transport's local jobs; the detached worker performs remote delivery. The split cannot provide transactional exactly-once delivery: a crash after a -remote flush and before acknowledgement can retry a deterministic span. +remote flush and before acknowledgement can retry a deterministic span. The +transport's own durable claims (`otel_export.turn_export_sent` / +`accounting_export_sent`) are what let this module tell "already confirmed +delivered" apart from "merely queued" across restarts, since the worker +deletes its own job file on success. """ from __future__ import annotations @@ -20,10 +24,13 @@ from .constants import PLATFORM_NAME from .export_state import ( + clear_placement_error, + clear_turn_error, initialize_eligibility, is_accounting_eligible, is_turn_eligible, mark_placement_error, + mark_turn_error, record_placement, update_export_state, ) @@ -37,8 +44,8 @@ def _directory(config: Config, stored_session_id: str) -> Path: return session_dir(config.root, PLATFORM_NAME, stored_session_id) -def _terminal(turn: dict[str, Any]) -> bool: - return turn.get("status") in _TERMINAL +def _terminal(turn: dict[str, Any] | None) -> bool: + return bool(turn) and turn.get("status") in _TERMINAL def _walk_turns(turns: list[dict[str, Any]]) -> list[dict[str, Any]]: @@ -61,6 +68,37 @@ def _main_terminal_turns(projection: Projection) -> list[dict[str, Any]]: ] +def _root_owner_index(projection: Projection) -> dict[str, dict[str, Any]]: + """Map every turn id, main or nested, to its owning main ``TurnSpanDict``. + + Eligibility and "is this history" are properties of a whole interaction + (a main turn and everything nested under it, exported together as one + job), never of a nested child's own id — the child is never exported as + an independent top-level job, so its id alone is never a member of the + boundary lists built from ``_main_terminal_turns``. + """ + index: dict[str, dict[str, Any]] = {} + + def walk(turn: dict[str, Any], root: dict[str, Any]) -> None: + turn_id = turn.get("turn_id") + if isinstance(turn_id, str): + index[turn_id] = root + for child in turn.get("subagents") or []: + if isinstance(child, dict): + walk(child, root) + + for turn in projection["turns"]: + if isinstance(turn, dict): + walk(turn, turn) + return index + + +def _root_for(root_index: dict[str, dict[str, Any]], turn_id: object) -> dict[str, Any] | None: + if isinstance(turn_id, str): + return root_index.get(turn_id) + return None + + def _accounting_calls(projection: Projection) -> dict[str, tuple[dict[str, Any], dict[str, Any]]]: """Map an identity to its owner turn and serialized generic accounting.""" found: dict[str, tuple[dict[str, Any], dict[str, Any]]] = {} @@ -75,10 +113,30 @@ def _accounting_calls(projection: Projection) -> dict[str, tuple[dict[str, Any], return found -def _usage_ids(projection: Projection) -> list[str]: - ids = [row.call_id for row in projection["usage_rows"]] - ids.extend(item["logical_call_id"] for item in projection["attributions"]) - return sorted({item for item in ids if isinstance(item, str) and item}) +def _historical_accounting_ids( + projection: Projection, root_index: dict[str, dict[str, Any]] +) -> list[str]: + """Accounting identities that are already part of terminal history. + + An identity owned by a still-open interaction must be excluded from this + set (and therefore stay eligible for later export once that interaction + completes). An identity with no interaction to gate on at all (an + ownerless usage row, or a ``stored_turn_id`` this projection cannot + resolve) has no way to become "no longer historical" later, so it keeps + the conservative default of being treated as already-seen history. + """ + owners: dict[str, dict[str, Any] | None] = {} + for accounting_id, (owner, _item) in _accounting_calls(projection).items(): + owners[accounting_id] = _root_for(root_index, owner.get("turn_id")) + for attribution in projection["attributions"]: + accounting_id = attribution["logical_call_id"] + if accounting_id not in owners: + owners[accounting_id] = _root_for(root_index, attribution["stored_turn_id"]) + for row in projection["usage_rows"]: + owners.setdefault(row.call_id, None) + return sorted( + accounting_id for accounting_id, root in owners.items() if root is None or _terminal(root) + ) def _span_id(session_id: str, turn_id: str | None, accounting_id: str) -> str: @@ -103,10 +161,32 @@ def update(state: dict[str, Any]) -> dict[str, Any]: update_export_state(config, stored_session_id, update) +def _turn_error_state(config: Config, stored_session_id: str, turn_id: str, message: str) -> None: + def update(state: dict[str, Any]) -> dict[str, Any]: + return mark_turn_error(state, turn_id, message) + + update_export_state(config, stored_session_id, update) + + +def _clear_error_state(config: Config, stored_session_id: str, accounting_id: str) -> None: + def update(state: dict[str, Any]) -> dict[str, Any]: + return clear_placement_error(state, accounting_id) + + update_export_state(config, stored_session_id, update) + + +def _clear_turn_error_state(config: Config, stored_session_id: str, turn_id: str) -> None: + def update(state: dict[str, Any]) -> dict[str, Any]: + return clear_turn_error(state, turn_id) + + update_export_state(config, stored_session_id, update) + + def _eligible_state( config: Config, stored_session_id: str, projection: Projection, + root_index: dict[str, dict[str, Any]], *, include_history: bool, ) -> dict[str, Any]: @@ -116,7 +196,7 @@ def update(state: dict[str, Any]) -> dict[str, Any]: return initialize_eligibility( state, terminal_turn_ids=terminal_ids, - accounting_ids=_usage_ids(projection), + accounting_ids=_historical_accounting_ids(projection, root_index), include_history=include_history, ) @@ -131,6 +211,7 @@ def _place( destination: str, span_id: str, usage: dict[str, Any], + delivered: bool, ) -> tuple[dict[str, Any] | None, bool]: captured: dict[str, Any] = {} @@ -141,6 +222,7 @@ def update(state: dict[str, Any]) -> dict[str, Any]: destination=destination, span_id=span_id, usage=usage, + delivered=delivered, ) captured["entry"] = entry captured["accepted"] = accepted @@ -169,9 +251,7 @@ def _turn_with_placed_accounting( return result -def _with_deterministic_turn_ids( - turn: dict[str, Any], stored_session_id: str -) -> dict[str, Any]: +def _with_deterministic_turn_ids(turn: dict[str, Any], stored_session_id: str) -> dict[str, Any]: """Finalize a complete turn tree with ids stable across archive replays. Copilot's stable turn identity is a source-derived string rather than the @@ -210,8 +290,9 @@ def queue_exports( meta = read_meta(meta_path(directory)) if meta is None: raise ValueError(f"unknown Copilot session: {stored_session_id}") + root_index = _root_owner_index(projection) state = _eligible_state( - config, stored_session_id, projection, include_history=include_history + config, stored_session_id, projection, root_index, include_history=include_history ) if not _configured(config): return 0 @@ -225,22 +306,35 @@ def queue_exports( # fallback charge onto a chat span. for accounting_id, (owner, item) in accounting.items(): owner_id = owner.get("turn_id") + root = _root_for(root_index, owner_id) usage = item["usage"] if ( not isinstance(owner_id, str) - or not _terminal(owner) + or root is None + or not _terminal(root) or item.get("attribution_status") not in _EXPORTABLE_ATTRIBUTIONS ): continue - if not is_turn_eligible(state, owner_id) or not is_accounting_eligible(state, accounting_id): + root_id = str(root["turn_id"]) + if not is_turn_eligible(state, root_id) or not is_accounting_eligible(state, accounting_id): continue call_id = item.get("call_id") - if item.get("attribution_status") == "matched" and isinstance(call_id, str) and call_id: + # A chat span already flushed to Logfire is immutable history: usage + # that resolves to "matched" only *after* that flush can no longer + # land on it and must use the turn-owned fallback span instead. + chat_available = not otel_export.turn_export_sent(directory, root_id) + if ( + chat_available + and item.get("attribution_status") == "matched" + and isinstance(call_id, str) + and call_id + ): destination = "chat-span" span_id = str(call_id) else: destination = "turn-accounting-span" span_id = _span_id(stored_session_id, owner_id, accounting_id) + delivered = otel_export.accounting_export_sent(directory, accounting_id) entry, accepted = _place( config, stored_session_id, @@ -248,14 +342,15 @@ def queue_exports( destination=destination, span_id=span_id, usage=usage, + delivered=delivered, ) if not accepted: continue if entry is not None: placements[accounting_id] = entry if entry is not None and entry.get("emitted"): - # ``emitted`` means a delivery acknowledgement, unlike a queued - # job. Never recreate an accounting job after that point. + # Confirmed delivered, whether just now or on an earlier pass. + # Never recreate an accounting job after that point. continue if destination != "turn-accounting-span" or entry is None: continue @@ -271,6 +366,8 @@ def queue_exports( ) if sent: queued += 1 + if entry.get("last_error") is not None: + _clear_error_state(config, stored_session_id, accounting_id) else: _error_state(config, stored_session_id, accounting_id, "accounting job was not queued") @@ -291,6 +388,7 @@ def queue_exports( if usage is None: continue span_id = _span_id(stored_session_id, None, accounting_id) + delivered = otel_export.accounting_export_sent(directory, accounting_id) entry, accepted = _place( config, stored_session_id, @@ -298,6 +396,7 @@ def queue_exports( destination="session-accounting-span", span_id=span_id, usage=usage, + delivered=delivered, ) if not accepted or entry is None: continue @@ -320,6 +419,8 @@ def queue_exports( ) if sent: queued += 1 + if entry.get("last_error") is not None: + _clear_error_state(config, stored_session_id, accounting_id) else: _error_state(config, stored_session_id, accounting_id, "accounting job was not queued") @@ -332,8 +433,13 @@ def queue_exports( assembled = _with_deterministic_turn_ids( _turn_with_placed_accounting(turn, placements), stored_session_id ) - otel_export.export_turn( + sent = otel_export.export_turn( config, directory, stored_session_id, PLATFORM_NAME, meta.cwd, assembled ) - queued += 1 + if sent: + queued += 1 + if turn_id in (state.get("turn_errors") or {}): + _clear_turn_error_state(config, stored_session_id, turn_id) + else: + _turn_error_state(config, stored_session_id, turn_id, "turn export job was not queued") return queued diff --git a/src/thirdeye/platforms/copilot/export_state.py b/src/thirdeye/platforms/copilot/export_state.py index b7211d9..ec2abc0 100644 --- a/src/thirdeye/platforms/copilot/export_state.py +++ b/src/thirdeye/platforms/copilot/export_state.py @@ -6,6 +6,14 @@ atomically acknowledge a remote collector and this file, so an absent job is never treated as proof of delivery. A crash after a remote flush can still lead to a deterministic retry and therefore a duplicate remote span. + +``record_placement`` takes an explicit ``delivered`` flag from the caller +(who reads the generic transport's independent, durable +``otel_export.accounting_export_sent`` claim before calling in). That flag, +not merely a changed destination, is what turns a correction into a +quarantined conflict: a correction to a job that never left this machine is +always safe to replace, while a correction after confirmed delivery can no +longer relocate tokens the remote collector already has. """ from __future__ import annotations @@ -49,6 +57,7 @@ def empty_export_state() -> dict[str, Any]: "excluded_accounting_ids": [], "placements": {}, "conflicts": {}, + "turn_errors": {}, } @@ -62,32 +71,50 @@ def _mapping(value: object) -> dict[str, Any]: return value if isinstance(value, dict) else {} -def _state(value: dict[str, Any] | None) -> dict[str, Any]: - if value is None or value.get("schema_version") != EXPORT_STATE_SCHEMA_VERSION: - return empty_export_state() +def _normalize(value: dict[str, Any]) -> dict[str, Any]: + """Fill defaults on a dict already known to carry the current schema. + + Only safe for state this module itself produced (loaded and version + checked by :func:`_read`, or built fresh by another function here) — + never call this directly on unvalidated bytes off disk. + """ placements = _mapping(value.get("placements")) conflicts = _mapping(value.get("conflicts")) + turn_errors = _mapping(value.get("turn_errors")) return { "schema_version": EXPORT_STATE_SCHEMA_VERSION, "activated": bool(value.get("activated", False)), "excluded_turn_ids": _ids(value.get("excluded_turn_ids")), "excluded_accounting_ids": _ids(value.get("excluded_accounting_ids")), - "placements": {key: item for key, item in placements.items() if isinstance(key, str) and isinstance(item, dict)}, - "conflicts": {key: item for key, item in conflicts.items() if isinstance(key, str) and isinstance(item, dict)}, + "placements": { + key: item for key, item in placements.items() if isinstance(key, str) and isinstance(item, dict) + }, + "conflicts": { + key: item for key, item in conflicts.items() if isinstance(key, str) and isinstance(item, dict) + }, + "turn_errors": { + key: value2 for key, value2 in turn_errors.items() if isinstance(key, str) and isinstance(value2, str) + }, } def _read(directory: Path) -> dict[str, Any]: - try: - return _state( - read_json_object( - export_state_path(directory), invalid_message="invalid Copilot export ledger" - ) - ) - except ValueError: - # Export accounting is not disposable. Do not overwrite a corrupt - # ledger and risk emitting tokens at another location. - raise ValueError("invalid Copilot export ledger") from None + """Load the ledger, failing closed on anything but a genuinely absent file. + + A missing file is the only case that legitimately means "no export has + ever run for this session" and may start from empty state. Malformed + JSON, a non-object document, or an unrecognized ``schema_version`` are + all corruption or a future/unknown format from this module's point of + view: silently treating them as empty would forget every placement this + ledger recorded and risk emitting tokens at a second location. + """ + raw = read_json_object(export_state_path(directory), invalid_message="invalid Copilot export ledger") + if raw is None: + return empty_export_state() + version = raw.get("schema_version") + if version != EXPORT_STATE_SCHEMA_VERSION: + raise ValueError(f"unsupported Copilot export ledger schema_version: {version!r}") + return _normalize(raw) def _write(directory: Path, state: dict[str, Any]) -> None: @@ -105,9 +132,7 @@ def load_export_state(config: Config, stored_session_id: str) -> dict[str, Any]: return json.loads(json.dumps(_read(directory))) -def update_export_state( - config: Config, stored_session_id: str, update: Any -) -> dict[str, Any]: +def update_export_state(config: Config, stored_session_id: str, update: Any) -> dict[str, Any]: """Atomically apply ``update(state)`` and return the published state.""" directory = _directory(config, stored_session_id) with locked(export_lock_path(directory), LockMode.EXCLUSIVE): @@ -115,7 +140,7 @@ def update_export_state( updated = update(state) if not isinstance(updated, dict): raise TypeError("Copilot export state update must return a dictionary") - state = _state(updated) + state = _normalize(updated) _write(directory, state) return json.loads(json.dumps(state)) @@ -147,8 +172,14 @@ def initialize_eligibility( export opts the supplied completed turns and accounting identities in by removing them from that boundary. New evidence is absent from both lists and is therefore eligible after restart. + + Callers are responsible for passing only turn ids and accounting ids that + are *actually* already historical (a terminal main interaction, or + accounting owned by one) — an interaction still open at first activation + must not appear here, or it would stay excluded forever even once it + later completes. """ - result = _state(state) + result = _normalize(state) turn_ids = set(_ids(terminal_turn_ids)) usage_ids = set(_ids(accounting_ids)) if not result["activated"]: @@ -159,9 +190,7 @@ def initialize_eligibility( return result if include_history: result["excluded_turn_ids"] = sorted(set(result["excluded_turn_ids"]) - turn_ids) - result["excluded_accounting_ids"] = sorted( - set(result["excluded_accounting_ids"]) - usage_ids - ) + result["excluded_accounting_ids"] = sorted(set(result["excluded_accounting_ids"]) - usage_ids) return result @@ -172,42 +201,77 @@ def record_placement( destination: str, span_id: str, usage: dict[str, Any], + delivered: bool = False, ) -> tuple[dict[str, Any], dict[str, Any] | None, bool]: """Persist the first token location for an accounting identity. - Returns ``(state, entry, accepted)``. A source correction or a changed - destination after placement is a durable conflict: silently replacing a - queued job could produce two different token totals at the same span. + ``delivered`` is the caller's fresh read of the generic transport's + durable delivery claim for this identity (see + ``otel_export.accounting_export_sent``), not this ledger's own possibly + stale ``emitted`` flag — the local job file backing that flag is deleted + by the worker on success, so this ledger cannot detect delivery on its + own and must be told. + + Returns ``(state, entry, accepted)``. A correction is only a durable + conflict when the existing placement was (or is now known to have been) + delivered: rebinding a destination or usage value the remote collector + already received would produce two different token totals for the same + logical call. A correction to a placement that was only ever queued + locally is always safe to replace/requeue. """ - result = _state(state) + result = _normalize(state) digest = usage_digest(usage) placements = result["placements"] existing = _mapping(placements.get(accounting_id)) if existing: + already_delivered = bool(existing.get("emitted")) or delivered same = ( existing.get("destination") == destination and existing.get("span_id") == span_id and existing.get("usage_digest") == digest ) if same: + if delivered and not existing.get("emitted"): + existing = {**existing, "emitted": True, "last_error": None} + placements[accounting_id] = existing return result, existing, True - result["conflicts"][accounting_id] = { + if already_delivered: + if delivered and not existing.get("emitted"): + # The candidate is rejected, but the fresh delivery read is + # still new information about the *existing* placement — + # record it so the ledger stops looking like it was only + # ever queued. + existing = {**existing, "emitted": True} + placements[accounting_id] = existing + result["conflicts"][accounting_id] = { + "accounting_id": accounting_id, + "reason": "accounting placement or usage changed after confirmed delivery", + "existing": existing, + "candidate": { + "destination": destination, + "span_id": span_id, + "usage_digest": digest, + }, + } + return result, existing, False + # Nothing was ever confirmed delivered for this identity: replace the + # queued/failed placement outright rather than quarantining it. + entry = { "accounting_id": accounting_id, - "reason": "accounting placement or usage changed after queueing", - "existing": existing, - "candidate": { - "destination": destination, - "span_id": span_id, - "usage_digest": digest, - }, + "destination": destination, + "span_id": span_id, + "usage_digest": digest, + "emitted": False, + "last_error": None, } - return result, existing, False + placements[accounting_id] = entry + return result, entry, True entry = { "accounting_id": accounting_id, "destination": destination, "span_id": span_id, "usage_digest": digest, - "emitted": False, + "emitted": bool(delivered), "last_error": None, } placements[accounting_id] = entry @@ -215,7 +279,7 @@ def record_placement( def mark_placement_error(state: dict[str, Any], accounting_id: str, error: str) -> dict[str, Any]: - result = _state(state) + result = _normalize(state) entry = _mapping(result["placements"].get(accounting_id)) if entry: entry["last_error"] = error @@ -223,14 +287,25 @@ def mark_placement_error(state: dict[str, Any], accounting_id: str, error: str) return result +def clear_placement_error(state: dict[str, Any], accounting_id: str) -> dict[str, Any]: + """Drop a stale ``last_error`` once a retried job successfully queues.""" + result = _normalize(state) + entry = _mapping(result["placements"].get(accounting_id)) + if entry and entry.get("last_error") is not None: + entry["last_error"] = None + result["placements"][accounting_id] = entry + return result + + def mark_placement_delivered(state: dict[str, Any], accounting_id: str) -> dict[str, Any]: """Record an externally confirmed delivery, never inferred from a job file. - The current detached worker has no transactional callback into this - ledger. This hook is deliberately available for a future confirmed - transport, while ordinary queueing leaves ``emitted`` false. + Ordinary reconciliation no longer needs to call this directly — + ``record_placement``'s ``delivered`` flag self-heals ``emitted`` from the + durable transport claim on every pass — but it remains available for a + caller with its own confirmed-delivery source. """ - result = _state(state) + result = _normalize(state) entry = _mapping(result["placements"].get(accounting_id)) if entry: entry["emitted"] = True @@ -239,6 +314,24 @@ def mark_placement_delivered(state: dict[str, Any], accounting_id: str) -> dict[ return result +def mark_turn_error(state: dict[str, Any], turn_id: str, error: str) -> dict[str, Any]: + """Record that a whole-turn export job failed to queue locally. + + Kept separate from ``placements`` (keyed by accounting id, not turn id) + so a turn-job failure is visible without inventing a fake accounting + record for it. + """ + result = _normalize(state) + result["turn_errors"][turn_id] = error + return result + + +def clear_turn_error(state: dict[str, Any], turn_id: str) -> dict[str, Any]: + result = _normalize(state) + result["turn_errors"].pop(turn_id, None) + return result + + def remove_export_state(config: Config, stored_session_id: str) -> None: """Remove export state only for an explicit export-ledger reset. diff --git a/tests/platforms/copilot/test_export.py b/tests/platforms/copilot/test_export.py index 35be70e..c2c994d 100644 --- a/tests/platforms/copilot/test_export.py +++ b/tests/platforms/copilot/test_export.py @@ -216,8 +216,9 @@ def export_calls(monkeypatch: pytest.MonkeyPatch) -> dict[str, list[Any]]: "session_accounting": [], } - def _turn(*args: Any, **kwargs: Any) -> None: + def _turn(*args: Any, **kwargs: Any) -> bool: calls["turn"].append(args[5]) + return True def _turn_accounting(*args: Any, **kwargs: Any) -> bool: calls["turn_accounting"].append(args[6]) @@ -293,7 +294,11 @@ def test_record_placement_is_idempotent_for_same_decision(self) -> None: assert same_accepted is True assert same_entry == entry - def test_record_placement_conflicts_on_changed_destination(self) -> None: + def test_record_placement_replaces_undelivered_placement_on_changed_destination( + self, + ) -> None: + """Nothing was ever confirmed delivered, so a corrected destination + just replaces the queued-but-unconfirmed placement outright.""" usage = _usage_row().to_dict() state, _, accepted = record_placement( empty_export_state(), @@ -303,6 +308,56 @@ def test_record_placement_conflicts_on_changed_destination(self) -> None: usage=usage, ) assert accepted is True + replaced, entry, accepted_again = record_placement( + state, + accounting_id="acct-1", + destination="chat-span", + span_id="call-1", + usage=usage, + ) + assert accepted_again is True + assert entry is not None + assert entry["destination"] == "chat-span" + assert replaced["placements"]["acct-1"]["destination"] == "chat-span" + assert "acct-1" not in replaced["conflicts"] + + def test_record_placement_replaces_undelivered_placement_on_usage_correction( + self, + ) -> None: + usage = _usage_row().to_dict() + state, _, accepted = record_placement( + empty_export_state(), + accounting_id="acct-1", + destination="chat-span", + span_id="call-1", + usage=usage, + ) + assert accepted is True + corrected = dict(usage) + corrected["output_tokens"] = 999 + replaced, entry, accepted_again = record_placement( + state, + accounting_id="acct-1", + destination="chat-span", + span_id="call-1", + usage=corrected, + ) + assert accepted_again is True + assert entry is not None + assert entry["usage_digest"] == usage_digest(corrected) + assert "acct-1" not in replaced["conflicts"] + + def test_record_placement_conflicts_on_changed_destination_after_delivery(self) -> None: + usage = _usage_row().to_dict() + state, _, accepted = record_placement( + empty_export_state(), + accounting_id="acct-1", + destination="turn-accounting-span", + span_id="span-a", + usage=usage, + delivered=True, + ) + assert accepted is True conflicted, existing, rejected = record_placement( state, accounting_id="acct-1", @@ -315,7 +370,7 @@ def test_record_placement_conflicts_on_changed_destination(self) -> None: assert existing["destination"] == "turn-accounting-span" assert "acct-1" in conflicted["conflicts"] - def test_record_placement_conflicts_on_usage_correction(self) -> None: + def test_record_placement_conflicts_on_usage_correction_after_delivery(self) -> None: usage = _usage_row().to_dict() state, _, accepted = record_placement( empty_export_state(), @@ -323,6 +378,7 @@ def test_record_placement_conflicts_on_usage_correction(self) -> None: destination="chat-span", span_id="call-1", usage=usage, + delivered=True, ) assert accepted is True corrected = dict(usage) @@ -340,6 +396,34 @@ def test_record_placement_conflicts_on_usage_correction(self) -> None: corrected ) + def test_record_placement_self_heals_emitted_flag_when_delivery_confirmed(self) -> None: + """Same destination/span/usage as before, but the caller now has a + fresh durable-claim read showing delivery succeeded: the ledger's own + ``emitted`` flag was never told directly, so this is the only place + it catches up.""" + usage = _usage_row().to_dict() + state, entry, _ = record_placement( + empty_export_state(), + accounting_id="acct-1", + destination="chat-span", + span_id="call-1", + usage=usage, + ) + assert entry is not None + assert entry["emitted"] is False + healed, healed_entry, accepted = record_placement( + state, + accounting_id="acct-1", + destination="chat-span", + span_id="call-1", + usage=usage, + delivered=True, + ) + assert accepted is True + assert healed_entry is not None + assert healed_entry["emitted"] is True + assert healed["placements"]["acct-1"]["emitted"] is True + class TestQueueExports: def test_unknown_session_raises(self, enabled_config: Config) -> None: @@ -363,6 +447,26 @@ def test_unconfigured_returns_zero_but_activates( assert TURN_ONE in state["excluded_turn_ids"] assert ACCOUNTING_MATCHED in state["excluded_accounting_ids"] + def test_unsupported_ledger_schema_version_fails_closed( + self, config: Config, paths: SourcePaths + ) -> None: + """A missing ledger file legitimately starts from empty state, but a + present file with an unrecognized ``schema_version`` must never be + silently treated as empty -- that would forget every recorded + placement and risk emitting tokens at a second location.""" + stored = _seed_session(config, paths) + directory = _directory(config, stored) + directory.mkdir(parents=True, exist_ok=True) + export_state_path(directory).write_text( + json.dumps({"schema_version": 999, "placements": {"acct-1": {"emitted": True}}}), + encoding="utf-8", + ) + with pytest.raises(ValueError, match="unsupported Copilot export ledger schema_version"): + load_export_state(config, stored) + projection = _projection(turns=[_main_turn()]) + with pytest.raises(ValueError, match="unsupported Copilot export ledger schema_version"): + queue_exports(config, stored, projection) + def test_first_activation_without_history_queues_nothing( self, enabled_config: Config, @@ -534,13 +638,7 @@ def test_pending_and_conflicting_attributions_are_not_exported( assert "pending-call" not in state["placements"] assert "conflict-call" not in state["placements"] - def test_fallback_placement_prevents_later_chat_relocation( - self, - enabled_config: Config, - paths: SourcePaths, - export_calls: dict[str, list[Any]], - ) -> None: - stored = _seed_session(enabled_config, paths) + def _ambiguous_then_matched_projections(self) -> tuple[Projection, Projection]: ambiguous_projection = _projection( turns=[ _main_turn( @@ -563,10 +661,6 @@ def test_fallback_placement_prevents_later_chat_relocation( ) ], ) - queue_exports(enabled_config, stored, ambiguous_projection, include_history=True) - export_calls["turn"].clear() - export_calls["turn_accounting"].clear() - matched_projection = _projection( turns=[ _main_turn( @@ -583,16 +677,93 @@ def test_fallback_placement_prevents_later_chat_relocation( usage_rows=[_usage_row(call_id=ACCOUNTING_UNMATCHED, input_tokens=6587, output_tokens=5)], attributions=[_attribution(logical_call_id=ACCOUNTING_UNMATCHED, call_id=CALL_MATCHED)], ) + return ambiguous_projection, matched_projection + + def test_pre_delivery_correction_relocates_to_chat_without_conflict( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + ) -> None: + """Nothing was ever confirmed delivered for the fallback placement + (no real worker ran), so improved local matching is still free to + relocate the tokens onto the now-known chat span.""" + stored = _seed_session(enabled_config, paths) + ambiguous_projection, matched_projection = self._ambiguous_then_matched_projections() + queue_exports(enabled_config, stored, ambiguous_projection, include_history=True) + export_calls["turn"].clear() + export_calls["turn_accounting"].clear() + + queue_exports(enabled_config, stored, matched_projection, include_history=True) + + state = load_export_state(enabled_config, stored) + placement = state["placements"][ACCOUNTING_UNMATCHED] + assert placement["destination"] == "chat-span" + assert placement["span_id"] == CALL_MATCHED + assert ACCOUNTING_UNMATCHED not in state["conflicts"] + turn = export_calls["turn"][0] + assert turn["accounting_calls"][0]["accounting_id"] == ACCOUNTING_UNMATCHED + + def test_fallback_placement_prevents_later_chat_relocation_after_confirmed_delivery( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + ) -> None: + """Once the durable transport claim shows the fallback span was + actually flushed, later local matching can no longer move those + tokens onto the chat span — that would double the emitted total.""" + stored = _seed_session(enabled_config, paths) + ambiguous_projection, matched_projection = self._ambiguous_then_matched_projections() + queue_exports(enabled_config, stored, ambiguous_projection, include_history=True) + + directory = _directory(enabled_config, stored) + otel_export._mark_accounting_sent(directory, ACCOUNTING_UNMATCHED) + + export_calls["turn"].clear() + export_calls["turn_accounting"].clear() + queue_exports(enabled_config, stored, matched_projection, include_history=True) state = load_export_state(enabled_config, stored) placement = state["placements"][ACCOUNTING_UNMATCHED] assert placement["destination"] == "turn-accounting-span" + assert placement["emitted"] is True assert ACCOUNTING_UNMATCHED in state["conflicts"] assert export_calls["turn_accounting"] == [] turn = export_calls["turn"][0] assert turn["accounting_calls"] == [] + def test_confirmed_turn_delivery_routes_new_match_to_fallback( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + ) -> None: + """Even with no prior accounting placement at all, a chat span that + the durable turn claim shows was already flushed is immutable: a + newly-matched usage row must land on a turn-accounting fallback + span, never on that already-sent chat span.""" + stored = _seed_session(enabled_config, paths) + turn_only_projection = _projection(turns=[_main_turn()]) + queue_exports(enabled_config, stored, turn_only_projection, include_history=True) + + directory = _directory(enabled_config, stored) + claim_path = otel_export._turn_claim_path(directory, TURN_ONE) + claim_path.parent.mkdir(parents=True, exist_ok=True) + claim_path.write_text("sent", encoding="utf-8", newline="\n") + + export_calls["turn"].clear() + _, matched_projection = self._ambiguous_then_matched_projections() + queue_exports(enabled_config, stored, matched_projection, include_history=True) + + state = load_export_state(enabled_config, stored) + placement = state["placements"][ACCOUNTING_UNMATCHED] + assert placement["destination"] == "turn-accounting-span" + assert len(export_calls["turn_accounting"]) == 1 + turn = export_calls["turn"][0] + assert turn["accounting_calls"] == [] + def test_emitted_placement_skips_requeue( self, enabled_config: Config, @@ -681,7 +852,7 @@ def test_accounting_job_failure_records_error( monkeypatch: pytest.MonkeyPatch, ) -> None: stored = _seed_session(enabled_config, paths) - monkeypatch.setattr(otel_export, "export_turn", lambda *args, **kwargs: None) + monkeypatch.setattr(otel_export, "export_turn", lambda *args, **kwargs: True) monkeypatch.setattr(otel_export, "export_turn_accounting", lambda *args, **kwargs: False) monkeypatch.setattr(otel_export, "export_session_accounting", lambda *args, **kwargs: False) @@ -714,6 +885,71 @@ def test_accounting_job_failure_records_error( "accounting job was not queued" ) + def test_turn_job_failure_is_not_counted_and_is_recorded( + self, + enabled_config: Config, + paths: SourcePaths, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """`otel_export.export_turn` now reports durable queue acceptance, + same contract as `export_spans`; a spawn/write failure there must not + be silently counted as a successful queue.""" + stored = _seed_session(enabled_config, paths) + monkeypatch.setattr(otel_export, "export_turn", lambda *args, **kwargs: False) + + projection = _projection(turns=[_main_turn()]) + queued = queue_exports(enabled_config, stored, projection, include_history=True) + assert queued == 0 + state = load_export_state(enabled_config, stored) + assert state["turn_errors"][TURN_ONE] == "turn export job was not queued" + + def test_errors_clear_once_a_retry_succeeds( + self, + enabled_config: Config, + paths: SourcePaths, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """A stale `last_error`/turn error from an earlier failed attempt must + not linger once a later reconciliation successfully queues the job.""" + stored = _seed_session(enabled_config, paths) + monkeypatch.setattr(otel_export, "export_turn", lambda *args, **kwargs: False) + monkeypatch.setattr(otel_export, "export_turn_accounting", lambda *args, **kwargs: False) + + projection = _projection( + turns=[ + _main_turn( + accounting_calls=[ + _accounting_call( + accounting_id=ACCOUNTING_UNMATCHED, + attribution_status="ambiguous", + call_id=None, + usage=_usage_row(call_id=ACCOUNTING_UNMATCHED).to_dict(), + ) + ] + ) + ], + usage_rows=[_usage_row(call_id=ACCOUNTING_UNMATCHED, input_tokens=6587, output_tokens=5)], + attributions=[ + _attribution( + logical_call_id=ACCOUNTING_UNMATCHED, status="ambiguous", call_id=None + ) + ], + ) + queue_exports(enabled_config, stored, projection, include_history=True) + state = load_export_state(enabled_config, stored) + assert state["turn_errors"][TURN_ONE] == "turn export job was not queued" + assert state["placements"][ACCOUNTING_UNMATCHED]["last_error"] == ( + "accounting job was not queued" + ) + + monkeypatch.setattr(otel_export, "export_turn", lambda *args, **kwargs: True) + monkeypatch.setattr(otel_export, "export_turn_accounting", lambda *args, **kwargs: True) + queue_exports(enabled_config, stored, projection, include_history=True) + + state = load_export_state(enabled_config, stored) + assert TURN_ONE not in state["turn_errors"] + assert state["placements"][ACCOUNTING_UNMATCHED]["last_error"] is None + def test_restart_preserves_ledger_and_boundary( self, enabled_config: Config, @@ -755,6 +991,252 @@ def test_non_terminal_turns_are_not_exported( assert queued == 0 assert export_calls["turn"] == [] + def test_open_interaction_accounting_becomes_eligible_once_turn_completes( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + ) -> None: + """First activation must only wall off *already terminal* history. + + A usage row owned by an interaction that is still open at first + activation is not history yet — it must stay eligible once that + interaction later completes, without an explicit ``--export``. + """ + stored = _seed_session(enabled_config, paths) + open_projection = _projection( + turns=[ + _main_turn( + status="in_progress", + accounting_calls=[_accounting_call()], + ) + ], + usage_rows=[_usage_row()], + attributions=[_attribution(logical_call_id=ACCOUNTING_MATCHED)], + ) + queued = queue_exports(enabled_config, stored, open_projection) + assert queued == 0 + + completed_projection = _projection( + turns=[_main_turn(status="completed", accounting_calls=[_accounting_call()])], + usage_rows=[_usage_row()], + attributions=[_attribution(logical_call_id=ACCOUNTING_MATCHED)], + ) + queued = queue_exports(enabled_config, stored, completed_projection) + assert queued == 1 + state = load_export_state(enabled_config, stored) + assert ACCOUNTING_MATCHED in state["placements"] + assert state["placements"][ACCOUNTING_MATCHED]["destination"] == "chat-span" + + def test_pending_attribution_on_open_turn_is_eligible_once_resolved( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + ) -> None: + """A pending (unresolved) attribution owned by a still-open turn at + first activation must not be forever excluded once the interaction + finishes and the attribution later resolves to ambiguous/matched.""" + stored = _seed_session(enabled_config, paths) + open_projection = _projection( + turns=[ + _main_turn( + status="in_progress", + accounting_calls=[ + _accounting_call( + accounting_id=ACCOUNTING_UNMATCHED, + attribution_status="pending", + call_id=None, + usage=_usage_row(call_id=ACCOUNTING_UNMATCHED).to_dict(), + ) + ], + ) + ], + usage_rows=[_usage_row(call_id=ACCOUNTING_UNMATCHED)], + attributions=[ + _attribution( + logical_call_id=ACCOUNTING_UNMATCHED, status="pending", call_id=None + ) + ], + ) + queue_exports(enabled_config, stored, open_projection) + + resolved_projection = _projection( + turns=[ + _main_turn( + status="completed", + accounting_calls=[ + _accounting_call( + accounting_id=ACCOUNTING_UNMATCHED, + attribution_status="ambiguous", + call_id=None, + usage=_usage_row(call_id=ACCOUNTING_UNMATCHED).to_dict(), + ) + ], + ) + ], + usage_rows=[_usage_row(call_id=ACCOUNTING_UNMATCHED)], + attributions=[ + _attribution( + logical_call_id=ACCOUNTING_UNMATCHED, status="ambiguous", call_id=None + ) + ], + ) + queued = queue_exports(enabled_config, stored, resolved_projection) + assert queued == 2 + state = load_export_state(enabled_config, stored) + assert ACCOUNTING_UNMATCHED in state["placements"] + + def test_child_turn_accounting_stays_excluded_with_its_main_interaction( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + ) -> None: + """A nested child (subagent) turn is never exported as its own job, + so its own id is never a member of the boundary lists built from main + turns. Eligibility for its accounting must resolve to the owning main + interaction, not the child's own id -- otherwise usage discovered + later for a child of an already-excluded historical main turn would + wrongly become eligible on its own.""" + stored = _seed_session(enabled_config, paths) + child_turn_id = f"{TURN_ONE}:agent-1" + first_projection = _projection( + turns=[_main_turn(subagents=[_main_turn(turn_id=child_turn_id)])], + ) + queued = queue_exports(enabled_config, stored, first_projection) + assert queued == 0 + state = load_export_state(enabled_config, stored) + assert TURN_ONE in state["excluded_turn_ids"] + assert child_turn_id not in state["excluded_turn_ids"] + + late_child_accounting_projection = _projection( + turns=[ + _main_turn( + subagents=[ + _main_turn( + turn_id=child_turn_id, + accounting_calls=[_accounting_call()], + ) + ] + ) + ], + usage_rows=[_usage_row()], + attributions=[ + _attribution(logical_call_id=ACCOUNTING_MATCHED, stored_turn_id=child_turn_id) + ], + ) + queued = queue_exports(enabled_config, stored, late_child_accounting_projection) + assert queued == 0 + assert export_calls["turn_accounting"] == [] + state = load_export_state(enabled_config, stored) + assert ACCOUNTING_MATCHED not in state["placements"] + + +class TestWorkerConfirmedDelivery: + """`queue_exports` against a real (locally flushed, never remote) Logfire + instance and the real detached worker, to verify the durable delivery + claim actually prevents a second emission across a simulated restart.""" + + @pytest.fixture(autouse=True) + def _reset_otel_state(self): + pytest.importorskip("logfire") + from thirdeye import otel_export as _otel_export + + _otel_export._state["attempted"] = False + _otel_export._state["instance"] = None + _otel_export._state["id_generator"] = None + yield + _otel_export._state["attempted"] = False + _otel_export._state["instance"] = None + _otel_export._state["id_generator"] = None + + @pytest.fixture + def exporter(self): + from logfire.testing import TestExporter + + return TestExporter() + + @pytest.fixture + def wired_instance(self, exporter, monkeypatch: pytest.MonkeyPatch): + import logfire + from opentelemetry.sdk.trace.export import SimpleSpanProcessor + + instance = logfire.configure( + send_to_logfire=False, + console=False, + additional_span_processors=[SimpleSpanProcessor(exporter)], + advanced=logfire.AdvancedOptions(id_generator=otel_export._id_generator()), + ) + monkeypatch.setattr(otel_export, "_get_instance", lambda config, platform: instance) + return instance + + @pytest.fixture(autouse=True) + def _synchronous_worker(self, monkeypatch: pytest.MonkeyPatch, enabled_config: Config): + """Run the detached worker in-process instead of spawning a real + child, same as the generic transport's own worker tests do.""" + from thirdeye import otel_worker + + monkeypatch.setattr(Config, "load", lambda: enabled_config) + + def _run(job_path: Path) -> None: + otel_worker.main([str(job_path)]) + + monkeypatch.setattr(otel_export, "_spawn", _run) + + def test_confirmed_accounting_delivery_survives_restart_without_double_emission( + self, + enabled_config: Config, + paths: SourcePaths, + wired_instance, + exporter, + ) -> None: + stored = _seed_session(enabled_config, paths) + projection = _projection( + turns=[ + _main_turn( + accounting_calls=[ + _accounting_call( + accounting_id=ACCOUNTING_UNMATCHED, + attribution_status="ambiguous", + call_id=None, + usage=_usage_row(call_id=ACCOUNTING_UNMATCHED).to_dict(), + ) + ] + ) + ], + usage_rows=[_usage_row(call_id=ACCOUNTING_UNMATCHED, input_tokens=6587, output_tokens=5)], + attributions=[ + _attribution( + logical_call_id=ACCOUNTING_UNMATCHED, status="ambiguous", call_id=None + ) + ], + ) + + queue_exports(enabled_config, stored, projection, include_history=True) + directory = _directory(enabled_config, stored) + assert otel_export.accounting_export_sent(directory, ACCOUNTING_UNMATCHED) is True + first_accounting_spans = [ + span for span in exporter.exported_spans_as_dict() if span["name"] == "accounting" + ] + assert len(first_accounting_spans) == 1 + + # Simulate a restart: fresh Config/instance state, same durable + # ledger and durable transport claim on disk. + reloaded = Config( + root=enabled_config.root, + logfire=LogfireSettings(enabled=True, token="fake-token"), + ) + queued = queue_exports(reloaded, stored, projection, include_history=True) + assert queued == 1 # the turn job re-queues; harmless, first-wins claim there too + + second_accounting_spans = [ + span for span in exporter.exported_spans_as_dict() if span["name"] == "accounting" + ] + assert len(second_accounting_spans) == 1 # unchanged: no second accounting emission + state = load_export_state(enabled_config, stored) + assert state["placements"][ACCOUNTING_UNMATCHED]["emitted"] is True + def _drain_cli_transcript(home: Path) -> list[SourceRecord]: session_dir = home / "session-state" / NATIVE_SESSION_ID From d5574f2cfaa4d1fd209bfec3bcc84d1dbddff1ef Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Sat, 12 Sep 2026 07:26:03 -0700 Subject: [PATCH 74/88] Close remaining atomicity, concurrency, and replay-equivalence gaps in Copilot reconciliation. Add an optimistic-concurrency commit_sequence to projection state so a reconcile_archive call built from state read before a concurrent writer committed is refused (ProjectionConflictError) instead of silently overwriting newer derived state. Document that a post-validation I/O failure while publishing the usage-sidecar mirror is a distinct, self-healing failure mode from a pre-publish validation failure (the projection document itself is already durably committed by that point), rather than papering over it with an inaccurate "always fully atomic" claim. Strengthen reconcile test coverage to compare full projection indexes (usage/attributions/pending/diagnostics), not just turn counts, between incremental and full-replay paths, and to assert idempotent re-reconciliation does not duplicate any raw index content. --- .../platforms/copilot/projection_state.py | 1 + .../platforms/copilot/projection_store.py | 54 ++++- src/thirdeye/platforms/copilot/reconcile.py | 31 ++- src/thirdeye/platforms/copilot/types.py | 8 +- tests/platforms/copilot/test_reconcile.py | 198 +++++++++++++++++- 5 files changed, 282 insertions(+), 10 deletions(-) diff --git a/src/thirdeye/platforms/copilot/projection_state.py b/src/thirdeye/platforms/copilot/projection_state.py index e8d7a24..78c6140 100644 --- a/src/thirdeye/platforms/copilot/projection_state.py +++ b/src/thirdeye/platforms/copilot/projection_state.py @@ -52,6 +52,7 @@ def empty_projection_state() -> dict[str, Any]: "semantic_state": {"open_interactions": {}}, "accounting_state": {"logical_calls": {}}, "projection_revision": "", + "commit_sequence": 0, } diff --git a/src/thirdeye/platforms/copilot/projection_store.py b/src/thirdeye/platforms/copilot/projection_store.py index 6ee2c2e..0c3214c 100644 --- a/src/thirdeye/platforms/copilot/projection_store.py +++ b/src/thirdeye/platforms/copilot/projection_store.py @@ -41,6 +41,17 @@ _INDEX_NAMES = ("events", "turns", "usage", "attributions", "pending", "diagnostics") +class ProjectionConflictError(RuntimeError): + """A commit observed a newer ``commit_sequence`` than it was based on. + + Raised instead of silently overwriting: a second writer already + committed a projection built from more current builder state, so this + (older) attempt is discarded rather than winning a last-write-races + against the newer one. The caller should reload projection state and + retry. + """ + + def _directory(config: Config, stored_session_id: str) -> Path: return session_dir(config.root, PLATFORM_NAME, stored_session_id) @@ -391,6 +402,7 @@ def commit_projection( next_state: dict[str, Any], *, replace: bool = False, + base_commit_sequence: int | None = None, ) -> dict[str, int]: """Atomically merge or replace a DTO projection into replayable derived state. @@ -400,15 +412,48 @@ def commit_projection( ``replace=True`` discards prior derived indexes as part of this same locked operation instead of requiring a separate reset call. Nothing is - written to disk until every validation below succeeds, so a rebuild that - fails partway (a malformed usage row, an IO error) leaves the previous - projection completely untouched rather than losing it to a non-atomic - delete-then-commit sequence. + written to disk until every validation above this point succeeds, so a + commit that fails validation (a malformed usage row, an unknown-schema + document) leaves the previous projection completely untouched rather + than losing it to a non-atomic delete-then-commit sequence. + + Durability past that validation point has two distinct failure modes. + The projection document is the source of truth and is published with + write-ahead journaling (see ``publish_projection_document``): once that + call returns, the new document is durably committed, full stop. The + usage sidecar published immediately after is a derived, self-healing + mirror of the document's own usage index -- kept as a separate JSONL + file only so ``UsageIndex`` can query it without parsing the whole + document. If that second, mirror-only publish raises (a disk error, not + a validation error), this function still raises so the caller learns of + it, but the already-committed document is *not* rolled back: it reflects + the new projection, and the next ``load_projection_state`` call + republishes a sidecar that matches it. A caller must not assume a raise + from this function always means "nothing changed" -- check which phase + failed via ``read_projection_status``/``load_projection_state`` if that + distinction matters. + + ``base_commit_sequence``, when given, must equal the ``commit_sequence`` + a caller observed from an earlier ``load_projection_state`` call. A + mismatch means another writer has committed since that read -- this + raises :class:`ProjectionConflictError` instead of overwriting the newer + projection with one built from the stale builder state, closing the + otherwise unlocked gap between reading prior state, building a new + projection from it, and committing here. This check itself happens + before any write in this call, so a conflict never touches disk. """ directory = _directory(config, stored_session_id) _require_session(directory, stored_session_id) with locked(projection_lock_path(directory), LockMode.EXCLUSIVE): document = read_projection_document(directory) + current_sequence = _mapping(document.get("state")).get("commit_sequence") + current_sequence = current_sequence if isinstance(current_sequence, int) else 0 + if base_commit_sequence is not None and base_commit_sequence != current_sequence: + raise ProjectionConflictError( + f"Copilot projection for {stored_session_id!r} advanced from commit " + f"{base_commit_sequence} to {current_sequence} since it was loaded; " + "reload projection state and retry" + ) indexes = {} if replace else _mapping(document.get("indexes")) merged_indexes = {name: dict(_mapping(indexes.get(name))) for name in indexes} for name in (*_INDEX_NAMES, "usage_identities"): @@ -429,6 +474,7 @@ def commit_projection( ] = item state = _replace_state(next_state) + state["commit_sequence"] = current_sequence + 1 identities = _collect_identities( merged_indexes["usage_identities"], merged_indexes["attributions"], diff --git a/src/thirdeye/platforms/copilot/reconcile.py b/src/thirdeye/platforms/copilot/reconcile.py index eb44315..004c87f 100644 --- a/src/thirdeye/platforms/copilot/reconcile.py +++ b/src/thirdeye/platforms/copilot/reconcile.py @@ -92,8 +92,24 @@ def reconcile_archive( reproducible projection state as part of the same locked commit used for an incremental merge (see ``commit_projection(..., replace=True)``): raw archive records and the separate export ledger are untouched, and nothing - is written to disk until the replacement projection is fully validated, - so a rebuild that fails partway cannot erase the last readable local view. + is written to disk until the replacement projection passes validation, so + a rebuild that fails validation (a malformed usage row, a stale schema) + cannot erase the last readable local view. A failure *after* that point + -- the projection document itself durably publishes, but the derived + usage-sidecar mirror then hits an I/O error -- still reports an error + here, but the counts below reflect the document that was, in fact, + committed; see ``commit_projection`` for why that distinction is not a + bug. Either way the next reconciliation call self-heals the sidecar. + + Loading prior state, building the new projection from it, and committing + are not one locked operation -- ``build_projection`` runs unlocked so a + slow archive replay does not hold the projection lock. To close the gap + that leaves, the ``commit_sequence`` observed here is passed through to + ``commit_projection`` as ``base_commit_sequence``: if another writer + committed in the meantime, the commit is refused (``ProjectionConflictError``, + reported below as an error) instead of overwriting newer derived state + with one built from what is now stale builder state. This is checked + even for ``rebuild``, which otherwise discards ``prior_state`` entirely. Errors are reported as counts instead of escaping so source capture can remain operational when a derived projection is malformed. Export is a @@ -101,12 +117,19 @@ def reconcile_archive( """ try: - prior_state = {} if rebuild else load_projection_state(config, stored_session_id) + loaded_state = load_projection_state(config, stored_session_id) + base_commit_sequence = loaded_state.get("commit_sequence") + prior_state = {} if rebuild else loaded_state records = list(iter_captured_records(config, stored_session_id)) projection, next_state = build_projection(records, prior_state) next_state["archive_source_ids"] = [record["source_id"] for record in records] counts = commit_projection( - config, stored_session_id, projection, next_state, replace=rebuild + config, + stored_session_id, + projection, + next_state, + replace=rebuild, + base_commit_sequence=base_commit_sequence, ) except Exception: return _failure_result(config, stored_session_id) diff --git a/src/thirdeye/platforms/copilot/types.py b/src/thirdeye/platforms/copilot/types.py index bd0517c..07e95bd 100644 --- a/src/thirdeye/platforms/copilot/types.py +++ b/src/thirdeye/platforms/copilot/types.py @@ -578,7 +578,12 @@ class ProjectionState(TypedDict): of the committed indexes when the builder leaves it empty. ``index_totals`` is the size of each derived index after the commit, not a per-commit delta. A stale schema version is discarded so a rebuild - can reconstruct derived indexes from the V1 archive. + can reconstruct derived indexes from the V1 archive. ``commit_sequence`` + is a storage-owned counter incremented on every successful + ``commit_projection`` call; it is never set by a builder and exists so a + caller that read state via ``load_projection_state`` can pass it back as + ``base_commit_sequence`` to detect (and refuse) overwriting a projection + that a concurrent writer has already advanced. """ projection_schema_version: int @@ -587,3 +592,4 @@ class ProjectionState(TypedDict): accounting_state: AccountingProjectionState projection_revision: str index_totals: NotRequired[dict[str, int]] + commit_sequence: NotRequired[int] diff --git a/tests/platforms/copilot/test_reconcile.py b/tests/platforms/copilot/test_reconcile.py index 8858d71..eb3545e 100644 --- a/tests/platforms/copilot/test_reconcile.py +++ b/tests/platforms/copilot/test_reconcile.py @@ -11,10 +11,16 @@ import pytest from thirdeye.config import Config +from thirdeye.paths import session_dir +from thirdeye.platforms.copilot import projection_store from thirdeye.platforms.copilot.archive import commit_batch, iter_captured_records +from thirdeye.platforms.copilot.constants import PLATFORM_NAME from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id from thirdeye.platforms.copilot.projection import build_projection as _real_build_projection +from thirdeye.platforms.copilot.projection_state import projection_state_path from thirdeye.platforms.copilot.projection_store import ( + ProjectionConflictError, + commit_projection, load_projection_state, read_projected_turns, ) @@ -46,6 +52,22 @@ def _load_json(path: Path) -> Any: return json.loads(path.read_text(encoding="utf-8")) +def _directory(config: Config, stored: str) -> Path: + return session_dir(config.root, PLATFORM_NAME, stored) + + +def _document(config: Config, stored: str) -> dict[str, Any]: + return json.loads(projection_state_path(_directory(config, stored)).read_text()) + + +def _normalized_index_values( + config: Config, stored: str, native_session_id: str, index_name: str +) -> list[Any]: + index = _document(config, stored)["indexes"][index_name] + normalized = [_rewrite_native_id(item, native_session_id) for item in index.values()] + return sorted(normalized, key=lambda item: json.dumps(item, sort_keys=True, default=str)) + + def _drain_cli_transcript( home: Path, *, native_session_id: str = NATIVE_SESSION_ID ) -> list[SourceRecord]: @@ -288,13 +310,34 @@ def test_reconcile_archive_is_idempotent( second = reconcile_archive(config, stored) second_turns = read_projected_turns(config, stored) second_state = load_projection_state(config, stored) + first_document = _document(config, stored) + second_document = _document(config, stored) assert first == second assert first_turns == second_turns - assert first_state == second_state + # commit_sequence is a storage-owned counter that advances on every + # successful commit_projection call, whether or not its content changed + # -- it must not be idempotent, unlike everything else in builder state. + assert second_state["commit_sequence"] == first_state["commit_sequence"] + 1 + assert {k: v for k, v in first_state.items() if k != "commit_sequence"} == { + k: v for k, v in second_state.items() if k != "commit_sequence" + } for turn in second_turns: seqs = [event.get("seq") for event in turn.get("events") or []] assert len(seqs) == len(set(seqs)), "re-reconciling must not duplicate turn events" + # Full raw indexes (including each turn's nested accounting_calls, which + # read_projected_turns intentionally does not surface) must be byte-for- + # byte identical across the two re-derivations: re-running reconcile + # must not accumulate duplicate accounting calls or usage/attribution + # entries inside any index. + normalized_first = { + k: v for k, v in first_document["state"].items() if k != "commit_sequence" + } + normalized_second = { + k: v for k, v in second_document["state"].items() if k != "commit_sequence" + } + assert normalized_first == normalized_second + assert first_document["indexes"] == second_document["indexes"] def test_rebuild_is_idempotent_and_matches_initial_reconcile( @@ -348,6 +391,21 @@ def test_incremental_reconcile_matches_full_replay( normalized_full = _rewrite_native_id(full_turns, FULL_NATIVE_SESSION_ID) assert normalized_incremental == normalized_full + # Index keys can fall back to a content digest computed over pre- + # normalization payloads (see _index_key), so two sessions built from the + # same content under different native IDs are not guaranteed to share + # digest-fallback keys even though their *content* is equivalent. + # Comparing normalized values as an order-independent multiset avoids + # that false negative while still catching a real divergence (a usage + # row, attribution, pending item, or diagnostic present under one path + # and not the other, or duplicated under either). + for name in ("usage", "attributions", "pending", "diagnostics"): + incremental_values = _normalized_index_values( + config, stored_incremental, NATIVE_SESSION_ID, name + ) + full_values = _normalized_index_values(config, stored_full, FULL_NATIVE_SESSION_ID, name) + assert incremental_values == full_values, f"{name} index diverged between replay paths" + def test_reconcile_default_does_not_export( config: Config, @@ -480,6 +538,144 @@ def build_with_invalid_usage_row( assert load_projection_state(config, stored) == prior_state +def test_commit_projection_rejects_a_stale_base_commit_sequence( + config: Config, + paths: SourcePaths, + cli_transcript_records: list[SourceRecord], +) -> None: + """A commit built from state read before a concurrent writer committed + + must be refused rather than silently overwriting that newer commit. + This exercises commit_projection's optimistic-concurrency check + directly: build a projection from state observed at commit_sequence 1, + let another writer advance the stored session to commit_sequence 2, then + attempt to commit the stale one with its now-outdated base sequence. + """ + stored = _seed_full_corpus(config, paths, cli_transcript_records) + reconcile_archive(config, stored) + stale_state = load_projection_state(config, stored) + assert stale_state["commit_sequence"] == 1 + records = list(iter_captured_records(config, stored)) + stale_projection, stale_next_state = _real_build_projection(records, stale_state) + stale_next_state["archive_source_ids"] = [record["source_id"] for record in records] + + reconcile_archive(config, stored, rebuild=True) + newer_turns = read_projected_turns(config, stored) + newer_state = load_projection_state(config, stored) + assert newer_state["commit_sequence"] == 2 + + with pytest.raises(ProjectionConflictError): + commit_projection( + config, + stored, + stale_projection, + stale_next_state, + base_commit_sequence=stale_state["commit_sequence"], + ) + + assert read_projected_turns(config, stored) == newer_turns + assert load_projection_state(config, stored) == newer_state + + +def test_reconcile_reports_error_when_projection_advances_concurrently( + config: Config, + paths: SourcePaths, + cli_transcript_records: list[SourceRecord], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """reconcile_archive itself must surface a concurrent-writer conflict + + as an error rather than corrupt state, even though its own load, build, + and commit are not one held lock. A second writer (simulated here by + recursively calling reconcile_archive from inside a patched + commit_projection, guarded so it only races once) finishes a full + rebuild between this call's load and its commit; the racing call's own + stale commit must then be refused, and the session must be left exactly + as the interleaved rebuild left it. + """ + stored = _seed_full_corpus(config, paths, cli_transcript_records) + reconcile_archive(config, stored) + baseline_turns = read_projected_turns(config, stored) + + real_commit = projection_store.commit_projection + raced = {"done": False} + + def racing_commit( + cfg: Config, sid: str, projection: Any, next_state: dict[str, Any], **kwargs: Any + ) -> dict[str, int]: + if not raced["done"]: + raced["done"] = True + reconcile_archive(cfg, sid, rebuild=True) + return real_commit(cfg, sid, projection, next_state, **kwargs) + + monkeypatch.setattr("thirdeye.platforms.copilot.reconcile.commit_projection", racing_commit) + + result = reconcile_archive(config, stored) + + assert result["errors"] == 1 + assert read_projected_turns(config, stored) == baseline_turns + + +def test_usage_sidecar_publish_failure_reports_the_committed_document( + config: Config, + paths: SourcePaths, + cli_transcript_records: list[SourceRecord], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A post-validation I/O failure while publishing the usage-sidecar + + mirror is a distinct failure mode from a validation failure: by the time + it can happen, the projection document has already durably published + (see commit_projection's docstring), so the counts reconcile_archive + reports must reflect that new document -- not the prior one -- with the + error counted on top. The next reconcile call must self-heal the + sidecar without reprocessing anything new. + """ + stored = _seed_full_corpus(config, paths, cli_transcript_records) + baseline = reconcile_archive(config, stored) + prior_turns = read_projected_turns(config, stored) + baseline_sequence = load_projection_state(config, stored)["commit_sequence"] + + original_publish_usage = projection_store._publish_usage + + def selective_boom( + cfg: Config, sid: str, directory: Path, usage_index: dict[str, Any], *, force: bool = False + ) -> None: + # load_projection_state's self-heal always passes force=True; only + # commit_projection's own (non-forced) publish should fail here, so + # this reaches the specific "document committed, sidecar mirror + # failed" state the docstring describes rather than failing before + # commit_projection is ever entered. + if force: + original_publish_usage(cfg, sid, directory, usage_index, force=force) + return + raise OSError("disk full while rewriting usage sidecar") + + monkeypatch.setattr( + "thirdeye.platforms.copilot.projection_store._publish_usage", selective_boom + ) + + result = reconcile_archive(config, stored, rebuild=True) + + assert result["errors"] == baseline["errors"] + 1 + assert result["turns"] == baseline["turns"] + assert result["usage"] == baseline["usage"] + # The document committed despite the sidecar failure -- proven by the + # storage-owned commit_sequence advancing even though this call reported + # an error -- so turns already reflect the (content-equivalent, since + # this rebuilds the same archive) new projection rather than being stuck + # on the old one. Read the raw document rather than load_projection_state + # here: that call would itself retry the still-patched, still-failing + # sidecar publish as part of its own self-heal. + assert _document(config, stored)["state"]["commit_sequence"] == baseline_sequence + 1 + assert read_projected_turns(config, stored) == prior_turns + + monkeypatch.undo() + healed = reconcile_archive(config, stored) + assert healed["errors"] == 0 + assert read_projected_turns(config, stored) == prior_turns + + def test_reconcile_unknown_session_reports_error_without_creating_paths( config: Config, ) -> None: From db66a84263c8f08be2e0f7a84fe43517f8a6e150 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Sat, 12 Sep 2026 11:09:33 -0700 Subject: [PATCH 75/88] Fix archive reconciliation review gaps --- .../platforms/copilot/projection_store.py | 22 ++----- tests/platforms/copilot/test_reconcile.py | 57 ++++++++++--------- 2 files changed, 36 insertions(+), 43 deletions(-) diff --git a/src/thirdeye/platforms/copilot/projection_store.py b/src/thirdeye/platforms/copilot/projection_store.py index 0c3214c..cf85a41 100644 --- a/src/thirdeye/platforms/copilot/projection_store.py +++ b/src/thirdeye/platforms/copilot/projection_store.py @@ -417,21 +417,11 @@ def commit_projection( document) leaves the previous projection completely untouched rather than losing it to a non-atomic delete-then-commit sequence. - Durability past that validation point has two distinct failure modes. - The projection document is the source of truth and is published with - write-ahead journaling (see ``publish_projection_document``): once that - call returns, the new document is durably committed, full stop. The - usage sidecar published immediately after is a derived, self-healing - mirror of the document's own usage index -- kept as a separate JSONL - file only so ``UsageIndex`` can query it without parsing the whole - document. If that second, mirror-only publish raises (a disk error, not - a validation error), this function still raises so the caller learns of - it, but the already-committed document is *not* rolled back: it reflects - the new projection, and the next ``load_projection_state`` call - republishes a sidecar that matches it. A caller must not assume a raise - from this function always means "nothing changed" -- check which phase - failed via ``read_projection_status``/``load_projection_state`` if that - distinction matters. + The usage JSONL and ``UsageIndex`` are derived mirrors, but publishing + them can fail. They are therefore updated before the projection document: + a sidecar failure leaves the last readable projection untouched. The + projection document remains the commit point and is published with + write-ahead journaling (see ``publish_projection_document``). ``base_commit_sequence``, when given, must equal the ``commit_sequence`` a caller observed from an earlier ``load_projection_state`` call. A @@ -528,8 +518,8 @@ def commit_projection( "state": state, "indexes": merged_indexes, } - publish_projection_document(directory, next_document) _publish_usage(config, stored_session_id, directory, merged_indexes["usage"]) + publish_projection_document(directory, next_document) return counts diff --git a/tests/platforms/copilot/test_reconcile.py b/tests/platforms/copilot/test_reconcile.py index eb3545e..e2bc8a6 100644 --- a/tests/platforms/copilot/test_reconcile.py +++ b/tests/platforms/copilot/test_reconcile.py @@ -197,7 +197,9 @@ def _rewrite_native_id(value: Any, native_session_id: str, placeholder: str = "N offsets/inode generation are per-file-instance, not semantic content). """ if isinstance(value, str): - return value.replace(native_session_id, placeholder) + for identity in {native_session_id, NATIVE_SESSION_ID, FULL_NATIVE_SESSION_ID}: + value = value.replace(identity, placeholder) + return value if isinstance(value, list): return [_rewrite_native_id(item, native_session_id, placeholder) for item in value] if isinstance(value, dict): @@ -306,11 +308,11 @@ def test_reconcile_archive_is_idempotent( first = reconcile_archive(config, stored) first_turns = read_projected_turns(config, stored) first_state = load_projection_state(config, stored) + first_document = _document(config, stored) second = reconcile_archive(config, stored) second_turns = read_projected_turns(config, stored) second_state = load_projection_state(config, stored) - first_document = _document(config, stored) second_document = _document(config, stored) assert first == second @@ -338,6 +340,10 @@ def test_reconcile_archive_is_idempotent( } assert normalized_first == normalized_second assert first_document["indexes"] == second_document["indexes"] + for record in second_document["indexes"]["turns"].values(): + calls = record["span"].get("accounting_calls", []) + call_ids = [call["accounting_id"] for call in calls] + assert len(call_ids) == len(set(call_ids)) def test_rebuild_is_idempotent_and_matches_initial_reconcile( @@ -385,12 +391,25 @@ def test_incremental_reconcile_matches_full_replay( ) full = reconcile_archive(config, stored_full) full_turns = read_projected_turns(config, stored_full) + incremental_document = _document(config, stored_incremental) + full_document = _document(config, stored_full) assert incremental == full normalized_incremental = _rewrite_native_id(incremental_turns, NATIVE_SESSION_ID) normalized_full = _rewrite_native_id(full_turns, FULL_NATIVE_SESSION_ID) assert normalized_incremental == normalized_full + # Builder state must agree independently of the storage-owned commit + # counter and identity-dependent document digest. + incremental_state = copy.deepcopy(incremental_document["state"]) + full_state = copy.deepcopy(full_document["state"]) + for state in (incremental_state, full_state): + state.pop("commit_sequence", None) + state.pop("projection_revision", None) + assert _rewrite_native_id(incremental_state, NATIVE_SESSION_ID) == _rewrite_native_id( + full_state, FULL_NATIVE_SESSION_ID + ) + # Index keys can fall back to a content digest computed over pre- # normalization payloads (see _index_key), so two sessions built from the # same content under different native IDs are not guaranteed to share @@ -399,7 +418,9 @@ def test_incremental_reconcile_matches_full_replay( # that false negative while still catching a real divergence (a usage # row, attribution, pending item, or diagnostic present under one path # and not the other, or duplicated under either). - for name in ("usage", "attributions", "pending", "diagnostics"): + # Comparing every index's normalized values includes raw stored turns and + # their nested accounting_calls, plus events and the usage identity map. + for name in incremental_document["indexes"]: incremental_values = _normalized_index_values( config, stored_incremental, NATIVE_SESSION_ID, name ) @@ -616,36 +637,25 @@ def racing_commit( assert read_projected_turns(config, stored) == baseline_turns -def test_usage_sidecar_publish_failure_reports_the_committed_document( +def test_usage_sidecar_publish_failure_preserves_prior_projection( config: Config, paths: SourcePaths, cli_transcript_records: list[SourceRecord], monkeypatch: pytest.MonkeyPatch, ) -> None: - """A post-validation I/O failure while publishing the usage-sidecar - - mirror is a distinct failure mode from a validation failure: by the time - it can happen, the projection document has already durably published - (see commit_projection's docstring), so the counts reconcile_archive - reports must reflect that new document -- not the prior one -- with the - error counted on top. The next reconcile call must self-heal the - sidecar without reprocessing anything new. - """ + """A usage-sidecar I/O failure must not replace the readable projection.""" stored = _seed_full_corpus(config, paths, cli_transcript_records) baseline = reconcile_archive(config, stored) prior_turns = read_projected_turns(config, stored) - baseline_sequence = load_projection_state(config, stored)["commit_sequence"] + prior_document = _document(config, stored) original_publish_usage = projection_store._publish_usage def selective_boom( cfg: Config, sid: str, directory: Path, usage_index: dict[str, Any], *, force: bool = False ) -> None: - # load_projection_state's self-heal always passes force=True; only - # commit_projection's own (non-forced) publish should fail here, so - # this reaches the specific "document committed, sidecar mirror - # failed" state the docstring describes rather than failing before - # commit_projection is ever entered. + # Let readers inspect the prior projection normally. Only the commit's + # attempt to publish the new sidecar fails. if force: original_publish_usage(cfg, sid, directory, usage_index, force=force) return @@ -660,14 +670,7 @@ def selective_boom( assert result["errors"] == baseline["errors"] + 1 assert result["turns"] == baseline["turns"] assert result["usage"] == baseline["usage"] - # The document committed despite the sidecar failure -- proven by the - # storage-owned commit_sequence advancing even though this call reported - # an error -- so turns already reflect the (content-equivalent, since - # this rebuilds the same archive) new projection rather than being stuck - # on the old one. Read the raw document rather than load_projection_state - # here: that call would itself retry the still-patched, still-failing - # sidecar publish as part of its own self-heal. - assert _document(config, stored)["state"]["commit_sequence"] == baseline_sequence + 1 + assert _document(config, stored) == prior_document assert read_projected_turns(config, stored) == prior_turns monkeypatch.undo() From 89c9035d0fa4be402934d92b17d673a900302526 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Sat, 12 Sep 2026 11:13:27 -0700 Subject: [PATCH 76/88] Fix duplicate accounting delivery, eligibility, and in-flight races in Copilot export Address four issues from review attempt 2: 1. Matched usage embedded on a chat span was never marked delivered by the generic transport, so once the turn's own claim was "sent" a later pass saw the accounting as undelivered and relocated it to a new fallback span, duplicating tokens. `_export_turn_inner`/`_export_subagent_turn_inner` now mark every accounting id carried in the exported turn tree as sent. `queue_exports`'s chat-availability check also no longer misreads an already-settled chat placement as a conflicting correction. 2. A usage row with a still-`pending` (unresolved) attribution and no current owner was locked into the first-activation historical boundary forever, even after it later resolved to a real, completed-later interaction. `_historical_accounting_ids` no longer treats a pending, ownerless identity as history. 3. A permanently-failed deterministic accounting job (exhausted retries) was indistinguishable from a healthy one, since success only meant the local write+dispatch succeeded. `queue_exports` now consults the job's own on-disk state (`otel_export.accounting_job_status`) before clearing errors or counting a queue as successful. 4. Relocating an accounting identity to a new destination could leave its old deterministic job on disk, still eligible for an already-dispatched worker to deliver after the ledger moved on -- and a job merely `"claimed"` (in flight) was treated the same as one proven never queued. `record_placement` now takes the old job's state and quarantines a relocation attempted while in flight; `queue_exports` cancels a stale job proven still `"queued"` before recording the new placement. Co-Authored-By: Claude Sonnet 5 --- src/thirdeye/otel_export.py | 68 ++++ src/thirdeye/platforms/copilot/export.py | 91 +++++- .../platforms/copilot/export_state.py | 34 +- tests/platforms/copilot/test_export.py | 302 ++++++++++++++++++ 4 files changed, 473 insertions(+), 22 deletions(-) diff --git a/src/thirdeye/otel_export.py b/src/thirdeye/otel_export.py index 1720898..deb852b 100644 --- a/src/thirdeye/otel_export.py +++ b/src/thirdeye/otel_export.py @@ -545,6 +545,47 @@ def _write_accounting_job(thirdeye_home: Path, payload: dict[str, Any]) -> Path: return job_path +def accounting_job_status(thirdeye_home: Path, job_id: str) -> dict[str, Any] | None: + """Read the current on-disk state of a deterministic accounting job. + + Returns ``None`` when no job file exists for this id: either it was + never queued, or the worker already claimed, emitted, and deleted it + (see ``_run_accounting_job``). Callers that need to know "confirmed + delivered" rather than "no job file present" must consult + ``accounting_export_sent`` separately -- that durable claim is written + before the job file is removed, so it survives this function returning + ``None`` for an already-delivered identity. + """ + digest = hashlib.sha256(job_id.encode("utf-8")).hexdigest() + job_path = otel_jobs_dir(thirdeye_home) / f"accounting-{digest}.json" + try: + payload = json.loads(job_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return None + return {"state": payload.get("state"), "attempt": payload.get("attempt")} + + +def cancel_accounting_job(thirdeye_home: Path, job_id: str) -> None: + """Best-effort removal of a still-queued deterministic accounting job. + + Only meant to be called right after ``accounting_job_status`` reported + ``"queued"`` for this id -- a caller relocating an accounting identity to + a new destination/span uses this to keep the stale job (keyed by the old + span id) from being picked up by an already-dispatched worker and + delivering tokens nobody's ledger points at anymore. If a worker wins the + race and claims the job between that read and this delete, the delete is + still safe: the worker holds its payload in memory and only ever + rewrites the same path, so removing the file first does not resurrect a + delivery that wasn't already in flight, and a worker that instead loses + the race hits a benign missing-file read and exits (see + ``otel_worker.main``). + """ + digest = hashlib.sha256(job_id.encode("utf-8")).hexdigest() + job_path = otel_jobs_dir(thirdeye_home) / f"accounting-{digest}.json" + fsops.unlink(job_path, missing_ok=True) + fsops.unlink(job_path.with_suffix(f"{job_path.suffix}.claim"), missing_ok=True) + + def _spawn(job_path: Path) -> None: """Hand a job file to a detached ``thirdeye.otel_worker``. @@ -613,6 +654,31 @@ def _mark_accounting_sent(session_dir_: Path, accounting_id: str) -> None: path.write_text("sent", encoding="utf-8", newline="\n") +def _embedded_accounting_ids(turn: TurnSpanDict) -> list[str]: + """Collect every accounting id a turn's own job carries, recursively. + + ``_export_turn_subtree`` delivers every one of these within the same + flush as the turn itself -- merged onto a matching chat span's usage + fields, or (if its call id doesn't match a chat span in this turn) as an + inline turn-owned accounting span -- so once that flush is confirmed, + every id collected here is confirmed delivered too, the same as one + delivered through the separate fallback accounting job path. + """ + ids: list[str] = [] + for accounting in turn.get("accounting_calls") or []: + accounting_id = accounting.get("accounting_id") + if isinstance(accounting_id, str) and accounting_id: + ids.append(accounting_id) + for subagent in turn.get("subagents") or []: + ids.extend(_embedded_accounting_ids(subagent)) + return ids + + +def _mark_turn_accounting_delivered(session_dir_: Path, turn: TurnSpanDict) -> None: + for accounting_id in _embedded_accounting_ids(turn): + _mark_accounting_sent(session_dir_, accounting_id) + + def _claim_turn_export(session_dir_: Path, turn_id: str) -> bool: """First-wins claim on exporting this turn's span tree, ever, for this session. A replayed/duplicate hook invocation for the same turn (e.g. the @@ -986,6 +1052,7 @@ def _export_turn_inner( fsops.unlink(claim_path, missing_ok=True) raise claim_path.write_text("sent", encoding="utf-8", newline="\n") + _mark_turn_accounting_delivered(session_dir_, turn) def _export_subagent_turn_inner( @@ -1036,6 +1103,7 @@ def _export_subagent_turn_inner( fsops.unlink(claim_path, missing_ok=True) raise claim_path.write_text("sent", encoding="utf-8", newline="\n") + _mark_turn_accounting_delivered(session_dir_, turn) def _require_export_instance(config: Config, platform: str): diff --git a/src/thirdeye/platforms/copilot/export.py b/src/thirdeye/platforms/copilot/export.py index c0e996d..40d64fd 100644 --- a/src/thirdeye/platforms/copilot/export.py +++ b/src/thirdeye/platforms/copilot/export.py @@ -123,16 +123,29 @@ def _historical_accounting_ids( completes). An identity with no interaction to gate on at all (an ownerless usage row, or a ``stored_turn_id`` this projection cannot resolve) has no way to become "no longer historical" later, so it keeps - the conservative default of being treated as already-seen history. + the conservative default of being treated as already-seen history -- + *unless* its attribution is still ``pending``. A pending join has not + told us anything about ownership yet: it may still turn out to belong to + an interaction that is open right now, or resolve to a confirmed + ownerless status later. Locking it into history the moment it happens to + be observed with no owner would exclude it forever, since only an + explicit ``--export`` ever removes an id from this boundary once set. """ owners: dict[str, dict[str, Any] | None] = {} for accounting_id, (owner, _item) in _accounting_calls(projection).items(): owners[accounting_id] = _root_for(root_index, owner.get("turn_id")) + pending_unowned: set[str] = set() for attribution in projection["attributions"]: accounting_id = attribution["logical_call_id"] - if accounting_id not in owners: - owners[accounting_id] = _root_for(root_index, attribution["stored_turn_id"]) + if accounting_id in owners: + continue + if attribution["status"] == "pending": + pending_unowned.add(accounting_id) + continue + owners[accounting_id] = _root_for(root_index, attribution["stored_turn_id"]) for row in projection["usage_rows"]: + if row.call_id in pending_unowned: + continue owners.setdefault(row.call_id, None) return sorted( accounting_id for accounting_id, root in owners.items() if root is None or _terminal(root) @@ -216,6 +229,19 @@ def _place( captured: dict[str, Any] = {} def update(state: dict[str, Any]) -> dict[str, Any]: + existing = state.get("placements", {}).get(accounting_id) or {} + old_span_id = existing.get("span_id") + old_job_state: str | None = None + if isinstance(old_span_id, str) and old_span_id and old_span_id != span_id: + # Relocating to a different span: find out whether the job under + # the *old* span id is provably inert before deciding it is safe + # to replace, and if it is, cancel it so an already-dispatched + # worker can never deliver tokens this ledger no longer points + # at once it decides on the new destination below. + status = otel_export.accounting_job_status(config.root, old_span_id) + old_job_state = status.get("state") if status else None + if old_job_state == "queued": + otel_export.cancel_accounting_job(config.root, old_span_id) next_state, entry, accepted = record_placement( state, accounting_id=accounting_id, @@ -223,6 +249,7 @@ def update(state: dict[str, Any]) -> dict[str, Any]: span_id=span_id, usage=usage, delivered=delivered, + old_job_state=old_job_state, ) captured["entry"] = entry captured["accepted"] = accepted @@ -232,6 +259,37 @@ def update(state: dict[str, Any]) -> dict[str, Any]: return captured.get("entry"), bool(captured.get("accepted")) +def _reflect_accounting_job_health( + config: Config, stored_session_id: str, accounting_id: str, span_id: str +) -> bool: + """Fold the deterministic accounting job's own on-disk state into the + ledger's error tracking after a local write+dispatch reported success. + + A local write succeeding only means the job file exists and a worker was + spawned -- it says nothing about whether that worker (this call, or an + earlier one that already ran and gave up) ever actually delivered it. + Without this check, a job that exhausted its retries and is stuck in a + permanent ``"failed"`` state on disk would be silently reported as + healthy and have any earlier ``last_error`` cleared on every subsequent + reconciliation, even though the generic transport will never retry it + again on its own (see ``otel_worker._claim_job``). + + Returns whether this counts as a successful queue for this pass. + """ + status = otel_export.accounting_job_status(config.root, span_id) + if status is not None and status.get("state") == "failed": + attempt = status.get("attempt") + _error_state( + config, + stored_session_id, + accounting_id, + f"accounting job permanently failed after {attempt} attempts", + ) + return False + _clear_error_state(config, stored_session_id, accounting_id) + return True + + def _turn_with_placed_accounting( turn: dict[str, Any], placements: dict[str, dict[str, Any]] ) -> dict[str, Any]: @@ -319,10 +377,20 @@ def queue_exports( if not is_turn_eligible(state, root_id) or not is_accounting_eligible(state, accounting_id): continue call_id = item.get("call_id") + delivered = otel_export.accounting_export_sent(directory, accounting_id) + existing_entry = (state.get("placements") or {}).get(accounting_id) or {} # A chat span already flushed to Logfire is immutable history: usage # that resolves to "matched" only *after* that flush can no longer - # land on it and must use the turn-owned fallback span instead. - chat_available = not otel_export.turn_export_sent(directory, root_id) + # land on it and must use the turn-owned fallback span instead. But + # if *this* identity was itself the one embedded on that chat span + # (confirmed via the durable delivery claim plus the ledger's own + # record of where it landed), the turn having been sent is exactly + # what delivered it there -- re-deriving "unavailable" from that same + # fact would misread an already-settled placement as a conflicting + # correction. + chat_available = not otel_export.turn_export_sent(directory, root_id) or ( + delivered and existing_entry.get("destination") == "chat-span" + ) if ( chat_available and item.get("attribution_status") == "matched" @@ -334,7 +402,6 @@ def queue_exports( else: destination = "turn-accounting-span" span_id = _span_id(stored_session_id, owner_id, accounting_id) - delivered = otel_export.accounting_export_sent(directory, accounting_id) entry, accepted = _place( config, stored_session_id, @@ -364,11 +431,9 @@ def queue_exports( item, turn_span_id=_turn_span_id(stored_session_id, owner_id), ) - if sent: + if sent and _reflect_accounting_job_health(config, stored_session_id, accounting_id, span_id): queued += 1 - if entry.get("last_error") is not None: - _clear_error_state(config, stored_session_id, accounting_id) - else: + elif not sent: _error_state(config, stored_session_id, accounting_id, "accounting job was not queued") # Usage with no known user-turn owner is intentionally a session accounting @@ -417,11 +482,9 @@ def queue_exports( sent = otel_export.export_session_accounting( config, directory, stored_session_id, PLATFORM_NAME, meta.cwd, item ) - if sent: + if sent and _reflect_accounting_job_health(config, stored_session_id, accounting_id, span_id): queued += 1 - if entry.get("last_error") is not None: - _clear_error_state(config, stored_session_id, accounting_id) - else: + elif not sent: _error_state(config, stored_session_id, accounting_id, "accounting job was not queued") # Generic transport provides a persistent completed-turn claim. Sending diff --git a/src/thirdeye/platforms/copilot/export_state.py b/src/thirdeye/platforms/copilot/export_state.py index ec2abc0..bc5429e 100644 --- a/src/thirdeye/platforms/copilot/export_state.py +++ b/src/thirdeye/platforms/copilot/export_state.py @@ -202,6 +202,7 @@ def record_placement( span_id: str, usage: dict[str, Any], delivered: bool = False, + old_job_state: str | None = None, ) -> tuple[dict[str, Any], dict[str, Any] | None, bool]: """Persist the first token location for an accounting identity. @@ -212,12 +213,23 @@ def record_placement( by the worker on success, so this ledger cannot detect delivery on its own and must be told. + ``old_job_state`` is the caller's fresh read (``otel_export. + accounting_job_status``) of the *existing* placement's own job file, + when relocating to a different destination/span. ``None``/``"queued"``/ + ``"failed"`` all mean nothing is currently in flight for the old job (it + was never queued, is still sitting untouched, or permanently gave up), + so relocating is safe. ``"claimed"`` means a worker holds it right now + and could deliver at any moment — proven neither safe to relocate nor + known to already be delivered, so it is quarantined the same as a + confirmed-delivered correction rather than guessed either way. + Returns ``(state, entry, accepted)``. A correction is only a durable conflict when the existing placement was (or is now known to have been) - delivered: rebinding a destination or usage value the remote collector - already received would produce two different token totals for the same - logical call. A correction to a placement that was only ever queued - locally is always safe to replace/requeue. + delivered, or when its job might still be in flight: rebinding a + destination or usage value the remote collector already has, or might + still receive, risks two different token totals for the same logical + call. A correction to a placement proven to have never been queued, or + proven to have permanently failed, is always safe to replace/requeue. """ result = _normalize(state) digest = usage_digest(usage) @@ -235,7 +247,7 @@ def record_placement( existing = {**existing, "emitted": True, "last_error": None} placements[accounting_id] = existing return result, existing, True - if already_delivered: + if already_delivered or old_job_state == "claimed": if delivered and not existing.get("emitted"): # The candidate is rejected, but the fresh delivery read is # still new information about the *existing* placement — @@ -243,9 +255,14 @@ def record_placement( # ever queued. existing = {**existing, "emitted": True} placements[accounting_id] = existing + reason = ( + "accounting placement or usage changed after confirmed delivery" + if already_delivered + else "accounting placement or usage changed while the prior job was in flight" + ) result["conflicts"][accounting_id] = { "accounting_id": accounting_id, - "reason": "accounting placement or usage changed after confirmed delivery", + "reason": reason, "existing": existing, "candidate": { "destination": destination, @@ -254,8 +271,9 @@ def record_placement( }, } return result, existing, False - # Nothing was ever confirmed delivered for this identity: replace the - # queued/failed placement outright rather than quarantining it. + # Nothing was ever confirmed delivered, and no job is provably in + # flight, for this identity: replace the queued/failed placement + # outright rather than quarantining it. entry = { "accounting_id": accounting_id, "destination": destination, diff --git a/tests/platforms/copilot/test_export.py b/tests/platforms/copilot/test_export.py index c2c994d..9565274 100644 --- a/tests/platforms/copilot/test_export.py +++ b/tests/platforms/copilot/test_export.py @@ -2,6 +2,7 @@ from __future__ import annotations +import hashlib import json import shutil from pathlib import Path @@ -424,6 +425,63 @@ def test_record_placement_self_heals_emitted_flag_when_delivery_confirmed(self) assert healed_entry["emitted"] is True assert healed["placements"]["acct-1"]["emitted"] is True + def test_record_placement_conflicts_when_old_job_is_claimed_in_flight(self) -> None: + """The old job's own worker could deliver at any moment -- relocating + while it is ``"claimed"`` is proven neither safe nor already known to + be delivered, so it must be quarantined rather than guessed either + way, exactly like a confirmed-delivered correction.""" + usage = _usage_row().to_dict() + state, _, accepted = record_placement( + empty_export_state(), + accounting_id="acct-1", + destination="turn-accounting-span", + span_id="span-a", + usage=usage, + ) + assert accepted is True + conflicted, existing, rejected = record_placement( + state, + accounting_id="acct-1", + destination="chat-span", + span_id="call-1", + usage=usage, + old_job_state="claimed", + ) + assert rejected is False + assert existing is not None + assert existing["destination"] == "turn-accounting-span" + assert "acct-1" in conflicted["conflicts"] + assert "in flight" in conflicted["conflicts"]["acct-1"]["reason"] + + def test_record_placement_replaces_when_old_job_proven_inert(self) -> None: + """Unlike ``"claimed"``, a ``None`` (never queued), ``"queued"`` + (untouched by any worker), or ``"failed"`` (permanently gave up, will + never be retried by the transport) old job state proves nothing is + in flight, so relocating is exactly as safe as it always was for an + undelivered placement.""" + usage = _usage_row().to_dict() + for old_state in (None, "queued", "failed"): + state, _, accepted = record_placement( + empty_export_state(), + accounting_id="acct-1", + destination="turn-accounting-span", + span_id="span-a", + usage=usage, + ) + assert accepted is True + replaced, entry, accepted_again = record_placement( + state, + accounting_id="acct-1", + destination="chat-span", + span_id="call-1", + usage=usage, + old_job_state=old_state, + ) + assert accepted_again is True, old_state + assert entry is not None + assert entry["destination"] == "chat-span" + assert "acct-1" not in replaced["conflicts"] + class TestQueueExports: def test_unknown_session_raises(self, enabled_config: Config) -> None: @@ -1132,6 +1190,192 @@ def test_child_turn_accounting_stays_excluded_with_its_main_interaction( state = load_export_state(enabled_config, stored) assert ACCOUNTING_MATCHED not in state["placements"] + def test_pending_ownerless_usage_is_eligible_once_it_resolves_to_a_completed_turn( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + ) -> None: + """At first activation a usage row's attribution may still be + ``"pending"`` with no ``stored_turn_id`` at all -- unresolved, not + confirmed ownerless. That must not be locked into the historical + boundary the way a *resolved* ownerless usage row would be: once it + later resolves to a real, completed owning turn, it must still be + exportable, exactly like any other accounting attached late to an + interaction that was open (or simply not yet observed) at + activation.""" + stored = _seed_session(enabled_config, paths) + pending_projection = _projection( + usage_rows=[_usage_row()], + attributions=[ + _attribution( + logical_call_id=ACCOUNTING_MATCHED, + status="pending", + stored_turn_id=None, + call_id=None, + ) + ], + ) + queued = queue_exports(enabled_config, stored, pending_projection) + assert queued == 0 + state = load_export_state(enabled_config, stored) + assert ACCOUNTING_MATCHED not in state["excluded_accounting_ids"] + + resolved_projection = _projection( + turns=[_main_turn(accounting_calls=[_accounting_call()])], + usage_rows=[_usage_row()], + attributions=[_attribution(logical_call_id=ACCOUNTING_MATCHED)], + ) + queued = queue_exports(enabled_config, stored, resolved_projection) + assert queued == 1 # the turn job; chat-span placement has no separate job + state = load_export_state(enabled_config, stored) + assert ACCOUNTING_MATCHED in state["placements"] + assert state["placements"][ACCOUNTING_MATCHED]["destination"] == "chat-span" + + def test_relocation_cancels_stale_queued_job_at_old_span( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + ) -> None: + """A prior reconciliation placed unmatched usage on a turn-accounting + fallback span and dispatched its deterministic job, which is still + sitting on disk untouched (``"queued"``) because no worker has + claimed it yet. When a later reconciliation discovers a real match + and relocates the tokens to the chat span, the stale fallback job + must be cancelled -- otherwise the already-dispatched worker could + still pick it up later and deliver the same tokens a second time.""" + stored = _seed_session(enabled_config, paths) + old_span_id = f"accounting:{stored}:{TURN_ONE}:{ACCOUNTING_UNMATCHED}" + jobs_dir = otel_export.otel_jobs_dir(enabled_config.root) + jobs_dir.mkdir(parents=True, exist_ok=True) + digest = hashlib.sha256(old_span_id.encode("utf-8")).hexdigest() + job_path = jobs_dir / f"accounting-{digest}.json" + job_path.write_text(json.dumps({"state": "queued", "attempt": 0}), encoding="utf-8") + + def _seed(state: dict[str, Any]) -> dict[str, Any]: + placed, _, _ = record_placement( + state, + accounting_id=ACCOUNTING_UNMATCHED, + destination="turn-accounting-span", + span_id=old_span_id, + usage=_usage_row(call_id=ACCOUNTING_UNMATCHED).to_dict(), + ) + return initialize_eligibility( + placed, + terminal_turn_ids=[TURN_ONE], + accounting_ids=[], + include_history=True, + ) + + update_export_state(enabled_config, stored, _seed) + assert job_path.exists() + + _, matched_projection = self._ambiguous_then_matched_projections() + queue_exports(enabled_config, stored, matched_projection, include_history=True) + + assert not job_path.exists() + state = load_export_state(enabled_config, stored) + placement = state["placements"][ACCOUNTING_UNMATCHED] + assert placement["destination"] == "chat-span" + assert ACCOUNTING_UNMATCHED not in state["conflicts"] + + def test_relocation_is_quarantined_while_old_job_is_claimed( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + ) -> None: + """Unlike the merely-``"queued"`` case, a job a worker has already + ``"claimed"`` could deliver at any moment. Relocating anyway risks a + duplicate if it does; leaving the old placement in place and + recording it as an inconclusive conflict is the only safe response, + the same way a confirmed-delivered correction is quarantined rather + than guessed.""" + stored = _seed_session(enabled_config, paths) + old_span_id = f"accounting:{stored}:{TURN_ONE}:{ACCOUNTING_UNMATCHED}" + jobs_dir = otel_export.otel_jobs_dir(enabled_config.root) + jobs_dir.mkdir(parents=True, exist_ok=True) + digest = hashlib.sha256(old_span_id.encode("utf-8")).hexdigest() + job_path = jobs_dir / f"accounting-{digest}.json" + job_path.write_text(json.dumps({"state": "claimed", "attempt": 0}), encoding="utf-8") + + def _seed(state: dict[str, Any]) -> dict[str, Any]: + placed, _, _ = record_placement( + state, + accounting_id=ACCOUNTING_UNMATCHED, + destination="turn-accounting-span", + span_id=old_span_id, + usage=_usage_row(call_id=ACCOUNTING_UNMATCHED).to_dict(), + ) + return initialize_eligibility( + placed, + terminal_turn_ids=[TURN_ONE], + accounting_ids=[], + include_history=True, + ) + + update_export_state(enabled_config, stored, _seed) + + _, matched_projection = self._ambiguous_then_matched_projections() + queue_exports(enabled_config, stored, matched_projection, include_history=True) + + assert job_path.exists() + assert json.loads(job_path.read_text(encoding="utf-8"))["state"] == "claimed" + state = load_export_state(enabled_config, stored) + placement = state["placements"][ACCOUNTING_UNMATCHED] + assert placement["destination"] == "turn-accounting-span" + assert ACCOUNTING_UNMATCHED in state["conflicts"] + assert "in flight" in state["conflicts"][ACCOUNTING_UNMATCHED]["reason"] + + def test_permanently_failed_accounting_job_is_reported_and_not_cleared( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + ) -> None: + """The generic transport gives up on a deterministic accounting job + after exhausting its retries and marks it permanently ``"failed"`` on + disk (see ``otel_worker._claim_job``) -- it will never retry that job + again on its own. A local write+dispatch reporting success only means + the job file exists and a worker was spawned at some point; it must + not be conflated with actual delivery health, or a permanently stuck + job would silently look fine and have its error cleared forever.""" + stored = _seed_session(enabled_config, paths) + projection = _projection( + turns=[ + _main_turn( + accounting_calls=[ + _accounting_call( + accounting_id=ACCOUNTING_UNMATCHED, + attribution_status="ambiguous", + call_id=None, + usage=_usage_row(call_id=ACCOUNTING_UNMATCHED).to_dict(), + ) + ] + ) + ], + usage_rows=[_usage_row(call_id=ACCOUNTING_UNMATCHED, input_tokens=6587, output_tokens=5)], + attributions=[ + _attribution( + logical_call_id=ACCOUNTING_UNMATCHED, status="ambiguous", call_id=None + ) + ], + ) + span_id = f"accounting:{stored}:{TURN_ONE}:{ACCOUNTING_UNMATCHED}" + jobs_dir = otel_export.otel_jobs_dir(enabled_config.root) + jobs_dir.mkdir(parents=True, exist_ok=True) + digest = hashlib.sha256(span_id.encode("utf-8")).hexdigest() + job_path = jobs_dir / f"accounting-{digest}.json" + job_path.write_text(json.dumps({"state": "failed", "attempt": 5}), encoding="utf-8") + + queued = queue_exports(enabled_config, stored, projection, include_history=True) + assert queued == 1 # only the turn job; the permanently-failed accounting job does not count + state = load_export_state(enabled_config, stored) + assert state["placements"][ACCOUNTING_UNMATCHED]["last_error"] == ( + "accounting job permanently failed after 5 attempts" + ) + class TestWorkerConfirmedDelivery: """`queue_exports` against a real (locally flushed, never remote) Logfire @@ -1237,6 +1481,64 @@ def test_confirmed_accounting_delivery_survives_restart_without_double_emission( state = load_export_state(enabled_config, stored) assert state["placements"][ACCOUNTING_UNMATCHED]["emitted"] is True + def test_chat_embedded_accounting_survives_restart_without_relocation_or_duplicate( + self, + enabled_config: Config, + paths: SourcePaths, + wired_instance, + exporter, + ) -> None: + """Matched usage placed on the chat span is delivered as part of the + turn's own job -- there is no separate fallback accounting job for + it. The durable transport claim must still record that delivery + (``otel_export.accounting_export_sent``), or a later reconciliation + would see the turn as already sent (so the chat span is no longer + available) but the accounting as never delivered, and would + "recover" by relocating it to a brand-new turn-accounting fallback + span -- duplicating the tokens that were already flushed on the chat + span the first time.""" + stored = _seed_session(enabled_config, paths) + projection = _projection( + turns=[_main_turn(accounting_calls=[_accounting_call()])], + usage_rows=[_usage_row()], + attributions=[_attribution(logical_call_id=ACCOUNTING_MATCHED, call_id=CALL_MATCHED)], + ) + + queue_exports(enabled_config, stored, projection, include_history=True) + directory = _directory(enabled_config, stored) + assert otel_export.turn_export_sent(directory, TURN_ONE) is True + assert otel_export.accounting_export_sent(directory, ACCOUNTING_MATCHED) is True + chat_spans = [ + span for span in exporter.exported_spans_as_dict() if span["name"].startswith("chat") + ] + assert len(chat_spans) == 1 + accounting_spans = [ + span for span in exporter.exported_spans_as_dict() if span["name"] == "accounting" + ] + assert len(accounting_spans) == 0 + + # Simulate a restart: fresh Config/instance state, same durable + # ledger, turn claim, and accounting delivery claim on disk. + reloaded = Config( + root=enabled_config.root, + logfire=LogfireSettings(enabled=True, token="fake-token"), + ) + queue_exports(reloaded, stored, projection, include_history=True) + + chat_spans_after = [ + span for span in exporter.exported_spans_as_dict() if span["name"].startswith("chat") + ] + accounting_spans_after = [ + span for span in exporter.exported_spans_as_dict() if span["name"] == "accounting" + ] + assert len(chat_spans_after) == 1 # unchanged: the turn's own claim is first-wins + assert len(accounting_spans_after) == 0 # never relocated to a fallback span + state = load_export_state(enabled_config, stored) + placement = state["placements"][ACCOUNTING_MATCHED] + assert placement["destination"] == "chat-span" + assert placement["emitted"] is True + assert ACCOUNTING_MATCHED not in state["conflicts"] + def _drain_cli_transcript(home: Path) -> list[SourceRecord]: session_dir = home / "session-state" / NATIVE_SESSION_ID From 830b673a1b02b4bdf39f02650e921f5af33457f5 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Sat, 12 Sep 2026 11:42:34 -0700 Subject: [PATCH 77/88] Fix Copilot export reconciliation races --- src/thirdeye/platforms/copilot/export.py | 58 ++- .../platforms/copilot/export_state.py | 40 +- .../platforms/copilot/export_transport.py | 376 ++++++++++++++++++ tests/platforms/copilot/test_export.py | 180 +++++++-- .../copilot/test_export_transport.py | 191 +++++++++ 5 files changed, 788 insertions(+), 57 deletions(-) create mode 100644 src/thirdeye/platforms/copilot/export_transport.py create mode 100644 tests/platforms/copilot/test_export_transport.py diff --git a/src/thirdeye/platforms/copilot/export.py b/src/thirdeye/platforms/copilot/export.py index 40d64fd..2aca7dc 100644 --- a/src/thirdeye/platforms/copilot/export.py +++ b/src/thirdeye/platforms/copilot/export.py @@ -22,6 +22,7 @@ from thirdeye.paths import meta_path, session_dir from thirdeye.span_ids import chat_span_id +from . import export_transport from .constants import PLATFORM_NAME from .export_state import ( clear_placement_error, @@ -30,6 +31,7 @@ is_accounting_eligible, is_turn_eligible, mark_placement_error, + mark_placement_job_status, mark_turn_error, record_placement, update_export_state, @@ -238,10 +240,8 @@ def update(state: dict[str, Any]) -> dict[str, Any]: # to replace, and if it is, cancel it so an already-dispatched # worker can never deliver tokens this ledger no longer points # at once it decides on the new destination below. - status = otel_export.accounting_job_status(config.root, old_span_id) - old_job_state = status.get("state") if status else None - if old_job_state == "queued": - otel_export.cancel_accounting_job(config.root, old_span_id) + status = export_transport.cancel(config.root, old_span_id) + old_job_state = status.get("state") next_state, entry, accepted = record_placement( state, accounting_id=accounting_id, @@ -276,18 +276,30 @@ def _reflect_accounting_job_health( Returns whether this counts as a successful queue for this pass. """ - status = otel_export.accounting_job_status(config.root, span_id) - if status is not None and status.get("state") == "failed": - attempt = status.get("attempt") - _error_state( - config, - stored_session_id, - accounting_id, - f"accounting job permanently failed after {attempt} attempts", - ) - return False - _clear_error_state(config, stored_session_id, accounting_id) - return True + status = export_transport.status(config.root, span_id) + if status is None: + if otel_export.accounting_export_sent( + _directory(config, stored_session_id), accounting_id + ): + status = {"state": "emitted", "attempt": None, "last_error": None} + else: + _clear_error_state(config, stored_session_id, accounting_id) + return True + + if status.get("state") == "failed" and not status.get("last_error"): + status = { + **status, + "last_error": ( + f"accounting job permanently failed after {status.get('attempt')} attempts" + ), + } + + update_export_state( + config, + stored_session_id, + lambda state: mark_placement_job_status(state, accounting_id, status), + ) + return status.get("state") != "failed" def _turn_with_placed_accounting( @@ -377,8 +389,11 @@ def queue_exports( if not is_turn_eligible(state, root_id) or not is_accounting_eligible(state, accounting_id): continue call_id = item.get("call_id") - delivered = otel_export.accounting_export_sent(directory, accounting_id) existing_entry = (state.get("placements") or {}).get(accounting_id) or {} + turn_sent = otel_export.turn_export_sent(directory, root_id) + delivered = otel_export.accounting_export_sent(directory, accounting_id) or ( + turn_sent and existing_entry.get("destination") == "chat-span" + ) # A chat span already flushed to Logfire is immutable history: usage # that resolves to "matched" only *after* that flush can no longer # land on it and must use the turn-owned fallback span instead. But @@ -388,7 +403,7 @@ def queue_exports( # what delivered it there -- re-deriving "unavailable" from that same # fact would misread an already-settled placement as a conflicting # correction. - chat_available = not otel_export.turn_export_sent(directory, root_id) or ( + chat_available = not turn_sent or ( delivered and existing_entry.get("destination") == "chat-span" ) if ( @@ -421,11 +436,10 @@ def queue_exports( continue if destination != "turn-accounting-span" or entry is None: continue - sent = otel_export.export_turn_accounting( + sent = export_transport.queue_turn_accounting( config, directory, stored_session_id, - PLATFORM_NAME, meta.cwd, owner_id, item, @@ -479,8 +493,8 @@ def queue_exports( "evidence": list(attribution["evidence"]), }, } - sent = otel_export.export_session_accounting( - config, directory, stored_session_id, PLATFORM_NAME, meta.cwd, item + sent = export_transport.queue_session_accounting( + config, directory, stored_session_id, meta.cwd, item ) if sent and _reflect_accounting_job_health(config, stored_session_id, accounting_id, span_id): queued += 1 diff --git a/src/thirdeye/platforms/copilot/export_state.py b/src/thirdeye/platforms/copilot/export_state.py index bc5429e..e2c5d42 100644 --- a/src/thirdeye/platforms/copilot/export_state.py +++ b/src/thirdeye/platforms/copilot/export_state.py @@ -2,7 +2,7 @@ This state intentionally has no relationship to projection state. Projection state is disposable and rebuilt from the V1 archive; export placement is an -accounting decision and survives rebuilds. The generic OTel worker cannot +accounting decision and survives rebuilds. The accounting worker cannot atomically acknowledge a remote collector and this file, so an absent job is never treated as proof of delivery. A crash after a remote flush can still lead to a deterministic retry and therefore a duplicate remote span. @@ -213,15 +213,11 @@ def record_placement( by the worker on success, so this ledger cannot detect delivery on its own and must be told. - ``old_job_state`` is the caller's fresh read (``otel_export. - accounting_job_status``) of the *existing* placement's own job file, - when relocating to a different destination/span. ``None``/``"queued"``/ - ``"failed"`` all mean nothing is currently in flight for the old job (it - was never queued, is still sitting untouched, or permanently gave up), - so relocating is safe. ``"claimed"`` means a worker holds it right now - and could deliver at any moment — proven neither safe to relocate nor - known to already be delivered, so it is quarantined the same as a - confirmed-delivered correction rather than guessed either way. + ``old_job_state`` is the Copilot transport's result after atomically + attempting to cancel the *existing* placement's own job. ``"cancelled"`` + means relocation is safe. ``"claimed"`` means a worker holds it right now + and could deliver at any moment; ``"emitted"`` means it already completed. + Both states quarantine the correction. Returns ``(state, entry, accepted)``. A correction is only a durable conflict when the existing placement was (or is now known to have been) @@ -247,7 +243,7 @@ def record_placement( existing = {**existing, "emitted": True, "last_error": None} placements[accounting_id] = existing return result, existing, True - if already_delivered or old_job_state == "claimed": + if already_delivered or old_job_state in {"claimed", "emitted"}: if delivered and not existing.get("emitted"): # The candidate is rejected, but the fresh delivery read is # still new information about the *existing* placement — @@ -315,6 +311,28 @@ def clear_placement_error(state: dict[str, Any], accounting_id: str) -> dict[str return result +def mark_placement_job_status( + state: dict[str, Any], accounting_id: str, status: dict[str, Any] +) -> dict[str, Any]: + """Persist the worker lifecycle state and its latest delivery error.""" + result = _normalize(state) + entry = _mapping(result["placements"].get(accounting_id)) + if not entry: + return result + entry["job_state"] = status.get("state") + entry["job_attempt"] = status.get("attempt") + worker_error = status.get("last_error") + if isinstance(worker_error, str) and worker_error: + entry["last_error"] = worker_error + elif status.get("state") in {"queued", "claimed"}: + entry["last_error"] = None + if status.get("state") == "emitted": + entry["emitted"] = True + entry["last_error"] = None + result["placements"][accounting_id] = entry + return result + + def mark_placement_delivered(state: dict[str, Any], accounting_id: str) -> dict[str, Any]: """Record an externally confirmed delivery, never inferred from a job file. diff --git a/src/thirdeye/platforms/copilot/export_transport.py b/src/thirdeye/platforms/copilot/export_transport.py new file mode 100644 index 0000000..c46b0c1 --- /dev/null +++ b/src/thirdeye/platforms/copilot/export_transport.py @@ -0,0 +1,376 @@ +"""Copilot-owned transport for deterministic accounting exports. + +Copilot accounting can move as transcript and usage evidence is reconciled. +Its jobs therefore need an atomic cancel/claim protocol and observable retry +state that the generic fire-and-forget OTel transport does not promise. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import sys +import tempfile +import time +from pathlib import Path +from typing import Any + +from thirdeye import otel_export +from thirdeye._compat import fsops, proc +from thirdeye.config import Config +from thirdeye.usage.errlog import log_capture_error + +from .jsonio import atomic_write_json + +_CLAIM_STALE_S = 30.0 +_MAX_ATTEMPTS = 5 +_KINDS = frozenset({"session_accounting", "turn_accounting"}) + + +def jobs_dir(root: Path) -> Path: + return root / "logs" / "copilot-otel-jobs" + + +def job_path(root: Path, job_id: str) -> Path: + digest = hashlib.sha256(job_id.encode("utf-8")).hexdigest() + return jobs_dir(root) / f"accounting-{digest}.json" + + +def claim_path(path: Path) -> Path: + return path.with_suffix(f"{path.suffix}.claim") + + +def _atomic_create(path: Path, text: str) -> bool: + path.parent.mkdir(parents=True, exist_ok=True) + try: + descriptor = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY) + except FileExistsError: + return False + try: + os.write(descriptor, text.encode("utf-8")) + except BaseException: + fsops.unlink(path, missing_ok=True) + raise + finally: + os.close(descriptor) + return True + + +def _take_claim(path: Path, text: str, *, recover_stale: bool) -> bool: + if _atomic_create(path, text): + return True + if not recover_stale: + return False + try: + stale = time.time() - path.stat().st_mtime > _CLAIM_STALE_S + except OSError: + stale = True + if not stale: + return False + fsops.unlink(path, missing_ok=True) + return _atomic_create(path, text) + + +def _read_job(path: Path) -> dict[str, Any] | None: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (FileNotFoundError, json.JSONDecodeError): + return None + return value if isinstance(value, dict) else None + + +def _write_job(path: Path, payload: dict[str, Any]) -> None: + atomic_write_json(path, payload) + + +def _create_job(path: Path, payload: dict[str, Any]) -> bool: + """Publish a complete job atomically, preserving the first writer.""" + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary_name = tempfile.mkstemp( + dir=path.parent, prefix=f".{path.name}.", suffix=".tmp" + ) + temporary = Path(temporary_name) + try: + with os.fdopen(descriptor, "w", encoding="utf-8", newline="\n") as stream: + json.dump(payload, stream, default=str, sort_keys=True, separators=(",", ":")) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + try: + os.link(temporary, path) + except FileExistsError: + return False + fsops.sync_directory(path.parent) + return True + finally: + fsops.unlink(temporary, missing_ok=True) + + +def status(root: Path, job_id: str) -> dict[str, Any] | None: + path = job_path(root, job_id) + payload = _read_job(path) + if payload is None: + return None + state = "claimed" if claim_path(path).exists() else payload.get("state") + return { + "state": state, + "attempt": payload.get("attempt"), + "last_error": payload.get("last_error"), + } + + +def cancel(root: Path, job_id: str) -> dict[str, Any]: + """Cancel only after acquiring the same exclusive claim as the worker.""" + path = job_path(root, job_id) + owner = claim_path(path) + if not _atomic_create(owner, f"cancel:{os.getpid()}"): + return {**(status(root, job_id) or {}), "state": "claimed"} + try: + payload = _read_job(path) + if payload is None: + return {"state": "cancelled", "attempt": None, "last_error": None} + result = { + "state": payload.get("state"), + "attempt": payload.get("attempt"), + "last_error": payload.get("last_error"), + } + if result["state"] == "emitted": + return result + fsops.unlink(path, missing_ok=True) + return {**result, "state": "cancelled"} + finally: + fsops.unlink(owner, missing_ok=True) + + +def _spawn(path: Path) -> None: + proc.spawn_detached([sys.executable, "-m", "thirdeye.platforms.copilot.export_transport", str(path)]) + + +def _placement_is_current(config: Config, payload: dict[str, Any]) -> bool: + from .export_state import load_export_state + + state = load_export_state(config, str(payload["session_id"])) + placement = (state.get("placements") or {}).get(str(payload["accounting_id"])) or {} + return placement.get("span_id") == payload.get("job_id") + + +def _queue(config: Config, payload: dict[str, Any]) -> bool: + if not config.logfire.enabled or not config.logfire.token: + return False + try: + if payload.get("kind") not in _KINDS: + raise ValueError(f"unsupported Copilot accounting job kind: {payload.get('kind')!r}") + path = job_path(config.root, str(payload["job_id"])) + queued = { + **payload, + "captured_attributes": otel_export._resolve_captured_attributes(config, None), + "state": "queued", + "attempt": 0, + } + owner = claim_path(path) + if not _take_claim(owner, f"queue:{os.getpid()}", recover_stale=True): + return True + dispatch = False + try: + if not _placement_is_current(config, payload): + return True + delivered = otel_export.accounting_export_sent( + Path(str(payload["session_dir"])), str(payload["accounting_id"]) + ) + if not delivered: + _create_job(path, queued) + dispatch = True + finally: + fsops.unlink(owner, missing_ok=True) + if dispatch: + _spawn(path) + return True + except Exception as exc: + log_capture_error( + thirdeye_home=config.root, + phase="copilot_accounting_export_spawn", + error=exc, + platform="copilot", + session_id=str(payload.get("session_id") or ""), + ) + return False + + +def queue_session_accounting( + config: Config, + session_dir: Path, + session_id: str, + cwd: str, + accounting: dict[str, Any], +) -> bool: + accounting_id = str(accounting["accounting_id"]) + logical_span_id = f"accounting:{session_id}:{accounting_id}" + return _queue( + config, + { + "job_id": logical_span_id, + "kind": "session_accounting", + "session_dir": str(session_dir), + "session_id": session_id, + "cwd": cwd, + "accounting_id": accounting_id, + "usage": dict(accounting["usage"]), + "attribution_status": str(accounting["attribution_status"]), + "agent_id": accounting.get("agent_id"), + "attributes": dict(accounting.get("attributes") or {}), + }, + ) + + +def queue_turn_accounting( + config: Config, + session_dir: Path, + session_id: str, + cwd: str, + turn_id: str, + accounting: dict[str, Any], + *, + turn_span_id: str, +) -> bool: + accounting_id = str(accounting["accounting_id"]) + logical_span_id = f"accounting:{session_id}:{turn_id}:{accounting_id}" + return _queue( + config, + { + "job_id": logical_span_id, + "kind": "turn_accounting", + "session_dir": str(session_dir), + "session_id": session_id, + "cwd": cwd, + "turn_id": turn_id, + "turn_span_id": turn_span_id, + "accounting_id": accounting_id, + "usage": dict(accounting["usage"]), + "attribution_status": str(accounting["attribution_status"]), + "agent_id": accounting.get("agent_id"), + "attributes": dict(accounting.get("attributes") or {}), + }, + ) + + +def _acquire(path: Path) -> dict[str, Any] | None: + owner = claim_path(path) + if not _take_claim(owner, str(os.getpid()), recover_stale=True): + return None + + # Re-read only after owning the claim. A cancellation can happen after a + # worker's initial argv read but before this point; stale in-memory bytes + # must never resurrect the cancelled job. + payload = _read_job(path) + if payload is None or payload.get("state") in {"failed", "emitted"}: + fsops.unlink(owner, missing_ok=True) + return None + claimed = {**payload, "state": "claimed"} + _write_job(path, claimed) + return claimed + + +def _accounting(payload: dict[str, Any]) -> dict[str, Any]: + return { + "accounting_id": payload["accounting_id"], + "usage": payload["usage"], + "attribution_status": payload["attribution_status"], + "agent_id": payload.get("agent_id"), + "attributes": payload.get("attributes") or {}, + "call_id": payload.get("call_id"), + } + + +def _deliver(config: Config, payload: dict[str, Any]) -> None: + token = otel_export._captured_attributes.set(payload.get("captured_attributes") or {}) + try: + common = { + "config": config, + "session_dir_": Path(payload["session_dir"]), + "session_id": payload["session_id"], + "platform": "copilot", + "cwd": payload["cwd"], + "accounting": _accounting(payload), + } + if payload["kind"] == "session_accounting": + otel_export._export_session_accounting_inner(**common) + elif payload["kind"] == "turn_accounting": + otel_export._export_turn_accounting_inner( + **common, + turn_id=str(payload["turn_id"]), + turn_span_id=payload.get("turn_span_id"), + ) + else: + raise ValueError(f"unsupported Copilot accounting job kind: {payload.get('kind')!r}") + finally: + otel_export._captured_attributes.reset(token) + + +def run(path: Path) -> None: + root = path.parents[2] if len(path.parents) > 2 else path.parent + try: + claimed = _acquire(path) + except Exception as exc: + log_capture_error( + thirdeye_home=root, + phase="copilot_accounting_claim_failed", + error=exc, + platform="copilot", + ) + return + if claimed is None: + return + owner = claim_path(path) + try: + _deliver(Config.load(), claimed) + except Exception as exc: + attempt = int(claimed.get("attempt", 0)) + 1 + retry = { + **claimed, + "state": "failed" if attempt >= _MAX_ATTEMPTS else "retrying", + "attempt": attempt, + "last_error": f"{type(exc).__name__}: {exc}", + } + try: + _write_job(path, retry) + except Exception as state_error: + log_capture_error( + thirdeye_home=root, + phase="copilot_accounting_state_failed", + error=state_error, + platform="copilot", + session_id=str(claimed.get("session_id") or ""), + ) + log_capture_error( + thirdeye_home=root, + phase="copilot_accounting_export_failed", + error=exc, + platform="copilot", + session_id=str(claimed.get("session_id") or ""), + message=f"accounting_id={claimed.get('accounting_id')} kind={claimed.get('kind')}", + ) + else: + try: + _write_job(path, {**claimed, "state": "emitted", "last_error": None}) + fsops.unlink(path, missing_ok=True) + except Exception as exc: + log_capture_error( + thirdeye_home=root, + phase="copilot_accounting_ack_failed", + error=exc, + platform="copilot", + session_id=str(claimed.get("session_id") or ""), + ) + finally: + fsops.unlink(owner, missing_ok=True) + + +def main(argv: list[str] | None = None) -> None: + values = sys.argv[1:] if argv is None else argv + if values: + run(Path(values[0])) + + +if __name__ == "__main__": + main() diff --git a/tests/platforms/copilot/test_export.py b/tests/platforms/copilot/test_export.py index 9565274..f974ff0 100644 --- a/tests/platforms/copilot/test_export.py +++ b/tests/platforms/copilot/test_export.py @@ -2,7 +2,6 @@ from __future__ import annotations -import hashlib import json import shutil from pathlib import Path @@ -13,6 +12,7 @@ from thirdeye import otel_export from thirdeye.config import Config, LogfireSettings from thirdeye.paths import session_dir +from thirdeye.platforms.copilot import export_transport from thirdeye.platforms.copilot.archive import commit_batch from thirdeye.platforms.copilot.constants import PLATFORM_NAME from thirdeye.platforms.copilot.export import queue_exports @@ -222,16 +222,16 @@ def _turn(*args: Any, **kwargs: Any) -> bool: return True def _turn_accounting(*args: Any, **kwargs: Any) -> bool: - calls["turn_accounting"].append(args[6]) + calls["turn_accounting"].append(args[5]) return True def _session_accounting(*args: Any, **kwargs: Any) -> bool: - calls["session_accounting"].append(args[5]) + calls["session_accounting"].append(args[4]) return True monkeypatch.setattr(otel_export, "export_turn", _turn) - monkeypatch.setattr(otel_export, "export_turn_accounting", _turn_accounting) - monkeypatch.setattr(otel_export, "export_session_accounting", _session_accounting) + monkeypatch.setattr(export_transport, "queue_turn_accounting", _turn_accounting) + monkeypatch.setattr(export_transport, "queue_session_accounting", _session_accounting) return calls @@ -822,6 +822,34 @@ def test_confirmed_turn_delivery_routes_new_match_to_fallback( turn = export_calls["turn"][0] assert turn["accounting_calls"] == [] + def test_turn_claim_confirms_previously_embedded_chat_accounting( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + ) -> None: + """The turn acknowledgement is sufficient for accounting that the + Copilot ledger already placed inside that turn's chat span. This + survives a crash before the shared exporter writes its secondary + per-accounting acknowledgement.""" + stored = _seed_session(enabled_config, paths) + _, matched_projection = self._ambiguous_then_matched_projections() + queue_exports(enabled_config, stored, matched_projection, include_history=True) + + directory = _directory(enabled_config, stored) + claim_path = otel_export._turn_claim_path(directory, TURN_ONE) + claim_path.parent.mkdir(parents=True, exist_ok=True) + claim_path.write_text("sent", encoding="utf-8", newline="\n") + export_calls["turn"].clear() + export_calls["turn_accounting"].clear() + + queue_exports(enabled_config, stored, matched_projection, include_history=True) + + placement = load_export_state(enabled_config, stored)["placements"][ACCOUNTING_UNMATCHED] + assert placement["destination"] == "chat-span" + assert placement["emitted"] is True + assert export_calls["turn_accounting"] == [] + def test_emitted_placement_skips_requeue( self, enabled_config: Config, @@ -911,8 +939,12 @@ def test_accounting_job_failure_records_error( ) -> None: stored = _seed_session(enabled_config, paths) monkeypatch.setattr(otel_export, "export_turn", lambda *args, **kwargs: True) - monkeypatch.setattr(otel_export, "export_turn_accounting", lambda *args, **kwargs: False) - monkeypatch.setattr(otel_export, "export_session_accounting", lambda *args, **kwargs: False) + monkeypatch.setattr( + export_transport, "queue_turn_accounting", lambda *args, **kwargs: False + ) + monkeypatch.setattr( + export_transport, "queue_session_accounting", lambda *args, **kwargs: False + ) projection = _projection( turns=[ @@ -971,7 +1003,9 @@ def test_errors_clear_once_a_retry_succeeds( not linger once a later reconciliation successfully queues the job.""" stored = _seed_session(enabled_config, paths) monkeypatch.setattr(otel_export, "export_turn", lambda *args, **kwargs: False) - monkeypatch.setattr(otel_export, "export_turn_accounting", lambda *args, **kwargs: False) + monkeypatch.setattr( + export_transport, "queue_turn_accounting", lambda *args, **kwargs: False + ) projection = _projection( turns=[ @@ -1001,7 +1035,9 @@ def test_errors_clear_once_a_retry_succeeds( ) monkeypatch.setattr(otel_export, "export_turn", lambda *args, **kwargs: True) - monkeypatch.setattr(otel_export, "export_turn_accounting", lambda *args, **kwargs: True) + monkeypatch.setattr( + export_transport, "queue_turn_accounting", lambda *args, **kwargs: True + ) queue_exports(enabled_config, stored, projection, include_history=True) state = load_export_state(enabled_config, stored) @@ -1247,10 +1283,8 @@ def test_relocation_cancels_stale_queued_job_at_old_span( still pick it up later and deliver the same tokens a second time.""" stored = _seed_session(enabled_config, paths) old_span_id = f"accounting:{stored}:{TURN_ONE}:{ACCOUNTING_UNMATCHED}" - jobs_dir = otel_export.otel_jobs_dir(enabled_config.root) - jobs_dir.mkdir(parents=True, exist_ok=True) - digest = hashlib.sha256(old_span_id.encode("utf-8")).hexdigest() - job_path = jobs_dir / f"accounting-{digest}.json" + job_path = export_transport.job_path(enabled_config.root, old_span_id) + job_path.parent.mkdir(parents=True, exist_ok=True) job_path.write_text(json.dumps({"state": "queued", "attempt": 0}), encoding="utf-8") def _seed(state: dict[str, Any]) -> dict[str, Any]: @@ -1294,11 +1328,11 @@ def test_relocation_is_quarantined_while_old_job_is_claimed( than guessed.""" stored = _seed_session(enabled_config, paths) old_span_id = f"accounting:{stored}:{TURN_ONE}:{ACCOUNTING_UNMATCHED}" - jobs_dir = otel_export.otel_jobs_dir(enabled_config.root) - jobs_dir.mkdir(parents=True, exist_ok=True) - digest = hashlib.sha256(old_span_id.encode("utf-8")).hexdigest() - job_path = jobs_dir / f"accounting-{digest}.json" + job_path = export_transport.job_path(enabled_config.root, old_span_id) + job_path.parent.mkdir(parents=True, exist_ok=True) job_path.write_text(json.dumps({"state": "claimed", "attempt": 0}), encoding="utf-8") + claim_path = export_transport.claim_path(job_path) + claim_path.write_text("worker-42", encoding="utf-8") def _seed(state: dict[str, Any]) -> dict[str, Any]: placed, _, _ = record_placement( @@ -1321,6 +1355,7 @@ def _seed(state: dict[str, Any]) -> dict[str, Any]: queue_exports(enabled_config, stored, matched_projection, include_history=True) assert job_path.exists() + assert claim_path.exists() assert json.loads(job_path.read_text(encoding="utf-8"))["state"] == "claimed" state = load_export_state(enabled_config, stored) placement = state["placements"][ACCOUNTING_UNMATCHED] @@ -1328,6 +1363,45 @@ def _seed(state: dict[str, Any]) -> dict[str, Any]: assert ACCOUNTING_UNMATCHED in state["conflicts"] assert "in flight" in state["conflicts"][ACCOUNTING_UNMATCHED]["reason"] + def test_relocation_is_quarantined_when_worker_claims_during_cancellation( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + monkeypatch: pytest.MonkeyPatch, + ) -> None: + stored = _seed_session(enabled_config, paths) + old_span_id = f"accounting:{stored}:{TURN_ONE}:{ACCOUNTING_UNMATCHED}" + + def _seed(state: dict[str, Any]) -> dict[str, Any]: + placed, _, _ = record_placement( + state, + accounting_id=ACCOUNTING_UNMATCHED, + destination="turn-accounting-span", + span_id=old_span_id, + usage=_usage_row(call_id=ACCOUNTING_UNMATCHED).to_dict(), + ) + return initialize_eligibility( + placed, + terminal_turn_ids=[TURN_ONE], + accounting_ids=[], + include_history=True, + ) + + update_export_state(enabled_config, stored, _seed) + monkeypatch.setattr( + export_transport, + "cancel", + lambda *args: {"state": "claimed", "attempt": 0, "last_error": None}, + ) + + _, matched_projection = self._ambiguous_then_matched_projections() + queue_exports(enabled_config, stored, matched_projection, include_history=True) + + state = load_export_state(enabled_config, stored) + assert state["placements"][ACCOUNTING_UNMATCHED]["span_id"] == old_span_id + assert ACCOUNTING_UNMATCHED in state["conflicts"] + def test_permanently_failed_accounting_job_is_reported_and_not_cleared( self, enabled_config: Config, @@ -1363,19 +1437,72 @@ def test_permanently_failed_accounting_job_is_reported_and_not_cleared( ], ) span_id = f"accounting:{stored}:{TURN_ONE}:{ACCOUNTING_UNMATCHED}" - jobs_dir = otel_export.otel_jobs_dir(enabled_config.root) - jobs_dir.mkdir(parents=True, exist_ok=True) - digest = hashlib.sha256(span_id.encode("utf-8")).hexdigest() - job_path = jobs_dir / f"accounting-{digest}.json" - job_path.write_text(json.dumps({"state": "failed", "attempt": 5}), encoding="utf-8") + job_path = export_transport.job_path(enabled_config.root, span_id) + job_path.parent.mkdir(parents=True, exist_ok=True) + job_path.write_text( + json.dumps( + {"state": "failed", "attempt": 5, "last_error": "TimeoutError: collector stalled"} + ), + encoding="utf-8", + ) queued = queue_exports(enabled_config, stored, projection, include_history=True) assert queued == 1 # only the turn job; the permanently-failed accounting job does not count state = load_export_state(enabled_config, stored) - assert state["placements"][ACCOUNTING_UNMATCHED]["last_error"] == ( - "accounting job permanently failed after 5 attempts" + placement = state["placements"][ACCOUNTING_UNMATCHED] + assert placement["job_state"] == "failed" + assert placement["job_attempt"] == 5 + assert placement["last_error"] == "TimeoutError: collector stalled" + + @pytest.mark.parametrize( + ("job_state", "last_error"), + [ + ("queued", None), + ("claimed", None), + ("retrying", "ConnectionError: collector unavailable"), + ], + ) + def test_accounting_worker_lifecycle_is_reflected_in_ledger( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + monkeypatch: pytest.MonkeyPatch, + job_state: str, + last_error: str | None, + ) -> None: + stored = _seed_session(enabled_config, paths) + projection = _projection( + turns=[ + _main_turn( + accounting_calls=[ + _accounting_call( + accounting_id=ACCOUNTING_UNMATCHED, + attribution_status="ambiguous", + call_id=None, + usage=_usage_row(call_id=ACCOUNTING_UNMATCHED).to_dict(), + ) + ] + ) + ], + usage_rows=[_usage_row(call_id=ACCOUNTING_UNMATCHED)], + attributions=[ + _attribution(logical_call_id=ACCOUNTING_UNMATCHED, status="ambiguous", call_id=None) + ], + ) + monkeypatch.setattr( + export_transport, + "status", + lambda *args: {"state": job_state, "attempt": 2, "last_error": last_error}, ) + queue_exports(enabled_config, stored, projection, include_history=True) + + placement = load_export_state(enabled_config, stored)["placements"][ACCOUNTING_UNMATCHED] + assert placement["job_state"] == job_state + assert placement["job_attempt"] == 2 + assert placement["last_error"] == last_error + class TestWorkerConfirmedDelivery: """`queue_exports` against a real (locally flushed, never remote) Logfire @@ -1428,6 +1555,11 @@ def _run(job_path: Path) -> None: monkeypatch.setattr(otel_export, "_spawn", _run) + def _run_accounting(job_path: Path) -> None: + export_transport.main([str(job_path)]) + + monkeypatch.setattr(export_transport, "_spawn", _run_accounting) + def test_confirmed_accounting_delivery_survives_restart_without_double_emission( self, enabled_config: Config, diff --git a/tests/platforms/copilot/test_export_transport.py b/tests/platforms/copilot/test_export_transport.py new file mode 100644 index 0000000..76fcaa0 --- /dev/null +++ b/tests/platforms/copilot/test_export_transport.py @@ -0,0 +1,191 @@ +"""Copilot accounting transport claim, cancellation, and retry behavior.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from thirdeye import otel_export +from thirdeye.config import Config, LogfireSettings +from thirdeye.platforms.copilot import export_transport +from thirdeye.platforms.copilot.export_state import record_placement, update_export_state + + +@pytest.fixture +def config(tmp_path: Path) -> Config: + return Config( + root=tmp_path / "thirdeye", + logfire=LogfireSettings(enabled=True, token="fake-token"), + ) + + +def _payload(**overrides: Any) -> dict[str, Any]: + value: dict[str, Any] = { + "job_id": "accounting:stored-session:acct-1", + "kind": "session_accounting", + "session_dir": "/traces/copilot/stored-session", + "session_id": "stored-session", + "cwd": "/proj", + "accounting_id": "acct-1", + "usage": {}, + "attribution_status": "ambiguous", + "state": "queued", + "attempt": 0, + } + value.update(overrides) + return value + + +def _write(config: Config, payload: dict[str, Any]) -> Path: + path = export_transport.job_path(config.root, str(payload["job_id"])) + export_transport._write_job(path, payload) + return path + + +def _seed_placement( + config: Config, payload: dict[str, Any], *, span_id: str | None = None +) -> None: + def _record(state: dict[str, Any]) -> dict[str, Any]: + updated, _, _ = record_placement( + state, + accounting_id=str(payload["accounting_id"]), + destination="session-accounting-span", + span_id=span_id or str(payload["job_id"]), + usage={}, + ) + return updated + + update_export_state(config, str(payload["session_id"]), _record) + + +def test_queue_is_deterministic(config: Config, monkeypatch: pytest.MonkeyPatch) -> None: + spawned: list[Path] = [] + monkeypatch.setattr(export_transport, "_spawn", spawned.append) + accounting = { + "accounting_id": "acct-1", + "usage": {}, + "attribution_status": "ambiguous", + } + _seed_placement(config, _payload()) + + assert export_transport.queue_session_accounting( + config, Path("/traces/copilot/stored-session"), "stored-session", "/proj", accounting + ) + assert export_transport.queue_session_accounting( + config, Path("/traces/copilot/stored-session"), "stored-session", "/proj", accounting + ) + + jobs = list(export_transport.jobs_dir(config.root).glob("accounting-*.json")) + assert len(jobs) == 1 + assert len(spawned) == 2 + + +def test_queue_does_not_recreate_confirmed_delivery( + config: Config, monkeypatch: pytest.MonkeyPatch +) -> None: + spawned: list[Path] = [] + monkeypatch.setattr(export_transport, "_spawn", spawned.append) + session_dir = config.root / "traces" / "copilot" / "stored-session" + otel_export._mark_accounting_sent(session_dir, "acct-1") + accounting = { + "accounting_id": "acct-1", + "usage": {}, + "attribution_status": "ambiguous", + } + _seed_placement(config, _payload()) + + assert export_transport.queue_session_accounting( + config, session_dir, "stored-session", "/proj", accounting + ) + + assert list(export_transport.jobs_dir(config.root).glob("accounting-*.json")) == [] + assert spawned == [] + + +def test_queue_does_not_publish_a_stale_placement( + config: Config, monkeypatch: pytest.MonkeyPatch +) -> None: + spawned: list[Path] = [] + monkeypatch.setattr(export_transport, "_spawn", spawned.append) + payload = _payload() + _seed_placement(config, payload, span_id="accounting:stored-session:new-placement") + + assert export_transport._queue(config, payload) + + assert not export_transport.job_path(config.root, payload["job_id"]).exists() + assert spawned == [] + + +def test_queue_does_not_publish_while_another_owner_holds_the_claim( + config: Config, monkeypatch: pytest.MonkeyPatch +) -> None: + spawned: list[Path] = [] + monkeypatch.setattr(export_transport, "_spawn", spawned.append) + payload = _payload() + path = export_transport.job_path(config.root, payload["job_id"]) + owner = export_transport.claim_path(path) + owner.parent.mkdir(parents=True, exist_ok=True) + owner.write_text("cancel-in-progress", encoding="utf-8") + + assert export_transport._queue(config, payload) + + assert not path.exists() + assert spawned == [] + + +def test_cancel_preserves_worker_owned_job(config: Config) -> None: + payload = _payload() + path = _write(config, payload) + export_transport.claim_path(path).write_text("worker-42", encoding="utf-8") + + assert export_transport.status(config.root, payload["job_id"])["state"] == "claimed" + result = export_transport.cancel(config.root, payload["job_id"]) + + assert result["state"] == "claimed" + assert path.exists() + assert export_transport.claim_path(path).exists() + + +def test_cancelled_job_cannot_be_resurrected_by_a_stale_reader(config: Config) -> None: + payload = _payload() + path = _write(config, payload) + + result = export_transport.cancel(config.root, payload["job_id"]) + claimed = export_transport._acquire(path) + + assert result["state"] == "cancelled" + assert claimed is None + assert not path.exists() + assert not export_transport.claim_path(path).exists() + + +@pytest.mark.parametrize( + ("attempt", "expected_state"), + [(0, "retrying"), (export_transport._MAX_ATTEMPTS - 1, "failed")], +) +def test_worker_failure_persists_state_and_error( + config: Config, + monkeypatch: pytest.MonkeyPatch, + attempt: int, + expected_state: str, +) -> None: + payload = _payload(attempt=attempt) + path = _write(config, payload) + monkeypatch.setattr(Config, "load", lambda: config) + + def _fail(config: Config, payload: dict[str, Any]) -> None: + raise TimeoutError("collector stalled") + + monkeypatch.setattr(export_transport, "_deliver", _fail) + + export_transport.run(path) + + status = export_transport.status(config.root, payload["job_id"]) + assert status == { + "state": expected_state, + "attempt": attempt + 1, + "last_error": "TimeoutError: collector stalled", + } + assert not export_transport.claim_path(path).exists() From f9c14e600670b645447726bf85c0a1a13d382510 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Sat, 12 Sep 2026 12:14:33 -0700 Subject: [PATCH 78/88] Keep export reconciliation Copilot-specific --- src/thirdeye/otel_export.py | 114 +----------------- src/thirdeye/platforms/copilot/export.py | 19 ++- .../platforms/copilot/export_state.py | 11 +- .../platforms/copilot/export_transport.py | 63 +++++++++- tests/platforms/copilot/test_export.py | 32 ++--- .../copilot/test_export_transport.py | 3 +- 6 files changed, 91 insertions(+), 151 deletions(-) diff --git a/src/thirdeye/otel_export.py b/src/thirdeye/otel_export.py index deb852b..bbb202d 100644 --- a/src/thirdeye/otel_export.py +++ b/src/thirdeye/otel_export.py @@ -545,47 +545,6 @@ def _write_accounting_job(thirdeye_home: Path, payload: dict[str, Any]) -> Path: return job_path -def accounting_job_status(thirdeye_home: Path, job_id: str) -> dict[str, Any] | None: - """Read the current on-disk state of a deterministic accounting job. - - Returns ``None`` when no job file exists for this id: either it was - never queued, or the worker already claimed, emitted, and deleted it - (see ``_run_accounting_job``). Callers that need to know "confirmed - delivered" rather than "no job file present" must consult - ``accounting_export_sent`` separately -- that durable claim is written - before the job file is removed, so it survives this function returning - ``None`` for an already-delivered identity. - """ - digest = hashlib.sha256(job_id.encode("utf-8")).hexdigest() - job_path = otel_jobs_dir(thirdeye_home) / f"accounting-{digest}.json" - try: - payload = json.loads(job_path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): - return None - return {"state": payload.get("state"), "attempt": payload.get("attempt")} - - -def cancel_accounting_job(thirdeye_home: Path, job_id: str) -> None: - """Best-effort removal of a still-queued deterministic accounting job. - - Only meant to be called right after ``accounting_job_status`` reported - ``"queued"`` for this id -- a caller relocating an accounting identity to - a new destination/span uses this to keep the stale job (keyed by the old - span id) from being picked up by an already-dispatched worker and - delivering tokens nobody's ledger points at anymore. If a worker wins the - race and claims the job between that read and this delete, the delete is - still safe: the worker holds its payload in memory and only ever - rewrites the same path, so removing the file first does not resurrect a - delivery that wasn't already in flight, and a worker that instead loses - the race hits a benign missing-file read and exits (see - ``otel_worker.main``). - """ - digest = hashlib.sha256(job_id.encode("utf-8")).hexdigest() - job_path = otel_jobs_dir(thirdeye_home) / f"accounting-{digest}.json" - fsops.unlink(job_path, missing_ok=True) - fsops.unlink(job_path.with_suffix(f"{job_path.suffix}.claim"), missing_ok=True) - - def _spawn(job_path: Path) -> None: """Hand a job file to a detached ``thirdeye.otel_worker``. @@ -621,64 +580,6 @@ def turn_export_sent(session_dir_: Path, turn_id: str) -> bool: return False -def _accounting_claim_path(session_dir_: Path, accounting_id: str) -> Path: - # Same hashed-filename rationale as `_turn_claim_path`: an accounting id - # is caller-derived and not guaranteed filesystem-safe. - digest = hashlib.sha256(accounting_id.encode()).hexdigest() - return session_dir_ / "otel-accounting-sent" / f"{digest}.json" - - -def accounting_export_sent(session_dir_: Path, accounting_id: str) -> bool: - """Whether one accounting identity's fallback/chat tokens were ever - confirmed flushed to Logfire. - - The deterministic accounting job file the worker writes is deleted once - delivery succeeds (see `otel_worker._run_accounting_job`), so it cannot - be reused as a durable "already delivered" signal — a later caller would - see no job file and wrongly conclude nothing was ever sent, and requeue - a duplicate. This claim persists independently of that job file, mirroring - `_turn_claim_path`, so a caller like Copilot's export ledger can tell a - confirmed delivery apart from one that is merely queued or still retrying. - """ - try: - return _accounting_claim_path(session_dir_, accounting_id).read_text( - encoding="utf-8" - ) == "sent" - except OSError: - return False - - -def _mark_accounting_sent(session_dir_: Path, accounting_id: str) -> None: - path = _accounting_claim_path(session_dir_, accounting_id) - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text("sent", encoding="utf-8", newline="\n") - - -def _embedded_accounting_ids(turn: TurnSpanDict) -> list[str]: - """Collect every accounting id a turn's own job carries, recursively. - - ``_export_turn_subtree`` delivers every one of these within the same - flush as the turn itself -- merged onto a matching chat span's usage - fields, or (if its call id doesn't match a chat span in this turn) as an - inline turn-owned accounting span -- so once that flush is confirmed, - every id collected here is confirmed delivered too, the same as one - delivered through the separate fallback accounting job path. - """ - ids: list[str] = [] - for accounting in turn.get("accounting_calls") or []: - accounting_id = accounting.get("accounting_id") - if isinstance(accounting_id, str) and accounting_id: - ids.append(accounting_id) - for subagent in turn.get("subagents") or []: - ids.extend(_embedded_accounting_ids(subagent)) - return ids - - -def _mark_turn_accounting_delivered(session_dir_: Path, turn: TurnSpanDict) -> None: - for accounting_id in _embedded_accounting_ids(turn): - _mark_accounting_sent(session_dir_, accounting_id) - - def _claim_turn_export(session_dir_: Path, turn_id: str) -> bool: """First-wins claim on exporting this turn's span tree, ever, for this session. A replayed/duplicate hook invocation for the same turn (e.g. the @@ -716,21 +617,16 @@ def export_turn( turn: TurnSpanDict, *, captured_env: dict[str, str] | None = None, -) -> bool: +) -> None: """Hand a completed turn off for background export. Never raises, never blocks on network I/O — the actual Logfire call happens in a detached child process this spawns and does not wait for. See module docstring. ``captured_env`` lets a caller whose own ``os.environ`` is unreliable supply the raw opted-in env dict; when omitted it is read here. - - Returns whether the local job was durably written and dispatched, same - contract as :func:`export_spans` — never whether Logfire ever received it. - No existing caller inspects the return value, so this is backward - compatible with every platform still calling this positionally. """ if not config.logfire.enabled or not config.logfire.token: - return False + return try: job_path = _write_job( config.root, @@ -745,7 +641,6 @@ def export_turn( }, ) _spawn(job_path) - return True except Exception as exc: log_capture_error( thirdeye_home=config.root, @@ -754,7 +649,6 @@ def export_turn( platform=platform, session_id=session_id, ) - return False def export_spans( @@ -1052,7 +946,6 @@ def _export_turn_inner( fsops.unlink(claim_path, missing_ok=True) raise claim_path.write_text("sent", encoding="utf-8", newline="\n") - _mark_turn_accounting_delivered(session_dir_, turn) def _export_subagent_turn_inner( @@ -1103,7 +996,6 @@ def _export_subagent_turn_inner( fsops.unlink(claim_path, missing_ok=True) raise claim_path.write_text("sent", encoding="utf-8", newline="\n") - _mark_turn_accounting_delivered(session_dir_, turn) def _require_export_instance(config: Config, platform: str): @@ -1193,7 +1085,6 @@ def _export_session_accounting_inner( ) if instance.force_flush(timeout_millis=_FLUSH_TIMEOUT_MS) is False: raise RuntimeError("session accounting export was not flushed") - _mark_accounting_sent(session_dir_, str(accounting["accounting_id"])) def _export_turn_accounting_inner( @@ -1235,7 +1126,6 @@ def _export_turn_accounting_inner( ) if instance.force_flush(timeout_millis=_FLUSH_TIMEOUT_MS) is False: raise RuntimeError("turn accounting export was not flushed") - _mark_accounting_sent(session_dir_, str(accounting["accounting_id"])) @lru_cache(maxsize=128) diff --git a/src/thirdeye/platforms/copilot/export.py b/src/thirdeye/platforms/copilot/export.py index 2aca7dc..a6bb645 100644 --- a/src/thirdeye/platforms/copilot/export.py +++ b/src/thirdeye/platforms/copilot/export.py @@ -4,10 +4,9 @@ generic transport's local jobs; the detached worker performs remote delivery. The split cannot provide transactional exactly-once delivery: a crash after a remote flush and before acknowledgement can retry a deterministic span. The -transport's own durable claims (`otel_export.turn_export_sent` / -`accounting_export_sent`) are what let this module tell "already confirmed -delivered" apart from "merely queued" across restarts, since the worker -deletes its own job file on success. +transport's durable turn claims and Copilot's accounting delivery claims are +what let this module tell "already confirmed delivered" apart from "merely +queued" across restarts, since workers delete their job files on success. """ from __future__ import annotations @@ -278,9 +277,7 @@ def _reflect_accounting_job_health( """ status = export_transport.status(config.root, span_id) if status is None: - if otel_export.accounting_export_sent( - _directory(config, stored_session_id), accounting_id - ): + if export_transport.delivery_sent(_directory(config, stored_session_id), accounting_id): status = {"state": "emitted", "attempt": None, "last_error": None} else: _clear_error_state(config, stored_session_id, accounting_id) @@ -391,7 +388,7 @@ def queue_exports( call_id = item.get("call_id") existing_entry = (state.get("placements") or {}).get(accounting_id) or {} turn_sent = otel_export.turn_export_sent(directory, root_id) - delivered = otel_export.accounting_export_sent(directory, accounting_id) or ( + delivered = export_transport.delivery_sent(directory, accounting_id) or ( turn_sent and existing_entry.get("destination") == "chat-span" ) # A chat span already flushed to Logfire is immutable history: usage @@ -467,7 +464,7 @@ def queue_exports( if usage is None: continue span_id = _span_id(stored_session_id, None, accounting_id) - delivered = otel_export.accounting_export_sent(directory, accounting_id) + delivered = export_transport.delivery_sent(directory, accounting_id) entry, accepted = _place( config, stored_session_id, @@ -510,8 +507,8 @@ def queue_exports( assembled = _with_deterministic_turn_ids( _turn_with_placed_accounting(turn, placements), stored_session_id ) - sent = otel_export.export_turn( - config, directory, stored_session_id, PLATFORM_NAME, meta.cwd, assembled + sent = export_transport.queue_turn( + config, directory, stored_session_id, meta.cwd, assembled ) if sent: queued += 1 diff --git a/src/thirdeye/platforms/copilot/export_state.py b/src/thirdeye/platforms/copilot/export_state.py index e2c5d42..202de37 100644 --- a/src/thirdeye/platforms/copilot/export_state.py +++ b/src/thirdeye/platforms/copilot/export_state.py @@ -8,9 +8,8 @@ lead to a deterministic retry and therefore a duplicate remote span. ``record_placement`` takes an explicit ``delivered`` flag from the caller -(who reads the generic transport's independent, durable -``otel_export.accounting_export_sent`` claim before calling in). That flag, -not merely a changed destination, is what turns a correction into a +(who reads Copilot's independent, durable accounting delivery claim before +calling in). That flag, not merely a changed destination, is what turns a correction into a quarantined conflict: a correction to a job that never left this machine is always safe to replace, while a correction after confirmed delivery can no longer relocate tokens the remote collector already has. @@ -206,9 +205,9 @@ def record_placement( ) -> tuple[dict[str, Any], dict[str, Any] | None, bool]: """Persist the first token location for an accounting identity. - ``delivered`` is the caller's fresh read of the generic transport's - durable delivery claim for this identity (see - ``otel_export.accounting_export_sent``), not this ledger's own possibly + ``delivered`` is the caller's fresh read of Copilot's durable delivery + claim for this identity (see ``export_transport.delivery_sent``), not + this ledger's own possibly stale ``emitted`` flag — the local job file backing that flag is deleted by the worker on success, so this ledger cannot detect delivery on its own and must be told. diff --git a/src/thirdeye/platforms/copilot/export_transport.py b/src/thirdeye/platforms/copilot/export_transport.py index c46b0c1..e5a3599 100644 --- a/src/thirdeye/platforms/copilot/export_transport.py +++ b/src/thirdeye/platforms/copilot/export_transport.py @@ -41,6 +41,24 @@ def claim_path(path: Path) -> Path: return path.with_suffix(f"{path.suffix}.claim") +def delivery_claim_path(session_dir: Path, accounting_id: str) -> Path: + digest = hashlib.sha256(accounting_id.encode("utf-8")).hexdigest() + return session_dir / "copilot-accounting-sent" / f"{digest}.json" + + +def delivery_sent(session_dir: Path, accounting_id: str) -> bool: + try: + return delivery_claim_path(session_dir, accounting_id).read_text(encoding="utf-8") == "sent" + except OSError: + return False + + +def _mark_delivered(session_dir: Path, accounting_id: str) -> None: + path = delivery_claim_path(session_dir, accounting_id) + if _atomic_create(path, "sent"): + fsops.sync_directory(path.parent) + + def _atomic_create(path: Path, text: str) -> bool: path.parent.mkdir(parents=True, exist_ok=True) try: @@ -147,6 +165,42 @@ def _spawn(path: Path) -> None: proc.spawn_detached([sys.executable, "-m", "thirdeye.platforms.copilot.export_transport", str(path)]) +def queue_turn( + config: Config, + session_dir: Path, + session_id: str, + cwd: str, + turn: dict[str, Any], +) -> bool: + """Queue a Copilot turn while preserving the generic worker job shape.""" + if not config.logfire.enabled or not config.logfire.token: + return False + try: + path = otel_export._write_job( + config.root, + { + "kind": "turn", + "captured_attributes": otel_export._resolve_captured_attributes(config, None), + "session_dir": str(session_dir), + "session_id": session_id, + "platform": "copilot", + "cwd": cwd, + "turn": turn, + }, + ) + otel_export._spawn(path) + return True + except Exception as exc: + log_capture_error( + thirdeye_home=config.root, + phase="copilot_turn_export_spawn", + error=exc, + platform="copilot", + session_id=session_id, + ) + return False + + def _placement_is_current(config: Config, payload: dict[str, Any]) -> bool: from .export_state import load_export_state @@ -175,7 +229,7 @@ def _queue(config: Config, payload: dict[str, Any]) -> bool: try: if not _placement_is_current(config, payload): return True - delivered = otel_export.accounting_export_sent( + delivered = delivery_sent( Path(str(payload["session_dir"])), str(payload["accounting_id"]) ) if not delivered: @@ -266,6 +320,10 @@ def _acquire(path: Path) -> dict[str, Any] | None: if payload is None or payload.get("state") in {"failed", "emitted"}: fsops.unlink(owner, missing_ok=True) return None + if delivery_sent(Path(str(payload["session_dir"])), str(payload["accounting_id"])): + fsops.unlink(path, missing_ok=True) + fsops.unlink(owner, missing_ok=True) + return None claimed = {**payload, "state": "claimed"} _write_job(path, claimed) return claimed @@ -352,6 +410,9 @@ def run(path: Path) -> None: ) else: try: + _mark_delivered( + Path(str(claimed["session_dir"])), str(claimed["accounting_id"]) + ) _write_job(path, {**claimed, "state": "emitted", "last_error": None}) fsops.unlink(path, missing_ok=True) except Exception as exc: diff --git a/tests/platforms/copilot/test_export.py b/tests/platforms/copilot/test_export.py index f974ff0..c86dc7f 100644 --- a/tests/platforms/copilot/test_export.py +++ b/tests/platforms/copilot/test_export.py @@ -218,7 +218,7 @@ def export_calls(monkeypatch: pytest.MonkeyPatch) -> dict[str, list[Any]]: } def _turn(*args: Any, **kwargs: Any) -> bool: - calls["turn"].append(args[5]) + calls["turn"].append(args[4]) return True def _turn_accounting(*args: Any, **kwargs: Any) -> bool: @@ -229,7 +229,7 @@ def _session_accounting(*args: Any, **kwargs: Any) -> bool: calls["session_accounting"].append(args[4]) return True - monkeypatch.setattr(otel_export, "export_turn", _turn) + monkeypatch.setattr(export_transport, "queue_turn", _turn) monkeypatch.setattr(export_transport, "queue_turn_accounting", _turn_accounting) monkeypatch.setattr(export_transport, "queue_session_accounting", _session_accounting) return calls @@ -776,7 +776,7 @@ def test_fallback_placement_prevents_later_chat_relocation_after_confirmed_deliv queue_exports(enabled_config, stored, ambiguous_projection, include_history=True) directory = _directory(enabled_config, stored) - otel_export._mark_accounting_sent(directory, ACCOUNTING_UNMATCHED) + export_transport._mark_delivered(directory, ACCOUNTING_UNMATCHED) export_calls["turn"].clear() export_calls["turn_accounting"].clear() @@ -938,7 +938,7 @@ def test_accounting_job_failure_records_error( monkeypatch: pytest.MonkeyPatch, ) -> None: stored = _seed_session(enabled_config, paths) - monkeypatch.setattr(otel_export, "export_turn", lambda *args, **kwargs: True) + monkeypatch.setattr(export_transport, "queue_turn", lambda *args, **kwargs: True) monkeypatch.setattr( export_transport, "queue_turn_accounting", lambda *args, **kwargs: False ) @@ -981,11 +981,10 @@ def test_turn_job_failure_is_not_counted_and_is_recorded( paths: SourcePaths, monkeypatch: pytest.MonkeyPatch, ) -> None: - """`otel_export.export_turn` now reports durable queue acceptance, - same contract as `export_spans`; a spawn/write failure there must not - be silently counted as a successful queue.""" + """A Copilot turn spawn/write failure must not be counted as a + successful queue.""" stored = _seed_session(enabled_config, paths) - monkeypatch.setattr(otel_export, "export_turn", lambda *args, **kwargs: False) + monkeypatch.setattr(export_transport, "queue_turn", lambda *args, **kwargs: False) projection = _projection(turns=[_main_turn()]) queued = queue_exports(enabled_config, stored, projection, include_history=True) @@ -1002,7 +1001,7 @@ def test_errors_clear_once_a_retry_succeeds( """A stale `last_error`/turn error from an earlier failed attempt must not linger once a later reconciliation successfully queues the job.""" stored = _seed_session(enabled_config, paths) - monkeypatch.setattr(otel_export, "export_turn", lambda *args, **kwargs: False) + monkeypatch.setattr(export_transport, "queue_turn", lambda *args, **kwargs: False) monkeypatch.setattr( export_transport, "queue_turn_accounting", lambda *args, **kwargs: False ) @@ -1034,7 +1033,7 @@ def test_errors_clear_once_a_retry_succeeds( "accounting job was not queued" ) - monkeypatch.setattr(otel_export, "export_turn", lambda *args, **kwargs: True) + monkeypatch.setattr(export_transport, "queue_turn", lambda *args, **kwargs: True) monkeypatch.setattr( export_transport, "queue_turn_accounting", lambda *args, **kwargs: True ) @@ -1591,7 +1590,7 @@ def test_confirmed_accounting_delivery_survives_restart_without_double_emission( queue_exports(enabled_config, stored, projection, include_history=True) directory = _directory(enabled_config, stored) - assert otel_export.accounting_export_sent(directory, ACCOUNTING_UNMATCHED) is True + assert export_transport.delivery_sent(directory, ACCOUNTING_UNMATCHED) is True first_accounting_spans = [ span for span in exporter.exported_spans_as_dict() if span["name"] == "accounting" ] @@ -1622,13 +1621,9 @@ def test_chat_embedded_accounting_survives_restart_without_relocation_or_duplica ) -> None: """Matched usage placed on the chat span is delivered as part of the turn's own job -- there is no separate fallback accounting job for - it. The durable transport claim must still record that delivery - (``otel_export.accounting_export_sent``), or a later reconciliation - would see the turn as already sent (so the chat span is no longer - available) but the accounting as never delivered, and would - "recover" by relocating it to a brand-new turn-accounting fallback - span -- duplicating the tokens that were already flushed on the chat - span the first time.""" + it. Copilot reconciles the durable turn claim with its chat placement; + otherwise a later pass could relocate the accounting to a fallback + span and duplicate tokens already flushed on the chat span.""" stored = _seed_session(enabled_config, paths) projection = _projection( turns=[_main_turn(accounting_calls=[_accounting_call()])], @@ -1639,7 +1634,6 @@ def test_chat_embedded_accounting_survives_restart_without_relocation_or_duplica queue_exports(enabled_config, stored, projection, include_history=True) directory = _directory(enabled_config, stored) assert otel_export.turn_export_sent(directory, TURN_ONE) is True - assert otel_export.accounting_export_sent(directory, ACCOUNTING_MATCHED) is True chat_spans = [ span for span in exporter.exported_spans_as_dict() if span["name"].startswith("chat") ] diff --git a/tests/platforms/copilot/test_export_transport.py b/tests/platforms/copilot/test_export_transport.py index 76fcaa0..3be7a2a 100644 --- a/tests/platforms/copilot/test_export_transport.py +++ b/tests/platforms/copilot/test_export_transport.py @@ -7,7 +7,6 @@ import pytest -from thirdeye import otel_export from thirdeye.config import Config, LogfireSettings from thirdeye.platforms.copilot import export_transport from thirdeye.platforms.copilot.export_state import record_placement, update_export_state @@ -88,7 +87,7 @@ def test_queue_does_not_recreate_confirmed_delivery( spawned: list[Path] = [] monkeypatch.setattr(export_transport, "_spawn", spawned.append) session_dir = config.root / "traces" / "copilot" / "stored-session" - otel_export._mark_accounting_sent(session_dir, "acct-1") + export_transport._mark_delivered(session_dir, "acct-1") accounting = { "accounting_id": "acct-1", "usage": {}, From 3ce259e90b64569dd80c5e02588d5bd06ae5635e Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Sun, 13 Sep 2026 10:35:42 -0700 Subject: [PATCH 79/88] Restore projection recovery commit order --- .../platforms/copilot/projection_store.py | 22 ++++++++++---- tests/platforms/copilot/test_reconcile.py | 30 +++++++++++++++---- 2 files changed, 40 insertions(+), 12 deletions(-) diff --git a/src/thirdeye/platforms/copilot/projection_store.py b/src/thirdeye/platforms/copilot/projection_store.py index cf85a41..0c3214c 100644 --- a/src/thirdeye/platforms/copilot/projection_store.py +++ b/src/thirdeye/platforms/copilot/projection_store.py @@ -417,11 +417,21 @@ def commit_projection( document) leaves the previous projection completely untouched rather than losing it to a non-atomic delete-then-commit sequence. - The usage JSONL and ``UsageIndex`` are derived mirrors, but publishing - them can fail. They are therefore updated before the projection document: - a sidecar failure leaves the last readable projection untouched. The - projection document remains the commit point and is published with - write-ahead journaling (see ``publish_projection_document``). + Durability past that validation point has two distinct failure modes. + The projection document is the source of truth and is published with + write-ahead journaling (see ``publish_projection_document``): once that + call returns, the new document is durably committed, full stop. The + usage sidecar published immediately after is a derived, self-healing + mirror of the document's own usage index -- kept as a separate JSONL + file only so ``UsageIndex`` can query it without parsing the whole + document. If that second, mirror-only publish raises (a disk error, not + a validation error), this function still raises so the caller learns of + it, but the already-committed document is *not* rolled back: it reflects + the new projection, and the next ``load_projection_state`` call + republishes a sidecar that matches it. A caller must not assume a raise + from this function always means "nothing changed" -- check which phase + failed via ``read_projection_status``/``load_projection_state`` if that + distinction matters. ``base_commit_sequence``, when given, must equal the ``commit_sequence`` a caller observed from an earlier ``load_projection_state`` call. A @@ -518,8 +528,8 @@ def commit_projection( "state": state, "indexes": merged_indexes, } - _publish_usage(config, stored_session_id, directory, merged_indexes["usage"]) publish_projection_document(directory, next_document) + _publish_usage(config, stored_session_id, directory, merged_indexes["usage"]) return counts diff --git a/tests/platforms/copilot/test_reconcile.py b/tests/platforms/copilot/test_reconcile.py index e2bc8a6..bf3edd9 100644 --- a/tests/platforms/copilot/test_reconcile.py +++ b/tests/platforms/copilot/test_reconcile.py @@ -637,25 +637,36 @@ def racing_commit( assert read_projected_turns(config, stored) == baseline_turns -def test_usage_sidecar_publish_failure_preserves_prior_projection( +def test_usage_sidecar_publish_failure_reports_the_committed_document( config: Config, paths: SourcePaths, cli_transcript_records: list[SourceRecord], monkeypatch: pytest.MonkeyPatch, ) -> None: - """A usage-sidecar I/O failure must not replace the readable projection.""" + """A post-validation I/O failure while publishing the usage-sidecar + + mirror is a distinct failure mode from a validation failure: by the time + it can happen, the projection document has already durably published + (see commit_projection's docstring), so the counts reconcile_archive + reports must reflect that new document -- not the prior one -- with the + error counted on top. The next reconcile call must self-heal the + sidecar without reprocessing anything new. + """ stored = _seed_full_corpus(config, paths, cli_transcript_records) baseline = reconcile_archive(config, stored) prior_turns = read_projected_turns(config, stored) - prior_document = _document(config, stored) + baseline_sequence = load_projection_state(config, stored)["commit_sequence"] original_publish_usage = projection_store._publish_usage def selective_boom( cfg: Config, sid: str, directory: Path, usage_index: dict[str, Any], *, force: bool = False ) -> None: - # Let readers inspect the prior projection normally. Only the commit's - # attempt to publish the new sidecar fails. + # load_projection_state's self-heal always passes force=True; only + # commit_projection's own (non-forced) publish should fail here, so + # this reaches the specific "document committed, sidecar mirror + # failed" state the docstring describes rather than failing before + # commit_projection is ever entered. if force: original_publish_usage(cfg, sid, directory, usage_index, force=force) return @@ -670,7 +681,14 @@ def selective_boom( assert result["errors"] == baseline["errors"] + 1 assert result["turns"] == baseline["turns"] assert result["usage"] == baseline["usage"] - assert _document(config, stored) == prior_document + # The document committed despite the sidecar failure -- proven by the + # storage-owned commit_sequence advancing even though this call reported + # an error -- so turns already reflect the (content-equivalent, since + # this rebuilds the same archive) new projection rather than being stuck + # on the old one. Read the raw document rather than load_projection_state + # here: that call would itself retry the still-patched, still-failing + # sidecar publish as part of its own self-heal. + assert _document(config, stored)["state"]["commit_sequence"] == baseline_sequence + 1 assert read_projected_turns(config, stored) == prior_turns monkeypatch.undo() From 11364ed46b50dcccb222d8f9b30413e8ed66bf39 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Sun, 13 Sep 2026 10:44:48 -0700 Subject: [PATCH 80/88] Integrate Copilot V2 runtime reconciliation --- src/thirdeye/commands/copilot.py | 77 ++++++++++++++++++- src/thirdeye/platforms/copilot/followup.py | 16 ++++ src/thirdeye/platforms/copilot/reconcile.py | 17 ++++- src/thirdeye/platforms/copilot/runtime.py | 82 +++++++++++++++++++++ src/thirdeye/platforms/copilot/status.py | 62 +++++++++++++--- src/thirdeye/platforms/copilot/watch.py | 10 ++- 6 files changed, 248 insertions(+), 16 deletions(-) create mode 100644 src/thirdeye/platforms/copilot/runtime.py diff --git a/src/thirdeye/commands/copilot.py b/src/thirdeye/commands/copilot.py index 14030f9..8b57bf4 100644 --- a/src/thirdeye/commands/copilot.py +++ b/src/thirdeye/commands/copilot.py @@ -15,6 +15,8 @@ from thirdeye.platforms.copilot.capture import sync as capture_sync from thirdeye.platforms.copilot.constants import COPILOT_HOME_ENV from thirdeye.platforms.copilot.identity import resolve_sources, validate_native_id +from thirdeye.platforms.copilot.reconcile import reconcile_archive +from thirdeye.platforms.copilot.runtime import reconcile_archived_sessions from thirdeye.platforms.copilot.status import capture_status from thirdeye.platforms.copilot.types import SourcePaths, SyncResult from thirdeye.platforms.copilot.watch import watch as watch_loop @@ -153,6 +155,20 @@ def _print_sync_result(result: SyncResult) -> None: click.echo(_counts_line(result)) +def _print_reconcile_result(result: dict[str, int]) -> None: + click.echo( + "reconcile " + f"events={result.get('events', 0)} " + f"usage={result.get('usage', 0)} " + f"turns={result.get('turns', 0)} " + f"exports_queued={result.get('exports', 0)} " + f"pending={result.get('pending', 0)} " + f"ambiguous={result.get('ambiguous', 0)} " + f"conflicting={result.get('conflicting', 0)} " + f"errors={result.get('errors', 0)}" + ) + + def _print_sync_followup(config: Config, paths: SourcePaths, result: SyncResult) -> None: if result["errors"] == 0 and result["pending"] == 0: return @@ -194,6 +210,25 @@ def _print_status(status: dict[str, Any]) -> None: f"leases={pending.get('leases', 0)} " f"journals={pending.get('journals', 0)}" ) + projections = [ + item.get("projection") for item in status.get("sessions") or [] if isinstance(item, dict) + ] + exports = [ + item.get("export") for item in status.get("sessions") or [] if isinstance(item, dict) + ] + click.echo( + "Projection: " + f"pending={sum(item.get('pending', 0) for item in projections if isinstance(item, dict))} " + f"ambiguous={sum(item.get('ambiguous', 0) for item in projections if isinstance(item, dict))} " + f"conflicting={sum(item.get('conflicting', 0) for item in projections if isinstance(item, dict))}" + ) + click.echo( + "Export: " + f"activated={sum(1 for item in exports if isinstance(item, dict) and item.get('activated'))} " + f"queued={sum(item.get('queued', 0) for item in exports if isinstance(item, dict))} " + f"delivered={sum(item.get('delivered', 0) for item in exports if isinstance(item, dict))} " + f"errors={sum(item.get('errors', 0) for item in exports if isinstance(item, dict))}" + ) errors = status.get("errors") or [] informational = [error for error in errors if not _status_error_affects_exit(error)] blocking = [error for error in errors if _status_error_affects_exit(error)] @@ -220,8 +255,15 @@ def copilot_group() -> None: @copilot_group.command("sync", help="One-shot local ingestion of Copilot CLI recordings.") @click.option("--session-id", default=None, help="Exact native Copilot session ID.") +@click.option( + "--export", + "export_history", + is_flag=True, + default=False, + help="Queue completed archived history for export after local reconciliation.", +) @_source_home_option -def sync_cmd(session_id: str | None, source_home: Path | None) -> None: +def sync_cmd(session_id: str | None, export_history: bool, source_home: Path | None) -> None: config = Config.load() paths = _resolve_paths(source_home) try: @@ -231,6 +273,15 @@ def sync_cmd(session_id: str | None, source_home: Path | None) -> None: except ValueError as exc: raise click.ClickException(str(exc)) from exc _print_sync_result(result) + # Every sync refreshes only local V2 projections. Explicit opt-in is the + # only sync path that can include completed history in export eligibility. + if export_history: + for derived in reconcile_archived_sessions( + config, paths, export=True, include_history=True + ).values(): + _print_reconcile_result(derived) + else: + reconcile_archived_sessions(config, paths) _print_sync_followup(config, paths, result) if session_id is not None and result["sessions"] == 0: raise click.ClickException( @@ -238,6 +289,30 @@ def sync_cmd(session_id: str | None, source_home: Path | None) -> None: ) +@copilot_group.command("reconcile", help="Build Copilot V2 projections from retained local archives.") +@click.option("--session-id", required=True, help="Exact stored Thirdeye Copilot session ID.") +@click.option("--rebuild", is_flag=True, default=False, help="Rebuild derived state only.") +@click.option( + "--export", + "export_history", + is_flag=True, + default=False, + help="Queue completed archived history for export after local reconciliation.", +) +def reconcile_cmd(session_id: str, rebuild: bool, export_history: bool) -> None: + config = Config.load() + result = reconcile_archive( + config, + session_id, + rebuild=rebuild, + export=export_history, + include_history=export_history, + ) + _print_reconcile_result(result) + if result.get("errors", 0): + raise click.ClickException("Copilot reconciliation completed with errors") + + @copilot_group.command("watch", help="Poll local Copilot CLI recordings until interrupted.") @_source_home_option @click.option( diff --git a/src/thirdeye/platforms/copilot/followup.py b/src/thirdeye/platforms/copilot/followup.py index 996e81a..f1e05e4 100644 --- a/src/thirdeye/platforms/copilot/followup.py +++ b/src/thirdeye/platforms/copilot/followup.py @@ -221,6 +221,22 @@ def _run(config: Config, paths: SourcePaths, native_id: str, generation: str) -> silent_fallback=True, ) else: + if result is not None: + # Derived work is coalesced here, never in the hook + # process. A projection problem cannot affect capture. + try: + from thirdeye.platforms.copilot.runtime import reconcile_session + + reconcile_session(config, paths, native_id, export=True) + except Exception as exc: + log_capture_error( + thirdeye_home=config.root, + phase="copilot_followup_reconcile", + error=exc, + platform=_PLATFORM, + session_id=native_id, + silent_fallback=True, + ) if result is not None and result["errors"] == 0 and result["pending"] == 0: return remaining = deadline - time.monotonic() diff --git a/src/thirdeye/platforms/copilot/reconcile.py b/src/thirdeye/platforms/copilot/reconcile.py index 004c87f..d919c20 100644 --- a/src/thirdeye/platforms/copilot/reconcile.py +++ b/src/thirdeye/platforms/copilot/reconcile.py @@ -62,7 +62,13 @@ def _diagnostic_errors(projection: Projection) -> int: ) -def queue_exports(config: Config, stored_session_id: str, projection: Projection) -> int: +def queue_exports( + config: Config, + stored_session_id: str, + projection: Projection, + *, + include_history: bool = False, +) -> int: """Queue export work only when an explicit caller requests it. Export assembly is intentionally not an import-time dependency of local @@ -74,8 +80,8 @@ def queue_exports(config: Config, stored_session_id: str, projection: Projection enqueue = module.queue_exports if not callable(enqueue): raise TypeError("Copilot export assembly does not provide queue_exports") - exporter: Callable[[Config, str, Projection], int] = enqueue - return exporter(config, stored_session_id, projection) + exporter: Callable[..., int] = enqueue + return exporter(config, stored_session_id, projection, include_history=include_history) def reconcile_archive( @@ -84,6 +90,7 @@ def reconcile_archive( *, rebuild: bool = False, export: bool = False, + include_history: bool = False, ) -> dict[str, int]: """Rebuild local Copilot projections from the immutable V1 archive. @@ -149,7 +156,9 @@ def reconcile_archive( return result try: - result["exports"] = queue_exports(config, stored_session_id, projection) + result["exports"] = queue_exports( + config, stored_session_id, projection, include_history=include_history + ) except Exception: # Export delivery must never roll back a successful local projection. result["errors"] += 1 diff --git a/src/thirdeye/platforms/copilot/runtime.py b/src/thirdeye/platforms/copilot/runtime.py new file mode 100644 index 0000000..0aa9ebc --- /dev/null +++ b/src/thirdeye/platforms/copilot/runtime.py @@ -0,0 +1,82 @@ +"""Runtime composition for archive reconciliation after V1 capture. + +Capture remains the durable, local V1 boundary while reconciliation and +export eligibility are V2 derived work. A projection failure must never turn +a successful hook receipt or source import into a failed capture. +""" + +from __future__ import annotations + +from thirdeye.config import Config +from thirdeye.paths import platform_dir + +from .constants import PLATFORM_NAME +from .identity import stored_session_id +from .reconcile import reconcile_archive +from .state import read_json, state_path +from .types import SourcePaths + + +def exports_configured(config: Config) -> bool: + """Whether runtime activity may queue detached live export work.""" + return bool(config.logfire.enabled and config.logfire.token) + + +def archived_session_ids(config: Config, paths: SourcePaths) -> list[str]: + """Return this source home's retained V1 sessions without reading sources.""" + root = platform_dir(config.root, PLATFORM_NAME) + prefix = f"copilot-{paths['source_key'][:16]}-" + try: + directories = sorted(entry for entry in root.iterdir() if entry.is_dir()) + except OSError: + return [] + result: list[str] = [] + for directory in directories: + if not directory.name.startswith(prefix): + continue + try: + state = read_json(state_path(directory)) or {} + except ValueError: + continue + if state.get("source_key") == paths["source_key"]: + result.append(directory.name) + return result + + +def reconcile_session( + config: Config, + paths: SourcePaths, + native_session_id: str, + *, + export: bool = False, + include_history: bool = False, +) -> dict[str, int]: + """Derive one retained session, optionally queueing detached live export.""" + return reconcile_archive( + config, + stored_session_id(paths, native_session_id), + export=export and exports_configured(config), + include_history=include_history, + ) + + +def reconcile_archived_sessions( + config: Config, + paths: SourcePaths, + *, + export: bool = False, + include_history: bool = False, +) -> dict[str, dict[str, int]]: + """Replay retained sessions even after their live sources disappear.""" + return { + session_id: reconcile_archive( + config, + session_id, + export=export and exports_configured(config), + include_history=include_history, + ) + for session_id in archived_session_ids(config, paths) + } + + +__all__ = ["archived_session_ids", "exports_configured", "reconcile_archived_sessions", "reconcile_session"] diff --git a/src/thirdeye/platforms/copilot/status.py b/src/thirdeye/platforms/copilot/status.py index b81c7d2..960636a 100644 --- a/src/thirdeye/platforms/copilot/status.py +++ b/src/thirdeye/platforms/copilot/status.py @@ -14,7 +14,9 @@ from .archive import _record_from_event from .constants import FOLLOWUP_LEASE_FILENAME, PLATFORM_NAME from .database import read_database +from .export_state import load_export_state from .install import CopilotPlatform +from .projection_store import read_projection_status from .spool import read_spool from .state import journal_path, read_json, state_path from .types import SourcePaths, SourceRecord @@ -266,16 +268,58 @@ def _archive_status( if _followup_lease_pending(directory): pending_followup += 1 active_leases += 1 - sessions.append( - { - "stored_session_id": directory.name, - "native_session_id": state.get("native_session_id"), - "cursor": state.get("cursor", {}), - "last_successful_import": health.get("last_successful_import"), - "diagnostics": diagnostics, - "journal_pending": journal_path(directory).is_file(), + session = { + "stored_session_id": directory.name, + "native_session_id": state.get("native_session_id"), + "cursor": state.get("cursor", {}), + "last_successful_import": health.get("last_successful_import"), + "diagnostics": diagnostics, + "journal_pending": journal_path(directory).is_file(), + } + try: + session["projection"] = read_projection_status(config, directory.name) + except Exception as error: + session["projection"] = {"errors": 1} + errors.append( + { + "kind": "copilot_projection_unreadable", + "session": directory.name, + "reason": type(error).__name__, + } + ) + try: + ledger = load_export_state(config, directory.name) + placements = ledger.get("placements") if isinstance(ledger.get("placements"), dict) else {} + session["export"] = { + "activated": bool(ledger.get("activated")), + "queued": sum( + 1 + for item in placements.values() + if isinstance(item, dict) and not item.get("emitted") + ), + "delivered": sum( + 1 + for item in placements.values() + if isinstance(item, dict) and item.get("emitted") + ), + "errors": sum( + 1 + for item in placements.values() + if isinstance(item, dict) and item.get("last_error") + ) + + len(ledger.get("conflicts") if isinstance(ledger.get("conflicts"), dict) else {}) + + len(ledger.get("turn_errors") if isinstance(ledger.get("turn_errors"), dict) else {}), } - ) + except Exception as error: + session["export"] = {"activated": False, "queued": 0, "delivered": 0, "errors": 1} + errors.append( + { + "kind": "copilot_export_ledger_unreadable", + "session": directory.name, + "reason": type(error).__name__, + } + ) + sessions.append(session) for diagnostic in diagnostics: if isinstance(diagnostic, dict): errors.append({"session": directory.name, **diagnostic}) diff --git a/src/thirdeye/platforms/copilot/watch.py b/src/thirdeye/platforms/copilot/watch.py index 81b39de..6b168f0 100644 --- a/src/thirdeye/platforms/copilot/watch.py +++ b/src/thirdeye/platforms/copilot/watch.py @@ -18,6 +18,7 @@ from .capture import sync from .database import discover_database_sessions from .identity import validate_native_id +from .runtime import reconcile_archived_sessions, reconcile_session from .sources import discover_sessions from .types import SourcePaths @@ -143,8 +144,9 @@ def watch(config: Config, paths: SourcePaths, *, interval: float = 1.0) -> None: The initial sync drains the bounded source snapshot. Later cycles invoke per-session sync only after a transcript, database/WAL, or spool position changes (or after a retryable result), avoiding repeated parsing of quiet - completed sessions. All capture remains local; this function never - exports data and never starts a background service. + completed sessions. Capture and derived replay remain local. When live + export is configured, reconciliation may queue detached jobs; this + foreground loop never delivers remotely itself. """ if not isinstance(interval, (int, float)) or isinstance(interval, bool): @@ -158,6 +160,9 @@ def watch(config: Config, paths: SourcePaths, *, interval: float = 1.0) -> None: # on the first poll ensures that append receives a later capture. previous = _source_snapshot(config, paths) initial = sync(config, paths) + # Activation replays retained V1 evidence, including sessions whose + # transcript/database was removed after capture. + reconcile_archived_sessions(config, paths, export=True) retry = set(previous["sessions"]) if _result_needs_retry(initial, present=True) else set() while True: _SLEEP(float(interval)) @@ -168,6 +173,7 @@ def watch(config: Config, paths: SourcePaths, *, interval: float = 1.0) -> None: # KeyboardInterrupt is intentionally checked between sessions; # a current archive commit remains crash-recoverable. result = sync(config, paths, session_id=native_id) + reconcile_session(config, paths, native_id, export=True) if _result_needs_retry(result, present=native_id in current["sessions"]): retry.add(native_id) previous = current From 55d510ad1f242946dcd018fcc4300b9d86fa9f5b Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Sun, 13 Sep 2026 10:46:13 -0700 Subject: [PATCH 81/88] Add runtime integration tests for Copilot reconcile wiring. Cover reconcile CLI, sync export opt-in, watch/followup coalesced reconciliation, status projection/export summaries, and archive replay without live sources. Co-authored-by: Cursor --- tests/platforms/copilot/test_command.py | 153 ++++++++++- tests/platforms/copilot/test_runtime.py | 350 ++++++++++++++++++++++++ tests/platforms/copilot/test_watch.py | 17 ++ 3 files changed, 512 insertions(+), 8 deletions(-) create mode 100644 tests/platforms/copilot/test_runtime.py diff --git a/tests/platforms/copilot/test_command.py b/tests/platforms/copilot/test_command.py index 0a4b756..c3db31a 100644 --- a/tests/platforms/copilot/test_command.py +++ b/tests/platforms/copilot/test_command.py @@ -228,7 +228,7 @@ def test_copilot_group_appears_in_main_help() -> None: def test_copilot_help_lists_subcommands() -> None: result = CliRunner().invoke(main, ["copilot", "--help"]) assert result.exit_code == 0, result.output - for subcommand in ("sync", "watch", "status"): + for subcommand in ("sync", "reconcile", "watch", "status"): assert subcommand in result.output @@ -237,6 +237,15 @@ def test_sync_help_documents_flags() -> None: assert result.exit_code == 0, result.output assert "--session-id" in result.output assert "--source-home" in result.output + assert "--export" in result.output + + +def test_reconcile_help_documents_flags() -> None: + result = CliRunner().invoke(main, ["copilot", "reconcile", "--help"]) + assert result.exit_code == 0, result.output + assert "--session-id" in result.output + assert "--rebuild" in result.output + assert "--export" in result.output def test_watch_help_documents_interval_and_source_home() -> None: @@ -252,14 +261,9 @@ def test_status_help_documents_source_home() -> None: assert "--source-home" in result.output -def test_copilot_commands_have_no_export_flag() -> None: +def test_watch_and_status_help_have_no_export_flag() -> None: runner = CliRunner() - for args in ( - ["copilot", "--help"], - ["copilot", "sync", "--help"], - ["copilot", "watch", "--help"], - ["copilot", "status", "--help"], - ): + for args in (["copilot", "watch", "--help"], ["copilot", "status", "--help"]): result = runner.invoke(main, args) assert result.exit_code == 0, result.output assert "--export" not in result.output @@ -288,6 +292,7 @@ def test_copilot_hook_entrypoint_is_importable() -> None: def test_sync_invokes_capture_sync(isolated_home: Path, monkeypatch: pytest.MonkeyPatch) -> None: calls: list[tuple[Any, ...]] = [] + reconcile_calls: list[tuple[bool, bool]] = [] def fake_sync( config: Config, paths: SourcePaths, *, session_id: str | None = None @@ -295,11 +300,26 @@ def fake_sync( calls.append((config.root, paths["home"], session_id)) return _empty_result(sessions=2, records_written=5) + def fake_reconcile_archived( + _config: Config, + _paths: SourcePaths, + *, + export: bool = False, + include_history: bool = False, + ) -> dict[str, dict[str, int]]: + reconcile_calls.append((export, include_history)) + return {} + monkeypatch.setattr("thirdeye.commands.copilot.capture_sync", fake_sync) + monkeypatch.setattr( + "thirdeye.commands.copilot.reconcile_archived_sessions", + fake_reconcile_archived, + ) result = CliRunner().invoke(main, ["copilot", "sync"]) assert result.exit_code == 0, result.output assert len(calls) == 1 assert calls[0][2] is None + assert reconcile_calls == [(False, False)] assert _counts_line(sessions=2, records_written=5) in result.output @@ -618,6 +638,123 @@ def test_sync_path_escape_is_click_error(isolated_home: Path, tmp_path: Path) -> assert str(home) in result.output +def test_sync_export_requests_history_reconciliation( + isolated_home: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + reconcile_calls: list[tuple[bool, bool]] = [] + + def fake_sync( + _config: Config, _paths: SourcePaths, *, session_id: str | None = None + ) -> SyncResult: + return _empty_result(sessions=1) + + def fake_reconcile_archived( + _config: Config, + _paths: SourcePaths, + *, + export: bool = False, + include_history: bool = False, + ) -> dict[str, dict[str, int]]: + reconcile_calls.append((export, include_history)) + return {"stored-1": {"events": 3, "usage": 2, "turns": 1, "exports": 0, "pending": 0, "ambiguous": 0, "conflicting": 0, "errors": 0}} + + monkeypatch.setattr("thirdeye.commands.copilot.capture_sync", fake_sync) + monkeypatch.setattr( + "thirdeye.commands.copilot.reconcile_archived_sessions", + fake_reconcile_archived, + ) + result = CliRunner().invoke(main, ["copilot", "sync", "--export"]) + assert result.exit_code == 0, result.output + assert reconcile_calls == [(True, True)] + assert "reconcile events=3 usage=2 turns=1" in result.output + + +def test_reconcile_command_invokes_reconcile_archive( + isolated_home: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[tuple[str, bool, bool, bool]] = [] + + def fake_reconcile_archive( + _config: Config, + stored_session_id: str, + *, + rebuild: bool = False, + export: bool = False, + include_history: bool = False, + ) -> dict[str, int]: + calls.append((stored_session_id, rebuild, export, include_history)) + return { + "events": 4, + "usage": 6, + "turns": 2, + "exports": 0, + "pending": 1, + "ambiguous": 0, + "conflicting": 0, + "errors": 0, + } + + monkeypatch.setattr("thirdeye.commands.copilot.reconcile_archive", fake_reconcile_archive) + result = CliRunner().invoke( + main, + ["copilot", "reconcile", "--session-id", "copilot-abc-stored", "--rebuild", "--export"], + ) + assert result.exit_code == 0, result.output + assert calls == [("copilot-abc-stored", True, True, True)] + assert "reconcile events=4 usage=6 turns=2 exports_queued=0 pending=1" in result.output + + +def test_reconcile_command_exits_nonzero_on_errors( + isolated_home: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + "thirdeye.commands.copilot.reconcile_archive", + lambda *_args, **_kwargs: { + "events": 0, + "usage": 0, + "turns": 0, + "exports": 0, + "pending": 0, + "ambiguous": 0, + "conflicting": 0, + "errors": 1, + }, + ) + result = CliRunner().invoke( + main, + ["copilot", "reconcile", "--session-id", "copilot-abc-stored"], + ) + assert result.exit_code != 0, result.output + assert "completed with errors" in result.output + + +def test_status_prints_projection_and_export_summaries( + isolated_home: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + status = _minimal_status(configured=True) + status["sessions"] = [ + { + "stored_session_id": "stored-1", + "projection": {"pending": 2, "ambiguous": 1, "conflicting": 0}, + "export": {"activated": True, "queued": 3, "delivered": 1, "errors": 0}, + }, + { + "stored_session_id": "stored-2", + "projection": {"pending": 0, "ambiguous": 0, "conflicting": 1}, + "export": {"activated": False, "queued": 0, "delivered": 0, "errors": 2}, + }, + ] + monkeypatch.setattr("thirdeye.commands.copilot.capture_status", lambda _config, _paths: status) + result = CliRunner().invoke(main, ["copilot", "status"]) + assert result.exit_code == 0, result.output + assert "Projection: pending=2 ambiguous=1 conflicting=1" in result.output + assert "Export: activated=1 queued=3 delivered=1 errors=2" in result.output + + # -- watch --------------------------------------------------------------------- diff --git a/tests/platforms/copilot/test_runtime.py b/tests/platforms/copilot/test_runtime.py new file mode 100644 index 0000000..2ce52d8 --- /dev/null +++ b/tests/platforms/copilot/test_runtime.py @@ -0,0 +1,350 @@ +"""Integration tests for Copilot runtime reconciliation wiring.""" + +from __future__ import annotations + +import json +import shutil +from pathlib import Path +from typing import Any + +import pytest + +from thirdeye.config import Config, LogfireSettings +from thirdeye.paths import session_dir +from thirdeye.platforms.copilot.archive import commit_batch +from thirdeye.platforms.copilot.constants import PLATFORM_NAME +from thirdeye.platforms.copilot.followup import _claim_lease, _run +from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id +from thirdeye.platforms.copilot.projection_store import read_projection_status +from thirdeye.platforms.copilot.runtime import ( + archived_session_ids, + exports_configured, + reconcile_archived_sessions, + reconcile_session, +) +from thirdeye.platforms.copilot.state import state_path +from thirdeye.platforms.copilot import watch as watch_mod + +FIXTURES = Path(__file__).parent / "fixtures" +NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" +OTHER_SESSION_ID = "a1b2c3d4-e5f6-7890-abcd-ef1234567890" + + +@pytest.fixture +def copilot_env(tmp_path: Path) -> tuple[Config, SourcePaths]: + home = tmp_path / "copilot-home" + home.mkdir() + config = Config(root=tmp_path / "thirdeye") + paths = resolve_sources(home) + return config, paths + + +def _record(source_id: str, *, native_session_id: str = NATIVE_SESSION_ID) -> SourceRecord: + return { + "source_id": source_id, + "source_kind": "transcript", + "native_session_id": native_session_id, + "ts": "2026-09-10T17:08:24.000Z", + "observed_at": "2026-09-10T17:08:25.000Z", + "payload": {"schema_version": 1, "type": "user.message"}, + "locator": {"file": "events.jsonl", "offset": 0}, + } + + +def _batch( + paths: SourcePaths, + records: list[SourceRecord], + *, + native_session_id: str = NATIVE_SESSION_ID, +) -> SourceBatch: + return { + "source_key": paths["source_key"], + "native_session_id": native_session_id, + "cwd": "/proj", + "records": records, + "next_cursor": {"generation": 1}, + "diagnostics": [], + } + + +def _seed_archive( + config: Config, + paths: SourcePaths, + *, + native_session_id: str = NATIVE_SESSION_ID, +) -> str: + commit_batch( + config, + paths, + _batch( + paths, + [_record("runtime/archived", native_session_id=native_session_id)], + native_session_id=native_session_id, + ), + ) + return stored_session_id(paths, native_session_id) + + +def _empty_result(**overrides: int) -> SyncResult: + result: SyncResult = { + "sessions": 0, + "records_written": 0, + "duplicate_records": 0, + "pending": 0, + "errors": 0, + } + result.update(overrides) # type: ignore[typeddict-item] + return result + + +def test_exports_configured_requires_enabled_logfire_with_token() -> None: + assert exports_configured(Config(root=Path("/tmp/thirdeye"))) is False + assert ( + exports_configured( + Config( + root=Path("/tmp/thirdeye"), + logfire=LogfireSettings(enabled=True, token="fake-token"), + ) + ) + is True + ) + assert ( + exports_configured( + Config( + root=Path("/tmp/thirdeye"), + logfire=LogfireSettings(enabled=True, token=""), + ) + ) + is False + ) + + +def test_archived_session_ids_lists_sessions_for_source_home( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + stored = _seed_archive(config, paths) + _seed_archive(config, paths, native_session_id=OTHER_SESSION_ID) + + assert archived_session_ids(config, paths) == sorted([stored, stored_session_id(paths, OTHER_SESSION_ID)]) + + +def test_archived_session_ids_ignores_other_source_keys( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + stored = _seed_archive(config, paths) + directory = session_dir(config.root, PLATFORM_NAME, stored) + state = json.loads(state_path(directory).read_text(encoding="utf-8")) + state["source_key"] = "x" * 64 + state_path(directory).write_text(json.dumps(state) + "\n", encoding="utf-8") + + assert archived_session_ids(config, paths) == [] + + +def test_reconcile_session_maps_native_id_to_stored_session( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + stored = _seed_archive(config, paths) + calls: list[tuple[str, bool, bool]] = [] + + def fake_reconcile_archive( + _config: Config, + stored_session_id: str, + *, + rebuild: bool = False, + export: bool = False, + include_history: bool = False, + ) -> dict[str, int]: + calls.append((stored_session_id, export, include_history)) + return {"events": 1, "usage": 0, "turns": 1, "exports": 0, "pending": 0, "ambiguous": 0, "conflicting": 0, "errors": 0} + + monkeypatch.setattr("thirdeye.platforms.copilot.runtime.reconcile_archive", fake_reconcile_archive) + result = reconcile_session(config, paths, NATIVE_SESSION_ID, export=True, include_history=True) + + assert result["turns"] == 1 + assert calls == [(stored, False, True)] + + +def test_reconcile_session_skips_export_when_logfire_not_configured( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + _seed_archive(config, paths) + calls: list[bool] = [] + + def fake_reconcile_archive( + _config: Config, + _stored_session_id: str, + *, + rebuild: bool = False, + export: bool = False, + include_history: bool = False, + ) -> dict[str, int]: + calls.append(export) + return {"events": 0, "usage": 0, "turns": 0, "exports": 0, "pending": 0, "ambiguous": 0, "conflicting": 0, "errors": 0} + + monkeypatch.setattr("thirdeye.platforms.copilot.runtime.reconcile_archive", fake_reconcile_archive) + reconcile_session(config, paths, NATIVE_SESSION_ID, export=True) + + assert calls == [False] + + +def test_reconcile_archived_sessions_replays_all_retained_sessions( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + first = _seed_archive(config, paths) + second = _seed_archive(config, paths, native_session_id=OTHER_SESSION_ID) + seen: list[str] = [] + + def fake_reconcile_archive( + _config: Config, + stored_session_id: str, + *, + rebuild: bool = False, + export: bool = False, + include_history: bool = False, + ) -> dict[str, int]: + seen.append(stored_session_id) + return {"events": 0, "usage": 0, "turns": 0, "exports": 0, "pending": 0, "ambiguous": 0, "conflicting": 0, "errors": 0} + + monkeypatch.setattr("thirdeye.platforms.copilot.runtime.reconcile_archive", fake_reconcile_archive) + results = reconcile_archived_sessions(config, paths) + + assert set(seen) == {first, second} + assert set(results) == {first, second} + + +def test_reconcile_archived_sessions_works_after_source_removal( + copilot_env: tuple[Config, SourcePaths], + tmp_path: Path, +) -> None: + config, paths = copilot_env + home = Path(paths["home"]) + session_dir_path = home / "session-state" / NATIVE_SESSION_ID + session_dir_path.mkdir(parents=True) + shutil.copy(FIXTURES / "events.jsonl", session_dir_path / "events.jsonl") + (session_dir_path / "workspace.yaml").write_text("cwd: /sanitized/workspace\n", encoding="utf-8") + + from thirdeye.platforms.copilot.capture import sync + + sync(config, paths, session_id=NATIVE_SESSION_ID) + stored = stored_session_id(paths, NATIVE_SESSION_ID) + shutil.rmtree(home / "session-state") + (home / "session-store.db").unlink(missing_ok=True) + + result = reconcile_archived_sessions(config, paths)[stored] + + assert result["errors"] == 0 + assert read_projection_status(config, stored)["errors"] == 0 + + +def test_followup_reconciles_after_successful_capture( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + generation = _claim_lease(config, paths, NATIVE_SESSION_ID) + assert generation is not None + reconcile_calls: list[tuple[str, bool]] = [] + + def fake_capture(_config: Config, _paths: SourcePaths, _native_id: str) -> SyncResult: + return _empty_result(sessions=1, records_written=1) + + def fake_reconcile_session( + _config: Config, + _paths: SourcePaths, + native_session_id: str, + *, + export: bool = False, + include_history: bool = False, + ) -> dict[str, int]: + reconcile_calls.append((native_session_id, export)) + return {"events": 0, "usage": 0, "turns": 0, "exports": 0, "pending": 0, "ambiguous": 0, "conflicting": 0, "errors": 0} + + monkeypatch.setattr("thirdeye.platforms.copilot.capture.capture_session", fake_capture) + monkeypatch.setattr( + "thirdeye.platforms.copilot.runtime.reconcile_session", + fake_reconcile_session, + ) + _run(config, paths, NATIVE_SESSION_ID, generation) + + assert reconcile_calls == [(NATIVE_SESSION_ID, True)] + + +def test_followup_reconcile_failure_does_not_block_capture_completion( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + from thirdeye.platforms.copilot.followup import _lease_path + + generation = _claim_lease(config, paths, NATIVE_SESSION_ID) + assert generation is not None + directory = session_dir(config.root, PLATFORM_NAME, stored_session_id(paths, NATIVE_SESSION_ID)) + + def fake_capture(_config: Config, _paths: SourcePaths, _native_id: str) -> SyncResult: + return _empty_result(sessions=1) + + def boom(*_args: Any, **_kwargs: Any) -> dict[str, int]: + raise RuntimeError("projection failed") + + monkeypatch.setattr("thirdeye.platforms.copilot.capture.capture_session", fake_capture) + monkeypatch.setattr("thirdeye.platforms.copilot.runtime.reconcile_session", boom) + _run(config, paths, NATIVE_SESSION_ID, generation) + + assert not _lease_path(directory).exists() + log = config.root / "logs" / "usage-errors.jsonl" + assert log.is_file() + entries = [json.loads(line) for line in log.read_text(encoding="utf-8").splitlines()] + assert any(entry.get("phase") == "copilot_followup_reconcile" for entry in entries) + + +def test_watch_activation_reconciles_archived_sessions( + monkeypatch: pytest.MonkeyPatch, + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + activation: list[bool] = [] + per_session: list[str] = [] + + def fake_sync(_config: Config, _paths: SourcePaths, *, session_id: str | None = None) -> SyncResult: + return _empty_result() + + def fake_reconcile_archived( + _config: Config, + _paths: SourcePaths, + *, + export: bool = False, + include_history: bool = False, + ) -> dict[str, dict[str, int]]: + activation.append(export) + return {} + + def fake_reconcile_one( + _config: Config, + _paths: SourcePaths, + native_session_id: str, + *, + export: bool = False, + include_history: bool = False, + ) -> dict[str, int]: + per_session.append(native_session_id) + return {"events": 0, "usage": 0, "turns": 0, "exports": 0, "pending": 0, "ambiguous": 0, "conflicting": 0, "errors": 0} + + monkeypatch.setattr(watch_mod, "sync", fake_sync) + monkeypatch.setattr(watch_mod, "reconcile_archived_sessions", fake_reconcile_archived) + monkeypatch.setattr(watch_mod, "reconcile_session", fake_reconcile_one) + monkeypatch.setattr( + watch_mod, "_SLEEP", lambda _interval: (_ for _ in ()).throw(KeyboardInterrupt) + ) + + watch_mod.watch(config, paths, interval=0.1) + + assert activation == [True] + assert per_session == [] diff --git a/tests/platforms/copilot/test_watch.py b/tests/platforms/copilot/test_watch.py index c137d9d..41a8a03 100644 --- a/tests/platforms/copilot/test_watch.py +++ b/tests/platforms/copilot/test_watch.py @@ -201,6 +201,8 @@ def stop_immediately(_interval: float) -> None: raise KeyboardInterrupt monkeypatch.setattr(watch_mod, "sync", tracking_sync) + monkeypatch.setattr(watch_mod, "reconcile_archived_sessions", lambda *_args, **_kwargs: {}) + monkeypatch.setattr(watch_mod, "reconcile_session", lambda *_args, **_kwargs: _empty_result()) monkeypatch.setattr(watch_mod, "_SLEEP", stop_immediately) watch(config, paths, interval=0.1) @@ -245,12 +247,24 @@ def test_watch_syncs_only_changed_transcript_session( _write_transcript(home, OTHER_SESSION_ID) events_path = home / "session-state" / NATIVE_SESSION_ID / "events.jsonl" calls: list[str | None] = [] + reconciled: list[str] = [] cycle = {"count": 0} def tracking_sync(cfg: Config, p: SourcePaths, *, session_id: str | None = None) -> SyncResult: calls.append(session_id) return _empty_result() + def track_reconcile( + _config: Config, + _paths: SourcePaths, + native_session_id: str, + *, + export: bool = False, + include_history: bool = False, + ) -> dict[str, int]: + reconciled.append(native_session_id) + return {"events": 0, "usage": 0, "turns": 0, "exports": 0, "pending": 0, "ambiguous": 0, "conflicting": 0, "errors": 0} + def append_during_poll(_interval: float) -> None: cycle["count"] += 1 if cycle["count"] == 1: @@ -260,6 +274,8 @@ def append_during_poll(_interval: float) -> None: raise KeyboardInterrupt monkeypatch.setattr(watch_mod, "sync", tracking_sync) + monkeypatch.setattr(watch_mod, "reconcile_archived_sessions", lambda *_args, **_kwargs: {}) + monkeypatch.setattr(watch_mod, "reconcile_session", track_reconcile) monkeypatch.setattr(watch_mod, "_SLEEP", append_during_poll) watch(config, paths, interval=0.1) @@ -267,6 +283,7 @@ def append_during_poll(_interval: float) -> None: assert calls[0] is None assert calls.count(NATIVE_SESSION_ID) == 1 assert OTHER_SESSION_ID not in calls + assert reconciled == [NATIVE_SESSION_ID] def test_watch_detects_database_wal_change( From 276d547a7cce5386b1cfa8e9d9d78b7283bd6717 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Sun, 13 Sep 2026 11:08:55 -0700 Subject: [PATCH 82/88] Keep Copilot V2 runtime export eligibility and status honest after review. Load persisted Logfire settings in follow-up, scope selected sync, persist the activation boundary without requiring export config, and report real job/claim health rather than ledger intent. Co-authored-by: Cursor --- src/thirdeye/commands/copilot.py | 82 ++++-- src/thirdeye/config.py | 8 +- src/thirdeye/platforms/copilot/followup.py | 2 +- src/thirdeye/platforms/copilot/identity.py | 28 +- src/thirdeye/platforms/copilot/runtime.py | 152 ++++++++++- src/thirdeye/platforms/copilot/status.py | 163 ++++++++++-- tests/platforms/copilot/test_command.py | 173 ++++++++++++- tests/platforms/copilot/test_runtime.py | 287 +++++++++++++++++++-- tests/platforms/copilot/test_watch.py | 28 +- 9 files changed, 826 insertions(+), 97 deletions(-) diff --git a/src/thirdeye/commands/copilot.py b/src/thirdeye/commands/copilot.py index 8b57bf4..33ba045 100644 --- a/src/thirdeye/commands/copilot.py +++ b/src/thirdeye/commands/copilot.py @@ -14,9 +14,17 @@ ) from thirdeye.platforms.copilot.capture import sync as capture_sync from thirdeye.platforms.copilot.constants import COPILOT_HOME_ENV -from thirdeye.platforms.copilot.identity import resolve_sources, validate_native_id -from thirdeye.platforms.copilot.reconcile import reconcile_archive -from thirdeye.platforms.copilot.runtime import reconcile_archived_sessions +from thirdeye.platforms.copilot.identity import ( + resolve_sources, + validate_native_id, + validate_stored_session_id, +) +from thirdeye.platforms.copilot.runtime import ( + all_archived_session_ids, + reconcile_archived_sessions, + reconcile_session, + reconcile_stored_session, +) from thirdeye.platforms.copilot.status import capture_status from thirdeye.platforms.copilot.types import SourcePaths, SyncResult from thirdeye.platforms.copilot.watch import watch as watch_loop @@ -148,7 +156,9 @@ def _status_error_affects_exit(error: object) -> bool: return False if _is_retryable_code(error.get("code")): return False - return error.get("kind") != "missing_source_time" + if error.get("kind") in {"missing_source_time", "copilot_reconcile_error"}: + return False + return True def _print_sync_result(result: SyncResult) -> None: @@ -275,13 +285,25 @@ def sync_cmd(session_id: str | None, export_history: bool, source_home: Path | N _print_sync_result(result) # Every sync refreshes only local V2 projections. Explicit opt-in is the # only sync path that can include completed history in export eligibility. - if export_history: - for derived in reconcile_archived_sessions( - config, paths, export=True, include_history=True - ).values(): + # A selected native ID never exports or rebuilds unrelated archives. + if session_id is None: + if export_history: + for derived in reconcile_archived_sessions( + config, paths, export=True, include_history=True + ).values(): + _print_reconcile_result(derived) + else: + reconcile_archived_sessions(config, paths) + elif result["sessions"] > 0: + derived = reconcile_session( + config, + paths, + session_id, + export=export_history, + include_history=export_history, + ) + if export_history: _print_reconcile_result(derived) - else: - reconcile_archived_sessions(config, paths) _print_sync_followup(config, paths, result) if session_id is not None and result["sessions"] == 0: raise click.ClickException( @@ -290,7 +312,7 @@ def sync_cmd(session_id: str | None, export_history: bool, source_home: Path | N @copilot_group.command("reconcile", help="Build Copilot V2 projections from retained local archives.") -@click.option("--session-id", required=True, help="Exact stored Thirdeye Copilot session ID.") +@click.option("--session-id", default=None, help="Exact stored Thirdeye Copilot session ID.") @click.option("--rebuild", is_flag=True, default=False, help="Rebuild derived state only.") @click.option( "--export", @@ -299,17 +321,32 @@ def sync_cmd(session_id: str | None, export_history: bool, source_home: Path | N default=False, help="Queue completed archived history for export after local reconciliation.", ) -def reconcile_cmd(session_id: str, rebuild: bool, export_history: bool) -> None: +def reconcile_cmd(session_id: str | None, rebuild: bool, export_history: bool) -> None: config = Config.load() - result = reconcile_archive( - config, - session_id, - rebuild=rebuild, - export=export_history, - include_history=export_history, - ) - _print_reconcile_result(result) - if result.get("errors", 0): + if session_id is None: + stored_ids = all_archived_session_ids(config) + else: + try: + validate_stored_session_id(session_id) + stored_ids = [session_id] + except ValueError as exc: + raise click.ClickException(str(exc)) from exc + failed = False + for stored_id in stored_ids: + try: + result = reconcile_stored_session( + config, + stored_id, + rebuild=rebuild, + export=export_history, + include_history=export_history, + ) + except ValueError as exc: + raise click.ClickException(str(exc)) from exc + _print_reconcile_result(result) + if result.get("errors", 0): + failed = True + if failed: raise click.ClickException("Copilot reconciliation completed with errors") @@ -327,7 +364,8 @@ def watch_cmd(source_home: Path | None, interval: float) -> None: config = Config.load() paths = _resolve_paths(source_home) click.echo( - f"Watching Copilot home {paths['home']} every {interval}s (local-only). Ctrl-C to stop." + f"Watching Copilot home {paths['home']} every {interval}s. " + "Configured exports are queued locally. Ctrl-C to stop." ) try: watch_loop(config, paths, interval=interval) diff --git a/src/thirdeye/config.py b/src/thirdeye/config.py index 1505024..d48449a 100644 --- a/src/thirdeye/config.py +++ b/src/thirdeye/config.py @@ -91,9 +91,9 @@ class Config: logfire: LogfireSettings = field(default_factory=LogfireSettings) @classmethod - def load(cls) -> Config: - root = default_root() - raw = _read_config_yaml(root / "config.yaml") + def load(cls, root: Path | None = None) -> Config: + resolved = Path(root) if root is not None else default_root() + raw = _read_config_yaml(resolved / "config.yaml") # THIRDEYE_CAPTURE_ENV wins when set, so a one-off run can still # override; otherwise fall back to config.yaml's ``capture_env`` so # capture does not depend on the launching shell exporting anything @@ -102,7 +102,7 @@ def load(cls) -> Config: if not patterns: patterns = _coerce_patterns(raw.get("capture_env")) return cls( - root=root, + root=resolved, capture_env_patterns=patterns, logfire=LogfireSettings.from_dict(raw.get("logfire")), ) diff --git a/src/thirdeye/platforms/copilot/followup.py b/src/thirdeye/platforms/copilot/followup.py index f1e05e4..303337d 100644 --- a/src/thirdeye/platforms/copilot/followup.py +++ b/src/thirdeye/platforms/copilot/followup.py @@ -258,7 +258,7 @@ def main() -> None: paths = resolve_sources(Path(args.source_home)) native_id = str(args.session_id) validate_native_id(native_id) - config = Config(root=Path(args.config_root)) + config = Config.load(root=Path(args.config_root)) _run(config, paths, native_id, str(args.generation)) except Exception: # This worker is deliberately silent: diagnostics are kept locally and diff --git a/src/thirdeye/platforms/copilot/identity.py b/src/thirdeye/platforms/copilot/identity.py index c22af3c..6d81009 100644 --- a/src/thirdeye/platforms/copilot/identity.py +++ b/src/thirdeye/platforms/copilot/identity.py @@ -77,17 +77,29 @@ def resolve_sources(source_home: Path | None = None) -> SourcePaths: return paths +def _validate_path_id(value: str, *, label: str) -> None: + if not isinstance(value, str) or not value or value.strip() != value: + raise ValueError(f"{label} must be a non-empty, trimmed string") + if value in {".", ".."}: + raise ValueError(f"{label} must not be a traversal segment") + if any(character in value for character in ("/", "\\", "\x00", ":")): + raise ValueError(f"{label} contains a path separator or invalid path character") + if any(ord(character) < 32 for character in value): + raise ValueError(f"{label} contains a control character") + + def validate_native_id(native_id: str) -> None: """Ensure a native session ID can never select a path outside its home.""" - if not isinstance(native_id, str) or not native_id or native_id.strip() != native_id: - raise ValueError("native session ID must be a non-empty, trimmed string") - if native_id in {".", ".."}: - raise ValueError("native session ID must not be a traversal segment") - if any(character in native_id for character in ("/", "\\", "\x00", ":")): - raise ValueError("native session ID contains a path separator or invalid path character") - if any(ord(character) < 32 for character in native_id): - raise ValueError("native session ID contains a control character") + _validate_path_id(native_id, label="native session ID") + + +def validate_stored_session_id(stored_id: str) -> None: + """Ensure a stored session ID cannot be used as a path-escape segment.""" + + _validate_path_id(stored_id, label="stored session ID") + if not stored_id.startswith("copilot-"): + raise ValueError("stored session ID must start with 'copilot-'") def stored_session_id(paths: SourcePaths, native_id: str) -> str: diff --git a/src/thirdeye/platforms/copilot/runtime.py b/src/thirdeye/platforms/copilot/runtime.py index 0aa9ebc..508090e 100644 --- a/src/thirdeye/platforms/copilot/runtime.py +++ b/src/thirdeye/platforms/copilot/runtime.py @@ -7,21 +7,97 @@ from __future__ import annotations +from pathlib import Path +from typing import Any + from thirdeye.config import Config -from thirdeye.paths import platform_dir +from thirdeye.paths import platform_dir, session_dir +from thirdeye.usage.errlog import log_capture_error from .constants import PLATFORM_NAME -from .identity import stored_session_id +from .identity import stored_session_id, validate_stored_session_id +from .jsonio import atomic_write_json, read_json_object from .reconcile import reconcile_archive from .state import read_json, state_path from .types import SourcePaths +RUNTIME_STATUS_FILENAME = "copilot.runtime.json" + def exports_configured(config: Config) -> bool: """Whether runtime activity may queue detached live export work.""" return bool(config.logfire.enabled and config.logfire.token) +def runtime_status_path(directory: Path) -> Path: + return directory / RUNTIME_STATUS_FILENAME + + +def stored_session_directory(config: Config, stored_id: str) -> Path: + """Return the archive directory for a stored ID after rejecting path escapes.""" + + validate_stored_session_id(stored_id) + root = platform_dir(config.root, PLATFORM_NAME).resolve() + directory = (root / stored_id).resolve() + try: + directory.relative_to(root) + except ValueError as exc: + raise ValueError("stored session ID escapes the Copilot archive") from exc + return directory + + +def load_runtime_status(config: Config, stored_session_id: str) -> dict[str, Any]: + """Return the last recorded derived-work error for a stored session.""" + + try: + directory = stored_session_directory(config, stored_session_id) + raw = read_json_object( + runtime_status_path(directory), + invalid_message="invalid Copilot runtime status", + ) + except (OSError, ValueError): + return {} + return raw if isinstance(raw, dict) else {} + + +def _note_reconcile_result( + config: Config, stored_id: str, result: dict[str, int] +) -> dict[str, int]: + """Persist/log derived failures without raising into capture.""" + + directory = session_dir(config.root, PLATFORM_NAME, stored_id) + path = runtime_status_path(directory) + if result.get("errors"): + log_capture_error( + thirdeye_home=config.root, + phase="copilot_reconcile", + message=f"errors={result.get('errors')}", + platform=PLATFORM_NAME, + session_id=stored_id, + silent_fallback=True, + ) + if directory.is_dir(): + try: + atomic_write_json( + path, + { + "last_error": { + "phase": "copilot_reconcile", + "errors": int(result.get("errors") or 0), + } + }, + ) + except OSError: + pass + return result + if directory.is_dir(): + try: + path.unlink(missing_ok=True) + except OSError: + pass + return result + + def archived_session_ids(config: Config, paths: SourcePaths) -> list[str]: """Return this source home's retained V1 sessions without reading sources.""" root = platform_dir(config.root, PLATFORM_NAME) @@ -35,6 +111,7 @@ def archived_session_ids(config: Config, paths: SourcePaths) -> list[str]: if not directory.name.startswith(prefix): continue try: + validate_stored_session_id(directory.name) state = read_json(state_path(directory)) or {} except ValueError: continue @@ -43,6 +120,50 @@ def archived_session_ids(config: Config, paths: SourcePaths) -> list[str]: return result +def all_archived_session_ids(config: Config) -> list[str]: + """Return every retained Copilot archive under this Thirdeye home.""" + + root = platform_dir(config.root, PLATFORM_NAME) + try: + directories = sorted(entry for entry in root.iterdir() if entry.is_dir()) + except OSError: + return [] + result: list[str] = [] + for directory in directories: + try: + validate_stored_session_id(directory.name) + state = read_json(state_path(directory)) + except ValueError: + continue + if state is not None: + result.append(directory.name) + return result + + +def reconcile_stored_session( + config: Config, + stored_session_id: str, + *, + rebuild: bool = False, + export: bool = False, + include_history: bool = False, +) -> dict[str, int]: + """Derive one stored archive after validating its identity.""" + + stored_session_directory(config, stored_session_id) + return _note_reconcile_result( + config, + stored_session_id, + reconcile_archive( + config, + stored_session_id, + rebuild=rebuild, + export=export, + include_history=include_history, + ), + ) + + def reconcile_session( config: Config, paths: SourcePaths, @@ -51,11 +172,16 @@ def reconcile_session( export: bool = False, include_history: bool = False, ) -> dict[str, int]: - """Derive one retained session, optionally queueing detached live export.""" - return reconcile_archive( + """Derive one retained session, optionally queueing detached live export. + + ``export`` is forwarded even when Logfire is not configured: export + assembly records the activation boundary before it decides whether any + jobs can be dispatched. + """ + return reconcile_stored_session( config, stored_session_id(paths, native_session_id), - export=export and exports_configured(config), + export=export, include_history=include_history, ) @@ -69,14 +195,24 @@ def reconcile_archived_sessions( ) -> dict[str, dict[str, int]]: """Replay retained sessions even after their live sources disappear.""" return { - session_id: reconcile_archive( + session_id: reconcile_stored_session( config, session_id, - export=export and exports_configured(config), + export=export, include_history=include_history, ) for session_id in archived_session_ids(config, paths) } -__all__ = ["archived_session_ids", "exports_configured", "reconcile_archived_sessions", "reconcile_session"] +__all__ = [ + "all_archived_session_ids", + "archived_session_ids", + "exports_configured", + "load_runtime_status", + "reconcile_archived_sessions", + "reconcile_session", + "reconcile_stored_session", + "runtime_status_path", + "stored_session_directory", +] diff --git a/src/thirdeye/platforms/copilot/status.py b/src/thirdeye/platforms/copilot/status.py index 960636a..e138940 100644 --- a/src/thirdeye/platforms/copilot/status.py +++ b/src/thirdeye/platforms/copilot/status.py @@ -8,15 +8,17 @@ from typing import Any from thirdeye.config import Config -from thirdeye.paths import platform_dir +from thirdeye.paths import otel_jobs_dir, platform_dir from thirdeye.reader import SessionReader +from . import export_transport from .archive import _record_from_event from .constants import FOLLOWUP_LEASE_FILENAME, PLATFORM_NAME from .database import read_database from .export_state import load_export_state from .install import CopilotPlatform from .projection_store import read_projection_status +from .runtime import load_runtime_status from .spool import read_spool from .state import journal_path, read_json, state_path from .types import SourcePaths, SourceRecord @@ -231,6 +233,134 @@ def _followup_lease_pending(directory: Path) -> bool: return float(expires_at) > time.time() +def _json_object(path: Path) -> dict[str, Any] | None: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError): + return None + return value if isinstance(value, dict) else None + + +def _iter_json_jobs(directory: Path) -> list[dict[str, Any]]: + payloads: list[dict[str, Any]] = [] + try: + entries = list(directory.iterdir()) + except OSError: + return payloads + for path in entries: + if not path.is_file() or path.suffix != ".json" or path.name.endswith(".claim"): + continue + payload = _json_object(path) + if payload is not None: + payloads.append(payload) + return payloads + + +def _export_health(config: Config, stored_session_id: str, directory: Path) -> dict[str, Any]: + """Summarize queue/delivery health from claims and jobs, not ledger intent.""" + + ledger = load_export_state(config, stored_session_id) + placements = ledger.get("placements") if isinstance(ledger.get("placements"), dict) else {} + conflicts = ledger.get("conflicts") if isinstance(ledger.get("conflicts"), dict) else {} + turn_errors = ledger.get("turn_errors") if isinstance(ledger.get("turn_errors"), dict) else {} + + delivered: set[str] = set() + queued: set[str] = set() + errored: set[str] = set() + + for accounting_id, item in placements.items(): + if not isinstance(accounting_id, str) or not isinstance(item, dict): + continue + key = f"acct:{accounting_id}" + span_id = item.get("span_id") + sent = export_transport.delivery_sent(directory, accounting_id) + job = ( + export_transport.status(config.root, span_id) + if isinstance(span_id, str) and span_id + else None + ) + job_state = job.get("state") if job else None + if sent or item.get("emitted") or job_state == "emitted": + delivered.add(key) + elif job_state == "failed": + queued.add(key) + errored.add(key) + elif job_state in {"queued", "claimed", "retrying"}: + queued.add(key) + else: + queued.add(key) + if item.get("last_error") or (job and job.get("last_error")): + errored.add(key) + + for payload in _iter_json_jobs(export_transport.jobs_dir(config.root)): + if payload.get("session_id") != stored_session_id: + continue + accounting_id = payload.get("accounting_id") + key = ( + f"acct:{accounting_id}" + if isinstance(accounting_id, str) and accounting_id + else f"acct-job:{payload.get('job_id')}" + ) + if key in delivered: + continue + state = payload.get("state") + if state == "emitted" or ( + isinstance(accounting_id, str) and export_transport.delivery_sent(directory, accounting_id) + ): + delivered.add(key) + queued.discard(key) + continue + queued.add(key) + if state == "failed" or payload.get("last_error"): + errored.add(key) + + sent_dir = directory / "otel-turns-sent" + try: + claim_files = list(sent_dir.iterdir()) if sent_dir.is_dir() else [] + except OSError: + claim_files = [] + for path in claim_files: + if not path.is_file(): + continue + try: + text = path.read_text(encoding="utf-8") + except OSError: + continue + key = f"turn-claim:{path.name}" + if text == "sent": + delivered.add(key) + else: + queued.add(key) + + for payload in _iter_json_jobs(otel_jobs_dir(config.root)): + if payload.get("session_id") != stored_session_id: + continue + if payload.get("kind") not in {"turn", "spans", "subagent_turn"}: + continue + queued.add(f"otel:{payload.get('job_id') or id(payload)}") + if payload.get("last_error"): + errored.add(f"otel:{payload.get('job_id') or id(payload)}") + + queued -= delivered + + runtime_status = load_runtime_status(config, stored_session_id) + last_error = runtime_status.get("last_error") + if isinstance(last_error, dict): + errored.add("runtime") + else: + last_error = None + + return { + "activated": bool(ledger.get("activated")), + "queued": len(queued), + "delivered": len(delivered), + "errors": len(errored) + + len(conflicts) + + len(turn_errors), + "last_error": last_error, + } + + def _archive_status( config: Config, paths: SourcePaths ) -> tuple[list[dict[str, Any]], SourceRecord | None, list[dict[str, Any]], int, int]: @@ -288,28 +418,17 @@ def _archive_status( } ) try: - ledger = load_export_state(config, directory.name) - placements = ledger.get("placements") if isinstance(ledger.get("placements"), dict) else {} - session["export"] = { - "activated": bool(ledger.get("activated")), - "queued": sum( - 1 - for item in placements.values() - if isinstance(item, dict) and not item.get("emitted") - ), - "delivered": sum( - 1 - for item in placements.values() - if isinstance(item, dict) and item.get("emitted") - ), - "errors": sum( - 1 - for item in placements.values() - if isinstance(item, dict) and item.get("last_error") + session["export"] = _export_health(config, directory.name, directory) + last_error = session["export"].get("last_error") + if isinstance(last_error, dict): + errors.append( + { + "kind": "copilot_reconcile_error", + "session": directory.name, + "reason": last_error.get("phase") or "copilot_reconcile", + "errors": last_error.get("errors"), + } ) - + len(ledger.get("conflicts") if isinstance(ledger.get("conflicts"), dict) else {}) - + len(ledger.get("turn_errors") if isinstance(ledger.get("turn_errors"), dict) else {}), - } except Exception as error: session["export"] = {"activated": False, "queued": 0, "delivered": 0, "errors": 1} errors.append( diff --git a/tests/platforms/copilot/test_command.py b/tests/platforms/copilot/test_command.py index c3db31a..f06745d 100644 --- a/tests/platforms/copilot/test_command.py +++ b/tests/platforms/copilot/test_command.py @@ -333,6 +333,19 @@ def fake_sync( return _empty_result(sessions=1) monkeypatch.setattr("thirdeye.commands.copilot.capture_sync", fake_sync) + monkeypatch.setattr( + "thirdeye.commands.copilot.reconcile_session", + lambda *_args, **_kwargs: { + "events": 0, + "usage": 0, + "turns": 0, + "exports": 0, + "pending": 0, + "ambiguous": 0, + "conflicting": 0, + "errors": 0, + }, + ) result = CliRunner().invoke(main, ["copilot", "sync", "--session-id", NATIVE_SESSION_ID]) assert result.exit_code == 0, result.output assert captured["session_id"] == NATIVE_SESSION_ID @@ -473,6 +486,19 @@ def fake_sync( return _empty_result(sessions=1, records_written=12, errors=1) monkeypatch.setattr("thirdeye.commands.copilot.capture_sync", fake_sync) + monkeypatch.setattr( + "thirdeye.commands.copilot.reconcile_session", + lambda *_args, **_kwargs: { + "events": 0, + "usage": 0, + "turns": 0, + "exports": 0, + "pending": 0, + "ambiguous": 0, + "conflicting": 0, + "errors": 0, + }, + ) result = CliRunner().invoke( main, ["copilot", "sync", "--session-id", NATIVE_SESSION_ID], @@ -508,6 +534,19 @@ def fake_status(_config: Config, _paths: SourcePaths) -> dict: monkeypatch.setattr("thirdeye.commands.copilot.capture_sync", fake_sync) monkeypatch.setattr("thirdeye.commands.copilot.capture_status", fake_status) + monkeypatch.setattr( + "thirdeye.commands.copilot.reconcile_session", + lambda *_args, **_kwargs: { + "events": 0, + "usage": 0, + "turns": 0, + "exports": 0, + "pending": 0, + "ambiguous": 0, + "conflicting": 0, + "errors": 0, + }, + ) result = CliRunner().invoke(main, ["copilot", "sync", "--session-id", NATIVE_SESSION_ID]) assert result.exit_code == 0, result.output assert "transcript_invalid_json" in result.output @@ -696,7 +735,7 @@ def fake_reconcile_archive( "errors": 0, } - monkeypatch.setattr("thirdeye.commands.copilot.reconcile_archive", fake_reconcile_archive) + monkeypatch.setattr("thirdeye.commands.copilot.reconcile_stored_session", fake_reconcile_archive) result = CliRunner().invoke( main, ["copilot", "reconcile", "--session-id", "copilot-abc-stored", "--rebuild", "--export"], @@ -711,7 +750,7 @@ def test_reconcile_command_exits_nonzero_on_errors( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr( - "thirdeye.commands.copilot.reconcile_archive", + "thirdeye.commands.copilot.reconcile_stored_session", lambda *_args, **_kwargs: { "events": 0, "usage": 0, @@ -755,6 +794,133 @@ def test_status_prints_projection_and_export_summaries( assert "Export: activated=1 queued=3 delivered=1 errors=2" in result.output +def test_sync_session_id_reconciles_only_selected_session( + isolated_home: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + archived_calls: list[tuple[bool, bool]] = [] + session_calls: list[tuple[str, bool, bool]] = [] + + def fake_sync( + _config: Config, _paths: SourcePaths, *, session_id: str | None = None + ) -> SyncResult: + return _empty_result(sessions=1) + + def fake_archived( + _config: Config, + _paths: SourcePaths, + *, + export: bool = False, + include_history: bool = False, + ) -> dict[str, dict[str, int]]: + archived_calls.append((export, include_history)) + return {"other-stored": {"events": 9, "usage": 0, "turns": 0, "exports": 1, "pending": 0, "ambiguous": 0, "conflicting": 0, "errors": 0}} + + def fake_session( + _config: Config, + _paths: SourcePaths, + native_session_id: str, + *, + export: bool = False, + include_history: bool = False, + ) -> dict[str, int]: + session_calls.append((native_session_id, export, include_history)) + return { + "events": 1, + "usage": 0, + "turns": 1, + "exports": 0, + "pending": 0, + "ambiguous": 0, + "conflicting": 0, + "errors": 0, + } + + monkeypatch.setattr("thirdeye.commands.copilot.capture_sync", fake_sync) + monkeypatch.setattr("thirdeye.commands.copilot.reconcile_archived_sessions", fake_archived) + monkeypatch.setattr("thirdeye.commands.copilot.reconcile_session", fake_session) + result = CliRunner().invoke( + main, ["copilot", "sync", "--session-id", NATIVE_SESSION_ID, "--export"] + ) + assert result.exit_code == 0, result.output + assert archived_calls == [] + assert session_calls == [(NATIVE_SESSION_ID, True, True)] + assert "other-stored" not in result.output + assert "reconcile events=1" in result.output + + +def test_sync_missing_session_does_not_reconcile_unrelated_history( + isolated_home: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + archived_calls: list[object] = [] + session_calls: list[object] = [] + + def fake_sync( + _config: Config, _paths: SourcePaths, *, session_id: str | None = None + ) -> SyncResult: + return _empty_result(sessions=0, errors=1) + + monkeypatch.setattr("thirdeye.commands.copilot.capture_sync", fake_sync) + monkeypatch.setattr( + "thirdeye.commands.copilot.reconcile_archived_sessions", + lambda *_a, **_k: archived_calls.append(True) or {}, + ) + monkeypatch.setattr( + "thirdeye.commands.copilot.reconcile_session", + lambda *_a, **_k: session_calls.append(True) or {}, + ) + result = CliRunner().invoke(main, ["copilot", "sync", "--session-id", NATIVE_SESSION_ID, "--export"]) + assert result.exit_code != 0, result.output + assert "was not found" in result.output + assert archived_calls == [] + assert session_calls == [] + + +def test_reconcile_without_session_id_reconciles_all_archives( + isolated_home: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[str] = [] + + def fake_reconcile( + _config: Config, + stored_session_id: str, + *, + rebuild: bool = False, + export: bool = False, + include_history: bool = False, + ) -> dict[str, int]: + calls.append(stored_session_id) + return { + "events": 1, + "usage": 0, + "turns": 1, + "exports": 0, + "pending": 0, + "ambiguous": 0, + "conflicting": 0, + "errors": 0, + } + + monkeypatch.setattr( + "thirdeye.commands.copilot.all_archived_session_ids", + lambda _config: ["copilot-aaa-one", "copilot-bbb-two"], + ) + monkeypatch.setattr("thirdeye.commands.copilot.reconcile_stored_session", fake_reconcile) + result = CliRunner().invoke(main, ["copilot", "reconcile", "--rebuild"]) + assert result.exit_code == 0, result.output + assert calls == ["copilot-aaa-one", "copilot-bbb-two"] + assert result.output.count("reconcile events=1") == 2 + + +def test_reconcile_rejects_path_escape_session_id(isolated_home: Path) -> None: + result = CliRunner().invoke(main, ["copilot", "reconcile", "--session-id", "../secret"]) + assert result.exit_code != 0, result.output + assert "Traceback" not in result.output + assert "path separator" in result.output or "stored session ID" in result.output + + # -- watch --------------------------------------------------------------------- @@ -788,7 +954,8 @@ def test_watch_prints_start_and_stop(isolated_home: Path, monkeypatch: pytest.Mo result = CliRunner().invoke(main, ["copilot", "watch", "--interval", "1.5"]) assert result.exit_code == 0, result.output assert "1.5" in result.output - assert "local-only" in result.output + assert "queued locally" in result.output + assert "local-only" not in result.output assert "Ctrl-C" in result.output assert "Stopped" in result.output diff --git a/tests/platforms/copilot/test_runtime.py b/tests/platforms/copilot/test_runtime.py index 2ce52d8..e04c8f7 100644 --- a/tests/platforms/copilot/test_runtime.py +++ b/tests/platforms/copilot/test_runtime.py @@ -4,26 +4,38 @@ import json import shutil +import sys from pathlib import Path from typing import Any import pytest from thirdeye.config import Config, LogfireSettings -from thirdeye.paths import session_dir +from thirdeye.paths import otel_jobs_dir, session_dir +from thirdeye.platforms.copilot import watch as watch_mod from thirdeye.platforms.copilot.archive import commit_batch from thirdeye.platforms.copilot.constants import PLATFORM_NAME +from thirdeye.platforms.copilot.export_state import ( + load_export_state, + record_placement, + update_export_state, +) +from thirdeye.platforms.copilot.export_transport import delivery_claim_path, job_path from thirdeye.platforms.copilot.followup import _claim_lease, _run +from thirdeye.platforms.copilot.followup import main as followup_main from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id from thirdeye.platforms.copilot.projection_store import read_projection_status from thirdeye.platforms.copilot.runtime import ( + all_archived_session_ids, archived_session_ids, exports_configured, + load_runtime_status, reconcile_archived_sessions, reconcile_session, ) from thirdeye.platforms.copilot.state import state_path -from thirdeye.platforms.copilot import watch as watch_mod +from thirdeye.platforms.copilot.status import capture_status +from thirdeye.platforms.copilot.types import SourceBatch, SourcePaths, SourceRecord, SyncResult FIXTURES = Path(__file__).parent / "fixtures" NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" @@ -126,7 +138,9 @@ def test_archived_session_ids_lists_sessions_for_source_home( stored = _seed_archive(config, paths) _seed_archive(config, paths, native_session_id=OTHER_SESSION_ID) - assert archived_session_ids(config, paths) == sorted([stored, stored_session_id(paths, OTHER_SESSION_ID)]) + assert archived_session_ids(config, paths) == sorted( + [stored, stored_session_id(paths, OTHER_SESSION_ID)] + ) def test_archived_session_ids_ignores_other_source_keys( @@ -142,6 +156,15 @@ def test_archived_session_ids_ignores_other_source_keys( assert archived_session_ids(config, paths) == [] +def test_all_archived_session_ids_lists_every_copilot_archive( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + first = _seed_archive(config, paths) + second = _seed_archive(config, paths, native_session_id=OTHER_SESSION_ID) + assert all_archived_session_ids(config) == sorted([first, second]) + + def test_reconcile_session_maps_native_id_to_stored_session( copilot_env: tuple[Config, SourcePaths], monkeypatch: pytest.MonkeyPatch, @@ -159,38 +182,37 @@ def fake_reconcile_archive( include_history: bool = False, ) -> dict[str, int]: calls.append((stored_session_id, export, include_history)) - return {"events": 1, "usage": 0, "turns": 1, "exports": 0, "pending": 0, "ambiguous": 0, "conflicting": 0, "errors": 0} + return { + "events": 1, + "usage": 0, + "turns": 1, + "exports": 0, + "pending": 0, + "ambiguous": 0, + "conflicting": 0, + "errors": 0, + } monkeypatch.setattr("thirdeye.platforms.copilot.runtime.reconcile_archive", fake_reconcile_archive) result = reconcile_session(config, paths, NATIVE_SESSION_ID, export=True, include_history=True) assert result["turns"] == 1 - assert calls == [(stored, False, True)] + assert calls == [(stored, True, True)] -def test_reconcile_session_skips_export_when_logfire_not_configured( +def test_reconcile_session_activates_eligibility_without_logfire( copilot_env: tuple[Config, SourcePaths], - monkeypatch: pytest.MonkeyPatch, ) -> None: config, paths = copilot_env - _seed_archive(config, paths) - calls: list[bool] = [] - - def fake_reconcile_archive( - _config: Config, - _stored_session_id: str, - *, - rebuild: bool = False, - export: bool = False, - include_history: bool = False, - ) -> dict[str, int]: - calls.append(export) - return {"events": 0, "usage": 0, "turns": 0, "exports": 0, "pending": 0, "ambiguous": 0, "conflicting": 0, "errors": 0} + stored = _seed_archive(config, paths) - monkeypatch.setattr("thirdeye.platforms.copilot.runtime.reconcile_archive", fake_reconcile_archive) - reconcile_session(config, paths, NATIVE_SESSION_ID, export=True) + result = reconcile_session(config, paths, NATIVE_SESSION_ID, export=True) - assert calls == [False] + assert result["exports"] == 0 + state = load_export_state(config, stored) + assert state["activated"] is True + restarted = load_export_state(config, stored) + assert restarted["activated"] is True def test_reconcile_archived_sessions_replays_all_retained_sessions( @@ -211,7 +233,16 @@ def fake_reconcile_archive( include_history: bool = False, ) -> dict[str, int]: seen.append(stored_session_id) - return {"events": 0, "usage": 0, "turns": 0, "exports": 0, "pending": 0, "ambiguous": 0, "conflicting": 0, "errors": 0} + return { + "events": 0, + "usage": 0, + "turns": 0, + "exports": 0, + "pending": 0, + "ambiguous": 0, + "conflicting": 0, + "errors": 0, + } monkeypatch.setattr("thirdeye.platforms.copilot.runtime.reconcile_archive", fake_reconcile_archive) results = reconcile_archived_sessions(config, paths) @@ -265,7 +296,16 @@ def fake_reconcile_session( include_history: bool = False, ) -> dict[str, int]: reconcile_calls.append((native_session_id, export)) - return {"events": 0, "usage": 0, "turns": 0, "exports": 0, "pending": 0, "ambiguous": 0, "conflicting": 0, "errors": 0} + return { + "events": 0, + "usage": 0, + "turns": 0, + "exports": 0, + "pending": 0, + "ambiguous": 0, + "conflicting": 0, + "errors": 0, + } monkeypatch.setattr("thirdeye.platforms.copilot.capture.capture_session", fake_capture) monkeypatch.setattr( @@ -305,6 +345,77 @@ def boom(*_args: Any, **_kwargs: Any) -> dict[str, int]: assert any(entry.get("phase") == "copilot_followup_reconcile" for entry in entries) +def test_followup_logs_returned_reconcile_errors( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + stored = _seed_archive(config, paths) + generation = _claim_lease(config, paths, NATIVE_SESSION_ID) + assert generation is not None + + def fake_capture(_config: Config, _paths: SourcePaths, _native_id: str) -> SyncResult: + return _empty_result(sessions=1) + + def fake_reconcile_archive( + *_args: Any, **_kwargs: Any + ) -> dict[str, int]: + return { + "events": 0, + "usage": 0, + "turns": 0, + "exports": 0, + "pending": 0, + "ambiguous": 0, + "conflicting": 0, + "errors": 2, + } + + monkeypatch.setattr("thirdeye.platforms.copilot.capture.capture_session", fake_capture) + monkeypatch.setattr("thirdeye.platforms.copilot.runtime.reconcile_archive", fake_reconcile_archive) + _run(config, paths, NATIVE_SESSION_ID, generation) + + status = load_runtime_status(config, stored) + assert status["last_error"]["errors"] == 2 + log = config.root / "logs" / "usage-errors.jsonl" + entries = [json.loads(line) for line in log.read_text(encoding="utf-8").splitlines()] + assert any(entry.get("phase") == "copilot_reconcile" for entry in entries) + + +def test_followup_entrypoint_loads_persisted_logfire_settings( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + config.write_logfire_settings(LogfireSettings(enabled=True, token="live-token")) + seen: list[Config] = [] + + def fake_run(loaded: Config, _paths: SourcePaths, _native_id: str, _generation: str) -> None: + seen.append(loaded) + + monkeypatch.setattr("thirdeye.platforms.copilot.followup._run", fake_run) + monkeypatch.setattr( + sys, + "argv", + [ + "followup", + "--source-home", + paths["home"], + "--session-id", + NATIVE_SESSION_ID, + "--config-root", + str(config.root), + "--generation", + "deadbeef", + ], + ) + followup_main() + + assert len(seen) == 1 + assert seen[0].logfire.enabled is True + assert seen[0].logfire.token == "live-token" + + def test_watch_activation_reconciles_archived_sessions( monkeypatch: pytest.MonkeyPatch, copilot_env: tuple[Config, SourcePaths], @@ -313,7 +424,9 @@ def test_watch_activation_reconciles_archived_sessions( activation: list[bool] = [] per_session: list[str] = [] - def fake_sync(_config: Config, _paths: SourcePaths, *, session_id: str | None = None) -> SyncResult: + def fake_sync( + _config: Config, _paths: SourcePaths, *, session_id: str | None = None + ) -> SyncResult: return _empty_result() def fake_reconcile_archived( @@ -335,7 +448,16 @@ def fake_reconcile_one( include_history: bool = False, ) -> dict[str, int]: per_session.append(native_session_id) - return {"events": 0, "usage": 0, "turns": 0, "exports": 0, "pending": 0, "ambiguous": 0, "conflicting": 0, "errors": 0} + return { + "events": 0, + "usage": 0, + "turns": 0, + "exports": 0, + "pending": 0, + "ambiguous": 0, + "conflicting": 0, + "errors": 0, + } monkeypatch.setattr(watch_mod, "sync", fake_sync) monkeypatch.setattr(watch_mod, "reconcile_archived_sessions", fake_reconcile_archived) @@ -348,3 +470,114 @@ def fake_reconcile_one( assert activation == [True] assert per_session == [] + + +def test_status_counts_delivery_claim_as_delivered_not_queued( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + stored = _seed_archive(config, paths) + directory = session_dir(config.root, PLATFORM_NAME, stored) + + def _record(state: dict[str, Any]) -> dict[str, Any]: + updated, _, _ = record_placement( + state, + accounting_id="acct-1", + destination="session-accounting-span", + span_id="accounting:stored:acct-1", + usage={}, + delivered=False, + ) + return updated + + update_export_state(config, stored, _record) + claim = delivery_claim_path(directory, "acct-1") + claim.parent.mkdir(parents=True, exist_ok=True) + claim.write_text("sent", encoding="utf-8") + + status = capture_status(config, paths) + export = status["sessions"][0]["export"] + assert export["delivered"] >= 1 + assert export["queued"] == 0 + + +def test_status_counts_failed_accounting_job_as_error_backlog( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + stored = _seed_archive(config, paths) + span_id = f"accounting:{stored}:acct-fail" + path = job_path(config.root, span_id) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps( + { + "job_id": span_id, + "kind": "session_accounting", + "session_id": stored, + "accounting_id": "acct-fail", + "state": "failed", + "attempt": 5, + "last_error": "TimeoutError: collector stalled", + } + ), + encoding="utf-8", + ) + + status = capture_status(config, paths) + export = status["sessions"][0]["export"] + assert export["errors"] >= 1 + assert export["queued"] >= 1 + assert export["delivered"] == 0 + + +def test_status_counts_turn_job_and_sent_claim( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + stored = _seed_archive(config, paths) + directory = session_dir(config.root, PLATFORM_NAME, stored) + jobs = otel_jobs_dir(config.root) + jobs.mkdir(parents=True, exist_ok=True) + (jobs / "turn-queued.json").write_text( + json.dumps({"kind": "turn", "session_id": stored, "job_id": "turn-queued"}), + encoding="utf-8", + ) + sent_dir = directory / "otel-turns-sent" + sent_dir.mkdir(parents=True, exist_ok=True) + (sent_dir / "delivered.json").write_text("sent", encoding="utf-8") + + status = capture_status(config, paths) + export = status["sessions"][0]["export"] + assert export["queued"] >= 1 + assert export["delivered"] >= 1 + + +def test_status_exposes_persisted_reconcile_last_error( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + stored = _seed_archive(config, paths) + + monkeypatch.setattr( + "thirdeye.platforms.copilot.runtime.reconcile_archive", + lambda *_args, **_kwargs: { + "events": 0, + "usage": 0, + "turns": 0, + "exports": 0, + "pending": 0, + "ambiguous": 0, + "conflicting": 0, + "errors": 1, + }, + ) + reconcile_session(config, paths, NATIVE_SESSION_ID) + + status = capture_status(config, paths) + export = status["sessions"][0]["export"] + assert export["errors"] >= 1 + assert export["last_error"]["errors"] == 1 + assert any(error.get("kind") == "copilot_reconcile_error" for error in status["errors"]) + assert stored in {session["stored_session_id"] for session in status["sessions"]} diff --git a/tests/platforms/copilot/test_watch.py b/tests/platforms/copilot/test_watch.py index 41a8a03..48a1471 100644 --- a/tests/platforms/copilot/test_watch.py +++ b/tests/platforms/copilot/test_watch.py @@ -45,6 +45,24 @@ def _empty_result(**overrides: int) -> SyncResult: return result +def _stub_reconcile(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(watch_mod, "reconcile_archived_sessions", lambda *_args, **_kwargs: {}) + monkeypatch.setattr( + watch_mod, + "reconcile_session", + lambda *_args, **_kwargs: { + "events": 0, + "usage": 0, + "turns": 0, + "exports": 0, + "pending": 0, + "ambiguous": 0, + "conflicting": 0, + "errors": 0, + }, + ) + + def _write_transcript(home: Path, native_id: str, *, events_path: Path | None = None) -> Path: session_dir = home / "session-state" / native_id session_dir.mkdir(parents=True, exist_ok=True) @@ -201,8 +219,7 @@ def stop_immediately(_interval: float) -> None: raise KeyboardInterrupt monkeypatch.setattr(watch_mod, "sync", tracking_sync) - monkeypatch.setattr(watch_mod, "reconcile_archived_sessions", lambda *_args, **_kwargs: {}) - monkeypatch.setattr(watch_mod, "reconcile_session", lambda *_args, **_kwargs: _empty_result()) + _stub_reconcile(monkeypatch) monkeypatch.setattr(watch_mod, "_SLEEP", stop_immediately) watch(config, paths, interval=0.1) @@ -230,6 +247,7 @@ def sleep_then_interrupt(_interval: float) -> None: raise KeyboardInterrupt monkeypatch.setattr(watch_mod, "sync", tracking_sync) + _stub_reconcile(monkeypatch) monkeypatch.setattr(watch_mod, "_SLEEP", sleep_then_interrupt) watch(config, paths, interval=0.1) @@ -317,6 +335,7 @@ def mutate_database(_interval: float) -> None: raise KeyboardInterrupt monkeypatch.setattr(watch_mod, "sync", tracking_sync) + _stub_reconcile(monkeypatch) monkeypatch.setattr(watch_mod, "_SLEEP", mutate_database) watch(config, paths, interval=0.1) @@ -345,6 +364,7 @@ def enqueue_during_poll(_interval: float) -> None: raise KeyboardInterrupt monkeypatch.setattr(watch_mod, "sync", tracking_sync) + _stub_reconcile(monkeypatch) monkeypatch.setattr(watch_mod, "_SLEEP", enqueue_during_poll) watch(config, paths, interval=0.1) @@ -377,6 +397,7 @@ def three_poll_cycles(_interval: float) -> None: raise KeyboardInterrupt monkeypatch.setattr(watch_mod, "sync", flaky_sync) + _stub_reconcile(monkeypatch) monkeypatch.setattr(watch_mod, "_SLEEP", three_poll_cycles) watch(config, paths, interval=0.1) @@ -391,6 +412,7 @@ def test_watch_exits_cleanly_on_keyboard_interrupt( config, paths = copilot_env monkeypatch.setattr(watch_mod, "sync", lambda *args, **kwargs: _empty_result()) + _stub_reconcile(monkeypatch) monkeypatch.setattr( watch_mod, "_SLEEP", lambda _interval: (_ for _ in ()).throw(KeyboardInterrupt) ) @@ -470,6 +492,7 @@ def delete_then_poll(_interval: float) -> None: raise KeyboardInterrupt monkeypatch.setattr(watch_mod, "sync", tracking_sync) + _stub_reconcile(monkeypatch) monkeypatch.setattr(watch_mod, "_SLEEP", delete_then_poll) watch(config, paths, interval=0.1) @@ -502,6 +525,7 @@ def delete_then_recreate(_interval: float) -> None: raise KeyboardInterrupt monkeypatch.setattr(watch_mod, "sync", tracking_sync) + _stub_reconcile(monkeypatch) monkeypatch.setattr(watch_mod, "_SLEEP", delete_then_recreate) watch(config, paths, interval=0.1) From 3d110865132ea09753b1c53f8bc84e3cf7e58dd0 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Sun, 13 Sep 2026 11:13:19 -0700 Subject: [PATCH 83/88] Report failed whole-turn Copilot exports from the worker error log. Generic turn jobs are deleted before delivery, so status must inspect usage-errors.jsonl or a failed export looks like success. Co-authored-by: Cursor --- src/thirdeye/platforms/copilot/status.py | 51 +++++++++++++++++++++++- tests/platforms/copilot/test_runtime.py | 42 +++++++++++++++++++ 2 files changed, 91 insertions(+), 2 deletions(-) diff --git a/src/thirdeye/platforms/copilot/status.py b/src/thirdeye/platforms/copilot/status.py index e138940..d4bb2d7 100644 --- a/src/thirdeye/platforms/copilot/status.py +++ b/src/thirdeye/platforms/copilot/status.py @@ -8,7 +8,7 @@ from typing import Any from thirdeye.config import Config -from thirdeye.paths import otel_jobs_dir, platform_dir +from thirdeye.paths import otel_jobs_dir, platform_dir, usage_log_path from thirdeye.reader import SessionReader from . import export_transport @@ -32,6 +32,7 @@ "copilot_database_read_failed", } ) +_WORKER_TURN_KINDS = frozenset({"turn", "spans", "subagent_turn"}) def _error_capability(path: Path, *, exists: bool, reason: str) -> dict[str, Any]: @@ -256,8 +257,49 @@ def _iter_json_jobs(directory: Path) -> list[dict[str, Any]]: return payloads +def _worker_kind(message: object) -> str: + text = str(message or "") + prefix = "kind=" + if not text.startswith(prefix): + return "" + return text[len(prefix) :].split(None, 1)[0] + + +def _iter_worker_turn_failures(config: Config, stored_session_id: str) -> list[dict[str, Any]]: + """Read deleted whole-turn export failures from the worker error log. + + Generic turn/spans/subagent jobs are unlinked before delivery. A crash or + export failure therefore leaves no job and no sent claim; the durable + breadcrumb is ``usage-errors.jsonl``. + """ + + path = usage_log_path(config.root) + try: + lines = path.read_text(encoding="utf-8").splitlines() + except (OSError, UnicodeDecodeError): + return [] + failures: list[dict[str, Any]] = [] + for line in lines: + if not line.strip(): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(entry, dict): + continue + if entry.get("phase") != "otel_worker_export_failed": + continue + if entry.get("session_id") != stored_session_id: + continue + if _worker_kind(entry.get("message")) not in _WORKER_TURN_KINDS: + continue + failures.append(entry) + return failures + + def _export_health(config: Config, stored_session_id: str, directory: Path) -> dict[str, Any]: - """Summarize queue/delivery health from claims and jobs, not ledger intent.""" + """Summarize queue/delivery health from claims, jobs, and worker logs.""" ledger = load_export_state(config, stored_session_id) placements = ledger.get("placements") if isinstance(ledger.get("placements"), dict) else {} @@ -341,6 +383,11 @@ def _export_health(config: Config, stored_session_id: str, directory: Path) -> d if payload.get("last_error"): errored.add(f"otel:{payload.get('job_id') or id(payload)}") + for index, entry in enumerate(_iter_worker_turn_failures(config, stored_session_id)): + key = f"otel-fail:{entry.get('ts') or index}:{entry.get('message')}" + queued.add(key) + errored.add(key) + queued -= delivered runtime_status = load_runtime_status(config, stored_session_id) diff --git a/tests/platforms/copilot/test_runtime.py b/tests/platforms/copilot/test_runtime.py index e04c8f7..65498d8 100644 --- a/tests/platforms/copilot/test_runtime.py +++ b/tests/platforms/copilot/test_runtime.py @@ -36,6 +36,7 @@ from thirdeye.platforms.copilot.state import state_path from thirdeye.platforms.copilot.status import capture_status from thirdeye.platforms.copilot.types import SourceBatch, SourcePaths, SourceRecord, SyncResult +from thirdeye.usage.errlog import log_capture_error FIXTURES = Path(__file__).parent / "fixtures" NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" @@ -553,6 +554,47 @@ def test_status_counts_turn_job_and_sent_claim( assert export["delivered"] >= 1 +def test_status_counts_failed_turn_export_from_worker_log( + copilot_env: tuple[Config, SourcePaths], +) -> None: + """A failed whole-turn export deletes the job before delivery. + + The only remaining evidence is usage-errors.jsonl. Status must still + report backlog and errors without a surviving job or sent claim. + """ + + config, paths = copilot_env + stored = _seed_archive(config, paths) + log_capture_error( + thirdeye_home=config.root, + phase="otel_worker_export_failed", + level="error", + platform=PLATFORM_NAME, + session_id=stored, + error=RuntimeError("logfire flush failed"), + message="kind=turn", + ) + log_capture_error( + thirdeye_home=config.root, + phase="otel_worker_export_failed", + level="error", + platform=PLATFORM_NAME, + session_id="copilot-other-session", + error=RuntimeError("other session"), + message="kind=turn", + ) + + jobs = otel_jobs_dir(config.root) + assert not jobs.exists() or not any(path.suffix == ".json" for path in jobs.iterdir()) + assert not (session_dir(config.root, PLATFORM_NAME, stored) / "otel-turns-sent").exists() + + status = capture_status(config, paths) + export = status["sessions"][0]["export"] + assert export["errors"] == 1 + assert export["queued"] == 1 + assert export["delivered"] == 0 + + def test_status_exposes_persisted_reconcile_last_error( copilot_env: tuple[Config, SourcePaths], monkeypatch: pytest.MonkeyPatch, From b8a68a00525924ab43c39ba310f2de4a28abf7f4 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Sun, 13 Sep 2026 16:47:14 -0700 Subject: [PATCH 84/88] docs: document Copilot V2 reconciliation --- README.md | 42 +++++-- docs/copilot-capture.md | 140 +++++++++++++++++++++ tests/platforms/copilot/fixtures/README.md | 24 +++- 3 files changed, 195 insertions(+), 11 deletions(-) create mode 100644 docs/copilot-capture.md diff --git a/README.md b/README.md index 0100d21..fd952d5 100644 --- a/README.md +++ b/README.md @@ -84,33 +84,55 @@ thirdeye add --claude # also: --cursor, --codex, --copilot To detach: `thirdeye remove --claude` (also `--cursor`, `--codex`, `--copilot`). -## Copilot CLI V1 +## Copilot CLI capture -GitHub Copilot CLI capture is a V1 immutable raw archive. V2 (reconstructed -turns, usage accounting, and OTel export) is out of scope. V1 does not export -Copilot content. +GitHub Copilot CLI capture keeps a durable, local V1 archive and derives V2 +views from that archive. V1 source records are immutable: V2 never rewrites +them, and it can rebuild its derived turns, usage accounting, and export queue +after the original Copilot files have gone away. ```bash thirdeye add --copilot thirdeye copilot status --source-home "$COPILOT_HOME" thirdeye copilot sync --source-home "$COPILOT_HOME" +thirdeye copilot reconcile +thirdeye copilot reconcile --session-id --rebuild thirdeye copilot watch --source-home "$COPILOT_HOME" --interval 1 thirdeye remove --copilot ``` `--source-home` is optional. Resolution is `--source-home`, then `COPILOT_HOME`, -then `~/.copilot`. V1 reads only that home's `session-state/**/events.jsonl`, +then `~/.copilot`. Capture reads only that home's `session-state/**/events.jsonl`, `workspace.yaml`, and `session-store.db` (`sessions`, `turns`, `assistant_usage_events`). It does not read credentials, token-bearing config, or other Copilot files. `thirdeye add --copilot` writes user-level hooks at `$COPILOT_HOME/hooks/thirdeye.json` (default `~/.copilot/hooks/thirdeye.json`). -The 1.0.83 live probe used repository hooks; V1 does not. `watch` is an explicit -foreground poller and is not started by add or setup. Captured content has the -same local sensitivity as other thirdeye sessions and is not exported in V1. -`sync` and `status` print recoverable source diagnostics (locations and reasons, -not prompt bodies). Passing unit tests is not live certification of Copilot CLI. +The 1.0.83 live probe used repository hooks; user-level installation is not a +claim that every Copilot runtime invokes them. `watch` is an explicit foreground +poller and is not started by add or setup. It can import transcripts and SQLite +rows even when hooks are absent. `sync` and `status` print recoverable source +diagnostics (locations and reasons, not prompt bodies). + +Normal `sync` and `reconcile` refresh local V2 projections only. To queue +already-completed retained history for remote delivery, opt in explicitly: + +```bash +thirdeye copilot sync --export --source-home "$COPILOT_HOME" +thirdeye copilot reconcile --export +``` + +Without that opt-in, V2 records an eligibility boundary: interactions already +complete when export is first activated remain local-only, while interactions +that complete afterward may be queued by watch/hooks when remote export is +configured. `--rebuild` resets only reproducible derived state; it preserves +the raw archive and the separate delivery ledger. See +[Copilot CLI capture and reconciliation](docs/copilot-capture.md) for the +archive schema, attribution rules, correction behavior, and delivery limits. + +Passing unit tests is not live certification of Copilot CLI, native VS Code, or +Copilot cloud integrations. ## Read your history diff --git a/docs/copilot-capture.md b/docs/copilot-capture.md new file mode 100644 index 0000000..ac2944c --- /dev/null +++ b/docs/copilot-capture.md @@ -0,0 +1,140 @@ +# Copilot CLI capture and reconciliation + +thirdeye's Copilot integration is local, archive-first CLI capture. It supports +GitHub Copilot CLI recordings; it does not claim support for native VS Code, +Copilot cloud history, or Copilot as an evaluator/Ask backend. + +## V1 archive to V2 projections + +V1 capture stores immutable source envelopes in the normal local session Store. +Their event types are `copilot_transcript`, `copilot_database`, `copilot_hook`, +and `copilot_metadata`; each has a version-1 envelope and preserves its source +ID, native session ID, source timestamp when available, observation time, +payload, and locator. Transcript records retain native event IDs and content; +database records retain table/row/revision identities; hook records remain +observations rather than asserted tool executions. + +V2 is a separate, versioned derived state. It reads the retained archive rather +than the current Copilot home, so it can reprocess an older captured session +without launching an agent or retaining the original transcript/SQLite files. +It does not duplicate the raw records in default semantic views. + +```bash +# Rebuild every retained Copilot archive locally. +thirdeye copilot reconcile + +# Rebuild one archive's derived indexes only. +thirdeye copilot reconcile --session-id --rebuild +``` + +`--rebuild` replaces derived projections only. It does not delete V1 raw source +events, change stored-session/source identities, or reset export eligibility or +delivery history. + +## Raw and derived views + +Generic event commands continue to expose the retained raw evidence. Copilot +main-turn views, searches, evaluation inputs, and usage views consume V2's +normalized projections, so raw transcript/database/hook evidence does not show +up a second time as a semantic event. Completed main interactions are the +projected user-turn records; child-agent events and tools remain nested evidence +within their owning main interaction. + +The `status` command separates source capability/ingestion diagnostics from +projection and export state. In particular, pending, ambiguous, and conflicting +joins are local accounting states, and a configured or queued export is not a +successful remote delivery. + +## Source evidence and turn reconstruction + +For the observed Copilot CLI 1.0.83 corpus, a user interaction is identified by +`interactionId` with agent identity, not by a bare transcript `turnId` (which +resets). A child is attached through `agentId`, `parentToolCallId`, and the +parent task's `subagent.started.data.toolCallId`; interleaved transcript events +therefore remain in the correct subtree. Tool requests/executions are paired by +their invocation IDs, which distinguishes simultaneous identical calls. + +The retained external hook stream is supplementary. Its pre/post tool payloads +do not provide an invocation ID, prompt/stop hooks can use a child agent ID in +the session field, and its coverage need not equal transcript hook coverage. +It must not be used as proof of an exact tool pairing or as a session-ID-only +turn state machine. A prompt may also arrive before `SessionStart`. + +Unknown transcript schemas/events remain source evidence and searchable raw +records. A missing identity, missing completion, permission decision, abort, +compaction, or otherwise incomplete record produces an explicit pending or +diagnostic result rather than an invented completed turn. + +## Usage accounting and attribution + +`assistant_usage_events` database rows are the primary per-call accounting +source. Input counts include cache tokens when supplied; missing usage stays +missing rather than becoming zero. Checkpoints and shutdown totals validate the +database ledger but never add a second charge. Auxiliary `model.*` title +generation is classified separately and cannot inflate conversation totals. +Unknown model providers remain `unknown`; Copilot nano-AI-unit billing is kept +separate from any estimated USD model price. + +Each logical database call has a stable accounting identity across revisions. +An authoritative row correction replaces its local derived `UsageRow`; a +generation/row-ID reuse or incompatible correction is quarantined as a +conflict, not counted as another call. Delayed rows remain eligible for a later +reconciliation. + +The observed database schema has user `turn_index`, agent ID, parent tool-call +ID, model, ordering, finish/tool evidence, and token/billing/latency fields, +but no shared assistant-message/provider-call ID. As a result, attribution is +deliberately conservative: + +- `matched` is direct only with direct evidence, or inferred only when the + interaction, agent, model, order, and finish/tool evidence form one uniquely + consistent candidate. Its evidence is retained with the attribution. +- `pending` means more source evidence may resolve ownership. +- `ambiguous` lists competing candidates and chooses none. +- `conflicting` quarantines incompatible revision or join evidence. + +An unmatched terminal usage row still appears in local accounting. It can be +represented as an explicit user-turn/agent accounting span, or as session-level +accounting when no user-turn ownership is known; thirdeye never fabricates a +user turn merely to place tokens. + +## Export eligibility, retries, and corrections + +Local reconciliation never exports history by default. `sync --export` or +`reconcile --export` explicitly opts the selected completed retained history +into export eligibility. When live export is configured, first activation also +records a durable boundary: already-terminal history stays local-only, while +an interaction open at activation becomes eligible if it completes later. +The boundary survives restart and derived-state rebuild. + +Export assembly writes durable, deterministic local jobs. Matched accounting is +placed on its chat span; unmatched accounting uses one explicit accounting span. +The placement ledger prevents a logical token identity from being emitted in +both locations. If fallback accounting has already been delivered, a later +local match cannot re-export those tokens on the chat span. Pending rows stay +retryable; a correction after confirmed delivery is reported as an accounting +conflict rather than silently adding a new charge. + +Remote OpenTelemetry delivery is not transactionally exactly once. A process +can crash after a remote flush and before durable acknowledgement, so a retry +may resend a deterministic span. Durable jobs, deterministic IDs, and delivery +claims minimize that window and allow recovery, but cannot prove remote receipt +in every crash case. `status` reports queued work and last errors separately +from confirmed delivery. + +## Known source limitations + +The checked-in 1.0.83 corpus establishes two main interactions, one explore +child, five tool executions, twenty external hook observations, and six +database usage calls. The six database calls reconcile to the observed shutdown +totals, but the lack of a native shared assistant-message/provider-call ID +means comprehensive local capture does not make every message join exact. + +Interactive, folder-trusted capture produced hooks in the probe. Earlier +non-interactive probes did not; the cause was not proven, so this is not a +general claim that non-interactive hooks are unsupported. Watch/import remains +useful without hooks. Synthetic schema-derived fixtures cover permission, +compaction, abort, unknown-version, delayed-row, revision, retry, identical +concurrent-tool, nested-child, and uncertain-attribution behavior. They are +deterministic regression inputs, not live validation. Optional future live +tests remain separate and require no CI credentials. diff --git a/tests/platforms/copilot/fixtures/README.md b/tests/platforms/copilot/fixtures/README.md index a43a168..50c95bb 100644 --- a/tests/platforms/copilot/fixtures/README.md +++ b/tests/platforms/copilot/fixtures/README.md @@ -1,6 +1,6 @@ # Copilot CLI capture probe -Captured 2026-09-10 on macOS using Copilot CLI 1.0.83, with automatic model selection (resolved to gpt-5.6-luna). These are observed fixtures for adapter design, not an implemented adapter or a regression test suite. +Captured 2026-09-10 on macOS using Copilot CLI 1.0.83, with automatic model selection (resolved to gpt-5.6-luna). These are observed fixtures for adapter design and archive-replay regression input; they are not live validation of another Copilot runtime, native VS Code, or cloud history. ## Successful scenario @@ -49,3 +49,25 @@ Persisted usage checkpoint events contain aggregate billing and cache-frontier d The raw transcript also contains two auxiliary model.model_call_success records for gpt-4o-mini session-title generation, with request/response content, usage and latency. These were omitted from the sanitized transcript. Do not treat them as the main-agent model-call history or add their usage to the six-row total without explicitly accounting for auxiliary calls. The database also has sessions, turns, checkpoints, session_files, session_refs and full-text search tables. Our session has two complete turns but no session_files rows despite four file reads, so the database's discovery/index tables do not replace raw tool execution events. Files beside the transcript include workspace.yaml (identity/repo/title), checkpoints/index.md (empty here), and rewind-file-snapshots/tracking.json (tracking metadata only here). + +## V2 replay use + +V2 tests construct a V1 archive from this corpus through capture APIs, then +reconcile that archive. They do not require a generated archive fixture or the +original Copilot home. The six `assistant_usage_events` rows are the primary +accounting ledger: checkpoint/shutdown values only validate their totals, and +the raw `model.*` title-generation records must remain auxiliary. + +The reconciler uses source identities, interaction/agent identifiers, tool-call +IDs, row revisions, and retained evidence. It does not treat bare transcript +`turnId`, hook timing, tool name, row count, or a nearest timestamp as proof of +an exact relationship. Since the observed SQLite rows lack a shared +assistant-message/provider-call ID, a uniquely consistent mapping is marked +inferred with evidence; competing candidates stay ambiguous and unresolved rows +stay pending or conflicting. + +`reconciliation-cases/` contains schema-derived synthetic fixtures for +permission outcomes, compaction, aborts, unknown versions, delayed/revised +database rows, retries, identical concurrent tools, nested children, and +uncertain attribution. Those cases are synthetic regression inputs, not claims +about live Copilot behavior. From b7a586878bf4e1fe564e03185ddf1f9a01aa035e Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Sun, 13 Sep 2026 16:54:11 -0700 Subject: [PATCH 85/88] docs: correct Copilot V2 view, hook, reuse, and export claims Align release notes with the implemented raw/projected split, child-hook identities, accounting reuse diagnostics, and export-eligibility triggers. Co-authored-by: Cursor --- README.md | 15 +++-- docs/copilot-capture.md | 69 ++++++++++++++-------- tests/platforms/copilot/fixtures/README.md | 9 ++- 3 files changed, 60 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index fd952d5..0278968 100644 --- a/README.md +++ b/README.md @@ -115,17 +115,22 @@ poller and is not started by add or setup. It can import transcripts and SQLite rows even when hooks are absent. `sync` and `status` print recoverable source diagnostics (locations and reasons, not prompt bodies). -Normal `sync` and `reconcile` refresh local V2 projections only. To queue -already-completed retained history for remote delivery, opt in explicitly: +Normal `sync` and `reconcile` refresh local V2 projections only; they do not +initialize export eligibility. To mark already-completed retained history as +export-eligible, opt in explicitly: ```bash thirdeye copilot sync --export --source-home "$COPILOT_HOME" thirdeye copilot reconcile --export ``` -Without that opt-in, V2 records an eligibility boundary: interactions already -complete when export is first activated remain local-only, while interactions -that complete afterward may be queued by watch/hooks when remote export is +Watch and hook follow-up activate live-style export without that history +opt-in. Their first activation records a durable eligibility boundary even if +remote export is not presently configured: interactions already complete stay +local-only, while interactions still open at activation may be queued later if +they complete and remote export is configured. `--export` instead activates +with retained completed history included, or later removes those identities +from an existing boundary. Jobs are dispatched only when remote export is configured. `--rebuild` resets only reproducible derived state; it preserves the raw archive and the separate delivery ledger. See [Copilot CLI capture and reconciliation](docs/copilot-capture.md) for the diff --git a/docs/copilot-capture.md b/docs/copilot-capture.md index ac2944c..e72bcad 100644 --- a/docs/copilot-capture.md +++ b/docs/copilot-capture.md @@ -33,12 +33,15 @@ delivery history. ## Raw and derived views -Generic event commands continue to expose the retained raw evidence. Copilot -main-turn views, searches, evaluation inputs, and usage views consume V2's -normalized projections, so raw transcript/database/hook evidence does not show -up a second time as a semantic event. Completed main interactions are the -projected user-turn records; child-agent events and tools remain nested evidence -within their owning main interaction. +Generic event commands (`events`, `show`, `tail`, `search`) continue to expose +the retained raw Store evidence. Session-scoped evaluation timelines also read +those raw events. Copilot user-turn views, turn-scoped dataset inputs, and +usage dashboards consume V2's derived projections: completed main interactions +via the projected turn index, and per-call usage via the rewritten `usage.jsonl` +sidecar. Raw transcript, database, and hook records therefore remain searchable +as captured, and they do not appear a second time as semantic events in default +turn views. Child-agent events and tools remain nested evidence within their +owning main interaction. The `status` command separates source capability/ingestion diagnostics from projection and export state. In particular, pending, ambiguous, and conflicting @@ -55,15 +58,20 @@ therefore remain in the correct subtree. Tool requests/executions are paired by their invocation IDs, which distinguishes simultaneous identical calls. The retained external hook stream is supplementary. Its pre/post tool payloads -do not provide an invocation ID, prompt/stop hooks can use a child agent ID in -the session field, and its coverage need not equal transcript hook coverage. -It must not be used as proof of an exact tool pairing or as a session-ID-only -turn state machine. A prompt may also arrive before `SessionStart`. +do not provide an invocation ID, and its coverage need not equal transcript +hook coverage. Child `userPromptSubmitted` and `agentStop` hooks put the child +ID in `sessionId`; `subagentStart`/`subagentStop` retain the parent session ID +and expose the child separately as `agentId`. Hooks must not be used as proof +of an exact tool pairing or as a session-ID-only turn state machine. A prompt +may also arrive before `SessionStart`. Unknown transcript schemas/events remain source evidence and searchable raw -records. A missing identity, missing completion, permission decision, abort, -compaction, or otherwise incomplete record produces an explicit pending or -diagnostic result rather than an invented completed turn. +records. Permission decisions and compaction produce normalized semantic +events; they are not inferred as tool executions or extra usage calls. +`assistant.abort` terminates a reconstructed turn with `status="interrupted"`. +A missing identity, missing completion, unknown version, or otherwise +incomplete record produces an explicit pending or diagnostic result rather +than an invented completed turn. ## Usage accounting and attribution @@ -75,11 +83,14 @@ generation is classified separately and cannot inflate conversation totals. Unknown model providers remain `unknown`; Copilot nano-AI-unit billing is kept separate from any estimated USD model price. -Each logical database call has a stable accounting identity across revisions. -An authoritative row correction replaces its local derived `UsageRow`; a -generation/row-ID reuse or incompatible correction is quarantined as a -conflict, not counted as another call. Delayed rows remain eligible for a later -reconciliation. +Each logical database call has a stable accounting identity across revisions +of the same generation and row. An authoritative row correction replaces its +local derived `UsageRow` under that identity. Database generation/row-ID reuse +is a different call: V2 keeps both logical identities and emits a +`usage_row_id_reuse` warning rather than relabeling the first call. An +incompatible correction of one logical call is quarantined as +`usage_revision_conflict` and is not counted as another charge. Delayed rows +remain eligible for a later reconciliation. The observed database schema has user `turn_index`, agent ID, parent tool-call ID, model, ordering, finish/tool evidence, and token/billing/latency fields, @@ -91,7 +102,9 @@ deliberately conservative: consistent candidate. Its evidence is retained with the attribution. - `pending` means more source evidence may resolve ownership. - `ambiguous` lists competing candidates and chooses none. -- `conflicting` quarantines incompatible revision or join evidence. +- `conflicting` quarantines incompatible join evidence. Incompatible database + revisions are a separate `usage_revision_conflict` diagnostic, not this + attribution status. An unmatched terminal usage row still appears in local accounting. It can be represented as an explicit user-turn/agent accounting span, or as session-level @@ -100,12 +113,18 @@ user turn merely to place tokens. ## Export eligibility, retries, and corrections -Local reconciliation never exports history by default. `sync --export` or -`reconcile --export` explicitly opts the selected completed retained history -into export eligibility. When live export is configured, first activation also -records a durable boundary: already-terminal history stays local-only, while -an interaction open at activation becomes eligible if it completes later. -The boundary survives restart and derived-state rebuild. +Local reconciliation never exports history by default. Plain `sync` and +`reconcile` refresh local projections only and do not initialize the export +ledger. `sync --export` or `reconcile --export` calls export assembly with +history included: that first activation marks the selected completed retained +history as eligible. Watch and hook follow-up also call export assembly, but +without the history opt-in. Their first activation records a durable boundary +even when remote export is not presently configured: already-terminal history +stays local-only, while an interaction open at activation becomes eligible if +it completes later. A later `--export` can remove those identities from an +existing boundary. The boundary survives restart and derived-state rebuild. +Jobs are dispatched only when remote export is configured; a configured or +queued export is not a successful delivery. Export assembly writes durable, deterministic local jobs. Matched accounting is placed on its chat span; unmatched accounting uses one explicit accounting span. diff --git a/tests/platforms/copilot/fixtures/README.md b/tests/platforms/copilot/fixtures/README.md index 50c95bb..653f486 100644 --- a/tests/platforms/copilot/fixtures/README.md +++ b/tests/platforms/copilot/fixtures/README.md @@ -64,10 +64,13 @@ IDs, row revisions, and retained evidence. It does not treat bare transcript an exact relationship. Since the observed SQLite rows lack a shared assistant-message/provider-call ID, a uniquely consistent mapping is marked inferred with evidence; competing candidates stay ambiguous and unresolved rows -stay pending or conflicting. +stay pending or conflicting. Database generation/row-ID reuse keeps both +logical identities and emits `usage_row_id_reuse`; incompatible revisions of +one call are `usage_revision_conflict`. `reconciliation-cases/` contains schema-derived synthetic fixtures for permission outcomes, compaction, aborts, unknown versions, delayed/revised database rows, retries, identical concurrent tools, nested children, and -uncertain attribution. Those cases are synthetic regression inputs, not claims -about live Copilot behavior. +uncertain attribution. Permission and compaction cases emit semantic events; +abort closes a reconstructed turn as `interrupted`. Those cases are synthetic +regression inputs, not claims about live Copilot behavior. From 34f33176f9a703942b0e06ce66fc811602a17f5b Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Sun, 13 Sep 2026 17:38:23 -0700 Subject: [PATCH 86/88] Close Copilot V2 review gaps in native billing, generation-reuse quarantine, and labeled synthetic cases. Native nano-AIU never reached exported spans or usage dashboards, reused row generations were billed twice, and undelivered same-span corrections left stale job bytes; align those paths plus unknown-version/identical-args fixtures and static acceptance with the digest. Co-authored-by: Cursor --- docs/copilot-capture.md | 20 +- src/thirdeye/platforms/copilot/events.py | 13 + src/thirdeye/platforms/copilot/export.py | 48 +++- src/thirdeye/platforms/copilot/projection.py | 33 ++- .../platforms/copilot/projection_store.py | 33 +++ src/thirdeye/platforms/copilot/turns.py | 23 ++ src/thirdeye/platforms/copilot/types.py | 2 +- src/thirdeye/platforms/copilot/usage.py | 65 +++-- src/thirdeye/web/routes/usage.py | 37 ++- src/thirdeye/web/templates/usage/global.html | 4 + src/thirdeye/web/templates/usage/session.html | 10 + tests/platforms/copilot/fixtures/README.md | 2 +- .../reconciliation-cases/attributions.json | 4 +- .../fixtures/reconciliation-cases/cases.json | 224 +++++++++++++++++- tests/platforms/copilot/test_attribution.py | 99 +++----- tests/platforms/copilot/test_export.py | 182 +++++++++++--- tests/platforms/copilot/test_tracing.py | 73 +++++- tests/platforms/copilot/test_usage.py | 22 +- tests/shared/copilot_projection_fixtures.py | 78 ++++++ tests/web/test_copilot_capture_views.py | 20 ++ tests/web/test_routes_usage.py | 1 + 21 files changed, 847 insertions(+), 146 deletions(-) diff --git a/docs/copilot-capture.md b/docs/copilot-capture.md index e72bcad..8da2ed5 100644 --- a/docs/copilot-capture.md +++ b/docs/copilot-capture.md @@ -86,8 +86,8 @@ separate from any estimated USD model price. Each logical database call has a stable accounting identity across revisions of the same generation and row. An authoritative row correction replaces its local derived `UsageRow` under that identity. Database generation/row-ID reuse -is a different call: V2 keeps both logical identities and emits a -`usage_row_id_reuse` warning rather than relabeling the first call. An +is a conflict: V2 quarantines both logical identities, drops their `UsageRow`s, +and emits a `usage_row_id_reuse` error rather than charging either call. An incompatible correction of one logical call is quarantined as `usage_revision_conflict` and is not counted as another charge. Delayed rows remain eligible for a later reconciliation. @@ -157,3 +157,19 @@ compaction, abort, unknown-version, delayed-row, revision, retry, identical concurrent-tool, nested-child, and uncertain-attribution behavior. They are deterministic regression inputs, not live validation. Optional future live tests remain separate and require no CI credentials. + +## V2 static acceptance + +This document is the V2 acceptance record for Copilot CLI archive capture. +It covers V1-to-V2 upgrade (immutable source envelopes, versioned derived +indexes, `--rebuild` without resetting the export ledger), cross-source +correctness (transcript/hook/database reconstruction without live Copilot +files), exact join evidence (no timestamp/row-count/tool-name proof), +accounting corrections (revision replacement vs generation-reuse quarantine, +pre-delivery job rewrite vs post-delivery conflict), history export opt-in +(`--export` defaulting false; enqueue is not delivery), and unresolved source +limitations (no native VS Code/cloud capture, no shared assistant-message ID, +hooks without invocation IDs). No pytest run is required for this static +acceptance, and it does not claim native VS Code or Copilot cloud support. + +VERDICT: PASS diff --git a/src/thirdeye/platforms/copilot/events.py b/src/thirdeye/platforms/copilot/events.py index 88b6ed9..363a4ee 100644 --- a/src/thirdeye/platforms/copilot/events.py +++ b/src/thirdeye/platforms/copilot/events.py @@ -14,6 +14,7 @@ from copy import deepcopy from typing import Any +from .constants import SOURCE_SCHEMA_VERSION from .types import ( EventClassification, NormalizedEvent, @@ -135,6 +136,15 @@ def _identity_attributes(record: SourceRecord) -> dict[str, Any]: } +def unknown_source_schema(record: SourceRecord) -> bool: + """Return whether the archived payload names a schema other than V1.""" + + payload = record.get("payload") + if not isinstance(payload, dict) or "schema_version" not in payload: + return False + return payload.get("schema_version") != SOURCE_SCHEMA_VERSION + + def _unknown(record: SourceRecord, native_type: str | None) -> list[NormalizedEvent]: attrs: dict[str, Any] = {"raw_payload": deepcopy(record.get("payload"))} if native_type is not None: @@ -150,6 +160,9 @@ def normalize_record(record: SourceRecord) -> list[NormalizedEvent]: if record.get("source_kind") not in _TRANSCRIPT_OR_HOOK: return [] + if unknown_source_schema(record): + return _unknown(record, record_type(record)) + native_type = record_type(record) data = record_data(record) identity = _identity_attributes(record) diff --git a/src/thirdeye/platforms/copilot/export.py b/src/thirdeye/platforms/copilot/export.py index a6bb645..e768af2 100644 --- a/src/thirdeye/platforms/copilot/export.py +++ b/src/thirdeye/platforms/copilot/export.py @@ -34,6 +34,7 @@ mark_turn_error, record_placement, update_export_state, + usage_digest, ) from .types import Projection @@ -41,6 +42,22 @@ _EXPORTABLE_ATTRIBUTIONS = frozenset({"matched", "ambiguous"}) +def _copilot_supplemental_export_attributes(candidate: dict[str, Any] | None) -> dict[str, Any]: + metrics = (candidate or {}).get("supplemental_metrics") or {} + attributes: dict[str, Any] = {} + if "total_nano_aiu" in metrics: + attributes["copilot.billing.nano_aiu"] = metrics["total_nano_aiu"] + if "duration_ms" in metrics: + attributes["copilot.latency.duration_ms"] = metrics["duration_ms"] + if "output_ttft_ms" in metrics: + attributes["copilot.latency.output_ttft_ms"] = metrics["output_ttft_ms"] + if "time_to_first_token_ms" in metrics: + attributes["copilot.latency.time_to_first_token_ms"] = metrics["time_to_first_token_ms"] + if "inter_token_latency_ms" in metrics: + attributes["copilot.latency.inter_token_latency_ms"] = metrics["inter_token_latency_ms"] + return attributes + + def _directory(config: Config, stored_session_id: str) -> Path: return session_dir(config.root, PLATFORM_NAME, stored_session_id) @@ -233,6 +250,8 @@ def update(state: dict[str, Any]) -> dict[str, Any]: existing = state.get("placements", {}).get(accounting_id) or {} old_span_id = existing.get("span_id") old_job_state: str | None = None + digest = usage_digest(usage) + usage_changed = existing.get("usage_digest") != digest if isinstance(old_span_id, str) and old_span_id and old_span_id != span_id: # Relocating to a different span: find out whether the job under # the *old* span id is provably inert before deciding it is safe @@ -241,6 +260,21 @@ def update(state: dict[str, Any]) -> dict[str, Any]: # at once it decides on the new destination below. status = export_transport.cancel(config.root, old_span_id) old_job_state = status.get("state") + elif ( + isinstance(old_span_id, str) + and old_span_id == span_id + and usage_changed + and not delivered + ): + # Same span, new usage, still undelivered: rewrite the queued + # payload. A claimed in-flight job cannot be overwritten. + current = export_transport.status(config.root, old_span_id) + current_state = (current or {}).get("state") + if current_state == "claimed": + old_job_state = "claimed" + elif current_state in {"queued", "retrying", "failed"}: + status = export_transport.cancel(config.root, old_span_id) + old_job_state = status.get("state") next_state, entry, accepted = record_placement( state, accounting_id=accounting_id, @@ -442,7 +476,9 @@ def queue_exports( item, turn_span_id=_turn_span_id(stored_session_id, owner_id), ) - if sent and _reflect_accounting_job_health(config, stored_session_id, accounting_id, span_id): + if sent and _reflect_accounting_job_health( + config, stored_session_id, accounting_id, span_id + ): queued += 1 elif not sent: _error_state(config, stored_session_id, accounting_id, "accounting job was not queued") @@ -450,6 +486,11 @@ def queue_exports( # Usage with no known user-turn owner is intentionally a session accounting # job. It never manufactures a prompt/turn merely to satisfy tracing. rows = {row.call_id: row.to_dict() for row in projection["usage_rows"]} + candidates = { + item["logical_call_id"]: item + for item in projection.get("accounting_candidates") or [] + if isinstance(item, dict) and isinstance(item.get("logical_call_id"), str) + } attached_ids = set(accounting) for attribution in projection["attributions"]: accounting_id = attribution["logical_call_id"] @@ -488,12 +529,15 @@ def queue_exports( "logical_call_id": accounting_id, "usage_source_id": attribution["usage_source_id"], "evidence": list(attribution["evidence"]), + **_copilot_supplemental_export_attributes(candidates.get(accounting_id)), }, } sent = export_transport.queue_session_accounting( config, directory, stored_session_id, meta.cwd, item ) - if sent and _reflect_accounting_job_health(config, stored_session_id, accounting_id, span_id): + if sent and _reflect_accounting_job_health( + config, stored_session_id, accounting_id, span_id + ): queued += 1 elif not sent: _error_state(config, stored_session_id, accounting_id, "accounting job was not queued") diff --git a/src/thirdeye/platforms/copilot/projection.py b/src/thirdeye/platforms/copilot/projection.py index 8726e23..f90cc37 100644 --- a/src/thirdeye/platforms/copilot/projection.py +++ b/src/thirdeye/platforms/copilot/projection.py @@ -7,7 +7,7 @@ from .attribution import join_usage from .tracing import build_semantics -from .types import Attribution, Projection, ProjectionDiagnostic, SourceRecord +from .types import AccountingCandidate, Attribution, Projection, ProjectionDiagnostic, SourceRecord from .usage import build_accounting @@ -34,7 +34,29 @@ def _find_turn(turns: list[dict[str, Any]], identity: str) -> dict[str, Any] | N return None -def _accounting_call(attribution: Attribution, row: object) -> dict[str, Any] | None: +def _copilot_supplemental_export_attributes( + candidate: AccountingCandidate | None, +) -> dict[str, Any]: + """Copy native Copilot billing/latency onto export attributes, never as USD.""" + + metrics = (candidate or {}).get("supplemental_metrics") or {} + attributes: dict[str, Any] = {} + if "total_nano_aiu" in metrics: + attributes["copilot.billing.nano_aiu"] = metrics["total_nano_aiu"] + if "duration_ms" in metrics: + attributes["copilot.latency.duration_ms"] = metrics["duration_ms"] + if "output_ttft_ms" in metrics: + attributes["copilot.latency.output_ttft_ms"] = metrics["output_ttft_ms"] + if "time_to_first_token_ms" in metrics: + attributes["copilot.latency.time_to_first_token_ms"] = metrics["time_to_first_token_ms"] + if "inter_token_latency_ms" in metrics: + attributes["copilot.latency.inter_token_latency_ms"] = metrics["inter_token_latency_ms"] + return attributes + + +def _accounting_call( + attribution: Attribution, row: object, candidate: AccountingCandidate | None +) -> dict[str, Any] | None: if row is None or not hasattr(row, "to_dict"): return None return { @@ -48,6 +70,7 @@ def _accounting_call(attribution: Attribution, row: object) -> dict[str, Any] | "usage_source_id": attribution["usage_source_id"], "join_kind": attribution["join_kind"], "evidence": list(attribution["evidence"]), + **_copilot_supplemental_export_attributes(candidate), }, } @@ -62,12 +85,15 @@ def build_projection( attributions = join_usage(semantic, accounting) turns = deepcopy(semantic["turns"]) rows = {row.call_id: row for row in accounting["usage_rows"]} + candidates = {item["logical_call_id"]: item for item in accounting["candidates"]} pending = [*semantic["pending"]] diagnostics: list[ProjectionDiagnostic] = [*semantic["diagnostics"], *accounting["diagnostics"]] for attribution in attributions: row = rows.get(attribution["logical_call_id"]) - accounting_call = _accounting_call(attribution, row) + accounting_call = _accounting_call( + attribution, row, candidates.get(attribution["logical_call_id"]) + ) if attribution["status"] == "matched": if attribution["join_kind"] == "inferred": diagnostics.append( @@ -110,6 +136,7 @@ def build_projection( "attributions": attributions, "pending": pending, "diagnostics": diagnostics, + "accounting_candidates": accounting["candidates"], }, {"semantic_state": semantic_state, "accounting_state": accounting_state}, ) diff --git a/src/thirdeye/platforms/copilot/projection_store.py b/src/thirdeye/platforms/copilot/projection_store.py index 0c3214c..8f77baa 100644 --- a/src/thirdeye/platforms/copilot/projection_store.py +++ b/src/thirdeye/platforms/copilot/projection_store.py @@ -569,6 +569,39 @@ def read_projected_turns(config: Config, stored_session_id: str) -> list[dict[st return json.loads(_canonical(values)) +def _collect_usage_labels(turn: dict[str, Any], labels: dict[str, dict[str, Any]]) -> None: + for call in _items(turn.get("accounting_calls")): + mapped = _mapping(call) + accounting_id = mapped.get("accounting_id") + attributes = _mapping(mapped.get("attributes")) + extra = { + key: value + for key, value in attributes.items() + if str(key).startswith("copilot.billing") or str(key).startswith("copilot.latency") + } + if isinstance(accounting_id, str) and extra: + labels[accounting_id] = extra + for child in _items(turn.get("subagents")): + if isinstance(child, dict): + _collect_usage_labels(child, labels) + + +def read_usage_labels(config: Config, stored_session_id: str) -> dict[str, dict[str, Any]]: + """Read Copilot native billing/latency labels from stored turn accounting.""" + + directory = _existing_session_dir(config, stored_session_id) + if directory is None: + return {} + with locked(projection_lock_path(directory), LockMode.EXCLUSIVE): + document = read_projection_document(directory) + turns = _mapping(_mapping(document.get("indexes")).get("turns")) + labels: dict[str, dict[str, Any]] = {} + for record in turns.values(): + if isinstance(record, dict): + _collect_usage_labels(_mapping(record.get("span")), labels) + return json.loads(_canonical(labels)) + + _STATUS_KEYS = ("events", "usage", "turns", "pending", "ambiguous", "conflicting", "errors") diff --git a/src/thirdeye/platforms/copilot/turns.py b/src/thirdeye/platforms/copilot/turns.py index 18705ec..defbdf7 100644 --- a/src/thirdeye/platforms/copilot/turns.py +++ b/src/thirdeye/platforms/copilot/turns.py @@ -21,6 +21,7 @@ record_type, source_reference, tool_call_id, + unknown_source_schema, ) from .types import CallCandidate, PendingItem, ProjectionDiagnostic, SourceRecord @@ -230,6 +231,28 @@ def drop_native_turn(agent: str | None, native_turn: Any, owner: str | None) -> for record in records: if record.get("source_kind") != "transcript": continue + if unknown_source_schema(record): + payload = record.get("payload") + version = payload.get("schema_version") if isinstance(payload, dict) else None + pending.append( + { + "id": f"pending:unknown-version:{record['source_id']}", + "kind": "missing_source_capability", + "reason": "unknown transcript schema_version", + "source_ids": [record["source_id"]], + "evidence": [f"schema_version:{version}"], + } + ) + diagnostics.append( + { + "code": "capability_gap", + "severity": "warning", + "message": "unknown transcript schema_version; raw payload retained", + "source_ids": [record["source_id"]], + "details": {"schema_version": version}, + } + ) + continue native_type = record_type(record) data = record_data(record) agent = agent_id(record) diff --git a/src/thirdeye/platforms/copilot/types.py b/src/thirdeye/platforms/copilot/types.py index 07e95bd..0eea005 100644 --- a/src/thirdeye/platforms/copilot/types.py +++ b/src/thirdeye/platforms/copilot/types.py @@ -218,7 +218,6 @@ class SourceSlice(TypedDict): "missing_source_capability", "missing_identity", "incomplete_tool_pair", - "delayed_row", ] DiagnosticCode = Literal[ @@ -498,6 +497,7 @@ class Projection(TypedDict): attributions: list[Attribution] pending: list[PendingItem] diagnostics: list[ProjectionDiagnostic] + accounting_candidates: NotRequired[list[AccountingCandidate]] class ProjectedTurnRecord(TypedDict): diff --git a/src/thirdeye/platforms/copilot/usage.py b/src/thirdeye/platforms/copilot/usage.py index 9f93657..08367be 100644 --- a/src/thirdeye/platforms/copilot/usage.py +++ b/src/thirdeye/platforms/copilot/usage.py @@ -444,9 +444,7 @@ def _seed_revision_sources( if logical_id in revision_sources: return previous = inherited_calls.get(logical_id) - revision_sources[logical_id] = ( - _prior_source_ids(previous) if isinstance(previous, dict) else [] - ) + revision_sources[logical_id] = _prior_source_ids(previous) if isinstance(previous, dict) else [] def _logical_call_entry( @@ -555,21 +553,20 @@ def build_accounting( revision_sources[logical_id].append(record["source_id"]) prior_digests[logical_id] = _metrics_digest(payload["row"]) - for key in sorted(row_generations): - generations = row_generations[key] - if len(generations) > 1: - table, primary_key = key.split("\x1f", 1) - diagnostics.append( - _diagnostic( - "usage_row_id_reuse", - "warning", - "database row ID was reused by a different database generation", - list(row_sources.get(key, [])), - table=table, - primary_key=primary_key, - generations=sorted(generations), - ) + reuse_keys = {key for key, generations in row_generations.items() if len(generations) > 1} + for key in sorted(reuse_keys): + table, primary_key = key.split("\x1f", 1) + diagnostics.append( + _diagnostic( + "usage_row_id_reuse", + "error", + "database row ID was reused by a different database generation; quarantined as conflict", + list(row_sources.get(key, [])), + table=table, + primary_key=primary_key, + generations=sorted(row_generations[key]), ) + ) candidates: list[AccountingCandidate] = [] usage_rows: list[UsageRow] = [] @@ -579,6 +576,28 @@ def build_accounting( for key, value in inherited_calls.items() if isinstance(key, str) and isinstance(value, dict) } + reused_logical_ids: set[str] = set() + for logical_id, call in logical_calls.items(): + table = call.get("table") if isinstance(call.get("table"), str) else _USAGE_TABLE + primary_key = call.get("primary_key") + if isinstance(primary_key, str) and _row_key(table, primary_key) in reuse_keys: + reused_logical_ids.add(logical_id) + for logical_id, (_record, revision, _selected_candidate, _digest) in selected.items(): + if _row_key(revision["table"], revision["primary_key"]) in reuse_keys: + reused_logical_ids.add(logical_id) + for logical_id in reused_logical_ids: + if logical_id in selected: + continue + previous = logical_calls.get(logical_id) + if not isinstance(previous, dict): + continue + old_metrics = _metrics_from_call(previous) + _add_metrics(accounted, old_metrics, sign=-1) + previous_agent = previous.get("agent_id") + old_agent = _agent_key(previous_agent if isinstance(previous_agent, str) else None) + if old_agent in accounted_by_agent: + _add_metrics(accounted_by_agent[old_agent], old_metrics, sign=-1) + logical_calls[logical_id] = {**previous, "metrics": {}, "quarantined": True} unknown_provider_ids: list[str] = [] for logical_id, (record, revision, candidate, previous_digest) in selected.items(): payload = record["payload"] @@ -594,6 +613,18 @@ def build_accounting( if old_agent in accounted_by_agent: _add_metrics(accounted_by_agent[old_agent], old_metrics, sign=-1) digest = _metrics_digest(row) + if logical_id in reused_logical_ids: + candidates.append(candidate) + logical_calls[logical_id] = _logical_call_entry( + logical_id, + revision, + record, + row, + candidate, + revision_sources[logical_id], + quarantined=True, + ) + continue if _row_is_incompatible(row): details: dict[str, Any] = { "logical_call_id": logical_id, diff --git a/src/thirdeye/web/routes/usage.py b/src/thirdeye/web/routes/usage.py index 54483ff..f037a52 100644 --- a/src/thirdeye/web/routes/usage.py +++ b/src/thirdeye/web/routes/usage.py @@ -15,6 +15,29 @@ from thirdeye.usage.read import iter_calls +def _copilot_usage_display(rows: list, labels: dict[str, dict]) -> list[dict]: + display = [] + for row in rows: + extra = labels.get(row.call_id, {}) + display.append( + { + "seq": row.seq, + "response_model": row.response_model, + "input_tokens": row.input_tokens, + "output_tokens": row.output_tokens, + "cache_read_input_tokens": row.cache_read_input_tokens, + "cache_creation_input_tokens": row.cache_creation_input_tokens, + "reasoning_output_tokens": row.reasoning_output_tokens, + "total_tokens": row.total_tokens, + "ts": row.ts, + "copilot_billing_nano_aiu": extra.get("copilot.billing.nano_aiu"), + "duration_ms": extra.get("copilot.latency.duration_ms"), + "output_ttft_ms": extra.get("copilot.latency.output_ttft_ms"), + } + ) + return display + + async def _session_usage(request: Request) -> HTMLResponse: prefix = request.path_params["sid"] store = request.app.state.store @@ -25,12 +48,24 @@ async def _session_usage(request: Request) -> HTMLResponse: raise HTTPException(status_code=404, detail=str(e)) from e sdir = session_dir(config.root, platform, sid) rows = list(iter_calls(sdir)) + labels = {} + if platform == "copilot": + from thirdeye.platforms.copilot.projection_store import read_usage_labels + + labels = read_usage_labels(config, sid) + rows = _copilot_usage_display(rows, labels) aggregate = store.stats(session_id=sid) templates = request.app.state.templates return templates.TemplateResponse( request, "usage/session.html", - {"rows": rows, "aggregate": aggregate, "sid": sid, "platform": platform}, + { + "rows": rows, + "aggregate": aggregate, + "sid": sid, + "platform": platform, + "show_copilot_labels": platform == "copilot", + }, ) diff --git a/src/thirdeye/web/templates/usage/global.html b/src/thirdeye/web/templates/usage/global.html index e028f4a..4826500 100644 --- a/src/thirdeye/web/templates/usage/global.html +++ b/src/thirdeye/web/templates/usage/global.html @@ -35,6 +35,10 @@
{{ report.totals.input_tokens }}
output tokens
{{ report.totals.output_tokens }}
+ {% if filters.platform == "copilot" %} +
copilot native billing (nano-AIU)
+
session usage
+ {% endif %} diff --git a/src/thirdeye/web/templates/usage/session.html b/src/thirdeye/web/templates/usage/session.html index ede284e..a73da53 100644 --- a/src/thirdeye/web/templates/usage/session.html +++ b/src/thirdeye/web/templates/usage/session.html @@ -32,6 +32,11 @@

session usage

 + {% if show_copilot_labels %} + + + + {% endif %} @@ -46,6 +51,11 @@

session usage

+ {% if show_copilot_labels %} + + + + {% endif %} diff --git a/tests/platforms/copilot/fixtures/README.md b/tests/platforms/copilot/fixtures/README.md index 653f486..5999e34 100644 --- a/tests/platforms/copilot/fixtures/README.md +++ b/tests/platforms/copilot/fixtures/README.md @@ -64,7 +64,7 @@ IDs, row revisions, and retained evidence. It does not treat bare transcript an exact relationship. Since the observed SQLite rows lack a shared assistant-message/provider-call ID, a uniquely consistent mapping is marked inferred with evidence; competing candidates stay ambiguous and unresolved rows -stay pending or conflicting. Database generation/row-ID reuse keeps both +stay pending or conflicting. Database generation/row-ID reuse quarantines both logical identities and emits `usage_row_id_reuse`; incompatible revisions of one call are `usage_revision_conflict`. diff --git a/tests/platforms/copilot/fixtures/reconciliation-cases/attributions.json b/tests/platforms/copilot/fixtures/reconciliation-cases/attributions.json index e07b66a..23295bf 100644 --- a/tests/platforms/copilot/fixtures/reconciliation-cases/attributions.json +++ b/tests/platforms/copilot/fixtures/reconciliation-cases/attributions.json @@ -30,8 +30,8 @@ "join_kind": null, "evidence": [ "logical_call_id:copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:14", - "turn_index:0", - "delayed_row:true" + "interaction_id:6d2b89fd-a653-430c-b532-b0936d72eb42", + "turn_index:0" ] }, "ambiguous": { diff --git a/tests/platforms/copilot/fixtures/reconciliation-cases/cases.json b/tests/platforms/copilot/fixtures/reconciliation-cases/cases.json index 2055b1e..cd46a50 100644 --- a/tests/platforms/copilot/fixtures/reconciliation-cases/cases.json +++ b/tests/platforms/copilot/fixtures/reconciliation-cases/cases.json @@ -935,8 +935,8 @@ "join_kind": null, "evidence": [ "logical_call_id:copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:14", - "turn_index:0", - "delayed_row:true" + "interaction_id:6d2b89fd-a653-430c-b532-b0936d72eb42", + "turn_index:0" ] } ] @@ -1518,5 +1518,225 @@ } ] } + }, + "unknown_version": { + "observed": false, + "required": "retain raw payload as unknown; pending/diagnostic rather than a completed turn; no fabricated grouping", + "input_records": [ + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-unknown-version-1", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:22.203Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "user.message", + "data": { + "content": "hello from an unsupported transcript schema", + "interactionId": "6d2b89fd-a653-430c-b532-b0936d72eb42", + "turnId": "0" + }, + "id": "synthetic-unknown-version-1", + "timestamp": "2026-09-10T17:08:22.203Z", + "parentId": null, + "schema_version": 2 + }, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 0, + "byte_length": 240, + "native_event_id": "synthetic-unknown-version-1" + } + } + ], + "expected": { + "events": [ + { + "id": "copilot:event:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-unknown-version-1", + "kind": "unknown", + "classification": "main", + "ts": "2026-09-10T17:08:22.203Z", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-unknown-version-1" + ], + "attributes": { + "native_type": "user.message", + "raw_payload": { + "type": "user.message", + "data": { + "content": "hello from an unsupported transcript schema", + "interactionId": "6d2b89fd-a653-430c-b532-b0936d72eb42", + "turnId": "0" + }, + "id": "synthetic-unknown-version-1", + "timestamp": "2026-09-10T17:08:22.203Z", + "parentId": null, + "schema_version": 2 + } + } + } + ], + "turns": [], + "call_candidates": [], + "pending": [ + { + "id": "pending:unknown-version:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-unknown-version-1", + "kind": "missing_source_capability", + "reason": "unknown transcript schema_version", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-unknown-version-1" + ], + "evidence": [ + "schema_version:2" + ] + } + ], + "diagnostics": [ + { + "code": "capability_gap", + "severity": "warning", + "message": "unknown transcript schema_version; raw payload retained", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-unknown-version-1" + ], + "details": { + "schema_version": 2 + } + } + ] + } + }, + "identical_concurrent_identical_args": { + "observed": false, + "required": "pair concurrent identical view arguments by toolCallId only; arguments/tool_name/timestamp cannot prove pairing", + "input_records": [ + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-identical-args-asst", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.503Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "assistant.message", + "data": { + "messageId": "synthetic-identical-args-msg", + "model": "gpt-5.6-luna", + "content": "", + "toolRequests": [ + { + "toolCallId": "call_identical_args_alpha_1", + "name": "view", + "arguments": { + "path": "/fixture/workspace/alpha.txt" + }, + "type": "function", + "intentionSummary": "view the file at /fixture/workspace/alpha.txt." + }, + { + "toolCallId": "call_identical_args_alpha_2", + "name": "view", + "arguments": { + "path": "/fixture/workspace/alpha.txt" + }, + "type": "function", + "intentionSummary": "view the file at /fixture/workspace/alpha.txt." + } + ], + "interactionId": "6d2b89fd-a653-430c-b532-b0936d72eb42", + "turnId": "0", + "rte": true + }, + "id": "synthetic-identical-args-asst", + "timestamp": "2026-09-10T17:08:24.503Z", + "parentId": "64f9e436-651f-4a9d-919c-a4d7abad2652", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 3731, + "byte_length": 793, + "native_event_id": "synthetic-identical-args-asst" + } + }, + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-identical-args-exec-a", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.506Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "tool.execution_start", + "data": { + "toolCallId": "call_identical_args_alpha_1", + "toolName": "view", + "arguments": { + "path": "/fixture/workspace/alpha.txt" + }, + "turnId": "0", + "model": "gpt-5.6-luna" + }, + "id": "synthetic-identical-args-exec-a", + "timestamp": "2026-09-10T17:08:24.506Z", + "parentId": "synthetic-identical-args-asst", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 4524, + "byte_length": 344, + "native_event_id": "synthetic-identical-args-exec-a" + } + }, + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-identical-args-exec-b", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.506Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "tool.execution_start", + "data": { + "toolCallId": "call_identical_args_alpha_2", + "toolName": "view", + "arguments": { + "path": "/fixture/workspace/alpha.txt" + }, + "turnId": "0", + "model": "gpt-5.6-luna" + }, + "id": "synthetic-identical-args-exec-b", + "timestamp": "2026-09-10T17:08:24.506Z", + "parentId": "synthetic-identical-args-exec-a", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 4868, + "byte_length": 343, + "native_event_id": "synthetic-identical-args-exec-b" + } + } + ], + "expected": { + "tool_call_ids": [ + "call_identical_args_alpha_1", + "call_identical_args_alpha_2" + ], + "event_ids": [ + "copilot:event:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-identical-args-exec-a", + "copilot:event:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-identical-args-exec-b" + ], + "forbidden_join_keys": [ + "arguments", + "tool_name", + "timestamp", + "parentId", + "turnId" + ] + } } } diff --git a/tests/platforms/copilot/test_attribution.py b/tests/platforms/copilot/test_attribution.py index 50a9919..24afcd5 100644 --- a/tests/platforms/copilot/test_attribution.py +++ b/tests/platforms/copilot/test_attribution.py @@ -33,15 +33,9 @@ GENERATION = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" OBSERVED_AT = "2026-09-10T17:09:00.000Z" INTERACTION_ONE = "6d2b89fd-a653-430c-b532-b0936d72eb42" -TURN_ONE = ( - f"copilot:turn:{SOURCE_KEY}:{NATIVE_SESSION_ID}:{INTERACTION_ONE}" -) -CALL_A = ( - f"copilot:call:{SOURCE_KEY}/{NATIVE_SESSION_ID}/a4a17e63-7ba5-422f-8ee9-b495be417328" -) -CALL_B = ( - f"copilot:call:{SOURCE_KEY}/{NATIVE_SESSION_ID}/33cc6465-29e1-4a04-8bdb-00241474b4d2" -) +TURN_ONE = f"copilot:turn:{SOURCE_KEY}:{NATIVE_SESSION_ID}:{INTERACTION_ONE}" +CALL_A = f"copilot:call:{SOURCE_KEY}/{NATIVE_SESSION_ID}/a4a17e63-7ba5-422f-8ee9-b495be417328" +CALL_B = f"copilot:call:{SOURCE_KEY}/{NATIVE_SESSION_ID}/33cc6465-29e1-4a04-8bdb-00241474b4d2" TS_CYCLE_0 = "2026-09-10T17:08:24.503Z" TS_CYCLE_1 = "2026-09-10T17:08:25.624Z" INTERACTION_TWO = "793d3703-6f4a-4814-8877-34a7325848ce" @@ -139,9 +133,7 @@ def _align_attribution(expected: dict[str, Any], source_key: str) -> dict[str, A for field in ("call_id", "stored_turn_id", "logical_call_id", "usage_source_id"): if isinstance(aligned.get(field), str): aligned[field] = _substitute_source_key(aligned[field], source_key) - aligned["evidence"] = [ - _substitute_source_key(item, source_key) for item in aligned["evidence"] - ] + aligned["evidence"] = [_substitute_source_key(item, source_key) for item in aligned["evidence"]] return aligned @@ -224,8 +216,7 @@ def _accounting_candidate( f"sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:{row_id}" ) usage_source_id = ( - f"copilot-db:{SOURCE_KEY}:{NATIVE_SESSION_ID}:assistant_usage_events:" - f"{row_id}:{revision}" + f"copilot-db:{SOURCE_KEY}:{NATIVE_SESSION_ID}:assistant_usage_events:{row_id}:{revision}" ) supplemental: dict[str, Any] = {} if initiator is not None: @@ -317,9 +308,7 @@ def test_child_usage_maps_to_main_stored_turn(cli_transcript_records: list[Sourc source_key = _source_key(cli_transcript_records) records = cli_transcript_records + _six_call_records(source_key=source_key) projection, _state = build_projection(records, {}) - child = next( - item for item in projection["attributions"] if item["agent_id"] == CHILD_AGENT_ID - ) + child = next(item for item in projection["attributions"] if item["agent_id"] == CHILD_AGENT_ID) main_turn_two = ( f"copilot:turn:{source_key}:{NATIVE_SESSION_ID}:793d3703-6f4a-4814-8877-34a7325848ce" ) @@ -332,7 +321,9 @@ def test_child_usage_maps_to_main_stored_turn(cli_transcript_records: list[Sourc def test_attribution_fixture_matched_inferred(): examples = _load_json(RECON / "attributions.json") - semantic, _ = build_semantics(_load_json(RECON / "semantic-projection.json")["input_records"], {}) + semantic, _ = build_semantics( + _load_json(RECON / "semantic-projection.json")["input_records"], {} + ) accounting, _ = build_accounting( [ _usage_record( @@ -348,30 +339,9 @@ def test_attribution_fixture_matched_inferred(): def test_attribution_fixture_pending_without_matching_call(): examples = _load_json(RECON / "attributions.json") - semantic = _semantic( - calls=[ - _call_candidate( - call_id=CALL_A, - tool_call_ids=["call_YSSva4HCniiETlxdGGjcrHbh", "call_ayHplfzxjRFMTCpmTKEFhCSJ"], - ) - ] - ) - accounting = _accounting( - candidates=[ - _accounting_candidate( - row_id=14, - finish_reason="stop", - initiator="agent", - ) - ] - ) - attributions = join_usage(semantic, accounting) - assert attributions[0]["status"] == "pending" - assert attributions[0]["call_id"] is None - assert attributions[0]["stored_turn_id"] == examples["pending"]["stored_turn_id"] - assert "delayed_row:true" not in attributions[0]["evidence"] - assert "turn_index:0" in attributions[0]["evidence"] - assert f"interaction_id:{INTERACTION_ONE}" in attributions[0]["evidence"] + case = _load_json(RECON / "cases.json")["late_row"] + projection, _state = build_projection(case["input_records"], {}) + assert projection["attributions"] == [examples["pending"]] # --- direct / native joins --- @@ -483,9 +453,7 @@ def test_ambiguous_when_multiple_calls_fit_same_evidence(): def test_pending_when_turn_index_has_no_main_interaction(): semantic = _semantic(calls=[_call_candidate(call_id=CALL_A, tool_call_ids=["tool-a"])]) - accounting = _accounting( - candidates=[_accounting_candidate(row_id=13, turn_index=99)] - ) + accounting = _accounting(candidates=[_accounting_candidate(row_id=13, turn_index=99)]) attributions = join_usage(semantic, accounting) assert attributions[0]["status"] == "pending" assert attributions[0]["stored_turn_id"] is None @@ -511,9 +479,7 @@ def test_delayed_row_resolves_after_transcript_replay(): partial = case["input_records"] final_answer = _load_json(RECON / "cases.json")["ambiguous"]["input_records"][1] turn_end = { - "source_id": ( - f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/4a386a37-ca7e-4ebc-a746-cdda20f2a4bb" - ), + "source_id": (f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/4a386a37-ca7e-4ebc-a746-cdda20f2a4bb"), "source_kind": "transcript", "native_session_id": NATIVE_SESSION_ID, "ts": "2026-09-10T17:08:25.626Z", @@ -620,12 +586,8 @@ def test_matched_child_usage_attaches_to_nested_turn( source_key = _source_key(cli_transcript_records) records = cli_transcript_records + _six_call_records(source_key=source_key) projection, _ = build_projection(records, {}) - child = next( - item for item in projection["attributions"] if item["agent_id"] == CHILD_AGENT_ID - ) - main = next( - turn for turn in projection["turns"] if turn["turn_id"] == child["stored_turn_id"] - ) + child = next(item for item in projection["attributions"] if item["agent_id"] == CHILD_AGENT_ID) + main = next(turn for turn in projection["turns"] if turn["turn_id"] == child["stored_turn_id"]) nested = main["subagents"][0] nested_ids = [ item["call_id"] @@ -633,15 +595,11 @@ def test_matched_child_usage_attaches_to_nested_turn( if item["attribution_status"] == "matched" ] main_ids = [ - item["call_id"] - for item in main.get("accounting_calls") or [] - if item.get("call_id") + item["call_id"] for item in main.get("accounting_calls") or [] if item.get("call_id") ] assert child["call_id"] in nested_ids assert child["call_id"] not in main_ids - assert any( - llm.get("call_id") == child["call_id"] for llm in nested.get("llm_calls") or [] - ) + assert any(llm.get("call_id") == child["call_id"] for llm in nested.get("llm_calls") or []) def test_retry_reordering_same_interaction_keeps_call_assignments(): @@ -733,9 +691,9 @@ def test_partial_turn_incremental_state_matches_full_archive(): later = cases["abort"]["input_records"] full = partial + later first, state = build_projection(partial, {}) - assert "6d2b89fd-a653-430c-b532-b0936d72eb42|main" in state["semantic_state"][ - "open_interactions" - ] + assert ( + "6d2b89fd-a653-430c-b532-b0936d72eb42|main" in state["semantic_state"]["open_interactions"] + ) incremental, inc_state = build_projection(later, state) from_flat, flat_state = build_projection(later, state["semantic_state"]) complete, full_state = build_projection(full, {}) @@ -745,9 +703,10 @@ def test_partial_turn_incremental_state_matches_full_archive(): == full_state["semantic_state"]["open_interactions"].keys() == flat_state["semantic_state"]["open_interactions"].keys() ) - assert "6d2b89fd-a653-430c-b532-b0936d72eb42|main" in inc_state["semantic_state"][ - "open_interactions" - ] + assert ( + "6d2b89fd-a653-430c-b532-b0936d72eb42|main" + in inc_state["semantic_state"]["open_interactions"] + ) assert first["turns"] == [] @@ -870,9 +829,7 @@ def test_accounting_rows_persist_when_attribution_is_ambiguous(): row, content_revision="sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", ) - ambiguous, _ = build_projection( - [user, first_msg, first_end, second_msg, second_end, usage], {} - ) + ambiguous, _ = build_projection([user, first_msg, first_end, second_msg, second_end, usage], {}) matched, _ = build_projection([user, first_msg, first_end, usage], {}) assert ambiguous["attributions"][0]["status"] == "ambiguous" assert matched["attributions"][0]["status"] == "matched" @@ -964,9 +921,7 @@ def test_source_correction_replaces_attribution_target_after_replay( assert joined["usage_source_id"] == corrected["source_id"] assert joined["logical_call_id"] == original_join["logical_call_id"] assert _metric_totals(second["usage_rows"])["output_tokens"] == ( - _metric_totals(first["usage_rows"])["output_tokens"] - - original_row.output_tokens - + 999 + _metric_totals(first["usage_rows"])["output_tokens"] - original_row.output_tokens + 999 ) diff --git a/tests/platforms/copilot/test_export.py b/tests/platforms/copilot/test_export.py index c86dc7f..fc0fe05 100644 --- a/tests/platforms/copilot/test_export.py +++ b/tests/platforms/copilot/test_export.py @@ -39,9 +39,7 @@ GENERATION = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" OBSERVED_AT = "2026-09-10T17:09:00.000Z" TURN_ONE = f"copilot:turn:{SOURCE_KEY}:{NATIVE_SESSION_ID}:interaction-main-1" -CALL_MATCHED = ( - f"copilot:call:{SOURCE_KEY}/{NATIVE_SESSION_ID}/a4a17e63-7ba5-422f-8ee9-b495be417328" -) +CALL_MATCHED = f"copilot:call:{SOURCE_KEY}/{NATIVE_SESSION_ID}/a4a17e63-7ba5-422f-8ee9-b495be417328" ACCOUNTING_MATCHED = ( f"copilot:usage:{SOURCE_KEY}:assistant_usage_events:" f"sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13" @@ -610,7 +608,9 @@ def test_ambiguous_terminal_usage_queues_turn_accounting_job( ] ) ], - usage_rows=[_usage_row(call_id=ACCOUNTING_UNMATCHED, input_tokens=6587, output_tokens=5)], + usage_rows=[ + _usage_row(call_id=ACCOUNTING_UNMATCHED, input_tokens=6587, output_tokens=5) + ], attributions=[ _attribution( logical_call_id=ACCOUNTING_UNMATCHED, @@ -710,7 +710,9 @@ def _ambiguous_then_matched_projections(self) -> tuple[Projection, Projection]: ] ) ], - usage_rows=[_usage_row(call_id=ACCOUNTING_UNMATCHED, input_tokens=6587, output_tokens=5)], + usage_rows=[ + _usage_row(call_id=ACCOUNTING_UNMATCHED, input_tokens=6587, output_tokens=5) + ], attributions=[ _attribution( logical_call_id=ACCOUNTING_UNMATCHED, @@ -732,7 +734,9 @@ def _ambiguous_then_matched_projections(self) -> tuple[Projection, Projection]: ] ) ], - usage_rows=[_usage_row(call_id=ACCOUNTING_UNMATCHED, input_tokens=6587, output_tokens=5)], + usage_rows=[ + _usage_row(call_id=ACCOUNTING_UNMATCHED, input_tokens=6587, output_tokens=5) + ], attributions=[_attribution(logical_call_id=ACCOUNTING_UNMATCHED, call_id=CALL_MATCHED)], ) return ambiguous_projection, matched_projection @@ -893,7 +897,9 @@ def _mark_delivered(state: dict[str, Any]) -> dict[str, Any]: ] ) ], - usage_rows=[_usage_row(call_id=ACCOUNTING_UNMATCHED, input_tokens=6587, output_tokens=5)], + usage_rows=[ + _usage_row(call_id=ACCOUNTING_UNMATCHED, input_tokens=6587, output_tokens=5) + ], attributions=[ _attribution( logical_call_id=ACCOUNTING_UNMATCHED, @@ -959,7 +965,9 @@ def test_accounting_job_failure_records_error( ] ) ], - usage_rows=[_usage_row(call_id=ACCOUNTING_UNMATCHED, input_tokens=6587, output_tokens=5)], + usage_rows=[ + _usage_row(call_id=ACCOUNTING_UNMATCHED, input_tokens=6587, output_tokens=5) + ], attributions=[ _attribution( logical_call_id=ACCOUNTING_UNMATCHED, @@ -1019,11 +1027,11 @@ def test_errors_clear_once_a_retry_succeeds( ] ) ], - usage_rows=[_usage_row(call_id=ACCOUNTING_UNMATCHED, input_tokens=6587, output_tokens=5)], + usage_rows=[ + _usage_row(call_id=ACCOUNTING_UNMATCHED, input_tokens=6587, output_tokens=5) + ], attributions=[ - _attribution( - logical_call_id=ACCOUNTING_UNMATCHED, status="ambiguous", call_id=None - ) + _attribution(logical_call_id=ACCOUNTING_UNMATCHED, status="ambiguous", call_id=None) ], ) queue_exports(enabled_config, stored, projection, include_history=True) @@ -1034,9 +1042,7 @@ def test_errors_clear_once_a_retry_succeeds( ) monkeypatch.setattr(export_transport, "queue_turn", lambda *args, **kwargs: True) - monkeypatch.setattr( - export_transport, "queue_turn_accounting", lambda *args, **kwargs: True - ) + monkeypatch.setattr(export_transport, "queue_turn_accounting", lambda *args, **kwargs: True) queue_exports(enabled_config, stored, projection, include_history=True) state = load_export_state(enabled_config, stored) @@ -1066,7 +1072,9 @@ def test_restart_preserves_ledger_and_boundary( ) queued = queue_exports(reloaded, stored, projection, include_history=True) assert queued == 1 - assert json.loads(ledger_path.read_text(encoding="utf-8"))["placements"] == saved["placements"] + assert ( + json.loads(ledger_path.read_text(encoding="utf-8"))["placements"] == saved["placements"] + ) def test_non_terminal_turns_are_not_exported( self, @@ -1147,9 +1155,7 @@ def test_pending_attribution_on_open_turn_is_eligible_once_resolved( ], usage_rows=[_usage_row(call_id=ACCOUNTING_UNMATCHED)], attributions=[ - _attribution( - logical_call_id=ACCOUNTING_UNMATCHED, status="pending", call_id=None - ) + _attribution(logical_call_id=ACCOUNTING_UNMATCHED, status="pending", call_id=None) ], ) queue_exports(enabled_config, stored, open_projection) @@ -1170,9 +1176,7 @@ def test_pending_attribution_on_open_turn_is_eligible_once_resolved( ], usage_rows=[_usage_row(call_id=ACCOUNTING_UNMATCHED)], attributions=[ - _attribution( - logical_call_id=ACCOUNTING_UNMATCHED, status="ambiguous", call_id=None - ) + _attribution(logical_call_id=ACCOUNTING_UNMATCHED, status="ambiguous", call_id=None) ], ) queued = queue_exports(enabled_config, stored, resolved_projection) @@ -1428,11 +1432,11 @@ def test_permanently_failed_accounting_job_is_reported_and_not_cleared( ] ) ], - usage_rows=[_usage_row(call_id=ACCOUNTING_UNMATCHED, input_tokens=6587, output_tokens=5)], + usage_rows=[ + _usage_row(call_id=ACCOUNTING_UNMATCHED, input_tokens=6587, output_tokens=5) + ], attributions=[ - _attribution( - logical_call_id=ACCOUNTING_UNMATCHED, status="ambiguous", call_id=None - ) + _attribution(logical_call_id=ACCOUNTING_UNMATCHED, status="ambiguous", call_id=None) ], ) span_id = f"accounting:{stored}:{TURN_ONE}:{ACCOUNTING_UNMATCHED}" @@ -1446,7 +1450,9 @@ def test_permanently_failed_accounting_job_is_reported_and_not_cleared( ) queued = queue_exports(enabled_config, stored, projection, include_history=True) - assert queued == 1 # only the turn job; the permanently-failed accounting job does not count + assert ( + queued == 1 + ) # only the turn job; the permanently-failed accounting job does not count state = load_export_state(enabled_config, stored) placement = state["placements"][ACCOUNTING_UNMATCHED] assert placement["job_state"] == "failed" @@ -1580,11 +1586,11 @@ def test_confirmed_accounting_delivery_survives_restart_without_double_emission( ] ) ], - usage_rows=[_usage_row(call_id=ACCOUNTING_UNMATCHED, input_tokens=6587, output_tokens=5)], + usage_rows=[ + _usage_row(call_id=ACCOUNTING_UNMATCHED, input_tokens=6587, output_tokens=5) + ], attributions=[ - _attribution( - logical_call_id=ACCOUNTING_UNMATCHED, status="ambiguous", call_id=None - ) + _attribution(logical_call_id=ACCOUNTING_UNMATCHED, status="ambiguous", call_id=None) ], ) @@ -1665,6 +1671,120 @@ def test_chat_embedded_accounting_survives_restart_without_relocation_or_duplica assert placement["emitted"] is True assert ACCOUNTING_MATCHED not in state["conflicts"] + def test_undelivered_same_span_token_correction_rewrites_job_and_delivers_once( + self, + enabled_config: Config, + paths: SourcePaths, + wired_instance, + exporter, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + stored = _seed_session(enabled_config, paths) + delayed: list[Path] = [] + + def _hold(job_path: Path) -> None: + delayed.append(job_path) + + monkeypatch.setattr(export_transport, "_spawn", _hold) + monkeypatch.setattr(otel_export, "_spawn", lambda job_path: None) + + first = _projection( + turns=[ + _main_turn( + accounting_calls=[ + _accounting_call( + accounting_id=ACCOUNTING_UNMATCHED, + attribution_status="ambiguous", + call_id=None, + usage=_usage_row( + call_id=ACCOUNTING_UNMATCHED, input_tokens=6587, output_tokens=5 + ).to_dict(), + ) + ] + ) + ], + usage_rows=[ + _usage_row(call_id=ACCOUNTING_UNMATCHED, input_tokens=6587, output_tokens=5) + ], + attributions=[ + _attribution(logical_call_id=ACCOUNTING_UNMATCHED, status="ambiguous", call_id=None) + ], + ) + queue_exports(enabled_config, stored, first, include_history=True) + span_id = f"accounting:{stored}:{TURN_ONE}:{ACCOUNTING_UNMATCHED}" + job_path = export_transport.job_path(enabled_config.root, span_id) + assert job_path.exists() + first_job = json.loads(job_path.read_text(encoding="utf-8")) + assert first_job["usage"]["gen_ai.usage.output_tokens"] == 5 + + corrected_usage = _usage_row( + call_id=ACCOUNTING_UNMATCHED, input_tokens=6587, output_tokens=999 + ).to_dict() + second = _projection( + turns=[ + _main_turn( + accounting_calls=[ + _accounting_call( + accounting_id=ACCOUNTING_UNMATCHED, + attribution_status="ambiguous", + call_id=None, + usage=corrected_usage, + ) + ] + ) + ], + usage_rows=[ + _usage_row(call_id=ACCOUNTING_UNMATCHED, input_tokens=6587, output_tokens=999) + ], + attributions=[ + _attribution(logical_call_id=ACCOUNTING_UNMATCHED, status="ambiguous", call_id=None) + ], + ) + queue_exports(enabled_config, stored, second, include_history=True) + rewritten = json.loads(job_path.read_text(encoding="utf-8")) + assert rewritten["usage"]["gen_ai.usage.output_tokens"] == 999 + assert rewritten["state"] == "queued" + + export_transport.main([str(job_path)]) + accounting_spans = [ + span for span in exporter.exported_spans_as_dict() if span["name"] == "accounting" + ] + assert len(accounting_spans) == 1 + assert accounting_spans[0]["attributes"]["gen_ai.usage.output_tokens"] == 999 + directory = _directory(enabled_config, stored) + assert export_transport.delivery_sent(directory, ACCOUNTING_UNMATCHED) is True + state = load_export_state(enabled_config, stored) + assert ACCOUNTING_UNMATCHED not in state["conflicts"] + + def test_observed_six_call_queue_exports_labels_native_billing_not_usd( + self, + enabled_config: Config, + wired_instance, + exporter, + tmp_path: Path, + ) -> None: + home = tmp_path / "copilot-home" + home.mkdir() + paths = resolve_sources(home) + commit_batch(enabled_config, paths, _batch(paths, _drain_cli_transcript(home))) + stored = stored_session_id(paths, NATIVE_SESSION_ID) + source_key = _source_key(_drain_cli_transcript(home)) + records = _drain_cli_transcript(home) + _six_call_records(source_key=source_key) + projection, _ = build_projection(records, {}) + + queue_exports(enabled_config, stored, projection, include_history=True) + + billed = [ + span + for span in exporter.exported_spans_as_dict() + if span["attributes"].get("thirdeye.accounting.billing.kind") == "copilot-native-unit" + ] + assert billed + for span in billed: + assert span["attributes"]["thirdeye.accounting.billing.kind"] == "copilot-native-unit" + assert "operation.cost" not in span["attributes"] + assert span["attributes"]["copilot.billing.nano_aiu"] + def _drain_cli_transcript(home: Path) -> list[SourceRecord]: session_dir = home / "session-state" / NATIVE_SESSION_ID diff --git a/tests/platforms/copilot/test_tracing.py b/tests/platforms/copilot/test_tracing.py index 1f6e6f6..faf1acb 100644 --- a/tests/platforms/copilot/test_tracing.py +++ b/tests/platforms/copilot/test_tracing.py @@ -11,7 +11,10 @@ import pytest -from thirdeye.platforms.copilot.identity import resolve_sources +from thirdeye.config import Config +from thirdeye.platforms.copilot.archive import commit_batch +from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id +from thirdeye.platforms.copilot.reconcile import reconcile_archive from thirdeye.platforms.copilot.tracing import build_semantics from thirdeye.platforms.copilot.transcript import read_transcript from thirdeye.platforms.copilot.turns import build_turns @@ -268,6 +271,27 @@ def test_identical_concurrent_tools_pair_by_tool_call_id() -> None: assert len({event["attributes"]["tool_call_id"] for event in start_events}) == 2 +def test_identical_concurrent_identical_args_pair_by_tool_call_id() -> None: + case = _load_json(RECON_CASES / "cases.json")["identical_concurrent_identical_args"] + assert case["observed"] is False + projection, _ = build_semantics(case["input_records"], {}) + expected = case["expected"] + + start_events = [ + event for event in projection["events"] if event["kind"] == "tool_execution_start" + ] + assert sorted(event["id"] for event in start_events) == sorted(expected["event_ids"]) + assert sorted(event["attributes"]["tool_call_id"] for event in start_events) == sorted( + expected["tool_call_ids"] + ) + arguments = [event["attributes"].get("arguments") for event in start_events] + assert arguments[0] == arguments[1] + assert {event["attributes"].get("name") for event in start_events} == {"view"} + candidate = projection["call_candidates"][0] + assert candidate["tool_call_ids"] == expected["tool_call_ids"] + assert len({event["attributes"]["tool_call_id"] for event in start_events}) == 2 + + # --- nested child ownership --- @@ -401,7 +425,7 @@ def test_retry_case_is_database_only_and_emits_no_semantic_events() -> None: assert projection["pending"] == [] -def test_observed_versus_synthetic_cases_run_through_build_semantics() -> None: +def test_observed_versus_synthetic_cases_run_through_build_semantics(tmp_path: Path) -> None: cases = _load_json(RECON_CASES / "cases.json") observed = {name for name, case in cases.items() if case["observed"]} synthetic = {name for name, case in cases.items() if not case["observed"]} @@ -410,7 +434,16 @@ def test_observed_versus_synthetic_cases_run_through_build_semantics() -> None: assert "permission" in synthetic assert "partial_turn" in synthetic assert "abort" in synthetic - for name in ("identical_concurrent_tools", "permission", "partial_turn", "abort"): + assert "unknown_version" in synthetic + assert "identical_concurrent_identical_args" in synthetic + for name in ( + "identical_concurrent_tools", + "permission", + "partial_turn", + "abort", + "unknown_version", + "identical_concurrent_identical_args", + ): projection, _ = build_semantics(cases[name]["input_records"], {}) assert "events" in projection _assert_expected_projection(projection, cases[name]["expected"]) @@ -419,6 +452,40 @@ def test_observed_versus_synthetic_cases_run_through_build_semantics() -> None: started = [event for event in nested["events"] if event["kind"] == "subagent_started"] assert started[0]["attributes"]["parent_tool_call_id"] == "call_qx4FH5DADTeT1qVLb37HNpBk" + unknown = cases["unknown_version"] + unknown_projection, _ = build_semantics(unknown["input_records"], {}) + assert unknown_projection["turns"] == [] + unknown_event = next( + event for event in unknown_projection["events"] if event["kind"] == "unknown" + ) + assert "raw_payload" in unknown_event["attributes"] + assert unknown_event["attributes"]["raw_payload"]["schema_version"] == 2 + assert any( + item["kind"] == "missing_source_capability" for item in unknown_projection["pending"] + ) + + config = Config(root=tmp_path / "thirdeye") + home = tmp_path / "copilot-home" + home.mkdir() + paths = resolve_sources(home) + native_session_id = unknown["input_records"][0]["native_session_id"] + commit_batch( + config, + paths, + { + "source_key": paths["source_key"], + "native_session_id": native_session_id, + "cwd": "/proj", + "records": unknown["input_records"], + "next_cursor": {"generation": 1}, + "diagnostics": [], + }, + ) + stored = stored_session_id(paths, native_session_id) + result = reconcile_archive(config, stored) + assert result["turns"] == 0 + assert result["pending"] >= 1 + def test_completed_child_without_parent_link_is_pending_not_dropped() -> None: child = "child-agent-1" diff --git a/tests/platforms/copilot/test_usage.py b/tests/platforms/copilot/test_usage.py index 1a11441..f7c56c3 100644 --- a/tests/platforms/copilot/test_usage.py +++ b/tests/platforms/copilot/test_usage.py @@ -453,15 +453,17 @@ def test_reused_row_id_across_generations_emits_warning(): assert "usage_row_id_reuse" in _diagnostic_codes(projection) reuse = next(item for item in projection["diagnostics"] if item["code"] == "usage_row_id_reuse") + assert reuse["severity"] == "error" assert first["source_id"] in reuse["source_ids"] assert second["source_id"] in reuse["source_ids"] - call_ids = [row.call_id for row in projection["usage_rows"]] - assert len(call_ids) == 2 - assert call_ids[0] != call_ids[1] - assert projection["usage_rows"][0].output_tokens == row_a["output_tokens"] - assert projection["usage_rows"][1].output_tokens == 50 - assert projection["candidates"][0]["usage_source_id"] == first["source_id"] + assert projection["usage_rows"] == [] + assert len(projection["candidates"]) == 2 + assert {candidate["usage_source_id"] for candidate in projection["candidates"]} == { + first["source_id"], + second["source_id"], + } assert len(state["logical_calls"]) == 2 + assert all(call["quarantined"] is True for call in state["logical_calls"].values()) def test_incompatible_metrics_quarantine_logical_call(): @@ -788,14 +790,16 @@ def test_row_id_reuse_is_detected_across_partitions(): generation="sha256:generation-b", ) _first_projection, state = build_accounting([first], {}) - projection, _next_state = build_accounting([second], state) + projection, next_state = build_accounting([second], state) + assert len(_first_projection["usage_rows"]) == 1 assert "usage_row_id_reuse" in _diagnostic_codes(projection) reuse = next(item for item in projection["diagnostics"] if item["code"] == "usage_row_id_reuse") + assert reuse["severity"] == "error" assert first["source_id"] in reuse["source_ids"] assert second["source_id"] in reuse["source_ids"] - assert len(projection["usage_rows"]) == 1 - assert projection["usage_rows"][0].output_tokens == 50 + assert projection["usage_rows"] == [] + assert all(call["quarantined"] is True for call in next_state["logical_calls"].values()) def test_integer_valued_float_row_metrics_do_not_false_mismatch_shutdown(): diff --git a/tests/shared/copilot_projection_fixtures.py b/tests/shared/copilot_projection_fixtures.py index 44cde08..bb57935 100644 --- a/tests/shared/copilot_projection_fixtures.py +++ b/tests/shared/copilot_projection_fixtures.py @@ -2,6 +2,8 @@ from __future__ import annotations +import json +import shutil from pathlib import Path from typing import Any @@ -9,8 +11,10 @@ from thirdeye.platforms.copilot.archive import commit_batch from thirdeye.platforms.copilot.constants import PLATFORM_NAME from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id +from thirdeye.platforms.copilot.projection import build_projection from thirdeye.platforms.copilot.projection_state import empty_projection_state from thirdeye.platforms.copilot.projection_store import commit_projection +from thirdeye.platforms.copilot.transcript import read_transcript from thirdeye.platforms.copilot.types import Projection, SourceBatch, SourceRecord from thirdeye.usage.types import UsageRow @@ -293,3 +297,77 @@ def seed_two_main_interaction_projection(config: Config, tmp_path: Path) -> str: } commit_projection(config, stored_id, projection, empty_projection_state()) return stored_id + + +_COPILOT_FIXTURES = Path(__file__).resolve().parents[1] / "platforms" / "copilot" / "fixtures" +_OBSERVED_AT = "2026-09-10T17:09:00.000Z" + + +def seed_observed_six_call_projection(config: Config, tmp_path: Path) -> str: + """Archive the observed CLI corpus and commit its six-call projection.""" + home = tmp_path / "copilot-home" + home.mkdir(parents=True, exist_ok=True) + session_path = home / "session-state" / NATIVE_ID + session_path.mkdir(parents=True, exist_ok=True) + shutil.copy(_COPILOT_FIXTURES / "events.jsonl", session_path / "events.jsonl") + (session_path / "workspace.yaml").write_text("cwd: /sanitized/workspace\n", encoding="utf-8") + paths = resolve_sources(home) + cursor: dict[str, Any] = {} + transcript: list[SourceRecord] = [] + while True: + slice_ = read_transcript(paths, NATIVE_ID, cursor) + transcript.extend(slice_["records"]) + cursor = slice_["next_cursor"] + if slice_["exhausted"]: + break + source_key = next( + record["source_id"].split("/", 1)[0] + for record in transcript + if record["source_kind"] == "transcript" + ) + rows = json.loads((_COPILOT_FIXTURES / "assistant-usage-events.json").read_text()) + revisions = { + call["row_id"]: call["usage_source_id"].rsplit(":", 1)[-1] + for call in json.loads( + (_COPILOT_FIXTURES / "reconciliation-cases" / "observed-six-calls.json").read_text() + )["calls"] + } + usage_records: list[SourceRecord] = [] + for row in rows: + content_revision = revisions[row["id"]] + usage_records.append( + { + "source_id": ( + f"copilot-db:{source_key}:{NATIVE_ID}:assistant_usage_events:" + f"{row['id']}:{content_revision}" + ), + "source_kind": "database", + "native_session_id": NATIVE_ID, + "ts": row.get("created_at"), + "observed_at": _OBSERVED_AT, + "payload": {"table": "assistant_usage_events", "row": row}, + "locator": { + "database": "/example/.copilot/session-store.db", + "table": "assistant_usage_events", + "primary_key": row["id"], + "content_revision": content_revision, + "generation": ( + "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + ), + }, + } + ) + records = transcript + usage_records + batch: SourceBatch = { + "source_key": paths["source_key"], + "native_session_id": NATIVE_ID, + "cwd": "/fixture/workspace", + "records": records, + "next_cursor": {"generation": 1}, + "diagnostics": [], + } + commit_batch(config, paths, batch) + stored_id = stored_session_id(paths, NATIVE_ID) + projection, next_state = build_projection(records, {}) + commit_projection(config, stored_id, projection, next_state) + return stored_id diff --git a/tests/web/test_copilot_capture_views.py b/tests/web/test_copilot_capture_views.py index 8fe2184..800ea50 100644 --- a/tests/web/test_copilot_capture_views.py +++ b/tests/web/test_copilot_capture_views.py @@ -16,6 +16,7 @@ USAGE_MODEL_TWO, USAGE_TOKENS_ONE, USAGE_TOKENS_TWO, + seed_observed_six_call_projection, seed_two_main_interaction_projection, ) from thirdeye.config import LogfireSettings @@ -224,6 +225,25 @@ def test_copilot_session_usage_page_shows_projected_usage_rows( assert str(USAGE_TOKENS_TWO * 2) not in body +def test_copilot_session_usage_page_labels_native_billing_and_latency( + client, web_config, tmp_path: Path +) -> None: + stored_id = seed_observed_six_call_projection(web_config, tmp_path) + + usage = client.get(f"/sessions/{stored_id}/usage") + + assert usage.status_code == 200 + body = usage.text + assert "copilot native billing (nano-AIU)" in body + assert "duration_ms" in body + assert "TTFT (ms)" in body + assert "174125000" in body + tbody = body.split("", 1)[1].split("", 1)[0] + assert tbody.count("") == 6 + assert "operation.cost" not in body + assert "$" not in tbody + + def test_copilot_projected_session_search_tag_and_eval_routes( client, app, web_config, tmp_path: Path, monkeypatch ) -> None: diff --git a/tests/web/test_routes_usage.py b/tests/web/test_routes_usage.py index f4eed35..c6e2700 100644 --- a/tests/web/test_routes_usage.py +++ b/tests/web/test_routes_usage.py @@ -106,6 +106,7 @@ def test_global_usage_platform_filter_shows_copilot_rows(client, web_config, tmp assert "selected>copilot<" in r.text.replace("\n", "") assert str(USAGE_TOKENS_ONE + USAGE_TOKENS_TWO) in r.text assert str(USAGE_TOKENS_TWO * 2) not in r.text + assert "copilot native billing (nano-AIU)" in r.text def test_session_usage_renders_per_call_rows_with_model(client, web_config): From d56e026c93d1242e23516cad9ebcf0796d72ccf1 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Sun, 13 Sep 2026 17:50:10 -0700 Subject: [PATCH 87/88] formatting --- src/thirdeye/commands/copilot.py | 4 ++- src/thirdeye/platforms/copilot/attribution.py | 12 ++----- .../platforms/copilot/export_state.py | 20 ++++++++--- .../platforms/copilot/export_transport.py | 8 ++--- src/thirdeye/platforms/copilot/reconcile.py | 4 +-- src/thirdeye/platforms/copilot/status.py | 7 ++-- tests/platforms/copilot/test_command.py | 34 ++++++++++++++++--- .../copilot/test_export_transport.py | 4 +-- tests/platforms/copilot/test_migration.py | 4 +-- .../copilot/test_projection_store.py | 4 +-- tests/platforms/copilot/test_reconcile.py | 16 +++------ tests/platforms/copilot/test_runtime.py | 20 +++++++---- tests/platforms/copilot/test_watch.py | 11 +++++- uv.lock | 24 +++++++++++++ 14 files changed, 113 insertions(+), 59 deletions(-) diff --git a/src/thirdeye/commands/copilot.py b/src/thirdeye/commands/copilot.py index 33ba045..1d75da8 100644 --- a/src/thirdeye/commands/copilot.py +++ b/src/thirdeye/commands/copilot.py @@ -311,7 +311,9 @@ def sync_cmd(session_id: str | None, export_history: bool, source_home: Path | N ) -@copilot_group.command("reconcile", help="Build Copilot V2 projections from retained local archives.") +@copilot_group.command( + "reconcile", help="Build Copilot V2 projections from retained local archives." +) @click.option("--session-id", default=None, help="Exact stored Thirdeye Copilot session ID.") @click.option("--rebuild", is_flag=True, default=False, help="Rebuild derived state only.") @click.option( diff --git a/src/thirdeye/platforms/copilot/attribution.py b/src/thirdeye/platforms/copilot/attribution.py index 30772b8..3ad04b2 100644 --- a/src/thirdeye/platforms/copilot/attribution.py +++ b/src/thirdeye/platforms/copilot/attribution.py @@ -36,9 +36,7 @@ def _direct_ids(candidate: dict[str, Any], *, semantic: bool) -> set[str]: """Read future native linkage fields without making one up today.""" result = { - value - for field in _DIRECT_ID_FIELDS - if (value := _string(candidate.get(field))) is not None + value for field in _DIRECT_ID_FIELDS if (value := _string(candidate.get(field))) is not None } # A future accounting producer may carry the semantic durable call ID. # The current accounting shape uses logical_call_id instead, so this does @@ -87,9 +85,7 @@ def _cycle_order(calls: list[CallCandidate]) -> list[CallCandidate]: ) -def _identity_group( - call: CallCandidate, calls: list[CallCandidate] -) -> list[CallCandidate]: +def _identity_group(call: CallCandidate, calls: list[CallCandidate]) -> list[CallCandidate]: group = [ item for item in calls @@ -172,9 +168,7 @@ def _turn_for_usage( return interaction, _string(root.get("stored_turn_id")) -def _base_attribution( - usage: AccountingCandidate, stored_turn_id: str | None -) -> Attribution: +def _base_attribution(usage: AccountingCandidate, stored_turn_id: str | None) -> Attribution: return { "usage_source_id": usage["usage_source_id"], "logical_call_id": usage["logical_call_id"], diff --git a/src/thirdeye/platforms/copilot/export_state.py b/src/thirdeye/platforms/copilot/export_state.py index 202de37..84856b0 100644 --- a/src/thirdeye/platforms/copilot/export_state.py +++ b/src/thirdeye/platforms/copilot/export_state.py @@ -86,13 +86,19 @@ def _normalize(value: dict[str, Any]) -> dict[str, Any]: "excluded_turn_ids": _ids(value.get("excluded_turn_ids")), "excluded_accounting_ids": _ids(value.get("excluded_accounting_ids")), "placements": { - key: item for key, item in placements.items() if isinstance(key, str) and isinstance(item, dict) + key: item + for key, item in placements.items() + if isinstance(key, str) and isinstance(item, dict) }, "conflicts": { - key: item for key, item in conflicts.items() if isinstance(key, str) and isinstance(item, dict) + key: item + for key, item in conflicts.items() + if isinstance(key, str) and isinstance(item, dict) }, "turn_errors": { - key: value2 for key, value2 in turn_errors.items() if isinstance(key, str) and isinstance(value2, str) + key: value2 + for key, value2 in turn_errors.items() + if isinstance(key, str) and isinstance(value2, str) }, } @@ -107,7 +113,9 @@ def _read(directory: Path) -> dict[str, Any]: view: silently treating them as empty would forget every placement this ledger recorded and risk emitting tokens at a second location. """ - raw = read_json_object(export_state_path(directory), invalid_message="invalid Copilot export ledger") + raw = read_json_object( + export_state_path(directory), invalid_message="invalid Copilot export ledger" + ) if raw is None: return empty_export_state() version = raw.get("schema_version") @@ -189,7 +197,9 @@ def initialize_eligibility( return result if include_history: result["excluded_turn_ids"] = sorted(set(result["excluded_turn_ids"]) - turn_ids) - result["excluded_accounting_ids"] = sorted(set(result["excluded_accounting_ids"]) - usage_ids) + result["excluded_accounting_ids"] = sorted( + set(result["excluded_accounting_ids"]) - usage_ids + ) return result diff --git a/src/thirdeye/platforms/copilot/export_transport.py b/src/thirdeye/platforms/copilot/export_transport.py index e5a3599..b7a53e9 100644 --- a/src/thirdeye/platforms/copilot/export_transport.py +++ b/src/thirdeye/platforms/copilot/export_transport.py @@ -162,7 +162,9 @@ def cancel(root: Path, job_id: str) -> dict[str, Any]: def _spawn(path: Path) -> None: - proc.spawn_detached([sys.executable, "-m", "thirdeye.platforms.copilot.export_transport", str(path)]) + proc.spawn_detached( + [sys.executable, "-m", "thirdeye.platforms.copilot.export_transport", str(path)] + ) def queue_turn( @@ -410,9 +412,7 @@ def run(path: Path) -> None: ) else: try: - _mark_delivered( - Path(str(claimed["session_dir"])), str(claimed["accounting_id"]) - ) + _mark_delivered(Path(str(claimed["session_dir"])), str(claimed["accounting_id"])) _write_job(path, {**claimed, "state": "emitted", "last_error": None}) fsops.unlink(path, missing_ok=True) except Exception as exc: diff --git a/src/thirdeye/platforms/copilot/reconcile.py b/src/thirdeye/platforms/copilot/reconcile.py index d919c20..70c8f25 100644 --- a/src/thirdeye/platforms/copilot/reconcile.py +++ b/src/thirdeye/platforms/copilot/reconcile.py @@ -56,9 +56,7 @@ def _attribution_counts(projection: Projection) -> tuple[int, int]: def _diagnostic_errors(projection: Projection) -> int: return sum( - 1 - for diagnostic in projection["diagnostics"] - if diagnostic.get("severity") == "error" + 1 for diagnostic in projection["diagnostics"] if diagnostic.get("severity") == "error" ) diff --git a/src/thirdeye/platforms/copilot/status.py b/src/thirdeye/platforms/copilot/status.py index d4bb2d7..09381eb 100644 --- a/src/thirdeye/platforms/copilot/status.py +++ b/src/thirdeye/platforms/copilot/status.py @@ -347,7 +347,8 @@ def _export_health(config: Config, stored_session_id: str, directory: Path) -> d continue state = payload.get("state") if state == "emitted" or ( - isinstance(accounting_id, str) and export_transport.delivery_sent(directory, accounting_id) + isinstance(accounting_id, str) + and export_transport.delivery_sent(directory, accounting_id) ): delivered.add(key) queued.discard(key) @@ -401,9 +402,7 @@ def _export_health(config: Config, stored_session_id: str, directory: Path) -> d "activated": bool(ledger.get("activated")), "queued": len(queued), "delivered": len(delivered), - "errors": len(errored) - + len(conflicts) - + len(turn_errors), + "errors": len(errored) + len(conflicts) + len(turn_errors), "last_error": last_error, } diff --git a/tests/platforms/copilot/test_command.py b/tests/platforms/copilot/test_command.py index f06745d..078bbb2 100644 --- a/tests/platforms/copilot/test_command.py +++ b/tests/platforms/copilot/test_command.py @@ -696,7 +696,18 @@ def fake_reconcile_archived( include_history: bool = False, ) -> dict[str, dict[str, int]]: reconcile_calls.append((export, include_history)) - return {"stored-1": {"events": 3, "usage": 2, "turns": 1, "exports": 0, "pending": 0, "ambiguous": 0, "conflicting": 0, "errors": 0}} + return { + "stored-1": { + "events": 3, + "usage": 2, + "turns": 1, + "exports": 0, + "pending": 0, + "ambiguous": 0, + "conflicting": 0, + "errors": 0, + } + } monkeypatch.setattr("thirdeye.commands.copilot.capture_sync", fake_sync) monkeypatch.setattr( @@ -735,7 +746,9 @@ def fake_reconcile_archive( "errors": 0, } - monkeypatch.setattr("thirdeye.commands.copilot.reconcile_stored_session", fake_reconcile_archive) + monkeypatch.setattr( + "thirdeye.commands.copilot.reconcile_stored_session", fake_reconcile_archive + ) result = CliRunner().invoke( main, ["copilot", "reconcile", "--session-id", "copilot-abc-stored", "--rebuild", "--export"], @@ -814,7 +827,18 @@ def fake_archived( include_history: bool = False, ) -> dict[str, dict[str, int]]: archived_calls.append((export, include_history)) - return {"other-stored": {"events": 9, "usage": 0, "turns": 0, "exports": 1, "pending": 0, "ambiguous": 0, "conflicting": 0, "errors": 0}} + return { + "other-stored": { + "events": 9, + "usage": 0, + "turns": 0, + "exports": 1, + "pending": 0, + "ambiguous": 0, + "conflicting": 0, + "errors": 0, + } + } def fake_session( _config: Config, @@ -870,7 +894,9 @@ def fake_sync( "thirdeye.commands.copilot.reconcile_session", lambda *_a, **_k: session_calls.append(True) or {}, ) - result = CliRunner().invoke(main, ["copilot", "sync", "--session-id", NATIVE_SESSION_ID, "--export"]) + result = CliRunner().invoke( + main, ["copilot", "sync", "--session-id", NATIVE_SESSION_ID, "--export"] + ) assert result.exit_code != 0, result.output assert "was not found" in result.output assert archived_calls == [] diff --git a/tests/platforms/copilot/test_export_transport.py b/tests/platforms/copilot/test_export_transport.py index 3be7a2a..3ae804f 100644 --- a/tests/platforms/copilot/test_export_transport.py +++ b/tests/platforms/copilot/test_export_transport.py @@ -43,9 +43,7 @@ def _write(config: Config, payload: dict[str, Any]) -> Path: return path -def _seed_placement( - config: Config, payload: dict[str, Any], *, span_id: str | None = None -) -> None: +def _seed_placement(config: Config, payload: dict[str, Any], *, span_id: str | None = None) -> None: def _record(state: dict[str, Any]) -> dict[str, Any]: updated, _, _ = record_placement( state, diff --git a/tests/platforms/copilot/test_migration.py b/tests/platforms/copilot/test_migration.py index 6cbc7ec..6ecee74 100644 --- a/tests/platforms/copilot/test_migration.py +++ b/tests/platforms/copilot/test_migration.py @@ -554,9 +554,7 @@ def test_stale_journal_schema_is_discarded_without_raising( assert not projection_journal_path(directory).exists() -def test_corrupt_journal_is_discarded_and_snapshot_used( - config: Config, paths: SourcePaths -) -> None: +def test_corrupt_journal_is_discarded_and_snapshot_used(config: Config, paths: SourcePaths) -> None: stored = _seed_v1_archive(config, paths) commit_projection(config, stored, _sample_projection(stored), empty_projection_state()) directory = _directory(config, stored) diff --git a/tests/platforms/copilot/test_projection_store.py b/tests/platforms/copilot/test_projection_store.py index 6eb7fac..68b5b65 100644 --- a/tests/platforms/copilot/test_projection_store.py +++ b/tests/platforms/copilot/test_projection_store.py @@ -315,9 +315,7 @@ def test_child_agent_top_level_turns_are_excluded(config: Config, paths: SourceP assert [turn["turn_id"] for turn in turns] == [main_turn["turn_id"]] -def test_nested_child_archive_events_stay_on_main_turn( - config: Config, paths: SourcePaths -) -> None: +def test_nested_child_archive_events_stay_on_main_turn(config: Config, paths: SourcePaths) -> None: stored = _seed_archive( config, paths, diff --git a/tests/platforms/copilot/test_reconcile.py b/tests/platforms/copilot/test_reconcile.py index bf3edd9..38153d1 100644 --- a/tests/platforms/copilot/test_reconcile.py +++ b/tests/platforms/copilot/test_reconcile.py @@ -332,9 +332,7 @@ def test_reconcile_archive_is_idempotent( # byte identical across the two re-derivations: re-running reconcile # must not accumulate duplicate accounting calls or usage/attribution # entries inside any index. - normalized_first = { - k: v for k, v in first_document["state"].items() if k != "commit_sequence" - } + normalized_first = {k: v for k, v in first_document["state"].items() if k != "commit_sequence"} normalized_second = { k: v for k, v in second_document["state"].items() if k != "commit_sequence" } @@ -735,18 +733,14 @@ def test_rebuild_preserves_immutable_archive_records( stored = _seed_full_corpus(config, paths, cli_transcript_records) before = { event["data"]["source_record"]["source_id"] - for event in SessionReader( - config.root / "traces" / "copilot" / stored - ).iter_events() + for event in SessionReader(config.root / "traces" / "copilot" / stored).iter_events() } reconcile_archive(config, stored, rebuild=True) after = { event["data"]["source_record"]["source_id"] - for event in SessionReader( - config.root / "traces" / "copilot" / stored - ).iter_events() + for event in SessionReader(config.root / "traces" / "copilot" / stored).iter_events() } assert before == after @@ -765,9 +759,7 @@ def test_late_row_archive_reconcile_resolves_after_full_replay( turn_end = copy.deepcopy(final_answer) turn_end.update( { - "source_id": ( - f"{source_key}/{NATIVE_SESSION_ID}/4a386a37-ca7e-4ebc-a746-cdda20f2a4bb" - ), + "source_id": (f"{source_key}/{NATIVE_SESSION_ID}/4a386a37-ca7e-4ebc-a746-cdda20f2a4bb"), "payload": { "type": "assistant.turn_end", "data": {"turnId": "1"}, diff --git a/tests/platforms/copilot/test_runtime.py b/tests/platforms/copilot/test_runtime.py index 65498d8..88b3e3c 100644 --- a/tests/platforms/copilot/test_runtime.py +++ b/tests/platforms/copilot/test_runtime.py @@ -194,7 +194,9 @@ def fake_reconcile_archive( "errors": 0, } - monkeypatch.setattr("thirdeye.platforms.copilot.runtime.reconcile_archive", fake_reconcile_archive) + monkeypatch.setattr( + "thirdeye.platforms.copilot.runtime.reconcile_archive", fake_reconcile_archive + ) result = reconcile_session(config, paths, NATIVE_SESSION_ID, export=True, include_history=True) assert result["turns"] == 1 @@ -245,7 +247,9 @@ def fake_reconcile_archive( "errors": 0, } - monkeypatch.setattr("thirdeye.platforms.copilot.runtime.reconcile_archive", fake_reconcile_archive) + monkeypatch.setattr( + "thirdeye.platforms.copilot.runtime.reconcile_archive", fake_reconcile_archive + ) results = reconcile_archived_sessions(config, paths) assert set(seen) == {first, second} @@ -261,7 +265,9 @@ def test_reconcile_archived_sessions_works_after_source_removal( session_dir_path = home / "session-state" / NATIVE_SESSION_ID session_dir_path.mkdir(parents=True) shutil.copy(FIXTURES / "events.jsonl", session_dir_path / "events.jsonl") - (session_dir_path / "workspace.yaml").write_text("cwd: /sanitized/workspace\n", encoding="utf-8") + (session_dir_path / "workspace.yaml").write_text( + "cwd: /sanitized/workspace\n", encoding="utf-8" + ) from thirdeye.platforms.copilot.capture import sync @@ -358,9 +364,7 @@ def test_followup_logs_returned_reconcile_errors( def fake_capture(_config: Config, _paths: SourcePaths, _native_id: str) -> SyncResult: return _empty_result(sessions=1) - def fake_reconcile_archive( - *_args: Any, **_kwargs: Any - ) -> dict[str, int]: + def fake_reconcile_archive(*_args: Any, **_kwargs: Any) -> dict[str, int]: return { "events": 0, "usage": 0, @@ -373,7 +377,9 @@ def fake_reconcile_archive( } monkeypatch.setattr("thirdeye.platforms.copilot.capture.capture_session", fake_capture) - monkeypatch.setattr("thirdeye.platforms.copilot.runtime.reconcile_archive", fake_reconcile_archive) + monkeypatch.setattr( + "thirdeye.platforms.copilot.runtime.reconcile_archive", fake_reconcile_archive + ) _run(config, paths, NATIVE_SESSION_ID, generation) status = load_runtime_status(config, stored) diff --git a/tests/platforms/copilot/test_watch.py b/tests/platforms/copilot/test_watch.py index 48a1471..b681052 100644 --- a/tests/platforms/copilot/test_watch.py +++ b/tests/platforms/copilot/test_watch.py @@ -281,7 +281,16 @@ def track_reconcile( include_history: bool = False, ) -> dict[str, int]: reconciled.append(native_session_id) - return {"events": 0, "usage": 0, "turns": 0, "exports": 0, "pending": 0, "ambiguous": 0, "conflicting": 0, "errors": 0} + return { + "events": 0, + "usage": 0, + "turns": 0, + "exports": 0, + "pending": 0, + "ambiguous": 0, + "conflicting": 0, + "errors": 0, + } def append_during_poll(_interval: float) -> None: cycle["count"] += 1 diff --git a/uv.lock b/uv.lock index 68856f5..aa2c6bd 100644 --- a/uv.lock +++ b/uv.lock @@ -341,6 +341,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/33/6b/e0547afaf41bf2c42e52430072fa5658766e3d65bd4b03a563d1b6336f57/distlib-0.4.0-py2.py3-none-any.whl", hash = "sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16", size = 469047, upload-time = "2025-07-17T16:51:58.613Z" }, ] +[[package]] +name = "execnet" +version = "2.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/89/780e11f9588d9e7128a3f87788354c7946a9cbb1401ad38a48c4db9a4f07/execnet-2.1.2.tar.gz", hash = "sha256:63d83bfdd9a23e35b9c6a3261412324f964c2ec8dcd8d3c6916ee9373e0befcd", size = 166622, upload-time = "2025-11-12T09:56:37.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708, upload-time = "2025-11-12T09:56:36.333Z" }, +] + [[package]] name = "executing" version = "2.2.1" @@ -1081,6 +1090,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" }, ] +[[package]] +name = "pytest-xdist" +version = "3.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "execnet" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/b4/439b179d1ff526791eb921115fca8e44e596a13efeda518b9d845a619450/pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1", size = 88069, upload-time = "2025-07-01T13:30:59.346Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396, upload-time = "2025-07-01T13:30:56.632Z" }, +] + [[package]] name = "python-discovery" version = "1.2.2" @@ -1242,6 +1264,7 @@ dev = [ { name = "pre-commit" }, { name = "pytest" }, { name = "pytest-cov" }, + { name = "pytest-xdist" }, { name = "ruff" }, ] logfire = [ @@ -1268,6 +1291,7 @@ requires-dist = [ { name = "pyaml", specifier = ">=23.0" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0" }, { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=5.0" }, + { name = "pytest-xdist", marker = "extra == 'dev'", specifier = ">=3.5" }, { name = "python-multipart", marker = "extra == 'ui'", specifier = ">=0.0.20" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.7" }, { name = "starlette", marker = "extra == 'ui'", specifier = ">=0.36" }, From 676e7d3b23d2a7eaa535c28ccc42012f13748558 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Sun, 13 Sep 2026 18:04:11 -0700 Subject: [PATCH 88/88] docs: remove Copilot acceptance protocol --- docs/copilot-capture.md | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/docs/copilot-capture.md b/docs/copilot-capture.md index 8da2ed5..f935d30 100644 --- a/docs/copilot-capture.md +++ b/docs/copilot-capture.md @@ -157,19 +157,3 @@ compaction, abort, unknown-version, delayed-row, revision, retry, identical concurrent-tool, nested-child, and uncertain-attribution behavior. They are deterministic regression inputs, not live validation. Optional future live tests remain separate and require no CI credentials. - -## V2 static acceptance - -This document is the V2 acceptance record for Copilot CLI archive capture. -It covers V1-to-V2 upgrade (immutable source envelopes, versioned derived -indexes, `--rebuild` without resetting the export ledger), cross-source -correctness (transcript/hook/database reconstruction without live Copilot -files), exact join evidence (no timestamp/row-count/tool-name proof), -accounting corrections (revision replacement vs generation-reuse quarantine, -pre-delivery job rewrite vs post-delivery conflict), history export opt-in -(`--export` defaulting false; enqueue is not delivery), and unresolved source -limitations (no native VS Code/cloud capture, no shared assistant-message ID, -hooks without invocation IDs). No pytest run is required for this static -acceptance, and it does not claim native VS Code or Copilot cloud support. - -VERDICT: PASS
cache read cache creation reasoningcopilot native billing (nano-AIU)duration_msTTFT (ms)total ts
{{ row.cache_read_input_tokens if row.cache_read_input_tokens is not none else "-" }} {{ row.cache_creation_input_tokens if row.cache_creation_input_tokens is not none else "-" }} {{ row.reasoning_output_tokens if row.reasoning_output_tokens is not none else "-" }}{{ row.copilot_billing_nano_aiu if row.copilot_billing_nano_aiu is not none else "-" }}{{ row.duration_ms if row.duration_ms is not none else "-" }}{{ row.output_ttft_ms if row.output_ttft_ms is not none else "-" }}{{ row.total_tokens }} {{ row.ts }}