diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 465bf713..1636f302 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -37,12 +37,9 @@ jobs: python-version: "3.13" - os: ubuntu-latest python-version: "3.14" - # Windows support is experimental: minimum supported version plus a - # recent stable, to bound runner cost. + # Exercise Windows on the project's primary Python version. - os: windows-latest - python-version: "3.11" - - os: windows-latest - python-version: "3.13" + python-version: "3.12" steps: - uses: actions/checkout@v5 diff --git a/README.md b/README.md index 6bff94fb..02789680 100644 --- a/README.md +++ b/README.md @@ -9,14 +9,14 @@ [![Python](https://img.shields.io/pypi/pyversions/thrdi.svg)](https://pypi.org/project/thrdi/) [![License](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE) -Trace every agent session on your machine — Claude Code, Codex, Cursor — into one history you and your agents can manage, search, and evaluate. +Trace every agent session on your machine — Claude Code, Codex, Cursor, GitHub Copilot CLI — into one history you and your agents can manage, search, and evaluate. ## Install > **Windows support is experimental.** The test suite runs on Windows in CI, and -> Claude Code tracing is the verified integration there. The Codex CLI and Cursor -> installers are implemented but have not been verified against those tools on -> Windows. Please report Windows problems at the +> Claude Code tracing is the verified integration there. The Codex CLI, Cursor, +> and Copilot CLI installers are implemented but have not been live-certified +> against those tools on Windows. Please report Windows problems at the > [issue tracker](https://github.com/duncankmckinnon/thirdeye/issues). See > [docs/windows.md](docs/windows.md) for the deliberate platform differences. @@ -79,10 +79,65 @@ rerun `thirdeye skills add --force` to refresh them. See ## Enable tracing ```bash -thirdeye add --claude # also: --cursor, --codex +thirdeye add --claude # also: --cursor, --codex, --copilot ``` -To detach: `thirdeye remove --claude`. +To detach: `thirdeye remove --claude` (also `--cursor`, `--codex`, `--copilot`). + +## Copilot CLI capture + +GitHub Copilot CLI capture keeps a durable, local V1 archive and derives V2 +views from that archive. V1 source records are immutable: V2 never rewrites +them, and it can rebuild its derived turns, usage accounting, and export queue +after the original Copilot files have gone away. + +```bash +thirdeye add --copilot +thirdeye copilot status --source-home "$COPILOT_HOME" +thirdeye copilot sync --source-home "$COPILOT_HOME" +thirdeye copilot reconcile +thirdeye copilot reconcile --session-id --rebuild +thirdeye copilot watch --source-home "$COPILOT_HOME" --interval 1 +thirdeye remove --copilot +``` + +`--source-home` is optional. Resolution is `--source-home`, then `COPILOT_HOME`, +then `~/.copilot`. Capture reads only that home's `session-state/**/events.jsonl`, +`workspace.yaml`, and `session-store.db` (`sessions`, `turns`, +`assistant_usage_events`). It does not read credentials, token-bearing config, +or other Copilot files. + +`thirdeye add --copilot` writes user-level hooks at +`$COPILOT_HOME/hooks/thirdeye.json` (default `~/.copilot/hooks/thirdeye.json`). +The 1.0.83 live probe used repository hooks; user-level installation is not a +claim that every Copilot runtime invokes them. `watch` is an explicit foreground +poller and is not started by add or setup. It can import transcripts and SQLite +rows even when hooks are absent. `sync` and `status` print recoverable source +diagnostics (locations and reasons, not prompt bodies). + +Normal `sync` and `reconcile` refresh local V2 projections only; they do not +initialize export eligibility. To mark already-completed retained history as +export-eligible, opt in explicitly: + +```bash +thirdeye copilot sync --export --source-home "$COPILOT_HOME" +thirdeye copilot reconcile --export +``` + +Watch and hook follow-up activate live-style export without that history +opt-in. Their first activation records a durable eligibility boundary even if +remote export is not presently configured: interactions already complete stay +local-only, while interactions still open at activation may be queued later if +they complete and remote export is configured. `--export` instead activates +with retained completed history included, or later removes those identities +from an existing boundary. Jobs are dispatched only when remote export is +configured. `--rebuild` resets only reproducible derived state; it preserves +the raw archive and the separate delivery ledger. See +[Copilot CLI capture and reconciliation](docs/copilot-capture.md) for the +archive schema, attribution rules, correction behavior, and delivery limits. + +Passing unit tests is not live certification of Copilot CLI, native VS Code, or +Copilot cloud integrations. ## Read your history diff --git a/docs/copilot-capture.md b/docs/copilot-capture.md new file mode 100644 index 00000000..f935d308 --- /dev/null +++ b/docs/copilot-capture.md @@ -0,0 +1,159 @@ +# Copilot CLI capture and reconciliation + +thirdeye's Copilot integration is local, archive-first CLI capture. It supports +GitHub Copilot CLI recordings; it does not claim support for native VS Code, +Copilot cloud history, or Copilot as an evaluator/Ask backend. + +## V1 archive to V2 projections + +V1 capture stores immutable source envelopes in the normal local session Store. +Their event types are `copilot_transcript`, `copilot_database`, `copilot_hook`, +and `copilot_metadata`; each has a version-1 envelope and preserves its source +ID, native session ID, source timestamp when available, observation time, +payload, and locator. Transcript records retain native event IDs and content; +database records retain table/row/revision identities; hook records remain +observations rather than asserted tool executions. + +V2 is a separate, versioned derived state. It reads the retained archive rather +than the current Copilot home, so it can reprocess an older captured session +without launching an agent or retaining the original transcript/SQLite files. +It does not duplicate the raw records in default semantic views. + +```bash +# Rebuild every retained Copilot archive locally. +thirdeye copilot reconcile + +# Rebuild one archive's derived indexes only. +thirdeye copilot reconcile --session-id --rebuild +``` + +`--rebuild` replaces derived projections only. It does not delete V1 raw source +events, change stored-session/source identities, or reset export eligibility or +delivery history. + +## Raw and derived views + +Generic event commands (`events`, `show`, `tail`, `search`) continue to expose +the retained raw Store evidence. Session-scoped evaluation timelines also read +those raw events. Copilot user-turn views, turn-scoped dataset inputs, and +usage dashboards consume V2's derived projections: completed main interactions +via the projected turn index, and per-call usage via the rewritten `usage.jsonl` +sidecar. Raw transcript, database, and hook records therefore remain searchable +as captured, and they do not appear a second time as semantic events in default +turn views. Child-agent events and tools remain nested evidence within their +owning main interaction. + +The `status` command separates source capability/ingestion diagnostics from +projection and export state. In particular, pending, ambiguous, and conflicting +joins are local accounting states, and a configured or queued export is not a +successful remote delivery. + +## Source evidence and turn reconstruction + +For the observed Copilot CLI 1.0.83 corpus, a user interaction is identified by +`interactionId` with agent identity, not by a bare transcript `turnId` (which +resets). A child is attached through `agentId`, `parentToolCallId`, and the +parent task's `subagent.started.data.toolCallId`; interleaved transcript events +therefore remain in the correct subtree. Tool requests/executions are paired by +their invocation IDs, which distinguishes simultaneous identical calls. + +The retained external hook stream is supplementary. Its pre/post tool payloads +do not provide an invocation ID, and its coverage need not equal transcript +hook coverage. Child `userPromptSubmitted` and `agentStop` hooks put the child +ID in `sessionId`; `subagentStart`/`subagentStop` retain the parent session ID +and expose the child separately as `agentId`. Hooks must not be used as proof +of an exact tool pairing or as a session-ID-only turn state machine. A prompt +may also arrive before `SessionStart`. + +Unknown transcript schemas/events remain source evidence and searchable raw +records. Permission decisions and compaction produce normalized semantic +events; they are not inferred as tool executions or extra usage calls. +`assistant.abort` terminates a reconstructed turn with `status="interrupted"`. +A missing identity, missing completion, unknown version, or otherwise +incomplete record produces an explicit pending or diagnostic result rather +than an invented completed turn. + +## Usage accounting and attribution + +`assistant_usage_events` database rows are the primary per-call accounting +source. Input counts include cache tokens when supplied; missing usage stays +missing rather than becoming zero. Checkpoints and shutdown totals validate the +database ledger but never add a second charge. Auxiliary `model.*` title +generation is classified separately and cannot inflate conversation totals. +Unknown model providers remain `unknown`; Copilot nano-AI-unit billing is kept +separate from any estimated USD model price. + +Each logical database call has a stable accounting identity across revisions +of the same generation and row. An authoritative row correction replaces its +local derived `UsageRow` under that identity. Database generation/row-ID reuse +is a conflict: V2 quarantines both logical identities, drops their `UsageRow`s, +and emits a `usage_row_id_reuse` error rather than charging either call. An +incompatible correction of one logical call is quarantined as +`usage_revision_conflict` and is not counted as another charge. Delayed rows +remain eligible for a later reconciliation. + +The observed database schema has user `turn_index`, agent ID, parent tool-call +ID, model, ordering, finish/tool evidence, and token/billing/latency fields, +but no shared assistant-message/provider-call ID. As a result, attribution is +deliberately conservative: + +- `matched` is direct only with direct evidence, or inferred only when the + interaction, agent, model, order, and finish/tool evidence form one uniquely + consistent candidate. Its evidence is retained with the attribution. +- `pending` means more source evidence may resolve ownership. +- `ambiguous` lists competing candidates and chooses none. +- `conflicting` quarantines incompatible join evidence. Incompatible database + revisions are a separate `usage_revision_conflict` diagnostic, not this + attribution status. + +An unmatched terminal usage row still appears in local accounting. It can be +represented as an explicit user-turn/agent accounting span, or as session-level +accounting when no user-turn ownership is known; thirdeye never fabricates a +user turn merely to place tokens. + +## Export eligibility, retries, and corrections + +Local reconciliation never exports history by default. Plain `sync` and +`reconcile` refresh local projections only and do not initialize the export +ledger. `sync --export` or `reconcile --export` calls export assembly with +history included: that first activation marks the selected completed retained +history as eligible. Watch and hook follow-up also call export assembly, but +without the history opt-in. Their first activation records a durable boundary +even when remote export is not presently configured: already-terminal history +stays local-only, while an interaction open at activation becomes eligible if +it completes later. A later `--export` can remove those identities from an +existing boundary. The boundary survives restart and derived-state rebuild. +Jobs are dispatched only when remote export is configured; a configured or +queued export is not a successful delivery. + +Export assembly writes durable, deterministic local jobs. Matched accounting is +placed on its chat span; unmatched accounting uses one explicit accounting span. +The placement ledger prevents a logical token identity from being emitted in +both locations. If fallback accounting has already been delivered, a later +local match cannot re-export those tokens on the chat span. Pending rows stay +retryable; a correction after confirmed delivery is reported as an accounting +conflict rather than silently adding a new charge. + +Remote OpenTelemetry delivery is not transactionally exactly once. A process +can crash after a remote flush and before durable acknowledgement, so a retry +may resend a deterministic span. Durable jobs, deterministic IDs, and delivery +claims minimize that window and allow recovery, but cannot prove remote receipt +in every crash case. `status` reports queued work and last errors separately +from confirmed delivery. + +## Known source limitations + +The checked-in 1.0.83 corpus establishes two main interactions, one explore +child, five tool executions, twenty external hook observations, and six +database usage calls. The six database calls reconcile to the observed shutdown +totals, but the lack of a native shared assistant-message/provider-call ID +means comprehensive local capture does not make every message join exact. + +Interactive, folder-trusted capture produced hooks in the probe. Earlier +non-interactive probes did not; the cause was not proven, so this is not a +general claim that non-interactive hooks are unsupported. Watch/import remains +useful without hooks. Synthetic schema-derived fixtures cover permission, +compaction, abort, unknown-version, delayed-row, revision, retry, identical +concurrent-tool, nested-child, and uncertain-attribution behavior. They are +deterministic regression inputs, not live validation. Optional future live +tests remain separate and require no CI credentials. diff --git a/docs/windows.md b/docs/windows.md index 0f275314..81c4535c 100644 --- a/docs/windows.md +++ b/docs/windows.md @@ -2,9 +2,10 @@ Windows support is **experimental**. The test suite runs on `windows-latest` in CI across Python 3.11 and 3.13, and Claude Code tracing is the verified -integration. The Codex CLI and Cursor installers are implemented but have not -been exercised against those tools on Windows. Please file Windows issues at the -[issue tracker](https://github.com/duncankmckinnon/thirdeye/issues). +integration. The Codex CLI, Cursor, and Copilot CLI installers are implemented +but have not been live-certified against those tools on Windows. Passing Copilot +unit tests is not live certification of Copilot CLI. Please file Windows issues +at the [issue tracker](https://github.com/duncankmckinnon/thirdeye/issues). Install with `pipx` or `uv` — Homebrew stays macOS/Linux only. @@ -35,12 +36,14 @@ A copied skill does not track a thirdeye upgrade the way a symlink does. After `pipx upgrade thrdi` (or `uv tool upgrade thrdi`), rerun `thirdeye skills add --force` to refresh the copied skills. -### 3. Unverified Codex and Cursor hooks +### 3. Unverified Codex, Cursor, and Copilot hooks -The Codex CLI and Cursor installers are correct by construction but have not been -run against the real tools on Windows. How each tool invokes a hook command -there — `cmd.exe`, PowerShell, or a direct `CreateProcess` — is unconfirmed, and -therefore so is whether a path containing spaces needs quoting. +The Codex CLI, Cursor, and Copilot CLI installers are correct by construction +but have not been run against the real tools on Windows. How each tool invokes a +hook command there — `cmd.exe`, PowerShell, or a direct `CreateProcess` — is +unconfirmed, and therefore so is whether a path containing spaces needs quoting. +Copilot V1 still writes user-level `$COPILOT_HOME/hooks/thirdeye.json` (default +`~/.copilot/hooks/thirdeye.json`); the 1.0.83 probe used repository hooks. To sidestep the question, on Windows only, when the resolved hook binary path contains a space thirdeye writes the bare binary name into the tool's config diff --git a/pyproject.toml b/pyproject.toml index 082e8195..71ee0693 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,6 +60,7 @@ thirdeye-codex-pre-compact = "thirdeye.platforms.codex.hooks_json:pre_compact" thirdeye-codex-post-compact = "thirdeye.platforms.codex.hooks_json:post_compact" thirdeye-codex-session-end = "thirdeye.platforms.codex.hooks_json:session_end" thirdeye-cursor-hook = "thirdeye.platforms.cursor.hook:main" +thirdeye-copilot-hook = "thirdeye.platforms.copilot.hooks:main" [build-system] requires = ["setuptools>=64", "setuptools-scm>=8", "wheel"] diff --git a/src/thirdeye/_compat/fsops.py b/src/thirdeye/_compat/fsops.py index 947429d2..2a6a6dee 100644 --- a/src/thirdeye/_compat/fsops.py +++ b/src/thirdeye/_compat/fsops.py @@ -62,6 +62,27 @@ def unlink(path: Path, *, missing_ok: bool = False) -> None: path.unlink(missing_ok=missing_ok) +def sync_directory(path: Path | str) -> None: + """Best-effort ``fsync`` of a directory after ``replace`` or ``unlink``. + + Publishing a file with tmp-plus-replace (or removing a journal) is not + durable until the directory entry itself is synced. Some platforms, + notably Windows, reject directory ``fsync``; those errors are ignored so + callers stay portable. + """ + path = Path(path) + try: + fd = os.open(path, os.O_RDONLY) + except OSError: + return + try: + os.fsync(fd) + except OSError: + return + finally: + os.close(fd) + + def read_text(path: Path | str, *, encoding: str = "utf-8") -> str: """``Path.read_text`` with bounded retry on Windows ``PermissionError``. diff --git a/src/thirdeye/cli.py b/src/thirdeye/cli.py index d50f6402..b0260b04 100644 --- a/src/thirdeye/cli.py +++ b/src/thirdeye/cli.py @@ -7,6 +7,7 @@ from thirdeye.commands.add import add, remove from thirdeye.commands.agent import agent_cmd from thirdeye.commands.capture_env import capture_env_group +from thirdeye.commands.copilot import copilot_group from thirdeye.commands.eval import eval_group from thirdeye.commands.ingest import ingest from thirdeye.commands.logfire_cmd import logfire_group @@ -28,6 +29,7 @@ def main() -> None: main.add_command(add) main.add_command(remove) main.add_command(ingest) +main.add_command(copilot_group) main.add_command(list_sessions) main.add_command(show) main.add_command(events) diff --git a/src/thirdeye/commands/add.py b/src/thirdeye/commands/add.py index ed5f7f80..2b526db5 100644 --- a/src/thirdeye/commands/add.py +++ b/src/thirdeye/commands/add.py @@ -9,12 +9,14 @@ from thirdeye.platforms.base import Platform from thirdeye.platforms.claude.install import ClaudePlatform from thirdeye.platforms.codex.install import CodexPlatform +from thirdeye.platforms.copilot.install import CopilotPlatform from thirdeye.platforms.cursor.install import CursorPlatform PLATFORMS: dict[str, type[Platform]] = { "claude": ClaudePlatform, "codex": CodexPlatform, "cursor": CursorPlatform, + "copilot": CopilotPlatform, } # Config files that may still reference console scripts for platforms this @@ -25,6 +27,9 @@ def _platform_options(fn): + fn = click.option( + "--copilot", "platform_flag", flag_value="copilot", help="GitHub Copilot CLI." + )(fn) fn = click.option("--cursor", "platform_flag", flag_value="cursor", help="Cursor.")(fn) fn = click.option("--codex", "platform_flag", flag_value="codex", help="Codex CLI.")(fn) fn = click.option("--claude", "platform_flag", flag_value="claude", help="Claude Code.")(fn) @@ -33,7 +38,7 @@ def _platform_options(fn): def _resolve_platform(platform_flag: str | None, *, force: bool = False) -> Platform: if not platform_flag: - raise click.UsageError("Pick a platform: --claude, --codex, --cursor") + raise click.UsageError("Pick a platform: " + ", ".join(f"--{name}" for name in PLATFORMS)) platform_cls = PLATFORMS[platform_flag] if platform_flag == "codex" and force: return platform_cls(force=True) diff --git a/src/thirdeye/commands/copilot.py b/src/thirdeye/commands/copilot.py new file mode 100644 index 00000000..1d75da82 --- /dev/null +++ b/src/thirdeye/commands/copilot.py @@ -0,0 +1,389 @@ +from __future__ import annotations + +import math +import os +from pathlib import Path +from typing import Any + +import click + +from thirdeye.config import Config +from thirdeye.platforms.copilot.capture import ( + _is_absence_diagnostic, + _is_retryable_code, +) +from thirdeye.platforms.copilot.capture import sync as capture_sync +from thirdeye.platforms.copilot.constants import COPILOT_HOME_ENV +from thirdeye.platforms.copilot.identity import ( + resolve_sources, + validate_native_id, + validate_stored_session_id, +) +from thirdeye.platforms.copilot.runtime import ( + all_archived_session_ids, + reconcile_archived_sessions, + reconcile_session, + reconcile_stored_session, +) +from thirdeye.platforms.copilot.status import capture_status +from thirdeye.platforms.copilot.types import SourcePaths, SyncResult +from thirdeye.platforms.copilot.watch import watch as watch_loop + +_MIN_WATCH_INTERVAL = 0.1 +_LOCATOR_KEYS = ( + "file", + "path", + "table", + "row_id", + "offset", + "byte_offset", + "generation", + "file_generation", +) + + +def _source_home_option(fn): + return click.option( + "--source-home", + type=click.Path(path_type=Path), + default=None, + help="Copilot home directory. Overrides COPILOT_HOME and ~/.copilot.", + )(fn) + + +def _validate_interval(_ctx: click.Context, _param: click.Parameter, value: float) -> float: + if not math.isfinite(value) or value < _MIN_WATCH_INTERVAL: + raise click.BadParameter( + f"interval must be finite and at least {_MIN_WATCH_INTERVAL} seconds" + ) + return value + + +def _requested_home(source_home: Path | None) -> Path: + if source_home is not None: + return source_home + configured = os.environ.get(COPILOT_HOME_ENV) + if configured: + return Path(configured) + return Path.home() / ".copilot" + + +def _resolve_paths(source_home: Path | None) -> SourcePaths: + try: + return resolve_sources(source_home) + except ValueError as exc: + raise click.ClickException( + f"Cannot resolve Copilot sources at {_requested_home(source_home)}: {exc}" + ) from exc + + +def _counts_line(result: SyncResult) -> str: + return ( + f"sessions={result['sessions']} " + f"records_written={result['records_written']} " + f"duplicate_records={result['duplicate_records']} " + f"pending={result['pending']} " + f"errors={result['errors']}" + ) + + +def _format_locator(locator: dict[str, Any]) -> str | None: + parts: list[str] = [] + for key in _LOCATOR_KEYS: + value = locator.get(key) + if isinstance(value, (str, int, float)) and not isinstance(value, bool): + parts.append(f"{key}={value}") + return ",".join(parts) if parts else None + + +def _format_diagnostic(error: object) -> str: + if not isinstance(error, dict): + return str(error) + code = error.get("code") or error.get("kind") or "error" + if not isinstance(code, str) or not code: + code = "error" + message = error.get("message") or error.get("reason") + bits = [f"{code}: {message}" if isinstance(message, str) and message else code] + session = error.get("session") or error.get("native_session_id") + if isinstance(session, str) and session: + bits.append(f"session={session}") + path = error.get("path") + if isinstance(path, str) and path: + bits.append(f"path={path}") + source_id = error.get("source_id") + if isinstance(source_id, str) and source_id: + bits.append(f"source_id={source_id}") + locator = error.get("locator") + if isinstance(locator, dict): + formatted = _format_locator(locator) + if formatted: + bits.append(f"locator={formatted}") + return " ".join(bits) + + +def _format_last_hook(record: object) -> str: + if not isinstance(record, dict): + return "none" + payload = record.get("payload") if isinstance(record.get("payload"), dict) else {} + locator = record.get("locator") if isinstance(record.get("locator"), dict) else {} + event = payload.get("event") or locator.get("event") or "unknown" + observed_at = record.get("observed_at") + session = record.get("native_session_id") + observed = observed_at if isinstance(observed_at, str) and observed_at else "unknown" + native_id = session if isinstance(session, str) and session else "unknown" + return f"observed_at={observed} event={event} session={native_id}" + + +def _format_timestamp(value: object) -> str: + return value if isinstance(value, str) and value else "none" + + +def _print_capability(label: str, capability: object) -> None: + data = capability if isinstance(capability, dict) else {} + path = data.get("path", "") + click.echo( + f"{label}: exists={data.get('exists', False)} " + f"readable={data.get('readable', False)} path={path}" + ) + + +def _status_error_affects_exit(error: object) -> bool: + """Reuse capture's absence/retryable classifiers; missing timestamps are diagnoses.""" + + if not isinstance(error, dict): + return True + if _is_absence_diagnostic(error): + return False + if _is_retryable_code(error.get("code")): + return False + if error.get("kind") in {"missing_source_time", "copilot_reconcile_error"}: + return False + return True + + +def _print_sync_result(result: SyncResult) -> None: + click.echo(_counts_line(result)) + + +def _print_reconcile_result(result: dict[str, int]) -> None: + click.echo( + "reconcile " + f"events={result.get('events', 0)} " + f"usage={result.get('usage', 0)} " + f"turns={result.get('turns', 0)} " + f"exports_queued={result.get('exports', 0)} " + f"pending={result.get('pending', 0)} " + f"ambiguous={result.get('ambiguous', 0)} " + f"conflicting={result.get('conflicting', 0)} " + f"errors={result.get('errors', 0)}" + ) + + +def _print_sync_followup(config: Config, paths: SourcePaths, result: SyncResult) -> None: + if result["errors"] == 0 and result["pending"] == 0: + return + if result["errors"] > 0 and result["sessions"] > 0: + click.echo(f"imported with {result['errors']} source diagnostics") + if result["errors"] > 0: + status = capture_status(config, paths) + for error in status.get("errors") or []: + click.echo(f" {_format_diagnostic(error)}") + click.echo("See `thirdeye copilot status` for source diagnostics.") + if result["pending"] > 0: + click.echo("Pending work remains; rerun sync or use `thirdeye copilot watch`.") + + +def _print_status(status: dict[str, Any]) -> None: + paths = status.get("paths") or {} + installation = status.get("installation") or {} + pending = status.get("pending") or {} + capabilities = status.get("capabilities") or {} + configured = bool(installation.get("configured")) + click.echo(f"Copilot home: {paths.get('home', '')}") + click.echo(f"Session root: {paths.get('session_root', '')}") + click.echo(f"Database: {paths.get('database', '')}") + click.echo(f"Hooks file: {installation.get('hooks_file', '')}") + if configured: + click.echo("Hooks: configured") + else: + click.echo("Hooks: not configured (informational; persisted import still works)") + click.echo("Install hooks with: thirdeye add --copilot") + _print_capability("Transcripts", capabilities.get("transcripts")) + _print_capability("Database", capabilities.get("database")) + _print_capability("Database WAL", capabilities.get("database_wal")) + click.echo(f"Last observed hook: {_format_last_hook(status.get('last_observed_hook'))}") + click.echo(f"Last successful import: {_format_timestamp(status.get('last_successful_import'))}") + click.echo( + "Pending: " + f"spool_records={pending.get('spool_records', 0)} " + f"followup={pending.get('followup', 0)} " + f"leases={pending.get('leases', 0)} " + f"journals={pending.get('journals', 0)}" + ) + projections = [ + item.get("projection") for item in status.get("sessions") or [] if isinstance(item, dict) + ] + exports = [ + item.get("export") for item in status.get("sessions") or [] if isinstance(item, dict) + ] + click.echo( + "Projection: " + f"pending={sum(item.get('pending', 0) for item in projections if isinstance(item, dict))} " + f"ambiguous={sum(item.get('ambiguous', 0) for item in projections if isinstance(item, dict))} " + f"conflicting={sum(item.get('conflicting', 0) for item in projections if isinstance(item, dict))}" + ) + click.echo( + "Export: " + f"activated={sum(1 for item in exports if isinstance(item, dict) and item.get('activated'))} " + f"queued={sum(item.get('queued', 0) for item in exports if isinstance(item, dict))} " + f"delivered={sum(item.get('delivered', 0) for item in exports if isinstance(item, dict))} " + f"errors={sum(item.get('errors', 0) for item in exports if isinstance(item, dict))}" + ) + errors = status.get("errors") or [] + informational = [error for error in errors if not _status_error_affects_exit(error)] + blocking = [error for error in errors if _status_error_affects_exit(error)] + if informational: + click.echo("Informational:") + for error in informational: + click.echo(f" {_format_diagnostic(error)}") + if blocking: + click.echo("Source errors:") + for error in blocking: + click.echo(f" {_format_diagnostic(error)}") + else: + click.echo("Source errors: none") + click.echo( + "Start an interactive Copilot CLI session in a trusted folder, then rerun; " + "'Last observed hook' should update." + ) + + +@click.group(name="copilot", help="Ingest local GitHub Copilot CLI recordings.") +def copilot_group() -> None: + pass + + +@copilot_group.command("sync", help="One-shot local ingestion of Copilot CLI recordings.") +@click.option("--session-id", default=None, help="Exact native Copilot session ID.") +@click.option( + "--export", + "export_history", + is_flag=True, + default=False, + help="Queue completed archived history for export after local reconciliation.", +) +@_source_home_option +def sync_cmd(session_id: str | None, export_history: bool, source_home: Path | None) -> None: + config = Config.load() + paths = _resolve_paths(source_home) + try: + if session_id is not None: + validate_native_id(session_id) + result = capture_sync(config, paths, session_id=session_id) + except ValueError as exc: + raise click.ClickException(str(exc)) from exc + _print_sync_result(result) + # Every sync refreshes only local V2 projections. Explicit opt-in is the + # only sync path that can include completed history in export eligibility. + # A selected native ID never exports or rebuilds unrelated archives. + if session_id is None: + if export_history: + for derived in reconcile_archived_sessions( + config, paths, export=True, include_history=True + ).values(): + _print_reconcile_result(derived) + else: + reconcile_archived_sessions(config, paths) + elif result["sessions"] > 0: + derived = reconcile_session( + config, + paths, + session_id, + export=export_history, + include_history=export_history, + ) + if export_history: + _print_reconcile_result(derived) + _print_sync_followup(config, paths, result) + if session_id is not None and result["sessions"] == 0: + raise click.ClickException( + f"Copilot session {session_id} was not found or could not be imported" + ) + + +@copilot_group.command( + "reconcile", help="Build Copilot V2 projections from retained local archives." +) +@click.option("--session-id", default=None, help="Exact stored Thirdeye Copilot session ID.") +@click.option("--rebuild", is_flag=True, default=False, help="Rebuild derived state only.") +@click.option( + "--export", + "export_history", + is_flag=True, + default=False, + help="Queue completed archived history for export after local reconciliation.", +) +def reconcile_cmd(session_id: str | None, rebuild: bool, export_history: bool) -> None: + config = Config.load() + if session_id is None: + stored_ids = all_archived_session_ids(config) + else: + try: + validate_stored_session_id(session_id) + stored_ids = [session_id] + except ValueError as exc: + raise click.ClickException(str(exc)) from exc + failed = False + for stored_id in stored_ids: + try: + result = reconcile_stored_session( + config, + stored_id, + rebuild=rebuild, + export=export_history, + include_history=export_history, + ) + except ValueError as exc: + raise click.ClickException(str(exc)) from exc + _print_reconcile_result(result) + if result.get("errors", 0): + failed = True + if failed: + raise click.ClickException("Copilot reconciliation completed with errors") + + +@copilot_group.command("watch", help="Poll local Copilot CLI recordings until interrupted.") +@_source_home_option +@click.option( + "--interval", + type=float, + default=1.0, + show_default=True, + callback=_validate_interval, + help="Seconds between source polls. Must be finite and at least 0.1.", +) +def watch_cmd(source_home: Path | None, interval: float) -> None: + config = Config.load() + paths = _resolve_paths(source_home) + click.echo( + f"Watching Copilot home {paths['home']} every {interval}s. " + "Configured exports are queued locally. Ctrl-C to stop." + ) + try: + watch_loop(config, paths, interval=interval) + except KeyboardInterrupt: + pass + except ValueError as exc: + raise click.ClickException(str(exc)) from exc + click.echo("Stopped watching Copilot recordings.") + + +@copilot_group.command("status", help="Show Copilot CLI capture paths, hooks, and health.") +@_source_home_option +def status_cmd(source_home: Path | None) -> None: + config = Config.load() + paths = _resolve_paths(source_home) + status = capture_status(config, paths) + _print_status(status) + if any(_status_error_affects_exit(error) for error in status.get("errors") or []): + raise click.ClickException("Copilot source errors prevent a healthy status") diff --git a/src/thirdeye/commands/setup.py b/src/thirdeye/commands/setup.py index 179f4d9c..fad21535 100644 --- a/src/thirdeye/commands/setup.py +++ b/src/thirdeye/commands/setup.py @@ -16,6 +16,7 @@ "claude": "Claude Code", "codex": "Codex CLI", "cursor": "Cursor", + "copilot": "GitHub Copilot CLI", } _SKILL_TARGET_LABELS = { diff --git a/src/thirdeye/config.py b/src/thirdeye/config.py index 1505024b..d48449ad 100644 --- a/src/thirdeye/config.py +++ b/src/thirdeye/config.py @@ -91,9 +91,9 @@ class Config: logfire: LogfireSettings = field(default_factory=LogfireSettings) @classmethod - def load(cls) -> Config: - root = default_root() - raw = _read_config_yaml(root / "config.yaml") + def load(cls, root: Path | None = None) -> Config: + resolved = Path(root) if root is not None else default_root() + raw = _read_config_yaml(resolved / "config.yaml") # THIRDEYE_CAPTURE_ENV wins when set, so a one-off run can still # override; otherwise fall back to config.yaml's ``capture_env`` so # capture does not depend on the launching shell exporting anything @@ -102,7 +102,7 @@ def load(cls) -> Config: if not patterns: patterns = _coerce_patterns(raw.get("capture_env")) return cls( - root=root, + root=resolved, capture_env_patterns=patterns, logfire=LogfireSettings.from_dict(raw.get("logfire")), ) diff --git a/src/thirdeye/otel_export.py b/src/thirdeye/otel_export.py index b3885e5f..bbb202db 100644 --- a/src/thirdeye/otel_export.py +++ b/src/thirdeye/otel_export.py @@ -532,6 +532,19 @@ def _write_job(thirdeye_home: Path, payload: dict[str, Any]) -> Path: return job_path +def _write_accounting_job(thirdeye_home: Path, payload: dict[str, Any]) -> Path: + """Persist one deterministic accounting job without creating duplicates.""" + jobs_dir = otel_jobs_dir(thirdeye_home) + jobs_dir.mkdir(parents=True, exist_ok=True) + job_id = str(payload["job_id"]) + digest = hashlib.sha256(job_id.encode("utf-8")).hexdigest() + job_path = jobs_dir / f"accounting-{digest}.json" + queued = {**payload, "state": "queued", "attempt": int(payload.get("attempt", 0))} + if not _atomic_create(job_path, json.dumps(queued, default=str)): + return job_path + return job_path + + def _spawn(job_path: Path) -> None: """Hand a job file to a detached ``thirdeye.otel_worker``. @@ -693,6 +706,105 @@ def export_spans( return False +def export_session_accounting( + config: Config, + session_dir_: Path, + session_id: str, + platform: str, + cwd: str, + accounting: dict[str, Any], + *, + captured_env: dict[str, str] | None = None, +) -> bool: + """Queue accounting that has no owning user turn.""" + if not config.logfire.enabled or not config.logfire.token: + return False + try: + accounting_id = str(accounting["accounting_id"]) + logical_span_id = f"accounting:{session_id}:{accounting_id}" + job_path = _write_accounting_job( + config.root, + { + "job_id": logical_span_id, + "kind": "session_accounting", + "session_dir": str(session_dir_), + "session_id": session_id, + "platform": platform, + "cwd": cwd, + "captured_attributes": _resolve_captured_attributes(config, captured_env), + "accounting_id": accounting_id, + "destination": "session-accounting-span", + "usage": dict(accounting["usage"]), + "attribution_status": str(accounting["attribution_status"]), + "agent_id": accounting.get("agent_id"), + "attributes": dict(accounting.get("attributes") or {}), + "span_id": logical_span_id, + }, + ) + _spawn(job_path) + return True + except Exception as exc: + log_capture_error( + thirdeye_home=config.root, + phase="logfire_session_accounting_export_spawn", + error=exc, + platform=platform, + session_id=session_id, + ) + return False + + +def export_turn_accounting( + config: Config, + session_dir_: Path, + session_id: str, + platform: str, + cwd: str, + turn_id: str, + accounting: dict[str, Any], + *, + turn_span_id: str | int | None = None, + captured_env: dict[str, str] | None = None, +) -> bool: + """Queue unmatched accounting owned by an already-exported user turn.""" + if not config.logfire.enabled or not config.logfire.token: + return False + try: + accounting_id = str(accounting["accounting_id"]) + logical_span_id = f"accounting:{session_id}:{turn_id}:{accounting_id}" + payload: dict[str, Any] = { + "job_id": logical_span_id, + "kind": "turn_accounting", + "session_dir": str(session_dir_), + "session_id": session_id, + "platform": platform, + "cwd": cwd, + "captured_attributes": _resolve_captured_attributes(config, captured_env), + "turn_id": turn_id, + "accounting_id": accounting_id, + "destination": "turn-accounting-span", + "usage": dict(accounting["usage"]), + "attribution_status": str(accounting["attribution_status"]), + "agent_id": accounting.get("agent_id"), + "attributes": dict(accounting.get("attributes") or {}), + "span_id": logical_span_id, + } + if turn_span_id is not None: + payload["turn_span_id"] = str(turn_span_id) + job_path = _write_accounting_job(config.root, payload) + _spawn(job_path) + return True + except Exception as exc: + log_capture_error( + thirdeye_home=config.root, + phase="logfire_turn_accounting_export_spawn", + error=exc, + platform=platform, + session_id=session_id, + ) + return False + + def export_subagent_turn( config: Config, session_dir_: Path, @@ -886,6 +998,136 @@ def _export_subagent_turn_inner( claim_path.write_text("sent", encoding="utf-8", newline="\n") +def _require_export_instance(config: Config, platform: str): + instance = _get_instance(config, platform) + if instance is None: + raise RuntimeError("logfire instance unavailable for accounting export") + return instance + + +def _resolve_session_parent( + *, + instance: Any, + session_dir_: Path, + session_id: str, + platform: str, + cwd: str, + fallback_ts: str, +) -> tuple[Any, tuple[int, int]]: + """Return (tracer, (trace_id, root_span_id)), creating the session root if needed.""" + tracer = instance.config.get_tracer_provider().get_tracer("thirdeye") + root_path = otel_state_path(session_dir_) + parent, root_lock = _root_or_ownership(root_path) + try: + if parent is None and root_lock is None: + raise RuntimeError("could not resolve or create session root") + if parent is None: + root_ns = _ts_to_ns(fallback_ts) + derived = ( + trace_id_for_session(platform, session_id), + root_span_id_for_session(platform, session_id), + ) + parent, created_root = _create_root_atomic(root_path, *derived) + if created_root: + root_span = _start_span_with_id( + tracer, + "session", + derived[1], + trace_id=derived[0], + start_time=root_ns, + attributes=_flatten_attrs( + { + **(_captured_attributes.get() or {}), + **_identity_attributes( + session_id=session_id, platform=platform, cwd=cwd + ), + } + ), + ) + root_span.end(end_time=root_ns) + finally: + if root_lock is not None: + fsops.unlink(root_lock, missing_ok=True) + if parent is None: + raise RuntimeError("could not resolve or create session root") + return tracer, parent + + +def _export_session_accounting_inner( + *, + config: Config, + session_dir_: Path, + session_id: str, + platform: str, + cwd: str, + accounting: dict[str, Any], +) -> None: + """Emit a session-owned accounting span under the durable session root.""" + instance = _require_export_instance(config, platform) + usage = dict(accounting.get("usage") or {}) + fallback_ts = _accounting_timestamp(usage, "1970-01-01T00:00:00.000Z") + tracer, parent = _resolve_session_parent( + instance=instance, + session_dir_=session_dir_, + session_id=session_id, + platform=platform, + cwd=cwd, + fallback_ts=fallback_ts, + ) + _export_accounting_span( + tracer, + _parent_context(*parent), + accounting, + platform=platform, + session_id=session_id, + cwd=cwd, + fallback_ts=fallback_ts, + ) + if instance.force_flush(timeout_millis=_FLUSH_TIMEOUT_MS) is False: + raise RuntimeError("session accounting export was not flushed") + + +def _export_turn_accounting_inner( + *, + config: Config, + session_dir_: Path, + session_id: str, + platform: str, + cwd: str, + turn_id: str, + accounting: dict[str, Any], + turn_span_id: str | int | None = None, +) -> None: + """Emit a turn-owned accounting span under existing turn/session context.""" + instance = _require_export_instance(config, platform) + usage = dict(accounting.get("usage") or {}) + fallback_ts = _accounting_timestamp(usage, "1970-01-01T00:00:00.000Z") + tracer, parent = _resolve_session_parent( + instance=instance, + session_dir_=session_dir_, + session_id=session_id, + platform=platform, + cwd=cwd, + fallback_ts=fallback_ts, + ) + if turn_span_id is not None and str(turn_span_id) != "": + parent_ctx = _parent_context(parent[0], int(turn_span_id)) + else: + parent_ctx = _parent_context(*parent) + _export_accounting_span( + tracer, + parent_ctx, + accounting, + platform=platform, + session_id=session_id, + cwd=cwd, + fallback_ts=fallback_ts, + turn_id=turn_id, + ) + if instance.force_flush(timeout_millis=_FLUSH_TIMEOUT_MS) is False: + raise RuntimeError("turn accounting export was not flushed") + + @lru_cache(maxsize=128) def _repo_name(cwd: str) -> str | None: """The name of the git repository `cwd` sits in, or None outside one. @@ -1046,21 +1288,8 @@ def _cost_attributes(attributes: dict[str, Any]) -> dict[str, Any]: return {} -def _chat_attributes( - call_or_attributes: dict[str, Any], - *, - session_id: str, - platform: str, - cwd: str, - turn_id: Any = None, - turn_span_id: Any = None, -) -> dict[str, Any]: - """Build flattened attributes for a chat span. - - Completed-turn export passes an LLM-call record, while live batch export - passes the already-built semantic attributes from its job. Accepting both - forms keeps the vocabulary and JSON handling in one place. - """ +def _chat_core_attributes(call_or_attributes: dict[str, Any]) -> tuple[dict[str, Any], str | None]: + """Return unflattened chat attributes and the provider name they imply.""" if all( key in call_or_attributes for key in ("input_messages", "output_messages", "provider", "usage") @@ -1090,10 +1319,27 @@ def _chat_attributes( for source, target in _USAGE_KEYS.items(): if source in usage: attributes[target] = usage[source] - else: - attributes = call_or_attributes - # Already-built attributes carry the provider under its final name. - provider = attributes.get("gen_ai.provider.name") + return attributes, provider + attributes = dict(call_or_attributes) + return attributes, attributes.get("gen_ai.provider.name") + + +def _chat_attributes( + call_or_attributes: dict[str, Any], + *, + session_id: str, + platform: str, + cwd: str, + turn_id: Any = None, + turn_span_id: Any = None, +) -> dict[str, Any]: + """Build flattened attributes for a chat span. + + Completed-turn export passes an LLM-call record, while live batch export + passes the already-built semantic attributes from its job. Accepting both + forms keeps the vocabulary and JSON handling in one place. + """ + attributes, provider = _chat_core_attributes(call_or_attributes) return _flatten_attrs( _merge_raw( attributes, @@ -1110,6 +1356,89 @@ def _chat_attributes( ) +def _has_copilot_native_billing(attributes: dict[str, Any]) -> bool: + return any( + key.startswith("copilot.billing") or "nano_aiu" in str(key).lower() for key in attributes + ) + + +def _accounting_attributes_raw(accounting: dict[str, Any]) -> dict[str, Any]: + """Unflattened projection of one usage row onto its export span.""" + usage = dict(accounting.get("usage") or {}) + attributes = _merge_raw( + usage, + accounting.get("attributes"), + { + "thirdeye.accounting.id": str(accounting["accounting_id"]), + "thirdeye.accounting.attribution_status": str(accounting["attribution_status"]), + "thirdeye.accounting.agent_id": accounting.get("agent_id"), + "thirdeye.accounting.usage": usage, + }, + ) + # Native Copilot billing units are not an estimated USD model price. + if _has_copilot_native_billing(attributes): + attributes["thirdeye.accounting.billing.kind"] = "copilot-native-unit" + return attributes + + +def _accounting_attributes(accounting: dict[str, Any]) -> dict[str, Any]: + """Project an immutable usage row onto its one allowed export span.""" + return _flatten_attrs(_accounting_attributes_raw(accounting)) + + +def _accounting_span_id( + platform: str, session_id: str, accounting_id: str, turn_id: str | None = None +) -> int: + """Return an OTel-safe deterministic ID for an accounting fallback span.""" + return chat_span_id(platform, session_id, f"accounting:{turn_id or 'session'}:{accounting_id}") + + +def _accounting_timestamp(usage: dict[str, Any], fallback: str) -> str: + value = usage.get("ts") + if isinstance(value, str): + try: + _ts_to_ns(value) + except ValueError: + pass + else: + return value + return fallback + + +def _export_accounting_span( + tracer: Any, + parent_ctx: Any, + accounting: dict[str, Any], + *, + platform: str, + session_id: str, + cwd: str, + fallback_ts: str, + turn_id: str | None = None, +) -> None: + usage = dict(accounting.get("usage") or {}) + ts = _accounting_timestamp(usage, fallback_ts) + span = _start_span_with_id( + tracer, + "accounting", + _accounting_span_id(platform, session_id, str(accounting["accounting_id"]), turn_id), + parent_ctx=parent_ctx, + start_time=_ts_to_ns(ts), + attributes=_flatten_attrs( + _merge_raw( + _identity_attributes( + session_id=session_id, + platform=platform, + cwd=cwd, + turn_id=turn_id, + ), + _accounting_attributes_raw(accounting), + ) + ), + ) + span.end(end_time=_ts_to_ns(ts)) + + def _tool_attributes( attributes: dict[str, Any], *, @@ -1335,6 +1664,11 @@ def _export_turn_subtree( turn_span.end(end_time=_ts_to_ns(turn["end_ts"])) turn_ctx = turn_span.get_span_context() turn_parent_ctx = _parent_context(turn_ctx.trace_id, turn_ctx.span_id) + accounting_by_call = { + str(accounting["call_id"]): accounting + for accounting in turn.get("accounting_calls") or [] + if accounting.get("call_id") is not None + } for interaction in turn.get("interactions") or []: kind = interaction["kind"] @@ -1362,21 +1696,36 @@ def _export_turn_subtree( for llm_call in turn["llm_calls"]: model = llm_call.get("model") or "" - call_span = _start_span_with_id( - tracer, - f"chat {model}" if model else "chat", - chat_span_id(platform, session_id, llm_call["call_id"]), - parent_ctx=turn_parent_ctx, - start_time=_ts_to_ns(llm_call["start_ts"]), - attributes=_chat_attributes( - llm_call, + core_attrs, provider = _chat_core_attributes(llm_call) + raw_call_attrs = _merge_raw( + core_attrs, + _identity_attributes( session_id=session_id, platform=platform, cwd=cwd, turn_id=turn["turn_id"], turn_span_id=turn.get("turn_span_id"), + provider=provider, ), ) + accounting = accounting_by_call.get(str(llm_call["call_id"])) + if accounting is not None: + # Actual accounting, rather than the semantic LLM record, owns + # the token fields for this chat span. Merge raw then flatten once + # so one logfire.json_schema covers chat messages and usage JSON. + raw_call_attrs = _merge_raw(raw_call_attrs, _accounting_attributes_raw(accounting)) + if _has_copilot_native_billing(raw_call_attrs): + raw_call_attrs["thirdeye.accounting.billing.kind"] = "copilot-native-unit" + else: + raw_call_attrs = _merge_raw(raw_call_attrs, _cost_attributes(raw_call_attrs)) + call_span = _start_span_with_id( + tracer, + f"chat {model}" if model else "chat", + chat_span_id(platform, session_id, llm_call["call_id"]), + parent_ctx=turn_parent_ctx, + start_time=_ts_to_ns(llm_call["start_ts"]), + attributes=_flatten_attrs(raw_call_attrs), + ) call_span.end(end_time=_ts_to_ns(llm_call["end_ts"])) call_ctx = call_span.get_span_context() call_parent_ctx = _parent_context(call_ctx.trace_id, call_ctx.span_id) @@ -1402,6 +1751,24 @@ def _export_turn_subtree( ) tool_span.end(end_time=_ts_to_ns(tool_call["end_ts"])) + known_call_ids = {str(call["call_id"]) for call in turn["llm_calls"]} + for accounting in turn.get("accounting_calls") or []: + call_id = accounting.get("call_id") + if call_id is not None and str(call_id) in known_call_ids: + continue + # Unknown call ids do not prove chat-span ownership. Keep the usage on + # a concrete user-turn accounting span instead of inventing a chat. + _export_accounting_span( + tracer, + turn_parent_ctx, + accounting, + platform=platform, + session_id=session_id, + cwd=cwd, + fallback_ts=turn["end_ts"], + turn_id=turn["turn_id"], + ) + for orphan in turn.get("orphan_tool_calls") or []: parent_call_id = orphan["parent_call_id"] tool_call = orphan["tool_call"] diff --git a/src/thirdeye/otel_worker.py b/src/thirdeye/otel_worker.py index 2733310b..2323dacf 100644 --- a/src/thirdeye/otel_worker.py +++ b/src/thirdeye/otel_worker.py @@ -19,12 +19,161 @@ from __future__ import annotations import json +import os import sys +import time from pathlib import Path from typing import Any from thirdeye._compat import fsops +_JOB_CLAIM_STALE_S = 30.0 +_JOB_MAX_ATTEMPTS = 5 +_ACCOUNTING_KINDS = frozenset({"session_accounting", "turn_accounting"}) + + +def _write_job_state(job_path: Path, payload: dict[str, Any]) -> None: + """Atomically publish a claim/retry state without partial JSON.""" + temporary = job_path.with_name(f".{job_path.name}.{os.getpid()}.tmp") + temporary.write_text(json.dumps(payload, default=str), encoding="utf-8", newline="\n") + fsops.replace(temporary, job_path) + fsops.sync_directory(job_path.parent) + + +def _job_claim_path(job_path: Path) -> Path: + return job_path.with_suffix(f"{job_path.suffix}.claim") + + +def _create_job_claim(path: Path) -> bool: + try: + with path.open("x", encoding="utf-8") as handle: + handle.write(str(os.getpid())) + except FileExistsError: + return False + return True + + +def _claim_job(job_path: Path, payload: dict[str, Any]) -> dict[str, Any] | None: + """Recover a claimed-but-unsent job and claim it for this worker. + + Remote flush and local deletion cannot be a single transaction. A crash in + between can retry a deterministic span, reducing but not eliminating + remote duplicates. + """ + state = payload.get("state") + if state == "emitted": + fsops.unlink(job_path, missing_ok=True) + _release_job_claim(job_path) + return None + if state == "failed": + _release_job_claim(job_path) + return None + claim_path = _job_claim_path(job_path) + if not _create_job_claim(claim_path): + try: + stale = time.time() - claim_path.stat().st_mtime > _JOB_CLAIM_STALE_S + except OSError: + stale = True + if not stale: + return None + fsops.unlink(claim_path, missing_ok=True) + if not _create_job_claim(claim_path): + return None + claimed = dict(payload) + claimed["state"] = "claimed" + claimed["attempt"] = int(payload.get("attempt", 0)) + _write_job_state(job_path, claimed) + return claimed + + +def _release_job_claim(job_path: Path) -> None: + fsops.unlink(_job_claim_path(job_path), missing_ok=True) + + +def _retry_job(job_path: Path, payload: dict[str, Any]) -> None: + next_attempt = int(payload.get("attempt", 0)) + 1 + retry = dict(payload) + retry["attempt"] = next_attempt + retry["state"] = "failed" if next_attempt >= _JOB_MAX_ATTEMPTS else "queued" + _write_job_state(job_path, retry) + + +def _mark_emitted(job_path: Path, payload: dict[str, Any]) -> None: + emitted = dict(payload) + emitted["state"] = "emitted" + _write_job_state(job_path, emitted) + + +def _accounting_from_payload(payload: dict[str, Any]) -> dict[str, Any]: + return { + "accounting_id": payload["accounting_id"], + "usage": payload["usage"], + "attribution_status": payload["attribution_status"], + "agent_id": payload.get("agent_id"), + "attributes": payload.get("attributes") or {}, + "call_id": payload.get("call_id"), + } + + +def _run_accounting_job(job_path: Path, payload: dict[str, Any]) -> None: + """Claim, export, and ack one accounting job. Retryable on failure.""" + try: + claimed = _claim_job(job_path, payload) + except Exception as exc: + _log_worker_failure(kind="job_claim", payload=payload, error=exc) + return + if claimed is None: + return + + from thirdeye.otel_export import _captured_attributes + + token = _captured_attributes.set(claimed.get("captured_attributes") or {}) + kind = claimed.get("kind") + delivered = False + try: + from thirdeye.config import Config + from thirdeye.otel_export import ( + _export_session_accounting_inner, + _export_turn_accounting_inner, + ) + + config = Config.load() + if kind == "session_accounting": + _export_session_accounting_inner( + config=config, + session_dir_=Path(claimed["session_dir"]), + session_id=claimed["session_id"], + platform=claimed["platform"], + cwd=claimed["cwd"], + accounting=_accounting_from_payload(claimed), + ) + elif kind == "turn_accounting": + _export_turn_accounting_inner( + config=config, + session_dir_=Path(claimed["session_dir"]), + session_id=claimed["session_id"], + platform=claimed["platform"], + cwd=claimed["cwd"], + turn_id=str(claimed["turn_id"]), + accounting=_accounting_from_payload(claimed), + turn_span_id=claimed.get("turn_span_id"), + ) + else: + raise RuntimeError(f"unhandled accounting job kind {kind!r}") + _mark_emitted(job_path, claimed) + delivered = True + except Exception as exc: + try: + _retry_job(job_path, claimed) + except Exception: + pass + _log_worker_failure(kind=str(kind or ""), payload=claimed, error=exc) + finally: + _captured_attributes.reset(token) + _release_job_claim(job_path) + if delivered: + fsops.unlink(job_path, missing_ok=True) + def main(argv: list[str] | None = None) -> None: argv = sys.argv[1:] if argv is None else argv @@ -35,14 +184,22 @@ def main(argv: list[str] | None = None) -> None: payload = json.loads(job_path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError) as exc: _log_worker_failure(kind="job_read", payload={}, error=exc) - return - finally: fsops.unlink(job_path, missing_ok=True) + return + + kind = payload.get("kind") + if kind in _ACCOUNTING_KINDS: + _run_accounting_job(job_path, payload) + return + + # Existing platforms delete the ULID job as soon as it is readable. There + # is no scanner to respawn retained turn/spans/subagent jobs, so a crash + # or export failure must not leave poison pills on disk. + fsops.unlink(job_path, missing_ok=True) from thirdeye.otel_export import _captured_attributes token = _captured_attributes.set(payload.get("captured_attributes") or {}) - kind = payload.get("kind") try: from thirdeye.config import Config diff --git a/src/thirdeye/platforms/copilot/__init__.py b/src/thirdeye/platforms/copilot/__init__.py new file mode 100644 index 00000000..9dbe5b4e --- /dev/null +++ b/src/thirdeye/platforms/copilot/__init__.py @@ -0,0 +1,17 @@ +"""Read-only capture support for GitHub Copilot CLI recordings.""" + +from __future__ import annotations + +from .identity import resolve_sources, stored_session_id, validate_native_id +from .types import SourceBatch, SourcePaths, SourceRecord, SourceSlice, SyncResult + +__all__ = [ + "SourceBatch", + "SourcePaths", + "SourceRecord", + "SourceSlice", + "SyncResult", + "resolve_sources", + "stored_session_id", + "validate_native_id", +] diff --git a/src/thirdeye/platforms/copilot/archive.py b/src/thirdeye/platforms/copilot/archive.py new file mode 100644 index 00000000..750a7c00 --- /dev/null +++ b/src/thirdeye/platforms/copilot/archive.py @@ -0,0 +1,572 @@ +"""Durable, local archive for raw GitHub Copilot CLI source evidence. + +This module is intentionally below composition: it knows neither how records +are read nor how hooks are invoked. Its only input is a fully composed +``SourceBatch`` and its output is the generic thirdeye event log plus a small, +recoverable Copilot checkpoint beside that log. +""" + +from __future__ import annotations + +import json +from collections.abc import Callable, Iterator +from datetime import datetime +from pathlib import Path +from typing import Any + +from thirdeye._compat.locking import LockMode, locked +from thirdeye.config import Config +from thirdeye.meta import read_meta, write_meta +from thirdeye.paths import meta_path, session_dir +from thirdeye.reader import SessionReader +from thirdeye.store import Store +from thirdeye.writer import SessionWriter, utc_iso_ms + +from .constants import PLATFORM_NAME, SOURCE_SCHEMA_VERSION +from .identity import stored_session_id, validate_native_id +from .state import ( + STATE_SCHEMA_VERSION, + clear_journal, + journal_path, + lock_path, + read_json, + state_path, + write_journal, + write_state, +) +from .types import SourceBatch, SourcePaths, SourceRecord, SyncResult + +_EVENT_TYPES = { + "transcript": "copilot_transcript", + "database": "copilot_database", + "hook": "copilot_hook", + "metadata": "copilot_metadata", +} +_CURSOR_CONTROL_KEYS = ("base_cursor", "_base_cursor") + +# Tests and embedding applications may replace this with a deterministic +# callback that raises at a journal boundary. It is deliberately private: no +# runtime behaviour relies on fault injection. +_fault_injector: Callable[[str], None] | None = None + + +def _fault(point: str) -> None: + if _fault_injector is not None: + _fault_injector(point) + + +def _result( + *, + sessions: int = 0, + written: int = 0, + duplicates: int = 0, + pending: int = 0, + errors: int = 0, +) -> SyncResult: + return { + "sessions": sessions, + "records_written": written, + "duplicate_records": duplicates, + "pending": pending, + "errors": errors, + } + + +def _archive_dir(config: Config, stored_id: str) -> Path: + return session_dir(config.root, PLATFORM_NAME, stored_id) + + +def _valid_timestamp(value: object) -> str | None: + if not isinstance(value, str) or not value: + return None + candidate = value[:-1] + "+00:00" if value.endswith("Z") else value + try: + datetime.fromisoformat(candidate) + except ValueError: + return None + return value + + +def _source_cursor(cursor: object) -> dict[str, Any]: + """Return the source reader cursor, without archive contention markers.""" + if not isinstance(cursor, dict): + return {} + return {key: value for key, value in cursor.items() if key not in _CURSOR_CONTROL_KEYS} + + +def _cursor_copy(cursor: object) -> dict[str, Any]: + return json.loads(json.dumps(_source_cursor(cursor))) + + +def _event_type(record: SourceRecord) -> str: + return _EVENT_TYPES.get(record["source_kind"], "copilot_metadata") + + +def _envelope(record: SourceRecord) -> dict[str, Any]: + # ``source_record`` is named so later derived-state versions can add their + # own fields without ever changing the raw record's shape. + return {"schema_version": SOURCE_SCHEMA_VERSION, "source_record": record} + + +def _record_from_event(event: dict[str, Any]) -> SourceRecord | None: + if event.get("t") not in _EVENT_TYPES.values(): + return None + data = event.get("data") + if not isinstance(data, dict) or data.get("schema_version") != SOURCE_SCHEMA_VERSION: + return None + record = data.get("source_record") + if not isinstance(record, dict) or not isinstance(record.get("source_id"), str): + return None + return record # type: ignore[return-value] + + +def _committed_records(directory: Path) -> dict[str, SourceRecord]: + if not directory.exists(): + return {} + committed: dict[str, SourceRecord] = {} + for event in SessionReader(directory).iter_events(types=_EVENT_TYPES.values()): + record = _record_from_event(event) + if record is not None: + committed[record["source_id"]] = record + return committed + + +def _new_state(paths: SourcePaths, native_id: str) -> dict[str, Any]: + return { + "schema_version": STATE_SCHEMA_VERSION, + "source_key": paths["source_key"], + "source_home": paths["home"], + "native_session_id": native_id, + "cursor": {}, + "health": {"diagnostics": [], "last_successful_import": None, "capabilities": {}}, + } + + +def _validate_state(state: dict[str, Any], paths: SourcePaths, native_id: str) -> None: + if state.get("schema_version") != STATE_SCHEMA_VERSION: + raise ValueError("unsupported Copilot archive state schema") + if state.get("source_key") != paths["source_key"]: + raise ValueError("Copilot source-key prefix collision for stored session ID") + if state.get("native_session_id") != native_id: + raise ValueError("Copilot archive native session identity does not match stored session ID") + + +def _ensure_meta(config: Config, paths: SourcePaths, native_id: str, cwd: str | None) -> Path: + stored_id = stored_session_id(paths, native_id) + directory = _archive_dir(config, stored_id) + meta = read_meta(meta_path(directory)) if directory.exists() else None + if meta is not None: + identity = meta.extra.get("copilot") if isinstance(meta.extra, dict) else None + if not isinstance(identity, dict) or identity.get("source_key") != paths["source_key"]: + raise ValueError("Copilot source-key prefix collision for stored session ID") + if identity.get("native_session_id") != native_id: + raise ValueError( + "Copilot archive native session identity does not match stored session ID" + ) + return directory + + # Store owns generic session metadata. Creating it here also lets a hook + # establish a provisional session before any transcript exists. + writer = Store(config).open_session( + stored_id, + platform=PLATFORM_NAME, + cwd=cwd or "", + extra={ + "copilot": { + "schema_version": SOURCE_SCHEMA_VERSION, + "source_key": paths["source_key"], + "source_home": paths["home"], + "native_session_id": native_id, + } + }, + ) + writer.flush_and_detach() + return directory + + +def _base_cursor(batch: SourceBatch) -> dict[str, Any] | None: + """Read an optional composition cursor without imposing a reader format. + + The public SourceBatch schema remains JSON-compatible and intentionally + has no reader-specific field. Composition can include either spelling in + its cursor object when it needs optimistic contention detection. + """ + cursor = batch["next_cursor"] + for name in _CURSOR_CONTROL_KEYS: + value = cursor.get(name) + if isinstance(value, dict): + return value + return None + + +def _observation_errors(records: list[SourceRecord]) -> list[dict[str, Any]]: + errors: list[dict[str, Any]] = [] + for record in records: + if _valid_timestamp(record.get("observed_at")) is None: + source_id = record.get("source_id") + errors.append( + { + "kind": "invalid_observed_at", + "source_id": source_id if isinstance(source_id, str) else None, + } + ) + return errors + + +def _set_health( + state: dict[str, Any], diagnostics: list[dict[str, Any]], *, successful: bool +) -> None: + health = state.setdefault("health", {}) + health["diagnostics"] = diagnostics + if successful: + health["last_successful_import"] = utc_iso_ms() + + +def _writer_for_append( + config: Config, directory: Path, stored_id: str, cwd: str | None +) -> SessionWriter: + """Append into an existing archive without reopening Store session metadata.""" + meta = read_meta(meta_path(directory)) + if meta is None: + return Store(config).open_session(stored_id, platform=PLATFORM_NAME, cwd=cwd or "") + return SessionWriter(directory, meta) + + +def _append_records( + config: Config, + directory: Path, + stored_id: str, + cwd: str | None, + records: list[SourceRecord], + committed: dict[str, SourceRecord], +) -> tuple[int, int, list[dict[str, Any]], bool]: + writer: SessionWriter | None = None + written = 0 + duplicates = 0 + blocked = False + diagnostics: list[dict[str, Any]] = [] + try: + for record in records: + source_id = record.get("source_id") + if not isinstance(source_id, str) or not source_id: + diagnostics.append({"kind": "invalid_source_record", "reason": "missing source_id"}) + continue + if source_id in committed: + duplicates += 1 + continue + observed_at = _valid_timestamp(record.get("observed_at")) + if observed_at is None: + diagnostics.append({"kind": "invalid_observed_at", "source_id": source_id}) + blocked = True + continue + source_ts = _valid_timestamp(record.get("ts")) + event_ts = source_ts or observed_at + if source_ts is None: + diagnostics.append({"kind": "missing_source_time", "source_id": source_id}) + if writer is None: + writer = _writer_for_append(config, directory, stored_id, cwd) + writer.append(_event_type(record), _envelope(record), ts=event_ts) + committed[source_id] = record + written += 1 + if writer is not None: + writer.flush_and_detach() + except BaseException: + if writer is not None: + writer.flush_and_detach() + raise + return written, duplicates, diagnostics, blocked + + +def _identity_paths( + directory: Path, + state: dict[str, Any] | None, + journal: dict[str, Any] | None, +) -> tuple[SourcePaths, str]: + identity: dict[str, Any] = {} + if isinstance(state, dict): + identity = { + "source_key": state.get("source_key"), + "source_home": state.get("source_home"), + "native_session_id": state.get("native_session_id"), + } + if not isinstance(identity.get("source_key"), str) or not isinstance( + identity.get("native_session_id"), str + ): + meta = read_meta(meta_path(directory)) + extra = ( + meta.extra.get("copilot") if meta is not None and isinstance(meta.extra, dict) else None + ) + if isinstance(extra, dict): + identity = { + "source_key": extra.get("source_key"), + "source_home": extra.get("source_home"), + "native_session_id": extra.get("native_session_id"), + } + native_id = identity.get("native_session_id") + source_key = identity.get("source_key") + source_home = identity.get("source_home") + if not isinstance(native_id, str) and isinstance(journal, dict): + native_id = journal.get("native_session_id") + if not isinstance(source_key, str) and isinstance(journal, dict): + source_key = journal.get("source_key") + if ( + not isinstance(native_id, str) + or not isinstance(source_key, str) + or not isinstance(source_home, str) + ): + raise ValueError("Copilot archive identity is missing; cannot finish journal recovery") + home = Path(source_home) + paths: SourcePaths = { + "home": source_home, + "source_key": source_key, + "session_root": str(home / "session-state"), + "database": str(home / "session-store.db"), + } + return paths, native_id + + +def _checkpoint( + directory: Path, + state: dict[str, Any], + *, + next_cursor: dict[str, Any], + diagnostics: list[dict[str, Any]], + records: list[Any], +) -> dict[str, Any]: + _apply_lifecycle(directory, records) + _fault("after_lifecycle") + state["cursor"] = _source_cursor(next_cursor) + _set_health(state, diagnostics, successful=True) + write_state(directory, state) + _fault("after_checkpoint") + clear_journal(directory) + _fault("after_journal_clear") + return state + + +def _recover( + config: Config, + paths: SourcePaths, + native_id: str, + directory: Path, + state: dict[str, Any], +) -> tuple[dict[str, Any], int, int]: + journal = read_json(journal_path(directory)) + if journal is None: + return state, 0, 0 + if ( + journal.get("source_key") != paths["source_key"] + or journal.get("native_session_id") != native_id + ): + raise ValueError("Copilot archive journal belongs to another source session") + records = journal.get("records") + next_cursor = journal.get("next_cursor") + if not isinstance(records, list) or not isinstance(next_cursor, dict): + raise ValueError("invalid Copilot archive journal") + committed = _committed_records(directory) + written, duplicates, diagnostics, blocked = _append_records( + config, directory, directory.name, journal.get("cwd"), records, committed + ) + _fault("after_recovery_append") + if blocked: + _set_health(state, list(journal.get("diagnostics", [])) + diagnostics, successful=False) + write_state(directory, state) + clear_journal(directory) + return state, written, duplicates + state = _checkpoint( + directory, + state, + next_cursor=next_cursor, + diagnostics=list(journal.get("diagnostics", [])) + diagnostics, + records=records, + ) + return state, written, duplicates + + +def _recover_directory(config: Config, directory: Path) -> None: + journal = read_json(journal_path(directory)) + if journal is None: + return + state = read_json(state_path(directory)) + paths, native_id = _identity_paths(directory, state, journal) + if state is None: + state = _new_state(paths, native_id) + else: + _validate_state(state, paths, native_id) + _recover(config, paths, native_id, directory, state) + + +def load_cursor(config: Config, paths: SourcePaths, native_id: str) -> dict: + """Return a copy of the last committed cursor for one native session.""" + validate_native_id(native_id) + directory = _archive_dir(config, stored_session_id(paths, native_id)) + if not directory.exists(): + return {} + with locked(lock_path(directory), LockMode.EXCLUSIVE): + journal_exists = journal_path(directory).is_file() + state = read_json(state_path(directory)) + if state is None and not journal_exists: + return {} + if state is None: + state = _new_state(paths, native_id) + else: + _validate_state(state, paths, native_id) + state, _, _ = _recover(config, paths, native_id, directory, state) + return _cursor_copy(state.get("cursor", {})) + + +def commit_batch(config: Config, paths: SourcePaths, batch: SourceBatch) -> SyncResult: + """Append one raw evidence batch and atomically advance its cursor.""" + native_id = batch["native_session_id"] + validate_native_id(native_id) + if batch["source_key"] != paths["source_key"]: + raise ValueError("SourceBatch source_key does not match selected Copilot source home") + for record in batch["records"]: + if record.get("native_session_id") != native_id: + raise ValueError("SourceBatch contains a record for another native session") + stored_id = stored_session_id(paths, native_id) + directory = _ensure_meta(config, paths, native_id, batch.get("cwd")) + with locked(lock_path(directory), LockMode.EXCLUSIVE): + state = read_json(state_path(directory)) or _new_state(paths, native_id) + _validate_state(state, paths, native_id) + state, recovered_written, recovered_duplicates = _recover( + config, paths, native_id, directory, state + ) + + expected = _base_cursor(batch) + current = _source_cursor(state.get("cursor", {})) + if expected is not None and expected != current: + diagnostic = { + "kind": "stale_cursor", + "reason": "archive cursor advanced while batch was being read", + "expected_cursor": expected, + "committed_cursor": current, + } + _set_health(state, list(batch["diagnostics"]) + [diagnostic], successful=False) + write_state(directory, state) + return _result( + sessions=1, + written=recovered_written, + duplicates=recovered_duplicates, + pending=len(batch["records"]), + errors=1, + ) + + observation_errors = _observation_errors(batch["records"]) + if observation_errors: + _set_health(state, list(batch["diagnostics"]) + observation_errors, successful=False) + write_state(directory, state) + return _result( + sessions=1, + written=recovered_written, + duplicates=recovered_duplicates, + pending=len(batch["records"]), + errors=1, + ) + + next_cursor = _source_cursor(batch["next_cursor"]) + journal = { + "schema_version": STATE_SCHEMA_VERSION, + "source_key": paths["source_key"], + "native_session_id": native_id, + "cwd": batch.get("cwd"), + "records": batch["records"], + "next_cursor": next_cursor, + "diagnostics": batch["diagnostics"], + } + write_journal(directory, journal) + _fault("after_journal") + committed = _committed_records(directory) + written, duplicates, diagnostics, blocked = _append_records( + config, directory, stored_id, batch.get("cwd"), batch["records"], committed + ) + _fault("after_append") + if blocked: + _set_health(state, list(batch["diagnostics"]) + diagnostics, successful=False) + write_state(directory, state) + clear_journal(directory) + return _result( + sessions=1, + written=recovered_written + written, + duplicates=recovered_duplicates + duplicates, + pending=sum( + 1 + for record in batch["records"] + if isinstance(record.get("source_id"), str) + and record["source_id"] not in committed + ), + errors=1, + ) + _checkpoint( + directory, + state, + next_cursor=next_cursor, + diagnostics=list(batch["diagnostics"]) + diagnostics, + records=batch["records"], + ) + return _result( + sessions=1, + written=recovered_written + written, + duplicates=recovered_duplicates + duplicates, + ) + + +def _is_child_hook(payload: dict[str, Any]) -> bool: + """Child identity lives on hook_payload (and, historically, context).""" + + for mapping in (payload.get("hook_payload"), payload.get("context")): + if not isinstance(mapping, dict): + continue + if mapping.get("agentId") or mapping.get("agent_id"): + return True + if mapping.get("parentToolCallId") or mapping.get("parent_tool_call_id"): + return True + return False + + +def _apply_lifecycle(directory: Path, records: list[Any]) -> None: + """Apply only explicit top-level lifecycle evidence; child stops never close.""" + decision: str | None = None + for record in records: + if not isinstance(record, dict) or record.get("source_kind") != "hook": + continue + payload = record.get("payload", {}) + if not isinstance(payload, dict): + continue + event = payload.get("event") + if event in {"sessionEnd", "shutdown", "session_end"} and not _is_child_hook(payload): + decision = "close" + elif event in { + "sessionStart", + "resume", + "activity", + "session_start", + "userPromptSubmitted", + }: + decision = "reopen" + if decision is None: + return + meta_file = meta_path(directory) + meta = read_meta(meta_file) + if meta is None: + return + if decision == "reopen": + meta.status = "open" + meta.ended_at = None + else: + meta.status = "closed" + meta.ended_at = utc_iso_ms() + write_meta(meta_file, meta) + + +def iter_captured_records(config: Config, stored_session_id: str) -> Iterator[SourceRecord]: + """Yield immutable raw Copilot records retained in a local stored session.""" + directory = _archive_dir(config, stored_session_id) + if not directory.exists(): + return + with locked(lock_path(directory), LockMode.EXCLUSIVE): + _recover_directory(config, directory) + for event in SessionReader(directory).iter_events(types=_EVENT_TYPES.values()): + record = _record_from_event(event) + if record is not None: + yield record diff --git a/src/thirdeye/platforms/copilot/attribution.py b/src/thirdeye/platforms/copilot/attribution.py new file mode 100644 index 00000000..3ad04b25 --- /dev/null +++ b/src/thirdeye/platforms/copilot/attribution.py @@ -0,0 +1,305 @@ +"""Defensible joins between Copilot transcript calls and database usage. + +SQLite usage rows deliberately do not share an assistant-message identifier +with the transcript. This module therefore treats a join as an assertion +that must be supported by every available identity signal; a convenient +timestamp is never evidence. +""" + +from __future__ import annotations + +from collections import defaultdict +from typing import Any + +from .types import ( + AccountingCandidate, + AccountingProjection, + Attribution, + CallCandidate, + SemanticProjection, +) + +_DIRECT_ID_FIELDS = ( + "provider_call_id", + "assistant_message_id", + "native_call_id", + "native_message_id", + "message_id", +) + + +def _string(value: object) -> str | None: + return value if isinstance(value, str) and value else None + + +def _direct_ids(candidate: dict[str, Any], *, semantic: bool) -> set[str]: + """Read future native linkage fields without making one up today.""" + + result = { + value for field in _DIRECT_ID_FIELDS if (value := _string(candidate.get(field))) is not None + } + # A future accounting producer may carry the semantic durable call ID. + # The current accounting shape uses logical_call_id instead, so this does + # not turn a database identity into an accidental match. + call_id = _string(candidate.get("call_id")) + if call_id and (semantic or call_id.startswith("copilot:call:")): + result.add(call_id) + return result + + +def _tool_index(calls: list[CallCandidate]) -> dict[str, list[CallCandidate]]: + by_tool: dict[str, list[CallCandidate]] = defaultdict(list) + for call in calls: + for tool_id in call.get("tool_call_ids") or []: + if isinstance(tool_id, str) and tool_id: + by_tool[tool_id].append(call) + return by_tool + + +def _root_candidate( + candidate: CallCandidate, by_tool: dict[str, list[CallCandidate]] +) -> CallCandidate | None: + """Follow a child task's parent tool back to its main interaction.""" + + current = candidate + seen: set[str] = set() + while current.get("agent_id") is not None: + call_id = current["call_id"] + if call_id in seen: + return None + seen.add(call_id) + parent_tool = _string(current.get("parent_tool_call_id")) + parents = by_tool.get(parent_tool or "", []) + if len(parents) != 1: + return None + current = parents[0] + return current + + +def _cycle_order(calls: list[CallCandidate]) -> list[CallCandidate]: + """Order model cycles inside one identity group by source start_ts.""" + + return sorted( + calls, + key=lambda call: (str(call.get("start_ts") or ""), call["call_id"]), + ) + + +def _identity_group(call: CallCandidate, calls: list[CallCandidate]) -> list[CallCandidate]: + group = [ + item + for item in calls + if item.get("interaction_id") == call.get("interaction_id") + and item.get("agent_id") == call.get("agent_id") + and item.get("parent_tool_call_id") == call.get("parent_tool_call_id") + ] + return _cycle_order(group) + + +def _interaction_indexes( + calls: list[CallCandidate], + by_tool: dict[str, list[CallCandidate]], +) -> tuple[dict[str, int], dict[str, CallCandidate]]: + """Build only the verified main interaction order used by DB turn_index.""" + + indexes: dict[str, int] = {} + roots: dict[str, CallCandidate] = {} + for call in calls: + root = _root_candidate(call, by_tool) + if root is None: + continue + interaction = _string(root.get("interaction_id")) + if interaction is None: + continue + # A main interaction appears in archive order. Repeated model cycles + # intentionally retain its first position; this is not a bare turnId. + # Failed child walks never contribute a slot. + if interaction not in indexes: + indexes[interaction] = len(indexes) + roots[interaction] = root + return indexes, roots + + +def _finish_matches(usage: AccountingCandidate, call: CallCandidate) -> bool: + reason = _string(usage.get("finish_reason")) + if reason is None: + return True + if reason == "tool_calls": + return bool(call.get("tool_call_ids")) + if reason in {"stop", "length", "content_filter"}: + return not call.get("tool_call_ids") and bool(call.get("finish_evidence")) + return False + + +def _initiator_matches( + usage: AccountingCandidate, + call: CallCandidate, + candidates: list[CallCandidate], +) -> bool: + initiator = (usage.get("supplemental_metrics") or {}).get("initiator") + if initiator is None: + return True + if not isinstance(initiator, str): + return False + group = _identity_group(call, candidates) + call_order = group.index(call) + if initiator == "user": + return call_order == 0 + if initiator == "sub-agent": + return call.get("agent_id") is not None + if initiator == "agent": + return call_order > 0 and call.get("agent_id") is None + return False + + +def _turn_for_usage( + usage: AccountingCandidate, indexes: dict[str, int], roots: dict[str, CallCandidate] +) -> tuple[str | None, str | None]: + """Return (main interaction, main stored turn) only when DB index is sound.""" + + index = usage.get("turn_index") + if not isinstance(index, int) or isinstance(index, bool): + return None, None + matches = [interaction for interaction, value in indexes.items() if value == index] + if len(matches) != 1: + return None, None + interaction = matches[0] + root = roots[interaction] + return interaction, _string(root.get("stored_turn_id")) + + +def _base_attribution(usage: AccountingCandidate, stored_turn_id: str | None) -> Attribution: + return { + "usage_source_id": usage["usage_source_id"], + "logical_call_id": usage["logical_call_id"], + "stored_turn_id": stored_turn_id, + "agent_id": usage.get("agent_id"), + "call_id": None, + "status": "pending", + "join_kind": None, + "evidence": [f"logical_call_id:{usage['logical_call_id']}"], + } + + +def join_usage(semantic: SemanticProjection, accounting: AccountingProjection) -> list[Attribution]: + """Join each accounting candidate at most once, retaining uncertainty. + + Direct provider/message identifiers win when future source formats expose + them. Current CLI rows are inferred only after their main interaction is + established from main order and child parent-tool lineage. + """ + + calls = list(semantic.get("call_candidates") or []) + by_tool = _tool_index(calls) + indexes, roots = _interaction_indexes(calls, by_tool) + preliminary: list[tuple[Attribution, CallCandidate | None]] = [] + + for usage in accounting.get("candidates") or []: + interaction, stored_turn_id = _turn_for_usage(usage, indexes, roots) + result = _base_attribution(usage, stored_turn_id) + direct = _direct_ids(usage, semantic=False) + direct_matches = [call for call in calls if direct & _direct_ids(call, semantic=True)] + if direct_matches: + if len(direct_matches) != 1: + result.update( + status="conflicting", + evidence=[ + f"logical_call_id:{usage['logical_call_id']}", + *(f"call_id:{call['call_id']}" for call in direct_matches), + ], + ) + preliminary.append((result, None)) + continue + call = direct_matches[0] + root = _root_candidate(call, by_tool) or call + result.update( + stored_turn_id=_string(root.get("stored_turn_id")), + agent_id=call.get("agent_id"), + call_id=call["call_id"], + status="matched", + join_kind="direct", + evidence=["join_kind:direct", f"call_id:{call['call_id']}"], + ) + preliminary.append((result, call)) + continue + + if interaction is None: + result["evidence"].append(f"turn_index:{usage.get('turn_index')}") + preliminary.append((result, None)) + continue + + same_turn = [ + call + for call in calls + if (_root_candidate(call, by_tool) or call).get("interaction_id") == interaction + ] + possible = [ + call + for call in same_turn + if call.get("agent_id") == usage.get("agent_id") + and call.get("parent_tool_call_id") == usage.get("parent_tool_call_id") + and call.get("model") == usage.get("model") + and _finish_matches(usage, call) + and _initiator_matches(usage, call, calls) + ] + if len(possible) != 1: + if possible: + result.update( + status="ambiguous", + evidence=[ + *(f"call_id:{call['call_id']}" for call in possible), + f"interaction_id:{interaction}", + f"model:{usage.get('model')}", + f"turn_index:{usage.get('turn_index')}", + ], + ) + else: + result["evidence"].extend( + [ + f"interaction_id:{interaction}", + f"turn_index:{usage.get('turn_index')}", + ] + ) + preliminary.append((result, None)) + continue + + call = possible[0] + order = _identity_group(call, calls).index(call) + initiator = (usage.get("supplemental_metrics") or {}).get("initiator") + result.update( + call_id=call["call_id"], + status="matched", + join_kind="inferred", + evidence=[ + "join_kind:inferred", + f"interaction_id:{interaction}", + f"turn_index:{usage.get('turn_index')}", + f"agent_id:{usage.get('agent_id')}", + f"parent_tool_call_id:{usage.get('parent_tool_call_id')}", + f"model:{usage.get('model')}", + f"finish:{usage.get('finish_reason')}", + f"initiator:{initiator}", + f"order:{order}", + f"call_id:{call['call_id']}", + ], + ) + preliminary.append((result, call)) + + claimed: dict[str, list[Attribution]] = defaultdict(list) + for attribution, call in preliminary: + if call is not None and attribution["status"] == "matched": + claimed[call["call_id"]].append(attribution) + for call_id, entries in claimed.items(): + if len(entries) < 2: + continue + for entry in entries: + entry.update( + call_id=None, + status="conflicting", + join_kind=None, + evidence=[ + f"logical_call_id:{entry['logical_call_id']}", + f"call_id:{call_id}", + ], + ) + return [attribution for attribution, _ in preliminary] diff --git a/src/thirdeye/platforms/copilot/capture.py b/src/thirdeye/platforms/copilot/capture.py new file mode 100644 index 00000000..f920c74b --- /dev/null +++ b/src/thirdeye/platforms/copilot/capture.py @@ -0,0 +1,327 @@ +"""Bounded local composition of Copilot source capture and hook spooling.""" + +from __future__ import annotations + +from copy import deepcopy +from datetime import UTC, datetime +from pathlib import Path +from typing import Any +from uuid import uuid4 + +from thirdeye.config import Config + +from .archive import commit_batch, iter_captured_records, load_cursor +from .hook_payload import parse_hook +from .identity import validate_native_id +from .sources import _discover_with_diagnostics, read_batch +from .spool import ack_spool, enqueue_hook, read_spool +from .types import SourceBatch, SourcePaths, SourceRecord, SyncResult + +_RETRYABLE_SUFFIXES = ( + "_unavailable", + "_missing", + "_busy", + "_unreadable", + "_read_failed", + "_incompatible", +) +_ABSENCE_CODES = frozenset({"transcript_unavailable", "copilot_database_missing"}) +_MAX_DRAIN_PAGES = 256 + + +def _result() -> SyncResult: + return { + "sessions": 0, + "records_written": 0, + "duplicate_records": 0, + "pending": 0, + "errors": 0, + } + + +def _add(total: SyncResult, result: SyncResult) -> None: + for key in total: + total[key] += result[key] + + +def _spool_sessions(config: Config, paths: SourcePaths) -> list[str]: + """Discover hook-only sessions without reading their content directly.""" + + root = Path(config.root) / "spool" / "copilot" / paths["source_key"] + try: + entries = list(root.iterdir()) + except OSError: + return [] + sessions: list[str] = [] + for entry in entries: + try: + if not entry.is_dir(): + continue + validate_native_id(entry.name) + except (OSError, ValueError): + continue + sessions.append(entry.name) + return sorted(sessions) + + +def _with_spool(batch: SourceBatch, spool_records: list[SourceRecord]) -> SourceBatch: + """Add an invocation-time spool snapshot without changing reader cursors.""" + + return { + **batch, + "records": [*spool_records, *batch["records"]], + "next_cursor": deepcopy(batch["next_cursor"]), + "diagnostics": list(batch["diagnostics"]), + } + + +def _is_retryable_code(code: object) -> bool: + return isinstance(code, str) and code.endswith(_RETRYABLE_SUFFIXES) + + +def _diagnostic_count(batch: SourceBatch) -> int: + """Diagnostics are surfaced as source errors without dropping evidence.""" + + return len(batch["diagnostics"]) + + +def _diagnostic_pending(batch: SourceBatch) -> int: + """Count sources which must be retried without treating them as complete.""" + + return sum( + 1 for diagnostic in batch["diagnostics"] if _is_retryable_code(diagnostic.get("code")) + ) + + +def _source_retryable(batch: SourceBatch, name: str) -> bool: + return any( + isinstance(diagnostic.get("code"), str) + and name in diagnostic["code"] + and _is_retryable_code(diagnostic["code"]) + for diagnostic in batch["diagnostics"] + ) + + +def _pagination_pending(batch: SourceBatch) -> int: + """Count sources that still have unread snapshot pages.""" + + cursor = batch["next_cursor"] + pending = 0 + for name in ("transcript", "database"): + if cursor.get(f"{name}_exhausted", True): + continue + if _source_retryable(batch, name): + continue + pending += 1 + return pending + + +def _compose(archive: SyncResult, batch: SourceBatch) -> SyncResult: + result = dict(archive) + result["errors"] += _diagnostic_count(batch) + result["pending"] += _diagnostic_pending(batch) + _pagination_pending(batch) + return result # type: ignore[return-value] + + +def _merge_archive(first: SyncResult, second: SyncResult) -> SyncResult: + """Keep recovery writes from a stale attempt; take the retry's error/pending.""" + + return { + "sessions": max(first["sessions"], second["sessions"]), + "records_written": first["records_written"] + second["records_written"], + "duplicate_records": first["duplicate_records"] + second["duplicate_records"], + "pending": second["pending"], + "errors": second["errors"], + } + + +def _merge_diagnostics(items: list[dict[str, Any]]) -> list[dict[str, Any]]: + retryable: dict[str, dict[str, Any]] = {} + others: list[dict[str, Any]] = [] + for item in items: + code = item.get("code") + if _is_retryable_code(code) and isinstance(code, str): + retryable[code] = item + else: + others.append(item) + return others + list(retryable.values()) + + +def _stale_after_commit( + config: Config, + paths: SourcePaths, + native_session_id: str, + base_cursor: dict[str, Any], + result: SyncResult, +) -> bool: + """Identify archive's optimistic-race result without inspecting state files.""" + + return ( + result["errors"] > 0 + and result["pending"] > 0 + and load_cursor(config, paths, native_session_id) != base_cursor + ) + + +def _is_absence_diagnostic(diagnostic: dict[str, Any]) -> bool: + code = diagnostic.get("code") + if not isinstance(code, str): + return False + if code in _ABSENCE_CODES: + return True + return code.startswith("workspace_") + + +def _selected_source_missing(batch: SourceBatch) -> bool: + if batch["records"]: + return False + return all(_is_absence_diagnostic(diagnostic) for diagnostic in batch["diagnostics"]) + + +def _result_from_diagnostics(diagnostics: list[dict[str, Any]]) -> SyncResult: + return { + "sessions": 0, + "records_written": 0, + "duplicate_records": 0, + "pending": sum( + 1 for diagnostic in diagnostics if _is_retryable_code(diagnostic.get("code")) + ), + "errors": len(diagnostics), + } + + +def _capture_once( + config: Config, paths: SourcePaths, native_session_id: str +) -> tuple[SyncResult, SourceBatch, bool]: + """Commit one bounded source read plus the current durable hook spool. + + A concurrent capture may advance the archive cursor between read and + commit. In that case this retries exactly once from the newly committed + cursor, preserving the original spool snapshot and leaving any second race + pending for a later sync/watch cycle. + """ + + spool_records = read_spool(config, paths, native_session_id) + base_cursor = load_cursor(config, paths, native_session_id) + batch = _with_spool(read_batch(paths, native_session_id, base_cursor), spool_records) + archive = commit_batch(config, paths, batch) + unresolved_stale = False + + if _stale_after_commit(config, paths, native_session_id, base_cursor, archive): + retry_cursor = load_cursor(config, paths, native_session_id) + retry_batch = _with_spool(read_batch(paths, native_session_id, retry_cursor), spool_records) + retry_archive = commit_batch(config, paths, retry_batch) + archive = _merge_archive(archive, retry_archive) + batch = retry_batch + unresolved_stale = _stale_after_commit( + config, paths, native_session_id, retry_cursor, retry_archive + ) + + # A zero archive error means the journal checkpoint made every submitted + # hook source ID durable (including an already-durable duplicate). + if archive["errors"] == 0 and spool_records: + ack_spool(config, paths, [record["source_id"] for record in spool_records]) + + more_pages = (not unresolved_stale) and _pagination_pending(batch) > 0 + return archive, batch, more_pages + + +def capture_session(config: Config, paths: SourcePaths, native_session_id: str) -> SyncResult: + """Commit one bounded source read plus the current durable hook spool.""" + + validate_native_id(native_session_id) + archive, batch, _ = _capture_once(config, paths, native_session_id) + return _compose(archive, batch) + + +def _drain_session(config: Config, paths: SourcePaths, native_session_id: str) -> SyncResult: + """Drain one invocation-time snapshot without chasing a growing source.""" + + total = _result() + collected: list[dict[str, Any]] = [] + last_batch: SourceBatch | None = None + prev_cursor: object = object() + for _ in range(_MAX_DRAIN_PAGES): + archive, batch, more_pages = _capture_once(config, paths, native_session_id) + last_batch = batch + collected.extend(batch["diagnostics"]) + total["records_written"] += archive["records_written"] + total["duplicate_records"] += archive["duplicate_records"] + total["sessions"] = max(total["sessions"], archive["sessions"]) + total["errors"] += archive["errors"] + total["pending"] = archive["pending"] + current_cursor = load_cursor(config, paths, native_session_id) + if not more_pages or current_cursor == prev_cursor: + break + prev_cursor = current_cursor + assert last_batch is not None + merged: SourceBatch = { + **last_batch, + "diagnostics": _merge_diagnostics(collected), + } + return _compose(total, merged) + + +def sync(config: Config, paths: SourcePaths, *, session_id: str | None = None) -> SyncResult: + """Capture an invocation-time snapshot of discovered and spooled sessions.""" + + discovered, discovery_diagnostics = _discover_with_diagnostics(paths) + known = set(discovered) | set(_spool_sessions(config, paths)) + + if session_id is not None: + validate_native_id(session_id) + if session_id not in known: + probe = read_batch(paths, session_id, {}) + if _selected_source_missing(probe): + missing: SyncResult = { + "sessions": 0, + "records_written": 0, + "duplicate_records": 0, + "pending": 0, + "errors": 1, + } + if discovery_diagnostics: + _add(missing, _result_from_diagnostics(discovery_diagnostics)) + return missing + session_ids = [session_id] + else: + session_ids = sorted(known) + + total = _result_from_diagnostics(discovery_diagnostics) + if not session_ids: + return total + for native_session_id in session_ids: + _add(total, _drain_session(config, paths, native_session_id)) + return total + + +def _observed_at() -> str: + return datetime.now(UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z") + + +def record_hook( + config: Config, + paths: SourcePaths, + event: str, + payload: dict, + context: dict, +) -> SyncResult: + """Durably receive one hook observation, then make one bounded capture.""" + + record = parse_hook( + event, + payload, + context, + observed_at=_observed_at(), + observation_id=uuid4().hex, + ) + enqueue_hook(config, paths, record) + return capture_session(config, paths, record["native_session_id"]) + + +__all__ = [ + "capture_session", + "iter_captured_records", + "record_hook", + "sync", +] diff --git a/src/thirdeye/platforms/copilot/constants.py b/src/thirdeye/platforms/copilot/constants.py new file mode 100644 index 00000000..9e18b432 --- /dev/null +++ b/src/thirdeye/platforms/copilot/constants.py @@ -0,0 +1,51 @@ +"""Stable Copilot platform identifiers and CLI hook vocabulary. + +Paths are intentionally absent from this module. Source paths are resolved at +invocation time by :func:`thirdeye.platforms.copilot.identity.resolve_sources`. +""" + +from __future__ import annotations + +PLATFORM_NAME = "copilot" +DISPLAY_NAME = "GitHub Copilot CLI" +SOURCE_SCHEMA_VERSION = 1 +# Compatibility-friendly names for consumers that refer to the record/event +# envelope rather than the source format. All three are the same V1 contract. +SCHEMA_VERSION = SOURCE_SCHEMA_VERSION +SOURCE_RECORD_SCHEMA_VERSION = SOURCE_SCHEMA_VERSION +EVENT_ENVELOPE_VERSION = SOURCE_SCHEMA_VERSION + +COPILOT_HOME_ENV = "COPILOT_HOME" +HOOKS_DIRECTORY_NAME = "hooks" +OWNED_HOOK_FILENAME = "thirdeye.json" +FOLLOWUP_LEASE_FILENAME = "copilot.followup.json" +HOOK_CONFIG_VERSION = 1 +HOOK_TIMEOUT_S = 5 + +# This is a single dispatcher entrypoint. The event name is passed as an +# explicit argument rather than needing one installed binary per hook event. +# Runtime registration is deliberately owned by the later composition task. +HOOK_BIN_NAME = "thirdeye-copilot-hook" + +# Copilot CLI 1.0.83's documented hook names are camelCase. The aliases are +# intentionally data-only: installation and hook runtime decide which events +# they support without importing source-resolution code. +CLI_HOOK_EVENT_ALIASES: dict[str, str] = { + "sessionStart": "session_start", + "userPromptSubmitted": "user_prompt_submitted", + "preToolUse": "pre_tool_use", + "postToolUse": "post_tool_use", + "postToolUseFailure": "post_tool_use_failure", + "agentStop": "agent_stop", + "subagentStart": "subagent_start", + "subagentStop": "subagent_stop", + "sessionEnd": "session_end", + "errorOccurred": "error_occurred", + "permissionRequest": "permission_request", + "notification": "notification", + "preCompact": "pre_compact", +} + +CLI_HOOK_EVENTS = tuple(CLI_HOOK_EVENT_ALIASES) +HOOK_EVENT_ALIASES = CLI_HOOK_EVENT_ALIASES +HOOK_EVENTS = CLI_HOOK_EVENTS diff --git a/src/thirdeye/platforms/copilot/database.py b/src/thirdeye/platforms/copilot/database.py new file mode 100644 index 00000000..1e2826cb --- /dev/null +++ b/src/thirdeye/platforms/copilot/database.py @@ -0,0 +1,439 @@ +"""Read raw Copilot session evidence from its local SQLite database. + +This module deliberately has no knowledge of thirdeye's archive. It produces +lossless, revisioned source records; committing and deduplicating those records +is the archive's responsibility. +""" + +from __future__ import annotations + +import base64 +import hashlib +import json +import sqlite3 +from collections.abc import Iterable, Iterator, Sequence +from contextlib import contextmanager +from datetime import UTC, datetime +from pathlib import Path +from typing import Any +from urllib.parse import quote + +from .identity import validate_native_id +from .types import SourcePaths, SourceRecord, SourceSlice + +_TABLES = ("sessions", "turns", "assistant_usage_events") +_SESSION_ID_COLUMNS = { + "sessions": ("id", "session_id"), + "turns": ("session_id",), + "assistant_usage_events": ("session_id",), +} +_TIMESTAMP_COLUMNS = ("created_at", "createdAt", "timestamp", "updated_at", "updatedAt") +_CWD_COLUMNS = ("cwd", "working_directory", "workingDirectory") +_BUSY_TOKENS = ("locked", "busy") +_INCOMPATIBLE_TOKENS = ( + "file is not a database", + "malformed", + "corrupt", + "disk image is malformed", + "not a database", +) + + +def _diagnostic(code: str, message: str, **details: Any) -> dict[str, Any]: + """Return a content-free diagnostic suitable for CLI presentation.""" + + return {"code": code, "message": message, **details} + + +def _observed_at() -> str: + return datetime.now(UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z") + + +def _json_value(value: Any) -> Any: + """Make SQLite values JSON-compatible without dropping a column's value.""" + + if isinstance(value, bytes): + return {"encoding": "base64", "data": base64.b64encode(value).decode("ascii")} + if isinstance(value, memoryview): + return _json_value(value.tobytes()) + if isinstance(value, float) and (value != value or value in (float("inf"), float("-inf"))): + return {"encoding": "repr", "data": repr(value)} + if isinstance(value, dict): + return {str(key): _json_value(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_json_value(item) for item in value] + return value + + +def _canonical_json(value: Any) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False, default=str) + + +def _content_revision(row: dict[str, Any]) -> str: + return "sha256:" + hashlib.sha256(_canonical_json(row).encode("utf-8")).hexdigest() + + +def _quote_identifier(name: str) -> str: + # Names come from SQLite's schema, but quote anyway so a surprising schema + # cannot turn an identifier into executable SQL. + return '"' + name.replace('"', '""') + '"' + + +def _file_generation(database: Path) -> str: + """Identify the database file incarnation without consulting live WAL metadata. + + WAL size and mtime change independently of this session's rows. Device and + inode still change when the file is replaced, which is the signal needed to + detect row-ID reuse across a new database. + """ + + try: + stat = database.stat() + except FileNotFoundError: + payload: dict[str, Any] = {"path": database.name, "missing": True} + else: + payload = { + "path": database.name, + "device": stat.st_dev, + "inode": stat.st_ino, + # Windows may quickly reuse st_ino after unlink/recreate. Creation + # time remains stable for ordinary database writes and changes for + # a replacement file. It is also available as birth time on macOS. + "birthtime_ns": getattr(stat, "st_birthtime_ns", None), + } + return "sha256:" + hashlib.sha256(_canonical_json(payload).encode("utf-8")).hexdigest() + + +def _connect(database: Path) -> sqlite3.Connection: + # ``mode=ro`` keeps SQLite's normal WAL behaviour while preventing all + # writes. In particular, do not use immutable=1: it ignores live WAL data. + connection = sqlite3.connect(database.resolve().as_uri() + "?mode=ro", uri=True, timeout=0.1) + try: + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA busy_timeout = 100") + connection.execute("BEGIN") + return connection + except Exception: + connection.close() + raise + + +@contextmanager +def _readonly_connection(database: Path) -> Iterator[sqlite3.Connection]: + connection = _connect(database) + try: + yield connection + finally: + connection.close() + + +def _table_info(connection: sqlite3.Connection, table: str) -> list[sqlite3.Row]: + return list(connection.execute(f"PRAGMA table_info({_quote_identifier(table)})")) + + +def _column_names(info: Iterable[sqlite3.Row]) -> list[str]: + return [str(row["name"]) for row in info] + + +def _available_tables(connection: sqlite3.Connection) -> set[str]: + rows = connection.execute( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name IN (?, ?, ?)", _TABLES + ) + return {str(row[0]) for row in rows} + + +def _session_column(table: str, columns: Iterable[str]) -> str | None: + known = set(columns) + return next((name for name in _SESSION_ID_COLUMNS[table] if name in known), None) + + +def _primary_key_columns(info: Iterable[sqlite3.Row]) -> list[str] | None: + keyed = [(int(row["pk"]), str(row["name"])) for row in info if int(row["pk"]) > 0] + if not keyed: + return None + keyed.sort() + return [name for _, name in keyed] + + +def _session_scope_column( + table: str, columns: Iterable[str], pk_columns: list[str] | None +) -> str | None: + scoped = _session_column(table, columns) + if scoped is not None: + return scoped + # Observed Copilot sessions use ``id``; if a compatible schema names that + # single primary key differently, the key still *is* the native session ID. + if table == "sessions" and pk_columns is not None and len(pk_columns) == 1: + return pk_columns[0] + return None + + +def _row_identity(row: dict[str, Any], pk_columns: Sequence[str]) -> Any: + if len(pk_columns) == 1: + return row[pk_columns[0]] + return {name: row[name] for name in pk_columns} + + +def _valid_source_time(row: dict[str, Any]) -> str | None: + for column in _TIMESTAMP_COLUMNS: + value = row.get(column) + if not isinstance(value, str) or not value: + continue + try: + datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + continue + return value + return None + + +def _row_record( + paths: SourcePaths, + native_id: str, + table: str, + primary_key: Any, + row: dict[str, Any], + generation: str, + observed_at: str, +) -> SourceRecord: + revision = _content_revision(row) + primary_identity = _canonical_json(_json_value(primary_key)) + # Keep the row identity legible for normal integer IDs, while quoting it so + # arbitrary SQLite primary-key values never alter the source-ID structure. + primary_component = quote(primary_identity, safe="") + source_id = ( + f"copilot-db:{paths['source_key']}:{native_id}:{table}:{primary_component}:{revision}" + ) + return { + "source_id": source_id, + "source_kind": "database", + "native_session_id": native_id, + "ts": _valid_source_time(row), + "observed_at": observed_at, + "payload": {"table": table, "row": row}, + "locator": { + "database": paths["database"], + "table": table, + "primary_key": _json_value(primary_key), + "content_revision": revision, + "generation": generation, + }, + } + + +def _empty_slice(diagnostics: list[dict[str, Any]]) -> SourceSlice: + return { + "records": [], + "next_cursor": {}, + "diagnostics": diagnostics, + "cwd": None, + "exhausted": True, + } + + +def _operational_diagnostic(error: sqlite3.OperationalError, database: Path) -> dict[str, Any]: + message = str(error).lower() + if any(token in message for token in _BUSY_TOKENS): + return _diagnostic( + "copilot_database_busy", + "Copilot session database could not be read within 100ms; retry sync or watch", + path=str(database), + reason=str(error), + ) + if any(token in message for token in _INCOMPATIBLE_TOKENS): + return _diagnostic( + "copilot_database_incompatible", + "Copilot session database could not be read as SQLite evidence", + path=str(database), + reason=str(error), + ) + return _diagnostic( + "copilot_database_unreadable", + "Copilot session database could not be opened for read-only evidence", + path=str(database), + reason=str(error), + ) + + +def _session_ids_from_table( + connection: sqlite3.Connection, table: str, session_column: str +) -> list[str]: + rows = connection.execute( + f"SELECT {_quote_identifier(session_column)} FROM {_quote_identifier(table)} " + f"WHERE {_quote_identifier(session_column)} IS NOT NULL" + ) + return [str(row[0]) for row in rows if isinstance(row[0], str)] + + +def discover_database_sessions(paths: SourcePaths) -> list[str]: + """Return session IDs advertised by any readable allowlisted table. + + Discovery is intentionally conservative: a malformed or absent database is + simply not a discoverable source. ``read_database`` supplies the actionable + diagnostics needed by status and sync commands. + """ + + database = Path(paths["database"]) + if not database.is_file(): + return [] + try: + with _readonly_connection(database) as connection: + available = _available_tables(connection) + values: set[str] = set() + for table in _TABLES: + if table not in available: + continue + info = _table_info(connection, table) + columns = _column_names(info) + session_column = _session_scope_column(table, columns, _primary_key_columns(info)) + if session_column is None: + continue + values.update(_session_ids_from_table(connection, table, session_column)) + except sqlite3.Error: + return [] + return sorted(values) + + +def read_database( + paths: SourcePaths, + native_id: str, + cursor: dict, + *, + max_records: int = 1000, +) -> SourceSlice: + """Read a bounded, transactionally consistent raw SQLite snapshot. + + Every poll re-reads the selected session, because turns and sessions are + mutable. Pagination is keyed to the database file incarnation so live WAL + writes—including inserts and updates in this session—cannot starve later + rows. A replaced database file replays from the start so row-ID reuse + cannot silently continue a stale offset. Content revisions travel on each + record; unchanged snapshots still deduplicate after a later full read. + """ + + validate_native_id(native_id) + if max_records < 1: + raise ValueError("max_records must be at least 1") + + database = Path(paths["database"]) + if not database.exists(): + return _empty_slice( + [ + _diagnostic( + "copilot_database_missing", + "Copilot session database is not present", + path=str(database), + ) + ] + ) + if not database.is_file(): + return _empty_slice( + [ + _diagnostic( + "copilot_database_unreadable", + "Copilot session database is not a regular file", + path=str(database), + ) + ] + ) + + diagnostics: list[dict[str, Any]] = [] + observed_at = _observed_at() + file_generation = _file_generation(database) + rows_by_table: list[tuple[str, Any, dict[str, Any]]] = [] + cwd: str | None = None + try: + with _readonly_connection(database) as connection: + available = _available_tables(connection) + for table in _TABLES: + if table not in available: + diagnostics.append( + _diagnostic( + "copilot_database_table_missing", + "Expected Copilot table is unavailable", + table=table, + ) + ) + continue + info = _table_info(connection, table) + columns = _column_names(info) + pk_columns = _primary_key_columns(info) + session_column = _session_scope_column(table, columns, pk_columns) + if session_column is None: + diagnostics.append( + _diagnostic( + "copilot_database_missing_session_column", + "Copilot table cannot be scoped to a session", + table=table, + expected=list(_SESSION_ID_COLUMNS[table]), + columns=columns, + ) + ) + continue + if pk_columns is None: + diagnostics.append( + _diagnostic( + "copilot_database_missing_primary_key", + "Copilot table lacks a SQLite PRIMARY KEY that can identify rows", + table=table, + columns=columns, + ) + ) + continue + order = ", ".join(_quote_identifier(name) for name in pk_columns) + query = ( + f"SELECT * FROM {_quote_identifier(table)} " + f"WHERE {_quote_identifier(session_column)} = ? " + f"ORDER BY {order}" + ) + for sql_row in connection.execute(query, (native_id,)): + row = {key: _json_value(sql_row[key]) for key in sql_row.keys()} + if table == "sessions" and cwd is None: + cwd = next( + (row[name] for name in _CWD_COLUMNS if isinstance(row.get(name), str)), + None, + ) + rows_by_table.append((table, _row_identity(row, pk_columns), row)) + except sqlite3.OperationalError as error: + return _empty_slice([_operational_diagnostic(error, database)]) + except sqlite3.Error as error: + return _empty_slice( + [ + _diagnostic( + "copilot_database_incompatible", + "Copilot session database could not be read as SQLite evidence", + path=str(database), + reason=str(error), + ) + ] + ) + + # The table loop above has a deterministic table order; ordering row IDs by + # their JSON form makes heterogeneous SQLite primary key types deterministic. + rows_by_table.sort( + key=lambda item: (_TABLES.index(item[0]), _canonical_json(_json_value(item[1]))) + ) + generation = file_generation + incoming_generation = cursor.get("database_generation") if isinstance(cursor, dict) else None + incoming_offset = cursor.get("database_offset", 0) if isinstance(cursor, dict) else 0 + offset = ( + incoming_offset + if incoming_generation == generation and isinstance(incoming_offset, int) + else 0 + ) + offset = max(0, offset) + selected = rows_by_table[offset : offset + max_records] + records = [ + _row_record(paths, native_id, table, primary_key, row, file_generation, observed_at) + for table, primary_key, row in selected + ] + next_offset = offset + len(selected) + exhausted = next_offset >= len(rows_by_table) + next_cursor = {"database_generation": generation, "database_offset": next_offset} + return { + "records": records, + "next_cursor": next_cursor, + "diagnostics": diagnostics, + "cwd": cwd, + "exhausted": exhausted, + } diff --git a/src/thirdeye/platforms/copilot/events.py b/src/thirdeye/platforms/copilot/events.py new file mode 100644 index 00000000..363a4eed --- /dev/null +++ b/src/thirdeye/platforms/copilot/events.py @@ -0,0 +1,394 @@ +"""Lossless semantic normalization for archived Copilot source records. + +This module deliberately does not correlate records across sources. In +particular, a hook observation has no invocation ID in the observed corpus, +so it remains an observation instead of becoming a guessed tool span. + +Database and metadata records are accounting/identity evidence owned by +other V2 modules. They are skipped here so usage rows cannot appear as +``unknown`` semantic events in default turn views. +""" + +from __future__ import annotations + +from copy import deepcopy +from typing import Any + +from .constants import SOURCE_SCHEMA_VERSION +from .types import ( + EventClassification, + NormalizedEvent, + NormalizedEventKind, + SourceRecord, + SourceReference, + SourceReferenceRole, +) + +_TRANSCRIPT_OR_HOOK = frozenset({"transcript", "hook"}) +_SESSION_LIFECYCLE_NOTIFICATIONS = frozenset( + { + "notification", + "session.notification", + "session.model_change", + "session.auto_mode_resolved", + "hook.start", + "hook.end", + "preToolUse", + "postToolUse", + "postToolUseFailure", + "subagent.configured", + } +) +_ABORT_TYPES = frozenset({"assistant.abort", "session.abort", "abort"}) +_HOOK_KINDS: dict[str, tuple[NormalizedEventKind, SourceReferenceRole]] = { + "userPromptSubmitted": ("prompt_transformation", "hook"), + "agentStop": ("agent_stop", "finish"), + "subagentStart": ("subagent_started", "nested_child"), + "subagentStop": ("subagent_completed", "nested_child"), + "sessionStart": ("session_start", "hook"), + "sessionEnd": ("session_end", "hook"), +} + + +def record_type(record: SourceRecord) -> str | None: + """Return the native event name carried by an archived record.""" + payload = record.get("payload", {}) + if not isinstance(payload, dict): + return None + value = payload.get("type") if record.get("source_kind") != "hook" else payload.get("event") + return value if isinstance(value, str) and value else None + + +def record_data(record: SourceRecord) -> dict[str, Any]: + """Return the native data mapping without rewriting the raw record.""" + payload = record.get("payload", {}) + if not isinstance(payload, dict): + return {} + if record.get("source_kind") == "hook": + data = payload.get("hook_payload") + else: + data = payload.get("data") + return data if isinstance(data, dict) else {} + + +def agent_id(record: SourceRecord) -> str | None: + payload = record.get("payload", {}) + if isinstance(payload, dict) and isinstance(payload.get("agentId"), str): + return payload["agentId"] + data = record_data(record) + for key in ("agentId", "agent_id"): + value = data.get(key) + if value is not None and str(value): + return str(value) + return None + + +def interaction_id(record: SourceRecord) -> str | None: + value = record_data(record).get("interactionId") + return str(value) if value is not None and str(value) else None + + +def tool_call_id(record: SourceRecord) -> str | None: + data = record_data(record) + for key in ("toolCallId", "tool_call_id"): + value = data.get(key) + if value is not None and str(value): + return str(value) + return None + + +def source_reference(record: SourceRecord, role: SourceReferenceRole) -> SourceReference: + """Role-labelled pointer back to one immutable source record.""" + return { + "source_id": record["source_id"], + "source_kind": record["source_kind"], + "role": role, + } + + +def _event( + record: SourceRecord, + kind: NormalizedEventKind, + role: SourceReferenceRole, + *, + classification: EventClassification = "main", + suffix: str = "", + attributes: dict[str, Any] | None = None, +) -> NormalizedEvent: + return { + "id": f"copilot:event:{record['source_id']}{suffix}", + "kind": kind, + "classification": classification, + "ts": record.get("ts"), + "source_ids": [record["source_id"]], + "source_references": [source_reference(record, role)], + "attributes": attributes or {}, + } + + +def _identity_attributes(record: SourceRecord) -> dict[str, Any]: + data = record_data(record) + return { + "interaction_id": interaction_id(record), + "agent_id": agent_id(record), + "parent_tool_call_id": data.get("parentToolCallId"), + "turn_id": data.get("turnId"), + } + + +def unknown_source_schema(record: SourceRecord) -> bool: + """Return whether the archived payload names a schema other than V1.""" + + payload = record.get("payload") + if not isinstance(payload, dict) or "schema_version" not in payload: + return False + return payload.get("schema_version") != SOURCE_SCHEMA_VERSION + + +def _unknown(record: SourceRecord, native_type: str | None) -> list[NormalizedEvent]: + attrs: dict[str, Any] = {"raw_payload": deepcopy(record.get("payload"))} + if native_type is not None: + attrs["native_type"] = native_type + role: SourceReferenceRole = ( + "hook" if record.get("source_kind") == "hook" else "assistant_message" + ) + return [_event(record, "unknown", role, attributes=attrs)] + + +def normalize_record(record: SourceRecord) -> list[NormalizedEvent]: + """Normalize one record while retaining a direct pointer to its evidence.""" + if record.get("source_kind") not in _TRANSCRIPT_OR_HOOK: + return [] + + if unknown_source_schema(record): + return _unknown(record, record_type(record)) + + native_type = record_type(record) + data = record_data(record) + identity = _identity_attributes(record) + if native_type is None: + return _unknown(record, None) + + if native_type == "user.message": + attrs = { + **identity, + "delivery": data.get("delivery"), + "source": data.get("source"), + } + return [_event(record, "user_prompt", "user_prompt", attributes=attrs)] + + if native_type == "assistant.message": + attrs = {**identity, "model": data.get("model"), "phase": data.get("phase")} + events = [_event(record, "assistant_message", "assistant_message", attributes=attrs)] + requests = data.get("toolRequests") + if isinstance(requests, list): + for request in requests: + if not isinstance(request, dict) or not request.get("toolCallId"): + continue + call_id = str(request["toolCallId"]) + events.append( + _event( + record, + "tool_request", + "tool_request", + suffix=f":tool:{call_id}", + attributes={ + **identity, + "tool_call_id": call_id, + "name": request.get("name"), + "arguments": deepcopy(request.get("arguments")), + "intention_summary": request.get("intentionSummary"), + }, + ) + ) + return events + + if native_type == "assistant.turn_start": + return [_event(record, "assistant_turn_start", "assistant_message", attributes=identity)] + if native_type == "assistant.turn_end": + return [_event(record, "assistant_turn_end", "finish", attributes=identity)] + + if native_type == "tool.execution_start": + return [ + _event( + record, + "tool_execution_start", + "tool_execution", + attributes={ + **identity, + "tool_call_id": tool_call_id(record), + "name": data.get("toolName"), + "arguments": deepcopy(data.get("arguments")), + }, + ) + ] + if native_type in {"tool.execution_complete", "tool.execution_error", "tool.execution_failure"}: + failed = native_type != "tool.execution_complete" or data.get("success") is False + kind: NormalizedEventKind = ( + "tool_execution_failure" if failed else "tool_execution_complete" + ) + return [ + _event( + record, + kind, + "tool_result", + attributes={ + **identity, + "tool_call_id": tool_call_id(record), + "success": data.get("success"), + "result": deepcopy(data.get("result")), + }, + ) + ] + + if native_type in {"permission.request", "permissionRequest"}: + return [ + _event( + record, + "permission_request", + "permission_request", + attributes={ + **identity, + "tool_name": data.get("toolName"), + "arguments": deepcopy(data.get("toolArgs")), + }, + ) + ] + if native_type in {"permission.decision", "permissionDecision"}: + return [ + _event( + record, + "permission_decision", + "permission_decision", + attributes={ + **identity, + "decision": data.get("decision"), + "tool_name": data.get("toolName"), + }, + ) + ] + + if native_type in {"prompt.transformation", "prompt.transformed"}: + return [ + _event( + record, + "prompt_transformation", + "hook", + attributes={**identity, **deepcopy(data)}, + ) + ] + + if native_type in _ABORT_TYPES: + return [_event(record, "abort", "finish", attributes=identity)] + + if native_type.startswith("model."): + return [ + _event( + record, + "auxiliary_model_call", + "auxiliary_model", + classification="title_generation", + attributes={"native_type": native_type, **deepcopy(data)}, + ) + ] + + if native_type in {"session.start", "session.resume"}: + return [_event(record, "session_start", "hook", attributes=deepcopy(data))] + if native_type in {"session.end", "session.close"}: + return [_event(record, "session_end", "shutdown", attributes=deepcopy(data))] + if native_type == "session.shutdown": + return [ + _event( + record, + "session_shutdown", + "shutdown", + classification="shutdown_validation", + attributes=deepcopy(data), + ) + ] + if native_type in { + "session.usage_checkpoint", + "preCompact", + "session.compaction", + "context.compaction", + }: + checkpoint = native_type == "session.usage_checkpoint" + return [ + _event( + record, + "compaction", + "checkpoint" if checkpoint else "compaction", + classification="checkpoint" if checkpoint else "main", + attributes=deepcopy(data), + ) + ] + + if native_type == "subagent.started": + call = tool_call_id(record) + return [ + _event( + record, + "subagent_started", + "nested_child", + attributes={ + **identity, + **deepcopy(data), + "tool_call_id": call, + "parent_tool_call_id": call, + }, + ) + ] + if native_type == "subagent.completed": + call = tool_call_id(record) + return [ + _event( + record, + "subagent_completed", + "nested_child", + attributes={ + **identity, + **deepcopy(data), + "tool_call_id": call, + "parent_tool_call_id": call, + }, + ) + ] + + if native_type in _SESSION_LIFECYCLE_NOTIFICATIONS: + return [ + _event( + record, + "notification", + "hook", + attributes={**identity, "native_type": native_type, **deepcopy(data)}, + ) + ] + + mapped = _HOOK_KINDS.get(native_type) + if mapped is not None: + kind, role = mapped + return [ + _event( + record, + kind, + role, + attributes={**identity, "native_type": native_type, **deepcopy(data)}, + ) + ] + + if "error" in native_type.lower(): + return [ + _event( + record, + "error", + "finish", + attributes={**identity, "message": data.get("message") or data.get("error")}, + ) + ] + + return _unknown(record, native_type) + + +def normalize_records(records: list[SourceRecord]) -> list[NormalizedEvent]: + """Normalize archive records in replay order without deduplicating evidence.""" + return [event for record in records for event in normalize_record(record)] diff --git a/src/thirdeye/platforms/copilot/export.py b/src/thirdeye/platforms/copilot/export.py new file mode 100644 index 00000000..de5c6dc4 --- /dev/null +++ b/src/thirdeye/platforms/copilot/export.py @@ -0,0 +1,633 @@ +"""Assemble eligible Copilot projections into generic detached OTel jobs. + +No network operation happens here. This module only writes/dispatches the +generic transport's local jobs; the detached worker performs remote delivery. +The split cannot provide transactional exactly-once delivery: a crash after a +remote flush and before acknowledgement can retry a deterministic span. The +transport's durable turn claims and Copilot's accounting delivery claims are +what let this module tell "already confirmed delivered" apart from "merely +queued" across restarts, since workers delete their job files on success. +""" + +from __future__ import annotations + +from copy import deepcopy +from pathlib import Path +from typing import Any + +from thirdeye import otel_export +from thirdeye.config import Config +from thirdeye.meta import read_meta +from thirdeye.paths import meta_path, session_dir +from thirdeye.span_ids import chat_span_id + +from . import export_transport +from .constants import PLATFORM_NAME +from .export_state import ( + clear_placement_error, + clear_turn_error, + initialize_eligibility, + is_accounting_eligible, + is_turn_eligible, + mark_placement_error, + mark_placement_job_status, + mark_turn_error, + record_placement, + set_pending_unowned_accounting, + update_export_state, + usage_digest, +) +from .types import Projection + +_TERMINAL = frozenset({"completed", "interrupted", "errored"}) +_EXPORTABLE_ATTRIBUTIONS = frozenset({"matched", "ambiguous"}) + + +def _copilot_supplemental_export_attributes(candidate: dict[str, Any] | None) -> dict[str, Any]: + metrics = (candidate or {}).get("supplemental_metrics") or {} + attributes: dict[str, Any] = {} + if "total_nano_aiu" in metrics: + attributes["copilot.billing.nano_aiu"] = metrics["total_nano_aiu"] + if "duration_ms" in metrics: + attributes["copilot.latency.duration_ms"] = metrics["duration_ms"] + if "output_ttft_ms" in metrics: + attributes["copilot.latency.output_ttft_ms"] = metrics["output_ttft_ms"] + if "time_to_first_token_ms" in metrics: + attributes["copilot.latency.time_to_first_token_ms"] = metrics["time_to_first_token_ms"] + if "inter_token_latency_ms" in metrics: + attributes["copilot.latency.inter_token_latency_ms"] = metrics["inter_token_latency_ms"] + return attributes + + +def _directory(config: Config, stored_session_id: str) -> Path: + return session_dir(config.root, PLATFORM_NAME, stored_session_id) + + +def _terminal(turn: dict[str, Any] | None) -> bool: + return bool(turn) and turn.get("status") in _TERMINAL + + +def _walk_turns(turns: list[dict[str, Any]]) -> list[dict[str, Any]]: + found: list[dict[str, Any]] = [] + for turn in turns: + found.append(turn) + children = turn.get("subagents") + if isinstance(children, list): + found.extend(_walk_turns([child for child in children if isinstance(child, dict)])) + return found + + +def _main_terminal_turns(projection: Projection) -> list[dict[str, Any]]: + return [ + turn + for turn in projection["turns"] + if isinstance(turn, dict) + and _terminal(turn) + and not isinstance((turn.get("attributes") or {}).get("agent_id"), str) + ] + + +def _root_owner_index(projection: Projection) -> dict[str, dict[str, Any]]: + """Map every turn id, main or nested, to its owning main ``TurnSpanDict``. + + Eligibility and "is this history" are properties of a whole interaction + (a main turn and everything nested under it, exported together as one + job), never of a nested child's own id — the child is never exported as + an independent top-level job, so its id alone is never a member of the + boundary lists built from ``_main_terminal_turns``. + """ + index: dict[str, dict[str, Any]] = {} + + def walk(turn: dict[str, Any], root: dict[str, Any]) -> None: + turn_id = turn.get("turn_id") + if isinstance(turn_id, str): + index[turn_id] = root + for child in turn.get("subagents") or []: + if isinstance(child, dict): + walk(child, root) + + for turn in projection["turns"]: + if isinstance(turn, dict): + walk(turn, turn) + return index + + +def _root_for(root_index: dict[str, dict[str, Any]], turn_id: object) -> dict[str, Any] | None: + if isinstance(turn_id, str): + return root_index.get(turn_id) + return None + + +def _accounting_calls(projection: Projection) -> dict[str, tuple[dict[str, Any], dict[str, Any]]]: + """Map an identity to its owner turn and serialized generic accounting.""" + found: dict[str, tuple[dict[str, Any], dict[str, Any]]] = {} + for turn in _walk_turns([turn for turn in projection["turns"] if isinstance(turn, dict)]): + for item in turn.get("accounting_calls") or []: + if not isinstance(item, dict): + continue + accounting_id = item.get("accounting_id") + usage = item.get("usage") + if isinstance(accounting_id, str) and isinstance(usage, dict): + found[accounting_id] = (turn, item) + return found + + +def _historical_accounting_ids( + projection: Projection, + root_index: dict[str, dict[str, Any]], + *, + session_closed: bool, +) -> list[str]: + """Accounting identities that are already part of terminal history. + + An identity owned by a still-open interaction stays eligible for later + export once that interaction completes. Missing ownership in an open + session is likewise provisional and cannot become immutable history merely + because transcript reconstruction currently lags the usage row. Once the + session is explicitly closed, a non-pending row with no resolved root is + settled session history. A pending join remains unresolved in either case. + """ + owners: dict[str, dict[str, Any] | None] = {} + for accounting_id, (owner, _item) in _accounting_calls(projection).items(): + owners[accounting_id] = _root_for(root_index, owner.get("turn_id")) + unresolved: set[str] = set() + for attribution in projection["attributions"]: + accounting_id = attribution["logical_call_id"] + if accounting_id in owners: + continue + if attribution["status"] == "pending": + unresolved.add(accounting_id) + continue + root = _root_for(root_index, attribution["stored_turn_id"]) + if root is None and not session_closed: + unresolved.add(accounting_id) + continue + owners[accounting_id] = root + for row in projection["usage_rows"]: + if row.call_id in unresolved: + continue + if row.call_id not in owners and not session_closed: + continue + owners.setdefault(row.call_id, None) + return sorted( + accounting_id for accounting_id, root in owners.items() if root is None or _terminal(root) + ) + + +def _span_id(session_id: str, turn_id: str | None, accounting_id: str) -> str: + if turn_id is None: + return f"accounting:{session_id}:{accounting_id}" + return f"accounting:{session_id}:{turn_id}:{accounting_id}" + + +def _turn_span_id(stored_session_id: str, turn_id: str) -> str: + """Return the OTel-safe deterministic span id for a source-derived turn.""" + return str(chat_span_id(PLATFORM_NAME, stored_session_id, f"turn:{turn_id}")) + + +def _configured(config: Config) -> bool: + return config.logfire.enabled and bool(config.logfire.token) + + +def _error_state(config: Config, stored_session_id: str, accounting_id: str, message: str) -> None: + def update(state: dict[str, Any]) -> dict[str, Any]: + return mark_placement_error(state, accounting_id, message) + + update_export_state(config, stored_session_id, update) + + +def _turn_error_state(config: Config, stored_session_id: str, turn_id: str, message: str) -> None: + def update(state: dict[str, Any]) -> dict[str, Any]: + return mark_turn_error(state, turn_id, message) + + update_export_state(config, stored_session_id, update) + + +def _clear_error_state(config: Config, stored_session_id: str, accounting_id: str) -> None: + def update(state: dict[str, Any]) -> dict[str, Any]: + return clear_placement_error(state, accounting_id) + + update_export_state(config, stored_session_id, update) + + +def _clear_turn_error_state(config: Config, stored_session_id: str, turn_id: str) -> None: + def update(state: dict[str, Any]) -> dict[str, Any]: + return clear_turn_error(state, turn_id) + + update_export_state(config, stored_session_id, update) + + +def _eligible_state( + config: Config, + stored_session_id: str, + projection: Projection, + root_index: dict[str, dict[str, Any]], + *, + include_history: bool, + session_closed: bool, +) -> dict[str, Any]: + terminal_ids = [str(turn["turn_id"]) for turn in _main_terminal_turns(projection)] + + def update(state: dict[str, Any]) -> dict[str, Any]: + return initialize_eligibility( + state, + terminal_turn_ids=terminal_ids, + accounting_ids=_historical_accounting_ids( + projection, root_index, session_closed=session_closed + ), + include_history=include_history, + ) + + return update_export_state(config, stored_session_id, update) + + +def _place( + config: Config, + stored_session_id: str, + *, + accounting_id: str, + destination: str, + span_id: str, + usage: dict[str, Any], + delivered: bool, +) -> tuple[dict[str, Any] | None, bool]: + captured: dict[str, Any] = {} + + def update(state: dict[str, Any]) -> dict[str, Any]: + existing = state.get("placements", {}).get(accounting_id) or {} + old_span_id = existing.get("span_id") + old_job_state: str | None = None + digest = usage_digest(usage) + usage_changed = existing.get("usage_digest") != digest + if isinstance(old_span_id, str) and old_span_id and old_span_id != span_id: + # Relocating to a different span: find out whether the job under + # the *old* span id is provably inert before deciding it is safe + # to replace, and if it is, cancel it so an already-dispatched + # worker can never deliver tokens this ledger no longer points + # at once it decides on the new destination below. + status = export_transport.cancel(config.root, old_span_id) + old_job_state = status.get("state") + elif ( + isinstance(old_span_id, str) + and old_span_id == span_id + and usage_changed + and not delivered + ): + # Same span, new usage, still undelivered: rewrite the queued + # payload. A claimed in-flight job cannot be overwritten. + current = export_transport.status(config.root, old_span_id) + current_state = (current or {}).get("state") + if current_state == "claimed": + old_job_state = "claimed" + elif current_state in {"queued", "retrying", "failed"}: + status = export_transport.cancel(config.root, old_span_id) + old_job_state = status.get("state") + next_state, entry, accepted = record_placement( + state, + accounting_id=accounting_id, + destination=destination, + span_id=span_id, + usage=usage, + delivered=delivered, + old_job_state=old_job_state, + ) + captured["entry"] = entry + captured["accepted"] = accepted + return next_state + + update_export_state(config, stored_session_id, update) + return captured.get("entry"), bool(captured.get("accepted")) + + +def _reflect_accounting_job_health( + config: Config, stored_session_id: str, accounting_id: str, span_id: str +) -> bool: + """Fold the deterministic accounting job's own on-disk state into the + ledger's error tracking after a local write+dispatch reported success. + + A local write succeeding only means the job file exists and a worker was + spawned -- it says nothing about whether that worker (this call, or an + earlier one that already ran and gave up) ever actually delivered it. + Without this check, a job that exhausted its retries and is stuck in a + permanent ``"failed"`` state on disk would be silently reported as + healthy and have any earlier ``last_error`` cleared on every subsequent + reconciliation, even though the generic transport will never retry it + again on its own (see ``otel_worker._claim_job``). + + Returns whether this counts as a successful queue for this pass. + """ + status = export_transport.status(config.root, span_id) + if status is None: + if export_transport.delivery_sent(_directory(config, stored_session_id), accounting_id): + status = {"state": "emitted", "attempt": None, "last_error": None} + else: + _clear_error_state(config, stored_session_id, accounting_id) + return True + + if status.get("state") == "failed" and not status.get("last_error"): + status = { + **status, + "last_error": ( + f"accounting job permanently failed after {status.get('attempt')} attempts" + ), + } + + update_export_state( + config, + stored_session_id, + lambda state: mark_placement_job_status(state, accounting_id, status), + ) + return status.get("state") != "failed" + + +def _turn_with_placed_accounting( + turn: dict[str, Any], placements: dict[str, dict[str, Any]] +) -> dict[str, Any]: + """Return a recursive turn copy containing only its chat-placed tokens.""" + result = deepcopy(turn) + calls = [] + for accounting in result.get("accounting_calls") or []: + entry = placements.get(str(accounting.get("accounting_id"))) + if entry and entry.get("destination") == "chat-span": + calls.append(accounting) + result["accounting_calls"] = calls + result["subagents"] = [ + _turn_with_placed_accounting(child, placements) + for child in result.get("subagents") or [] + if isinstance(child, dict) + ] + return result + + +def _with_deterministic_turn_ids(turn: dict[str, Any], stored_session_id: str) -> dict[str, Any]: + """Finalize a complete turn tree with ids stable across archive replays. + + Copilot's stable turn identity is a source-derived string rather than the + integer sequence used by the older live adapters. ``chat_span_id`` is a + domain-separated deterministic 64-bit id function, and the ``turn:`` + prefix keeps this turn namespace distinct from model-call ids. + """ + result = deepcopy(turn) + turn_id = result.get("turn_id") + if isinstance(turn_id, str) and turn_id: + result["turn_span_id"] = _turn_span_id(stored_session_id, turn_id) + result["subagents"] = [ + _with_deterministic_turn_ids(child, stored_session_id) + for child in result.get("subagents") or [] + if isinstance(child, dict) + ] + return result + + +def _fallback_accounting_call( + attribution: dict[str, Any], + usage: dict[str, Any], + candidate: dict[str, Any] | None, +) -> dict[str, Any]: + """Serialize accounting whose exact chat placement never became available.""" + return { + "accounting_id": attribution["logical_call_id"], + "usage": usage, + "attribution_status": attribution["status"], + "agent_id": attribution["agent_id"], + "call_id": None, + "attributes": { + "logical_call_id": attribution["logical_call_id"], + "usage_source_id": attribution["usage_source_id"], + "evidence": list(attribution["evidence"]), + **_copilot_supplemental_export_attributes(candidate), + }, + } + + +def queue_exports( + config: Config, + stored_session_id: str, + projection: Projection, + *, + include_history: bool = False, +) -> int: + """Queue eligible terminal projection evidence without reading live Copilot data. + + ``include_history`` is the explicit ``--export`` opt-in supplied by + runtime composition. The default first activation records all currently + terminal evidence as local-only. Subsequent terminal interactions and + later accounting rows are eligible. The returned count is local jobs + accepted for dispatch, not confirmed remote delivery. + """ + directory = _directory(config, stored_session_id) + meta = read_meta(meta_path(directory)) + if meta is None: + raise ValueError(f"unknown Copilot session: {stored_session_id}") + session_closed = meta.status == "closed" + root_index = _root_owner_index(projection) + state = _eligible_state( + config, + stored_session_id, + projection, + root_index, + include_history=include_history, + session_closed=session_closed, + ) + if not _configured(config): + return 0 + + accounting = _accounting_calls(projection) + rows = {row.call_id: row.to_dict() for row in projection["usage_rows"]} + candidates = { + item["logical_call_id"]: item + for item in projection.get("accounting_candidates") or [] + if isinstance(item, dict) and isinstance(item.get("logical_call_id"), str) + } + placements: dict[str, dict[str, Any]] = {} + deferred_ids: set[str] = set() + blocked_root_ids: set[str] = set() + queued = 0 + + # Place terminal owned accounting first. This decision happens before a + # whole-turn job is assembled, so a future local match cannot move a + # fallback charge onto a chat span. + for accounting_id, (owner, item) in accounting.items(): + owner_id = owner.get("turn_id") + root = _root_for(root_index, owner_id) + usage = item["usage"] + if ( + not isinstance(owner_id, str) + or root is None + or not _terminal(root) + or item.get("attribution_status") not in _EXPORTABLE_ATTRIBUTIONS + ): + continue + root_id = str(root["turn_id"]) + if not is_turn_eligible(state, root_id) or not is_accounting_eligible(state, accounting_id): + continue + call_id = item.get("call_id") + if ( + item.get("attribution_status") == "matched" + and isinstance(call_id, str) + and call_id + and not any(call.get("call_id") == call_id for call in owner.get("llm_calls") or []) + ): + if not session_closed: + deferred_ids.add(accounting_id) + blocked_root_ids.add(root_id) + continue + item = {**item, "call_id": None} + call_id = None + existing_entry = (state.get("placements") or {}).get(accounting_id) or {} + turn_sent = otel_export.turn_export_sent(directory, root_id) + delivered = export_transport.delivery_sent(directory, accounting_id) or ( + turn_sent and existing_entry.get("destination") == "chat-span" + ) + # A chat span already flushed to Logfire is immutable history: usage + # that resolves to "matched" only *after* that flush can no longer + # land on it and must use the turn-owned fallback span instead. But + # if *this* identity was itself the one embedded on that chat span + # (confirmed via the durable delivery claim plus the ledger's own + # record of where it landed), the turn having been sent is exactly + # what delivered it there -- re-deriving "unavailable" from that same + # fact would misread an already-settled placement as a conflicting + # correction. + chat_available = not turn_sent or ( + delivered and existing_entry.get("destination") == "chat-span" + ) + if ( + chat_available + and item.get("attribution_status") == "matched" + and isinstance(call_id, str) + and call_id + ): + destination = "chat-span" + span_id = str(call_id) + else: + destination = "turn-accounting-span" + span_id = _span_id(stored_session_id, owner_id, accounting_id) + entry, accepted = _place( + config, + stored_session_id, + accounting_id=accounting_id, + destination=destination, + span_id=span_id, + usage=usage, + delivered=delivered, + ) + if not accepted: + continue + if entry is not None: + placements[accounting_id] = entry + if entry is not None and entry.get("emitted"): + # Confirmed delivered, whether just now or on an earlier pass. + # Never recreate an accounting job after that point. + continue + if destination != "turn-accounting-span" or entry is None: + continue + sent = export_transport.queue_turn_accounting( + config, + directory, + stored_session_id, + meta.cwd, + owner_id, + item, + turn_span_id=_turn_span_id(stored_session_id, owner_id), + ) + if sent and _reflect_accounting_job_health( + config, stored_session_id, accounting_id, span_id + ): + queued += 1 + elif not sent: + _error_state(config, stored_session_id, accounting_id, "accounting job was not queued") + + # An exportable attribution can temporarily precede reconstruction of its + # owner. Until explicit session completion, absence from ``accounting`` is + # incomplete evidence rather than proof of session-level ownership. + attached_ids = set(accounting) + for attribution in projection["attributions"]: + accounting_id = attribution["logical_call_id"] + if attribution["status"] not in _EXPORTABLE_ATTRIBUTIONS: + # Pending attribution is intentionally local-only until later + # archive evidence resolves it. A conflicting join is likewise + # quarantined rather than guessed into an accounting span. + continue + if accounting_id in attached_ids or not is_accounting_eligible(state, accounting_id): + continue + usage = rows.get(accounting_id) + if usage is None: + continue + owner_id = attribution.get("stored_turn_id") + root = _root_for(root_index, owner_id) + if not session_closed: + deferred_ids.add(accounting_id) + if root is not None: + blocked_root_ids.add(str(root["turn_id"])) + continue + + item = _fallback_accounting_call(attribution, usage, candidates.get(accounting_id)) + if isinstance(owner_id, str) and root is not None and _terminal(root): + root_id = str(root["turn_id"]) + if not is_turn_eligible(state, root_id): + continue + destination = "turn-accounting-span" + span_id = _span_id(stored_session_id, owner_id, accounting_id) + else: + destination = "session-accounting-span" + span_id = _span_id(stored_session_id, None, accounting_id) + delivered = export_transport.delivery_sent(directory, accounting_id) + entry, accepted = _place( + config, + stored_session_id, + accounting_id=accounting_id, + destination=destination, + span_id=span_id, + usage=usage, + delivered=delivered, + ) + if not accepted or entry is None: + continue + if entry.get("emitted"): + continue + if destination == "turn-accounting-span": + sent = export_transport.queue_turn_accounting( + config, + directory, + stored_session_id, + meta.cwd, + owner_id, + item, + turn_span_id=_turn_span_id(stored_session_id, owner_id), + ) + else: + sent = export_transport.queue_session_accounting( + config, directory, stored_session_id, meta.cwd, item + ) + if sent and _reflect_accounting_job_health( + config, stored_session_id, accounting_id, span_id + ): + queued += 1 + elif not sent: + _error_state(config, stored_session_id, accounting_id, "accounting job was not queued") + + state = update_export_state( + config, + stored_session_id, + lambda current: set_pending_unowned_accounting(current, sorted(deferred_ids)), + ) + + # Generic transport provides a persistent completed-turn claim. Sending + # a duplicate local job is harmless there; no network I/O occurs here. + for turn in _main_terminal_turns(projection): + turn_id = str(turn["turn_id"]) + if turn_id in blocked_root_ids or not is_turn_eligible(state, turn_id): + continue + assembled = _with_deterministic_turn_ids( + _turn_with_placed_accounting(turn, placements), stored_session_id + ) + sent = export_transport.queue_turn( + config, directory, stored_session_id, meta.cwd, assembled + ) + if sent: + queued += 1 + if turn_id in (state.get("turn_errors") or {}): + _clear_turn_error_state(config, stored_session_id, turn_id) + else: + _turn_error_state(config, stored_session_id, turn_id, "turn export job was not queued") + return queued diff --git a/src/thirdeye/platforms/copilot/export_state.py b/src/thirdeye/platforms/copilot/export_state.py new file mode 100644 index 00000000..e084730d --- /dev/null +++ b/src/thirdeye/platforms/copilot/export_state.py @@ -0,0 +1,399 @@ +"""Durable Copilot export eligibility and accounting-placement state. + +This state intentionally has no relationship to projection state. Projection +state is disposable and rebuilt from the V1 archive; export placement is an +accounting decision and survives rebuilds. The accounting worker cannot +atomically acknowledge a remote collector and this file, so an absent job is +never treated as proof of delivery. A crash after a remote flush can still +lead to a deterministic retry and therefore a duplicate remote span. + +``record_placement`` takes an explicit ``delivered`` flag from the caller +(who reads Copilot's independent, durable accounting delivery claim before +calling in). That flag, not merely a changed destination, is what turns a correction into a +quarantined conflict: a correction to a job that never left this machine is +always safe to replace, while a correction after confirmed delivery can no +longer relocate tokens the remote collector already has. +""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +from thirdeye._compat import fsops +from thirdeye._compat.locking import LockMode, locked +from thirdeye.config import Config +from thirdeye.paths import session_dir + +from .constants import PLATFORM_NAME +from .jsonio import atomic_write_json, read_json_object + +EXPORT_STATE_FILENAME = "copilot.export.ledger.json" +EXPORT_LOCK_FILENAME = "copilot.export.lock" +EXPORT_STATE_SCHEMA_VERSION = 1 + + +def export_state_path(directory: Path) -> Path: + return directory / EXPORT_STATE_FILENAME + + +def export_lock_path(directory: Path) -> Path: + return directory / EXPORT_LOCK_FILENAME + + +def _directory(config: Config, stored_session_id: str) -> Path: + return session_dir(config.root, PLATFORM_NAME, stored_session_id) + + +def empty_export_state() -> dict[str, Any]: + """Return the versioned state used before the first V2 export activation.""" + return { + "schema_version": EXPORT_STATE_SCHEMA_VERSION, + "activated": False, + "excluded_turn_ids": [], + "excluded_accounting_ids": [], + "placements": {}, + "conflicts": {}, + "turn_errors": {}, + "pending_unowned_accounting": [], + } + + +def _ids(values: object) -> list[str]: + if not isinstance(values, list): + return [] + return sorted({value for value in values if isinstance(value, str) and value}) + + +def _mapping(value: object) -> dict[str, Any]: + return value if isinstance(value, dict) else {} + + +def _normalize(value: dict[str, Any]) -> dict[str, Any]: + """Fill defaults on a dict already known to carry the current schema. + + Only safe for state this module itself produced (loaded and version + checked by :func:`_read`, or built fresh by another function here) — + never call this directly on unvalidated bytes off disk. + """ + placements = _mapping(value.get("placements")) + conflicts = _mapping(value.get("conflicts")) + turn_errors = _mapping(value.get("turn_errors")) + return { + "schema_version": EXPORT_STATE_SCHEMA_VERSION, + "activated": bool(value.get("activated", False)), + "excluded_turn_ids": _ids(value.get("excluded_turn_ids")), + "excluded_accounting_ids": _ids(value.get("excluded_accounting_ids")), + "pending_unowned_accounting": _ids(value.get("pending_unowned_accounting")), + "placements": { + key: item + for key, item in placements.items() + if isinstance(key, str) and isinstance(item, dict) + }, + "conflicts": { + key: item + for key, item in conflicts.items() + if isinstance(key, str) and isinstance(item, dict) + }, + "turn_errors": { + key: value2 + for key, value2 in turn_errors.items() + if isinstance(key, str) and isinstance(value2, str) + }, + } + + +def _read(directory: Path) -> dict[str, Any]: + """Load the ledger, failing closed on anything but a genuinely absent file. + + A missing file is the only case that legitimately means "no export has + ever run for this session" and may start from empty state. Malformed + JSON, a non-object document, or an unrecognized ``schema_version`` are + all corruption or a future/unknown format from this module's point of + view: silently treating them as empty would forget every placement this + ledger recorded and risk emitting tokens at a second location. + """ + raw = read_json_object( + export_state_path(directory), invalid_message="invalid Copilot export ledger" + ) + if raw is None: + return empty_export_state() + version = raw.get("schema_version") + if version != EXPORT_STATE_SCHEMA_VERSION: + raise ValueError(f"unsupported Copilot export ledger schema_version: {version!r}") + return _normalize(raw) + + +def _write(directory: Path, state: dict[str, Any]) -> None: + atomic_write_json(export_state_path(directory), state) + + +def load_export_state(config: Config, stored_session_id: str) -> dict[str, Any]: + """Read a snapshot of the independent export ledger. + + Callers that mutate the result must use :func:`update_export_state`; this + helper intentionally returns a detached JSON-safe dictionary. + """ + directory = _directory(config, stored_session_id) + with locked(export_lock_path(directory), LockMode.SHARED): + return json.loads(json.dumps(_read(directory))) + + +def update_export_state(config: Config, stored_session_id: str, update: Any) -> dict[str, Any]: + """Atomically apply ``update(state)`` and return the published state.""" + directory = _directory(config, stored_session_id) + with locked(export_lock_path(directory), LockMode.EXCLUSIVE): + state = _read(directory) + updated = update(state) + if not isinstance(updated, dict): + raise TypeError("Copilot export state update must return a dictionary") + state = _normalize(updated) + _write(directory, state) + return json.loads(json.dumps(state)) + + +def usage_digest(usage: dict[str, Any]) -> str: + """Stable correction detector for a serialized ``UsageRow``.""" + encoded = json.dumps(usage, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + return "sha256:" + hashlib.sha256(encoded.encode("utf-8")).hexdigest() + + +def is_turn_eligible(state: dict[str, Any], turn_id: str) -> bool: + return turn_id not in set(_ids(state.get("excluded_turn_ids"))) + + +def is_accounting_eligible(state: dict[str, Any], accounting_id: str) -> bool: + return accounting_id not in set(_ids(state.get("excluded_accounting_ids"))) + + +def set_pending_unowned_accounting( + state: dict[str, Any], accounting_ids: list[str] +) -> dict[str, Any]: + """Replace the accounting identities waiting for ownership evidence.""" + result = _normalize(state) + result["pending_unowned_accounting"] = _ids(accounting_ids) + return result + + +def initialize_eligibility( + state: dict[str, Any], + *, + terminal_turn_ids: list[str], + accounting_ids: list[str], + include_history: bool, +) -> dict[str, Any]: + """Set the first-activation boundary without changing any placement. + + Normal activation excludes terminal history. An explicit historical + export opts the supplied completed turns and accounting identities in by + removing them from that boundary. New evidence is absent from both lists + and is therefore eligible after restart. + + Callers are responsible for passing only turn ids and accounting ids that + are *actually* already historical (a terminal main interaction, or + accounting owned by one) — an interaction still open at first activation + must not appear here, or it would stay excluded forever even once it + later completes. + """ + result = _normalize(state) + turn_ids = set(_ids(terminal_turn_ids)) + usage_ids = set(_ids(accounting_ids)) + if not result["activated"]: + result["activated"] = True + if not include_history: + result["excluded_turn_ids"] = sorted(turn_ids) + result["excluded_accounting_ids"] = sorted(usage_ids) + return result + if include_history: + result["excluded_turn_ids"] = sorted(set(result["excluded_turn_ids"]) - turn_ids) + result["excluded_accounting_ids"] = sorted( + set(result["excluded_accounting_ids"]) - usage_ids + ) + return result + + +def record_placement( + state: dict[str, Any], + *, + accounting_id: str, + destination: str, + span_id: str, + usage: dict[str, Any], + delivered: bool = False, + old_job_state: str | None = None, +) -> tuple[dict[str, Any], dict[str, Any] | None, bool]: + """Persist the first token location for an accounting identity. + + ``delivered`` is the caller's fresh read of Copilot's durable delivery + claim for this identity (see ``export_transport.delivery_sent``), not + this ledger's own possibly + stale ``emitted`` flag — the local job file backing that flag is deleted + by the worker on success, so this ledger cannot detect delivery on its + own and must be told. + + ``old_job_state`` is the Copilot transport's result after atomically + attempting to cancel the *existing* placement's own job. ``"cancelled"`` + means relocation is safe. ``"claimed"`` means a worker holds it right now + and could deliver at any moment; ``"emitted"`` means it already completed. + Both states quarantine the correction. + + Returns ``(state, entry, accepted)``. A correction is only a durable + conflict when the existing placement was (or is now known to have been) + delivered, or when its job might still be in flight: rebinding a + destination or usage value the remote collector already has, or might + still receive, risks two different token totals for the same logical + call. A correction to a placement proven to have never been queued, or + proven to have permanently failed, is always safe to replace/requeue. + """ + result = _normalize(state) + digest = usage_digest(usage) + placements = result["placements"] + existing = _mapping(placements.get(accounting_id)) + if existing: + already_delivered = bool(existing.get("emitted")) or delivered + same = ( + existing.get("destination") == destination + and existing.get("span_id") == span_id + and existing.get("usage_digest") == digest + ) + if same: + if delivered and not existing.get("emitted"): + existing = {**existing, "emitted": True, "last_error": None} + placements[accounting_id] = existing + return result, existing, True + if already_delivered or old_job_state in {"claimed", "emitted"}: + if delivered and not existing.get("emitted"): + # The candidate is rejected, but the fresh delivery read is + # still new information about the *existing* placement — + # record it so the ledger stops looking like it was only + # ever queued. + existing = {**existing, "emitted": True} + placements[accounting_id] = existing + reason = ( + "accounting placement or usage changed after confirmed delivery" + if already_delivered + else "accounting placement or usage changed while the prior job was in flight" + ) + result["conflicts"][accounting_id] = { + "accounting_id": accounting_id, + "reason": reason, + "existing": existing, + "candidate": { + "destination": destination, + "span_id": span_id, + "usage_digest": digest, + }, + } + return result, existing, False + # Nothing was ever confirmed delivered, and no job is provably in + # flight, for this identity: replace the queued/failed placement + # outright rather than quarantining it. + entry = { + "accounting_id": accounting_id, + "destination": destination, + "span_id": span_id, + "usage_digest": digest, + "emitted": False, + "last_error": None, + } + placements[accounting_id] = entry + return result, entry, True + entry = { + "accounting_id": accounting_id, + "destination": destination, + "span_id": span_id, + "usage_digest": digest, + "emitted": bool(delivered), + "last_error": None, + } + placements[accounting_id] = entry + return result, entry, True + + +def mark_placement_error(state: dict[str, Any], accounting_id: str, error: str) -> dict[str, Any]: + result = _normalize(state) + entry = _mapping(result["placements"].get(accounting_id)) + if entry: + entry["last_error"] = error + result["placements"][accounting_id] = entry + return result + + +def clear_placement_error(state: dict[str, Any], accounting_id: str) -> dict[str, Any]: + """Drop a stale ``last_error`` once a retried job successfully queues.""" + result = _normalize(state) + entry = _mapping(result["placements"].get(accounting_id)) + if entry and entry.get("last_error") is not None: + entry["last_error"] = None + result["placements"][accounting_id] = entry + return result + + +def mark_placement_job_status( + state: dict[str, Any], accounting_id: str, status: dict[str, Any] +) -> dict[str, Any]: + """Persist the worker lifecycle state and its latest delivery error.""" + result = _normalize(state) + entry = _mapping(result["placements"].get(accounting_id)) + if not entry: + return result + entry["job_state"] = status.get("state") + entry["job_attempt"] = status.get("attempt") + worker_error = status.get("last_error") + if isinstance(worker_error, str) and worker_error: + entry["last_error"] = worker_error + elif status.get("state") in {"queued", "claimed"}: + entry["last_error"] = None + if status.get("state") == "emitted": + entry["emitted"] = True + entry["last_error"] = None + result["placements"][accounting_id] = entry + return result + + +def mark_placement_delivered(state: dict[str, Any], accounting_id: str) -> dict[str, Any]: + """Record an externally confirmed delivery, never inferred from a job file. + + Ordinary reconciliation no longer needs to call this directly — + ``record_placement``'s ``delivered`` flag self-heals ``emitted`` from the + durable transport claim on every pass — but it remains available for a + caller with its own confirmed-delivery source. + """ + result = _normalize(state) + entry = _mapping(result["placements"].get(accounting_id)) + if entry: + entry["emitted"] = True + entry["last_error"] = None + result["placements"][accounting_id] = entry + return result + + +def mark_turn_error(state: dict[str, Any], turn_id: str, error: str) -> dict[str, Any]: + """Record that a whole-turn export job failed to queue locally. + + Kept separate from ``placements`` (keyed by accounting id, not turn id) + so a turn-job failure is visible without inventing a fake accounting + record for it. + """ + result = _normalize(state) + result["turn_errors"][turn_id] = error + return result + + +def clear_turn_error(state: dict[str, Any], turn_id: str) -> dict[str, Any]: + result = _normalize(state) + result["turn_errors"].pop(turn_id, None) + return result + + +def remove_export_state(config: Config, stored_session_id: str) -> None: + """Remove export state only for an explicit export-ledger reset. + + Projection rebuilds must not call this function. + """ + directory = _directory(config, stored_session_id) + with locked(export_lock_path(directory), LockMode.EXCLUSIVE): + fsops.unlink(export_state_path(directory), missing_ok=True) + fsops.sync_directory(directory) diff --git a/src/thirdeye/platforms/copilot/export_transport.py b/src/thirdeye/platforms/copilot/export_transport.py new file mode 100644 index 00000000..b7a53e9d --- /dev/null +++ b/src/thirdeye/platforms/copilot/export_transport.py @@ -0,0 +1,437 @@ +"""Copilot-owned transport for deterministic accounting exports. + +Copilot accounting can move as transcript and usage evidence is reconciled. +Its jobs therefore need an atomic cancel/claim protocol and observable retry +state that the generic fire-and-forget OTel transport does not promise. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import sys +import tempfile +import time +from pathlib import Path +from typing import Any + +from thirdeye import otel_export +from thirdeye._compat import fsops, proc +from thirdeye.config import Config +from thirdeye.usage.errlog import log_capture_error + +from .jsonio import atomic_write_json + +_CLAIM_STALE_S = 30.0 +_MAX_ATTEMPTS = 5 +_KINDS = frozenset({"session_accounting", "turn_accounting"}) + + +def jobs_dir(root: Path) -> Path: + return root / "logs" / "copilot-otel-jobs" + + +def job_path(root: Path, job_id: str) -> Path: + digest = hashlib.sha256(job_id.encode("utf-8")).hexdigest() + return jobs_dir(root) / f"accounting-{digest}.json" + + +def claim_path(path: Path) -> Path: + return path.with_suffix(f"{path.suffix}.claim") + + +def delivery_claim_path(session_dir: Path, accounting_id: str) -> Path: + digest = hashlib.sha256(accounting_id.encode("utf-8")).hexdigest() + return session_dir / "copilot-accounting-sent" / f"{digest}.json" + + +def delivery_sent(session_dir: Path, accounting_id: str) -> bool: + try: + return delivery_claim_path(session_dir, accounting_id).read_text(encoding="utf-8") == "sent" + except OSError: + return False + + +def _mark_delivered(session_dir: Path, accounting_id: str) -> None: + path = delivery_claim_path(session_dir, accounting_id) + if _atomic_create(path, "sent"): + fsops.sync_directory(path.parent) + + +def _atomic_create(path: Path, text: str) -> bool: + path.parent.mkdir(parents=True, exist_ok=True) + try: + descriptor = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY) + except FileExistsError: + return False + try: + os.write(descriptor, text.encode("utf-8")) + except BaseException: + fsops.unlink(path, missing_ok=True) + raise + finally: + os.close(descriptor) + return True + + +def _take_claim(path: Path, text: str, *, recover_stale: bool) -> bool: + if _atomic_create(path, text): + return True + if not recover_stale: + return False + try: + stale = time.time() - path.stat().st_mtime > _CLAIM_STALE_S + except OSError: + stale = True + if not stale: + return False + fsops.unlink(path, missing_ok=True) + return _atomic_create(path, text) + + +def _read_job(path: Path) -> dict[str, Any] | None: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (FileNotFoundError, json.JSONDecodeError): + return None + return value if isinstance(value, dict) else None + + +def _write_job(path: Path, payload: dict[str, Any]) -> None: + atomic_write_json(path, payload) + + +def _create_job(path: Path, payload: dict[str, Any]) -> bool: + """Publish a complete job atomically, preserving the first writer.""" + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary_name = tempfile.mkstemp( + dir=path.parent, prefix=f".{path.name}.", suffix=".tmp" + ) + temporary = Path(temporary_name) + try: + with os.fdopen(descriptor, "w", encoding="utf-8", newline="\n") as stream: + json.dump(payload, stream, default=str, sort_keys=True, separators=(",", ":")) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + try: + os.link(temporary, path) + except FileExistsError: + return False + fsops.sync_directory(path.parent) + return True + finally: + fsops.unlink(temporary, missing_ok=True) + + +def status(root: Path, job_id: str) -> dict[str, Any] | None: + path = job_path(root, job_id) + payload = _read_job(path) + if payload is None: + return None + state = "claimed" if claim_path(path).exists() else payload.get("state") + return { + "state": state, + "attempt": payload.get("attempt"), + "last_error": payload.get("last_error"), + } + + +def cancel(root: Path, job_id: str) -> dict[str, Any]: + """Cancel only after acquiring the same exclusive claim as the worker.""" + path = job_path(root, job_id) + owner = claim_path(path) + if not _atomic_create(owner, f"cancel:{os.getpid()}"): + return {**(status(root, job_id) or {}), "state": "claimed"} + try: + payload = _read_job(path) + if payload is None: + return {"state": "cancelled", "attempt": None, "last_error": None} + result = { + "state": payload.get("state"), + "attempt": payload.get("attempt"), + "last_error": payload.get("last_error"), + } + if result["state"] == "emitted": + return result + fsops.unlink(path, missing_ok=True) + return {**result, "state": "cancelled"} + finally: + fsops.unlink(owner, missing_ok=True) + + +def _spawn(path: Path) -> None: + proc.spawn_detached( + [sys.executable, "-m", "thirdeye.platforms.copilot.export_transport", str(path)] + ) + + +def queue_turn( + config: Config, + session_dir: Path, + session_id: str, + cwd: str, + turn: dict[str, Any], +) -> bool: + """Queue a Copilot turn while preserving the generic worker job shape.""" + if not config.logfire.enabled or not config.logfire.token: + return False + try: + path = otel_export._write_job( + config.root, + { + "kind": "turn", + "captured_attributes": otel_export._resolve_captured_attributes(config, None), + "session_dir": str(session_dir), + "session_id": session_id, + "platform": "copilot", + "cwd": cwd, + "turn": turn, + }, + ) + otel_export._spawn(path) + return True + except Exception as exc: + log_capture_error( + thirdeye_home=config.root, + phase="copilot_turn_export_spawn", + error=exc, + platform="copilot", + session_id=session_id, + ) + return False + + +def _placement_is_current(config: Config, payload: dict[str, Any]) -> bool: + from .export_state import load_export_state + + state = load_export_state(config, str(payload["session_id"])) + placement = (state.get("placements") or {}).get(str(payload["accounting_id"])) or {} + return placement.get("span_id") == payload.get("job_id") + + +def _queue(config: Config, payload: dict[str, Any]) -> bool: + if not config.logfire.enabled or not config.logfire.token: + return False + try: + if payload.get("kind") not in _KINDS: + raise ValueError(f"unsupported Copilot accounting job kind: {payload.get('kind')!r}") + path = job_path(config.root, str(payload["job_id"])) + queued = { + **payload, + "captured_attributes": otel_export._resolve_captured_attributes(config, None), + "state": "queued", + "attempt": 0, + } + owner = claim_path(path) + if not _take_claim(owner, f"queue:{os.getpid()}", recover_stale=True): + return True + dispatch = False + try: + if not _placement_is_current(config, payload): + return True + delivered = delivery_sent( + Path(str(payload["session_dir"])), str(payload["accounting_id"]) + ) + if not delivered: + _create_job(path, queued) + dispatch = True + finally: + fsops.unlink(owner, missing_ok=True) + if dispatch: + _spawn(path) + return True + except Exception as exc: + log_capture_error( + thirdeye_home=config.root, + phase="copilot_accounting_export_spawn", + error=exc, + platform="copilot", + session_id=str(payload.get("session_id") or ""), + ) + return False + + +def queue_session_accounting( + config: Config, + session_dir: Path, + session_id: str, + cwd: str, + accounting: dict[str, Any], +) -> bool: + accounting_id = str(accounting["accounting_id"]) + logical_span_id = f"accounting:{session_id}:{accounting_id}" + return _queue( + config, + { + "job_id": logical_span_id, + "kind": "session_accounting", + "session_dir": str(session_dir), + "session_id": session_id, + "cwd": cwd, + "accounting_id": accounting_id, + "usage": dict(accounting["usage"]), + "attribution_status": str(accounting["attribution_status"]), + "agent_id": accounting.get("agent_id"), + "attributes": dict(accounting.get("attributes") or {}), + }, + ) + + +def queue_turn_accounting( + config: Config, + session_dir: Path, + session_id: str, + cwd: str, + turn_id: str, + accounting: dict[str, Any], + *, + turn_span_id: str, +) -> bool: + accounting_id = str(accounting["accounting_id"]) + logical_span_id = f"accounting:{session_id}:{turn_id}:{accounting_id}" + return _queue( + config, + { + "job_id": logical_span_id, + "kind": "turn_accounting", + "session_dir": str(session_dir), + "session_id": session_id, + "cwd": cwd, + "turn_id": turn_id, + "turn_span_id": turn_span_id, + "accounting_id": accounting_id, + "usage": dict(accounting["usage"]), + "attribution_status": str(accounting["attribution_status"]), + "agent_id": accounting.get("agent_id"), + "attributes": dict(accounting.get("attributes") or {}), + }, + ) + + +def _acquire(path: Path) -> dict[str, Any] | None: + owner = claim_path(path) + if not _take_claim(owner, str(os.getpid()), recover_stale=True): + return None + + # Re-read only after owning the claim. A cancellation can happen after a + # worker's initial argv read but before this point; stale in-memory bytes + # must never resurrect the cancelled job. + payload = _read_job(path) + if payload is None or payload.get("state") in {"failed", "emitted"}: + fsops.unlink(owner, missing_ok=True) + return None + if delivery_sent(Path(str(payload["session_dir"])), str(payload["accounting_id"])): + fsops.unlink(path, missing_ok=True) + fsops.unlink(owner, missing_ok=True) + return None + claimed = {**payload, "state": "claimed"} + _write_job(path, claimed) + return claimed + + +def _accounting(payload: dict[str, Any]) -> dict[str, Any]: + return { + "accounting_id": payload["accounting_id"], + "usage": payload["usage"], + "attribution_status": payload["attribution_status"], + "agent_id": payload.get("agent_id"), + "attributes": payload.get("attributes") or {}, + "call_id": payload.get("call_id"), + } + + +def _deliver(config: Config, payload: dict[str, Any]) -> None: + token = otel_export._captured_attributes.set(payload.get("captured_attributes") or {}) + try: + common = { + "config": config, + "session_dir_": Path(payload["session_dir"]), + "session_id": payload["session_id"], + "platform": "copilot", + "cwd": payload["cwd"], + "accounting": _accounting(payload), + } + if payload["kind"] == "session_accounting": + otel_export._export_session_accounting_inner(**common) + elif payload["kind"] == "turn_accounting": + otel_export._export_turn_accounting_inner( + **common, + turn_id=str(payload["turn_id"]), + turn_span_id=payload.get("turn_span_id"), + ) + else: + raise ValueError(f"unsupported Copilot accounting job kind: {payload.get('kind')!r}") + finally: + otel_export._captured_attributes.reset(token) + + +def run(path: Path) -> None: + root = path.parents[2] if len(path.parents) > 2 else path.parent + try: + claimed = _acquire(path) + except Exception as exc: + log_capture_error( + thirdeye_home=root, + phase="copilot_accounting_claim_failed", + error=exc, + platform="copilot", + ) + return + if claimed is None: + return + owner = claim_path(path) + try: + _deliver(Config.load(), claimed) + except Exception as exc: + attempt = int(claimed.get("attempt", 0)) + 1 + retry = { + **claimed, + "state": "failed" if attempt >= _MAX_ATTEMPTS else "retrying", + "attempt": attempt, + "last_error": f"{type(exc).__name__}: {exc}", + } + try: + _write_job(path, retry) + except Exception as state_error: + log_capture_error( + thirdeye_home=root, + phase="copilot_accounting_state_failed", + error=state_error, + platform="copilot", + session_id=str(claimed.get("session_id") or ""), + ) + log_capture_error( + thirdeye_home=root, + phase="copilot_accounting_export_failed", + error=exc, + platform="copilot", + session_id=str(claimed.get("session_id") or ""), + message=f"accounting_id={claimed.get('accounting_id')} kind={claimed.get('kind')}", + ) + else: + try: + _mark_delivered(Path(str(claimed["session_dir"])), str(claimed["accounting_id"])) + _write_job(path, {**claimed, "state": "emitted", "last_error": None}) + fsops.unlink(path, missing_ok=True) + except Exception as exc: + log_capture_error( + thirdeye_home=root, + phase="copilot_accounting_ack_failed", + error=exc, + platform="copilot", + session_id=str(claimed.get("session_id") or ""), + ) + finally: + fsops.unlink(owner, missing_ok=True) + + +def main(argv: list[str] | None = None) -> None: + values = sys.argv[1:] if argv is None else argv + if values: + run(Path(values[0])) + + +if __name__ == "__main__": + main() diff --git a/src/thirdeye/platforms/copilot/followup.py b/src/thirdeye/platforms/copilot/followup.py new file mode 100644 index 00000000..303337d5 --- /dev/null +++ b/src/thirdeye/platforms/copilot/followup.py @@ -0,0 +1,270 @@ +"""Short-lived, coalesced follow-up capture for Copilot hook receipts. + +The hook process must return promptly. A hook therefore starts at most one +detached worker per source session; the worker makes a few bounded attempts to +pick up transcript or SQLite data which arrived just after the hook. +""" + +from __future__ import annotations + +import argparse +import contextlib +import json +import os +import sys +import tempfile +import time +from collections.abc import Iterator +from pathlib import Path +from typing import Any +from uuid import uuid4 + +from thirdeye._compat import fsops, proc +from thirdeye._compat.locking import LockMode, LockTimeout, locked +from thirdeye.config import Config +from thirdeye.paths import session_dir +from thirdeye.platforms.copilot.constants import FOLLOWUP_LEASE_FILENAME, PLATFORM_NAME +from thirdeye.platforms.copilot.identity import stored_session_id, validate_native_id +from thirdeye.platforms.copilot.types import SourcePaths, SyncResult +from thirdeye.usage.errlog import log_capture_error + +_PLATFORM = PLATFORM_NAME +_LEASE_LOCK_FILENAME = "copilot.followup.lock" +_LEASE_SECONDS = 5.0 +_LOCK_PROBE_TIMEOUT = 0.0 +_INITIAL_BACKOFF_SECONDS = 0.05 +_MAX_BACKOFF_SECONDS = 0.5 + + +def _directory(config: Config, paths: SourcePaths, native_id: str) -> Path: + return session_dir(config.root, _PLATFORM, stored_session_id(paths, native_id)) + + +def _lease_path(directory: Path) -> Path: + return directory / FOLLOWUP_LEASE_FILENAME + + +def _lease_lock_path(directory: Path) -> Path: + return directory / _LEASE_LOCK_FILENAME + + +def _now() -> float: + return time.time() + + +def _read_lease(path: Path) -> dict[str, Any] | None: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError): + return None + return value if isinstance(value, dict) else None + + +def _write_lease(path: Path, value: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + fd, name = tempfile.mkstemp(dir=path.parent, prefix=f"{path.name}.", suffix=".tmp") + try: + with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as stream: + json.dump(value, stream, separators=(",", ":"), sort_keys=True) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + fsops.replace(name, path) + fsops.sync_directory(path.parent) + except BaseException: + fsops.unlink(Path(name), missing_ok=True) + raise + + +def _claim_lease(config: Config, paths: SourcePaths, native_id: str) -> str | None: + """Return a new generation, or ``None`` when a live worker owns it.""" + + directory = _directory(config, paths, native_id) + try: + with locked(_lease_lock_path(directory), LockMode.EXCLUSIVE, timeout=_LOCK_PROBE_TIMEOUT): + current = _read_lease(_lease_path(directory)) + if current is not None and isinstance(current.get("expires_at"), (int, float)): + if float(current["expires_at"]) > _now(): + return None + generation = uuid4().hex + _write_lease( + _lease_path(directory), + {"generation": generation, "expires_at": _now() + _LEASE_SECONDS}, + ) + return generation + except (LockTimeout, OSError): + return None + + +def _owns_lease(directory: Path, generation: str) -> bool: + try: + with locked(_lease_lock_path(directory), LockMode.EXCLUSIVE, timeout=_LOCK_PROBE_TIMEOUT): + current = _read_lease(_lease_path(directory)) + return bool(current and current.get("generation") == generation) + except (LockTimeout, OSError): + return False + + +def _release_lease(directory: Path, generation: str) -> None: + try: + with locked(_lease_lock_path(directory), LockMode.EXCLUSIVE, timeout=_LOCK_PROBE_TIMEOUT): + current = _read_lease(_lease_path(directory)) + if current is not None and current.get("generation") == generation: + fsops.unlink(_lease_path(directory), missing_ok=True) + fsops.sync_directory(directory) + except (LockTimeout, OSError): + return + + +def schedule_followup(config: Config, paths: SourcePaths, native_id: str) -> bool: + """Coalesce hook follow-ups and spawn one detached, finite worker.""" + + validate_native_id(native_id) + generation = _claim_lease(config, paths, native_id) + if generation is None: + return False + try: + proc.spawn_detached( + [ + sys.executable, + "-m", + "thirdeye.platforms.copilot.followup", + "--source-home", + paths["home"], + "--session-id", + native_id, + "--config-root", + str(config.root), + "--generation", + generation, + ] + ) + except Exception as exc: + _release_lease(_directory(config, paths, native_id), generation) + log_capture_error( + thirdeye_home=config.root, + phase="copilot_followup_spawn", + error=exc, + platform=_PLATFORM, + session_id=native_id, + silent_fallback=True, + ) + return False + return True + + +def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument("--source-home", required=True) + parser.add_argument("--session-id", required=True) + parser.add_argument("--config-root", required=True) + parser.add_argument("--generation", required=True) + return parser.parse_args(argv) + + +@contextlib.contextmanager +def nonblocking_archive_lock() -> Iterator[None]: + """Make archive lock acquisition fail immediately when the lock is busy. + + ``commit_batch`` / ``load_cursor`` take the session archive lock with no + timeout. Hook and follow-up callers must not wait on that lock, so this + temporarily forces those acquisitions to use a zero timeout. + """ + + from thirdeye.platforms.copilot import archive + + @contextlib.contextmanager + def _locked(path: Path, mode: LockMode, *, timeout: float | None = None) -> Iterator[None]: + with locked(path, mode, timeout=_LOCK_PROBE_TIMEOUT): + yield + + original = archive.locked + archive.locked = _locked + try: + yield + finally: + archive.locked = original + + +def try_capture_session(config: Config, paths: SourcePaths, native_id: str) -> SyncResult | None: + """Attempt one capture without waiting on a busy archive lock. + + Returns ``None`` when another worker already holds the lock. + """ + + from thirdeye.platforms.copilot.capture import capture_session + + try: + with nonblocking_archive_lock(): + return capture_session(config, paths, native_id) + except LockTimeout: + return None + + +def _run(config: Config, paths: SourcePaths, native_id: str, generation: str) -> None: + """Try follow-up capture for no longer than the lease window.""" + + directory = _directory(config, paths, native_id) + deadline = time.monotonic() + _LEASE_SECONDS + delay = _INITIAL_BACKOFF_SECONDS + try: + while time.monotonic() < deadline and _owns_lease(directory, generation): + try: + result = try_capture_session(config, paths, native_id) + except Exception as exc: + log_capture_error( + thirdeye_home=config.root, + phase="copilot_followup_capture", + error=exc, + platform=_PLATFORM, + session_id=native_id, + silent_fallback=True, + ) + else: + if result is not None: + # Derived work is coalesced here, never in the hook + # process. A projection problem cannot affect capture. + try: + from thirdeye.platforms.copilot.runtime import reconcile_session + + reconcile_session(config, paths, native_id, export=True) + except Exception as exc: + log_capture_error( + thirdeye_home=config.root, + phase="copilot_followup_reconcile", + error=exc, + platform=_PLATFORM, + session_id=native_id, + silent_fallback=True, + ) + if result is not None and result["errors"] == 0 and result["pending"] == 0: + return + remaining = deadline - time.monotonic() + if remaining <= 0: + break + time.sleep(min(delay, remaining)) + delay = min(delay * 2, _MAX_BACKOFF_SECONDS) + finally: + _release_lease(directory, generation) + + +def main() -> None: + """Detached entrypoint. Its arguments contain paths and identity only.""" + + try: + args = _parse_args() + from thirdeye.platforms.copilot.identity import resolve_sources + + paths = resolve_sources(Path(args.source_home)) + native_id = str(args.session_id) + validate_native_id(native_id) + config = Config.load(root=Path(args.config_root)) + _run(config, paths, native_id, str(args.generation)) + except Exception: + # This worker is deliberately silent: diagnostics are kept locally and + # a failed follow-up never changes Copilot's hook outcome. + return + + +if __name__ == "__main__": + main() diff --git a/src/thirdeye/platforms/copilot/hook_payload.py b/src/thirdeye/platforms/copilot/hook_payload.py new file mode 100644 index 00000000..9a10df92 --- /dev/null +++ b/src/thirdeye/platforms/copilot/hook_payload.py @@ -0,0 +1,139 @@ +"""Normalize Copilot hook observations into SourceRecord envelopes. + +This module is input-only: it must not open files, spawn processes, or talk +to SQLite. Callers own durable spooling and later capture. +""" + +from __future__ import annotations + +from copy import deepcopy +from datetime import UTC, datetime +from typing import Any + +from thirdeye.platforms.copilot.constants import CLI_HOOK_EVENT_ALIASES, SOURCE_SCHEMA_VERSION +from thirdeye.platforms.copilot.identity import validate_native_id +from thirdeye.platforms.copilot.types import SourceRecord + +# Claude-style PascalCase names accepted as input aliases. Canonical stored +# event names remain Copilot CLI camelCase from CLI_HOOK_EVENT_ALIASES. +_PASCAL_CASE_ALIASES: dict[str, str] = { + "SessionStart": "sessionStart", + "UserPromptSubmit": "userPromptSubmitted", + "PreToolUse": "preToolUse", + "PostToolUse": "postToolUse", + "Stop": "agentStop", + "SubagentStart": "subagentStart", + "SubagentStop": "subagentStop", + "SessionEnd": "sessionEnd", +} + +# Allowlisted hook-time context. ``env`` is the captured environment mapping; +# the remaining keys are optional explicit trace identifiers. +_CONTEXT_KEYS = ("env", "trace_id", "span_id", "parent_span_id", "trace_context", "traceparent") + +# Epoch milliseconds are >= 1e12 for dates after 2001-09-09. +_MILLISECOND_THRESHOLD = 1_000_000_000_000 + + +def _canonical_event(event: str) -> str: + if event in CLI_HOOK_EVENT_ALIASES: + return event + aliased = _PASCAL_CASE_ALIASES.get(event) + if aliased is not None: + return aliased + return event + + +def _session_id(payload: dict[str, Any]) -> str: + session_id = payload.get("sessionId") + if not isinstance(session_id, str): + raise ValueError("hook payload is missing a string sessionId") + validate_native_id(session_id) + return session_id + + +def _valid_iso_datetime(value: str) -> bool: + """Return True when *value* is a parseable ISO-8601 datetime.""" + + iso = value[:-1] + "+00:00" if value.endswith("Z") else value + try: + datetime.fromisoformat(iso) + except ValueError: + return False + return True + + +def _source_ts(payload: dict[str, Any]) -> str | None: + """Return the original source timestamp when it is valid; never invent one.""" + + if "timestamp" not in payload: + return None + value: Any = payload["timestamp"] + if isinstance(value, bool) or value is None: + return None + if isinstance(value, str): + stripped = value.strip() + if not stripped: + return None + if _valid_iso_datetime(stripped): + return stripped + try: + value = float(stripped) + except ValueError: + return None + if not isinstance(value, (int, float)): + return None + seconds = value / 1000.0 if abs(value) >= _MILLISECOND_THRESHOLD else float(value) + try: + dt = datetime.fromtimestamp(seconds, tz=UTC) + except (OSError, OverflowError, ValueError): + return None + return dt.isoformat(timespec="milliseconds").replace("+00:00", "Z") + + +def _allowlisted_context(context: dict[str, Any]) -> dict[str, Any]: + stored: dict[str, Any] = {} + for key in _CONTEXT_KEYS: + if key not in context: + continue + value = context[key] + stored[key] = deepcopy(value) + return stored + + +def parse_hook( + event: str, + payload: dict[str, Any], + context: dict[str, Any], + *, + observed_at: str, + observation_id: str, +) -> SourceRecord: + """Build one hook SourceRecord from a normalized observation. + + The raw *payload* is retained unmodified. Session routing uses Copilot's + ``sessionId`` and rejects invalid native IDs. Child hooks that share a + parent session ID are recorded as-is; this function does not guess + ownership. + """ + + native_session_id = _session_id(payload) + canonical_event = _canonical_event(event) + retained_payload = deepcopy(payload) + return { + "source_id": f"hook/{native_session_id}/{observation_id}", + "source_kind": "hook", + "native_session_id": native_session_id, + "ts": _source_ts(payload), + "observed_at": observed_at, + "payload": { + "schema_version": SOURCE_SCHEMA_VERSION, + "event": canonical_event, + "hook_payload": retained_payload, + "context": _allowlisted_context(context), + }, + "locator": { + "observation_id": observation_id, + "event": canonical_event, + }, + } diff --git a/src/thirdeye/platforms/copilot/hooks.py b/src/thirdeye/platforms/copilot/hooks.py new file mode 100644 index 00000000..075bd4ae --- /dev/null +++ b/src/thirdeye/platforms/copilot/hooks.py @@ -0,0 +1,231 @@ +"""Fail-open runtime for the ``thirdeye-copilot-hook EVENT`` dispatcher.""" + +from __future__ import annotations + +import contextlib +import io +import json +import sys +from collections.abc import Iterator +from types import SimpleNamespace +from typing import Any +from uuid import uuid4 + +from thirdeye._compat.locking import LockTimeout +from thirdeye.config import Config +from thirdeye.env_capture import capture_env, env_to_tag +from thirdeye.index import IndexReader +from thirdeye.paths import index_path, session_dir +from thirdeye.platforms.provenance import foreign_payload_reason +from thirdeye.reader import SessionReader +from thirdeye.tags import TagStore +from thirdeye.usage.errlog import log_capture_error + +from .capture import record_hook +from .constants import CLI_HOOK_EVENT_ALIASES, PLATFORM_NAME +from .followup import nonblocking_archive_lock, schedule_followup +from .identity import resolve_sources, stored_session_id +from .types import SourcePaths + +# Accepted PascalCase aliases. Canonicalization is owned by parse_hook; +# this set is filter-only so the explicit dispatcher argument is passed through. +_PASCAL_CASE_ALIASES = frozenset( + { + "SessionStart", + "UserPromptSubmit", + "PreToolUse", + "PostToolUse", + "Stop", + "SubagentStart", + "SubagentStop", + "SessionEnd", + } +) +_TRACE_CONTEXT_KEYS = ("trace_id", "span_id", "parent_span_id", "trace_context", "traceparent") +# Capture writes spool hooks first, then at most one transcript/database page. +# Tagging must not walk older session history during the hook process. +_MAX_TAG_SCAN = 64 + + +def _read_stdin() -> dict[str, Any]: + try: + buffer = getattr(sys.stdin, "buffer", None) + raw = ( + io.TextIOWrapper(buffer, encoding="utf-8").read() + if buffer is not None + else sys.stdin.read() + ) + value = json.loads(raw) if raw else {} + except (OSError, ValueError, UnicodeError, json.JSONDecodeError, RecursionError): + return {} + return value if isinstance(value, dict) else {} + + +def _accepted_event(event: str) -> bool: + return event in CLI_HOOK_EVENT_ALIASES or event in _PASCAL_CASE_ALIASES + + +def _context(config: Config, payload: dict[str, Any]) -> dict[str, Any]: + context: dict[str, Any] = {"env": capture_env(config.capture_env_patterns)} + # Trace data is supplied explicitly by Copilot/the invoking integration; + # never infer it by harvesting arbitrary process environment variables. + for key in _TRACE_CONTEXT_KEYS: + if key in payload: + context[key] = payload[key] + return context + + +@contextlib.contextmanager +def _bound_observation_id(observation_id: str) -> Iterator[None]: + """Force ``record_hook`` to persist this observation id so tags can target it.""" + + from thirdeye.platforms.copilot import capture as capture_mod + + original = capture_mod.uuid4 + capture_mod.uuid4 = lambda: SimpleNamespace(hex=observation_id) + try: + yield + finally: + capture_mod.uuid4 = original + + +def _session_event_count(config: Config, paths: SourcePaths, native_id: str) -> int: + directory = session_dir(config.root, PLATFORM_NAME, stored_session_id(paths, native_id)) + try: + return IndexReader(index_path(directory)).count() + except OSError: + return 0 + + +def _tag_observation( + config: Config, + paths: SourcePaths, + native_id: str, + env: dict[str, str], + *, + observation_id: str, + since_seq: int | None = None, +) -> None: + """Attach opt-in environment tags to the observation just written. + + Capture commits spool hooks at the front of the batch, then transcript and + database pages. Hook-time tagging therefore starts at ``since_seq`` (the + pre-capture index count) and stops after a bounded window so session + history cannot grow hook latency. + """ + + tags = [tag for name, value in env.items() if (tag := env_to_tag(name, value)) is not None] + if not tags: + return + directory = session_dir(config.root, PLATFORM_NAME, stored_session_id(paths, native_id)) + try: + reader = SessionReader(directory) + count = IndexReader(index_path(directory)).count() + start = since_seq if since_seq is not None else max(0, count - _MAX_TAG_SCAN) + start = max(0, min(start, count)) + end = min(count, start + _MAX_TAG_SCAN) + for seq in range(start, end): + event = reader.get_event(seq) + if event.get("t") != "copilot_hook": + continue + data = event.get("data") + if not isinstance(data, dict): + continue + record = data.get("source_record") + if not isinstance(record, dict): + continue + locator = record.get("locator") + if not isinstance(locator, dict): + continue + if locator.get("observation_id") != observation_id: + continue + store = TagStore(directory) + for tag in tags: + store.add(int(event["seq"]), tag, source="auto") + return + except Exception: + return + + +def _log(config: Config, phase: str, exc: BaseException, native_id: str = "") -> None: + try: + log_capture_error( + thirdeye_home=config.root, + phase=phase, + error=exc, + platform=PLATFORM_NAME, + session_id=native_id, + silent_fallback=True, + ) + except Exception: + return + + +def _schedule(config: Config, paths: SourcePaths, native_id: str) -> None: + try: + schedule_followup(config, paths, native_id) + except Exception as exc: + _log(config, "copilot_followup_schedule", exc, native_id) + + +def main() -> None: + """Receive one passive Copilot hook invocation and always return silently.""" + + try: + # The installed dispatcher provides the event explicitly. Do not + # classify a payload based on casing: that would confuse Copilot and + # Cursor hook conventions. + event = sys.argv[1] if len(sys.argv) > 1 else "" + if not _accepted_event(event): + return + payload = _read_stdin() + if foreign_payload_reason(payload, expected=PLATFORM_NAME) is not None: + return + config = Config.load() + paths = resolve_sources() + context = _context(config, payload) + native_id = payload.get("sessionId") + observation_id = uuid4().hex + since_seq = 0 + if isinstance(native_id, str): + try: + since_seq = _session_event_count(config, paths, native_id) + except Exception: + since_seq = 0 + try: + with _bound_observation_id(observation_id), nonblocking_archive_lock(): + record_hook(config, paths, event, payload, context) + except LockTimeout: + # Spool is durable before capture. A later worker retries import. + if isinstance(native_id, str): + _schedule(config, paths, native_id) + return + except Exception as exc: + _log( + config, + "copilot_hook_capture", + exc, + native_id if isinstance(native_id, str) else "", + ) + if isinstance(native_id, str): + _schedule(config, paths, native_id) + return + + if not isinstance(native_id, str): + return + _tag_observation( + config, + paths, + native_id, + context["env"], + observation_id=observation_id, + since_seq=since_seq, + ) + _schedule(config, paths, native_id) + except Exception: + # Hooks must never decide a Copilot permission or emit protocol output. + return + + +if __name__ == "__main__": + main() diff --git a/src/thirdeye/platforms/copilot/identity.py b/src/thirdeye/platforms/copilot/identity.py new file mode 100644 index 00000000..6d810098 --- /dev/null +++ b/src/thirdeye/platforms/copilot/identity.py @@ -0,0 +1,134 @@ +"""Pure source-home and native-session identity helpers.""" + +from __future__ import annotations + +import hashlib +import os +from pathlib import Path + +from .constants import COPILOT_HOME_ENV +from .types import SourcePaths + +# Stored session IDs keep the first 16 hex characters of the SHA-256 source key +# for path readability. The full 64-character digest remains the identity. +SOURCE_KEY_PREFIX_LEN = 16 +SOURCE_KEY_DIGEST_LEN = 64 + + +def _canonical_path(path: Path) -> Path: + """Return a stable, absolute path without requiring the path to exist.""" + + return path.expanduser().resolve(strict=False) + + +def _normalized_path(path: Path) -> str: + """Normalize the canonical path for source-home identity on this platform.""" + + return os.path.normcase(os.fspath(_canonical_path(path))) + + +def _source_key(home: Path) -> str: + return hashlib.sha256(_normalized_path(home).encode("utf-8")).hexdigest() + + +def _is_within(path: Path, root: Path) -> bool: + try: + path.relative_to(root) + except ValueError: + return False + return True + + +def _validate_paths(paths: SourcePaths) -> Path: + """Reject forged identities or recording paths outside their source home.""" + + home = _canonical_path(Path(paths["home"])) + if paths["source_key"] != _source_key(home): + raise ValueError("source_key does not match the canonical source home") + + for field in ("session_root", "database"): + candidate = _canonical_path(Path(paths[field])) + if not _is_within(candidate, home): + raise ValueError(f"{field} escapes the selected Copilot home") + return home + + +def resolve_sources(source_home: Path | None = None) -> SourcePaths: + """Resolve one Copilot home at invocation time. + + Explicit input wins over ``COPILOT_HOME``; the default is + ``Path.home() / '.copilot'``. The returned paths are canonical strings so + aliases resolving to the same home share a source identity. + """ + + requested = source_home + if requested is None: + configured = os.environ.get(COPILOT_HOME_ENV) + requested = Path(configured) if configured else Path.home() / ".copilot" + + home = _canonical_path(requested) + paths: SourcePaths = { + "home": os.fspath(home), + "source_key": _source_key(home), + "session_root": os.fspath(home / "session-state"), + "database": os.fspath(home / "session-store.db"), + } + _validate_paths(paths) + return paths + + +def _validate_path_id(value: str, *, label: str) -> None: + if not isinstance(value, str) or not value or value.strip() != value: + raise ValueError(f"{label} must be a non-empty, trimmed string") + if value in {".", ".."}: + raise ValueError(f"{label} must not be a traversal segment") + if any(character in value for character in ("/", "\\", "\x00", ":")): + raise ValueError(f"{label} contains a path separator or invalid path character") + if any(ord(character) < 32 for character in value): + raise ValueError(f"{label} contains a control character") + + +def validate_native_id(native_id: str) -> None: + """Ensure a native session ID can never select a path outside its home.""" + + _validate_path_id(native_id, label="native session ID") + + +def validate_stored_session_id(stored_id: str) -> None: + """Ensure a stored session ID cannot be used as a path-escape segment.""" + + _validate_path_id(stored_id, label="stored session ID") + if not stored_id.startswith("copilot-"): + raise ValueError("stored session ID must start with 'copilot-'") + + +def stored_session_id(paths: SourcePaths, native_id: str) -> str: + """Return the stable thirdeye ID for a native ID within one source home. + + The stored ID uses only :data:`SOURCE_KEY_PREFIX_LEN` hex characters of + the source key. This function checks that *one* ``SourcePaths`` value is + internally consistent: the full digest matches the canonical home, and + recording paths stay inside that home. That check does not detect two + legitimate homes whose SHA-256 digests share a prefix. Both would still + produce the same stored ID. On reuse the archive must compare the full + ``source_key`` retained in session metadata with the candidate home and + refuse to merge a prefix collision. + """ + + _validate_paths(paths) + validate_native_id(native_id) + return f"copilot-{paths['source_key'][:SOURCE_KEY_PREFIX_LEN]}-{native_id}" + + +def source_keys_share_stored_prefix(left: str, right: str) -> bool: + """Return True when two distinct full source keys would collide in stored IDs. + + Archive reuse must apply this comparison (or an equivalent full-key check + against retained metadata). :func:`stored_session_id` cannot: it never + sees the other home. + """ + + for key in (left, right): + if not isinstance(key, str) or len(key) != SOURCE_KEY_DIGEST_LEN: + raise ValueError("source_key must be a 64-character SHA-256 hex digest") + return left != right and left[:SOURCE_KEY_PREFIX_LEN] == right[:SOURCE_KEY_PREFIX_LEN] diff --git a/src/thirdeye/platforms/copilot/install.py b/src/thirdeye/platforms/copilot/install.py new file mode 100644 index 00000000..cecefcad --- /dev/null +++ b/src/thirdeye/platforms/copilot/install.py @@ -0,0 +1,286 @@ +"""Installation of thirdeye's passive GitHub Copilot CLI hooks. + +Copilot reads version-1 hook documents from ``$COPILOT_HOME/hooks``. This +module owns only ``thirdeye.json`` and only command entries whose executable +is thirdeye's dispatcher; it never changes Copilot-wide settings or hooks +owned by another integration. +""" + +from __future__ import annotations + +import json +import re +import shlex +import shutil +from pathlib import Path, PureWindowsPath +from typing import Any + +import click + +import thirdeye._compat as _compat +from thirdeye.platforms.base import Platform, command_basename, resolve_command + +from .constants import ( + CLI_HOOK_EVENTS, + DISPLAY_NAME, + HOOK_BIN_NAME, + HOOK_CONFIG_VERSION, + HOOK_TIMEOUT_S, + HOOKS_DIRECTORY_NAME, + OWNED_HOOK_FILENAME, + PLATFORM_NAME, +) +from .identity import resolve_sources + + +def _quote_bash(value: str) -> str: + """Quote one literal argument for Copilot's POSIX shell hook runner.""" + + return shlex.quote(value) + + +def _quote_powershell(value: str) -> str: + """Quote one literal argument for PowerShell's single-quote syntax.""" + + return "'" + value.replace("'", "''") + "'" + + +def _bash_command(entrypoint: str, event: str) -> str: + return f"{_quote_bash(entrypoint)} {_quote_bash(event)}" + + +def _powershell_command(entrypoint: str, event: str) -> str: + # ``&`` is required for a quoted executable path to be invoked rather + # than treated as a string expression. + return f"& {_quote_powershell(entrypoint)} {_quote_powershell(event)}" + + +def _command_executable(command: object, shell: str) -> str | None: + """Return the executable in one of our generated commands, if parseable.""" + + if not isinstance(command, str) or not command.strip(): + return None + if shell == "bash": + try: + parts = shlex.split(command, posix=True) + except ValueError: + return None + return parts[0] if parts else None + + # We generate ``& 'path' 'event'``. Accept both quote styles for a + # previous installation, but intentionally do not try to interpret an + # arbitrary PowerShell program: configuration ownership must be narrow. + match = re.match(r"^\s*&\s+(?:'((?:[^']|'')*)'|\"([^\"]*)\")", command) + if not match: + return None + quoted = match.group(1) if match.group(1) is not None else match.group(2) + return quoted.replace("''", "'") + + +def _is_our_command(command: object, shell: str) -> bool: + executable = _command_executable(command, shell) + if executable is None: + return False + # command_basename follows the host path rules. Also recognize a Windows + # executable when inspecting a fixture on a non-Windows host. + if command_basename(executable) == HOOK_BIN_NAME: + return True + name = PureWindowsPath(executable).name + return name.lower() in {HOOK_BIN_NAME.lower(), f"{HOOK_BIN_NAME}.exe".lower()} + + +def _entry_is_ours(entry: object) -> bool: + return isinstance(entry, dict) and any( + _is_our_command(entry.get(shell), shell) for shell in ("bash", "powershell") + ) + + +def _load_document(path: Path) -> dict[str, Any]: + """Load a valid version-1 Copilot hook document without normalizing it. + + Invalid documents are configuration errors, not empty documents: replacing + them would destroy an operator's hooks and conceal a broken setup. + """ + + if not path.exists(): + return {"version": HOOK_CONFIG_VERSION, "hooks": {}} + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise click.ClickException( + f"Cannot update Copilot hooks at {path}: the file is not valid JSON. " + "Fix or move it, then run the command again; it was left unchanged." + ) from exc + if not isinstance(data, dict): + raise click.ClickException( + f"Cannot update Copilot hooks at {path}: the document must be a JSON object. " + "It was left unchanged." + ) + if data.get("version") != HOOK_CONFIG_VERSION: + raise click.ClickException( + f"Cannot update Copilot hooks at {path}: expected version " + f"{HOOK_CONFIG_VERSION}, found {data.get('version')!r}. It was left unchanged." + ) + hooks = data.get("hooks") + if not isinstance(hooks, dict): + raise click.ClickException( + f"Cannot update Copilot hooks at {path}: 'hooks' must be an object. " + "It was left unchanged." + ) + for event, value in hooks.items(): + if not isinstance(value, list): + raise click.ClickException( + f"Cannot update Copilot hooks at {path}: hooks.{event} must be a list. " + "It was left unchanged." + ) + return data + + +def _save_document(path: Path, data: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8", newline="\n") + + +class CopilotPlatform(Platform): + """Passive Copilot CLI hook installation scoped to one source home.""" + + name = PLATFORM_NAME + display_name = DISPLAY_NAME + + def __init__( + self, + source_home: Path | None = None, + hooks_file: Path | None = None, + entrypoint: str | Path | None = None, + *, + windows: bool | None = None, + ) -> None: + self._source_home = source_home + self._hooks_file = hooks_file + self._entrypoint = str(entrypoint) if entrypoint is not None else None + self._windows = _compat.IS_WINDOWS if windows is None else windows + + @property + def hooks_file(self) -> Path: + if self._hooks_file is not None: + return self._hooks_file + paths = resolve_sources(self._source_home) + return Path(paths["home"]) / HOOKS_DIRECTORY_NAME / OWNED_HOOK_FILENAME + + @property + def entrypoint(self) -> str: + return self._entrypoint or resolve_command(HOOK_BIN_NAME) + + def _install_entrypoint(self) -> str: + """Return the dispatcher path that may be written into hook commands. + + An explicitly injected path is trusted so tests can supply a fixture + without installing the later runtime binary. Live installation + requires the dispatcher to be resolvable on PATH; Copilot itself is + not required. + """ + + if self._entrypoint: + return self._entrypoint + if shutil.which(HOOK_BIN_NAME) is None: + raise click.ClickException( + f"Cannot install Copilot hooks: {HOOK_BIN_NAME} is not on PATH. " + "Install thirdeye so the dispatcher entrypoint is available, then retry. " + "Copilot itself does not need to be on PATH." + ) + return resolve_command(HOOK_BIN_NAME) + + def _command(self, event: str, entrypoint: str | None = None) -> tuple[str, str]: + resolved = entrypoint if entrypoint is not None else self.entrypoint + if self._windows: + return "powershell", _powershell_command(resolved, event) + return "bash", _bash_command(resolved, event) + + def _report_install(self) -> None: + click.echo(f"Configured Copilot CLI hooks at {self.hooks_file}") + if self._hooks_file is None or self._source_home is not None: + home = resolve_sources(self._source_home)["home"] + click.echo(f"Scope: Copilot home {home}") + click.echo( + "Restart Copilot CLI or start a new interactive session so the hooks take effect." + ) + click.echo("Verify receipt with: thirdeye copilot status") + click.echo( + "If hooks are missing or not firing, ingest persisted recordings with: " + "thirdeye copilot watch" + ) + + def install(self) -> None: + entrypoint = self._install_entrypoint() + path = self.hooks_file + data = _load_document(path) + hooks = data["hooks"] + changed = False + for event in CLI_HOOK_EVENTS: + entries = hooks.get(event, []) + # The event type has already been validated by _load_document. + if not isinstance(entries, list): # Defensive for typed JSON input. + raise AssertionError(f"validated hook list changed shape for {event}") + retained = [entry for entry in entries if not _entry_is_ours(entry)] + shell, command = self._command(event, entrypoint) + desired = { + "type": "command", + shell: command, + "timeoutSec": HOOK_TIMEOUT_S, + } + if retained != entries or desired not in retained: + hooks[event] = [*retained, desired] + changed = True + if changed or not path.exists(): + _save_document(path, data) + self._report_install() + + def is_installed(self) -> bool: + path = self.hooks_file + try: + data = _load_document(path) + except click.ClickException: + return False + hooks = data["hooks"] + for event in CLI_HOOK_EVENTS: + entries = hooks.get(event) + if not isinstance(entries, list): + return False + shell, command = self._command(event) + if not any( + isinstance(entry, dict) + and entry.get("type") == "command" + and entry.get(shell) == command + and entry.get("timeoutSec") == HOOK_TIMEOUT_S + for entry in entries + ): + return False + return True + + def uninstall(self) -> None: + path = self.hooks_file + if not path.exists(): + return + data = _load_document(path) + hooks = data["hooks"] + changed = False + for event in list(hooks): + entries = hooks[event] + if not isinstance(entries, list): + # _load_document already rejected this shape; keep the guard so + # a concurrent rewrite cannot become a destructive repair. + continue + retained = [entry for entry in entries if not _entry_is_ours(entry)] + if retained == entries: + continue + changed = True + if retained: + hooks[event] = retained + else: + del hooks[event] + # Version and hooks are the only fields we create. An otherwise empty + # owned document can disappear; extra fields always remain untouched. + if not hooks and set(data) == {"version", "hooks"}: + path.unlink() + elif changed: + _save_document(path, data) diff --git a/src/thirdeye/platforms/copilot/jsonio.py b/src/thirdeye/platforms/copilot/jsonio.py new file mode 100644 index 00000000..1558840b --- /dev/null +++ b/src/thirdeye/platforms/copilot/jsonio.py @@ -0,0 +1,61 @@ +"""Atomic JSON publication shared by Copilot archive and projection state.""" + +from __future__ import annotations + +import json +import os +import tempfile +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from thirdeye._compat import fsops + + +def atomic_write_json( + path: Path, + value: dict[str, Any], + *, + on_replaced: Callable[[], None] | None = None, + on_synced: Callable[[], None] | None = None, +) -> None: + """Replace ``path`` with canonical JSON, fsyncing the file and directory. + + ``on_replaced`` runs after the durable rename and before the directory + sync so callers can inject crash boundaries at the same points as before. + """ + path.parent.mkdir(parents=True, exist_ok=True) + fd, temp_name = tempfile.mkstemp(dir=path.parent, prefix=f"{path.name}.", suffix=".tmp") + try: + with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as stream: + json.dump(value, stream, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + fsops.replace(temp_name, path) + if on_replaced is not None: + on_replaced() + fsops.sync_directory(path.parent) + if on_synced is not None: + on_synced() + except BaseException: + fsops.unlink(Path(temp_name), missing_ok=True) + raise + + +def read_json_object(path: Path, *, invalid_message: str) -> dict[str, Any] | None: + """Read a JSON object, distinguishing absence, corruption, and I/O errors. + + ``FileNotFoundError`` returns ``None``. Malformed JSON or a non-object + becomes ``ValueError``. Transient ``OSError`` (sharing violations, EIO) + propagates unchanged so callers do not treat a busy file as corruption. + """ + try: + raw = json.loads(fsops.read_text(path, encoding="utf-8")) + except FileNotFoundError: + return None + except json.JSONDecodeError: + raise ValueError(invalid_message) from None + if not isinstance(raw, dict): + raise ValueError(invalid_message) + return raw diff --git a/src/thirdeye/platforms/copilot/projection.py b/src/thirdeye/platforms/copilot/projection.py new file mode 100644 index 00000000..f90cc371 --- /dev/null +++ b/src/thirdeye/platforms/copilot/projection.py @@ -0,0 +1,142 @@ +"""Pure composition of Copilot semantic, accounting, and attribution views.""" + +from __future__ import annotations + +from copy import deepcopy +from typing import Any + +from .attribution import join_usage +from .tracing import build_semantics +from .types import AccountingCandidate, Attribution, Projection, ProjectionDiagnostic, SourceRecord +from .usage import build_accounting + + +def _unwrap_semantic_state(prior_state: dict[str, Any]) -> dict[str, Any]: + """Accept flat tracing state or a nested ProjectionState envelope.""" + + if isinstance(prior_state.get("open_interactions"), dict): + return prior_state + nested = prior_state.get("semantic_state") + if isinstance(nested, dict): + return nested + return {} + + +def _find_turn(turns: list[dict[str, Any]], identity: str) -> dict[str, Any] | None: + for turn in turns: + if turn.get("turn_id") == identity: + return turn + if any(call.get("call_id") == identity for call in turn.get("llm_calls") or []): + return turn + found = _find_turn(turn.get("subagents") or [], identity) + if found is not None: + return found + return None + + +def _copilot_supplemental_export_attributes( + candidate: AccountingCandidate | None, +) -> dict[str, Any]: + """Copy native Copilot billing/latency onto export attributes, never as USD.""" + + metrics = (candidate or {}).get("supplemental_metrics") or {} + attributes: dict[str, Any] = {} + if "total_nano_aiu" in metrics: + attributes["copilot.billing.nano_aiu"] = metrics["total_nano_aiu"] + if "duration_ms" in metrics: + attributes["copilot.latency.duration_ms"] = metrics["duration_ms"] + if "output_ttft_ms" in metrics: + attributes["copilot.latency.output_ttft_ms"] = metrics["output_ttft_ms"] + if "time_to_first_token_ms" in metrics: + attributes["copilot.latency.time_to_first_token_ms"] = metrics["time_to_first_token_ms"] + if "inter_token_latency_ms" in metrics: + attributes["copilot.latency.inter_token_latency_ms"] = metrics["inter_token_latency_ms"] + return attributes + + +def _accounting_call( + attribution: Attribution, row: object, candidate: AccountingCandidate | None +) -> dict[str, Any] | None: + if row is None or not hasattr(row, "to_dict"): + return None + return { + "accounting_id": attribution["logical_call_id"], + "usage": row.to_dict(), + "attribution_status": attribution["status"], + "agent_id": attribution["agent_id"], + "call_id": attribution["call_id"] if attribution["status"] == "matched" else None, + "attributes": { + "logical_call_id": attribution["logical_call_id"], + "usage_source_id": attribution["usage_source_id"], + "join_kind": attribution["join_kind"], + "evidence": list(attribution["evidence"]), + **_copilot_supplemental_export_attributes(candidate), + }, + } + + +def build_projection( + records: list[SourceRecord], prior_state: dict[str, Any] +) -> tuple[Projection, dict[str, Any]]: + """Build a local projection without mutating either source projection.""" + + semantic, semantic_state = build_semantics(records, _unwrap_semantic_state(prior_state)) + accounting, accounting_state = build_accounting(records, prior_state) + attributions = join_usage(semantic, accounting) + turns = deepcopy(semantic["turns"]) + rows = {row.call_id: row for row in accounting["usage_rows"]} + candidates = {item["logical_call_id"]: item for item in accounting["candidates"]} + pending = [*semantic["pending"]] + diagnostics: list[ProjectionDiagnostic] = [*semantic["diagnostics"], *accounting["diagnostics"]] + + for attribution in attributions: + row = rows.get(attribution["logical_call_id"]) + accounting_call = _accounting_call( + attribution, row, candidates.get(attribution["logical_call_id"]) + ) + if attribution["status"] == "matched": + if attribution["join_kind"] == "inferred": + diagnostics.append( + { + "code": "inferred_join", + "severity": "info", + "message": "usage joined by uniquely consistent evidence", + "source_ids": [attribution["usage_source_id"]], + "details": { + "logical_call_id": attribution["logical_call_id"], + "call_id": attribution["call_id"], + }, + } + ) + else: + pending.append( + { + "id": f"pending:usage:{attribution['logical_call_id']}", + "kind": "unmatched_usage", + "reason": f"usage attribution is {attribution['status']}", + "source_ids": [attribution["usage_source_id"]], + "evidence": list(attribution["evidence"]), + } + ) + if accounting_call is None: + continue + owner = None + if attribution["status"] == "matched" and attribution["call_id"] is not None: + owner = _find_turn(turns, attribution["call_id"]) + elif attribution["stored_turn_id"] is not None: + owner = _find_turn(turns, attribution["stored_turn_id"]) + if owner is not None: + owner.setdefault("accounting_calls", []).append(accounting_call) + + return ( + { + "normalized_events": semantic["events"], + "turns": turns, + "usage_rows": accounting["usage_rows"], + "attributions": attributions, + "pending": pending, + "diagnostics": diagnostics, + "accounting_candidates": accounting["candidates"], + }, + {"semantic_state": semantic_state, "accounting_state": accounting_state}, + ) diff --git a/src/thirdeye/platforms/copilot/projection_state.py b/src/thirdeye/platforms/copilot/projection_state.py new file mode 100644 index 00000000..78c61403 --- /dev/null +++ b/src/thirdeye/platforms/copilot/projection_state.py @@ -0,0 +1,173 @@ +"""Recoverable, versioned storage for Copilot's derived V2 projection. + +This state deliberately lives beside, rather than inside, the V1 capture +checkpoint. The capture archive is immutable evidence; deleting this file is +therefore a safe way to request a local projection rebuild. It must never +touch the export ledger (which has its own lock and lifecycle). +""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from thirdeye._compat import fsops + +from .jsonio import atomic_write_json, read_json_object +from .types import PROJECTION_SCHEMA_VERSION + +PROJECTION_STATE_FILENAME = "copilot.projection.state.json" +PROJECTION_JOURNAL_FILENAME = "copilot.projection.journal.json" +PROJECTION_LOCK_FILENAME = "copilot.projection.lock" +PROJECTION_JOURNAL_SCHEMA_VERSION = 1 + +# Kept private, just like the V1 archive fault hook. It gives durability tests +# precise crash boundaries without making fault injection a runtime feature. +_fault_injector: Callable[[str], None] | None = None + + +def _fault(point: str) -> None: + if _fault_injector is not None: + _fault_injector(point) + + +def projection_state_path(session_dir: Path) -> Path: + return session_dir / PROJECTION_STATE_FILENAME + + +def projection_journal_path(session_dir: Path) -> Path: + return session_dir / PROJECTION_JOURNAL_FILENAME + + +def projection_lock_path(session_dir: Path) -> Path: + return session_dir / PROJECTION_LOCK_FILENAME + + +def empty_projection_state() -> dict[str, Any]: + """Return the public state shape used as pure-builder input.""" + return { + "projection_schema_version": PROJECTION_SCHEMA_VERSION, + "archive_source_ids": [], + "semantic_state": {"open_interactions": {}}, + "accounting_state": {"logical_calls": {}}, + "projection_revision": "", + "commit_sequence": 0, + } + + +def empty_projection_document() -> dict[str, Any]: + """Return the private envelope around public state and replay indexes.""" + return { + "schema_version": PROJECTION_SCHEMA_VERSION, + "state": empty_projection_state(), + "indexes": { + "events": {}, + "turns": {}, + "usage": {}, + "usage_identities": {}, + "attributions": {}, + "pending": {}, + "diagnostics": {}, + }, + } + + +def _atomic_json(path: Path, value: dict[str, Any], *, fault_point: str) -> None: + atomic_write_json(path, value, on_synced=lambda: _fault(fault_point)) + + +def _read_json(path: Path) -> dict[str, Any] | None: + return read_json_object(path, invalid_message=f"invalid Copilot projection state: {path}") + + +def _schema_current(document: dict[str, Any]) -> bool: + if document.get("schema_version") != PROJECTION_SCHEMA_VERSION: + return False + state = document.get("state") + return isinstance(state, dict) and state.get("projection_schema_version") == ( + PROJECTION_SCHEMA_VERSION + ) + + +def _validate_current_document(document: dict[str, Any]) -> dict[str, Any]: + indexes = document.get("indexes") + if not isinstance(document.get("state"), dict) or not isinstance(indexes, dict): + raise ValueError("invalid Copilot projection state") + for name in ("events", "turns", "usage", "attributions", "pending", "diagnostics"): + if not isinstance(indexes.get(name), dict): + raise ValueError("invalid Copilot projection indexes") + if not isinstance(indexes.get("usage_identities", {}), dict): + raise ValueError("invalid Copilot projection indexes") + indexes.setdefault("usage_identities", {}) + return document + + +def read_projection_document(session_dir: Path) -> dict[str, Any]: + """Read the durable document, returning an empty one before first use. + + Callers holding :func:`projection_lock_path` may rely on this to finish a + previously published journal before reading. Keeping recovery here makes + it impossible for a later commit to merge against an incomplete snapshot. + + A stale schema version is disposable derived state: the files are removed + and the caller rebuilds from the immutable V1 archive. + """ + journal_path = projection_journal_path(session_dir) + try: + journal = _read_json(journal_path) + except ValueError: + # Derived journals are disposable: corrupt JSON is treated like a stale + # schema so recovery can fall through to the last snapshot. + journal = {} + if journal is not None: + document = journal.get("document") + journal_ok = journal.get("schema_version") == PROJECTION_JOURNAL_SCHEMA_VERSION + if journal_ok and isinstance(document, dict) and _schema_current(document): + _validate_current_document(document) + _atomic_json( + projection_state_path(session_dir), + document, + fault_point="after_projection_recovery", + ) + fsops.unlink(journal_path, missing_ok=True) + fsops.sync_directory(session_dir) + _fault("after_projection_recovery_clear") + return document + # Stale or unreadable journal: drop it and fall through to the snapshot. + fsops.unlink(journal_path, missing_ok=True) + fsops.sync_directory(session_dir) + + document = _read_json(projection_state_path(session_dir)) + if document is None: + return empty_projection_document() + if not _schema_current(document): + remove_projection_state(session_dir) + return empty_projection_document() + return _validate_current_document(document) + + +def publish_projection_document(session_dir: Path, document: dict[str, Any]) -> None: + """Publish ``document`` with write-ahead journaling. + + The caller owns the projection lock. If interrupted after either replace, + the next reader replays the complete immutable document from the journal. + """ + if not _schema_current(document): + raise ValueError("unsupported Copilot projection state schema") + _validate_current_document(document) + journal = {"schema_version": PROJECTION_JOURNAL_SCHEMA_VERSION, "document": document} + _atomic_json( + projection_journal_path(session_dir), journal, fault_point="after_projection_journal" + ) + _atomic_json(projection_state_path(session_dir), document, fault_point="after_projection_state") + fsops.unlink(projection_journal_path(session_dir), missing_ok=True) + fsops.sync_directory(session_dir) + _fault("after_projection_journal_clear") + + +def remove_projection_state(session_dir: Path) -> None: + """Remove only rebuildable local projection files, never V1 or export data.""" + fsops.unlink(projection_state_path(session_dir), missing_ok=True) + fsops.unlink(projection_journal_path(session_dir), missing_ok=True) + fsops.sync_directory(session_dir) diff --git a/src/thirdeye/platforms/copilot/projection_store.py b/src/thirdeye/platforms/copilot/projection_store.py new file mode 100644 index 00000000..8f77baa9 --- /dev/null +++ b/src/thirdeye/platforms/copilot/projection_store.py @@ -0,0 +1,659 @@ +"""Durable local storage for completed Copilot V2 projections. + +This module is intentionally a persistence boundary. It accepts already +constructed :class:`Projection` DTOs and archived Store events; it does not +read Copilot's live transcript/database files and does not perform semantic, +accounting, attribution, or export work. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import tempfile +from collections.abc import Iterable +from pathlib import Path +from typing import Any + +from thirdeye._compat import fsops +from thirdeye._compat.locking import LockMode, locked +from thirdeye.config import Config +from thirdeye.meta import read_meta +from thirdeye.paths import meta_path, session_dir, usage_jsonl_path +from thirdeye.reader import SessionReader +from thirdeye.usage.index import UsageIndex +from thirdeye.usage.types import UsageRow + +from .constants import PLATFORM_NAME, SOURCE_SCHEMA_VERSION +from .projection_state import ( + empty_projection_state, + projection_lock_path, + publish_projection_document, + read_projection_document, + remove_projection_state, +) +from .types import PROJECTION_SCHEMA_VERSION, Projection + +_RAW_EVENT_TYPES = frozenset( + {"copilot_transcript", "copilot_database", "copilot_hook", "copilot_metadata"} +) +_INDEX_NAMES = ("events", "turns", "usage", "attributions", "pending", "diagnostics") + + +class ProjectionConflictError(RuntimeError): + """A commit observed a newer ``commit_sequence`` than it was based on. + + Raised instead of silently overwriting: a second writer already + committed a projection built from more current builder state, so this + (older) attempt is discarded rather than winning a last-write-races + against the newer one. The caller should reload projection state and + retry. + """ + + +def _directory(config: Config, stored_session_id: str) -> Path: + return session_dir(config.root, PLATFORM_NAME, stored_session_id) + + +def _canonical(value: Any) -> str: + return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), default=str) + + +def _digest(value: Any) -> str: + return "sha256:" + hashlib.sha256(_canonical(value).encode("utf-8")).hexdigest() + + +def _mapping(value: object) -> dict[str, Any]: + return value if isinstance(value, dict) else {} + + +def _items(value: object) -> list[Any]: + return value if isinstance(value, list) else [] + + +def _index_key(item: dict[str, Any], field: str, *, prefix: str) -> str: + value = item.get(field) + return value if isinstance(value, str) and value else f"{prefix}:{_digest(item)}" + + +def _require_session(directory: Path, stored_session_id: str) -> None: + if read_meta(meta_path(directory)) is None: + raise ValueError(f"unknown Copilot session: {stored_session_id}") + + +def _existing_session_dir(config: Config, stored_session_id: str) -> Path | None: + """Return the session directory only when V1 meta is already on disk. + + ``locked()`` creates parent directories, so unknown ids must be rejected + before acquiring the projection lock. + """ + directory = _directory(config, stored_session_id) + if read_meta(meta_path(directory)) is None: + return None + return directory + + +def _source_id(event: dict[str, Any]) -> str | None: + if event.get("t") not in _RAW_EVENT_TYPES: + return None + data = _mapping(event.get("data")) + if data.get("schema_version") != SOURCE_SCHEMA_VERSION: + return None + record = _mapping(data.get("source_record")) + source_id = record.get("source_id") + return source_id if isinstance(source_id, str) else None + + +def _read_archived_events(directory: Path) -> dict[str, dict[str, Any]]: + if not directory.exists(): + return {} + indexed: dict[str, dict[str, Any]] = {} + for event in SessionReader(directory).iter_events(types=_RAW_EVENT_TYPES): + source_id = _source_id(event) + if source_id is not None: + indexed[source_id] = event + return indexed + + +def _add_source_ids(value: object, result: set[str]) -> None: + if not isinstance(value, dict): + return + ids = value.get("source_ids") + if isinstance(ids, list): + result.update(source_id for source_id in ids if isinstance(source_id, str)) + references = value.get("source_references") + if isinstance(references, list): + for reference in references: + source_id = _mapping(reference).get("source_id") + if isinstance(source_id, str): + result.add(source_id) + attributes = value.get("attributes") + if isinstance(attributes, dict): + _add_source_ids(attributes, result) + + +def _turn_source_ids(turn: dict[str, Any], events: Iterable[dict[str, Any]]) -> set[str]: + """Find archived evidence owned by a top-level reconstructed interaction. + + Producers can provide explicit source references on a turn/call. The + interaction-id fallback is evidence-preserving (not a guessed parent): it + includes all main and nested-child events bearing that exact native + interaction identity, which keeps child evidence inside its parent view. + """ + source_ids: set[str] = set() + _add_source_ids(turn, source_ids) + for call in _items(turn.get("llm_calls")): + _add_source_ids(_mapping(call), source_ids) + for call in _items(turn.get("accounting_calls")): + _add_source_ids(_mapping(call), source_ids) + for child in _items(turn.get("subagents")): + source_ids.update(_turn_source_ids(_mapping(child), ())) + + attributes = _mapping(turn.get("attributes")) + interaction_id = attributes.get("interaction_id") + turn_id = turn.get("turn_id") + for semantic_event in events: + event_attrs = _mapping(semantic_event.get("attributes")) + matched = ( + isinstance(interaction_id, str) and event_attrs.get("interaction_id") == interaction_id + ) or (isinstance(turn_id, str) and event_attrs.get("stored_turn_id") == turn_id) + if matched: + _add_source_ids(semantic_event, source_ids) + return source_ids + + +def _is_main_turn(turn: dict[str, Any]) -> bool: + return _mapping(turn.get("attributes")).get("agent_id") is None + + +def _replace_state(next_state: dict[str, Any]) -> dict[str, Any]: + """Treat ``next_state`` as an authoritative ProjectionState snapshot.""" + state = empty_projection_state() + archive_ids = next_state.get("archive_source_ids") + if isinstance(archive_ids, list): + state["archive_source_ids"] = [ + source_id for source_id in archive_ids if isinstance(source_id, str) + ] + semantic = _mapping(next_state.get("semantic_state")) + state["semantic_state"] = { + "open_interactions": dict(_mapping(semantic.get("open_interactions"))) + } + accounting = _mapping(next_state.get("accounting_state")) + state["accounting_state"] = {"logical_calls": dict(_mapping(accounting.get("logical_calls")))} + revision = next_state.get("projection_revision") + if isinstance(revision, str) and revision: + state["projection_revision"] = revision + state["projection_schema_version"] = PROJECTION_SCHEMA_VERSION + return state + + +def _collect_identities( + current: dict[str, Any], + attributions: dict[str, dict[str, Any]], + accounting_calls: dict[str, Any], +) -> dict[str, str]: + identities = { + key: value + for key, value in _mapping(current).items() + if isinstance(key, str) and isinstance(value, str) + } + for attribution in attributions.values(): + source_id = attribution.get("usage_source_id") + logical_id = attribution.get("logical_call_id") + if isinstance(source_id, str) and isinstance(logical_id, str) and logical_id: + identities[source_id] = logical_id + identities[logical_id] = logical_id + for call in accounting_calls.values(): + mapped = _mapping(call) + source_id = mapped.get("usage_source_id") + logical_id = mapped.get("logical_call_id") + if isinstance(logical_id, str) and logical_id: + identities[logical_id] = logical_id + if isinstance(source_id, str): + identities[source_id] = logical_id + return identities + + +def _usage_identity(row: UsageRow, identities: dict[str, str]) -> str: + return identities.get(row.call_id, row.call_id) + + +def _drop_aliased_usage_keys(usage_index: dict[str, Any], identities: dict[str, str]) -> None: + """Move a source-id row onto its logical id instead of deleting the call. + + An identity-only later commit may learn ``usage_source_id -> logical_call_id`` + without sending the ``UsageRow`` again. Popping the alias without copying + would drop the only persisted call. + """ + for source_id, logical_id in identities.items(): + if source_id == logical_id: + continue + existing = usage_index.pop(source_id, None) + if existing is None or logical_id in usage_index or not isinstance(existing, dict): + continue + relocated = dict(existing) + relocated["call_id"] = logical_id + usage_index[logical_id] = relocated + + +def _usage_seq( + logical_id: str, + row: UsageRow, + identities: dict[str, str], + archived_events: dict[str, dict[str, Any]], +) -> int: + candidates = [ + source_id + for source_id, mapped in identities.items() + if mapped == logical_id and source_id in archived_events + ] + if row.call_id in archived_events: + candidates.append(row.call_id) + if not candidates: + return 0 + return max(int(archived_events[source_id].get("seq", 0)) for source_id in candidates) + + +def _usage_line(row: dict[str, Any]) -> str: + return json.dumps(row, ensure_ascii=False, separators=(",", ":")) + + +def _sidecar_rows(usage_index: dict[str, Any]) -> list[dict[str, Any]]: + return [ + usage_index[logical_id] + for logical_id in sorted(usage_index) + if isinstance(usage_index[logical_id], dict) + ] + + +def _sidecar_payload(usage_index: dict[str, Any]) -> str: + return "".join(_usage_line(row) + "\n" for row in _sidecar_rows(usage_index)) + + +def _sidecar_matches(path: Path, usage_index: dict[str, Any]) -> bool: + if not path.exists(): + return not usage_index + try: + lines = [ + json.loads(line) + for line in path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + except (OSError, json.JSONDecodeError): + return False + return lines == _sidecar_rows(usage_index) + + +def _write_usage_index(directory: Path, usage_index: dict[str, Any]) -> None: + """Materialize one latest serialized row per logical call atomically. + + UsageStore is append-only and its generic reader is last-wins, which is + insufficient when a correction must replace a row under a re-keyed + identity. This derived-only rewrite preserves UsageRow.to_dict JSON + while the projection index supplies replacement semantics. + """ + path = usage_jsonl_path(directory) + path.parent.mkdir(parents=True, exist_ok=True) + fd, temporary = tempfile.mkstemp(dir=path.parent, prefix=f"{path.name}.", suffix=".tmp") + try: + with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as stream: + stream.write(_sidecar_payload(usage_index)) + stream.flush() + os.fsync(stream.fileno()) + fsops.replace(temporary, path) + fsops.sync_directory(path.parent) + except BaseException: + fsops.unlink(Path(temporary), missing_ok=True) + raise + + +def _invalidate_usage_index(config: Config, stored_session_id: str, directory: Path) -> None: + """Force UsageIndex to re-read a rewritten sidecar of unchanged size.""" + index = UsageIndex(config.root) + connection = index.connect() + try: + connection.execute("DELETE FROM usage WHERE session_id = ?", (stored_session_id,)) + connection.execute("DELETE FROM usage_sync WHERE session_id = ?", (stored_session_id,)) + index.refresh_session(connection, stored_session_id, directory) + finally: + connection.close() + + +def _publish_usage( + config: Config, + stored_session_id: str, + directory: Path, + usage_index: dict[str, Any], + *, + force: bool = False, +) -> None: + path = usage_jsonl_path(directory) + matches = _sidecar_matches(path, usage_index) + if not matches: + if not usage_index: + if path.exists(): + fsops.unlink(path, missing_ok=True) + fsops.sync_directory(directory) + else: + _write_usage_index(directory, usage_index) + elif not force: + return + _invalidate_usage_index(config, stored_session_id, directory) + + +def _commit_counts(indexes: dict[str, Any]) -> dict[str, int]: + return {name: len(_mapping(indexes.get(name))) for name in _INDEX_NAMES} + + +def _stored_turn( + turn: dict[str, Any], + source_ids: set[str], + prior: dict[str, Any] | None, +) -> dict[str, Any]: + turn_id = _index_key(turn, "turn_id", prefix="turn") + merged_ids = set(source_ids) + if prior is not None: + merged_ids.update( + source_id for source_id in _items(prior.get("source_ids")) if isinstance(source_id, str) + ) + status = turn.get("status") + return { + "turn_id": turn_id, + "status": status if isinstance(status, str) else "", + "source_ids": sorted(merged_ids), + "span": turn, + } + + +def _hydrate_turn( + stored_session_id: str, + record: dict[str, Any], + archived_events: dict[str, dict[str, Any]], + cwd: str, +) -> dict[str, Any]: + turn_id = record.get("turn_id") if isinstance(record.get("turn_id"), str) else "" + source_ids = [item for item in _items(record.get("source_ids")) if isinstance(item, str)] + events = [ + archived_events[source_id] for source_id in source_ids if source_id in archived_events + ] + events.sort(key=lambda event: int(event.get("seq", -1))) + span = _mapping(record.get("span")) + start_ts = span.get("start_ts") if isinstance(span.get("start_ts"), str) else None + end_ts = span.get("end_ts") if isinstance(span.get("end_ts"), str) else None + return { + "id": f"{stored_session_id}:{turn_id}", + "turn_id": turn_id, + "session_id": stored_session_id, + "platform": PLATFORM_NAME, + "cwd": cwd, + "start_seq": events[0].get("seq") if events else None, + "end_seq": events[-1].get("seq") if events else None, + "start_ts": events[0].get("ts") if events else start_ts, + "end_ts": events[-1].get("ts") if events else end_ts, + "events": events, + } + + +def commit_projection( + config: Config, + stored_session_id: str, + projection: Projection, + next_state: dict[str, Any], + *, + replace: bool = False, + base_commit_sequence: int | None = None, +) -> dict[str, int]: + """Atomically merge or replace a DTO projection into replayable derived state. + + V1 events are read only from the captured Store archive to form the generic + turn view. No source reader is invoked, so a captured session remains + projectable after Copilot removes its original files. + + ``replace=True`` discards prior derived indexes as part of this same + locked operation instead of requiring a separate reset call. Nothing is + written to disk until every validation above this point succeeds, so a + commit that fails validation (a malformed usage row, an unknown-schema + document) leaves the previous projection completely untouched rather + than losing it to a non-atomic delete-then-commit sequence. + + Durability past that validation point has two distinct failure modes. + The projection document is the source of truth and is published with + write-ahead journaling (see ``publish_projection_document``): once that + call returns, the new document is durably committed, full stop. The + usage sidecar published immediately after is a derived, self-healing + mirror of the document's own usage index -- kept as a separate JSONL + file only so ``UsageIndex`` can query it without parsing the whole + document. If that second, mirror-only publish raises (a disk error, not + a validation error), this function still raises so the caller learns of + it, but the already-committed document is *not* rolled back: it reflects + the new projection, and the next ``load_projection_state`` call + republishes a sidecar that matches it. A caller must not assume a raise + from this function always means "nothing changed" -- check which phase + failed via ``read_projection_status``/``load_projection_state`` if that + distinction matters. + + ``base_commit_sequence``, when given, must equal the ``commit_sequence`` + a caller observed from an earlier ``load_projection_state`` call. A + mismatch means another writer has committed since that read -- this + raises :class:`ProjectionConflictError` instead of overwriting the newer + projection with one built from the stale builder state, closing the + otherwise unlocked gap between reading prior state, building a new + projection from it, and committing here. This check itself happens + before any write in this call, so a conflict never touches disk. + """ + directory = _directory(config, stored_session_id) + _require_session(directory, stored_session_id) + with locked(projection_lock_path(directory), LockMode.EXCLUSIVE): + document = read_projection_document(directory) + current_sequence = _mapping(document.get("state")).get("commit_sequence") + current_sequence = current_sequence if isinstance(current_sequence, int) else 0 + if base_commit_sequence is not None and base_commit_sequence != current_sequence: + raise ProjectionConflictError( + f"Copilot projection for {stored_session_id!r} advanced from commit " + f"{base_commit_sequence} to {current_sequence} since it was loaded; " + "reload projection state and retry" + ) + indexes = {} if replace else _mapping(document.get("indexes")) + merged_indexes = {name: dict(_mapping(indexes.get(name))) for name in indexes} + for name in (*_INDEX_NAMES, "usage_identities"): + merged_indexes.setdefault(name, {}) + + event_items = [ + item for item in projection.get("normalized_events", []) if isinstance(item, dict) + ] + for item in event_items: + merged_indexes["events"][_index_key(item, "id", prefix="event")] = item + + attribution_items = [ + item for item in projection.get("attributions", []) if isinstance(item, dict) + ] + for item in attribution_items: + merged_indexes["attributions"][ + _index_key(item, "logical_call_id", prefix="attribution") + ] = item + + state = _replace_state(next_state) + state["commit_sequence"] = current_sequence + 1 + identities = _collect_identities( + merged_indexes["usage_identities"], + merged_indexes["attributions"], + _mapping(_mapping(state.get("accounting_state")).get("logical_calls")), + ) + + archived_events = _read_archived_events(directory) + for item in projection.get("usage_rows", []): + if not isinstance(item, UsageRow): + raise TypeError("projection usage_rows must contain UsageRow instances") + if item.session_id != stored_session_id: + raise ValueError("usage row session_id does not match stored session") + logical_id = _usage_identity(item, identities) + identities[item.call_id] = logical_id + identities[logical_id] = logical_id + row = item.to_dict() + row["call_id"] = logical_id + row["seq"] = _usage_seq(logical_id, item, identities, archived_events) + merged_indexes["usage"][logical_id] = row + _drop_aliased_usage_keys(merged_indexes["usage"], identities) + merged_indexes["usage_identities"] = identities + + for item in projection.get("turns", []): + if not isinstance(item, dict) or not _is_main_turn(item): + continue + turn_id = _index_key(item, "turn_id", prefix="turn") + source_ids = _turn_source_ids(item, merged_indexes["events"].values()) + merged_indexes["turns"][turn_id] = _stored_turn( + item, source_ids, _mapping(merged_indexes["turns"].get(turn_id)) or None + ) + + merged_indexes["pending"] = { + _index_key(item, "id", prefix="pending"): item + for item in projection.get("pending", []) + if isinstance(item, dict) + } + merged_indexes["diagnostics"] = { + _index_key(item, "id", prefix="diagnostic"): item + for item in projection.get("diagnostics", []) + if isinstance(item, dict) + } + + counts = _commit_counts(merged_indexes) + if not state["projection_revision"]: + state["projection_revision"] = _digest( + {key: merged_indexes.get(key) for key in (*_INDEX_NAMES, "usage_identities")} + ) + state["index_totals"] = counts + next_document = { + "schema_version": PROJECTION_SCHEMA_VERSION, + "state": state, + "indexes": merged_indexes, + } + publish_projection_document(directory, next_document) + _publish_usage(config, stored_session_id, directory, merged_indexes["usage"]) + return counts + + +def load_projection_state(config: Config, stored_session_id: str) -> dict[str, Any]: + """Return a copy of derived builder state, completing journal recovery.""" + directory = _existing_session_dir(config, stored_session_id) + if directory is None: + return empty_projection_state() + with locked(projection_lock_path(directory), LockMode.EXCLUSIVE): + document = read_projection_document(directory) + usage_index = _mapping(_mapping(document.get("indexes")).get("usage")) + _publish_usage(config, stored_session_id, directory, usage_index, force=True) + return json.loads(_canonical(_mapping(document.get("state")))) + + +def read_projected_turns(config: Config, stored_session_id: str) -> list[dict[str, Any]]: + """Read completed main interaction records without semantic duplicates.""" + directory = _existing_session_dir(config, stored_session_id) + if directory is None: + return [] + with locked(projection_lock_path(directory), LockMode.EXCLUSIVE): + document = read_projection_document(directory) + meta = read_meta(meta_path(directory)) + cwd = meta.cwd if meta is not None else "" + archived_events = _read_archived_events(directory) + turns = _mapping(_mapping(document.get("indexes")).get("turns")) + values = [] + for record in turns.values(): + if not isinstance(record, dict): + continue + if record.get("status") != "completed": + continue + values.append(_hydrate_turn(stored_session_id, record, archived_events, cwd)) + values.sort( + key=lambda turn: (str(turn.get("start_ts") or ""), str(turn.get("turn_id") or "")) + ) + return json.loads(_canonical(values)) + + +def _collect_usage_labels(turn: dict[str, Any], labels: dict[str, dict[str, Any]]) -> None: + for call in _items(turn.get("accounting_calls")): + mapped = _mapping(call) + accounting_id = mapped.get("accounting_id") + attributes = _mapping(mapped.get("attributes")) + extra = { + key: value + for key, value in attributes.items() + if str(key).startswith("copilot.billing") or str(key).startswith("copilot.latency") + } + if isinstance(accounting_id, str) and extra: + labels[accounting_id] = extra + for child in _items(turn.get("subagents")): + if isinstance(child, dict): + _collect_usage_labels(child, labels) + + +def read_usage_labels(config: Config, stored_session_id: str) -> dict[str, dict[str, Any]]: + """Read Copilot native billing/latency labels from stored turn accounting.""" + + directory = _existing_session_dir(config, stored_session_id) + if directory is None: + return {} + with locked(projection_lock_path(directory), LockMode.EXCLUSIVE): + document = read_projection_document(directory) + turns = _mapping(_mapping(document.get("indexes")).get("turns")) + labels: dict[str, dict[str, Any]] = {} + for record in turns.values(): + if isinstance(record, dict): + _collect_usage_labels(_mapping(record.get("span")), labels) + return json.loads(_canonical(labels)) + + +_STATUS_KEYS = ("events", "usage", "turns", "pending", "ambiguous", "conflicting", "errors") + + +def read_projection_status(config: Config, stored_session_id: str) -> dict[str, int]: + """Return persisted projection counts without attempting a new derive. + + A caller whose current reconciliation attempt failed can use this to + report the status of the last successfully published projection (which + remains on disk and readable) instead of fabricating zeroed-out counts. + """ + directory = _existing_session_dir(config, stored_session_id) + if directory is None: + return dict.fromkeys(_STATUS_KEYS, 0) + with locked(projection_lock_path(directory), LockMode.EXCLUSIVE): + document = read_projection_document(directory) + indexes = _mapping(document.get("indexes")) + attributions = _mapping(indexes.get("attributions")).values() + diagnostics = _mapping(indexes.get("diagnostics")).values() + return { + "events": len(_mapping(indexes.get("events"))), + "usage": len(_mapping(indexes.get("usage"))), + "turns": len(_mapping(indexes.get("turns"))), + "pending": len(_mapping(indexes.get("pending"))), + "ambiguous": sum( + 1 + for item in attributions + if isinstance(item, dict) and item.get("status") == "ambiguous" + ), + "conflicting": sum( + 1 + for item in attributions + if isinstance(item, dict) and item.get("status") == "conflicting" + ), + "errors": sum( + 1 + for item in diagnostics + if isinstance(item, dict) and item.get("severity") == "error" + ), + } + + +def reset_projection_state(config: Config, stored_session_id: str) -> None: + """Delete only reproducible projection state for a local rebuild. + + The caller must subsequently commit a full replay. This intentionally + does not import, reset, or otherwise interact with export-state files. + """ + directory = _existing_session_dir(config, stored_session_id) + if directory is None: + return + with locked(projection_lock_path(directory), LockMode.EXCLUSIVE): + remove_projection_state(directory) + fsops.unlink(usage_jsonl_path(directory), missing_ok=True) + _invalidate_usage_index(config, stored_session_id, directory) + fsops.sync_directory(directory) diff --git a/src/thirdeye/platforms/copilot/reconcile.py b/src/thirdeye/platforms/copilot/reconcile.py new file mode 100644 index 00000000..70c8f25d --- /dev/null +++ b/src/thirdeye/platforms/copilot/reconcile.py @@ -0,0 +1,163 @@ +"""Archive-only composition for Copilot's local V2 projection. + +Reconciliation deliberately reads the immutable V1 archive rather than the +live Copilot home. That makes it safe to run after a session has ended (or +after Copilot has removed its transient transcript files), and keeps capture +failures independent from derived-state failures. +""" + +from __future__ import annotations + +from collections.abc import Callable +from importlib import import_module + +from thirdeye.config import Config + +from .archive import iter_captured_records +from .projection import build_projection +from .projection_store import commit_projection, load_projection_state, read_projection_status +from .types import Projection + +_COUNT_KEYS = ("events", "usage", "turns", "exports", "pending", "ambiguous", "conflicting") + + +def _empty_result(*, errors: int = 0) -> dict[str, int]: + result = {key: 0 for key in _COUNT_KEYS} + result["errors"] = errors + return result + + +def _failure_result(config: Config, stored_session_id: str) -> dict[str, int]: + """Report the last successfully published projection's status on failure. + + The local projection on disk (if any) is untouched by a failed attempt, + so its counts -- not zeros -- describe what a reader will actually see. + """ + try: + status = read_projection_status(config, stored_session_id) + except Exception: + return _empty_result(errors=1) + result = {**_empty_result(), **status} + result["errors"] = status.get("errors", 0) + 1 + return result + + +def _attribution_counts(projection: Projection) -> tuple[int, int]: + ambiguous = 0 + conflicting = 0 + for attribution in projection["attributions"]: + status = attribution.get("status") + if status == "ambiguous": + ambiguous += 1 + elif status == "conflicting": + conflicting += 1 + return ambiguous, conflicting + + +def _diagnostic_errors(projection: Projection) -> int: + return sum( + 1 for diagnostic in projection["diagnostics"] if diagnostic.get("severity") == "error" + ) + + +def queue_exports( + config: Config, + stored_session_id: str, + projection: Projection, + *, + include_history: bool = False, +) -> int: + """Queue export work only when an explicit caller requests it. + + Export assembly is intentionally not an import-time dependency of local + reconciliation. Runtime integration can call this boundary after the + export implementation is installed; archive-only callers never load it. + """ + + module = import_module(".export", package=__package__) + enqueue = module.queue_exports + if not callable(enqueue): + raise TypeError("Copilot export assembly does not provide queue_exports") + exporter: Callable[..., int] = enqueue + return exporter(config, stored_session_id, projection, include_history=include_history) + + +def reconcile_archive( + config: Config, + stored_session_id: str, + *, + rebuild: bool = False, + export: bool = False, + include_history: bool = False, +) -> dict[str, int]: + """Rebuild local Copilot projections from the immutable V1 archive. + + A normal reconciliation is a complete archive replay. Stable derived + identities make committing that replay idempotent. ``rebuild`` discards + reproducible projection state as part of the same locked commit used for + an incremental merge (see ``commit_projection(..., replace=True)``): raw + archive records and the separate export ledger are untouched, and nothing + is written to disk until the replacement projection passes validation, so + a rebuild that fails validation (a malformed usage row, a stale schema) + cannot erase the last readable local view. A failure *after* that point + -- the projection document itself durably publishes, but the derived + usage-sidecar mirror then hits an I/O error -- still reports an error + here, but the counts below reflect the document that was, in fact, + committed; see ``commit_projection`` for why that distinction is not a + bug. Either way the next reconciliation call self-heals the sidecar. + + Loading prior state, building the new projection from it, and committing + are not one locked operation -- ``build_projection`` runs unlocked so a + slow archive replay does not hold the projection lock. To close the gap + that leaves, the ``commit_sequence`` observed here is passed through to + ``commit_projection`` as ``base_commit_sequence``: if another writer + committed in the meantime, the commit is refused (``ProjectionConflictError``, + reported below as an error) instead of overwriting newer derived state + with one built from what is now stale builder state. This is checked + even for ``rebuild``, which otherwise discards ``prior_state`` entirely. + + Errors are reported as counts instead of escaping so source capture can + remain operational when a derived projection is malformed. Export is a + second, opt-in phase performed only after the local commit succeeds. + """ + + try: + loaded_state = load_projection_state(config, stored_session_id) + base_commit_sequence = loaded_state.get("commit_sequence") + prior_state = {} if rebuild else loaded_state + records = list(iter_captured_records(config, stored_session_id)) + projection, next_state = build_projection(records, prior_state) + next_state["archive_source_ids"] = [record["source_id"] for record in records] + counts = commit_projection( + config, + stored_session_id, + projection, + next_state, + replace=rebuild, + base_commit_sequence=base_commit_sequence, + ) + except Exception: + return _failure_result(config, stored_session_id) + + ambiguous, conflicting = _attribution_counts(projection) + result = { + "events": counts["events"], + "usage": counts["usage"], + "turns": counts["turns"], + "exports": 0, + "pending": counts["pending"], + "ambiguous": ambiguous, + "conflicting": conflicting, + "errors": _diagnostic_errors(projection), + } + if not export: + return result + + try: + result["exports"] = queue_exports( + config, stored_session_id, projection, include_history=include_history + ) + except Exception: + # Export delivery must never roll back a successful local projection. + result["errors"] += 1 + return result diff --git a/src/thirdeye/platforms/copilot/runtime.py b/src/thirdeye/platforms/copilot/runtime.py new file mode 100644 index 00000000..508090e7 --- /dev/null +++ b/src/thirdeye/platforms/copilot/runtime.py @@ -0,0 +1,218 @@ +"""Runtime composition for archive reconciliation after V1 capture. + +Capture remains the durable, local V1 boundary while reconciliation and +export eligibility are V2 derived work. A projection failure must never turn +a successful hook receipt or source import into a failed capture. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from thirdeye.config import Config +from thirdeye.paths import platform_dir, session_dir +from thirdeye.usage.errlog import log_capture_error + +from .constants import PLATFORM_NAME +from .identity import stored_session_id, validate_stored_session_id +from .jsonio import atomic_write_json, read_json_object +from .reconcile import reconcile_archive +from .state import read_json, state_path +from .types import SourcePaths + +RUNTIME_STATUS_FILENAME = "copilot.runtime.json" + + +def exports_configured(config: Config) -> bool: + """Whether runtime activity may queue detached live export work.""" + return bool(config.logfire.enabled and config.logfire.token) + + +def runtime_status_path(directory: Path) -> Path: + return directory / RUNTIME_STATUS_FILENAME + + +def stored_session_directory(config: Config, stored_id: str) -> Path: + """Return the archive directory for a stored ID after rejecting path escapes.""" + + validate_stored_session_id(stored_id) + root = platform_dir(config.root, PLATFORM_NAME).resolve() + directory = (root / stored_id).resolve() + try: + directory.relative_to(root) + except ValueError as exc: + raise ValueError("stored session ID escapes the Copilot archive") from exc + return directory + + +def load_runtime_status(config: Config, stored_session_id: str) -> dict[str, Any]: + """Return the last recorded derived-work error for a stored session.""" + + try: + directory = stored_session_directory(config, stored_session_id) + raw = read_json_object( + runtime_status_path(directory), + invalid_message="invalid Copilot runtime status", + ) + except (OSError, ValueError): + return {} + return raw if isinstance(raw, dict) else {} + + +def _note_reconcile_result( + config: Config, stored_id: str, result: dict[str, int] +) -> dict[str, int]: + """Persist/log derived failures without raising into capture.""" + + directory = session_dir(config.root, PLATFORM_NAME, stored_id) + path = runtime_status_path(directory) + if result.get("errors"): + log_capture_error( + thirdeye_home=config.root, + phase="copilot_reconcile", + message=f"errors={result.get('errors')}", + platform=PLATFORM_NAME, + session_id=stored_id, + silent_fallback=True, + ) + if directory.is_dir(): + try: + atomic_write_json( + path, + { + "last_error": { + "phase": "copilot_reconcile", + "errors": int(result.get("errors") or 0), + } + }, + ) + except OSError: + pass + return result + if directory.is_dir(): + try: + path.unlink(missing_ok=True) + except OSError: + pass + return result + + +def archived_session_ids(config: Config, paths: SourcePaths) -> list[str]: + """Return this source home's retained V1 sessions without reading sources.""" + root = platform_dir(config.root, PLATFORM_NAME) + prefix = f"copilot-{paths['source_key'][:16]}-" + try: + directories = sorted(entry for entry in root.iterdir() if entry.is_dir()) + except OSError: + return [] + result: list[str] = [] + for directory in directories: + if not directory.name.startswith(prefix): + continue + try: + validate_stored_session_id(directory.name) + state = read_json(state_path(directory)) or {} + except ValueError: + continue + if state.get("source_key") == paths["source_key"]: + result.append(directory.name) + return result + + +def all_archived_session_ids(config: Config) -> list[str]: + """Return every retained Copilot archive under this Thirdeye home.""" + + root = platform_dir(config.root, PLATFORM_NAME) + try: + directories = sorted(entry for entry in root.iterdir() if entry.is_dir()) + except OSError: + return [] + result: list[str] = [] + for directory in directories: + try: + validate_stored_session_id(directory.name) + state = read_json(state_path(directory)) + except ValueError: + continue + if state is not None: + result.append(directory.name) + return result + + +def reconcile_stored_session( + config: Config, + stored_session_id: str, + *, + rebuild: bool = False, + export: bool = False, + include_history: bool = False, +) -> dict[str, int]: + """Derive one stored archive after validating its identity.""" + + stored_session_directory(config, stored_session_id) + return _note_reconcile_result( + config, + stored_session_id, + reconcile_archive( + config, + stored_session_id, + rebuild=rebuild, + export=export, + include_history=include_history, + ), + ) + + +def reconcile_session( + config: Config, + paths: SourcePaths, + native_session_id: str, + *, + export: bool = False, + include_history: bool = False, +) -> dict[str, int]: + """Derive one retained session, optionally queueing detached live export. + + ``export`` is forwarded even when Logfire is not configured: export + assembly records the activation boundary before it decides whether any + jobs can be dispatched. + """ + return reconcile_stored_session( + config, + stored_session_id(paths, native_session_id), + export=export, + include_history=include_history, + ) + + +def reconcile_archived_sessions( + config: Config, + paths: SourcePaths, + *, + export: bool = False, + include_history: bool = False, +) -> dict[str, dict[str, int]]: + """Replay retained sessions even after their live sources disappear.""" + return { + session_id: reconcile_stored_session( + config, + session_id, + export=export, + include_history=include_history, + ) + for session_id in archived_session_ids(config, paths) + } + + +__all__ = [ + "all_archived_session_ids", + "archived_session_ids", + "exports_configured", + "load_runtime_status", + "reconcile_archived_sessions", + "reconcile_session", + "reconcile_stored_session", + "runtime_status_path", + "stored_session_directory", +] diff --git a/src/thirdeye/platforms/copilot/sources.py b/src/thirdeye/platforms/copilot/sources.py new file mode 100644 index 00000000..d6597e5e --- /dev/null +++ b/src/thirdeye/platforms/copilot/sources.py @@ -0,0 +1,250 @@ +"""Compose Copilot's independent persisted recording sources. + +This module remains above the individual readers and below the archive. Its +only durable contract is a :class:`SourceBatch`: reader cursors stay namespaced +so a transcript replacement cannot reset SQLite pagination (and vice versa). +""" + +from __future__ import annotations + +from collections.abc import Callable +from copy import deepcopy +from pathlib import Path +from typing import Any + +from .database import discover_database_sessions, read_database +from .identity import resolve_sources, validate_native_id +from .transcript import discover_transcripts, read_transcript +from .types import SourceBatch, SourcePaths, SourceSlice + +__all__ = ["discover_sessions", "read_batch", "resolve_sources"] + +_DISCOVERY_PROBE_ID = "copilot-discovery-probe" +_MAX_DATABASE_SNAPSHOT_PROBES = 16 +_FILE_LEVEL_DATABASE_CODES = frozenset( + { + "copilot_database_unreadable", + "copilot_database_busy", + "copilot_database_incompatible", + "copilot_database_read_failed", + } +) + + +def _discovery_diagnostic(name: str, error: BaseException) -> dict[str, Any]: + return { + "code": f"copilot_{name}_unreadable", + "message": "Copilot source could not be discovered; retry sync or watch", + "reason": type(error).__name__, + } + + +def _discover_with_diagnostics( + paths: SourcePaths, +) -> tuple[list[str], list[dict[str, Any]]]: + """Return session IDs plus discovery diagnostics that sync must surface.""" + + discovered: set[str] = set() + diagnostics: list[dict[str, Any]] = [] + + try: + discovered.update(discover_transcripts(paths)) + except (OSError, ValueError) as error: + diagnostics.append(_discovery_diagnostic("transcript", error)) + + database_ids: list[str] | None + try: + database_ids = list(discover_database_sessions(paths)) + discovered.update(database_ids) + except (OSError, ValueError) as error: + diagnostics.append(_discovery_diagnostic("database", error)) + database_ids = None + + # discover_database_sessions treats an unreadable file as "no sessions". + # Probe the file-level diagnostic so all-session sync is not a silent no-op. + if database_ids == [] and Path(paths["database"]).is_file(): + probe = _read_slice("database", read_database, paths, _DISCOVERY_PROBE_ID, {}) + for diagnostic in probe["diagnostics"]: + code = diagnostic.get("code") + if isinstance(code, str) and code in _FILE_LEVEL_DATABASE_CODES: + diagnostics.append(diagnostic) + + return sorted(discovered), diagnostics + + +def discover_sessions(paths: SourcePaths) -> list[str]: + """Return the deterministic union of readable transcript and DB sessions.""" + + sessions, _ = _discover_with_diagnostics(paths) + return sessions + + +def _cursor_part(cursor: dict[str, Any], name: str) -> dict[str, Any]: + value = cursor.get(name) + return deepcopy(value) if isinstance(value, dict) else {} + + +def _failed_slice(cursor: dict[str, Any], name: str, error: BaseException) -> SourceSlice: + return { + "records": [], + "next_cursor": deepcopy(cursor), + "diagnostics": [ + { + "code": f"copilot_{name}_read_failed", + "message": "Copilot source could not be read; retry sync or watch", + "reason": type(error).__name__, + } + ], + "cwd": None, + "exhausted": False, + } + + +def _read_slice( + name: str, + reader: Callable[[SourcePaths, str, dict[str, Any]], SourceSlice], + paths: SourcePaths, + native_session_id: str, + cursor: dict[str, Any], +) -> SourceSlice: + try: + return reader(paths, native_session_id, cursor) + except (OSError, ValueError) as error: + return _failed_slice(cursor, name, error) + + +def _is_database_cursor(cursor: dict[str, Any]) -> bool: + return isinstance(cursor.get("database_offset"), int) or isinstance( + cursor.get("database_generation"), str + ) + + +def _page_start(slice_: SourceSlice, incoming_offset: object) -> int: + live_offset = slice_["next_cursor"].get("database_offset") + page_len = len(slice_["records"]) + if isinstance(live_offset, int) and live_offset >= page_len: + return live_offset - page_len + return incoming_offset if isinstance(incoming_offset, int) else 0 + + +def _observe_database_snapshot_end(paths: SourcePaths, native_id: str, slice_: SourceSlice) -> int: + """Freeze the live row count observed for this invocation. + + Additional probes learn how far the current SQLite snapshot extends, but + they stop after a bounded number of pages so a source that keeps growing + cannot postpone snapshot creation. + """ + + cursor = slice_["next_cursor"] + offset = cursor.get("database_offset", len(slice_["records"])) + if not isinstance(offset, int) or offset < 0: + offset = len(slice_["records"]) + if slice_["exhausted"]: + return offset + generation = cursor.get("database_generation") + observed = offset + for _ in range(_MAX_DATABASE_SNAPSHOT_PROBES): + try: + probe = read_database( + paths, + native_id, + {"database_generation": generation, "database_offset": observed}, + ) + except (OSError, ValueError): + return observed + nxt = probe["next_cursor"].get("database_offset", observed) + if not isinstance(nxt, int) or nxt <= observed: + return observed + observed = nxt + if probe["exhausted"]: + return observed + return observed + + +def _with_database_snapshot( + paths: SourcePaths, + native_id: str, + slice_: SourceSlice, + incoming: dict[str, Any], +) -> SourceSlice: + """Keep database pagination inside the invocation-time row bound. + + The SQLite reader re-queries live rows on every page, so composition must + freeze ``snapshot_end`` the way the transcript reader freezes file size. + """ + + next_cursor = deepcopy(slice_["next_cursor"]) + if not _is_database_cursor(next_cursor) and not _is_database_cursor(incoming): + return slice_ + + records = list(slice_["records"]) + incoming_end = incoming.get("snapshot_end") + incoming_offset = incoming.get("database_offset", 0) + incoming_gen = incoming.get("database_generation") + live_gen = next_cursor.get("database_generation") + start_offset = _page_start(slice_, incoming_offset) + draining = ( + incoming_gen == live_gen + and isinstance(incoming_end, int) + and isinstance(incoming_offset, int) + and incoming_offset < incoming_end + ) + if draining: + snapshot_end = incoming_end + else: + snapshot_end = _observe_database_snapshot_end(paths, native_id, slice_) + + allowed = max(0, snapshot_end - start_offset) + if len(records) > allowed: + records = records[:allowed] + next_offset = start_offset + len(records) + next_cursor["database_offset"] = next_offset + next_cursor["snapshot_end"] = snapshot_end + return { + **slice_, + "records": records, + "next_cursor": next_cursor, + "exhausted": next_offset >= snapshot_end, + } + + +def read_batch(paths: SourcePaths, native_session_id: str, cursor: dict) -> SourceBatch: + """Read one bounded, lossless slice from each persisted source. + + Sources deliberately do not share a cursor. A source failure is retained + as a diagnostic while a healthy sibling source still contributes records. + ``base_cursor`` is an optimistic archive marker and is stripped before the + archive persists the next source cursor. Exhaustion flags stay on the + composition cursor so capture can page or report pending without confusing + the per-source readers. Database cursors also carry ``snapshot_end`` so a + later SQLite insert cannot extend this invocation's drain. + """ + + validate_native_id(native_session_id) + source_cursor: dict[str, Any] = dict(cursor) if isinstance(cursor, dict) else {} + transcript_cursor = _cursor_part(source_cursor, "transcript") + database_cursor = _cursor_part(source_cursor, "database") + transcript = _read_slice( + "transcript", read_transcript, paths, native_session_id, transcript_cursor + ) + database = _with_database_snapshot( + paths, + native_session_id, + _read_slice("database", read_database, paths, native_session_id, database_cursor), + database_cursor, + ) + next_cursor: dict[str, Any] = { + "transcript": deepcopy(transcript["next_cursor"]), + "database": deepcopy(database["next_cursor"]), + "base_cursor": deepcopy(source_cursor), + "transcript_exhausted": bool(transcript["exhausted"]), + "database_exhausted": bool(database["exhausted"]), + } + return { + "source_key": paths["source_key"], + "native_session_id": native_session_id, + "cwd": transcript["cwd"] if transcript["cwd"] is not None else database["cwd"], + "records": [*transcript["records"], *database["records"]], + "next_cursor": next_cursor, + "diagnostics": [*transcript["diagnostics"], *database["diagnostics"]], + } diff --git a/src/thirdeye/platforms/copilot/spool.py b/src/thirdeye/platforms/copilot/spool.py new file mode 100644 index 00000000..86621847 --- /dev/null +++ b/src/thirdeye/platforms/copilot/spool.py @@ -0,0 +1,133 @@ +"""Durable per-record spool for Copilot hook observations. + +Enqueue is independent of archive capture: each observation is written +atomically to its own file without taking the session archive lock. Read +returns complete records; acknowledge deletes only the named committed +source IDs. Malformed neighbors stay on disk and do not drop valid records. +""" + +from __future__ import annotations + +import json +import os +import tempfile +import uuid +from pathlib import Path +from typing import Any + +from thirdeye._compat import fsops +from thirdeye.config import Config +from thirdeye.platforms.copilot.identity import validate_native_id +from thirdeye.platforms.copilot.types import SourcePaths, SourceRecord + +_MAX_DIAGNOSTIC_REASON = 200 + + +def _spool_root(config: Config, paths: SourcePaths) -> Path: + return Path(config.root) / "spool" / "copilot" / paths["source_key"] + + +def _spool_dir(config: Config, paths: SourcePaths, native_id: str) -> Path: + validate_native_id(native_id) + return _spool_root(config, paths) / native_id + + +def _write_diagnostic(path: Path, reason: str) -> None: + """Record a bounded location/reason diagnostic; never prompt bodies.""" + + diag_path = path.with_name(f"{path.name}.diag") + clipped = reason[:_MAX_DIAGNOSTIC_REASON] + payload = json.dumps({"path": path.name, "reason": clipped}, ensure_ascii=True) + try: + diag_path.write_text(payload + "\n", encoding="utf-8") + except OSError: + return + + +def _record_error(data: Any, *, expected_native_id: str) -> str | None: + """Return a diagnostic reason when *data* is not a complete hook SourceRecord.""" + + if not isinstance(data, dict): + return "spool file is not a SourceRecord object" + for key in ("source_id", "source_kind", "native_session_id", "observed_at"): + value = data.get(key) + if not isinstance(value, str) or not value: + return "spool file is not a complete SourceRecord object" + if "ts" not in data or (data["ts"] is not None and not isinstance(data["ts"], str)): + return "spool file is not a complete SourceRecord object" + if not isinstance(data.get("payload"), dict) or not isinstance(data.get("locator"), dict): + return "spool file is not a complete SourceRecord object" + if data["source_kind"] != "hook": + return "spool file source_kind is not hook" + if data["native_session_id"] != expected_native_id: + return "native_session_id does not match spool directory" + return None + + +def _load_record(path: Path, *, expected_native_id: str) -> SourceRecord | None: + try: + raw = fsops.read_text(path, encoding="utf-8") + data: Any = json.loads(raw) + except (OSError, UnicodeDecodeError, json.JSONDecodeError, RecursionError) as exc: + _write_diagnostic(path, f"{type(exc).__name__}: {exc}") + return None + reason = _record_error(data, expected_native_id=expected_native_id) + if reason is not None: + _write_diagnostic(path, reason) + return None + return data # type: ignore[return-value] + + +def enqueue_hook(config: Config, paths: SourcePaths, record: SourceRecord) -> str: + """Atomically persist one hook observation and return its spool path.""" + + native_id = record["native_session_id"] + directory = _spool_dir(config, paths, native_id) + directory.mkdir(parents=True, exist_ok=True) + dest = directory / f"{uuid.uuid4().hex}.json" + fd, tmp_name = tempfile.mkstemp(dir=directory, prefix=".", suffix=".json.tmp") + try: + with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as handle: + json.dump(record, handle, ensure_ascii=False) + handle.write("\n") + handle.flush() + os.fsync(handle.fileno()) + fsops.replace(tmp_name, dest) + except BaseException: + fsops.unlink(Path(tmp_name), missing_ok=True) + raise + return str(dest) + + +def read_spool(config: Config, paths: SourcePaths, native_id: str) -> list[SourceRecord]: + """Return complete spool records for *native_id*, skipping malformed files.""" + + directory = _spool_dir(config, paths, native_id) + if not directory.is_dir(): + return [] + records: list[SourceRecord] = [] + for path in sorted(directory.glob("*.json")): + record = _load_record(path, expected_native_id=native_id) + if record is not None: + records.append(record) + return records + + +def ack_spool(config: Config, paths: SourcePaths, source_ids: list[str]) -> None: + """Delete spool files whose committed source IDs are listed. + + Unknown IDs are no-ops. Malformed files are left in place. + """ + + wanted = {item for item in source_ids if item} + if not wanted: + return + root = _spool_root(config, paths) + if not root.is_dir(): + return + for path in root.glob("*/*.json"): + record = _load_record(path, expected_native_id=path.parent.name) + if record is None: + continue + if record["source_id"] in wanted: + fsops.unlink(path, missing_ok=True) diff --git a/src/thirdeye/platforms/copilot/state.py b/src/thirdeye/platforms/copilot/state.py new file mode 100644 index 00000000..f9087111 --- /dev/null +++ b/src/thirdeye/platforms/copilot/state.py @@ -0,0 +1,74 @@ +"""Private, atomically-published state for the Copilot evidence archive.""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from thirdeye._compat import fsops + +from .jsonio import atomic_write_json, read_json_object + +STATE_SCHEMA_VERSION = 1 +STATE_FILENAME = "copilot.state.json" +JOURNAL_FILENAME = "copilot.journal.json" +LOCK_FILENAME = "copilot.archive.lock" + +# Tests may replace this with a deterministic callback that raises at a +# publication boundary. Runtime behaviour never depends on fault injection. +_fault_injector: Callable[[str], None] | None = None + + +def _fault(point: str) -> None: + if _fault_injector is not None: + _fault_injector(point) + + +def state_path(session_dir: Path) -> Path: + return session_dir / STATE_FILENAME + + +def journal_path(session_dir: Path) -> Path: + return session_dir / JOURNAL_FILENAME + + +def lock_path(session_dir: Path) -> Path: + return session_dir / LOCK_FILENAME + + +def _atomic_json(path: Path, value: dict[str, Any]) -> None: + after_replace = None + if path.name == STATE_FILENAME: + after_replace = "after_state_replace" + elif path.name == JOURNAL_FILENAME: + after_replace = "after_journal_replace" + atomic_write_json( + path, + value, + on_replaced=(lambda: _fault(after_replace)) if after_replace else None, + on_synced=lambda: _fault("after_dirsync"), + ) + + +def read_json(path: Path) -> dict[str, Any] | None: + try: + return read_json_object(path, invalid_message=f"invalid Copilot archive state: {path}") + except OSError as exc: + # A malformed or unreadable state file is never silently used as a + # fresh cursor. The archive caller turns this into a diagnostic and + # leaves the evidence log itself readable. + raise ValueError(f"invalid Copilot archive state: {path}") from exc + + +def write_state(session_dir: Path, value: dict[str, Any]) -> None: + _atomic_json(state_path(session_dir), value) + + +def write_journal(session_dir: Path, value: dict[str, Any]) -> None: + _atomic_json(journal_path(session_dir), value) + + +def clear_journal(session_dir: Path) -> None: + fsops.unlink(journal_path(session_dir), missing_ok=True) + fsops.sync_directory(session_dir) diff --git a/src/thirdeye/platforms/copilot/status.py b/src/thirdeye/platforms/copilot/status.py new file mode 100644 index 00000000..09381eb4 --- /dev/null +++ b/src/thirdeye/platforms/copilot/status.py @@ -0,0 +1,560 @@ +"""Local health reporting for the Copilot CLI capture archive.""" + +from __future__ import annotations + +import json +import time +from pathlib import Path +from typing import Any + +from thirdeye.config import Config +from thirdeye.paths import otel_jobs_dir, platform_dir, usage_log_path +from thirdeye.reader import SessionReader + +from . import export_transport +from .archive import _record_from_event +from .constants import FOLLOWUP_LEASE_FILENAME, PLATFORM_NAME +from .database import read_database +from .export_state import load_export_state +from .install import CopilotPlatform +from .projection_store import read_projection_status +from .runtime import load_runtime_status +from .spool import read_spool +from .state import journal_path, read_json, state_path +from .types import SourcePaths, SourceRecord + +_STATUS_PROBE_ID = "copilot-status-probe" +_FILE_LEVEL_DATABASE_CODES = frozenset( + { + "copilot_database_unreadable", + "copilot_database_busy", + "copilot_database_incompatible", + "copilot_database_read_failed", + } +) +_WORKER_TURN_KINDS = frozenset({"turn", "spans", "subagent_turn"}) + + +def _error_capability(path: Path, *, exists: bool, reason: str) -> dict[str, Any]: + return { + "path": str(path), + "exists": exists, + "readable": False, + "error": {"kind": "source_unreadable", "path": str(path), "reason": reason}, + } + + +def _missing_capability(path: Path) -> dict[str, Any]: + return {"path": str(path), "exists": False, "readable": False} + + +def _directory_capability(path: Path) -> dict[str, Any]: + """Describe a directory by enumerating it, not just by a successful stat().""" + + try: + exists = path.exists() + except OSError as error: + return _error_capability(path, exists=False, reason=type(error).__name__) + if not exists: + return _missing_capability(path) + if not path.is_dir(): + return _error_capability(path, exists=True, reason="not a directory") + try: + next(iter(path.iterdir()), None) + except OSError as error: + return _error_capability(path, exists=True, reason=type(error).__name__) + return {"path": str(path), "exists": True, "readable": True} + + +def _file_capability(path: Path) -> dict[str, Any]: + """Describe a regular file by opening it for a bounded read.""" + + try: + exists = path.exists() + except OSError as error: + return _error_capability(path, exists=False, reason=type(error).__name__) + if not exists: + return _missing_capability(path) + if not path.is_file(): + return _error_capability(path, exists=True, reason="not a file") + try: + with path.open("rb") as handle: + handle.read(1) + except OSError as error: + return _error_capability(path, exists=True, reason=type(error).__name__) + return {"path": str(path), "exists": True, "readable": True} + + +def _database_capability(paths: SourcePaths) -> dict[str, Any]: + """Open the session database read-only; absence is not a source error.""" + + path = Path(paths["database"]) + try: + exists = path.exists() + except OSError as error: + return _error_capability(path, exists=False, reason=type(error).__name__) + if not exists: + return _missing_capability(path) + try: + probe = read_database(paths, _STATUS_PROBE_ID, {}) + except (OSError, ValueError) as error: + return _error_capability(path, exists=True, reason=type(error).__name__) + for diagnostic in probe["diagnostics"]: + code = diagnostic.get("code") + if code == "copilot_database_missing": + return _missing_capability(path) + if isinstance(code, str) and code in _FILE_LEVEL_DATABASE_CODES: + return _error_capability(path, exists=True, reason=code) + if not path.is_file(): + return _error_capability(path, exists=True, reason="not a file") + return {"path": str(path), "exists": True, "readable": True} + + +def _archive_directories(config: Config, paths: SourcePaths) -> list[Path]: + root = platform_dir(config.root, PLATFORM_NAME) + prefix = f"copilot-{paths['source_key'][:16]}-" + try: + return sorted( + (entry for entry in root.iterdir() if entry.is_dir() and entry.name.startswith(prefix)), + key=lambda entry: entry.name, + ) + except OSError: + return [] + + +def _record_hook(record: SourceRecord, latest: SourceRecord | None) -> SourceRecord | None: + if record["source_kind"] != "hook": + return latest + if latest is None or record["observed_at"] > latest["observed_at"]: + return record + return latest + + +def _spool_file_errors(directory: Path) -> list[dict[str, Any]]: + """Surface spool .diag files (locations/reasons only, never payloads).""" + + errors: list[dict[str, Any]] = [] + try: + diagnostics = sorted(directory.glob("*.diag")) + except OSError as error: + return [ + { + "kind": "spool_unreadable", + "session": directory.name, + "path": str(directory), + "reason": type(error).__name__, + } + ] + for diag_path in diagnostics: + try: + payload = json.loads(diag_path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError): + errors.append( + { + "kind": "spool_unreadable", + "session": directory.name, + "path": diag_path.name, + } + ) + continue + reason = payload.get("reason") if isinstance(payload, dict) else None + loc = payload.get("path") if isinstance(payload, dict) else None + errors.append( + { + "kind": "spool_unreadable", + "session": directory.name, + "path": loc if isinstance(loc, str) else diag_path.name, + "reason": reason if isinstance(reason, str) else "invalid spool diagnostic", + } + ) + return errors + + +def _spool_sessions( + config: Config, paths: SourcePaths +) -> tuple[int, list[str], SourceRecord | None, list[dict[str, Any]]]: + root = Path(config.root) / "spool" / "copilot" / paths["source_key"] + count = 0 + sessions: list[str] = [] + latest: SourceRecord | None = None + errors: list[dict[str, Any]] = [] + try: + entries = sorted(root.iterdir()) + except FileNotFoundError: + return count, sessions, latest, errors + except OSError as error: + return ( + count, + sessions, + latest, + [ + { + "kind": "spool_unreadable", + "path": str(root), + "reason": type(error).__name__, + } + ], + ) + for entry in entries: + if not entry.is_dir(): + continue + try: + records = read_spool(config, paths, entry.name) + json_files = list(entry.glob("*.json")) + except (OSError, ValueError) as error: + errors.append( + { + "kind": "spool_unreadable", + "session": entry.name, + "reason": str(error) if isinstance(error, ValueError) else type(error).__name__, + } + ) + continue + if json_files: + sessions.append(entry.name) + count += len(json_files) + for record in records: + latest = _record_hook(record, latest) + errors.extend(_spool_file_errors(entry)) + return count, sessions, latest, errors + + +def _followup_lease_pending(directory: Path) -> bool: + """True when a live follow-up lease file exists beside the session.""" + + try: + payload = json.loads((directory / FOLLOWUP_LEASE_FILENAME).read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError): + return False + if not isinstance(payload, dict): + return False + expires_at = payload.get("expires_at") + if not isinstance(expires_at, (int, float)): + return False + return float(expires_at) > time.time() + + +def _json_object(path: Path) -> dict[str, Any] | None: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError): + return None + return value if isinstance(value, dict) else None + + +def _iter_json_jobs(directory: Path) -> list[dict[str, Any]]: + payloads: list[dict[str, Any]] = [] + try: + entries = list(directory.iterdir()) + except OSError: + return payloads + for path in entries: + if not path.is_file() or path.suffix != ".json" or path.name.endswith(".claim"): + continue + payload = _json_object(path) + if payload is not None: + payloads.append(payload) + return payloads + + +def _worker_kind(message: object) -> str: + text = str(message or "") + prefix = "kind=" + if not text.startswith(prefix): + return "" + return text[len(prefix) :].split(None, 1)[0] + + +def _iter_worker_turn_failures(config: Config, stored_session_id: str) -> list[dict[str, Any]]: + """Read deleted whole-turn export failures from the worker error log. + + Generic turn/spans/subagent jobs are unlinked before delivery. A crash or + export failure therefore leaves no job and no sent claim; the durable + breadcrumb is ``usage-errors.jsonl``. + """ + + path = usage_log_path(config.root) + try: + lines = path.read_text(encoding="utf-8").splitlines() + except (OSError, UnicodeDecodeError): + return [] + failures: list[dict[str, Any]] = [] + for line in lines: + if not line.strip(): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(entry, dict): + continue + if entry.get("phase") != "otel_worker_export_failed": + continue + if entry.get("session_id") != stored_session_id: + continue + if _worker_kind(entry.get("message")) not in _WORKER_TURN_KINDS: + continue + failures.append(entry) + return failures + + +def _export_health(config: Config, stored_session_id: str, directory: Path) -> dict[str, Any]: + """Summarize queue/delivery health from claims, jobs, and worker logs.""" + + ledger = load_export_state(config, stored_session_id) + placements = ledger.get("placements") if isinstance(ledger.get("placements"), dict) else {} + conflicts = ledger.get("conflicts") if isinstance(ledger.get("conflicts"), dict) else {} + turn_errors = ledger.get("turn_errors") if isinstance(ledger.get("turn_errors"), dict) else {} + + delivered: set[str] = set() + queued: set[str] = set() + errored: set[str] = set() + + for accounting_id, item in placements.items(): + if not isinstance(accounting_id, str) or not isinstance(item, dict): + continue + key = f"acct:{accounting_id}" + span_id = item.get("span_id") + sent = export_transport.delivery_sent(directory, accounting_id) + job = ( + export_transport.status(config.root, span_id) + if isinstance(span_id, str) and span_id + else None + ) + job_state = job.get("state") if job else None + if sent or item.get("emitted") or job_state == "emitted": + delivered.add(key) + elif job_state == "failed": + queued.add(key) + errored.add(key) + elif job_state in {"queued", "claimed", "retrying"}: + queued.add(key) + else: + queued.add(key) + if item.get("last_error") or (job and job.get("last_error")): + errored.add(key) + + for payload in _iter_json_jobs(export_transport.jobs_dir(config.root)): + if payload.get("session_id") != stored_session_id: + continue + accounting_id = payload.get("accounting_id") + key = ( + f"acct:{accounting_id}" + if isinstance(accounting_id, str) and accounting_id + else f"acct-job:{payload.get('job_id')}" + ) + if key in delivered: + continue + state = payload.get("state") + if state == "emitted" or ( + isinstance(accounting_id, str) + and export_transport.delivery_sent(directory, accounting_id) + ): + delivered.add(key) + queued.discard(key) + continue + queued.add(key) + if state == "failed" or payload.get("last_error"): + errored.add(key) + + sent_dir = directory / "otel-turns-sent" + try: + claim_files = list(sent_dir.iterdir()) if sent_dir.is_dir() else [] + except OSError: + claim_files = [] + for path in claim_files: + if not path.is_file(): + continue + try: + text = path.read_text(encoding="utf-8") + except OSError: + continue + key = f"turn-claim:{path.name}" + if text == "sent": + delivered.add(key) + else: + queued.add(key) + + for payload in _iter_json_jobs(otel_jobs_dir(config.root)): + if payload.get("session_id") != stored_session_id: + continue + if payload.get("kind") not in {"turn", "spans", "subagent_turn"}: + continue + queued.add(f"otel:{payload.get('job_id') or id(payload)}") + if payload.get("last_error"): + errored.add(f"otel:{payload.get('job_id') or id(payload)}") + + for index, entry in enumerate(_iter_worker_turn_failures(config, stored_session_id)): + key = f"otel-fail:{entry.get('ts') or index}:{entry.get('message')}" + queued.add(key) + errored.add(key) + + queued -= delivered + + runtime_status = load_runtime_status(config, stored_session_id) + last_error = runtime_status.get("last_error") + if isinstance(last_error, dict): + errored.add("runtime") + else: + last_error = None + + return { + "activated": bool(ledger.get("activated")), + "queued": len(queued), + "delivered": len(delivered), + "errors": len(errored) + len(conflicts) + len(turn_errors), + "last_error": last_error, + } + + +def _archive_status( + config: Config, paths: SourcePaths +) -> tuple[list[dict[str, Any]], SourceRecord | None, list[dict[str, Any]], int, int]: + """Read archive health/state without attempting an import or source read.""" + + sessions: list[dict[str, Any]] = [] + errors: list[dict[str, Any]] = [] + latest_hook: SourceRecord | None = None + pending_followup = 0 + active_leases = 0 + for directory in _archive_directories(config, paths): + try: + state = read_json(state_path(directory)) + except ValueError as error: + errors.append( + {"kind": "invalid_archive_state", "session": directory.name, "reason": str(error)} + ) + continue + if state is None: + state = {} + stored_key = state.get("source_key") + if isinstance(stored_key, str) and stored_key != paths["source_key"]: + errors.append( + { + "kind": "source_key_collision", + "session": directory.name, + "reason": "Copilot source-key prefix collision for stored session ID", + } + ) + continue + health = state.get("health") if isinstance(state.get("health"), dict) else {} + diagnostics = ( + health.get("diagnostics") if isinstance(health.get("diagnostics"), list) else [] + ) + if _followup_lease_pending(directory): + pending_followup += 1 + active_leases += 1 + session = { + "stored_session_id": directory.name, + "native_session_id": state.get("native_session_id"), + "cursor": state.get("cursor", {}), + "last_successful_import": health.get("last_successful_import"), + "diagnostics": diagnostics, + "journal_pending": journal_path(directory).is_file(), + } + try: + session["projection"] = read_projection_status(config, directory.name) + except Exception as error: + session["projection"] = {"errors": 1} + errors.append( + { + "kind": "copilot_projection_unreadable", + "session": directory.name, + "reason": type(error).__name__, + } + ) + try: + session["export"] = _export_health(config, directory.name, directory) + last_error = session["export"].get("last_error") + if isinstance(last_error, dict): + errors.append( + { + "kind": "copilot_reconcile_error", + "session": directory.name, + "reason": last_error.get("phase") or "copilot_reconcile", + "errors": last_error.get("errors"), + } + ) + except Exception as error: + session["export"] = {"activated": False, "queued": 0, "delivered": 0, "errors": 1} + errors.append( + { + "kind": "copilot_export_ledger_unreadable", + "session": directory.name, + "reason": type(error).__name__, + } + ) + sessions.append(session) + for diagnostic in diagnostics: + if isinstance(diagnostic, dict): + errors.append({"session": directory.name, **diagnostic}) + try: + for event in SessionReader(directory).iter_events(types=("copilot_hook",)): + record = _record_from_event(event) + if record is not None: + latest_hook = _record_hook(record, latest_hook) + except (OSError, ValueError): + errors.append({"kind": "archive_events_unreadable", "session": directory.name}) + return sessions, latest_hook, errors, pending_followup, active_leases + + +def capture_status(config: Config, paths: SourcePaths) -> dict: + """Return local capture installation, progress, and health information. + + An absent Copilot home/database and uninstalled owned hooks are reported as + capabilities/configuration, not errors. Archive/source diagnostics remain + in ``errors`` so a command layer can choose a nonzero status for them. + """ + + home = Path(paths["home"]) + session_root = _directory_capability(Path(paths["session_root"])) + database = _database_capability(paths) + wal = _file_capability(Path(paths["database"]).with_name(f"{Path(paths['database']).name}-wal")) + sessions, archived_hook, archive_errors, pending_followup, active_leases = _archive_status( + config, paths + ) + spool_count, spool_sessions, spooled_hook, spool_errors = _spool_sessions(config, paths) + last_hook = archived_hook + if spooled_hook is not None: + last_hook = _record_hook(spooled_hook, last_hook) + + source_errors = [ + item["error"] + for item in (session_root, database, wal) + if isinstance(item.get("error"), dict) + ] + last_successful = max( + ( + value + for session in sessions + if isinstance((value := session.get("last_successful_import")), str) + ), + default=None, + ) + installer = CopilotPlatform(source_home=home) + return { + "paths": dict(paths), + "installation": { + "configured": installer.is_installed(), + "hooks_file": str(installer.hooks_file), + }, + "capabilities": { + "transcripts": session_root, + "database": database, + "database_wal": wal, + }, + "last_observed_hook": last_hook, + "last_successful_import": last_successful, + "sessions": sessions, + "pending": { + "spool_records": spool_count, + "spool_sessions": spool_sessions, + "followup": pending_followup, + "leases": active_leases, + "journals": sum(1 for session in sessions if session["journal_pending"]), + }, + "errors": [*source_errors, *archive_errors, *spool_errors], + } + + +__all__ = ["capture_status"] diff --git a/src/thirdeye/platforms/copilot/tracing.py b/src/thirdeye/platforms/copilot/tracing.py new file mode 100644 index 00000000..16fdb7c6 --- /dev/null +++ b/src/thirdeye/platforms/copilot/tracing.py @@ -0,0 +1,58 @@ +"""Pure semantic projection for V1 Copilot archives.""" + +from __future__ import annotations + +from copy import deepcopy +from typing import Any + +from .events import normalize_records +from .turns import build_turns +from .types import ProjectionDiagnostic, SemanticProjection, SourceRecord + + +def build_semantics( + records: list[SourceRecord], prior_state: dict[str, Any] +) -> tuple[SemanticProjection, dict[str, Any]]: + """Replay immutable source records into semantic events and main turns. + + ``prior_state`` is not evidence. It retains still-open interactions + (including completed nested children) whose ``source_ids`` are absent + from this partition. Replaying the complete archive with an empty + prior state is always authoritative. + """ + events = normalize_records(records) + turns, call_candidates, pending, diagnostics, semantic_state = build_turns( + records, prior_state if isinstance(prior_state, dict) else {} + ) + diagnostics = [*_auxiliary_diagnostics(events), *diagnostics] + return ( + { + "events": events, + "turns": turns, + "call_candidates": call_candidates, + "pending": pending, + "diagnostics": diagnostics, + }, + deepcopy(semantic_state), + ) + + +def _auxiliary_diagnostics(events: list[dict[str, Any]]) -> list[ProjectionDiagnostic]: + found: list[ProjectionDiagnostic] = [] + for event in events: + if event.get("kind") != "auxiliary_model_call": + continue + attributes = event.get("attributes") or {} + found.append( + { + "code": "auxiliary_excluded_from_main", + "severity": "info", + "message": "title-generation model.* record classified auxiliary", + "source_ids": list(event.get("source_ids") or []), + "details": { + "classification": event.get("classification"), + "model": attributes.get("model"), + }, + } + ) + return found diff --git a/src/thirdeye/platforms/copilot/transcript.py b/src/thirdeye/platforms/copilot/transcript.py new file mode 100644 index 00000000..df0a7d56 --- /dev/null +++ b/src/thirdeye/platforms/copilot/transcript.py @@ -0,0 +1,596 @@ +"""Lossless, bounded reads of Copilot CLI transcript recordings. + +This module deliberately knows nothing about thirdeye's archive. It turns the +``events.jsonl`` and the small, adjacent ``workspace.yaml`` document into raw +source evidence that a later composition layer can commit durably. +""" + +from __future__ import annotations + +import base64 +import hashlib +import json +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import yaml + +from .constants import SOURCE_SCHEMA_VERSION +from .identity import validate_native_id +from .types import SourcePaths, SourceRecord, SourceSlice + +_EVENTS_FILENAME = "events.jsonl" +_WORKSPACE_FILENAME = "workspace.yaml" +_IO_CHUNK = 65536 + + +def _observed_at() -> str: + return datetime.now(UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z") + + +def _diagnostic(code: str, message: str, **locator: Any) -> dict[str, Any]: + return {"code": code, "message": message, "locator": locator} + + +def _within(candidate: Path, root: Path) -> bool: + try: + candidate.relative_to(root) + except ValueError: + return False + return True + + +def _session_directory(paths: SourcePaths, native_id: str) -> Path: + """Return a checked session directory without allowing path traversal.""" + + validate_native_id(native_id) + root = Path(paths["session_root"]).expanduser().resolve(strict=False) + session = (root / native_id).resolve(strict=False) + if not _within(session, root): + raise ValueError("native session ID escapes the Copilot session root") + return session + + +def _source_file_status(path: Path, directory: Path) -> tuple[Path | None, bool]: + """Return ``(resolved_file, escaped)`` for a candidate source file. + + Symlinks are resolved before use. A target outside *directory* is + rejected so injected source roots cannot archive arbitrary files. + """ + + try: + present = path.is_symlink() or path.exists() + except OSError: + return None, False + if not present: + return None, False + try: + resolved = path.resolve(strict=False) + if not _within(resolved, directory): + return None, True + if resolved.is_file(): + return resolved, False + except OSError: + return None, False + return None, False + + +def _generation(path: Path) -> str: + """Identify the current file object, while remaining stable for appends.""" + + stat = path.stat() + # st_dev/st_ino distinguishes most atomic replacements. Windows may + # quickly reuse st_ino after unlink/recreate, so include creation time when + # the platform exposes it. All three values remain stable for appends. + birthtime_ns = getattr(stat, "st_birthtime_ns", None) + suffix = f"-{birthtime_ns:x}" if isinstance(birthtime_ns, int) else "" + return f"{stat.st_dev:x}-{stat.st_ino:x}{suffix}" + + +def _valid_timestamp(value: Any) -> str | None: + if not isinstance(value, str) or not value: + return None + try: + datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + return value + + +def _prefix_digest(path: Path, offset: int) -> str: + """SHA-256 of the consumed prefix ``[0, offset)``.""" + + hasher = hashlib.sha256() + if offset <= 0: + return hasher.hexdigest() + with path.open("rb") as stream: + remaining = offset + while remaining: + chunk = stream.read(min(remaining, _IO_CHUNK)) + if not chunk: + break + hasher.update(chunk) + remaining -= len(chunk) + return hasher.hexdigest() + + +def _contains_complete_line(path: Path, start: int, end: int) -> bool: + """Return True when ``[start, end)`` contains a newline-terminated line.""" + + if end <= start: + return False + with path.open("rb") as stream: + stream.seek(start) + remaining = end - start + while remaining: + chunk = stream.read(min(remaining, _IO_CHUNK)) + if not chunk: + return False + if b"\n" in chunk: + return True + remaining -= len(chunk) + return False + + +def _json_value(value: Any) -> Any: + """Reject YAML-only values so metadata remains a JSON-compatible contract.""" + + if value is None or isinstance(value, (str, int, float, bool)): + return value + if isinstance(value, list): + return [_json_value(item) for item in value] + if isinstance(value, dict) and all(isinstance(key, str) for key in value): + return {key: _json_value(item) for key, item in value.items()} + raise TypeError(f"workspace metadata contains unsupported value {type(value).__name__}") + + +def _source_id( + paths: SourcePaths, + native_id: str, + event: dict[str, Any], + generation: str, + offset: int, + raw: bytes, +) -> tuple[str, dict[str, Any]]: + event_id = event.get("id") + if isinstance(event_id, (str, int)) and str(event_id): + value = str(event_id) + return ( + f"{paths['source_key']}/{native_id}/{value}", + {"native_event_id": value}, + ) + + digest = hashlib.sha256(raw).hexdigest() + return ( + f"{paths['source_key']}/{native_id}/{generation}/{offset}/{digest}", + {"content_digest": digest, "identity_confidence": "lower"}, + ) + + +def _record_for_line( + paths: SourcePaths, + native_id: str, + generation: str, + offset: int, + line: bytes, + observed_at: str, +) -> tuple[SourceRecord, dict[str, Any] | None]: + """Create evidence for one complete newline-terminated physical line.""" + + raw = line[:-1] if line.endswith(b"\n") else line + locator: dict[str, Any] = { + "file": _EVENTS_FILENAME, + "file_generation": generation, + "byte_offset": offset, + "byte_length": len(line), + } + try: + text = raw.decode("utf-8") + except UnicodeDecodeError: + digest = hashlib.sha256(raw).hexdigest() + locator["content_digest"] = digest + record: SourceRecord = { + "source_id": f"{paths['source_key']}/{native_id}/{generation}/{offset}/{digest}", + "source_kind": "transcript", + "native_session_id": native_id, + "ts": None, + "observed_at": observed_at, + "payload": { + "schema_version": SOURCE_SCHEMA_VERSION, + "malformed": "invalid_utf8", + "raw_bytes_base64": base64.b64encode(raw).decode("ascii"), + }, + "locator": locator, + } + return record, _diagnostic( + "transcript_invalid_utf8", "complete transcript line is not UTF-8", **locator + ) + + try: + value = json.loads(text) + except (json.JSONDecodeError, ValueError): + digest = hashlib.sha256(raw).hexdigest() + locator["content_digest"] = digest + record = { + "source_id": f"{paths['source_key']}/{native_id}/{generation}/{offset}/{digest}", + "source_kind": "transcript", + "native_session_id": native_id, + "ts": None, + "observed_at": observed_at, + "payload": { + "schema_version": SOURCE_SCHEMA_VERSION, + "malformed": "invalid_json", + "raw_line": text, + }, + "locator": locator, + } + return record, _diagnostic( + "transcript_invalid_json", "complete transcript line is not JSON", **locator + ) + + if not isinstance(value, dict): + digest = hashlib.sha256(raw).hexdigest() + locator["content_digest"] = digest + record = { + "source_id": f"{paths['source_key']}/{native_id}/{generation}/{offset}/{digest}", + "source_kind": "transcript", + "native_session_id": native_id, + "ts": None, + "observed_at": observed_at, + "payload": { + "schema_version": SOURCE_SCHEMA_VERSION, + "malformed": "non_object_json", + "raw_value": value, + }, + "locator": locator, + } + return record, _diagnostic( + "transcript_non_object", "complete transcript line is not a JSON object", **locator + ) + + source_id, event_locator = _source_id(paths, native_id, value, generation, offset, raw) + locator.update(event_locator) + timestamp = _valid_timestamp(value.get("timestamp")) + diagnostic = None + if value.get("timestamp") is not None and timestamp is None: + diagnostic = _diagnostic( + "transcript_invalid_timestamp", "event timestamp is not ISO-8601", **locator + ) + # Keep every top-level field, including unknown future fields. The schema + # version comes last so an untrusted event cannot alter our envelope. + payload = {**value, "schema_version": SOURCE_SCHEMA_VERSION} + record = { + "source_id": source_id, + "source_kind": "transcript", + "native_session_id": native_id, + "ts": timestamp, + "observed_at": observed_at, + "payload": payload, + "locator": locator, + } + return record, diagnostic + + +def _workspace_record( + paths: SourcePaths, native_id: str, directory: Path, observed_at: str +) -> tuple[SourceRecord | None, str | None, list[dict[str, Any]]]: + """Read only safe workspace metadata and return it as independent evidence.""" + + path = directory / _WORKSPACE_FILENAME + resolved, escaped = _source_file_status(path, directory) + if escaped: + return ( + None, + None, + [ + _diagnostic( + "workspace_path_escaped", + "workspace.yaml resolves outside the session directory", + file=str(path), + ) + ], + ) + if resolved is None: + return None, None, [] + try: + raw = resolved.read_bytes() + data = yaml.safe_load(raw.decode("utf-8")) + data = _json_value(data) + except (OSError, TypeError, UnicodeDecodeError, yaml.YAMLError) as exc: + return ( + None, + None, + [ + _diagnostic( + "workspace_metadata_invalid", + "workspace.yaml could not be safely parsed", + file=str(path), + reason=type(exc).__name__, + ) + ], + ) + if not isinstance(data, dict): + return ( + None, + None, + [ + _diagnostic( + "workspace_metadata_invalid", + "workspace.yaml must contain a mapping", + file=str(path), + ) + ], + ) + + digest = hashlib.sha256(raw).hexdigest() + try: + generation = _generation(resolved) + except OSError: + generation = f"content-{digest}" + cwd = data.get("cwd") + if not isinstance(cwd, str): + cwd = data.get("workspacePath") if isinstance(data.get("workspacePath"), str) else None + record: SourceRecord = { + "source_id": f"{paths['source_key']}/{native_id}/workspace/{digest}", + "source_kind": "metadata", + "native_session_id": native_id, + "ts": None, + "observed_at": observed_at, + "payload": { + "schema_version": SOURCE_SCHEMA_VERSION, + "file": _WORKSPACE_FILENAME, + "data": data, + }, + "locator": { + "file": _WORKSPACE_FILENAME, + "file_generation": generation, + "content_digest": digest, + }, + } + return record, cwd, [] + + +def discover_transcripts(paths: SourcePaths) -> list[str]: + """Return native session IDs with an immediately readable events file.""" + + root = Path(paths["session_root"]).expanduser().resolve(strict=False) + try: + candidates = list(root.iterdir()) + except OSError: + return [] + result: list[str] = [] + for candidate in candidates: + try: + resolved = candidate.resolve(strict=False) + if not _within(resolved, root) or not resolved.is_dir(): + continue + events_file = _source_file_status(resolved / _EVENTS_FILENAME, resolved)[0] + if events_file is None: + continue + validate_native_id(candidate.name) + except (OSError, ValueError): + continue + result.append(candidate.name) + return sorted(result) + + +def read_transcript( + paths: SourcePaths, + native_id: str, + cursor: dict, + *, + max_records: int = 1000, + max_bytes: int = 4_194_304, +) -> SourceSlice: + """Read a stable bounded snapshot of one transcript. + + A newline is the commit boundary: a trailing partial JSON or UTF-8 line is + left untouched for the next read. Cursors retain a snapshot endpoint so a + caller can drain the state observed at the start even while Copilot appends. + After that snapshot has only an incomplete tail left, the next call reopens + the endpoint to the current size so an appended newline can finish the line. + """ + + if max_records < 0 or max_bytes < 0: + raise ValueError("max_records and max_bytes must be non-negative") + directory = _session_directory(paths, native_id) + event_path = directory / _EVENTS_FILENAME + diagnostics: list[dict[str, Any]] = [] + observed_at = _observed_at() + workspace, cwd, workspace_diagnostics = _workspace_record( + paths, native_id, directory, observed_at + ) + diagnostics.extend(workspace_diagnostics) + records: list[SourceRecord] = [] + + resolved_events, events_escaped = _source_file_status(event_path, directory) + if events_escaped: + diagnostics.append( + _diagnostic( + "transcript_path_escaped", + "events.jsonl resolves outside the session directory", + file=str(event_path), + ) + ) + return { + "records": records, + "next_cursor": dict(cursor), + "diagnostics": diagnostics, + "cwd": cwd, + "exhausted": False, + } + if resolved_events is None: + diagnostics.append( + _diagnostic( + "transcript_unavailable", + "events.jsonl is unavailable; it is not considered complete", + file=str(event_path), + ) + ) + return { + "records": records, + "next_cursor": dict(cursor), + "diagnostics": diagnostics, + "cwd": cwd, + "exhausted": False, + } + + try: + stat = resolved_events.stat() + generation = _generation(resolved_events) + except OSError: + diagnostics.append( + _diagnostic( + "transcript_unavailable", + "events.jsonl is unavailable; it is not considered complete", + file=str(event_path), + ) + ) + return { + "records": records, + "next_cursor": dict(cursor), + "diagnostics": diagnostics, + "cwd": cwd, + "exhausted": False, + } + + size = stat.st_size + prior_generation = cursor.get("file_generation") + prior_offset = cursor.get("byte_offset", 0) + if not isinstance(prior_offset, int) or prior_offset < 0: + prior_offset = 0 + diagnostics.append( + _diagnostic( + "transcript_cursor_invalid", + "invalid byte offset; replaying transcript", + file=str(event_path), + ) + ) + reset = prior_generation is not None and prior_generation != generation + if prior_offset > size: + reset = True + prior_digest = cursor.get("continuity_digest") + if not reset and prior_offset and isinstance(prior_digest, str): + try: + current_digest = _prefix_digest(resolved_events, prior_offset) + except OSError: + current_digest = "" + if current_digest != prior_digest: + reset = True + if reset: + diagnostics.append( + _diagnostic( + "transcript_replaced", + "transcript was replaced or truncated; replaying from byte zero", + file=str(event_path), + previous_generation=prior_generation, + file_generation=generation, + ) + ) + prior_offset = 0 + + prior_end = cursor.get("snapshot_end") + if reset or not isinstance(prior_end, int) or prior_end < prior_offset: + snapshot_end = size + elif prior_offset < prior_end: + frozen_end = min(prior_end, size) + try: + reopen = not _contains_complete_line(resolved_events, prior_offset, frozen_end) + except OSError: + reopen = True + snapshot_end = size if reopen else frozen_end + else: + snapshot_end = size + + # Metadata is a snapshot too. It is included when changed, but it does + # not prevent an events-only caller from making progress at a tiny bound. + workspace_digest = workspace["locator"]["content_digest"] if workspace else None + workspace_emitted = False + if ( + workspace is not None + and cursor.get("workspace_digest") != workspace_digest + and max_records > 0 + ): + records.append(workspace) + workspace_emitted = True + + offset = prior_offset + consumed = 0 + limit_hit = False + try: + with resolved_events.open("rb") as stream: + stream.seek(offset) + while offset < snapshot_end: + remaining = snapshot_end - offset + line = stream.readline(remaining) + if not line or not line.endswith(b"\n"): + # A line extending beyond the observed snapshot is not yet + # a source record, even if its prefix happens to decode. + break + if len(records) >= max_records: + limit_hit = True + break + line_size = len(line) + if consumed and consumed + line_size > max_bytes: + limit_hit = True + break + if not consumed and line_size > max_bytes: + diagnostics.append( + _diagnostic( + "transcript_record_oversize", + "one complete transcript record exceeds max_bytes and was accepted for progress", + file=str(event_path), + byte_offset=offset, + byte_length=line_size, + max_bytes=max_bytes, + ) + ) + record, diagnostic = _record_for_line( + paths, native_id, generation, offset, line, observed_at + ) + records.append(record) + if diagnostic is not None: + diagnostics.append(diagnostic) + offset += line_size + consumed += line_size + except OSError: + diagnostics.append( + _diagnostic( + "transcript_read_failed", + "events.jsonl could not be read; it is not considered complete", + file=str(event_path), + ) + ) + return { + "records": records, + "next_cursor": dict(cursor), + "diagnostics": diagnostics, + "cwd": cwd, + "exhausted": False, + } + + next_cursor: dict[str, Any] = { + "byte_offset": offset, + "file_generation": generation, + "snapshot_end": snapshot_end, + "continuity_start": 0, + } + try: + next_cursor["continuity_digest"] = _prefix_digest(resolved_events, offset) + except OSError: + # The read above remains useful. A later invocation will report the + # unavailable source instead of pretending that it reached completion. + pass + if workspace_digest is not None and ( + workspace_emitted or cursor.get("workspace_digest") == workspace_digest + ): + next_cursor["workspace_digest"] = workspace_digest + exhausted = offset >= snapshot_end and not limit_hit + return { + "records": records, + "next_cursor": next_cursor, + "diagnostics": diagnostics, + "cwd": cwd, + "exhausted": exhausted, + } diff --git a/src/thirdeye/platforms/copilot/turns.py b/src/thirdeye/platforms/copilot/turns.py new file mode 100644 index 00000000..defbdf74 --- /dev/null +++ b/src/thirdeye/platforms/copilot/turns.py @@ -0,0 +1,724 @@ +"""Explicit-identity Copilot interaction and recursive child-tree assembly. + +Callers that partition an archive must include every record belonging to +still-open interactions. ``prior_state`` retains those open keys, including +completed nested child spans, when their ``source_ids`` are absent from the +current partition. A complete archive replay with ``prior_state={}`` is +always the authoritative result. +""" + +from __future__ import annotations + +from copy import deepcopy +from typing import Any + +from thirdeye.tracing.model import LlmCallSpanDict, ToolCallSpanDict, TurnSpanDict + +from .events import ( + agent_id, + interaction_id, + record_data, + record_type, + source_reference, + tool_call_id, + unknown_source_schema, +) +from .types import CallCandidate, PendingItem, ProjectionDiagnostic, SourceRecord + +_ABORT_TYPES = frozenset({"assistant.abort", "session.abort", "abort"}) +_PUBLIC_CANDIDATE_KEYS = ( + "call_id", + "stored_turn_id", + "interaction_id", + "agent_id", + "parent_tool_call_id", + "model", + "source_ids", + "source_references", + "start_ts", + "end_ts", + "tool_call_ids", + "finish_evidence", +) + + +def _source_key(record: SourceRecord) -> str: + return record["source_id"].split("/", 1)[0] + + +def _turn_id(record: SourceRecord, interaction: str, agent: str | None) -> str: + base = f"copilot:turn:{_source_key(record)}:{record['native_session_id']}:{interaction}" + if agent: + return f"{base}:{agent}" + return base + + +def _key(interaction: str, agent: str | None) -> str: + return f"{interaction}|{agent or 'main'}" + + +def _public_candidate(raw: dict[str, Any]) -> CallCandidate: + return {key: raw[key] for key in _PUBLIC_CANDIDATE_KEYS} # type: ignore[return-value] + + +def _text_parts(text: str) -> list[dict[str, Any]]: + return [{"type": "text", "content": text}] if text else [] + + +def _reasoning_parts(summary: str | None) -> list[dict[str, Any]]: + if not isinstance(summary, str) or not summary: + return [] + return [{"type": "reasoning", "content": summary}] + + +def _missing_identity(record: SourceRecord, reason: str, evidence: str) -> PendingItem: + return { + "id": f"pending:identity:{record['source_id']}", + "kind": "missing_identity", + "reason": reason, + "source_ids": [record["source_id"]], + "evidence": [evidence], + } + + +def _incomplete_tool(record: SourceRecord, call: str, reason: str) -> PendingItem: + return { + "id": f"pending:tool:{call}", + "kind": "incomplete_tool_pair", + "reason": reason, + "source_ids": [record["source_id"]], + "evidence": [f"tool_call_id:{call}"], + } + + +def _collect_nested_turn_ids(turns: list[TurnSpanDict]) -> set[str]: + found: set[str] = set() + stack = list(turns) + while stack: + turn = stack.pop() + found.add(turn["turn_id"]) + stack.extend(turn.get("subagents") or []) + return found + + +def build_turns( + records: list[SourceRecord], + prior_state: dict[str, Any] | None = None, +) -> tuple[ + list[TurnSpanDict], + list[CallCandidate], + list[PendingItem], + list[ProjectionDiagnostic], + dict[str, Any], +]: + """Build completed main turns and recursive child spans from transcript IDs. + + ``prior_state`` is not evidence. Unfinished interactions keep the + ``OpenInteractionState`` shape; reconstructing a partition still requires + the records that belong to those interactions. + """ + interactions: dict[str, dict[str, Any]] = {} + active_native_turns: dict[tuple[str | None, str, str], str] = {} + child_parent: dict[str, tuple[str, str]] = {} + tool_owner: dict[str, str] = {} + calls: dict[str, dict[str, Any]] = {} + tool_spans: dict[str, ToolCallSpanDict] = {} + pending: list[PendingItem] = [] + diagnostics: list[ProjectionDiagnostic] = [] + prior = prior_state if isinstance(prior_state, dict) else {} + raw_open = prior.get("open_interactions") + prior_open = raw_open if isinstance(raw_open, dict) else {} + + def ensure(record: SourceRecord, interaction: str, agent: str | None) -> dict[str, Any]: + key = _key(interaction, agent) + if key not in interactions: + interactions[key] = { + "key": key, + "interaction": interaction, + "agent": agent, + "turn_id": _turn_id(record, interaction, agent), + "source_ids": [], + "start_ts": record.get("ts"), + "end_ts": None, + "input": "", + "output": "", + "calls": [], + "permission_requests": [], + "status": "completed", + "complete": False, + "saw_final_answer": False, + } + item = interactions[key] + item["source_ids"].append(record["source_id"]) + if item["start_ts"] is None: + item["start_ts"] = record.get("ts") + return item + + def remember_parent(child: str | None, parent_call: str | None) -> None: + if child and parent_call and parent_call in tool_owner: + child_parent[child] = (tool_owner[parent_call], parent_call) + + def bind_native_turn( + item: dict[str, Any], agent: str | None, interaction: str | None, native_turn: Any + ) -> None: + if interaction is None or native_turn is None: + return + active_native_turns[(agent, interaction, str(native_turn))] = item["key"] + + def complete_item(item: dict[str, Any], ts: str | None) -> None: + item["complete"] = True + item["end_ts"] = ts or item["end_ts"] + + def attach_finish( + item: dict[str, Any], record: SourceRecord, kind: str, native_turn: Any + ) -> dict[str, Any] | None: + turn_key = str(native_turn) if native_turn is not None else None + matches: list[dict[str, Any]] = [] + if turn_key is not None: + for call_id in item["calls"]: + candidate = calls[call_id] + if candidate["finish_evidence"]: + continue + if candidate.get("native_turn_id") == turn_key: + matches.append(candidate) + if turn_key is None or len(matches) != 1: + pending.append( + { + "id": f"pending:identity:{record['source_id']}", + "kind": "missing_identity", + "reason": f"{kind} has no uniquely matching open model cycle", + "source_ids": [record["source_id"]], + "evidence": [f"turn_id:{native_turn}"], + } + ) + return None + last_call = matches[0] + last_call["end_ts"] = record.get("ts") + if record["source_id"] not in last_call["source_ids"]: + last_call["source_ids"].append(record["source_id"]) + last_call["source_references"].append(source_reference(record, "finish")) + last_call["finish_evidence"].append( + {"source_id": record["source_id"], "kind": kind, "value": None} + ) + return last_call + + def owner_for_native_turn( + agent: str | None, native_turn: Any, interaction: str | None + ) -> str | None: + if native_turn is None: + return None + turn_key = str(native_turn) + if interaction is not None: + return active_native_turns.get((agent, interaction, turn_key)) + matches = [ + key + for (bound_agent, _bound_ix, bound_turn), key in active_native_turns.items() + if bound_agent == agent and bound_turn == turn_key + ] + unique = list(dict.fromkeys(matches)) + if len(unique) == 1: + return unique[0] + return None + + def drop_native_turn(agent: str | None, native_turn: Any, owner: str | None) -> None: + if native_turn is None or owner is None: + return + turn_key = str(native_turn) + for bind_key, bind_owner in list(active_native_turns.items()): + if bind_owner == owner and bind_key[0] == agent and bind_key[2] == turn_key: + active_native_turns.pop(bind_key, None) + + for record in records: + if record.get("source_kind") != "transcript": + continue + if unknown_source_schema(record): + payload = record.get("payload") + version = payload.get("schema_version") if isinstance(payload, dict) else None + pending.append( + { + "id": f"pending:unknown-version:{record['source_id']}", + "kind": "missing_source_capability", + "reason": "unknown transcript schema_version", + "source_ids": [record["source_id"]], + "evidence": [f"schema_version:{version}"], + } + ) + diagnostics.append( + { + "code": "capability_gap", + "severity": "warning", + "message": "unknown transcript schema_version; raw payload retained", + "source_ids": [record["source_id"]], + "details": {"schema_version": version}, + } + ) + continue + native_type = record_type(record) + data = record_data(record) + agent = agent_id(record) + interaction = interaction_id(record) + + if native_type == "subagent.started": + remember_parent(agent, tool_call_id(record)) + continue + + if native_type == "subagent.completed": + child = agent + parent_call = tool_call_id(record) + remember_parent(child, parent_call) + if child: + for item in interactions.values(): + if item["agent"] == child and not item["complete"]: + complete_item(item, record.get("ts")) + continue + + if native_type == "user.message": + if interaction is None: + pending.append( + _missing_identity( + record, + "user message has no interactionId", + "native_type:user.message", + ) + ) + continue + item = ensure(record, interaction, agent) + item["input"] = str(data.get("content") or item["input"]) + continue + + if native_type == "assistant.turn_start": + if interaction is None: + pending.append( + _missing_identity( + record, + "assistant turn has no interactionId", + "native_type:assistant.turn_start", + ) + ) + continue + item = ensure(record, interaction, agent) + bind_native_turn(item, agent, interaction, data.get("turnId")) + continue + + if native_type == "assistant.message": + if interaction is None: + pending.append( + _missing_identity( + record, + "assistant message has no interactionId", + "native_type:assistant.message", + ) + ) + continue + item = ensure(record, interaction, agent) + remember_parent(agent, data.get("parentToolCallId")) + call_id = f"copilot:call:{record['source_id']}" + requests = ( + data.get("toolRequests") if isinstance(data.get("toolRequests"), list) else [] + ) + requested_ids = [ + str(request["toolCallId"]) + for request in requests + if isinstance(request, dict) and request.get("toolCallId") + ] + content = data.get("content") if isinstance(data.get("content"), str) else "" + reasoning = data.get("reasoningSummary") + native_turn = data.get("turnId") + bind_native_turn(item, agent, interaction, native_turn) + candidate = { + "call_id": call_id, + "stored_turn_id": item["turn_id"], + "interaction_id": interaction, + "agent_id": agent, + "parent_tool_call_id": data.get("parentToolCallId"), + "model": data.get("model"), + "source_ids": [record["source_id"]], + "source_references": [source_reference(record, "assistant_message")], + "start_ts": record.get("ts"), + "end_ts": None, + "tool_call_ids": requested_ids, + "finish_evidence": [], + "interaction_key": item["key"], + "native_turn_id": str(native_turn) if native_turn is not None else None, + "content": content, + "reasoning_summary": reasoning if isinstance(reasoning, str) else None, + "final_answer": data.get("phase") == "final_answer", + } + calls[call_id] = candidate + item["calls"].append(call_id) + if content: + item["output"] = content + if candidate["final_answer"]: + item["saw_final_answer"] = True + item["status"] = "completed" + for request in requests: + if not isinstance(request, dict) or not request.get("toolCallId"): + continue + call = str(request["toolCallId"]) + tool_owner[call] = item["key"] + tool_spans[call] = { + "tool_call_id": call, + "name": str(request.get("name") or ""), + "start_ts": str(record.get("ts") or ""), + "end_ts": "", + "attributes": { + "arguments": deepcopy(request.get("arguments")), + "intention_summary": request.get("intentionSummary"), + "request_source_id": record["source_id"], + }, + } + continue + + if native_type == "tool.execution_start": + call = tool_call_id(record) + if call and call in tool_spans: + span = tool_spans[call] + span["start_ts"] = str(record.get("ts") or span["start_ts"]) + span["attributes"].update( + { + "arguments": deepcopy(data.get("arguments")), + "execution_start_source_id": record["source_id"], + } + ) + elif call: + pending.append( + _incomplete_tool( + record, + call, + "tool execution start has no requesting assistant message", + ) + ) + continue + + if native_type in { + "tool.execution_complete", + "tool.execution_error", + "tool.execution_failure", + }: + call = tool_call_id(record) + if call and call in tool_spans: + span = tool_spans[call] + span["end_ts"] = str(record.get("ts") or "") + span["attributes"].update( + { + "result": deepcopy(data.get("result")), + "success": data.get("success"), + "execution_result_source_id": record["source_id"], + } + ) + elif call: + pending.append( + _incomplete_tool( + record, + call, + "tool result has no requesting assistant message", + ) + ) + continue + + if native_type in {"permission.request", "permissionRequest"}: + if interaction is None: + continue + item = ensure(record, interaction, agent) + item["permission_requests"].append( + { + "ts": str(record.get("ts") or ""), + "tool_name": str(data.get("toolName") or ""), + "attributes": { + "arguments": deepcopy(data.get("toolArgs")), + "source_id": record["source_id"], + }, + } + ) + continue + + if native_type in {"permission.decision", "permissionDecision"}: + if interaction is None: + continue + item = ensure(record, interaction, agent) + decision = data.get("decision") + tool_name = data.get("toolName") + for request in reversed(item["permission_requests"]): + if tool_name and request["tool_name"] != tool_name: + continue + request["attributes"]["decision"] = decision + request["attributes"]["decision_source_id"] = record["source_id"] + break + continue + + is_abort = native_type in _ABORT_TYPES + is_error = ( + bool(native_type) + and "error" in native_type.lower() + and not native_type.startswith("model.") + and not native_type.startswith("tool.") + ) + if is_abort or is_error: + native_turn = data.get("turnId") + owner = owner_for_native_turn(agent, native_turn, interaction) + if owner is None and interaction is not None: + owner = ( + _key(interaction, agent) if _key(interaction, agent) in interactions else None + ) + drop_native_turn(agent, native_turn, owner) + if owner and owner in interactions: + item = interactions[owner] + item["status"] = "interrupted" if is_abort else "errored" + item["source_ids"].append(record["source_id"]) + attach_finish(item, record, "abort" if is_abort else "error", native_turn) + # Abort closes the user turn. An error leaves it open so a + # later model cycle in the same interaction can retry. + if is_abort: + complete_item(item, record.get("ts")) + continue + + if native_type == "assistant.turn_end": + native_turn = data.get("turnId") + owner = owner_for_native_turn(agent, native_turn, interaction) + drop_native_turn(agent, native_turn, owner) + if owner is None: + pending.append( + { + "id": f"pending:identity:{record['source_id']}", + "kind": "missing_identity", + "reason": "assistant.turn_end has no matching open model cycle", + "source_ids": [record["source_id"]], + "evidence": [f"turn_id:{native_turn}"], + } + ) + continue + item = interactions[owner] + item["source_ids"].append(record["source_id"]) + item["end_ts"] = record.get("ts") + last_call = attach_finish(item, record, "assistant_turn_end", native_turn) + if ( + last_call is not None + and not last_call["tool_call_ids"] + and last_call.get("final_answer") + ): + complete_item(item, record.get("ts")) + continue + + if native_type in {"session.shutdown", "session.end", "session.close"}: + for item in interactions.values(): + if item["complete"]: + continue + if item["saw_final_answer"] or item["status"] != "completed": + complete_item(item, record.get("ts")) + continue + + def call_span(candidate: dict[str, Any], user_input: str) -> LlmCallSpanDict: + attached = [tool_spans[tool] for tool in candidate["tool_call_ids"] if tool in tool_spans] + output_parts = _text_parts(str(candidate.get("content") or "")) + _reasoning_parts( + candidate.get("reasoning_summary") + ) + return { + "call_id": candidate["call_id"], + "provider": "unknown", + "model": str(candidate.get("model") or "unknown"), + "start_ts": str(candidate.get("start_ts") or ""), + "end_ts": str(candidate.get("end_ts") or candidate.get("start_ts") or ""), + "input_messages": ( + [{"role": "user", "parts": _text_parts(user_input)}] if user_input else [] + ), + "output_messages": ( + [{"role": "assistant", "parts": output_parts}] if output_parts else [] + ), + "usage": {}, + "tool_calls": attached, + } + + spans: dict[str, TurnSpanDict] = {} + for key, item in interactions.items(): + if not item["complete"]: + continue + spans[key] = { + "turn_id": item["turn_id"], + "start_ts": str(item["start_ts"] or ""), + "end_ts": str(item["end_ts"] or item["start_ts"] or ""), + "input_message": item["input"], + "output_message": item["output"], + "status": item["status"], + "llm_calls": [call_span(calls[call], item["input"]) for call in item["calls"]], + "permission_requests": item["permission_requests"], + "subagents": [], + "attributes": { + "interaction_id": item["interaction"], + "agent_id": item["agent"], + }, + "accounting_calls": [], + } + + for child, (parent_key, parent_call) in child_parent.items(): + child_items = [ + item for item in interactions.values() if item["agent"] == child and item["complete"] + ] + for item in child_items: + child_span = spans.get(item["key"]) + if child_span is None: + continue + child_span["attributes"]["parent_tool_call_id"] = parent_call + parent_span = spans.get(parent_key) + if parent_span is not None: + parent_span["subagents"].append(child_span) + + def nested_children_for(parent_key: str) -> list[TurnSpanDict]: + found: list[TurnSpanDict] = [] + seen: set[str] = set() + for child, (pkey, _parent_call) in child_parent.items(): + if pkey != parent_key: + continue + for item in interactions.values(): + if item["agent"] != child or not item["complete"]: + continue + child_span = spans.get(item["key"]) + if child_span is None or child_span["turn_id"] in seen: + continue + found.append(deepcopy(child_span)) + seen.add(child_span["turn_id"]) + prior_item = prior_open.get(parent_key) if isinstance(prior_open, dict) else None + if isinstance(prior_item, dict): + for child in prior_item.get("nested_children") or []: + if not isinstance(child, dict): + continue + turn_id = child.get("turn_id") + if not isinstance(turn_id, str) or turn_id in seen: + continue + found.append(deepcopy(child)) + seen.add(turn_id) + return found + + for key, span in spans.items(): + existing = {child["turn_id"] for child in span.get("subagents") or []} + for child in nested_children_for(key): + if child["turn_id"] in existing: + continue + span.setdefault("subagents", []).append(child) + existing.add(child["turn_id"]) + + emitted_ids = _collect_nested_turn_ids( + [span for key, span in spans.items() if interactions[key]["agent"] is None] + ) + for item in interactions.values(): + if not item["complete"] or item["agent"] is None: + continue + if item["turn_id"] in emitted_ids: + continue + pending.append( + { + "id": f"pending:identity:{item['turn_id']}", + "kind": "missing_identity", + "reason": "completed child interaction has no resolved parent tool call", + "source_ids": item["source_ids"], + "evidence": [ + f"interaction_id:{item['interaction']}", + f"agent_id:{item['agent']}", + ], + } + ) + diagnostics.append( + { + "code": "capability_gap", + "severity": "warning", + "message": "completed child interaction has no resolved parent tool call", + "source_ids": item["source_ids"], + "details": { + "interaction_id": item["interaction"], + "agent_id": item["agent"], + "stored_turn_id": item["turn_id"], + }, + } + ) + for call_id in item["calls"]: + calls[call_id]["stored_turn_id"] = None + + open_state: dict[str, Any] = {} + current_source_ids = {record["source_id"] for record in records} + for item in interactions.values(): + if item["complete"]: + continue + pending_tools = [ + tool + for call in item["calls"] + for tool in calls[call]["tool_call_ids"] + if not tool_spans.get(tool, {}).get("end_ts") + ] + has_finish = any(calls[call]["finish_evidence"] for call in item["calls"]) + reason = ( + "user interaction has no completed final assistant turn" + if has_finish + else "user interaction has no assistant.turn_end" + ) + retained: dict[str, Any] = { + "interaction_id": item["interaction"], + "agent_id": item["agent"], + "stored_turn_id": item["turn_id"], + "source_ids": item["source_ids"], + "last_event_source_id": item["source_ids"][-1] if item["source_ids"] else None, + "start_ts": item["start_ts"], + "pending_tool_call_ids": pending_tools, + } + nested = nested_children_for(item["key"]) + if nested: + retained["nested_children"] = nested + open_state[item["key"]] = retained + pending.append( + { + "id": f"pending:{item['turn_id']}", + "kind": "open_interaction", + "reason": reason, + "source_ids": item["source_ids"], + "evidence": [f"interaction_id:{item['interaction']}"], + } + ) + diagnostics.append( + { + "code": "open_interaction", + "severity": "info", + "message": reason, + "source_ids": item["source_ids"], + "details": {"interaction_id": item["interaction"]}, + } + ) + + seen_incomplete_tools = { + item["id"] for item in pending if item["kind"] == "incomplete_tool_pair" + } + for candidate in calls.values(): + for tool in candidate["tool_call_ids"]: + span = tool_spans.get(tool) + if span and span.get("end_ts"): + continue + pending_id = f"pending:tool:{tool}" + if pending_id in seen_incomplete_tools: + continue + source_ids = list(candidate["source_ids"][:1]) + request_source = (span or {}).get("attributes", {}).get("request_source_id") + if request_source: + source_ids = [request_source] + pending.append( + { + "id": pending_id, + "kind": "incomplete_tool_pair", + "reason": "tool request has no execution result", + "source_ids": source_ids, + "evidence": [f"tool_call_id:{tool}"], + } + ) + seen_incomplete_tools.add(pending_id) + + if isinstance(prior_open, dict): + for key, prior_item in prior_open.items(): + if key in open_state or not isinstance(prior_item, dict): + continue + prior_sources = prior_item.get("source_ids") or [] + if prior_sources and any( + source_id in current_source_ids for source_id in prior_sources + ): + continue + open_state[key] = deepcopy(prior_item) + + main_turns = [span for key, span in spans.items() if interactions[key]["agent"] is None] + main_turns.sort(key=lambda turn: (turn["start_ts"] == "", turn["start_ts"])) + call_candidates = [_public_candidate(raw) for raw in calls.values()] + return main_turns, call_candidates, pending, diagnostics, {"open_interactions": open_state} diff --git a/src/thirdeye/platforms/copilot/types.py b/src/thirdeye/platforms/copilot/types.py new file mode 100644 index 00000000..0eea0059 --- /dev/null +++ b/src/thirdeye/platforms/copilot/types.py @@ -0,0 +1,595 @@ +"""Versioned source envelopes (V1) and derived projection contracts (V2). + +V1 TypedDicts describe immutable archived evidence. They remain the input to +every V2 algorithm. V2 TypedDicts describe replaceable derived state: +normalized events, turn reconstruction, independent database accounting, and +attribution. A source ID, database generation, and content revision are +always retained alongside a derived result rather than being replaced by a +timestamp or import-order identity. +""" + +from __future__ import annotations + +from typing import Any, Literal, NotRequired, TypedDict + +from thirdeye.tracing.model import TurnSpanDict +from thirdeye.usage.types import UsageRow + +from .constants import SOURCE_SCHEMA_VERSION + +# The value placed beside every archived SourceRecord in a thirdeye event data +# envelope. It is deliberately separate from a Copilot CLI version. +SCHEMA_VERSION = SOURCE_SCHEMA_VERSION + +# Derived projection state is versioned independently of the V1 envelope. +# Rebuilds may drop and recreate this document without rewriting raw events. +PROJECTION_SCHEMA_VERSION = 2 + + +class SourcePaths(TypedDict): + """Canonical source-home identity and its Copilot recording locations.""" + + home: str + source_key: str + session_root: str + database: str + + +class SourceRecord(TypedDict): + """One immutable observation from a Copilot source domain.""" + + source_id: str + source_kind: str # transcript | database | hook | metadata + native_session_id: str + ts: str | None # Source time when valid; never invented. + observed_at: str + payload: dict[str, Any] + locator: dict[str, Any] + + +class SourceBatch(TypedDict): + """The composed capture boundary consumed by the durable archive.""" + + source_key: str + native_session_id: str + cwd: str | None + records: list[SourceRecord] + next_cursor: dict[str, Any] + diagnostics: list[dict[str, Any]] + + +class SyncResult(TypedDict): + """Counts returned by a capture operation.""" + + sessions: int + records_written: int + duplicate_records: int + pending: int + errors: int + + +class SourceSlice(TypedDict): + """A bounded read from one source; composed into a :class:`SourceBatch`.""" + + records: list[SourceRecord] + next_cursor: dict[str, Any] + diagnostics: list[dict[str, Any]] + cwd: str | None + exhausted: bool + + +# V2 projection contracts --------------------------------------------------- +# +# Stable derived identities use native/source identities, never import time or +# mutable list indices. Encoding rules (see also the reconciliation README): +# +# Transcript source IDs are V1 ``{full_source_key}/{native_id}/{event_id}``. +# Database source IDs are V1 +# ``copilot-db:{full_source_key}:{native_id}:{table}:{quoted_canonical_pk}:{revision}`` +# and do not embed generation. Hook source IDs are V1 +# ``hook/{native_id}/{observation_id}``. +# +# ```` in every derived ID is the full 64-character SHA-256 source +# home digest, not the 16-character prefix used only in stored session IDs. +# +# Semantic event: ``copilot:event:`` for the primary event of that +# record. When one source record yields extra events, append a native suffix: +# ``copilot:event::tool:`` for each +# ``data.toolRequests[].toolCallId`` on ``assistant.message``. A tool +# start/complete pair already has two source IDs, so each uses the unsuffixed +# form. If no native suffix exists, append ``:``. +# +# Semantic call: ``copilot:call:``. +# Main turn: ``copilot:turn:::``. +# Child turn: append ``:`` so two agents that share an +# ``interactionId`` cannot collide. Main interactions omit the suffix +# because ``agent_id`` is null. +# +# Logical database call / accounting span: +# ``copilot:usage::::``. +# ``quoted-generation`` and ``quoted-canonical-pk`` are +# ``urllib.parse.quote(..., safe="")`` of the locator generation and of +# ``json.dumps(locator.primary_key, sort_keys=True, separators=(",", ":"))`` +# respectively, matching V1 percent-quoting of composite keys. Generation is +# ``sha256:`` and therefore contains a colon; quoting turns that colon +# into ``%3A``. Content revision is excluded. Because V1 source IDs omit +# generation, an identical row seen in a new database file deduplicates to the +# first archived record; the logical ID uses that first record's locator +# generation. A different generation with different content is a new logical +# call (row-ID reuse), never a relabel of the previous call. +# +# ``Attribution.usage_source_id`` is the newest revision's source ID. +# Durable attribution is keyed by ``logical_call_id``. +# +# ``UsageRow.call_id`` is that same durable logical call identity +# (``copilot:usage:...``), never a revision source ID, import sequence, or +# Store seq. Persistence keys the usage index and ``usage.jsonl`` by this +# value and stamps ``UsageRow.seq`` with the Store seq of the newest archived +# revision. A content correction replaces the existing row under that +# identity. Storage also keeps a ``usage_source_id -> logical_call_id`` map +# so a row committed before its attribution/accounting identity is known is +# re-keyed rather than counted twice. +# +# Copilot ``TurnSpanDict.attributes`` carries ``interaction_id`` (native user +# request) and ``agent_id`` (null/absent on a main interaction; non-null on a +# nested child that must not become a generic user turn). Copilot +# ``NormalizedEvent.attributes`` carries ``interaction_id``, optional +# ``stored_turn_id``, and ``agent_id``. Producers may also set auxiliary +# ``source_ids`` / ``source_references`` on a turn, llm call, subagent, or +# accounting call; storage joins those plus matching normalized events onto +# the archived Store records for ``read_projected_turns``. Missing evidence +# yields ``events: []`` and null seq/ts rather than guessing. +# +# ``commit_projection``'s ``next_state`` is an authoritative +# :class:`ProjectionState` snapshot: omitted keys stay at their empty +# defaults, and a now-closed interaction is absent from +# ``semantic_state.open_interactions``. Storage does not resurrect keys the +# builder removed. ``pending`` and ``diagnostics`` on a Projection replace +# those indexes in full (status is current, not cumulative). Semantic +# events, turns, usage, and attributions merge by stable id. +# +# Storage entry points: ``commit_projection``, ``load_projection_state``, +# ``read_projected_turns``, and ``reset_projection_state`` (deletes only +# rebuildable local projection files and the derived usage sidecar; never +# V1 evidence or the export ledger). + +AttributionStatus = Literal["matched", "pending", "ambiguous", "conflicting"] +AttributionJoinKind = Literal["direct", "inferred"] +DiagnosticSeverity = Literal["info", "warning", "error"] +EventClassification = Literal["main", "title_generation", "checkpoint", "shutdown_validation"] +Initiator = Literal["user", "agent", "sub-agent"] + +NormalizedEventKind = Literal[ + "user_prompt", + "assistant_message", + "assistant_turn_start", + "assistant_turn_end", + "tool_request", + "tool_execution_start", + "tool_execution_complete", + "tool_execution_failure", + "permission_request", + "permission_decision", + "notification", + "prompt_transformation", + "compaction", + "abort", + "error", + "agent_stop", + "session_start", + "session_end", + "session_shutdown", + "subagent_started", + "subagent_completed", + "auxiliary_model_call", + "unknown", +] + +SourceReferenceRole = Literal[ + "user_prompt", + "assistant_message", + "tool_request", + "tool_execution", + "tool_result", + "permission_request", + "permission_decision", + "usage_row", + "finish", + "hook", + "shutdown", + "checkpoint", + "compaction", + "auxiliary_model", + "nested_child", +] + +FinishEvidenceKind = Literal[ + "assistant_turn_end", + "agent_stop_hook", + "finish_reason_db", + "abort", + "error", +] + +PendingItemKind = Literal[ + "open_interaction", + "missing_usage", + "unmatched_usage", + "missing_source_capability", + "missing_identity", + "incomplete_tool_pair", +] + +DiagnosticCode = Literal[ + "usage_revision_conflict", + "usage_row_id_reuse", + "shutdown_total_mismatch", + "checkpoint_not_additive", + "auxiliary_excluded_from_main", + "capability_gap", + "inferred_join", + "unknown_provider", + "missing_usage_fields", + "open_interaction", +] + +# Attribution.evidence strings are ``key:value`` with these keys. ``call_id`` +# may repeat when status is ambiguous. Inferred joins always include +# ``join_kind:inferred``; a future native assistant-message/provider-call id +# uses ``join_kind:direct``. +AttributionEvidenceKey = Literal[ + "join_kind", + "interaction_id", + "agent_id", + "model", + "turn_index", + "parent_tool_call_id", + "finish", + "order", + "call_id", + "logical_call_id", + "usage_source_id", + "revision", + "initiator", + "tool_call_id", +] + + +class SourceReference(TypedDict): + """A role-labelled pointer back to immutable archived evidence. + + ``role`` describes evidence only; it never upgrades a heuristic correlation + to proof. + """ + + source_id: str + source_kind: str + role: SourceReferenceRole + + +class FinishEvidence(TypedDict): + """One completion observation supporting a semantic call candidate. + + Observed Copilot transcripts finish a model cycle with + ``assistant.turn_end``, which carries only ``turnId``. There is no + ``agent.stop`` transcript event and no finish reason on that record. + """ + + source_id: str + kind: FinishEvidenceKind + value: str | None + + +class DatabaseRevision(TypedDict): + """Database identity for one accounting snapshot. + + ``primary_key`` is the percent-quoted canonical JSON of the V1 locator's + ``primary_key`` value (an int for observed ``assistant_usage_events`` rows, + an object for composite keys). ``generation`` is the unquoted locator + generation (``sha256:``). ``logical_call_id`` quotes both. + ``content_revision`` selects the newest snapshot for that logical call; a + correction replaces its derived UsageRow instead of adding another charge. + """ + + table: str + primary_key: str + generation: str + content_revision: str + + +class CopilotEventAttributes(TypedDict, total=False): + """``NormalizedEvent.attributes`` keys Copilot storage uses for joins.""" + + interaction_id: str + stored_turn_id: str + agent_id: str | None + + +class CopilotTurnAttributes(TypedDict, total=False): + """``TurnSpanDict.attributes`` keys Copilot storage uses. + + ``agent_id`` is null or omitted on a main user interaction. A non-null + value marks a nested child span; storage never emits those as standalone + ``session_turns`` records. + """ + + interaction_id: str + agent_id: str | None + + +class NormalizedEvent(TypedDict): + """One semantic event with stable identity and source provenance. + + Auxiliary ``model.*`` title-generation records use + ``kind="auxiliary_model_call"`` and ``classification="title_generation"``. + They must not appear as main ``call_candidates`` and must not inflate + conversation token totals. + + Copilot ``attributes`` follow :class:`CopilotEventAttributes`. + """ + + id: str + kind: NormalizedEventKind + classification: EventClassification + ts: str | None + source_ids: list[str] + source_references: list[SourceReference] + attributes: dict[str, Any] + + +class CallCandidate(TypedDict): + """A possible assistant-call span reconstructed from transcript evidence. + + IDs are native/source-derived (for example a transcript assistant-message + event ID), never a bare ``turnId``, chronological parent ID, tool name, or + list position. Nullable fields remain null when the archive does not + establish them. ``tool_call_ids`` come from + ``data.toolRequests[].toolCallId``, not from a ``toolCallIds`` array. + """ + + call_id: str + stored_turn_id: str | None + interaction_id: str | None + agent_id: str | None + parent_tool_call_id: str | None + model: str | None + source_ids: list[str] + source_references: list[SourceReference] + start_ts: str | None + end_ts: str | None + tool_call_ids: list[str] + finish_evidence: list[FinishEvidence] + + +class SupplementalMetrics(TypedDict, total=False): + """Copilot-only row fields that are not UsageRow columns. + + Keys are omitted when the source value is missing or JSON-null. Do not + store nulls. Nano-AI-unit billing stays here and is never converted into + estimated model-price USD. + """ + + total_nano_aiu: int + request_multiplier: float + duration_ms: int + time_to_first_token_ms: float + output_ttft_ms: float + inter_token_latency_ms: float + initiator: Initiator + api_endpoint: str + reasoning_effort: str + content_filter_triggered: int + token_details_json: str + input_tokens: int + output_tokens: int + cache_read_tokens: int + cache_write_tokens: int + reasoning_tokens: int + + +class AccountingCandidate(TypedDict): + """A database call before it is attributed to an assistant message. + + ``turn_index`` is the native user-turn index and is not a transcript + ``turnId``. ``provider`` is null when the database has no provider; + normalization must preserve that uncertainty (a UsageRow may use the + explicit ``"unknown"`` provider sentinel required by its existing shape). + Missing usage produces no UsageRow rather than a zero-valued one. + + A UsageRow is emitted only when timestamp, model, input_tokens, and + output_tokens are all present. A null timestamp or model keeps the + candidate and adds ``missing_usage_fields``; no row. Partial usage + (input present, output missing) likewise yields no row: present counts + go into ``supplemental_metrics`` only. In-memory ``UsageRow.seq`` is ``0`` + until persistence stamps the Store seq of the newest archived revision. + The SQLite primary key is never ``seq``, including when it happens to be + an integer. + """ + + usage_source_id: str + logical_call_id: str + turn_index: int | None + agent_id: str | None + parent_tool_call_id: str | None + model: str | None + provider: str | None + source_ids: list[str] + source_references: list[SourceReference] + revision: DatabaseRevision + timestamp: str | None + finish_reason: str | None + supplemental_metrics: SupplementalMetrics + + +class PendingItem(TypedDict): + """An explicit capability gap or unresolved relationship.""" + + id: str + kind: PendingItemKind + reason: str + source_ids: list[str] + evidence: list[str] + + +class ProjectionDiagnostic(TypedDict): + """Content-free derived-state diagnostic safe for status output.""" + + code: DiagnosticCode + severity: DiagnosticSeverity + message: str + source_ids: list[str] + details: dict[str, Any] + + +class Attribution(TypedDict): + """The durable result of joining one accounting record to semantics. + + ``usage_source_id`` is the newest revision's source ID. ``logical_call_id`` + is the durable key across content revisions. ``join_kind`` is ``inferred`` + or ``direct`` only when ``status`` is ``matched``; otherwise null. + + ``status="conflicting"`` is a join conflict: competing semantic assignments + or a join that would rebind a logical call after tokens were emitted. + Incompatible database revisions are ``ProjectionDiagnostic`` + ``usage_revision_conflict`` and quarantine the logical call; they do not + use ``AttributionStatus`` by themselves. + """ + + usage_source_id: str + logical_call_id: str + stored_turn_id: str | None + agent_id: str | None + call_id: str | None + status: AttributionStatus + join_kind: AttributionJoinKind | None + evidence: list[str] + + +class SemanticProjection(TypedDict): + """Pure transcript/hook reconstruction; it does not account for tokens.""" + + events: list[NormalizedEvent] + turns: list[TurnSpanDict] + call_candidates: list[CallCandidate] + pending: list[PendingItem] + diagnostics: list[ProjectionDiagnostic] + + +class AccountingProjection(TypedDict): + """Pure database normalization; it does not claim a message join.""" + + usage_rows: list[UsageRow] + candidates: list[AccountingCandidate] + diagnostics: list[ProjectionDiagnostic] + + +class Projection(TypedDict): + """Combined local-only V2 projection, serializable via UsageRow.to_dict. + + ``pending`` and ``diagnostics`` are the complete current sets; a later + commit that omits an item retracts it. ``usage_rows`` use + ``UsageRow.call_id`` as the logical call identity. + """ + + normalized_events: list[NormalizedEvent] + turns: list[TurnSpanDict] + usage_rows: list[UsageRow] + attributions: list[Attribution] + pending: list[PendingItem] + diagnostics: list[ProjectionDiagnostic] + accounting_candidates: NotRequired[list[AccountingCandidate]] + + +class ProjectedTurnRecord(TypedDict): + """The existing ``session_turns`` view shape for completed main turns only. + + ``events`` are Store events (``seq``, ``t``, ``ts``, ``data``), not + normalized semantic stubs. ``t`` is ``copilot_transcript``, + ``copilot_database``, ``copilot_hook``, or ``copilot_metadata``. ``data`` + is the V1 envelope ``{schema_version, source_record}``. ``start_seq`` / + ``end_seq`` are those events' seq values. ``filter_turns`` and + ``logfire_dataset._turn_case`` consume this shape. + + Child-agent evidence remains nested in the main turn's events / trace and + is never emitted here as an independent human interaction. + """ + + id: str + turn_id: str + session_id: str + platform: str + cwd: str + start_seq: int | None + end_seq: int | None + start_ts: str | None + end_ts: str | None + events: list[dict[str, Any]] + + +class OpenInteractionState(TypedDict): + """Unfinished interaction retained across incremental archive partitions.""" + + interaction_id: str + agent_id: str | None + stored_turn_id: str + source_ids: list[str] + last_event_source_id: str | None + start_ts: str | None + pending_tool_call_ids: list[str] + nested_children: NotRequired[list[TurnSpanDict]] + + +class LogicalCallState(TypedDict): + """Durable accounting identity for one logical database call. + + ``generation`` is the first archived record's locator generation. + ``metrics_digest`` is ``sha256:`` of canonical JSON over the row's + input/output/cache/reasoning/nano-AIU fields so a later revision can be + compared without treating row-ID reuse as the same call. + """ + + logical_call_id: str + generation: str + content_revision: str + metrics_digest: str + usage_source_id: str + + +class SemanticProjectionState(TypedDict): + """Incremental semantic replay state stored under projection state.""" + + open_interactions: dict[str, OpenInteractionState] + + +class AccountingProjectionState(TypedDict): + """Incremental accounting replay state stored under projection state.""" + + logical_calls: dict[str, LogicalCallState] + + +class ProjectionState(TypedDict): + """Persisted derived state. Separate from the V1 archive envelope. + + ``projection_schema_version`` is :data:`PROJECTION_SCHEMA_VERSION`. It is + not the V1 ``schema_version`` field on source envelopes. + + Passed as ``next_state`` to ``commit_projection``, this snapshot replaces + the previously loaded builder state. ``projection_revision`` is a digest + of the committed indexes when the builder leaves it empty. + ``index_totals`` is the size of each derived index after the commit, not + a per-commit delta. A stale schema version is discarded so a rebuild + can reconstruct derived indexes from the V1 archive. ``commit_sequence`` + is a storage-owned counter incremented on every successful + ``commit_projection`` call; it is never set by a builder and exists so a + caller that read state via ``load_projection_state`` can pass it back as + ``base_commit_sequence`` to detect (and refuse) overwriting a projection + that a concurrent writer has already advanced. + """ + + projection_schema_version: int + archive_source_ids: list[str] + semantic_state: SemanticProjectionState + accounting_state: AccountingProjectionState + projection_revision: str + index_totals: NotRequired[dict[str, int]] + commit_sequence: NotRequired[int] diff --git a/src/thirdeye/platforms/copilot/usage.py b/src/thirdeye/platforms/copilot/usage.py new file mode 100644 index 00000000..08367bea --- /dev/null +++ b/src/thirdeye/platforms/copilot/usage.py @@ -0,0 +1,795 @@ +"""Normalize archived Copilot database usage without assigning it to messages. + +The SQLite rows are the accounting authority. Transcript checkpoints and +shutdown records can validate the result, but never create another charge. +""" + +from __future__ import annotations + +import hashlib +import json +from datetime import datetime +from typing import Any +from urllib.parse import quote + +from thirdeye.usage.types import UsageRow + +from .identity import SOURCE_KEY_DIGEST_LEN, SOURCE_KEY_PREFIX_LEN +from .types import ( + AccountingCandidate, + AccountingProjection, + DatabaseRevision, + DiagnosticCode, + DiagnosticSeverity, + ProjectionDiagnostic, + SourceRecord, +) + +_USAGE_TABLE = "assistant_usage_events" +_MAIN_AGENT_KEY = "main" +_METRIC_FIELDS = ( + "input_tokens", + "output_tokens", + "cache_read_tokens", + "cache_write_tokens", + "reasoning_tokens", + "total_nano_aiu", +) +_SUPPLEMENTAL_FIELDS = ( + "total_nano_aiu", + "request_multiplier", + "duration_ms", + "time_to_first_token_ms", + "output_ttft_ms", + "inter_token_latency_ms", + "initiator", + "api_endpoint", + "reasoning_effort", + "content_filter_triggered", + "token_details_json", +) +_TOKEN_FIELDS = ( + "input_tokens", + "output_tokens", + "cache_read_tokens", + "cache_write_tokens", + "reasoning_tokens", +) + + +def _canonical_json(value: Any) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False, default=str) + + +def _valid_timestamp(value: object) -> str | None: + if not isinstance(value, str) or not value: + return None + try: + datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + return value + + +def _string(value: object) -> str | None: + return value if isinstance(value, str) and value else None + + +def _integer(value: object) -> int | None: + """Return a non-negative integer without turning missing values into zero.""" + + if isinstance(value, bool): + return None + if isinstance(value, int) and value >= 0: + return value + return None + + +def _metric_number(value: object) -> int | None: + """Accept whole-number floats so shutdown ``totalNanoAiu`` can validate.""" + + if isinstance(value, bool): + return None + if isinstance(value, int) and value >= 0: + return value + if isinstance(value, float) and value >= 0 and value.is_integer(): + return int(value) + return None + + +def _source_key(record: SourceRecord) -> str | None: + prefix = "copilot-db:" + source_id = record["source_id"] + if not source_id.startswith(prefix): + return None + key = source_id[len(prefix) :].split(":", 1)[0] + return key if len(key) == SOURCE_KEY_DIGEST_LEN else None + + +def _revision(record: SourceRecord) -> DatabaseRevision | None: + locator = record.get("locator") + if not isinstance(locator, dict) or locator.get("table") != _USAGE_TABLE: + return None + generation = _string(locator.get("generation")) + content_revision = _string(locator.get("content_revision")) + if generation is None or content_revision is None or "primary_key" not in locator: + return None + return { + "table": _USAGE_TABLE, + "primary_key": quote(_canonical_json(locator["primary_key"]), safe=""), + "generation": generation, + "content_revision": content_revision, + } + + +def _logical_call_id(record: SourceRecord, revision: DatabaseRevision) -> str | None: + source_key = _source_key(record) + if source_key is None: + return None + return ( + f"copilot:usage:{source_key}:{revision['table']}:" + f"{quote(revision['generation'], safe='')}:{revision['primary_key']}" + ) + + +def _metrics_digest(row: dict[str, Any]) -> str: + metrics = {field: row.get(field) for field in _METRIC_FIELDS} + return "sha256:" + hashlib.sha256(_canonical_json(metrics).encode("utf-8")).hexdigest() + + +def _row_metrics(row: dict[str, Any]) -> dict[str, int]: + metrics: dict[str, int] = {} + for field in _METRIC_FIELDS: + value = _metric_number(row.get(field)) + if value is not None: + metrics[field] = value + return metrics + + +def _row_is_incompatible(row: dict[str, Any]) -> bool: + """Reject revisions that cannot describe a single completed request.""" + + input_tokens = _metric_number(row.get("input_tokens")) + cache_read = _metric_number(row.get("cache_read_tokens")) + cache_write = _metric_number(row.get("cache_write_tokens")) + if input_tokens is None: + return False + if (cache_read is not None and cache_read > input_tokens) or ( + cache_write is not None and cache_write > input_tokens + ): + return True + return ( + cache_read is not None + and cache_write is not None + and cache_read + cache_write > input_tokens + ) + + +def _row_key(table: str, primary_key: str) -> str: + return f"{table}\x1f{primary_key}" + + +def _agent_key(agent_id: str | None) -> str: + return agent_id or _MAIN_AGENT_KEY + + +def _candidate( + record: SourceRecord, revision: DatabaseRevision, logical_id: str +) -> AccountingCandidate: + payload = record.get("payload") + row = payload.get("row") if isinstance(payload, dict) else None + assert isinstance(row, dict) + timestamp = _valid_timestamp(record.get("ts")) or _valid_timestamp(row.get("created_at")) + supplemental = { + field: row[field] + for field in _SUPPLEMENTAL_FIELDS + if field in row and row[field] is not None + } + # Fully specified token usage belongs in UsageRow. Retain token values in + # the candidate only when a partial row cannot produce a UsageRow. + if ( + _metric_number(row.get("input_tokens")) is None + or _metric_number(row.get("output_tokens")) is None + ): + supplemental.update( + { + field: row[field] + for field in _TOKEN_FIELDS + if field in row and row[field] is not None + } + ) + turn_index = _integer(row.get("turn_index")) + return { + "usage_source_id": record["source_id"], + "logical_call_id": logical_id, + "turn_index": turn_index, + "agent_id": _string(row.get("agent_id")), + "parent_tool_call_id": _string(row.get("parent_tool_call_id")), + "model": _string(row.get("model")), + "provider": _string(row.get("provider")) or _string(row.get("provider_name")), + "source_ids": [record["source_id"]], + "source_references": [ + {"source_id": record["source_id"], "source_kind": "database", "role": "usage_row"} + ], + "revision": revision, + "timestamp": timestamp, + "finish_reason": _string(row.get("finish_reason")), + "supplemental_metrics": supplemental, + } + + +def _attach_revision_sources(candidate: AccountingCandidate, source_ids: list[str]) -> None: + candidate["source_ids"] = list(source_ids) + candidate["source_references"] = [ + {"source_id": source_id, "source_kind": "database", "role": "usage_row"} + for source_id in source_ids + ] + + +def _missing_fields(candidate: AccountingCandidate, row: dict[str, Any]) -> list[str]: + required = { + "timestamp": candidate["timestamp"], + "model": candidate["model"], + "input_tokens": _metric_number(row.get("input_tokens")), + "output_tokens": _metric_number(row.get("output_tokens")), + } + return [name for name, value in required.items() if value is None] + + +def _usage_row( + candidate: AccountingCandidate, session_id: str, row: dict[str, Any] +) -> UsageRow | None: + missing = _missing_fields(candidate, row) + if missing: + return None + input_tokens = _metric_number(row["input_tokens"]) + output_tokens = _metric_number(row["output_tokens"]) + assert input_tokens is not None and output_tokens is not None + return UsageRow( + session_id=session_id, + seq=0, + call_id=candidate["logical_call_id"], + ts=candidate["timestamp"], # guarded by _missing_fields + platform="copilot", + provider_name=candidate["provider"] or "unknown", + response_model=candidate["model"], # guarded by _missing_fields + input_tokens=input_tokens, + output_tokens=output_tokens, + cache_read_input_tokens=_metric_number(row.get("cache_read_tokens")), + cache_creation_input_tokens=_metric_number(row.get("cache_write_tokens")), + reasoning_output_tokens=_metric_number(row.get("reasoning_tokens")), + ) + + +def _diagnostic( + code: DiagnosticCode, + severity: DiagnosticSeverity, + message: str, + source_ids: list[str], + **details: Any, +) -> ProjectionDiagnostic: + return { + "code": code, + "severity": severity, + "message": message, + "source_ids": source_ids, + "details": details, + } + + +def _add_metrics(target: dict[str, int], metrics: dict[str, int], *, sign: int = 1) -> None: + for field, value in metrics.items(): + target[field] = target.get(field, 0) + sign * value + + +def _zero_metrics() -> dict[str, int]: + return {field: 0 for field in _METRIC_FIELDS} + + +def _metrics_from_call(call: dict[str, Any]) -> dict[str, int]: + if call.get("quarantined"): + return {} + metrics = call.get("metrics") + if not isinstance(metrics, dict): + return {} + parsed: dict[str, int] = {} + for field in _METRIC_FIELDS: + value = _metric_number(metrics.get(field)) + if value is not None: + parsed[field] = value + return parsed + + +def _parse_usage_block(usage: dict[str, Any], total_nano_aiu: object) -> dict[str, int] | None: + values = { + "input_tokens": usage.get("inputTokens"), + "output_tokens": usage.get("outputTokens"), + "cache_read_tokens": usage.get("cacheReadTokens"), + "cache_write_tokens": usage.get("cacheWriteTokens"), + "reasoning_tokens": usage.get("reasoningTokens"), + "total_nano_aiu": total_nano_aiu, + } + parsed = {name: _metric_number(value) for name, value in values.items()} + if any(value is None for value in parsed.values()): + return None + return {name: value for name, value in parsed.items() if value is not None} + + +def _aggregate_model_metrics(model_metrics: object) -> dict[str, int] | None: + if not isinstance(model_metrics, dict) or not model_metrics: + return None + aggregate = _zero_metrics() + for model in model_metrics.values(): + if not isinstance(model, dict): + return None + usage = model.get("usage") + if not isinstance(usage, dict): + return None + parsed = _parse_usage_block(usage, model.get("totalNanoAiu")) + if parsed is None: + return None + _add_metrics(aggregate, parsed) + return aggregate + + +def _agent_shutdown_totals(agent_metrics: object) -> dict[str, dict[str, int]] | None: + if not isinstance(agent_metrics, dict) or not agent_metrics: + return None + totals: dict[str, dict[str, int]] = {} + for agent_id, payload in agent_metrics.items(): + if not isinstance(agent_id, str) or not isinstance(payload, dict): + return None + parsed = _aggregate_model_metrics(payload.get("modelMetrics")) + if parsed is None: + return None + totals[agent_id] = parsed + return totals + + +def _unwrap_prior_state(prior_state: dict[str, Any]) -> dict[str, Any]: + """Accept a flat accounting state or a nested ProjectionState envelope. + + ``prior_state`` may be ``{"logical_calls": ...}`` (and optional cumulative + keys) or ``{"accounting_state": {"logical_calls": ...}}``. + """ + + inherited_calls = prior_state.get("logical_calls") + if isinstance(inherited_calls, dict): + return prior_state + nested = prior_state.get("accounting_state") + if isinstance(nested, dict): + return nested + return {} + + +def _hydrate_row_identity( + source: dict[str, Any], inherited_calls: dict[str, Any] +) -> tuple[dict[str, set[str]], dict[str, list[str]]]: + generations: dict[str, set[str]] = {} + sources: dict[str, list[str]] = {} + prior_generations = source.get("row_generations") + if isinstance(prior_generations, dict): + for key, values in prior_generations.items(): + if isinstance(key, str) and isinstance(values, list): + generations[key] = {item for item in values if isinstance(item, str)} + prior_sources = source.get("row_sources") + if isinstance(prior_sources, dict): + for key, values in prior_sources.items(): + if isinstance(key, str) and isinstance(values, list): + sources[key] = [item for item in values if isinstance(item, str)] + for call in inherited_calls.values(): + if not isinstance(call, dict): + continue + table = call.get("table") if isinstance(call.get("table"), str) else _USAGE_TABLE + primary_key = call.get("primary_key") + generation = call.get("generation") + if not isinstance(primary_key, str) or not isinstance(generation, str): + continue + key = _row_key(table, primary_key) + generations.setdefault(key, set()).add(generation) + usage_source_id = call.get("usage_source_id") + if isinstance(usage_source_id, str) and usage_source_id not in sources.setdefault(key, []): + sources[key].append(usage_source_id) + return generations, sources + + +def _hydrate_accounted( + source: dict[str, Any], inherited_calls: dict[str, Any] +) -> tuple[dict[str, int], dict[str, dict[str, int]]]: + accounted = _zero_metrics() + accounted_by_agent: dict[str, dict[str, int]] = {} + prior_accounted = source.get("accounted_metrics") + if isinstance(prior_accounted, dict) and any( + _metric_number(prior_accounted.get(field)) is not None for field in _METRIC_FIELDS + ): + for field in _METRIC_FIELDS: + value = _metric_number(prior_accounted.get(field)) + if value is not None: + accounted[field] = value + prior_agents = source.get("accounted_by_agent") + if isinstance(prior_agents, dict): + for agent_id, metrics in prior_agents.items(): + if isinstance(agent_id, str) and isinstance(metrics, dict): + bucket = _zero_metrics() + _add_metrics(bucket, _metrics_from_call({"metrics": metrics})) + accounted_by_agent[agent_id] = bucket + return accounted, accounted_by_agent + for call in inherited_calls.values(): + if not isinstance(call, dict): + continue + metrics = _metrics_from_call(call) + _add_metrics(accounted, metrics) + agent_id = call.get("agent_id") + key = _agent_key(agent_id if isinstance(agent_id, str) else None) + accounted_by_agent.setdefault(key, _zero_metrics()) + _add_metrics(accounted_by_agent[key], metrics) + return accounted, accounted_by_agent + + +def _prior_source_ids(call: dict[str, Any]) -> list[str]: + prior_ids = call.get("source_ids") + if isinstance(prior_ids, list): + seeded = [item for item in prior_ids if isinstance(item, str)] + if seeded: + return seeded + usage_source_id = call.get("usage_source_id") + return [usage_source_id] if isinstance(usage_source_id, str) else [] + + +def _seed_revision_sources( + revision_sources: dict[str, list[str]], + logical_id: str, + inherited_calls: dict[str, Any], +) -> None: + if logical_id in revision_sources: + return + previous = inherited_calls.get(logical_id) + revision_sources[logical_id] = _prior_source_ids(previous) if isinstance(previous, dict) else [] + + +def _logical_call_entry( + logical_id: str, + revision: DatabaseRevision, + record: SourceRecord, + row: dict[str, Any], + candidate: AccountingCandidate, + source_ids: list[str], + *, + quarantined: bool, +) -> dict[str, Any]: + return { + "logical_call_id": logical_id, + "generation": revision["generation"], + "content_revision": revision["content_revision"], + "metrics_digest": _metrics_digest(row), + "usage_source_id": record["source_id"], + "source_ids": list(source_ids), + "table": revision["table"], + "primary_key": revision["primary_key"], + "agent_id": candidate["agent_id"], + "metrics": {} if quarantined else _row_metrics(row), + "quarantined": quarantined, + } + + +def _mismatch_details(expected: dict[str, int], accounted: dict[str, int]) -> dict[str, int]: + details: dict[str, int] = {} + for field in _METRIC_FIELDS: + details[f"expected_{field}"] = expected[field] + details[f"accounted_{field}"] = accounted[field] + return details + + +def _metrics_match(expected: dict[str, int], accounted: dict[str, int]) -> bool: + return all(accounted.get(field, 0) == expected[field] for field in _METRIC_FIELDS) + + +def build_accounting( + records: list[SourceRecord], prior_state: dict[str, Any] +) -> tuple[AccountingProjection, dict[str, Any]]: + """Build independent database accounting from immutable V1 source records. + + Revisions are applied in archive order. A missing row is intentionally not + a deletion: only archived observations can replace an accounting result. + + ``prior_state`` may be a flat accounting document (``logical_calls`` plus + optional cumulative keys) or a full projection state with nested + ``accounting_state``. Cumulative ``accounted_metrics``, + ``accounted_by_agent``, ``row_generations``, and ``row_sources`` support + disjoint archive partitions; prefix replay remains equivalent to a full + pass. + """ + + source = _unwrap_prior_state(prior_state) + inherited_raw = source.get("logical_calls") + inherited_calls = inherited_raw if isinstance(inherited_raw, dict) else {} + selected: dict[str, tuple[SourceRecord, DatabaseRevision, AccountingCandidate, str | None]] = {} + revision_sources: dict[str, list[str]] = {} + diagnostics: list[ProjectionDiagnostic] = [] + row_generations, row_sources = _hydrate_row_identity(source, inherited_calls) + prior_digests = { + logical_id: call["metrics_digest"] + for logical_id, call in inherited_calls.items() + if isinstance(logical_id, str) + and isinstance(call, dict) + and isinstance(call.get("metrics_digest"), str) + } + + for record in records: + payload = record.get("payload") + if record.get("source_kind") != "database" or not isinstance(payload, dict): + continue + if payload.get("table") != _USAGE_TABLE or not isinstance(payload.get("row"), dict): + continue + revision = _revision(record) + logical_id = _logical_call_id(record, revision) if revision is not None else None + if revision is None or logical_id is None: + reason = ( + "source_id is not a 64-character copilot-db identity" + if revision is not None + else "locator is missing generation, content_revision, or primary_key" + ) + diagnostics.append( + _diagnostic( + "capability_gap", + "warning", + "usage row cannot be accounted because archive identity is incomplete", + [record["source_id"]], + reason=reason, + ) + ) + continue + candidate = _candidate(record, revision, logical_id) + key = _row_key(revision["table"], revision["primary_key"]) + row_generations.setdefault(key, set()).add(revision["generation"]) + if record["source_id"] not in row_sources.setdefault(key, []): + row_sources[key].append(record["source_id"]) + previous_digest = prior_digests.get(logical_id) + if logical_id in selected: + previous_digest = _metrics_digest(selected[logical_id][0]["payload"]["row"]) + selected[logical_id] = (record, revision, candidate, previous_digest) + _seed_revision_sources(revision_sources, logical_id, inherited_calls) + if record["source_id"] not in revision_sources[logical_id]: + revision_sources[logical_id].append(record["source_id"]) + prior_digests[logical_id] = _metrics_digest(payload["row"]) + + reuse_keys = {key for key, generations in row_generations.items() if len(generations) > 1} + for key in sorted(reuse_keys): + table, primary_key = key.split("\x1f", 1) + diagnostics.append( + _diagnostic( + "usage_row_id_reuse", + "error", + "database row ID was reused by a different database generation; quarantined as conflict", + list(row_sources.get(key, [])), + table=table, + primary_key=primary_key, + generations=sorted(row_generations[key]), + ) + ) + + candidates: list[AccountingCandidate] = [] + usage_rows: list[UsageRow] = [] + accounted, accounted_by_agent = _hydrate_accounted(source, inherited_calls) + logical_calls = { + key: value.copy() + for key, value in inherited_calls.items() + if isinstance(key, str) and isinstance(value, dict) + } + reused_logical_ids: set[str] = set() + for logical_id, call in logical_calls.items(): + table = call.get("table") if isinstance(call.get("table"), str) else _USAGE_TABLE + primary_key = call.get("primary_key") + if isinstance(primary_key, str) and _row_key(table, primary_key) in reuse_keys: + reused_logical_ids.add(logical_id) + for logical_id, (_record, revision, _selected_candidate, _digest) in selected.items(): + if _row_key(revision["table"], revision["primary_key"]) in reuse_keys: + reused_logical_ids.add(logical_id) + for logical_id in reused_logical_ids: + if logical_id in selected: + continue + previous = logical_calls.get(logical_id) + if not isinstance(previous, dict): + continue + old_metrics = _metrics_from_call(previous) + _add_metrics(accounted, old_metrics, sign=-1) + previous_agent = previous.get("agent_id") + old_agent = _agent_key(previous_agent if isinstance(previous_agent, str) else None) + if old_agent in accounted_by_agent: + _add_metrics(accounted_by_agent[old_agent], old_metrics, sign=-1) + logical_calls[logical_id] = {**previous, "metrics": {}, "quarantined": True} + unknown_provider_ids: list[str] = [] + for logical_id, (record, revision, candidate, previous_digest) in selected.items(): + payload = record["payload"] + row = payload["row"] + assert isinstance(row, dict) + _attach_revision_sources(candidate, revision_sources[logical_id]) + previous = logical_calls.get(logical_id) + if isinstance(previous, dict): + old_metrics = _metrics_from_call(previous) + _add_metrics(accounted, old_metrics, sign=-1) + previous_agent = previous.get("agent_id") + old_agent = _agent_key(previous_agent if isinstance(previous_agent, str) else None) + if old_agent in accounted_by_agent: + _add_metrics(accounted_by_agent[old_agent], old_metrics, sign=-1) + digest = _metrics_digest(row) + if logical_id in reused_logical_ids: + candidates.append(candidate) + logical_calls[logical_id] = _logical_call_entry( + logical_id, + revision, + record, + row, + candidate, + revision_sources[logical_id], + quarantined=True, + ) + continue + if _row_is_incompatible(row): + details: dict[str, Any] = { + "logical_call_id": logical_id, + "metrics_digest": digest, + } + if previous_digest is not None: + details["prior_metrics_digest"] = previous_digest + diagnostics.append( + _diagnostic( + "usage_revision_conflict", + "error", + "incompatible metrics for one logical call; quarantined", + revision_sources[logical_id], + **details, + ) + ) + candidates.append(candidate) + logical_calls[logical_id] = _logical_call_entry( + logical_id, + revision, + record, + row, + candidate, + revision_sources[logical_id], + quarantined=True, + ) + continue + candidates.append(candidate) + missing = _missing_fields(candidate, row) + if missing: + diagnostics.append( + _diagnostic( + "missing_usage_fields", + "warning", + "database usage row is incomplete", + [record["source_id"]], + logical_call_id=logical_id, + missing_fields=missing, + ) + ) + source_key = _source_key(record) + assert source_key is not None + session_id = f"copilot-{source_key[:SOURCE_KEY_PREFIX_LEN]}-{record['native_session_id']}" + usage_row = _usage_row(candidate, session_id, row) + if usage_row is not None: + usage_rows.append(usage_row) + if usage_row.provider_name == "unknown": + unknown_provider_ids.append(record["source_id"]) + # Account every present metric field, including partial rows that cannot + # emit a UsageRow. Mismatch diagnostics therefore describe archived + # observations, not only exported totals. + metrics = _row_metrics(row) + _add_metrics(accounted, metrics) + agent_key = _agent_key(candidate["agent_id"]) + accounted_by_agent.setdefault(agent_key, _zero_metrics()) + _add_metrics(accounted_by_agent[agent_key], metrics) + logical_calls[logical_id] = _logical_call_entry( + logical_id, + revision, + record, + row, + candidate, + revision_sources[logical_id], + quarantined=False, + ) + + if unknown_provider_ids: + diagnostics.append( + _diagnostic( + "unknown_provider", + "info", + "database usage rows do not name a provider; UsageRow uses the unknown sentinel", + unknown_provider_ids, + ) + ) + + for record in records: + payload = record.get("payload") + if not isinstance(payload, dict) or payload.get("type") != "session.usage_checkpoint": + continue + diagnostics.append( + _diagnostic( + "checkpoint_not_additive", + "info", + "checkpoint snapshot validates totals and is not a seventh call", + [record["source_id"]], + ) + ) + + for record in records: + payload = record.get("payload") + if not isinstance(payload, dict) or payload.get("type") != "session.shutdown": + continue + data = payload.get("data") + if not isinstance(data, dict): + diagnostics.append( + _diagnostic( + "capability_gap", + "warning", + "session.shutdown usage cannot be used to validate accounting", + [record["source_id"]], + reason="missing_shutdown_data", + ) + ) + continue + session_totals = _aggregate_model_metrics(data.get("modelMetrics")) + if session_totals is None: + diagnostics.append( + _diagnostic( + "capability_gap", + "warning", + "session.shutdown usage cannot be used to validate accounting", + [record["source_id"]], + reason="unparseable_model_metrics", + ) + ) + continue + if not _metrics_match(session_totals, accounted): + diagnostics.append( + _diagnostic( + "shutdown_total_mismatch", + "warning", + "per-call totals do not equal session.shutdown usage", + [record["source_id"]], + **_mismatch_details(session_totals, accounted), + ) + ) + agent_metrics = data.get("agentMetrics") + if agent_metrics is None: + continue + agent_totals = _agent_shutdown_totals(agent_metrics) + if agent_totals is None: + diagnostics.append( + _diagnostic( + "capability_gap", + "warning", + "session.shutdown agentMetrics cannot be used to validate per-agent accounting", + [record["source_id"]], + reason="unparseable_agent_metrics", + ) + ) + continue + for agent_id, expected in agent_totals.items(): + actual = accounted_by_agent.get(agent_id, _zero_metrics()) + if not _metrics_match(expected, actual): + diagnostics.append( + _diagnostic( + "shutdown_total_mismatch", + "warning", + "per-agent totals do not equal session.shutdown agentMetrics", + [record["source_id"]], + agent_id=agent_id, + **_mismatch_details(expected, actual), + ) + ) + + next_state: dict[str, Any] = { + "logical_calls": logical_calls, + "accounted_metrics": accounted, + "accounted_by_agent": accounted_by_agent, + "row_generations": {key: sorted(values) for key, values in row_generations.items()}, + "row_sources": row_sources, + } + return { + "usage_rows": usage_rows, + "candidates": candidates, + "diagnostics": diagnostics, + }, next_state diff --git a/src/thirdeye/platforms/copilot/watch.py b/src/thirdeye/platforms/copilot/watch.py new file mode 100644 index 00000000..6b168f05 --- /dev/null +++ b/src/thirdeye/platforms/copilot/watch.py @@ -0,0 +1,185 @@ +"""Foreground polling for local Copilot CLI evidence. + +The watcher intentionally watches filesystem *identity*, not Copilot's +process. This makes it useful after an editor-terminal session has ended and +also means it never needs to start, inspect, or authenticate a Copilot CLI. +""" + +from __future__ import annotations + +import math +import time +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from thirdeye.config import Config + +from .capture import sync +from .database import discover_database_sessions +from .identity import validate_native_id +from .runtime import reconcile_archived_sessions, reconcile_session +from .sources import discover_sessions +from .types import SourcePaths + +_EVENTS_FILENAME = "events.jsonl" +_SLEEP: Callable[[float], None] = time.sleep + + +def _file_stamp(path: Path) -> tuple[int, int, int, int] | None: + """Return a cheap replacement/append-sensitive stamp for one file.""" + + try: + stat = path.stat() + except OSError: + return None + if not path.is_file(): + return None + return (stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns) + + +def _transcript_stamps(paths: SourcePaths, session_ids: set[str]) -> dict[str, object]: + root = Path(paths["session_root"]) + stamps: dict[str, object] = {} + for native_id in session_ids: + try: + validate_native_id(native_id) + except ValueError: + continue + stamps[native_id] = _file_stamp(root / native_id / _EVENTS_FILENAME) + return stamps + + +def _database_stamp(paths: SourcePaths) -> tuple[object, object, object]: + """Include WAL and SHM changes: live SQLite commits need no DB mtime.""" + + database = Path(paths["database"]) + return ( + _file_stamp(database), + _file_stamp(database.with_name(f"{database.name}-wal")), + _file_stamp(database.with_name(f"{database.name}-shm")), + ) + + +def _spool_stamps(config: Config, paths: SourcePaths) -> dict[str, tuple[tuple[str, object], ...]]: + """Return per-session spool stamps without loading prompt-bearing records.""" + + root = Path(config.root) / "spool" / "copilot" / paths["source_key"] + try: + entries = list(root.iterdir()) + except OSError: + return {} + result: dict[str, tuple[tuple[str, object], ...]] = {} + for entry in entries: + try: + if not entry.is_dir(): + continue + validate_native_id(entry.name) + files = tuple((item.name, _file_stamp(item)) for item in sorted(entry.glob("*.json"))) + except (OSError, ValueError): + continue + result[entry.name] = files + return result + + +def _source_snapshot(config: Config, paths: SourcePaths) -> dict[str, Any]: + """Discover source IDs and cheap change positions for the next poll.""" + + # ``discover_sessions`` is deliberately the public composition discovery + # entrypoint. Database-specific discovery is additionally retained so a + # WAL-only change need not re-read transcript-only sessions. + all_sessions = set(discover_sessions(paths)) + database_sessions = set(discover_database_sessions(paths)) + all_sessions.update(database_sessions) + spool = _spool_stamps(config, paths) + all_sessions.update(spool) + return { + "sessions": all_sessions, + "database_sessions": database_sessions, + "transcripts": _transcript_stamps(paths, all_sessions), + "database": _database_stamp(paths), + "spool": spool, + } + + +def _changed_sessions(before: dict[str, Any], after: dict[str, Any]) -> set[str]: + """Choose only source positions that can have produced new evidence.""" + + before_sessions = before["sessions"] + after_sessions = after["sessions"] + changed = set(after_sessions - before_sessions) + + for native_id in after_sessions: + if before["transcripts"].get(native_id) != after["transcripts"].get(native_id): + changed.add(native_id) + if before["spool"].get(native_id) != after["spool"].get(native_id): + changed.add(native_id) + + if before["database"] != after["database"]: + # Retain IDs observed before the change as well: a transaction may + # delete/update rows and source disappearance is never session close. + changed.update(before["database_sessions"]) + changed.update(after["database_sessions"]) + return changed + + +def _result_needs_retry(result: dict[str, int], *, present: bool) -> bool: + """Retry incomplete or failed work only while the source is still discoverable. + + A missing selected ID is a one-shot error for that poll: source removal is + never session completion, but it also must not become a permanent retry + loop. Recreated files change stamps and are selected again. Pending work + (busy/unreadable/incomplete pages) retries even if discovery briefly drops + the ID, because those codes are not absence. + """ + + if result.get("pending", 0) > 0: + return True + return bool(present and result.get("errors", 0) > 0) + + +def watch(config: Config, paths: SourcePaths, *, interval: float = 1.0) -> None: + """Poll local recordings until interrupted. + + The initial sync drains the bounded source snapshot. Later cycles invoke + per-session sync only after a transcript, database/WAL, or spool position + changes (or after a retryable result), avoiding repeated parsing of quiet + completed sessions. Capture and derived replay remain local. When live + export is configured, reconciliation may queue detached jobs; this + foreground loop never delivers remotely itself. + """ + + if not isinstance(interval, (int, float)) or isinstance(interval, bool): + raise ValueError("interval must be a finite number of seconds, at least 0.1") + if not math.isfinite(interval) or interval < 0.1: + raise ValueError("interval must be a finite number of seconds, at least 0.1") + + try: + # Take the baseline before the initial drain. A source can append + # while that bounded drain is running; comparing its post-drain stamp + # on the first poll ensures that append receives a later capture. + previous = _source_snapshot(config, paths) + initial = sync(config, paths) + # Activation replays retained V1 evidence, including sessions whose + # transcript/database was removed after capture. + reconcile_archived_sessions(config, paths, export=True) + retry = set(previous["sessions"]) if _result_needs_retry(initial, present=True) else set() + while True: + _SLEEP(float(interval)) + current = _source_snapshot(config, paths) + selected = _changed_sessions(previous, current) | retry + retry = set() + for native_id in sorted(selected): + # KeyboardInterrupt is intentionally checked between sessions; + # a current archive commit remains crash-recoverable. + result = sync(config, paths, session_id=native_id) + reconcile_session(config, paths, native_id, export=True) + if _result_needs_retry(result, present=native_id in current["sessions"]): + retry.add(native_id) + previous = current + except KeyboardInterrupt: + # A foreground CLI command treats Ctrl-C as ordinary termination. + return + + +__all__ = ["watch"] diff --git a/src/thirdeye/platforms/provenance.py b/src/thirdeye/platforms/provenance.py index 43ce68dd..610667f3 100644 --- a/src/thirdeye/platforms/provenance.py +++ b/src/thirdeye/platforms/provenance.py @@ -4,8 +4,14 @@ from typing import Any -_KNOWN_PLATFORMS = frozenset({"claude", "codex", "cursor"}) +from thirdeye.platforms.copilot.constants import CLI_HOOK_EVENT_ALIASES + +_KNOWN_PLATFORMS = frozenset({"claude", "codex", "cursor", "copilot"}) _CURSOR_MARKERS = ("cursor_version", "composer_mode") +# Copilot CLI natively emits camelCase hook names. Those names are not Cursor +# evidence when the expected platform is Copilot; Cursor-only camelCase events +# remain foreign. +_COPILOT_CAMEL_EVENTS = frozenset(CLI_HOOK_EVENT_ALIASES) def foreign_payload_reason(payload: dict[str, Any], expected: str) -> str | None: @@ -28,6 +34,8 @@ def foreign_payload_reason(payload: dict[str, Any], expected: str) -> str | None first_character = event_name[0] if first_character.islower() and expected != "cursor": + if expected == "copilot" and event_name in _COPILOT_CAMEL_EVENTS: + return None return f"Cursor event {event_name} received for {expected}" if first_character.isupper() and expected == "cursor": return f"PascalCase event {event_name} received for cursor" diff --git a/src/thirdeye/skills/use-thirdeye/SKILL.md b/src/thirdeye/skills/use-thirdeye/SKILL.md index 49cb1ccb..8eee83ae 100644 --- a/src/thirdeye/skills/use-thirdeye/SKILL.md +++ b/src/thirdeye/skills/use-thirdeye/SKILL.md @@ -5,7 +5,7 @@ description: Use when an agent needs to inspect, search, or evaluate past agent ## Overview -`thirdeye` (PyPI: `thrdi`) captures events from agentic tools (Claude Code, Codex, Cursor) +`thirdeye` (PyPI: `thrdi`) captures events from agentic tools (Claude Code, Codex, Cursor, GitHub Copilot CLI) into a unified per-session event store on disk. Each session's data lives under `/traces///` and contains a sequential log of all events the agent emitted during that session. Sessions are addressable by any unique prefix of their session ID, @@ -22,11 +22,18 @@ for full instructions on installing hooks, verifying data flow, and removing hoo thirdeye add --claude thirdeye add --codex thirdeye add --cursor +thirdeye add --copilot + +# Copilot CLI V1: ingest and health (watch is foreground-only; not started by add) +thirdeye copilot status --source-home "$COPILOT_HOME" +thirdeye copilot sync --source-home "$COPILOT_HOME" +thirdeye copilot watch --source-home "$COPILOT_HOME" --interval 1 # Remove hooks thirdeye remove --claude thirdeye remove --codex thirdeye remove --cursor +thirdeye remove --copilot ``` ## Searching and retrieving session data diff --git a/src/thirdeye/skills/use-thirdeye/references/setup-and-tracing.md b/src/thirdeye/skills/use-thirdeye/references/setup-and-tracing.md index 7654a405..4249c534 100644 --- a/src/thirdeye/skills/use-thirdeye/references/setup-and-tracing.md +++ b/src/thirdeye/skills/use-thirdeye/references/setup-and-tracing.md @@ -19,15 +19,17 @@ The PyPI package is `thrdi`; the installed commands are `thirdeye` and `thrdi` thirdeye add --claude # Claude Code thirdeye add --codex # OpenAI Codex CLI thirdeye add --cursor # Cursor +thirdeye add --copilot # GitHub Copilot CLI ``` `thirdeye add` is idempotent — running it twice for the same platform leaves the existing hook entries in place rather than duplicating them. Hook entries are written into each platform's own config file: Claude Code -uses `~/.claude/settings.json`, Codex uses `~/.codex/config.toml`, and +uses `~/.claude/settings.json`, Codex uses `~/.codex/config.toml`, Cursor uses `~/.cursor/hooks.json` -(covering both the IDE chat and the `cursor-agent` CLI). +(covering both the IDE chat and the `cursor-agent` CLI), and GitHub Copilot CLI +uses `$COPILOT_HOME/hooks/thirdeye.json` (default `~/.copilot/hooks/thirdeye.json`). ### Cursor subagent hooks @@ -53,9 +55,40 @@ and this user-level setup does not promise remote capture. ```bash thirdeye remove --claude # remove only Claude hooks -thirdeye remove --codex # etc. +thirdeye remove --codex +thirdeye remove --cursor +thirdeye remove --copilot ``` +## Copilot CLI V1 + +GitHub Copilot CLI capture is a V1 immutable raw archive of recordings from the +selected Copilot home. V2 (reconstructed turns, usage accounting, and OTel +export) is out of scope. V1 does not export Copilot content. + +```bash +thirdeye add --copilot +thirdeye copilot status --source-home "$COPILOT_HOME" +thirdeye copilot sync --source-home "$COPILOT_HOME" +thirdeye copilot watch --source-home "$COPILOT_HOME" --interval 1 +thirdeye remove --copilot +``` + +`--source-home` is optional. Resolution is `--source-home`, then `COPILOT_HOME`, +then `~/.copilot`. V1 reads only that home's `session-state/**/events.jsonl`, +`workspace.yaml`, and `session-store.db` (`sessions`, `turns`, +`assistant_usage_events`). It does not read credentials, token-bearing config, +or other Copilot files. + +`thirdeye add --copilot` writes user-level hooks at +`$COPILOT_HOME/hooks/thirdeye.json` (default `~/.copilot/hooks/thirdeye.json`). +The 1.0.83 live probe used repository hooks; V1 does not. `watch` is an explicit +foreground poller and is not started by add or setup. Captured content has the +same local sensitivity as other thirdeye sessions and is not exported in V1. +`sync` and `status` print recoverable source diagnostics (locations and reasons, +not prompt bodies). Passing the Copilot unit tests is not live certification of +Copilot CLI. + ## Verify tracing is live After the next agent run, a new session should appear: @@ -64,6 +97,7 @@ After the next agent run, a new session should appear: thirdeye list # JSON-per-line, newest first thirdeye list --tree # human-readable thirdeye events # events for one session +thirdeye copilot status # Copilot source health, pending spool/leases ``` `` accepts any unique prefix — usually 4-8 characters is enough. diff --git a/src/thirdeye/tracing/__init__.py b/src/thirdeye/tracing/__init__.py index 7bd93094..f112f489 100644 --- a/src/thirdeye/tracing/__init__.py +++ b/src/thirdeye/tracing/__init__.py @@ -1,18 +1,28 @@ from __future__ import annotations from thirdeye.tracing.model import ( + AccountingCallSpanDict, + AccountingDestination, + AccountingLedgerEntryDict, LlmCallSpanDict, PermissionRequestSpanDict, + SessionAccountingJobDict, ToolCallSpanDict, + TurnAccountingJobDict, TurnSpanDict, TurnStatus, UsageDict, ) __all__ = [ + "AccountingCallSpanDict", + "AccountingDestination", + "AccountingLedgerEntryDict", "LlmCallSpanDict", "PermissionRequestSpanDict", + "SessionAccountingJobDict", "ToolCallSpanDict", + "TurnAccountingJobDict", "TurnSpanDict", "TurnStatus", "UsageDict", diff --git a/src/thirdeye/tracing/model.py b/src/thirdeye/tracing/model.py index 7deef0e6..3a8bddde 100644 --- a/src/thirdeye/tracing/model.py +++ b/src/thirdeye/tracing/model.py @@ -35,7 +35,13 @@ class ToolCallSpanDict(TypedDict): class LlmCallSpanDict(TypedDict): - """One model call within a turn, plus the tool calls it requested.""" + """One model call within a turn, plus the tool calls it requested. + + When tokens are carried on :class:`AccountingCallSpanDict`, ``usage`` must + stay empty so the generic exporter cannot emit the same tokens twice. + Copilot producers always leave this empty and place actual usage on + ``TurnSpanDict.accounting_calls``. + """ call_id: str provider: str @@ -82,6 +88,100 @@ class InteractionSpanDict(TypedDict): attributes: dict[str, Any] +AccountingDestination = Literal["chat-span", "turn-accounting-span", "session-accounting-span"] + + +class AccountingCallSpanDict(TypedDict): + """Generic accounting attached to a turn without importing platform types. + + ``usage`` is exactly :meth:`UsageRow.to_dict` output. ``accounting_id`` is + stable across source-row corrections and export retries. + + ``call_id`` is the matching :class:`LlmCallSpanDict` id when tokens export + on that chat span; null means a user-turn or agent accounting span. Set + and null are mutually exclusive export locations: never both. The parent + chat span's ``usage`` must stay empty whenever this record is present. + + Deterministic accounting span IDs (generic transport): + + - chat-span: existing chat span id for ``call_id``; no extra span + - turn-accounting-span: ``accounting:{session_id}:{turn_id}:{accounting_id}`` + - session-accounting-span: ``accounting:{session_id}:{accounting_id}`` + """ + + accounting_id: str + usage: dict[str, Any] + attribution_status: str + agent_id: str | None + call_id: str | None + attributes: dict[str, Any] + + +class SessionAccountingJobDict(TypedDict): + """Durable export job when no user turn owns the accounting record.""" + + job_id: str + kind: Literal["session_accounting"] + session_id: str + accounting_id: str + destination: Literal["session-accounting-span"] + attempt: int + state: Literal["queued", "claimed", "emitted", "failed"] + usage: dict[str, Any] + attribution_status: str + span_id: str + # The durable generic job can retain an agent owner even though there is + # intentionally no fabricated user-turn owner. + agent_id: NotRequired[str | None] + attributes: NotRequired[dict[str, Any]] + # Worker envelope fields remain optional so the serializable public job + # shape above is usable by placement ledgers without filesystem context. + session_dir: NotRequired[str] + platform: NotRequired[str] + cwd: NotRequired[str] + captured_attributes: NotRequired[dict[str, Any]] + + +class TurnAccountingJobDict(TypedDict): + """Durable export job for unmatched usage owned by a user turn or agent.""" + + job_id: str + kind: Literal["turn_accounting"] + session_id: str + turn_id: str + accounting_id: str + destination: Literal["turn-accounting-span"] + attempt: int + state: Literal["queued", "claimed", "emitted", "failed"] + usage: dict[str, Any] + attribution_status: str + agent_id: str | None + span_id: str + attributes: NotRequired[dict[str, Any]] + # Worker envelope fields remain optional so the serializable public job + # shape above is usable by placement ledgers without filesystem context. + session_dir: NotRequired[str] + platform: NotRequired[str] + cwd: NotRequired[str] + captured_attributes: NotRequired[dict[str, Any]] + turn_span_id: NotRequired[str] + + +class AccountingLedgerEntryDict(TypedDict): + """Export-eligibility ledger row; lives in a file/lock apart from projection. + + ``emitted`` is remote-delivery success, not "configured" or "queued". + Once true for a destination, later local matching cannot emit the same + ``accounting_id`` on a different destination. + """ + + accounting_id: str + destination: AccountingDestination + span_id: str + emitted: bool + last_error: str | None + + TurnStatus = Literal["completed", "interrupted", "errored"] @@ -129,3 +229,7 @@ class TurnSpanDict(TypedDict): attributes: dict[str, Any] # Optional: Cursor interactions exported as spans. interactions: NotRequired[list[InteractionSpanDict]] + # Optional local accounting that may be exported on the owning chat span + # (AccountingCallSpanDict.call_id set) or an explicit user-turn/agent + # accounting span (call_id null), never both. + accounting_calls: NotRequired[list[AccountingCallSpanDict]] diff --git a/src/thirdeye/turns.py b/src/thirdeye/turns.py index 55e825b5..6a749300 100644 --- a/src/thirdeye/turns.py +++ b/src/thirdeye/turns.py @@ -4,6 +4,7 @@ from typing import Any from thirdeye.meta import SessionMeta +from thirdeye.platforms.copilot.projection_store import read_projected_turns from thirdeye.store import Store @@ -29,6 +30,15 @@ def _record(meta: SessionMeta, events: list[dict[str, Any]]) -> dict[str, Any]: def session_turns(meta: SessionMeta, store: Store) -> list[dict[str, Any]]: """Return durable top-level turn slices reconstructed from stored events.""" + if meta.platform == "copilot": + # Copilot's transcript interleaves child-agent messages with the main + # interaction, and its native turn IDs reset for each model cycle. + # The V2 projection has already reconstructed completed main user + # interactions from explicit identities. Reading it here preserves + # the normal turn-record shape without treating an intermediate or + # child assistant message as the end of a user turn. + return read_projected_turns(store.config, meta.session_id) + events = list(store.reader(meta.session_id).iter_events()) if meta.platform == "codex": starts = [i for i, event in enumerate(events) if event.get("t") == "agent_turn"] diff --git a/src/thirdeye/web/routes/usage.py b/src/thirdeye/web/routes/usage.py index 54483ffd..f037a52c 100644 --- a/src/thirdeye/web/routes/usage.py +++ b/src/thirdeye/web/routes/usage.py @@ -15,6 +15,29 @@ from thirdeye.usage.read import iter_calls +def _copilot_usage_display(rows: list, labels: dict[str, dict]) -> list[dict]: + display = [] + for row in rows: + extra = labels.get(row.call_id, {}) + display.append( + { + "seq": row.seq, + "response_model": row.response_model, + "input_tokens": row.input_tokens, + "output_tokens": row.output_tokens, + "cache_read_input_tokens": row.cache_read_input_tokens, + "cache_creation_input_tokens": row.cache_creation_input_tokens, + "reasoning_output_tokens": row.reasoning_output_tokens, + "total_tokens": row.total_tokens, + "ts": row.ts, + "copilot_billing_nano_aiu": extra.get("copilot.billing.nano_aiu"), + "duration_ms": extra.get("copilot.latency.duration_ms"), + "output_ttft_ms": extra.get("copilot.latency.output_ttft_ms"), + } + ) + return display + + async def _session_usage(request: Request) -> HTMLResponse: prefix = request.path_params["sid"] store = request.app.state.store @@ -25,12 +48,24 @@ async def _session_usage(request: Request) -> HTMLResponse: raise HTTPException(status_code=404, detail=str(e)) from e sdir = session_dir(config.root, platform, sid) rows = list(iter_calls(sdir)) + labels = {} + if platform == "copilot": + from thirdeye.platforms.copilot.projection_store import read_usage_labels + + labels = read_usage_labels(config, sid) + rows = _copilot_usage_display(rows, labels) aggregate = store.stats(session_id=sid) templates = request.app.state.templates return templates.TemplateResponse( request, "usage/session.html", - {"rows": rows, "aggregate": aggregate, "sid": sid, "platform": platform}, + { + "rows": rows, + "aggregate": aggregate, + "sid": sid, + "platform": platform, + "show_copilot_labels": platform == "copilot", + }, ) diff --git a/src/thirdeye/web/templates/usage/global.html b/src/thirdeye/web/templates/usage/global.html index 4b5ee499..4826500b 100644 --- a/src/thirdeye/web/templates/usage/global.html +++ b/src/thirdeye/web/templates/usage/global.html @@ -11,7 +11,7 @@ + {% if show_copilot_labels %} + + + + {% endif %} @@ -46,6 +51,11 @@

session usage

+ {% if show_copilot_labels %} + + + + {% endif %} diff --git a/src/thirdeye/writer.py b/src/thirdeye/writer.py index 2be320c5..bda9cb3e 100644 --- a/src/thirdeye/writer.py +++ b/src/thirdeye/writer.py @@ -105,8 +105,15 @@ def open( write_meta(meta_path(session_dir), meta) return cls(session_dir, meta) - def append(self, t: str, data: Any = None) -> int: - ts = _utc_iso_ms() + def append(self, t: str, data: Any = None, *, ts: str | None = None) -> int: + """Append an event, optionally retaining a source-provided timestamp. + + Most producers use the default observation time. Importers which + archive an external, immutable event may supply its already validated + source timestamp instead; this deliberately does not change the + default writer behaviour for live capture. + """ + ts = ts or _utc_iso_ms() with self._locked(): # Every hook invocation is a fresh process constructing its own # SessionWriter, so `self._next_seq` can be stale by the time diff --git a/tests/test_codex_captured_env.py b/tests/platforms/codex/test_captured_env.py similarity index 100% rename from tests/test_codex_captured_env.py rename to tests/platforms/codex/test_captured_env.py diff --git a/tests/platforms/copilot/fixtures/README.md b/tests/platforms/copilot/fixtures/README.md index 5a2fa429..5999e340 100644 --- a/tests/platforms/copilot/fixtures/README.md +++ b/tests/platforms/copilot/fixtures/README.md @@ -1,6 +1,6 @@ # Copilot CLI capture probe -Captured 2026-09-10 on macOS using Copilot CLI 1.0.83, with automatic model selection (resolved to gpt-5.6-luna). These are observed fixtures for adapter design, not an implemented adapter or a regression test suite. +Captured 2026-09-10 on macOS using Copilot CLI 1.0.83, with automatic model selection (resolved to gpt-5.6-luna). These are observed fixtures for adapter design and archive-replay regression input; they are not live validation of another Copilot runtime, native VS Code, or cloud history. ## Successful scenario @@ -27,7 +27,7 @@ Sanitization replaces the local home/workspace paths and removes system messages - External pre/post tool hooks have no invocation ID in this run. The transcript provides `toolCallId` on requests and executions. Correlate parallel tools using the transcript, not just tool names. - `turnId` denotes a model/tool cycle and resets across user requests. Group user interactions using `interactionId` plus agent identity. - Subagent events are interleaved in the same transcript and carry top-level `agentId`; `subagent.started` links to the parent task via `data.toolCallId`. Child records also expose `parentToolCallId` where applicable. -- The child generated its own user-prompt and agent-stop hooks using the parent's session ID and transcript path. There are three prompt hooks and three stop hooks for two top-level user requests. External hook payloads alone cannot reliably distinguish these turns. +- The child generated its own user-prompt and agent-stop hooks. Those payloads set `sessionId` to the child agent ID (`bf8cb9f3-2097-4db0-a3c8-78a2653b2106`), not the parent session ID. Child pre/post tool hooks do the same. `transcriptPath` on the child's agentStop still points at the parent session's `events.jsonl`. `subagentStart`/`subagentStop` keep the parent session ID and name the child in `agentId`. Sanitization did not rewrite these identifiers; transcript `hook.start` input matches the external recorder. Hook `sessionId` is therefore not a native session ID for capture routing. Child prompt/stop hooks cannot drive a session-ID-only turn state machine. There are three prompt hooks and three stop hooks for two top-level user requests. - SessionStart arrived after the first user-prompt hook. Initialization must tolerate that ordering. - There are 20 external recorder files but only 18 hook.start/hook.end pairs in the final transcript. Do not assume those two streams have one-to-one coverage. - Assistant text and model names are present in assistant.message. Final session.shutdown includes input/output/cache/reasoning usage and per-agent metrics. Availability of per-call usage at Stop time has not been established by this probe. @@ -49,3 +49,28 @@ Persisted usage checkpoint events contain aggregate billing and cache-frontier d The raw transcript also contains two auxiliary model.model_call_success records for gpt-4o-mini session-title generation, with request/response content, usage and latency. These were omitted from the sanitized transcript. Do not treat them as the main-agent model-call history or add their usage to the six-row total without explicitly accounting for auxiliary calls. The database also has sessions, turns, checkpoints, session_files, session_refs and full-text search tables. Our session has two complete turns but no session_files rows despite four file reads, so the database's discovery/index tables do not replace raw tool execution events. Files beside the transcript include workspace.yaml (identity/repo/title), checkpoints/index.md (empty here), and rewind-file-snapshots/tracking.json (tracking metadata only here). + +## V2 replay use + +V2 tests construct a V1 archive from this corpus through capture APIs, then +reconcile that archive. They do not require a generated archive fixture or the +original Copilot home. The six `assistant_usage_events` rows are the primary +accounting ledger: checkpoint/shutdown values only validate their totals, and +the raw `model.*` title-generation records must remain auxiliary. + +The reconciler uses source identities, interaction/agent identifiers, tool-call +IDs, row revisions, and retained evidence. It does not treat bare transcript +`turnId`, hook timing, tool name, row count, or a nearest timestamp as proof of +an exact relationship. Since the observed SQLite rows lack a shared +assistant-message/provider-call ID, a uniquely consistent mapping is marked +inferred with evidence; competing candidates stay ambiguous and unresolved rows +stay pending or conflicting. Database generation/row-ID reuse quarantines both +logical identities and emits `usage_row_id_reuse`; incompatible revisions of +one call are `usage_revision_conflict`. + +`reconciliation-cases/` contains schema-derived synthetic fixtures for +permission outcomes, compaction, aborts, unknown versions, delayed/revised +database rows, retries, identical concurrent tools, nested children, and +uncertain attribution. Permission and compaction cases emit semantic events; +abort closes a reconstructed turn as `interrupted`. Those cases are synthetic +regression inputs, not claims about live Copilot behavior. diff --git a/tests/platforms/copilot/fixtures/cases/README.md b/tests/platforms/copilot/fixtures/cases/README.md new file mode 100644 index 00000000..dfeca25c --- /dev/null +++ b/tests/platforms/copilot/fixtures/cases/README.md @@ -0,0 +1,48 @@ +# Copilot synthetic contract cases + +Every file in this directory is synthetic. It documents capture contracts and +reader/archive edge cases; it is not evidence of Copilot CLI behavior. + +`source-slice.json` and `source-batch.json` are small, JSON-round-trippable +values for independent reader, archive, and installer tests. They include the +full source-record envelope with schema version `1` in each event payload. +`exhausted` belongs only to `SourceSlice`; `SourceBatch` must not carry it. + +`trailing-json.jsonl` ends with an incomplete JSON record. `trailing-utf8.hex` +is the bytes of a JSONL file whose final UTF-8 code point is incomplete; tests +that need raw bytes should decode the hex, rather than silently converting it +to replacement characters. A reader must defer both tails until a later read. + +`unknown-event-fields.jsonl` requires lossless retention of fields that V1 does +not interpret. `database-row-revisions.json` represents a row-ID reuse and a +changed row revision: equal logical snapshots deduplicate, changed content is a +new evidence record. `distinct-hook-observations.json` contains identical +payloads with distinct observation IDs; both records must survive. +`missing-event-id.jsonl` requires a locator built from file generation, byte +location, and content digest rather than an invented native event ID. +`source-key-prefix-collision.json` is the archive-reuse contract for two +homes whose SHA-256 digests share a 16-character display prefix. + +The observed sibling fixture has two main prompts and one child +prompt. The child prompt is retained as source evidence, not asserted to be a +third human request. Its six `assistant_usage_events` rows remain raw database +evidence and must not create UsageStore rows. + +## Identity and revision handoff + +The selected source home is canonicalized and SHA-256 hashed in full. Stored +session IDs use only the first 16 digest characters for readability. +`stored_session_id` checks that one `SourcePaths` value is internally +consistent (full digest matches that home). Two legitimate homes whose +digests share a 16-character prefix still produce the same stored ID; see +`source-key-prefix-collision.json`. On reuse the archive must compare the +full `source_key` retained in session metadata with the candidate home and +refuse to merge a prefix collision. Distinct native IDs from different homes +must not merge either. + +Transcript identity is source key + native session + native event ID. Without +an event ID it is file generation + byte location + content digest and is +explicitly lower confidence. Database identity is table + row primary key + +content revision; its generation remains separate to expose replacement or +row-ID reuse. Hook identity is a generated observation ID: same-content hooks +are independent observations, while rereading one spool entry is idempotent. diff --git a/tests/platforms/copilot/fixtures/cases/database-row-revisions.json b/tests/platforms/copilot/fixtures/cases/database-row-revisions.json new file mode 100644 index 00000000..c2872747 --- /dev/null +++ b/tests/platforms/copilot/fixtures/cases/database-row-revisions.json @@ -0,0 +1,5 @@ +[ + {"table":"turns","primary_key":7,"generation":"db-generation-a","content_revision":"sha256:aaa","row":{"id":7,"session_id":"session-a","content":"first"}}, + {"table":"turns","primary_key":7,"generation":"db-generation-a","content_revision":"sha256:bbb","row":{"id":7,"session_id":"session-a","content":"updated"}}, + {"table":"turns","primary_key":7,"generation":"db-generation-b","content_revision":"sha256:ccc","row":{"id":7,"session_id":"session-a","content":"reused after replacement"}} +] diff --git a/tests/platforms/copilot/fixtures/cases/distinct-hook-observations.json b/tests/platforms/copilot/fixtures/cases/distinct-hook-observations.json new file mode 100644 index 00000000..614ebeb2 --- /dev/null +++ b/tests/platforms/copilot/fixtures/cases/distinct-hook-observations.json @@ -0,0 +1,4 @@ +[ + {"observation_id":"hook-observation-1","event":"agentStop","payload":{"sessionId":"session-a","stopReason":"end_turn"}}, + {"observation_id":"hook-observation-2","event":"agentStop","payload":{"sessionId":"session-a","stopReason":"end_turn"}} +] diff --git a/tests/platforms/copilot/fixtures/cases/missing-event-id.jsonl b/tests/platforms/copilot/fixtures/cases/missing-event-id.jsonl new file mode 100644 index 00000000..ae7f8970 --- /dev/null +++ b/tests/platforms/copilot/fixtures/cases/missing-event-id.jsonl @@ -0,0 +1 @@ +{"type":"assistant.message","timestamp":"2026-09-10T17:08:24.000Z","data":{"content":"no native ID"}} diff --git a/tests/platforms/copilot/fixtures/cases/source-batch.json b/tests/platforms/copilot/fixtures/cases/source-batch.json new file mode 100644 index 00000000..ab7ec416 --- /dev/null +++ b/tests/platforms/copilot/fixtures/cases/source-batch.json @@ -0,0 +1,18 @@ +{ + "source_key": "3c27ee5a0430219b982434b040245491be12dc57e270dcd2461e65d3a1f49839", + "native_session_id": "session-a", + "cwd": "/fixture/workspace", + "records": [ + { + "source_id": "3c27ee5a0430219b982434b040245491be12dc57e270dcd2461e65d3a1f49839/session-a/row-7/revision-a", + "source_kind": "database", + "native_session_id": "session-a", + "ts": null, + "observed_at": "2026-09-10T17:08:25.000Z", + "payload": {"schema_version": 1, "table": "assistant_usage_events", "row": {"id": 7}}, + "locator": {"table": "assistant_usage_events", "primary_key": 7, "content_revision": "revision-a"} + } + ], + "next_cursor": {"database_revision": "fixture-revision-1"}, + "diagnostics": [] +} diff --git a/tests/platforms/copilot/fixtures/cases/source-key-prefix-collision.json b/tests/platforms/copilot/fixtures/cases/source-key-prefix-collision.json new file mode 100644 index 00000000..3f9cd990 --- /dev/null +++ b/tests/platforms/copilot/fixtures/cases/source-key-prefix-collision.json @@ -0,0 +1,17 @@ +{ + "label": "synthetic", + "native_session_id": "session-a", + "note": "Two distinct SHA-256 source keys that share a 16-character display prefix. stored_session_id would emit the same copilot-- value for both. Detecting the collision requires comparing the full source_key retained in session metadata on reuse; validating that one SourcePaths object is internally consistent is not sufficient.", + "homes": [ + { + "home": "/synthetic/copilot-home-a", + "source_key": "aaaaaaaaaaaaaaaa111111111111111111111111111111111111111111111111" + }, + { + "home": "/synthetic/copilot-home-b", + "source_key": "aaaaaaaaaaaaaaaa222222222222222222222222222222222222222222222222" + } + ], + "retained_metadata_source_key": "aaaaaaaaaaaaaaaa111111111111111111111111111111111111111111111111", + "colliding_stored_session_id": "copilot-aaaaaaaaaaaaaaaa-session-a" +} diff --git a/tests/platforms/copilot/fixtures/cases/source-slice.json b/tests/platforms/copilot/fixtures/cases/source-slice.json new file mode 100644 index 00000000..1fb1c36b --- /dev/null +++ b/tests/platforms/copilot/fixtures/cases/source-slice.json @@ -0,0 +1,17 @@ +{ + "records": [ + { + "source_id": "3c27ee5a0430219b982434b040245491be12dc57e270dcd2461e65d3a1f49839/session-a/event-1", + "source_kind": "transcript", + "native_session_id": "session-a", + "ts": "2026-09-10T17:08:24.000Z", + "observed_at": "2026-09-10T17:08:25.000Z", + "payload": {"schema_version": 1, "type": "user.message", "data": {"text": "fixture"}}, + "locator": {"file_generation": "generation-1", "byte_offset": 0, "native_event_id": "event-1"} + } + ], + "next_cursor": {"byte_offset": 128, "file_generation": "generation-1"}, + "diagnostics": [], + "cwd": "/fixture/workspace", + "exhausted": false +} diff --git a/tests/platforms/copilot/fixtures/cases/trailing-json.jsonl b/tests/platforms/copilot/fixtures/cases/trailing-json.jsonl new file mode 100644 index 00000000..40f7e3e7 --- /dev/null +++ b/tests/platforms/copilot/fixtures/cases/trailing-json.jsonl @@ -0,0 +1,2 @@ +{"id":"complete-1","type":"known"} +{"id":"incomplete-2","type":"unterminated" diff --git a/tests/platforms/copilot/fixtures/cases/trailing-utf8.hex b/tests/platforms/copilot/fixtures/cases/trailing-utf8.hex new file mode 100644 index 00000000..f311deb1 --- /dev/null +++ b/tests/platforms/copilot/fixtures/cases/trailing-utf8.hex @@ -0,0 +1 @@ +7b226964223a22636f6d706c6574652d31227d0a7b226964223a22747261696c696e672d75746638222c2274657874223a22e282 diff --git a/tests/platforms/copilot/fixtures/cases/unknown-event-fields.jsonl b/tests/platforms/copilot/fixtures/cases/unknown-event-fields.jsonl new file mode 100644 index 00000000..5eadbcd0 --- /dev/null +++ b/tests/platforms/copilot/fixtures/cases/unknown-event-fields.jsonl @@ -0,0 +1 @@ +{"id":"unknown-1","type":"future.event","timestamp":"2026-09-10T17:08:24.000Z","data":{"future_field":{"nested":[1,true,{"opaque":"retain"}]},"known":"also retain"},"top_level_future":"retain"} diff --git a/tests/platforms/copilot/fixtures/reconciliation-cases/README.md b/tests/platforms/copilot/fixtures/reconciliation-cases/README.md new file mode 100644 index 00000000..1cdbd47e --- /dev/null +++ b/tests/platforms/copilot/fixtures/reconciliation-cases/README.md @@ -0,0 +1,101 @@ +# Copilot V2 reconciliation contracts + +These JSON documents are serialized DTO examples for projection consumers. +`input_records` arrays are V1 `SourceRecord` envelopes and may be passed +directly to `build_semantics` / `build_accounting`. They are not generated +archives: production tests that need a Store still create archives through +the V1 capture APIs from the observed sibling corpus or from these records. + +Placeholders for the live home digest and database file-stat generation are +documented in `identities.json`. Observed token totals remain authoritative +in `../assistant-usage-events.json` and `../usage.json`. + +| File | Role | +|---|---| +| `identities.json` | Shared source-key / generation / ID templates | +| `semantic-projection.json` | Transcript `SourceRecord` input and `SemanticProjection` output, including a `TurnSpanDict` | +| `accounting-projection.json` | Database `SourceRecord` input and `AccountingProjection` output | +| `storage.json` | `ProjectionState` plus `read_projected_turns` Store-event shape | +| `transport.json` | Generic `AccountingCallSpanDict`, turn-owned unmatched job, session job, ledger | +| `observed-six-calls.json` | Six observed rows, inferred call mapping, shutdown totals | +| `attributions.json` | Matched (inferred), pending, ambiguous, conflicting examples | +| `diagnostics.json` | `PendingItem` and `ProjectionDiagnostic` examples | +| `cases.json` | Concrete records and expected outputs for every required edge case | + +## V1 source-record shapes + +Transcript IDs are `{full_source_key}/{native_id}/{event_id}`. Payloads are +the raw JSONL object plus `schema_version`. Event type is `type` (not +`event`); `id`, `timestamp`, `parentId`, and `agentId` are top-level. +Tool IDs are `data.toolRequests[].toolCallId`. Model cycles end with +`assistant.turn_end` (`data.turnId` only). Locators use `file`, +`file_generation`, `byte_offset`, `byte_length`, and `native_event_id`. + +Database IDs are +`copilot-db:{full_source_key}:{native_id}:{table}:{quoted_canonical_pk}:{revision}` +with no generation. Locators include `database`. `ts` comes from the row's +`created_at`. V1 source IDs omit generation, so an identical row in a new +database file deduplicates; the logical call ID uses the first archived +record's locator generation. + +Hook IDs are `hook/{native_id}/{observation_id}`. + +## Derived identities + +`` is always the full 64-character SHA-256 digest, never the +16-character stored-session prefix. + +- semantic event: `copilot:event:` +- extra events from one record: `copilot:event::tool:` + (or `:` when no native suffix exists) +- semantic call: `copilot:call:` +- main turn: `copilot:turn:::` +- child turn: the main form plus `:` so agents that share an + `interactionId` cannot collide. Main interactions omit the suffix. +- logical call / accounting span: + `copilot:usage::
cache read cache creation reasoningcopilot native billing (nano-AIU)duration_msTTFT (ms)total ts
{{ row.cache_read_input_tokens if row.cache_read_input_tokens is not none else "-" }} {{ row.cache_creation_input_tokens if row.cache_creation_input_tokens is not none else "-" }} {{ row.reasoning_output_tokens if row.reasoning_output_tokens is not none else "-" }}{{ row.copilot_billing_nano_aiu if row.copilot_billing_nano_aiu is not none else "-" }}{{ row.duration_ms if row.duration_ms is not none else "-" }}{{ row.output_ttft_ms if row.output_ttft_ms is not none else "-" }}{{ row.total_tokens }} {{ row.ts }}
::` + +`quoted-generation` and `quoted-canonical-pk` are `urllib.parse.quote(..., safe="")` +of the locator generation (`sha256:`, colon becomes `%3A`) and of +canonical JSON of `locator.primary_key`. Content revision is excluded from +the logical ID. `Attribution.usage_source_id` is the newest revision's source +ID; durable attribution is keyed by `logical_call_id`. + +## UsageRow gaps + +A `UsageRow` is emitted only when timestamp, model, input_tokens, and +output_tokens are all present. Otherwise keep the `AccountingCandidate` and +add `missing_usage_fields`. Present partial counts go in `supplemental_metrics`. +Omit JSON-null metric keys (absent, never null). In-memory `seq` is `0` until +persistence stamps the Store seq of the newest archived revision. The SQLite +primary key is never `seq`. Unknown provider is the UsageRow sentinel +`unknown`; candidate `provider` stays null when the database has none. +Nano-AI-unit billing stays in supplemental metrics. + +## Attribution + +Statuses are `matched`, `pending`, `ambiguous`, and `conflicting`. +`join_kind` is `inferred` or `direct` only on a match. Evidence strings are +`key:value` using `AttributionEvidenceKey`. Ambiguous listings include every +candidate `call_id:...` and choose none. + +`Attribution.status=conflicting` is a join conflict. Incompatible database +revisions are `ProjectionDiagnostic.code=usage_revision_conflict` and +quarantine the logical call. + +## Turns, storage, export + +`read_projected_turns` returns Store events (`seq` / `t` / `ts` / `data`) +for completed main interactions only. Child evidence stays inside them. + +`AccountingCallSpanDict.call_id` set means export on that chat span; null +means a user-turn or agent accounting span. `LlmCallSpanDict.usage` stays +empty whenever `accounting_calls` is present. Span IDs: + +- chat-span: existing chat span for `call_id` +- turn-accounting-span: `accounting:{session_id}:{turn_id}:{accounting_id}` +- session-accounting-span: `accounting:{session_id}:{accounting_id}` + +Queued or configured export is not successful delivery. Auxiliary +`model.*` title-generation records use `kind=auxiliary_model_call` and +`classification=title_generation` and cannot inflate the six-call total. diff --git a/tests/platforms/copilot/fixtures/reconciliation-cases/accounting-projection.json b/tests/platforms/copilot/fixtures/reconciliation-cases/accounting-projection.json new file mode 100644 index 00000000..a11851c6 --- /dev/null +++ b/tests/platforms/copilot/fixtures/reconciliation-cases/accounting-projection.json @@ -0,0 +1,121 @@ +{ + "note": "input_records are V1 database SourceRecords. Pure-function tests may pass them directly to build_accounting.", + "input_records": [ + { + "source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "source_kind": "database", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.498Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "table": "assistant_usage_events", + "row": { + "id": 13, + "session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "turn_index": 0, + "agent_id": null, + "parent_tool_call_id": null, + "model": "gpt-5.6-luna", + "input_tokens": 6452, + "output_tokens": 107, + "cache_read_tokens": 0, + "cache_write_tokens": 6449, + "reasoning_tokens": 29, + "total_nano_aiu": 174125000, + "request_multiplier": 1.0, + "duration_ms": 2263, + "time_to_first_token_ms": 1571.1477920000002, + "output_ttft_ms": 1571.1479590000001, + "inter_token_latency_ms": 7.393025117647059, + "initiator": "user", + "api_endpoint": "ws:/responses", + "reasoning_effort": "medium", + "finish_reason": "tool_calls", + "content_filter_triggered": 0, + "token_details_json": "[{\"batchSize\":1000000,\"costPerBatch\":20000000000,\"tokenCount\":3,\"tokenType\":\"input\"},{\"batchSize\":1000000,\"costPerBatch\":2000000000,\"tokenCount\":0,\"tokenType\":\"cache_read\"},{\"batchSize\":1000000,\"costPerBatch\":25000000000,\"tokenCount\":6449,\"tokenType\":\"cache_write\"},{\"batchSize\":1000000,\"costPerBatch\":120000000000,\"tokenCount\":107,\"tokenType\":\"output\"}]", + "created_at": "2026-09-10T17:08:24.498Z" + } + }, + "locator": { + "database": "/example/.copilot/session-store.db", + "table": "assistant_usage_events", + "primary_key": 13, + "content_revision": "sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "generation": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + } + } + ], + "expected": { + "usage_rows": [ + { + "session_id": "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "seq": 0, + "call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13", + "ts": "2026-09-10T17:08:24.498Z", + "platform": "copilot", + "gen_ai.conversation.id": "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "gen_ai.provider.name": "unknown", + "gen_ai.operation.name": "chat", + "gen_ai.response.model": "gpt-5.6-luna", + "gen_ai.usage.input_tokens": 6452, + "gen_ai.usage.output_tokens": 107, + "gen_ai.usage.cache_read.input_tokens": 0, + "gen_ai.usage.cache_creation.input_tokens": 6449, + "gen_ai.usage.reasoning.output_tokens": 29 + } + ], + "candidates": [ + { + "usage_source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13", + "turn_index": 0, + "agent_id": null, + "parent_tool_call_id": null, + "model": "gpt-5.6-luna", + "provider": null, + "source_ids": [ + "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47" + ], + "source_references": [ + { + "source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "source_kind": "database", + "role": "usage_row" + } + ], + "revision": { + "table": "assistant_usage_events", + "primary_key": "13", + "generation": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "content_revision": "sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47" + }, + "timestamp": "2026-09-10T17:08:24.498Z", + "finish_reason": "tool_calls", + "supplemental_metrics": { + "total_nano_aiu": 174125000, + "request_multiplier": 1.0, + "initiator": "user", + "api_endpoint": "ws:/responses", + "reasoning_effort": "medium", + "content_filter_triggered": 0, + "token_details_json": "[{\"batchSize\":1000000,\"costPerBatch\":20000000000,\"tokenCount\":3,\"tokenType\":\"input\"},{\"batchSize\":1000000,\"costPerBatch\":2000000000,\"tokenCount\":0,\"tokenType\":\"cache_read\"},{\"batchSize\":1000000,\"costPerBatch\":25000000000,\"tokenCount\":6449,\"tokenType\":\"cache_write\"},{\"batchSize\":1000000,\"costPerBatch\":120000000000,\"tokenCount\":107,\"tokenType\":\"output\"}]", + "duration_ms": 2263, + "time_to_first_token_ms": 1571.1477920000002, + "output_ttft_ms": 1571.1479590000001, + "inter_token_latency_ms": 7.393025117647059 + } + } + ], + "diagnostics": [ + { + "code": "unknown_provider", + "severity": "info", + "message": "database usage rows do not name a provider; UsageRow uses the unknown sentinel", + "source_ids": [ + "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47" + ], + "details": {} + } + ] + } +} diff --git a/tests/platforms/copilot/fixtures/reconciliation-cases/attributions.json b/tests/platforms/copilot/fixtures/reconciliation-cases/attributions.json new file mode 100644 index 00000000..23295bff --- /dev/null +++ b/tests/platforms/copilot/fixtures/reconciliation-cases/attributions.json @@ -0,0 +1,67 @@ +{ + "matched_inferred": { + "usage_source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13", + "stored_turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42", + "agent_id": null, + "call_id": "copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "status": "matched", + "join_kind": "inferred", + "evidence": [ + "join_kind:inferred", + "interaction_id:6d2b89fd-a653-430c-b532-b0936d72eb42", + "turn_index:0", + "agent_id:None", + "parent_tool_call_id:None", + "model:gpt-5.6-luna", + "finish:tool_calls", + "initiator:user", + "order:0", + "call_id:copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328" + ] + }, + "pending": { + "usage_source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:14:sha256:20cb0f98d6130c8d0e139391cfc8e245d1d0ac96d41469c4da60f26e6d9e0a26", + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:14", + "stored_turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42", + "agent_id": null, + "call_id": null, + "status": "pending", + "join_kind": null, + "evidence": [ + "logical_call_id:copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:14", + "interaction_id:6d2b89fd-a653-430c-b532-b0936d72eb42", + "turn_index:0" + ] + }, + "ambiguous": { + "usage_source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13", + "stored_turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42", + "agent_id": null, + "call_id": null, + "status": "ambiguous", + "join_kind": null, + "evidence": [ + "call_id:copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "call_id:copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/33cc6465-29e1-4a04-8bdb-00241474b4d2", + "interaction_id:6d2b89fd-a653-430c-b532-b0936d72eb42", + "model:gpt-5.6-luna", + "turn_index:0" + ] + }, + "conflicting_join": { + "usage_source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13", + "stored_turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42", + "agent_id": null, + "call_id": null, + "status": "conflicting", + "join_kind": null, + "evidence": [ + "call_id:copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "call_id:copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/33cc6465-29e1-4a04-8bdb-00241474b4d2", + "join_conflict:rebinding_after_emission" + ] + } +} diff --git a/tests/platforms/copilot/fixtures/reconciliation-cases/cases.json b/tests/platforms/copilot/fixtures/reconciliation-cases/cases.json new file mode 100644 index 00000000..cd46a50d --- /dev/null +++ b/tests/platforms/copilot/fixtures/reconciliation-cases/cases.json @@ -0,0 +1,1742 @@ +{ + "permission": { + "observed": false, + "required": "emit permission semantic event; do not infer a tool execution or usage call", + "input_records": [ + { + "source_id": "hook/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/obs-permission-1", + "source_kind": "hook", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.500Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "schema_version": 1, + "event": "permissionRequest", + "hook_payload": { + "sessionId": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "timestamp": 1789060104500, + "cwd": "/fixture/workspace", + "toolName": "view", + "toolArgs": { + "path": "/fixture/workspace/alpha.txt" + } + }, + "context": {} + }, + "locator": { + "observation_id": "obs-permission-1", + "event": "permissionRequest" + } + } + ], + "expected": { + "events": [ + { + "id": "copilot:event:hook/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/obs-permission-1", + "kind": "permission_request", + "classification": "main", + "ts": "2026-09-10T17:08:24.500Z", + "source_ids": [ + "hook/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/obs-permission-1" + ], + "source_references": [ + { + "source_id": "hook/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/obs-permission-1", + "source_kind": "hook", + "role": "permission_request" + } + ], + "attributes": { + "tool_name": "view" + } + } + ], + "call_candidates": [], + "usage_rows": [], + "attributions": [] + } + }, + "compaction": { + "observed": false, + "required": "retain source evidence and diagnostics; never add snapshot usage to per-call accounting", + "input_records": [ + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-checkpoint-1", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:25.700Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "session.usage_checkpoint", + "data": { + "inputTokens": 26416, + "outputTokens": 229, + "cacheReadTokens": 19522, + "cacheWriteTokens": 6882, + "reasoningTokens": 39, + "totalNanoAiu": 238814000 + }, + "id": "synthetic-checkpoint-1", + "timestamp": "2026-09-10T17:08:25.700Z", + "parentId": "0080e44c-ad62-4288-b2b2-061ec2b73d80", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 0, + "byte_length": 297, + "native_event_id": "synthetic-checkpoint-1" + } + }, + { + "source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "source_kind": "database", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.498Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "table": "assistant_usage_events", + "row": { + "id": 13, + "session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "turn_index": 0, + "agent_id": null, + "parent_tool_call_id": null, + "model": "gpt-5.6-luna", + "input_tokens": 6452, + "output_tokens": 107, + "cache_read_tokens": 0, + "cache_write_tokens": 6449, + "reasoning_tokens": 29, + "total_nano_aiu": 174125000, + "request_multiplier": 1.0, + "duration_ms": 2263, + "time_to_first_token_ms": 1571.1477920000002, + "output_ttft_ms": 1571.1479590000001, + "inter_token_latency_ms": 7.393025117647059, + "initiator": "user", + "api_endpoint": "ws:/responses", + "reasoning_effort": "medium", + "finish_reason": "tool_calls", + "content_filter_triggered": 0, + "token_details_json": "[{\"batchSize\":1000000,\"costPerBatch\":20000000000,\"tokenCount\":3,\"tokenType\":\"input\"},{\"batchSize\":1000000,\"costPerBatch\":2000000000,\"tokenCount\":0,\"tokenType\":\"cache_read\"},{\"batchSize\":1000000,\"costPerBatch\":25000000000,\"tokenCount\":6449,\"tokenType\":\"cache_write\"},{\"batchSize\":1000000,\"costPerBatch\":120000000000,\"tokenCount\":107,\"tokenType\":\"output\"}]", + "created_at": "2026-09-10T17:08:24.498Z" + } + }, + "locator": { + "database": "/example/.copilot/session-store.db", + "table": "assistant_usage_events", + "primary_key": 13, + "content_revision": "sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "generation": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + } + } + ], + "expected": { + "events": [ + { + "id": "copilot:event:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-checkpoint-1", + "kind": "compaction", + "classification": "checkpoint", + "ts": "2026-09-10T17:08:25.700Z", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-checkpoint-1" + ], + "source_references": [ + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-checkpoint-1", + "source_kind": "transcript", + "role": "checkpoint" + } + ], + "attributes": {} + } + ], + "usage_rows": [ + { + "session_id": "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "seq": 0, + "call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13", + "ts": "2026-09-10T17:08:24.498Z", + "platform": "copilot", + "gen_ai.conversation.id": "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "gen_ai.provider.name": "unknown", + "gen_ai.operation.name": "chat", + "gen_ai.response.model": "gpt-5.6-luna", + "gen_ai.usage.input_tokens": 6452, + "gen_ai.usage.output_tokens": 107, + "gen_ai.usage.cache_read.input_tokens": 0, + "gen_ai.usage.cache_creation.input_tokens": 6449, + "gen_ai.usage.reasoning.output_tokens": 29 + } + ], + "diagnostics": [ + { + "code": "checkpoint_not_additive", + "severity": "info", + "message": "checkpoint snapshot validates totals and is not a seventh call", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-checkpoint-1" + ], + "details": {} + } + ] + } + }, + "partial_turn": { + "observed": false, + "required": "keep a prompt-only interaction pending; missing usage is absent, not zero", + "input_records": [ + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/f07404d0-af52-4260-89fb-358a10e86034", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:22.203Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "user.message", + "data": { + "content": "Read alpha.txt and beta.txt with separate view calls in parallel and report their sum. Only read those two files.", + "messageId": "a856cb38-7609-45ab-8a55-645553155db3", + "supportedNativeDocumentMimeTypes": [], + "delivery": "idle", + "interactionId": "6d2b89fd-a653-430c-b532-b0936d72eb42", + "turnId": "0", + "parentAgentTaskId": "fd800f7a-8163-4304-9681-efde4731e879" + }, + "id": "f07404d0-af52-4260-89fb-358a10e86034", + "timestamp": "2026-09-10T17:08:22.203Z", + "parentId": "3942810f-1caf-4251-82fb-3bf72698147a", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 2122, + "byte_length": 548, + "native_event_id": "f07404d0-af52-4260-89fb-358a10e86034" + } + } + ], + "expected": { + "turns": [], + "usage_rows": [], + "pending": [ + { + "id": "pending:copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42", + "kind": "open_interaction", + "reason": "user interaction has no assistant.turn_end", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/f07404d0-af52-4260-89fb-358a10e86034" + ], + "evidence": [ + "interaction_id:6d2b89fd-a653-430c-b532-b0936d72eb42" + ] + } + ], + "semantic_state": { + "open_interactions": { + "6d2b89fd-a653-430c-b532-b0936d72eb42|main": { + "interaction_id": "6d2b89fd-a653-430c-b532-b0936d72eb42", + "agent_id": null, + "stored_turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/f07404d0-af52-4260-89fb-358a10e86034" + ], + "last_event_source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/f07404d0-af52-4260-89fb-358a10e86034", + "start_ts": "2026-09-10T17:08:22.203Z", + "pending_tool_call_ids": [] + } + } + } + } + }, + "abort": { + "observed": false, + "required": "assistant.abort closes the user turn as interrupted; do not treat a prompt-only slice as abort", + "input_records": [ + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-abort-user", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.400Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "user.message", + "data": { + "content": "hello", + "interactionId": "ix-abort", + "turnId": "0" + }, + "id": "synthetic-abort-user", + "timestamp": "2026-09-10T17:08:24.400Z", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "native_event_id": "synthetic-abort-user" + } + }, + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-abort-start", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.450Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "assistant.turn_start", + "data": { + "turnId": "0", + "interactionId": "ix-abort" + }, + "id": "synthetic-abort-start", + "timestamp": "2026-09-10T17:08:24.450Z", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "native_event_id": "synthetic-abort-start" + } + }, + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-abort-msg", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.503Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "assistant.message", + "data": { + "content": "partial", + "model": "gpt-5.6-luna", + "interactionId": "ix-abort", + "turnId": "0", + "reasoningSummary": "started answering", + "toolRequests": [] + }, + "id": "synthetic-abort-msg", + "timestamp": "2026-09-10T17:08:24.503Z", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "native_event_id": "synthetic-abort-msg" + } + }, + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-abort-event", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.800Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "assistant.abort", + "data": { + "turnId": "0", + "interactionId": "ix-abort" + }, + "id": "synthetic-abort-event", + "timestamp": "2026-09-10T17:08:24.800Z", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "native_event_id": "synthetic-abort-event" + } + } + ], + "expected": { + "events": [ + { + "id": "copilot:event:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-abort-user", + "kind": "user_prompt", + "classification": "main", + "ts": "2026-09-10T17:08:24.400Z", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-abort-user" + ], + "attributes": { + "interaction_id": "ix-abort" + } + }, + { + "id": "copilot:event:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-abort-event", + "kind": "abort", + "classification": "main", + "ts": "2026-09-10T17:08:24.800Z", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-abort-event" + ] + } + ], + "turns": [ + { + "turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:ix-abort", + "start_ts": "2026-09-10T17:08:24.400Z", + "end_ts": "2026-09-10T17:08:24.800Z", + "input_message": "hello", + "output_message": "partial", + "status": "interrupted", + "llm_calls": [ + { + "call_id": "copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-abort-msg", + "provider": "unknown", + "model": "gpt-5.6-luna", + "start_ts": "2026-09-10T17:08:24.503Z", + "end_ts": "2026-09-10T17:08:24.800Z", + "input_messages": [ + { + "role": "user", + "parts": [ + { + "type": "text", + "content": "hello" + } + ] + } + ], + "output_messages": [ + { + "role": "assistant", + "parts": [ + { + "type": "text", + "content": "partial" + }, + { + "type": "reasoning", + "content": "started answering" + } + ] + } + ], + "usage": {}, + "tool_calls": [] + } + ], + "permission_requests": [], + "subagents": [], + "attributes": { + "interaction_id": "ix-abort", + "agent_id": null + }, + "accounting_calls": [] + } + ], + "call_candidates": [ + { + "call_id": "copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-abort-msg", + "stored_turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:ix-abort", + "interaction_id": "ix-abort", + "agent_id": null, + "parent_tool_call_id": null, + "model": "gpt-5.6-luna", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-abort-msg", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-abort-event" + ], + "start_ts": "2026-09-10T17:08:24.503Z", + "end_ts": "2026-09-10T17:08:24.800Z", + "tool_call_ids": [] + } + ], + "pending": [], + "usage_rows": [] + } + }, + "retry": { + "observed": false, + "required": "stable source-derived IDs make projection replay equivalent without duplicate accounting", + "input_records": [ + { + "source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "source_kind": "database", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.498Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "table": "assistant_usage_events", + "row": { + "id": 13, + "session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "turn_index": 0, + "agent_id": null, + "parent_tool_call_id": null, + "model": "gpt-5.6-luna", + "input_tokens": 6452, + "output_tokens": 107, + "cache_read_tokens": 0, + "cache_write_tokens": 6449, + "reasoning_tokens": 29, + "total_nano_aiu": 174125000, + "request_multiplier": 1.0, + "duration_ms": 2263, + "time_to_first_token_ms": 1571.1477920000002, + "output_ttft_ms": 1571.1479590000001, + "inter_token_latency_ms": 7.393025117647059, + "initiator": "user", + "api_endpoint": "ws:/responses", + "reasoning_effort": "medium", + "finish_reason": "tool_calls", + "content_filter_triggered": 0, + "token_details_json": "[{\"batchSize\":1000000,\"costPerBatch\":20000000000,\"tokenCount\":3,\"tokenType\":\"input\"},{\"batchSize\":1000000,\"costPerBatch\":2000000000,\"tokenCount\":0,\"tokenType\":\"cache_read\"},{\"batchSize\":1000000,\"costPerBatch\":25000000000,\"tokenCount\":6449,\"tokenType\":\"cache_write\"},{\"batchSize\":1000000,\"costPerBatch\":120000000000,\"tokenCount\":107,\"tokenType\":\"output\"}]", + "created_at": "2026-09-10T17:08:24.498Z" + } + }, + "locator": { + "database": "/example/.copilot/session-store.db", + "table": "assistant_usage_events", + "primary_key": 13, + "content_revision": "sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "generation": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + } + }, + { + "source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "source_kind": "database", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.498Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "table": "assistant_usage_events", + "row": { + "id": 13, + "session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "turn_index": 0, + "agent_id": null, + "parent_tool_call_id": null, + "model": "gpt-5.6-luna", + "input_tokens": 6452, + "output_tokens": 107, + "cache_read_tokens": 0, + "cache_write_tokens": 6449, + "reasoning_tokens": 29, + "total_nano_aiu": 174125000, + "request_multiplier": 1.0, + "duration_ms": 2263, + "time_to_first_token_ms": 1571.1477920000002, + "output_ttft_ms": 1571.1479590000001, + "inter_token_latency_ms": 7.393025117647059, + "initiator": "user", + "api_endpoint": "ws:/responses", + "reasoning_effort": "medium", + "finish_reason": "tool_calls", + "content_filter_triggered": 0, + "token_details_json": "[{\"batchSize\":1000000,\"costPerBatch\":20000000000,\"tokenCount\":3,\"tokenType\":\"input\"},{\"batchSize\":1000000,\"costPerBatch\":2000000000,\"tokenCount\":0,\"tokenType\":\"cache_read\"},{\"batchSize\":1000000,\"costPerBatch\":25000000000,\"tokenCount\":6449,\"tokenType\":\"cache_write\"},{\"batchSize\":1000000,\"costPerBatch\":120000000000,\"tokenCount\":107,\"tokenType\":\"output\"}]", + "created_at": "2026-09-10T17:08:24.498Z" + } + }, + "locator": { + "database": "/example/.copilot/session-store.db", + "table": "assistant_usage_events", + "primary_key": 13, + "content_revision": "sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "generation": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + } + } + ], + "expected": { + "usage_rows": [ + { + "session_id": "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "seq": 0, + "call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13", + "ts": "2026-09-10T17:08:24.498Z", + "platform": "copilot", + "gen_ai.conversation.id": "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "gen_ai.provider.name": "unknown", + "gen_ai.operation.name": "chat", + "gen_ai.response.model": "gpt-5.6-luna", + "gen_ai.usage.input_tokens": 6452, + "gen_ai.usage.output_tokens": 107, + "gen_ai.usage.cache_read.input_tokens": 0, + "gen_ai.usage.cache_creation.input_tokens": 6449, + "gen_ai.usage.reasoning.output_tokens": 29 + } + ], + "candidates": [ + { + "usage_source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13", + "turn_index": 0, + "agent_id": null, + "parent_tool_call_id": null, + "model": "gpt-5.6-luna", + "provider": null, + "source_ids": [ + "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47" + ], + "source_references": [ + { + "source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "source_kind": "database", + "role": "usage_row" + } + ], + "revision": { + "table": "assistant_usage_events", + "primary_key": "13", + "generation": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "content_revision": "sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47" + }, + "timestamp": "2026-09-10T17:08:24.498Z", + "finish_reason": "tool_calls", + "supplemental_metrics": { + "total_nano_aiu": 174125000, + "request_multiplier": 1.0, + "initiator": "user", + "api_endpoint": "ws:/responses", + "reasoning_effort": "medium", + "content_filter_triggered": 0, + "token_details_json": "[{\"batchSize\":1000000,\"costPerBatch\":20000000000,\"tokenCount\":3,\"tokenType\":\"input\"},{\"batchSize\":1000000,\"costPerBatch\":2000000000,\"tokenCount\":0,\"tokenType\":\"cache_read\"},{\"batchSize\":1000000,\"costPerBatch\":25000000000,\"tokenCount\":6449,\"tokenType\":\"cache_write\"},{\"batchSize\":1000000,\"costPerBatch\":120000000000,\"tokenCount\":107,\"tokenType\":\"output\"}]", + "duration_ms": 2263, + "time_to_first_token_ms": 1571.1477920000002, + "output_ttft_ms": 1571.1479590000001, + "inter_token_latency_ms": 7.393025117647059 + } + } + ] + } + }, + "nested_child": { + "observed": true, + "required": "child evidence stays within its parent main interaction; session-ID-only ownership is not inferred", + "input_records": [ + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/6dff9fa5-c2df-41da-912b-d1ffd0706076", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:44.058Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "subagent.started", + "data": { + "toolCallId": "call_qx4FH5DADTeT1qVLb37HNpBk", + "agentName": "explore", + "agentDisplayName": "sum-alpha-beta", + "agentDescription": "Sum two text files", + "model": "gpt-5.6-luna", + "resumable": false, + "agentType": "explore", + "executionMode": "sync" + }, + "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", + "id": "6dff9fa5-c2df-41da-912b-d1ffd0706076", + "timestamp": "2026-09-10T17:08:44.058Z", + "parentId": "0049f918-7bb9-478a-8c91-b6f8353525a9", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 15821, + "byte_length": 474, + "native_event_id": "6dff9fa5-c2df-41da-912b-d1ffd0706076" + } + }, + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/d4e30206-94f5-47d5-b15e-7411b9708aec", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:44.557Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "user.message", + "data": { + "content": "Read only /fixture/workspace/alpha.txt and /fixture/workspace/beta.txt. Do not modify files. Do not access any other files or services. Report the sum of the numeric values in those two files.", + "source": "agent-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "supportedNativeDocumentMimeTypes": [], + "delivery": "idle", + "interactionId": "7c0fa097-c0e2-48da-b2b6-fcfc1ad83a6b", + "turnId": "0", + "parentAgentTaskId": "5a3e63ac-c073-40ba-b07a-010d534b363e" + }, + "agentId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", + "id": "d4e30206-94f5-47d5-b15e-7411b9708aec", + "timestamp": "2026-09-10T17:08:44.557Z", + "parentId": "3bd18700-90c6-4b2f-aaaa-63760bdfa89a", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 18718, + "byte_length": 681, + "native_event_id": "d4e30206-94f5-47d5-b15e-7411b9708aec" + } + }, + { + "source_id": "hook/bf8cb9f3-2097-4db0-a3c8-78a2653b2106/obs-child-prompt-1", + "source_kind": "hook", + "native_session_id": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", + "ts": "2026-09-10T17:08:44.530Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "schema_version": 1, + "event": "userPromptSubmitted", + "hook_payload": { + "sessionId": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", + "timestamp": 1789060124530, + "cwd": "/fixture/workspace", + "prompt": "Read only /fixture/workspace/alpha.txt and /fixture/workspace/beta.txt. Do not modify files. Do not access any other files or services. Report the sum of the numeric values in those two files." + }, + "context": {} + }, + "locator": { + "observation_id": "obs-child-prompt-1", + "event": "userPromptSubmitted" + } + } + ], + "expected": { + "main_turn_ids": [ + "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:793d3703-6f4a-4814-8877-34a7325848ce" + ], + "child_owned_by": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:793d3703-6f4a-4814-8877-34a7325848ce", + "child_hook_native_session_id": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", + "do_not_infer_from_session_id_alone": true, + "turns": [ + { + "turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:793d3703-6f4a-4814-8877-34a7325848ce", + "start_ts": "2026-09-10T17:08:42.192Z", + "end_ts": "2026-09-10T17:08:48.287Z", + "input_message": "Invoke one explore subagent to read only alpha.txt and beta.txt and report their sum. Do not modify files or access other files or services. Then report its answer.", + "output_message": "42", + "status": "completed", + "llm_calls": [], + "permission_requests": [], + "subagents": [ + { + "turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:7c0fa097-c0e2-48da-b2b6-fcfc1ad83a6b:bf8cb9f3-2097-4db0-a3c8-78a2653b2106", + "start_ts": "2026-09-10T17:08:44.557Z", + "end_ts": "2026-09-10T17:08:47.463Z", + "input_message": "Read only /fixture/workspace/alpha.txt and /fixture/workspace/beta.txt. Do not modify files. Do not access any other files or services. Report the sum of the numeric values in those two files.", + "output_message": "42", + "status": "completed", + "llm_calls": [], + "permission_requests": [], + "subagents": [], + "attributes": { + "agent_id": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", + "parent_tool_call_id": "call_qx4FH5DADTeT1qVLb37HNpBk", + "agent_name": "explore" + } + } + ], + "attributes": { + "interaction_id": "793d3703-6f4a-4814-8877-34a7325848ce", + "agent_id": null + } + } + ], + "events": [ + { + "id": "copilot:event:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/6dff9fa5-c2df-41da-912b-d1ffd0706076", + "kind": "subagent_started", + "classification": "main", + "ts": "2026-09-10T17:08:44.058Z", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/6dff9fa5-c2df-41da-912b-d1ffd0706076" + ], + "source_references": [ + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/6dff9fa5-c2df-41da-912b-d1ffd0706076", + "source_kind": "transcript", + "role": "nested_child" + } + ], + "attributes": { + "agent_id": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", + "parent_tool_call_id": "call_qx4FH5DADTeT1qVLb37HNpBk" + } + } + ] + } + }, + "late_row": { + "observed": false, + "required": "create/revise accounting locally and leave attribution pending unless evidence is uniquely consistent", + "input_records": [ + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/f07404d0-af52-4260-89fb-358a10e86034", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:22.203Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "user.message", + "data": { + "content": "Read alpha.txt and beta.txt with separate view calls in parallel and report their sum. Only read those two files.", + "messageId": "a856cb38-7609-45ab-8a55-645553155db3", + "supportedNativeDocumentMimeTypes": [], + "delivery": "idle", + "interactionId": "6d2b89fd-a653-430c-b532-b0936d72eb42", + "turnId": "0", + "parentAgentTaskId": "fd800f7a-8163-4304-9681-efde4731e879" + }, + "id": "f07404d0-af52-4260-89fb-358a10e86034", + "timestamp": "2026-09-10T17:08:22.203Z", + "parentId": "3942810f-1caf-4251-82fb-3bf72698147a", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 2122, + "byte_length": 548, + "native_event_id": "f07404d0-af52-4260-89fb-358a10e86034" + } + }, + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.503Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "assistant.message", + "data": { + "messageId": "5a29b09d-5e7d-4603-975e-7801ce54232b", + "model": "gpt-5.6-luna", + "content": "", + "toolRequests": [ + { + "toolCallId": "call_YSSva4HCniiETlxdGGjcrHbh", + "name": "view", + "arguments": { + "path": "/fixture/workspace/alpha.txt" + }, + "type": "function", + "intentionSummary": "view the file at /fixture/workspace/alpha.txt." + }, + { + "toolCallId": "call_ayHplfzxjRFMTCpmTKEFhCSJ", + "name": "view", + "arguments": { + "path": "/fixture/workspace/beta.txt" + }, + "type": "function", + "intentionSummary": "view the file at /fixture/workspace/beta.txt." + } + ], + "interactionId": "6d2b89fd-a653-430c-b532-b0936d72eb42", + "turnId": "0", + "rte": true + }, + "id": "a4a17e63-7ba5-422f-8ee9-b495be417328", + "timestamp": "2026-09-10T17:08:24.503Z", + "parentId": "64f9e436-651f-4a9d-919c-a4d7abad2652", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 3731, + "byte_length": 793, + "native_event_id": "a4a17e63-7ba5-422f-8ee9-b495be417328" + } + }, + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/0080e44c-ad62-4288-b2b2-061ec2b73d80", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.593Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "assistant.turn_end", + "data": { + "turnId": "0" + }, + "id": "0080e44c-ad62-4288-b2b2-061ec2b73d80", + "timestamp": "2026-09-10T17:08:24.593Z", + "parentId": "032cf87e-05b7-4e5f-96b9-1fda1a8242f5", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 9311, + "byte_length": 195, + "native_event_id": "0080e44c-ad62-4288-b2b2-061ec2b73d80" + } + }, + { + "source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:14:sha256:20cb0f98d6130c8d0e139391cfc8e245d1d0ac96d41469c4da60f26e6d9e0a26", + "source_kind": "database", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:25.618Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "table": "assistant_usage_events", + "row": { + "id": 14, + "session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "turn_index": 0, + "agent_id": null, + "parent_tool_call_id": null, + "model": "gpt-5.6-luna", + "input_tokens": 6587, + "output_tokens": 5, + "cache_read_tokens": 6449, + "cache_write_tokens": 135, + "reasoning_tokens": 0, + "total_nano_aiu": 16933000, + "request_multiplier": 1.0, + "duration_ms": 1017, + "time_to_first_token_ms": 942.6416250000001, + "output_ttft_ms": 942.642416, + "inter_token_latency_ms": null, + "initiator": "agent", + "api_endpoint": "ws:/responses", + "reasoning_effort": "medium", + "finish_reason": "stop", + "content_filter_triggered": 0, + "token_details_json": "[{\"batchSize\":1000000,\"costPerBatch\":20000000000,\"tokenCount\":3,\"tokenType\":\"input\"},{\"batchSize\":1000000,\"costPerBatch\":2000000000,\"tokenCount\":6449,\"tokenType\":\"cache_read\"},{\"batchSize\":1000000,\"costPerBatch\":25000000000,\"tokenCount\":135,\"tokenType\":\"cache_write\"},{\"batchSize\":1000000,\"costPerBatch\":120000000000,\"tokenCount\":5,\"tokenType\":\"output\"}]", + "created_at": "2026-09-10T17:08:25.618Z" + } + }, + "locator": { + "database": "/example/.copilot/session-store.db", + "table": "assistant_usage_events", + "primary_key": 14, + "content_revision": "sha256:20cb0f98d6130c8d0e139391cfc8e245d1d0ac96d41469c4da60f26e6d9e0a26", + "generation": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + } + } + ], + "expected": { + "usage_rows": [ + { + "session_id": "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "seq": 0, + "call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:14", + "ts": "2026-09-10T17:08:25.618Z", + "platform": "copilot", + "gen_ai.conversation.id": "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "gen_ai.provider.name": "unknown", + "gen_ai.operation.name": "chat", + "gen_ai.response.model": "gpt-5.6-luna", + "gen_ai.usage.input_tokens": 6587, + "gen_ai.usage.output_tokens": 5, + "gen_ai.usage.cache_read.input_tokens": 6449, + "gen_ai.usage.cache_creation.input_tokens": 135, + "gen_ai.usage.reasoning.output_tokens": 0 + } + ], + "attributions": [ + { + "usage_source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:14:sha256:20cb0f98d6130c8d0e139391cfc8e245d1d0ac96d41469c4da60f26e6d9e0a26", + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:14", + "stored_turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42", + "agent_id": null, + "call_id": null, + "status": "pending", + "join_kind": null, + "evidence": [ + "logical_call_id:copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:14", + "interaction_id:6d2b89fd-a653-430c-b532-b0936d72eb42", + "turn_index:0" + ] + } + ] + } + }, + "revision": { + "observed": false, + "required": "replace the existing logical UsageRow or quarantine conflict; do not add another charge", + "input_records": [ + { + "source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "source_kind": "database", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.498Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "table": "assistant_usage_events", + "row": { + "id": 13, + "session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "turn_index": 0, + "agent_id": null, + "parent_tool_call_id": null, + "model": "gpt-5.6-luna", + "input_tokens": 6452, + "output_tokens": 107, + "cache_read_tokens": 0, + "cache_write_tokens": 6449, + "reasoning_tokens": 29, + "total_nano_aiu": 174125000, + "request_multiplier": 1.0, + "duration_ms": 2263, + "time_to_first_token_ms": 1571.1477920000002, + "output_ttft_ms": 1571.1479590000001, + "inter_token_latency_ms": 7.393025117647059, + "initiator": "user", + "api_endpoint": "ws:/responses", + "reasoning_effort": "medium", + "finish_reason": "tool_calls", + "content_filter_triggered": 0, + "token_details_json": "[{\"batchSize\":1000000,\"costPerBatch\":20000000000,\"tokenCount\":3,\"tokenType\":\"input\"},{\"batchSize\":1000000,\"costPerBatch\":2000000000,\"tokenCount\":0,\"tokenType\":\"cache_read\"},{\"batchSize\":1000000,\"costPerBatch\":25000000000,\"tokenCount\":6449,\"tokenType\":\"cache_write\"},{\"batchSize\":1000000,\"costPerBatch\":120000000000,\"tokenCount\":107,\"tokenType\":\"output\"}]", + "created_at": "2026-09-10T17:08:24.498Z" + } + }, + "locator": { + "database": "/example/.copilot/session-store.db", + "table": "assistant_usage_events", + "primary_key": 13, + "content_revision": "sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "generation": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + } + }, + { + "source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:fe8383e7e562b4986c48b7b03ce880a54fd4ef03ae1721047ca41261e4ff85ca", + "source_kind": "database", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.498Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "table": "assistant_usage_events", + "row": { + "id": 13, + "session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "turn_index": 0, + "agent_id": null, + "parent_tool_call_id": null, + "model": "gpt-5.6-luna", + "input_tokens": 6452, + "output_tokens": 999, + "cache_read_tokens": 0, + "cache_write_tokens": 6449, + "reasoning_tokens": 29, + "total_nano_aiu": 174125000, + "request_multiplier": 1.0, + "duration_ms": 2263, + "time_to_first_token_ms": 1571.1477920000002, + "output_ttft_ms": 1571.1479590000001, + "inter_token_latency_ms": 7.393025117647059, + "initiator": "user", + "api_endpoint": "ws:/responses", + "reasoning_effort": "medium", + "finish_reason": "tool_calls", + "content_filter_triggered": 0, + "token_details_json": "[{\"batchSize\":1000000,\"costPerBatch\":20000000000,\"tokenCount\":3,\"tokenType\":\"input\"},{\"batchSize\":1000000,\"costPerBatch\":2000000000,\"tokenCount\":0,\"tokenType\":\"cache_read\"},{\"batchSize\":1000000,\"costPerBatch\":25000000000,\"tokenCount\":6449,\"tokenType\":\"cache_write\"},{\"batchSize\":1000000,\"costPerBatch\":120000000000,\"tokenCount\":107,\"tokenType\":\"output\"}]", + "created_at": "2026-09-10T17:08:24.498Z" + } + }, + "locator": { + "database": "/example/.copilot/session-store.db", + "table": "assistant_usage_events", + "primary_key": 13, + "content_revision": "sha256:fe8383e7e562b4986c48b7b03ce880a54fd4ef03ae1721047ca41261e4ff85ca", + "generation": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + } + } + ], + "expected": { + "logical_call_ids": [ + "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13" + ], + "usage_row_count": 1, + "usage_source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:fe8383e7e562b4986c48b7b03ce880a54fd4ef03ae1721047ca41261e4ff85ca", + "replaced_output_tokens": 999 + } + }, + "identical_concurrent_tools": { + "observed": true, + "required": "use transcript toolCallId/source identity; hooks and nearest timestamps cannot prove pairing", + "input_records": [ + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.503Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "assistant.message", + "data": { + "messageId": "5a29b09d-5e7d-4603-975e-7801ce54232b", + "model": "gpt-5.6-luna", + "content": "", + "toolRequests": [ + { + "toolCallId": "call_YSSva4HCniiETlxdGGjcrHbh", + "name": "view", + "arguments": { + "path": "/fixture/workspace/alpha.txt" + }, + "type": "function", + "intentionSummary": "view the file at /fixture/workspace/alpha.txt." + }, + { + "toolCallId": "call_ayHplfzxjRFMTCpmTKEFhCSJ", + "name": "view", + "arguments": { + "path": "/fixture/workspace/beta.txt" + }, + "type": "function", + "intentionSummary": "view the file at /fixture/workspace/beta.txt." + } + ], + "interactionId": "6d2b89fd-a653-430c-b532-b0936d72eb42", + "turnId": "0", + "rte": true + }, + "id": "a4a17e63-7ba5-422f-8ee9-b495be417328", + "timestamp": "2026-09-10T17:08:24.503Z", + "parentId": "64f9e436-651f-4a9d-919c-a4d7abad2652", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 3731, + "byte_length": 793, + "native_event_id": "a4a17e63-7ba5-422f-8ee9-b495be417328" + } + }, + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a7f7bf04-589e-4989-a4a2-7ee687279627", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.506Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "tool.execution_start", + "data": { + "toolCallId": "call_YSSva4HCniiETlxdGGjcrHbh", + "toolName": "view", + "arguments": { + "path": "/fixture/workspace/alpha.txt" + }, + "turnId": "0", + "model": "gpt-5.6-luna" + }, + "id": "a7f7bf04-589e-4989-a4a2-7ee687279627", + "timestamp": "2026-09-10T17:08:24.506Z", + "parentId": "a4a17e63-7ba5-422f-8ee9-b495be417328", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 4524, + "byte_length": 344, + "native_event_id": "a7f7bf04-589e-4989-a4a2-7ee687279627" + } + }, + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/39e9da78-9bc3-4e1a-9629-f6680d1aeb4d", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.506Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "tool.execution_start", + "data": { + "toolCallId": "call_ayHplfzxjRFMTCpmTKEFhCSJ", + "toolName": "view", + "arguments": { + "path": "/fixture/workspace/beta.txt" + }, + "turnId": "0", + "model": "gpt-5.6-luna" + }, + "id": "39e9da78-9bc3-4e1a-9629-f6680d1aeb4d", + "timestamp": "2026-09-10T17:08:24.506Z", + "parentId": "a7f7bf04-589e-4989-a4a2-7ee687279627", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 4868, + "byte_length": 343, + "native_event_id": "39e9da78-9bc3-4e1a-9629-f6680d1aeb4d" + } + } + ], + "expected": { + "tool_call_ids": [ + "call_YSSva4HCniiETlxdGGjcrHbh", + "call_ayHplfzxjRFMTCpmTKEFhCSJ" + ], + "event_ids": [ + "copilot:event:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a7f7bf04-589e-4989-a4a2-7ee687279627", + "copilot:event:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/39e9da78-9bc3-4e1a-9629-f6680d1aeb4d" + ], + "forbidden_join_keys": [ + "tool_name", + "timestamp", + "parentId", + "turnId" + ] + } + }, + "ambiguous": { + "observed": false, + "required": "Attribution status ambiguous with both candidate IDs in evidence; choose neither", + "input_records": [ + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.503Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "assistant.message", + "data": { + "messageId": "5a29b09d-5e7d-4603-975e-7801ce54232b", + "model": "gpt-5.6-luna", + "content": "", + "toolRequests": [ + { + "toolCallId": "call_YSSva4HCniiETlxdGGjcrHbh", + "name": "view", + "arguments": { + "path": "/fixture/workspace/alpha.txt" + }, + "type": "function", + "intentionSummary": "view the file at /fixture/workspace/alpha.txt." + }, + { + "toolCallId": "call_ayHplfzxjRFMTCpmTKEFhCSJ", + "name": "view", + "arguments": { + "path": "/fixture/workspace/beta.txt" + }, + "type": "function", + "intentionSummary": "view the file at /fixture/workspace/beta.txt." + } + ], + "interactionId": "6d2b89fd-a653-430c-b532-b0936d72eb42", + "turnId": "0", + "rte": true + }, + "id": "a4a17e63-7ba5-422f-8ee9-b495be417328", + "timestamp": "2026-09-10T17:08:24.503Z", + "parentId": "64f9e436-651f-4a9d-919c-a4d7abad2652", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 3731, + "byte_length": 793, + "native_event_id": "a4a17e63-7ba5-422f-8ee9-b495be417328" + } + }, + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/33cc6465-29e1-4a04-8bdb-00241474b4d2", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:25.624Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "assistant.message", + "data": { + "messageId": "162d92b0-5d31-444c-b96f-b6c551527d2b", + "model": "gpt-5.6-luna", + "content": "42", + "toolRequests": [], + "interactionId": "6d2b89fd-a653-430c-b532-b0936d72eb42", + "turnId": "1", + "phase": "final_answer", + "rte": true + }, + "id": "33cc6465-29e1-4a04-8bdb-00241474b4d2", + "timestamp": "2026-09-10T17:08:25.624Z", + "parentId": "667fe48a-70a1-473b-b005-454964022344", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 9760, + "byte_length": 404, + "native_event_id": "33cc6465-29e1-4a04-8bdb-00241474b4d2" + } + }, + { + "source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "source_kind": "database", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.498Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "table": "assistant_usage_events", + "row": { + "id": 13, + "session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "turn_index": 0, + "agent_id": null, + "parent_tool_call_id": null, + "model": "gpt-5.6-luna", + "input_tokens": 6452, + "output_tokens": 107, + "cache_read_tokens": 0, + "cache_write_tokens": 6449, + "reasoning_tokens": 29, + "total_nano_aiu": 174125000, + "request_multiplier": 1.0, + "duration_ms": 2263, + "time_to_first_token_ms": 1571.1477920000002, + "output_ttft_ms": 1571.1479590000001, + "inter_token_latency_ms": 7.393025117647059, + "initiator": "user", + "api_endpoint": "ws:/responses", + "reasoning_effort": "medium", + "finish_reason": "tool_calls", + "content_filter_triggered": 0, + "token_details_json": "[{\"batchSize\":1000000,\"costPerBatch\":20000000000,\"tokenCount\":3,\"tokenType\":\"input\"},{\"batchSize\":1000000,\"costPerBatch\":2000000000,\"tokenCount\":0,\"tokenType\":\"cache_read\"},{\"batchSize\":1000000,\"costPerBatch\":25000000000,\"tokenCount\":6449,\"tokenType\":\"cache_write\"},{\"batchSize\":1000000,\"costPerBatch\":120000000000,\"tokenCount\":107,\"tokenType\":\"output\"}]", + "created_at": "2026-09-10T17:08:24.498Z" + } + }, + "locator": { + "database": "/example/.copilot/session-store.db", + "table": "assistant_usage_events", + "primary_key": 13, + "content_revision": "sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "generation": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + } + } + ], + "expected": { + "attributions": [ + { + "usage_source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13", + "stored_turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42", + "agent_id": null, + "call_id": null, + "status": "ambiguous", + "join_kind": null, + "evidence": [ + "call_id:copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "call_id:copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/33cc6465-29e1-4a04-8bdb-00241474b4d2", + "interaction_id:6d2b89fd-a653-430c-b532-b0936d72eb42", + "model:gpt-5.6-luna", + "turn_index:0" + ] + } + ] + } + }, + "conflicting": { + "observed": false, + "required": "ProjectionDiagnostic usage_revision_conflict quarantines incompatible revisions. Attribution.status conflicting is a join conflict, not that diagnostic.", + "input_records": [ + { + "source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "source_kind": "database", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.498Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "table": "assistant_usage_events", + "row": { + "id": 13, + "session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "turn_index": 0, + "agent_id": null, + "parent_tool_call_id": null, + "model": "gpt-5.6-luna", + "input_tokens": 6452, + "output_tokens": 107, + "cache_read_tokens": 0, + "cache_write_tokens": 6449, + "reasoning_tokens": 29, + "total_nano_aiu": 174125000, + "request_multiplier": 1.0, + "duration_ms": 2263, + "time_to_first_token_ms": 1571.1477920000002, + "output_ttft_ms": 1571.1479590000001, + "inter_token_latency_ms": 7.393025117647059, + "initiator": "user", + "api_endpoint": "ws:/responses", + "reasoning_effort": "medium", + "finish_reason": "tool_calls", + "content_filter_triggered": 0, + "token_details_json": "[{\"batchSize\":1000000,\"costPerBatch\":20000000000,\"tokenCount\":3,\"tokenType\":\"input\"},{\"batchSize\":1000000,\"costPerBatch\":2000000000,\"tokenCount\":0,\"tokenType\":\"cache_read\"},{\"batchSize\":1000000,\"costPerBatch\":25000000000,\"tokenCount\":6449,\"tokenType\":\"cache_write\"},{\"batchSize\":1000000,\"costPerBatch\":120000000000,\"tokenCount\":107,\"tokenType\":\"output\"}]", + "created_at": "2026-09-10T17:08:24.498Z" + } + }, + "locator": { + "database": "/example/.copilot/session-store.db", + "table": "assistant_usage_events", + "primary_key": 13, + "content_revision": "sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "generation": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + } + }, + { + "source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:bc82d51eaf6061d225d88e973fdaa5549ad7177a50376de381a3a33aadacfd13", + "source_kind": "database", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.498Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "table": "assistant_usage_events", + "row": { + "id": 13, + "session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "turn_index": 0, + "agent_id": null, + "parent_tool_call_id": null, + "model": "gpt-5.6-luna", + "input_tokens": 1, + "output_tokens": 1, + "cache_read_tokens": 0, + "cache_write_tokens": 6449, + "reasoning_tokens": 29, + "total_nano_aiu": 1, + "request_multiplier": 1.0, + "duration_ms": 2263, + "time_to_first_token_ms": 1571.1477920000002, + "output_ttft_ms": 1571.1479590000001, + "inter_token_latency_ms": 7.393025117647059, + "initiator": "user", + "api_endpoint": "ws:/responses", + "reasoning_effort": "medium", + "finish_reason": "tool_calls", + "content_filter_triggered": 0, + "token_details_json": "[{\"batchSize\":1000000,\"costPerBatch\":20000000000,\"tokenCount\":3,\"tokenType\":\"input\"},{\"batchSize\":1000000,\"costPerBatch\":2000000000,\"tokenCount\":0,\"tokenType\":\"cache_read\"},{\"batchSize\":1000000,\"costPerBatch\":25000000000,\"tokenCount\":6449,\"tokenType\":\"cache_write\"},{\"batchSize\":1000000,\"costPerBatch\":120000000000,\"tokenCount\":107,\"tokenType\":\"output\"}]", + "created_at": "2026-09-10T17:08:24.498Z" + } + }, + "locator": { + "database": "/example/.copilot/session-store.db", + "table": "assistant_usage_events", + "primary_key": 13, + "content_revision": "sha256:bc82d51eaf6061d225d88e973fdaa5549ad7177a50376de381a3a33aadacfd13", + "generation": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + } + } + ], + "expected": { + "usage_row_count": 0, + "diagnostics": [ + { + "code": "usage_revision_conflict", + "severity": "error", + "message": "incompatible metrics for one logical call; quarantined", + "source_ids": [ + "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:bc82d51eaf6061d225d88e973fdaa5549ad7177a50376de381a3a33aadacfd13" + ], + "details": { + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13" + } + } + ], + "attributions": [ + { + "usage_source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13", + "stored_turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42", + "agent_id": null, + "call_id": null, + "status": "conflicting", + "join_kind": null, + "evidence": [ + "call_id:copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "call_id:copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/33cc6465-29e1-4a04-8bdb-00241474b4d2", + "join_conflict:rebinding_after_emission" + ] + } + ] + } + }, + "auxiliary_title_generation": { + "observed": false, + "required": "classify model.* title generation separately; do not add to the six-call total", + "input_records": [ + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-title-1", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:08.000Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "model.model_call_success", + "data": { + "model": "gpt-4o-mini", + "purpose": "session_title", + "input_tokens": 120, + "output_tokens": 8 + }, + "id": "synthetic-title-1", + "timestamp": "2026-09-10T17:08:08.000Z", + "parentId": null, + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 0, + "byte_length": 210, + "native_event_id": "synthetic-title-1" + } + } + ], + "expected": { + "events": [ + { + "id": "copilot:event:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-title-1", + "kind": "auxiliary_model_call", + "classification": "title_generation", + "ts": "2026-09-10T17:08:08.000Z", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-title-1" + ], + "source_references": [ + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-title-1", + "source_kind": "transcript", + "role": "auxiliary_model" + } + ], + "attributes": { + "model": "gpt-4o-mini" + } + } + ], + "call_candidates": [], + "usage_rows": [], + "diagnostics": [ + { + "code": "auxiliary_excluded_from_main", + "severity": "info", + "message": "title-generation model.* record classified auxiliary", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-title-1" + ], + "details": { + "classification": "title_generation", + "model": "gpt-4o-mini" + } + } + ] + } + }, + "unknown_version": { + "observed": false, + "required": "retain raw payload as unknown; pending/diagnostic rather than a completed turn; no fabricated grouping", + "input_records": [ + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-unknown-version-1", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:22.203Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "user.message", + "data": { + "content": "hello from an unsupported transcript schema", + "interactionId": "6d2b89fd-a653-430c-b532-b0936d72eb42", + "turnId": "0" + }, + "id": "synthetic-unknown-version-1", + "timestamp": "2026-09-10T17:08:22.203Z", + "parentId": null, + "schema_version": 2 + }, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 0, + "byte_length": 240, + "native_event_id": "synthetic-unknown-version-1" + } + } + ], + "expected": { + "events": [ + { + "id": "copilot:event:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-unknown-version-1", + "kind": "unknown", + "classification": "main", + "ts": "2026-09-10T17:08:22.203Z", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-unknown-version-1" + ], + "attributes": { + "native_type": "user.message", + "raw_payload": { + "type": "user.message", + "data": { + "content": "hello from an unsupported transcript schema", + "interactionId": "6d2b89fd-a653-430c-b532-b0936d72eb42", + "turnId": "0" + }, + "id": "synthetic-unknown-version-1", + "timestamp": "2026-09-10T17:08:22.203Z", + "parentId": null, + "schema_version": 2 + } + } + } + ], + "turns": [], + "call_candidates": [], + "pending": [ + { + "id": "pending:unknown-version:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-unknown-version-1", + "kind": "missing_source_capability", + "reason": "unknown transcript schema_version", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-unknown-version-1" + ], + "evidence": [ + "schema_version:2" + ] + } + ], + "diagnostics": [ + { + "code": "capability_gap", + "severity": "warning", + "message": "unknown transcript schema_version; raw payload retained", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-unknown-version-1" + ], + "details": { + "schema_version": 2 + } + } + ] + } + }, + "identical_concurrent_identical_args": { + "observed": false, + "required": "pair concurrent identical view arguments by toolCallId only; arguments/tool_name/timestamp cannot prove pairing", + "input_records": [ + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-identical-args-asst", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.503Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "assistant.message", + "data": { + "messageId": "synthetic-identical-args-msg", + "model": "gpt-5.6-luna", + "content": "", + "toolRequests": [ + { + "toolCallId": "call_identical_args_alpha_1", + "name": "view", + "arguments": { + "path": "/fixture/workspace/alpha.txt" + }, + "type": "function", + "intentionSummary": "view the file at /fixture/workspace/alpha.txt." + }, + { + "toolCallId": "call_identical_args_alpha_2", + "name": "view", + "arguments": { + "path": "/fixture/workspace/alpha.txt" + }, + "type": "function", + "intentionSummary": "view the file at /fixture/workspace/alpha.txt." + } + ], + "interactionId": "6d2b89fd-a653-430c-b532-b0936d72eb42", + "turnId": "0", + "rte": true + }, + "id": "synthetic-identical-args-asst", + "timestamp": "2026-09-10T17:08:24.503Z", + "parentId": "64f9e436-651f-4a9d-919c-a4d7abad2652", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 3731, + "byte_length": 793, + "native_event_id": "synthetic-identical-args-asst" + } + }, + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-identical-args-exec-a", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.506Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "tool.execution_start", + "data": { + "toolCallId": "call_identical_args_alpha_1", + "toolName": "view", + "arguments": { + "path": "/fixture/workspace/alpha.txt" + }, + "turnId": "0", + "model": "gpt-5.6-luna" + }, + "id": "synthetic-identical-args-exec-a", + "timestamp": "2026-09-10T17:08:24.506Z", + "parentId": "synthetic-identical-args-asst", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 4524, + "byte_length": 344, + "native_event_id": "synthetic-identical-args-exec-a" + } + }, + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-identical-args-exec-b", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.506Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "tool.execution_start", + "data": { + "toolCallId": "call_identical_args_alpha_2", + "toolName": "view", + "arguments": { + "path": "/fixture/workspace/alpha.txt" + }, + "turnId": "0", + "model": "gpt-5.6-luna" + }, + "id": "synthetic-identical-args-exec-b", + "timestamp": "2026-09-10T17:08:24.506Z", + "parentId": "synthetic-identical-args-exec-a", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 4868, + "byte_length": 343, + "native_event_id": "synthetic-identical-args-exec-b" + } + } + ], + "expected": { + "tool_call_ids": [ + "call_identical_args_alpha_1", + "call_identical_args_alpha_2" + ], + "event_ids": [ + "copilot:event:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-identical-args-exec-a", + "copilot:event:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-identical-args-exec-b" + ], + "forbidden_join_keys": [ + "arguments", + "tool_name", + "timestamp", + "parentId", + "turnId" + ] + } + } +} diff --git a/tests/platforms/copilot/fixtures/reconciliation-cases/diagnostics.json b/tests/platforms/copilot/fixtures/reconciliation-cases/diagnostics.json new file mode 100644 index 00000000..9039b2cc --- /dev/null +++ b/tests/platforms/copilot/fixtures/reconciliation-cases/diagnostics.json @@ -0,0 +1,62 @@ +{ + "pending": { + "open_interaction": { + "id": "pending:copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42", + "kind": "open_interaction", + "reason": "user interaction has no assistant.turn_end", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/f07404d0-af52-4260-89fb-358a10e86034" + ], + "evidence": [ + "interaction_id:6d2b89fd-a653-430c-b532-b0936d72eb42" + ] + }, + "missing_source_capability": { + "id": "pending:capability:hooks", + "kind": "missing_source_capability", + "reason": "archive has no hook observations; watch must still derive from transcript", + "source_ids": [], + "evidence": [ + "capability:hooks=absent" + ] + } + }, + "diagnostics": { + "usage_revision_conflict": { + "code": "usage_revision_conflict", + "severity": "error", + "message": "incompatible metrics for one logical call; quarantined", + "source_ids": [ + "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:bc82d51eaf6061d225d88e973fdaa5549ad7177a50376de381a3a33aadacfd13" + ], + "details": { + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13" + } + }, + "shutdown_total_mismatch": { + "code": "shutdown_total_mismatch", + "severity": "warning", + "message": "per-call totals do not equal session.shutdown usage", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/9dbdd079-e22a-4bd0-87e2-17fa2b246d23" + ], + "details": { + "expected_input_tokens": 35396, + "accounted_input_tokens": 0 + } + }, + "auxiliary_excluded_from_main": { + "code": "auxiliary_excluded_from_main", + "severity": "info", + "message": "title-generation model.* record classified auxiliary", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/synthetic-title-1" + ], + "details": { + "classification": "title_generation", + "model": "gpt-4o-mini" + } + } + } +} diff --git a/tests/platforms/copilot/fixtures/reconciliation-cases/identities.json b/tests/platforms/copilot/fixtures/reconciliation-cases/identities.json new file mode 100644 index 00000000..93fce546 --- /dev/null +++ b/tests/platforms/copilot/fixtures/reconciliation-cases/identities.json @@ -0,0 +1,26 @@ +{ + "source_key": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "source_key_is_placeholder": true, + "source_key_note": "Full 64-character SHA-256 of the canonical source home. Not the 16-character stored-session prefix. Live captures replace this.", + "source_key_prefix": "aaaaaaaaaaaaaaaa", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "child_agent_id": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", + "stored_session_id": "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "database_generation": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "database_generation_is_placeholder": true, + "database_generation_note": "Live V1 generation is sha256 of canonical JSON {path, device, inode, birthtime_ns} from the database file stat.", + "quoted_database_generation": "sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "transcript_file_generation": "1-abc", + "transcript_file_generation_note": "Live V1 file_generation is hex(st_dev)-hex(st_ino)[-hex(birthtime_ns)].", + "database_path": "/example/.copilot/session-store.db", + "cwd": "/fixture/workspace", + "id_templates": { + "semantic_event": "copilot:event:", + "semantic_event_tool_request": "copilot:event::tool:", + "semantic_call": "copilot:call:", + "main_turn": "copilot:turn:::", + "logical_call": "copilot:usage::
::", + "turn_accounting_span": "accounting:{session_id}:{turn_id}:{accounting_id}", + "session_accounting_span": "accounting:{session_id}:{accounting_id}" + } +} diff --git a/tests/platforms/copilot/fixtures/reconciliation-cases/observed-six-calls.json b/tests/platforms/copilot/fixtures/reconciliation-cases/observed-six-calls.json new file mode 100644 index 00000000..c90ecad8 --- /dev/null +++ b/tests/platforms/copilot/fixtures/reconciliation-cases/observed-six-calls.json @@ -0,0 +1,239 @@ +{ + "source_fixture": "../assistant-usage-events.json", + "source_key": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "source_key_is_placeholder": true, + "database_generation": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "database_generation_is_placeholder": true, + "table": "assistant_usage_events", + "shutdown_totals": { + "source": "../usage.json", + "input_tokens": 35396, + "output_tokens": 328, + "reasoning_tokens": 55, + "cache_read_tokens": 23948, + "cache_write_tokens": 11430, + "total_nano_aiu": 373366000 + }, + "calls": [ + { + "row_id": 13, + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13", + "usage_source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "turn_index": 0, + "agent_id": null, + "parent_tool_call_id": null, + "initiator": "user", + "model": "gpt-5.6-luna", + "finish_reason": "tool_calls", + "expected_call_id": "copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "expected_stored_turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42", + "interaction_id": "6d2b89fd-a653-430c-b532-b0936d72eb42" + }, + { + "row_id": 14, + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:14", + "usage_source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:14:sha256:20cb0f98d6130c8d0e139391cfc8e245d1d0ac96d41469c4da60f26e6d9e0a26", + "turn_index": 0, + "agent_id": null, + "parent_tool_call_id": null, + "initiator": "agent", + "model": "gpt-5.6-luna", + "finish_reason": "stop", + "expected_call_id": "copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/33cc6465-29e1-4a04-8bdb-00241474b4d2", + "expected_stored_turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42", + "interaction_id": "6d2b89fd-a653-430c-b532-b0936d72eb42" + }, + { + "row_id": 15, + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:15", + "usage_source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:15:sha256:c78b2b2ed28b8641c996bf9c84a05812d5a108776ffe091a54378f9ff79162d8", + "turn_index": 1, + "agent_id": null, + "parent_tool_call_id": null, + "initiator": "user", + "model": "gpt-5.6-luna", + "finish_reason": "tool_calls", + "expected_call_id": "copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/2c2e4ab8-f283-4837-957d-da992ba55e65", + "expected_stored_turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:793d3703-6f4a-4814-8877-34a7325848ce", + "interaction_id": "793d3703-6f4a-4814-8877-34a7325848ce" + }, + { + "row_id": 16, + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:16", + "usage_source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:16:sha256:be2e28cbe7e6071667598eb2156481465762532a6652196b7735a46ef412c67c", + "turn_index": 1, + "agent_id": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", + "parent_tool_call_id": "call_qx4FH5DADTeT1qVLb37HNpBk", + "initiator": "sub-agent", + "model": "gpt-5.6-luna", + "finish_reason": "tool_calls", + "expected_call_id": "copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/df600539-fdd4-4de2-bd42-7f7ff2c952ab", + "expected_stored_turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:793d3703-6f4a-4814-8877-34a7325848ce", + "interaction_id": "7c0fa097-c0e2-48da-b2b6-fcfc1ad83a6b" + }, + { + "row_id": 17, + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:17", + "usage_source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:17:sha256:92ca44e9db1e93336e94649492c5c59945ef4268ca1188f710b77e6e4b148435", + "turn_index": 1, + "agent_id": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", + "parent_tool_call_id": "call_qx4FH5DADTeT1qVLb37HNpBk", + "initiator": "sub-agent", + "model": "gpt-5.6-luna", + "finish_reason": "stop", + "expected_call_id": "copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/450a1f4c-de11-4d37-bf84-08f63d29588f", + "expected_stored_turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:793d3703-6f4a-4814-8877-34a7325848ce", + "interaction_id": "7c0fa097-c0e2-48da-b2b6-fcfc1ad83a6b" + }, + { + "row_id": 18, + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:18", + "usage_source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:18:sha256:5fb5f685b5495f2fcc1874ad3b563a6750fa35c863af4647ea91225dd3ec907a", + "turn_index": 1, + "agent_id": null, + "parent_tool_call_id": null, + "initiator": "agent", + "model": "gpt-5.6-luna", + "finish_reason": "stop", + "expected_call_id": "copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/10001566-2704-4f75-add3-2c044373eaba", + "expected_stored_turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:793d3703-6f4a-4814-8877-34a7325848ce", + "interaction_id": "793d3703-6f4a-4814-8877-34a7325848ce" + } + ], + "expected_attributions": [ + { + "usage_source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13", + "stored_turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42", + "agent_id": null, + "call_id": "copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "status": "matched", + "join_kind": "inferred", + "evidence": [ + "join_kind:inferred", + "interaction_id:6d2b89fd-a653-430c-b532-b0936d72eb42", + "turn_index:0", + "agent_id:None", + "parent_tool_call_id:None", + "model:gpt-5.6-luna", + "finish:tool_calls", + "initiator:user", + "order:0", + "call_id:copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328" + ] + }, + { + "usage_source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:14:sha256:20cb0f98d6130c8d0e139391cfc8e245d1d0ac96d41469c4da60f26e6d9e0a26", + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:14", + "stored_turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42", + "agent_id": null, + "call_id": "copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/33cc6465-29e1-4a04-8bdb-00241474b4d2", + "status": "matched", + "join_kind": "inferred", + "evidence": [ + "join_kind:inferred", + "interaction_id:6d2b89fd-a653-430c-b532-b0936d72eb42", + "turn_index:0", + "agent_id:None", + "parent_tool_call_id:None", + "model:gpt-5.6-luna", + "finish:stop", + "initiator:agent", + "order:1", + "call_id:copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/33cc6465-29e1-4a04-8bdb-00241474b4d2" + ] + }, + { + "usage_source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:15:sha256:c78b2b2ed28b8641c996bf9c84a05812d5a108776ffe091a54378f9ff79162d8", + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:15", + "stored_turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:793d3703-6f4a-4814-8877-34a7325848ce", + "agent_id": null, + "call_id": "copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/2c2e4ab8-f283-4837-957d-da992ba55e65", + "status": "matched", + "join_kind": "inferred", + "evidence": [ + "join_kind:inferred", + "interaction_id:793d3703-6f4a-4814-8877-34a7325848ce", + "turn_index:1", + "agent_id:None", + "parent_tool_call_id:None", + "model:gpt-5.6-luna", + "finish:tool_calls", + "initiator:user", + "order:2", + "call_id:copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/2c2e4ab8-f283-4837-957d-da992ba55e65" + ] + }, + { + "usage_source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:16:sha256:be2e28cbe7e6071667598eb2156481465762532a6652196b7735a46ef412c67c", + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:16", + "stored_turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:793d3703-6f4a-4814-8877-34a7325848ce", + "agent_id": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", + "call_id": "copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/df600539-fdd4-4de2-bd42-7f7ff2c952ab", + "status": "matched", + "join_kind": "inferred", + "evidence": [ + "join_kind:inferred", + "interaction_id:7c0fa097-c0e2-48da-b2b6-fcfc1ad83a6b", + "turn_index:1", + "agent_id:bf8cb9f3-2097-4db0-a3c8-78a2653b2106", + "parent_tool_call_id:call_qx4FH5DADTeT1qVLb37HNpBk", + "model:gpt-5.6-luna", + "finish:tool_calls", + "initiator:sub-agent", + "order:3", + "call_id:copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/df600539-fdd4-4de2-bd42-7f7ff2c952ab" + ] + }, + { + "usage_source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:17:sha256:92ca44e9db1e93336e94649492c5c59945ef4268ca1188f710b77e6e4b148435", + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:17", + "stored_turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:793d3703-6f4a-4814-8877-34a7325848ce", + "agent_id": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106", + "call_id": "copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/450a1f4c-de11-4d37-bf84-08f63d29588f", + "status": "matched", + "join_kind": "inferred", + "evidence": [ + "join_kind:inferred", + "interaction_id:7c0fa097-c0e2-48da-b2b6-fcfc1ad83a6b", + "turn_index:1", + "agent_id:bf8cb9f3-2097-4db0-a3c8-78a2653b2106", + "parent_tool_call_id:call_qx4FH5DADTeT1qVLb37HNpBk", + "model:gpt-5.6-luna", + "finish:stop", + "initiator:sub-agent", + "order:4", + "call_id:copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/450a1f4c-de11-4d37-bf84-08f63d29588f" + ] + }, + { + "usage_source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:18:sha256:5fb5f685b5495f2fcc1874ad3b563a6750fa35c863af4647ea91225dd3ec907a", + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:18", + "stored_turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:793d3703-6f4a-4814-8877-34a7325848ce", + "agent_id": null, + "call_id": "copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/10001566-2704-4f75-add3-2c044373eaba", + "status": "matched", + "join_kind": "inferred", + "evidence": [ + "join_kind:inferred", + "interaction_id:793d3703-6f4a-4814-8877-34a7325848ce", + "turn_index:1", + "agent_id:None", + "parent_tool_call_id:None", + "model:gpt-5.6-luna", + "finish:stop", + "initiator:agent", + "order:5", + "call_id:copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/10001566-2704-4f75-add3-2c044373eaba" + ] + } + ], + "invariants": { + "call_count": 6, + "main_agent_call_count": 4, + "child_agent_call_count": 2, + "shutdown_usage_is_validation_only": true, + "checkpoint_usage_is_additive": false, + "auxiliary_model_calls_are_main_usage": false + } +} diff --git a/tests/platforms/copilot/fixtures/reconciliation-cases/semantic-projection.json b/tests/platforms/copilot/fixtures/reconciliation-cases/semantic-projection.json new file mode 100644 index 00000000..9dd8ce53 --- /dev/null +++ b/tests/platforms/copilot/fixtures/reconciliation-cases/semantic-projection.json @@ -0,0 +1,303 @@ +{ + "note": "input_records are V1 SourceRecords. Pure-function tests may pass them directly to build_semantics. They are not a generated archive. This slice is a tool cycle: assistant.turn_end attaches finish_evidence without requiring assistant.turn_start, but outstanding tools keep the user interaction open. A completed TurnSpanDict is not emitted.", + "input_records": [ + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/f07404d0-af52-4260-89fb-358a10e86034", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:22.203Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "user.message", + "data": { + "content": "Read alpha.txt and beta.txt with separate view calls in parallel and report their sum. Only read those two files.", + "messageId": "a856cb38-7609-45ab-8a55-645553155db3", + "supportedNativeDocumentMimeTypes": [], + "delivery": "idle", + "interactionId": "6d2b89fd-a653-430c-b532-b0936d72eb42", + "turnId": "0", + "parentAgentTaskId": "fd800f7a-8163-4304-9681-efde4731e879" + }, + "id": "f07404d0-af52-4260-89fb-358a10e86034", + "timestamp": "2026-09-10T17:08:22.203Z", + "parentId": "3942810f-1caf-4251-82fb-3bf72698147a", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 2122, + "byte_length": 548, + "native_event_id": "f07404d0-af52-4260-89fb-358a10e86034" + } + }, + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.503Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "assistant.message", + "data": { + "messageId": "5a29b09d-5e7d-4603-975e-7801ce54232b", + "model": "gpt-5.6-luna", + "content": "", + "toolRequests": [ + { + "toolCallId": "call_YSSva4HCniiETlxdGGjcrHbh", + "name": "view", + "arguments": { + "path": "/fixture/workspace/alpha.txt" + }, + "type": "function", + "intentionSummary": "view the file at /fixture/workspace/alpha.txt." + }, + { + "toolCallId": "call_ayHplfzxjRFMTCpmTKEFhCSJ", + "name": "view", + "arguments": { + "path": "/fixture/workspace/beta.txt" + }, + "type": "function", + "intentionSummary": "view the file at /fixture/workspace/beta.txt." + } + ], + "interactionId": "6d2b89fd-a653-430c-b532-b0936d72eb42", + "turnId": "0", + "rte": true + }, + "id": "a4a17e63-7ba5-422f-8ee9-b495be417328", + "timestamp": "2026-09-10T17:08:24.503Z", + "parentId": "64f9e436-651f-4a9d-919c-a4d7abad2652", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 3731, + "byte_length": 793, + "native_event_id": "a4a17e63-7ba5-422f-8ee9-b495be417328" + } + }, + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/0080e44c-ad62-4288-b2b2-061ec2b73d80", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.593Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "assistant.turn_end", + "data": { + "turnId": "0" + }, + "id": "0080e44c-ad62-4288-b2b2-061ec2b73d80", + "timestamp": "2026-09-10T17:08:24.593Z", + "parentId": "032cf87e-05b7-4e5f-96b9-1fda1a8242f5", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 9311, + "byte_length": 195, + "native_event_id": "0080e44c-ad62-4288-b2b2-061ec2b73d80" + } + } + ], + "expected": { + "events": [ + { + "id": "copilot:event:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/f07404d0-af52-4260-89fb-358a10e86034", + "kind": "user_prompt", + "classification": "main", + "ts": "2026-09-10T17:08:22.203Z", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/f07404d0-af52-4260-89fb-358a10e86034" + ], + "source_references": [ + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/f07404d0-af52-4260-89fb-358a10e86034", + "source_kind": "transcript", + "role": "user_prompt" + } + ], + "attributes": { + "interaction_id": "6d2b89fd-a653-430c-b532-b0936d72eb42", + "agent_id": null + } + }, + { + "id": "copilot:event:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "kind": "assistant_message", + "classification": "main", + "ts": "2026-09-10T17:08:24.503Z", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328" + ], + "source_references": [ + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "source_kind": "transcript", + "role": "assistant_message" + } + ], + "attributes": { + "interaction_id": "6d2b89fd-a653-430c-b532-b0936d72eb42", + "agent_id": null + } + }, + { + "id": "copilot:event:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328:tool:call_YSSva4HCniiETlxdGGjcrHbh", + "kind": "tool_request", + "classification": "main", + "ts": "2026-09-10T17:08:24.503Z", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328" + ], + "source_references": [ + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "source_kind": "transcript", + "role": "tool_request" + } + ], + "attributes": { + "tool_call_id": "call_YSSva4HCniiETlxdGGjcrHbh", + "name": "view" + } + }, + { + "id": "copilot:event:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328:tool:call_ayHplfzxjRFMTCpmTKEFhCSJ", + "kind": "tool_request", + "classification": "main", + "ts": "2026-09-10T17:08:24.503Z", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328" + ], + "source_references": [ + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "source_kind": "transcript", + "role": "tool_request" + } + ], + "attributes": { + "tool_call_id": "call_ayHplfzxjRFMTCpmTKEFhCSJ", + "name": "view" + } + }, + { + "id": "copilot:event:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/0080e44c-ad62-4288-b2b2-061ec2b73d80", + "kind": "assistant_turn_end", + "classification": "main", + "ts": "2026-09-10T17:08:24.593Z", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/0080e44c-ad62-4288-b2b2-061ec2b73d80" + ], + "source_references": [ + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/0080e44c-ad62-4288-b2b2-061ec2b73d80", + "source_kind": "transcript", + "role": "finish" + } + ], + "attributes": { + "turn_id": "0" + } + } + ], + "turns": [], + "call_candidates": [ + { + "call_id": "copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "stored_turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42", + "interaction_id": "6d2b89fd-a653-430c-b532-b0936d72eb42", + "agent_id": null, + "parent_tool_call_id": null, + "model": "gpt-5.6-luna", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/0080e44c-ad62-4288-b2b2-061ec2b73d80" + ], + "source_references": [ + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "source_kind": "transcript", + "role": "assistant_message" + }, + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/0080e44c-ad62-4288-b2b2-061ec2b73d80", + "source_kind": "transcript", + "role": "finish" + } + ], + "start_ts": "2026-09-10T17:08:24.503Z", + "end_ts": "2026-09-10T17:08:24.593Z", + "tool_call_ids": [ + "call_YSSva4HCniiETlxdGGjcrHbh", + "call_ayHplfzxjRFMTCpmTKEFhCSJ" + ], + "finish_evidence": [ + { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/0080e44c-ad62-4288-b2b2-061ec2b73d80", + "kind": "assistant_turn_end", + "value": null + } + ] + } + ], + "pending": [ + { + "id": "pending:copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42", + "kind": "open_interaction", + "reason": "user interaction has no completed final assistant turn", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/f07404d0-af52-4260-89fb-358a10e86034", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/0080e44c-ad62-4288-b2b2-061ec2b73d80" + ], + "evidence": [ + "interaction_id:6d2b89fd-a653-430c-b532-b0936d72eb42" + ] + }, + { + "id": "pending:tool:call_YSSva4HCniiETlxdGGjcrHbh", + "kind": "incomplete_tool_pair", + "reason": "tool request has no execution result", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328" + ], + "evidence": [ + "tool_call_id:call_YSSva4HCniiETlxdGGjcrHbh" + ] + }, + { + "id": "pending:tool:call_ayHplfzxjRFMTCpmTKEFhCSJ", + "kind": "incomplete_tool_pair", + "reason": "tool request has no execution result", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328" + ], + "evidence": [ + "tool_call_id:call_ayHplfzxjRFMTCpmTKEFhCSJ" + ] + } + ], + "diagnostics": [ + { + "code": "open_interaction", + "severity": "info", + "message": "user interaction has no completed final assistant turn", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/f07404d0-af52-4260-89fb-358a10e86034", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/0080e44c-ad62-4288-b2b2-061ec2b73d80" + ], + "details": { + "interaction_id": "6d2b89fd-a653-430c-b532-b0936d72eb42" + } + } + ] + } +} diff --git a/tests/platforms/copilot/fixtures/reconciliation-cases/storage.json b/tests/platforms/copilot/fixtures/reconciliation-cases/storage.json new file mode 100644 index 00000000..63bd31e1 --- /dev/null +++ b/tests/platforms/copilot/fixtures/reconciliation-cases/storage.json @@ -0,0 +1,158 @@ +{ + "projection_state": { + "projection_schema_version": 2, + "archive_source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/f07404d0-af52-4260-89fb-358a10e86034", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47" + ], + "semantic_state": { + "open_interactions": {} + }, + "accounting_state": { + "logical_calls": { + "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13": { + "logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13", + "generation": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "content_revision": "sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + "metrics_digest": "sha256:feee488caa281c39aa67f248f5e9d45220c1db1ee47b1f733b2eed66de686a53", + "usage_source_id": "copilot-db:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:assistant_usage_events:13:sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47" + } + } + }, + "projection_revision": "sha256:derived-state-example" + }, + "open_interactions_key": "|", + "open_interactions_example": { + "6d2b89fd-a653-430c-b532-b0936d72eb42|main": { + "interaction_id": "6d2b89fd-a653-430c-b532-b0936d72eb42", + "agent_id": null, + "stored_turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42", + "source_ids": [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/f07404d0-af52-4260-89fb-358a10e86034" + ], + "last_event_source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/f07404d0-af52-4260-89fb-358a10e86034", + "start_ts": "2026-09-10T17:08:22.203Z", + "pending_tool_call_ids": [] + } + }, + "index_totals": { + "events": 2, + "turns": 1, + "usage": 1, + "attributions": 1, + "pending": 0, + "diagnostics": 0 + }, + "read_projected_turns": [ + { + "id": "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42", + "turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42", + "session_id": "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "platform": "copilot", + "cwd": "/fixture/workspace", + "start_seq": 0, + "end_seq": 1, + "start_ts": "2026-09-10T17:08:22.203Z", + "end_ts": "2026-09-10T17:08:24.503Z", + "events": [ + { + "t": "copilot_transcript", + "ts": "2026-09-10T17:08:22.203Z", + "seq": 0, + "data": { + "schema_version": 1, + "source_record": { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/f07404d0-af52-4260-89fb-358a10e86034", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:22.203Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "user.message", + "data": { + "content": "Read alpha.txt and beta.txt with separate view calls in parallel and report their sum. Only read those two files.", + "messageId": "a856cb38-7609-45ab-8a55-645553155db3", + "supportedNativeDocumentMimeTypes": [], + "delivery": "idle", + "interactionId": "6d2b89fd-a653-430c-b532-b0936d72eb42", + "turnId": "0", + "parentAgentTaskId": "fd800f7a-8163-4304-9681-efde4731e879" + }, + "id": "f07404d0-af52-4260-89fb-358a10e86034", + "timestamp": "2026-09-10T17:08:22.203Z", + "parentId": "3942810f-1caf-4251-82fb-3bf72698147a", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 2122, + "byte_length": 548, + "native_event_id": "f07404d0-af52-4260-89fb-358a10e86034" + } + } + } + }, + { + "t": "copilot_transcript", + "ts": "2026-09-10T17:08:24.503Z", + "seq": 1, + "data": { + "schema_version": 1, + "source_record": { + "source_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "source_kind": "transcript", + "native_session_id": "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "ts": "2026-09-10T17:08:24.503Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "type": "assistant.message", + "data": { + "messageId": "5a29b09d-5e7d-4603-975e-7801ce54232b", + "model": "gpt-5.6-luna", + "content": "", + "toolRequests": [ + { + "toolCallId": "call_YSSva4HCniiETlxdGGjcrHbh", + "name": "view", + "arguments": { + "path": "/fixture/workspace/alpha.txt" + }, + "type": "function", + "intentionSummary": "view the file at /fixture/workspace/alpha.txt." + }, + { + "toolCallId": "call_ayHplfzxjRFMTCpmTKEFhCSJ", + "name": "view", + "arguments": { + "path": "/fixture/workspace/beta.txt" + }, + "type": "function", + "intentionSummary": "view the file at /fixture/workspace/beta.txt." + } + ], + "interactionId": "6d2b89fd-a653-430c-b532-b0936d72eb42", + "turnId": "0", + "rte": true + }, + "id": "a4a17e63-7ba5-422f-8ee9-b495be417328", + "timestamp": "2026-09-10T17:08:24.503Z", + "parentId": "64f9e436-651f-4a9d-919c-a4d7abad2652", + "schema_version": 1 + }, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 3731, + "byte_length": 793, + "native_event_id": "a4a17e63-7ba5-422f-8ee9-b495be417328" + } + } + } + } + ] + } + ], + "events_note": "events are Store records with seq/t/ts/data. data is the V1 envelope {schema_version, source_record}." +} diff --git a/tests/platforms/copilot/fixtures/reconciliation-cases/transport.json b/tests/platforms/copilot/fixtures/reconciliation-cases/transport.json new file mode 100644 index 00000000..eac2aea4 --- /dev/null +++ b/tests/platforms/copilot/fixtures/reconciliation-cases/transport.json @@ -0,0 +1,186 @@ +{ + "rules": { + "call_id_set": "export tokens on the matching LlmCallSpanDict chat span", + "call_id_null": "export on a user-turn or agent accounting span, never a fabricated turn", + "llm_call_usage": "LlmCallSpanDict.usage stays empty whenever accounting_calls is present", + "never_both": "matched chat span and fallback accounting span are mutually exclusive", + "span_id_chat": "existing chat span id for call_id; no extra span", + "span_id_turn": "accounting:{session_id}:{turn_id}:{accounting_id}", + "span_id_session": "accounting:{session_id}:{accounting_id}" + }, + "turn_accounting_calls": [ + { + "accounting_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13", + "usage": { + "session_id": "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "seq": 0, + "call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13", + "ts": "2026-09-10T17:08:24.498Z", + "platform": "copilot", + "gen_ai.conversation.id": "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "gen_ai.provider.name": "unknown", + "gen_ai.operation.name": "chat", + "gen_ai.response.model": "gpt-5.6-luna", + "gen_ai.usage.input_tokens": 6452, + "gen_ai.usage.output_tokens": 107, + "gen_ai.usage.cache_read.input_tokens": 0, + "gen_ai.usage.cache_creation.input_tokens": 6449, + "gen_ai.usage.reasoning.output_tokens": 29 + }, + "attribution_status": "matched", + "agent_id": null, + "call_id": "copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "attributes": { + "accounting.destination": "chat-span", + "copilot.logical_call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13", + "copilot.evidence": [ + "join_kind:inferred", + "interaction_id:6d2b89fd-a653-430c-b532-b0936d72eb42", + "model:gpt-5.6-luna", + "turn_index:0", + "finish:tool_calls", + "initiator:user", + "order:0" + ] + } + } + ], + "turn_owned_unmatched_job": { + "job_id": "accounting:copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42:copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:14", + "kind": "turn_accounting", + "session_id": "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42", + "accounting_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:14", + "destination": "turn-accounting-span", + "attempt": 0, + "state": "queued", + "usage": { + "session_id": "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "seq": 0, + "call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:14", + "ts": "2026-09-10T17:08:25.618Z", + "platform": "copilot", + "gen_ai.conversation.id": "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "gen_ai.provider.name": "unknown", + "gen_ai.operation.name": "chat", + "gen_ai.response.model": "gpt-5.6-luna", + "gen_ai.usage.input_tokens": 6587, + "gen_ai.usage.output_tokens": 5, + "gen_ai.usage.cache_read.input_tokens": 6449, + "gen_ai.usage.cache_creation.input_tokens": 135, + "gen_ai.usage.reasoning.output_tokens": 0 + }, + "attribution_status": "pending", + "agent_id": null, + "span_id": "accounting:copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42:copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:14" + }, + "session_accounting_job": { + "job_id": "accounting:copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:14", + "kind": "session_accounting", + "session_id": "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "accounting_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:14", + "destination": "session-accounting-span", + "attempt": 0, + "state": "queued", + "usage": { + "session_id": "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "seq": 0, + "call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:14", + "ts": "2026-09-10T17:08:25.618Z", + "platform": "copilot", + "gen_ai.conversation.id": "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "gen_ai.provider.name": "unknown", + "gen_ai.operation.name": "chat", + "gen_ai.response.model": "gpt-5.6-luna", + "gen_ai.usage.input_tokens": 6587, + "gen_ai.usage.output_tokens": 5, + "gen_ai.usage.cache_read.input_tokens": 6449, + "gen_ai.usage.cache_creation.input_tokens": 135, + "gen_ai.usage.reasoning.output_tokens": 0 + }, + "attribution_status": "pending", + "span_id": "accounting:copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:14" + }, + "ledger_entry": { + "accounting_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:14", + "destination": "session-accounting-span", + "span_id": "accounting:copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:14", + "emitted": false, + "last_error": null + }, + "turn_span_with_accounting_calls": { + "turn_id": "copilot:turn:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6:6d2b89fd-a653-430c-b532-b0936d72eb42", + "start_ts": "2026-09-10T17:08:22.203Z", + "end_ts": "2026-09-10T17:08:25.626Z", + "input_message": "Read alpha.txt and beta.txt with separate view calls in parallel and report their sum. Only read those two files.", + "output_message": "42", + "status": "completed", + "llm_calls": [ + { + "call_id": "copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "provider": "unknown", + "model": "gpt-5.6-luna", + "start_ts": "2026-09-10T17:08:24.503Z", + "end_ts": "2026-09-10T17:08:24.593Z", + "input_messages": [], + "output_messages": [], + "usage": {}, + "tool_calls": [] + } + ], + "permission_requests": [], + "subagents": [], + "attributes": { + "interaction_id": "6d2b89fd-a653-430c-b532-b0936d72eb42", + "agent_id": null + }, + "accounting_calls": [ + { + "accounting_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13", + "usage": { + "session_id": "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "seq": 0, + "call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13", + "ts": "2026-09-10T17:08:24.498Z", + "platform": "copilot", + "gen_ai.conversation.id": "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "gen_ai.provider.name": "unknown", + "gen_ai.operation.name": "chat", + "gen_ai.response.model": "gpt-5.6-luna", + "gen_ai.usage.input_tokens": 6452, + "gen_ai.usage.output_tokens": 107, + "gen_ai.usage.cache_read.input_tokens": 0, + "gen_ai.usage.cache_creation.input_tokens": 6449, + "gen_ai.usage.reasoning.output_tokens": 29 + }, + "attribution_status": "matched", + "agent_id": null, + "call_id": "copilot:call:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/a4a17e63-7ba5-422f-8ee9-b495be417328", + "attributes": {"accounting.destination": "chat-span"} + }, + { + "accounting_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:14", + "usage": { + "session_id": "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "seq": 0, + "call_id": "copilot:usage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:assistant_usage_events:sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:14", + "ts": "2026-09-10T17:08:25.618Z", + "platform": "copilot", + "gen_ai.conversation.id": "copilot-aaaaaaaaaaaaaaaa-5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6", + "gen_ai.provider.name": "unknown", + "gen_ai.operation.name": "chat", + "gen_ai.response.model": "gpt-5.6-luna", + "gen_ai.usage.input_tokens": 6587, + "gen_ai.usage.output_tokens": 5, + "gen_ai.usage.cache_read.input_tokens": 6449, + "gen_ai.usage.cache_creation.input_tokens": 135, + "gen_ai.usage.reasoning.output_tokens": 0 + }, + "attribution_status": "pending", + "agent_id": null, + "call_id": null, + "attributes": {"accounting.destination": "turn-accounting-span"} + } + ] + } +} diff --git a/tests/platforms/copilot/test_archive.py b/tests/platforms/copilot/test_archive.py new file mode 100644 index 00000000..79309373 --- /dev/null +++ b/tests/platforms/copilot/test_archive.py @@ -0,0 +1,464 @@ +"""Behavioral tests for the Copilot evidence archive (commit, cursor, retrieval).""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +import thirdeye.platforms.copilot.archive as archive_mod +from thirdeye.config import Config +from thirdeye.meta import read_meta +from thirdeye.paths import meta_path, session_dir +from thirdeye.platforms.copilot.archive import commit_batch, iter_captured_records, load_cursor +from thirdeye.platforms.copilot.constants import PLATFORM_NAME, SOURCE_SCHEMA_VERSION +from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id +from thirdeye.platforms.copilot.state import read_json, state_path +from thirdeye.platforms.copilot.types import SourceBatch, SourcePaths, SourceRecord +from thirdeye.reader import SessionReader + +FIXTURES = Path(__file__).parent / "fixtures" / "cases" +NATIVE_ID = "session-a" + + +def _record( + source_id: str, + *, + source_kind: str = "transcript", + native_session_id: str = NATIVE_ID, + ts: str | None = "2026-09-10T17:08:24.000Z", + observed_at: str = "2026-09-10T17:08:25.000Z", + payload: dict[str, Any] | None = None, +) -> SourceRecord: + return { + "source_id": source_id, + "source_kind": source_kind, + "native_session_id": native_session_id, + "ts": ts, + "observed_at": observed_at, + "payload": payload or {"schema_version": 1, "type": "user.message"}, + "locator": {"file": "events.jsonl", "offset": 0}, + } + + +def _batch( + paths: SourcePaths, + records: list[SourceRecord], + *, + next_cursor: dict[str, Any] | None = None, + base_cursor: dict[str, Any] | None = None, + diagnostics: list[dict[str, Any]] | None = None, + cwd: str | None = "/proj", +) -> SourceBatch: + cursor = dict(next_cursor or {"generation": 1}) + if base_cursor is not None: + cursor["base_cursor"] = base_cursor + return { + "source_key": paths["source_key"], + "native_session_id": NATIVE_ID, + "cwd": cwd, + "records": records, + "next_cursor": cursor, + "diagnostics": diagnostics or [], + } + + +@pytest.fixture +def copilot_home(tmp_path: Path) -> Path: + home = tmp_path / "copilot-home" + home.mkdir() + return home + + +@pytest.fixture +def config(tmp_path: Path) -> Config: + return Config(root=tmp_path / "thirdeye") + + +@pytest.fixture +def paths(copilot_home: Path) -> SourcePaths: + return resolve_sources(copilot_home) + + +def _session_directory(config: Config, paths: SourcePaths) -> Path: + stored = stored_session_id(paths, NATIVE_ID) + return session_dir(config.root, PLATFORM_NAME, stored) + + +def test_commit_batch_writes_records_and_advances_cursor( + config: Config, paths: SourcePaths +) -> None: + records = [_record("key/a/event-1"), _record("key/a/event-2")] + result = commit_batch(config, paths, _batch(paths, records, next_cursor={"offset": 2})) + + assert result == { + "sessions": 1, + "records_written": 2, + "duplicate_records": 0, + "pending": 0, + "errors": 0, + } + assert load_cursor(config, paths, NATIVE_ID) == {"offset": 2} + captured = list(iter_captured_records(config, stored_session_id(paths, NATIVE_ID))) + assert {r["source_id"] for r in captured} == {"key/a/event-1", "key/a/event-2"} + + +def test_load_cursor_returns_empty_for_unknown_session(config: Config, paths: SourcePaths) -> None: + assert load_cursor(config, paths, NATIVE_ID) == {} + + +def test_load_cursor_returns_defensive_copy(config: Config, paths: SourcePaths) -> None: + commit_batch( + config, paths, _batch(paths, [_record("key/a/event-1")], next_cursor={"offset": 1}) + ) + cursor = load_cursor(config, paths, NATIVE_ID) + cursor["offset"] = 999 + assert load_cursor(config, paths, NATIVE_ID) == {"offset": 1} + + +def test_iter_captured_records_empty_when_session_missing(config: Config) -> None: + assert list(iter_captured_records(config, "copilot-missing-session")) == [] + + +@pytest.mark.parametrize( + ("source_kind", "event_type"), + [ + ("transcript", "copilot_transcript"), + ("database", "copilot_database"), + ("hook", "copilot_hook"), + ("metadata", "copilot_metadata"), + ("unknown", "copilot_metadata"), + ], +) +def test_event_type_mapping( + config: Config, + paths: SourcePaths, + source_kind: str, + event_type: str, +) -> None: + record = _record("key/a/typed", source_kind=source_kind) + commit_batch(config, paths, _batch(paths, [record])) + events = list(SessionReader(_session_directory(config, paths)).iter_events()) + assert len(events) == 1 + assert events[0]["t"] == event_type + envelope = events[0]["data"] + assert envelope["schema_version"] == SOURCE_SCHEMA_VERSION + assert envelope["source_record"]["source_id"] == "key/a/typed" + + +def test_original_source_timestamp_is_retained(config: Config, paths: SourcePaths) -> None: + source_ts = "2026-09-10T17:08:24.000Z" + commit_batch( + config, + paths, + _batch(paths, [_record("key/a/ts", ts=source_ts, observed_at="2026-09-10T17:08:25.000Z")]), + ) + event = SessionReader(_session_directory(config, paths)).get_event(0) + assert event["ts"] == source_ts + + +def test_invalid_observed_at_is_rejected_even_when_source_ts_is_valid( + config: Config, paths: SourcePaths +) -> None: + result = commit_batch( + config, + paths, + _batch( + paths, + [ + _record( + "key/a/bad-observed", + ts="2026-09-10T17:08:24.000Z", + observed_at="2026-09-10T17:08:99.000Z", + ) + ], + next_cursor={"offset": 1}, + ), + ) + assert result["records_written"] == 0 + assert result["pending"] == 1 + assert result["errors"] == 1 + assert list(iter_captured_records(config, stored_session_id(paths, NATIVE_ID))) == [] + + +def test_missing_source_time_falls_back_to_observed_at(config: Config, paths: SourcePaths) -> None: + observed = "2026-09-10T17:08:25.000Z" + commit_batch( + config, paths, _batch(paths, [_record("key/a/no-ts", ts=None, observed_at=observed)]) + ) + event = SessionReader(_session_directory(config, paths)).get_event(0) + assert event["ts"] == observed + + state = read_json(state_path(_session_directory(config, paths))) + assert state is not None + kinds = {item["kind"] for item in state["health"]["diagnostics"]} + assert "missing_source_time" in kinds + + +def test_duplicate_source_ids_are_skipped(config: Config, paths: SourcePaths) -> None: + record = _record("key/a/dup") + first = commit_batch(config, paths, _batch(paths, [record], next_cursor={"offset": 1})) + second = commit_batch( + config, + paths, + _batch(paths, [record], next_cursor={"offset": 2}, base_cursor={"offset": 1}), + ) + assert first["records_written"] == 1 + assert second["records_written"] == 0 + assert second["duplicate_records"] == 1 + assert len(list(iter_captured_records(config, stored_session_id(paths, NATIVE_ID)))) == 1 + + +def test_batch_source_key_must_match_paths(config: Config, paths: SourcePaths) -> None: + batch = _batch(paths, [_record("key/a/event-1")]) + batch["source_key"] = "0" * 64 + with pytest.raises(ValueError, match="source_key does not match"): + commit_batch(config, paths, batch) + + +def test_batch_rejects_foreign_native_session_id(config: Config, paths: SourcePaths) -> None: + foreign = _record("key/a/foreign", native_session_id="other-session") + with pytest.raises(ValueError, match="another native session"): + commit_batch(config, paths, _batch(paths, [foreign])) + + +def test_source_key_prefix_collision_is_rejected( + config: Config, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Archive reuse must compare full source_key metadata, not just stored ID prefix.""" + import hashlib + + from thirdeye.platforms.copilot import identity as identity_mod + from thirdeye.store import Store + + case = json.loads((FIXTURES / "source-key-prefix-collision.json").read_text(encoding="utf-8")) + second_home = tmp_path / "home-b" + second_home.mkdir() + second_key = case["homes"][1]["source_key"] + + original_source_key = identity_mod._source_key + + def keyed_source_key(home: Path) -> str: + if home.resolve() == second_home.resolve(): + return second_key + return original_source_key(home) + + monkeypatch.setattr(identity_mod, "_source_key", keyed_source_key) + + stored_id = case["colliding_stored_session_id"] + first_key = case["homes"][0]["source_key"] + Store(config).open_session( + stored_id, + platform=PLATFORM_NAME, + cwd="/proj", + extra={ + "copilot": { + "schema_version": SOURCE_SCHEMA_VERSION, + "source_key": first_key, + "source_home": case["homes"][0]["home"], + "native_session_id": NATIVE_ID, + } + }, + ).flush_and_detach() + + second_paths: SourcePaths = { + "home": str(second_home.resolve()), + "source_key": second_key, + "session_root": str((second_home / "session-state").resolve()), + "database": str((second_home / "session-store.db").resolve()), + } + assert hashlib.sha256(str(second_home.resolve()).encode()).hexdigest() != second_key + with pytest.raises(ValueError, match="source-key prefix collision"): + commit_batch(config, second_paths, _batch(second_paths, [_record("key/b/second")])) + + +def test_provisional_session_metadata_is_created(config: Config, paths: SourcePaths) -> None: + commit_batch(config, paths, _batch(paths, [_record("key/a/hook", source_kind="hook")])) + meta = read_meta(meta_path(_session_directory(config, paths))) + assert meta is not None + assert meta.platform == PLATFORM_NAME + identity = meta.extra["copilot"] + assert identity["source_key"] == paths["source_key"] + assert identity["native_session_id"] == NATIVE_ID + + +def test_committed_cursor_strips_optimistic_base_marker(config: Config, paths: SourcePaths) -> None: + commit_batch(config, paths, _batch(paths, [_record("key/a/one")], next_cursor={"offset": 1})) + assert load_cursor(config, paths, NATIVE_ID) == {"offset": 1} + + commit_batch( + config, + paths, + _batch(paths, [_record("key/a/two")], next_cursor={"offset": 2}, base_cursor={"offset": 1}), + ) + cursor = load_cursor(config, paths, NATIVE_ID) + assert cursor == {"offset": 2} + state = read_json(state_path(_session_directory(config, paths))) + assert state is not None + assert "base_cursor" not in state["cursor"] + assert "_base_cursor" not in state["cursor"] + + +def test_append_does_not_reopen_closed_session(config: Config, paths: SourcePaths) -> None: + close_record = _record( + "key/a/close", + source_kind="hook", + payload={"event": "sessionEnd", "context": {}}, + ) + commit_batch(config, paths, _batch(paths, [close_record], next_cursor={"generation": 1})) + closed = read_meta(meta_path(_session_directory(config, paths))) + assert closed is not None + assert closed.status == "closed" + ended_at = closed.ended_at + assert ended_at is not None + + commit_batch( + config, + paths, + _batch( + paths, + [_record("key/a/after-close")], + next_cursor={"generation": 2}, + base_cursor={"generation": 1}, + ), + ) + meta = read_meta(meta_path(_session_directory(config, paths))) + assert meta is not None + assert meta.status == "closed" + assert meta.ended_at == ended_at + + +def test_invalid_timestamps_are_rejected_without_inventing_time( + config: Config, paths: SourcePaths +) -> None: + result = commit_batch( + config, + paths, + _batch( + paths, + [_record("key/a/bad-ts", ts="not-a-timestamp", observed_at="also-invalid")], + next_cursor={"offset": 1}, + ), + ) + assert result["records_written"] == 0 + assert result["pending"] == 1 + assert result["errors"] == 1 + assert load_cursor(config, paths, NATIVE_ID) == {} + assert list(iter_captured_records(config, stored_session_id(paths, NATIVE_ID))) == [] + + state = read_json(state_path(_session_directory(config, paths))) + assert state is not None + assert any(item.get("kind") == "invalid_observed_at" for item in state["health"]["diagnostics"]) + + +def test_session_end_closes_and_resume_reopens(config: Config, paths: SourcePaths) -> None: + close_record = _record( + "key/a/close", + source_kind="hook", + payload={"event": "sessionEnd", "context": {}}, + ) + commit_batch(config, paths, _batch(paths, [close_record])) + meta = read_meta(meta_path(_session_directory(config, paths))) + assert meta is not None + assert meta.status == "closed" + assert meta.ended_at is not None + + reopen_record = _record( + "key/a/reopen", + source_kind="hook", + payload={"event": "resume", "context": {}}, + ) + commit_batch( + config, + paths, + _batch( + paths, [reopen_record], next_cursor={"generation": 2}, base_cursor={"generation": 1} + ), + ) + meta = read_meta(meta_path(_session_directory(config, paths))) + assert meta is not None + assert meta.status == "open" + assert meta.ended_at is None + + +def test_last_lifecycle_event_in_batch_wins(config: Config, paths: SourcePaths) -> None: + start = _record( + "key/a/start", + source_kind="hook", + payload={"event": "sessionStart", "hook_payload": {}, "context": {}}, + ) + end = _record( + "key/a/end", + source_kind="hook", + payload={"event": "sessionEnd", "hook_payload": {}, "context": {}}, + ) + commit_batch(config, paths, _batch(paths, [start, end])) + meta = read_meta(meta_path(_session_directory(config, paths))) + assert meta is not None + assert meta.status == "closed" + assert meta.ended_at is not None + + later_end = _record( + "key/a/end-again", + source_kind="hook", + payload={"event": "sessionEnd", "hook_payload": {}, "context": {}}, + ) + resume = _record( + "key/a/resume-after-end", + source_kind="hook", + payload={"event": "resume", "hook_payload": {}, "context": {}}, + ) + commit_batch( + config, + paths, + _batch( + paths, + [later_end, resume], + next_cursor={"generation": 2}, + base_cursor={"generation": 1}, + ), + ) + meta = read_meta(meta_path(_session_directory(config, paths))) + assert meta is not None + assert meta.status == "open" + assert meta.ended_at is None + + +def test_child_stop_does_not_close_session(config: Config, paths: SourcePaths) -> None: + child_stop = _record( + "key/a/child-stop", + source_kind="hook", + payload={ + "event": "sessionEnd", + "hook_payload": {"agentId": "child-agent", "parentToolCallId": "tool-1"}, + "context": {}, + }, + ) + commit_batch(config, paths, _batch(paths, [child_stop])) + meta = read_meta(meta_path(_session_directory(config, paths))) + assert meta is not None + assert meta.status == "open" + assert meta.ended_at is None + + +def test_archive_module_has_no_usage_store_or_otel_imports() -> None: + source = Path(archive_mod.__file__).read_text(encoding="utf-8") + assert "UsageStore" not in source + assert "usage_store" not in source + assert "otel" not in source.lower() + + +def test_fixture_source_batch_can_be_committed(config: Config, copilot_home: Path) -> None: + raw = json.loads((FIXTURES / "source-batch.json").read_text(encoding="utf-8")) + paths = resolve_sources(copilot_home) + raw["source_key"] = paths["source_key"] + result = commit_batch(config, paths, raw) + assert result["records_written"] == 1 + assert result["errors"] == 0 + record = next(iter_captured_records(config, stored_session_id(paths, raw["native_session_id"]))) + assert record["source_kind"] == "database" + assert record["ts"] is None diff --git a/tests/platforms/copilot/test_attribution.py b/tests/platforms/copilot/test_attribution.py new file mode 100644 index 00000000..24afcd52 --- /dev/null +++ b/tests/platforms/copilot/test_attribution.py @@ -0,0 +1,930 @@ +"""Behavioral tests for Copilot usage attribution and projection composition.""" + +from __future__ import annotations + +import copy +import json +import shutil +from pathlib import Path +from typing import Any + +import pytest + +from thirdeye.platforms.copilot.attribution import join_usage +from thirdeye.platforms.copilot.identity import resolve_sources +from thirdeye.platforms.copilot.projection import build_projection +from thirdeye.platforms.copilot.tracing import build_semantics +from thirdeye.platforms.copilot.transcript import read_transcript +from thirdeye.platforms.copilot.types import ( + AccountingCandidate, + AccountingProjection, + CallCandidate, + SemanticProjection, + SourceRecord, +) +from thirdeye.platforms.copilot.usage import build_accounting +from thirdeye.usage.types import UsageRow + +FIXTURES = Path(__file__).parent / "fixtures" +RECON = FIXTURES / "reconciliation-cases" +NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" +CHILD_AGENT_ID = "bf8cb9f3-2097-4db0-a3c8-78a2653b2106" +SOURCE_KEY = "a" * 64 +GENERATION = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" +OBSERVED_AT = "2026-09-10T17:09:00.000Z" +INTERACTION_ONE = "6d2b89fd-a653-430c-b532-b0936d72eb42" +TURN_ONE = f"copilot:turn:{SOURCE_KEY}:{NATIVE_SESSION_ID}:{INTERACTION_ONE}" +CALL_A = f"copilot:call:{SOURCE_KEY}/{NATIVE_SESSION_ID}/a4a17e63-7ba5-422f-8ee9-b495be417328" +CALL_B = f"copilot:call:{SOURCE_KEY}/{NATIVE_SESSION_ID}/33cc6465-29e1-4a04-8bdb-00241474b4d2" +TS_CYCLE_0 = "2026-09-10T17:08:24.503Z" +TS_CYCLE_1 = "2026-09-10T17:08:25.624Z" +INTERACTION_TWO = "793d3703-6f4a-4814-8877-34a7325848ce" + + +def _load_json(path: Path) -> Any: + return json.loads(path.read_text(encoding="utf-8")) + + +def _drain_cli_transcript(home: Path) -> list[SourceRecord]: + session_dir = home / "session-state" / NATIVE_SESSION_ID + session_dir.mkdir(parents=True, exist_ok=True) + shutil.copy(FIXTURES / "events.jsonl", session_dir / "events.jsonl") + (session_dir / "workspace.yaml").write_text("cwd: /sanitized/workspace\n", encoding="utf-8") + paths = resolve_sources(home) + cursor: dict[str, Any] = {} + records: list[SourceRecord] = [] + while True: + slice_ = read_transcript(paths, NATIVE_SESSION_ID, cursor) + records.extend(slice_["records"]) + cursor = slice_["next_cursor"] + if slice_["exhausted"]: + break + return [record for record in records if record["source_kind"] == "transcript"] + + +def _usage_record( + row: dict[str, Any], + *, + content_revision: str, + generation: str = GENERATION, + source_key: str = SOURCE_KEY, + observed_at: str = OBSERVED_AT, +) -> SourceRecord: + primary_key = row["id"] + source_id = ( + f"copilot-db:{source_key}:{NATIVE_SESSION_ID}:assistant_usage_events:" + f"{primary_key}:{content_revision}" + ) + return { + "source_id": source_id, + "source_kind": "database", + "native_session_id": NATIVE_SESSION_ID, + "ts": row.get("created_at"), + "observed_at": observed_at, + "payload": {"table": "assistant_usage_events", "row": row}, + "locator": { + "database": "/example/.copilot/session-store.db", + "table": "assistant_usage_events", + "primary_key": primary_key, + "content_revision": content_revision, + "generation": generation, + }, + } + + +def _source_key(records: list[SourceRecord]) -> str: + for record in records: + if record["source_kind"] == "transcript": + return record["source_id"].split("/", 1)[0] + pytest.fail("expected at least one transcript record") + + +def _substitute_source_key(value: str, source_key: str) -> str: + return value.replace(SOURCE_KEY, source_key) + + +def _rewrite_source_key(value: Any, source_key: str) -> Any: + if isinstance(value, str): + return _substitute_source_key(value, source_key) + if isinstance(value, list): + return [_rewrite_source_key(item, source_key) for item in value] + if isinstance(value, dict): + return {key: _rewrite_source_key(item, source_key) for key, item in value.items()} + return value + + +def _iter_turns(turns: list[dict[str, Any]]) -> list[dict[str, Any]]: + found: list[dict[str, Any]] = [] + for turn in turns: + found.append(turn) + found.extend(_iter_turns(turn.get("subagents") or [])) + return found + + +def _accounting_calls(turns: list[dict[str, Any]]) -> list[dict[str, Any]]: + found: list[dict[str, Any]] = [] + for turn in _iter_turns(turns): + found.extend(turn.get("accounting_calls") or []) + return found + + +def _align_attribution(expected: dict[str, Any], source_key: str) -> dict[str, Any]: + aligned = copy.deepcopy(expected) + for field in ("call_id", "stored_turn_id", "logical_call_id", "usage_source_id"): + if isinstance(aligned.get(field), str): + aligned[field] = _substitute_source_key(aligned[field], source_key) + aligned["evidence"] = [_substitute_source_key(item, source_key) for item in aligned["evidence"]] + return aligned + + +def _six_call_records(*, source_key: str = SOURCE_KEY) -> list[SourceRecord]: + rows = _load_json(FIXTURES / "assistant-usage-events.json") + revisions = { + call["row_id"]: call["usage_source_id"].rsplit(":", 1)[-1] + for call in _load_json(RECON / "observed-six-calls.json")["calls"] + } + return [ + _usage_record(row, content_revision=revisions[row["id"]], source_key=source_key) + for row in rows + ] + + +def _metric_totals(usage_rows: list[UsageRow]) -> dict[str, int]: + totals = { + "input_tokens": 0, + "output_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "reasoning_tokens": 0, + } + for row in usage_rows: + totals["input_tokens"] += row.input_tokens + totals["output_tokens"] += row.output_tokens + totals["cache_read_tokens"] += row.cache_read_input_tokens or 0 + totals["cache_write_tokens"] += row.cache_creation_input_tokens or 0 + totals["reasoning_tokens"] += row.reasoning_output_tokens or 0 + return totals + + +def _call_candidate( + *, + call_id: str, + stored_turn_id: str | None = TURN_ONE, + interaction_id: str = INTERACTION_ONE, + agent_id: str | None = None, + parent_tool_call_id: str | None = None, + model: str = "gpt-5.6-luna", + tool_call_ids: list[str] | None = None, + finish_evidence: list[dict[str, Any]] | None = None, + assistant_message_id: str | None = None, + start_ts: str = TS_CYCLE_0, + end_ts: str = "2026-09-10T17:08:24.593Z", +) -> CallCandidate: + candidate: CallCandidate = { + "call_id": call_id, + "stored_turn_id": stored_turn_id, + "interaction_id": interaction_id, + "agent_id": agent_id, + "parent_tool_call_id": parent_tool_call_id, + "model": model, + "source_ids": [call_id.rsplit("/", 1)[-1]], + "source_references": [], + "start_ts": start_ts, + "end_ts": end_ts, + "tool_call_ids": tool_call_ids or [], + "finish_evidence": finish_evidence or [], + } + if assistant_message_id is not None: + candidate["assistant_message_id"] = assistant_message_id # type: ignore[typeddict-unknown-key] + return candidate + + +def _accounting_candidate( + *, + row_id: int, + turn_index: int | None = 0, + agent_id: str | None = None, + parent_tool_call_id: str | None = None, + model: str = "gpt-5.6-luna", + finish_reason: str | None = "tool_calls", + initiator: str | None = "user", + revision: str = "sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + assistant_message_id: str | None = None, +) -> AccountingCandidate: + logical_call_id = ( + f"copilot:usage:{SOURCE_KEY}:assistant_usage_events:" + f"sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:{row_id}" + ) + usage_source_id = ( + f"copilot-db:{SOURCE_KEY}:{NATIVE_SESSION_ID}:assistant_usage_events:{row_id}:{revision}" + ) + supplemental: dict[str, Any] = {} + if initiator is not None: + supplemental["initiator"] = initiator + candidate: AccountingCandidate = { + "usage_source_id": usage_source_id, + "logical_call_id": logical_call_id, + "turn_index": turn_index, + "agent_id": agent_id, + "parent_tool_call_id": parent_tool_call_id, + "model": model, + "provider": None, + "source_ids": [usage_source_id], + "source_references": [], + "revision": { + "primary_key": row_id, + "content_revision": revision, + "generation": GENERATION, + }, + "timestamp": "2026-09-10T17:08:24.498Z", + "finish_reason": finish_reason, + "supplemental_metrics": supplemental, + } + if assistant_message_id is not None: + candidate["assistant_message_id"] = assistant_message_id # type: ignore[typeddict-unknown-key] + return candidate + + +def _semantic(*, calls: list[CallCandidate]) -> SemanticProjection: + return { + "events": [], + "turns": [ + { + "turn_id": TURN_ONE, + "stored_turn_id": TURN_ONE, + "status": "completed", + "output_message": "42", + "subagents": [], + "attributes": {"interaction_id": INTERACTION_ONE}, + } + ], + "call_candidates": calls, + "pending": [], + "diagnostics": [], + } + + +def _accounting(*, candidates: list[AccountingCandidate]) -> AccountingProjection: + return {"usage_rows": [], "candidates": candidates, "diagnostics": []} + + +def _attribution_by_logical(attributions: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + return {item["logical_call_id"]: item for item in attributions} + + +# --- observed six-call corpus --- + + +def test_six_call_corpus_joins_all_usage_rows_inferred(cli_transcript_records: list[SourceRecord]): + source_key = _source_key(cli_transcript_records) + records = cli_transcript_records + _six_call_records(source_key=source_key) + projection, _state = build_projection(records, {}) + expected = _load_json(RECON / "observed-six-calls.json")["expected_attributions"] + by_logical = _attribution_by_logical(projection["attributions"]) + + assert len(projection["attributions"]) == 6 + assert all(item["status"] == "matched" for item in projection["attributions"]) + assert all(item["join_kind"] == "inferred" for item in projection["attributions"]) + for wanted in expected: + aligned = _align_attribution(wanted, source_key) + actual = by_logical[aligned["logical_call_id"]] + assert actual["call_id"] == aligned["call_id"] + assert actual["stored_turn_id"] == aligned["stored_turn_id"] + assert actual["agent_id"] == aligned["agent_id"] + assert actual["evidence"][0] == "join_kind:inferred" + assert actual["evidence"][-1] == aligned["evidence"][-1] + assert f"turn_index:{aligned['evidence'][2].split(':', 1)[1]}" in actual["evidence"] + + +def test_six_call_usage_totals_unchanged_by_attribution(cli_transcript_records: list[SourceRecord]): + source_key = _source_key(cli_transcript_records) + records = cli_transcript_records + _six_call_records(source_key=source_key) + projection, _state = build_projection(records, {}) + assert len(projection["usage_rows"]) == 6 + assert _metric_totals(projection["usage_rows"])["input_tokens"] == 35396 + + +def test_child_usage_maps_to_main_stored_turn(cli_transcript_records: list[SourceRecord]): + source_key = _source_key(cli_transcript_records) + records = cli_transcript_records + _six_call_records(source_key=source_key) + projection, _state = build_projection(records, {}) + child = next(item for item in projection["attributions"] if item["agent_id"] == CHILD_AGENT_ID) + main_turn_two = ( + f"copilot:turn:{source_key}:{NATIVE_SESSION_ID}:793d3703-6f4a-4814-8877-34a7325848ce" + ) + assert child["stored_turn_id"] == main_turn_two + assert child["status"] == "matched" + + +# --- contract fixtures via join_usage --- + + +def test_attribution_fixture_matched_inferred(): + examples = _load_json(RECON / "attributions.json") + semantic, _ = build_semantics( + _load_json(RECON / "semantic-projection.json")["input_records"], {} + ) + accounting, _ = build_accounting( + [ + _usage_record( + _load_json(FIXTURES / "assistant-usage-events.json")[0], + content_revision="sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + ) + ], + {}, + ) + attributions = join_usage(semantic, accounting) + assert attributions == [examples["matched_inferred"]] + + +def test_attribution_fixture_pending_without_matching_call(): + examples = _load_json(RECON / "attributions.json") + case = _load_json(RECON / "cases.json")["late_row"] + projection, _state = build_projection(case["input_records"], {}) + assert projection["attributions"] == [examples["pending"]] + + +# --- direct / native joins --- + + +def test_direct_join_wins_over_inferred_evidence(): + shared_message_id = "native-msg-abc" + semantic = _semantic( + calls=[ + _call_candidate( + call_id=CALL_A, + tool_call_ids=["tool-a"], + assistant_message_id=shared_message_id, + ), + _call_candidate( + call_id=CALL_B, + tool_call_ids=[], + finish_evidence=[{"source_id": "finish"}], + start_ts=TS_CYCLE_1, + end_ts=TS_CYCLE_1, + ), + ] + ) + accounting = _accounting( + candidates=[_accounting_candidate(row_id=13, assistant_message_id=shared_message_id)] + ) + attributions = join_usage(semantic, accounting) + assert attributions[0]["status"] == "matched" + assert attributions[0]["join_kind"] == "direct" + assert attributions[0]["call_id"] == CALL_A + assert "join_kind:direct" in attributions[0]["evidence"] + + +def test_competing_direct_ids_are_conflicting(): + shared_message_id = "shared-native-id" + semantic = _semantic( + calls=[ + _call_candidate(call_id=CALL_A, assistant_message_id=shared_message_id), + _call_candidate(call_id=CALL_B, assistant_message_id=shared_message_id), + ] + ) + accounting = _accounting( + candidates=[_accounting_candidate(row_id=13, assistant_message_id=shared_message_id)] + ) + attributions = join_usage(semantic, accounting) + assert attributions[0]["status"] == "conflicting" + assert attributions[0]["call_id"] is None + assert f"call_id:{CALL_A}" in attributions[0]["evidence"] + assert f"call_id:{CALL_B}" in attributions[0]["evidence"] + assert not any(item.startswith("join_conflict:") for item in attributions[0]["evidence"]) + + +# --- inferred / ambiguous / pending --- + + +def test_inferred_join_requires_unique_consistent_evidence(): + semantic = _semantic( + calls=[ + _call_candidate( + call_id=CALL_A, + tool_call_ids=["tool-a"], + ), + _call_candidate( + call_id=CALL_B, + tool_call_ids=[], + finish_evidence=[{"source_id": "finish-b"}], + start_ts=TS_CYCLE_1, + end_ts=TS_CYCLE_1, + ), + ] + ) + accounting = _accounting( + candidates=[ + _accounting_candidate(row_id=13, finish_reason="tool_calls", initiator="user"), + _accounting_candidate( + row_id=14, + finish_reason="stop", + initiator="agent", + ), + ] + ) + attributions = join_usage(semantic, accounting) + by_row = {item["logical_call_id"].rsplit(":", 1)[-1]: item for item in attributions} + assert by_row["13"]["status"] == "matched" + assert by_row["13"]["join_kind"] == "inferred" + assert by_row["13"]["call_id"] == CALL_A + assert by_row["14"]["status"] == "matched" + assert by_row["14"]["call_id"] == CALL_B + + +def test_ambiguous_when_multiple_calls_fit_same_evidence(): + semantic = _semantic( + calls=[ + _call_candidate(call_id=CALL_A, tool_call_ids=["tool-a"]), + _call_candidate(call_id=CALL_B, tool_call_ids=["tool-b"], start_ts=TS_CYCLE_1), + ] + ) + accounting = _accounting( + candidates=[ + _accounting_candidate(row_id=13, finish_reason=None, initiator=None), + ] + ) + attributions = join_usage(semantic, accounting) + assert attributions[0]["status"] == "ambiguous" + assert attributions[0]["call_id"] is None + assert f"call_id:{CALL_A}" in attributions[0]["evidence"] + assert f"call_id:{CALL_B}" in attributions[0]["evidence"] + + +def test_pending_when_turn_index_has_no_main_interaction(): + semantic = _semantic(calls=[_call_candidate(call_id=CALL_A, tool_call_ids=["tool-a"])]) + accounting = _accounting(candidates=[_accounting_candidate(row_id=13, turn_index=99)]) + attributions = join_usage(semantic, accounting) + assert attributions[0]["status"] == "pending" + assert attributions[0]["stored_turn_id"] is None + assert attributions[0]["call_id"] is None + assert "turn_index:99" in attributions[0]["evidence"] + assert "delayed_row:true" not in attributions[0]["evidence"] + + +def test_late_row_case_stays_pending_until_semantics_catch_up(): + case = _load_json(RECON / "cases.json")["late_row"] + projection, _state = build_projection(case["input_records"], {}) + attribution = projection["attributions"][0] + assert attribution["status"] == "pending" + assert attribution["call_id"] is None + assert attribution["logical_call_id"] == case["expected"]["attributions"][0]["logical_call_id"] + assert "delayed_row:true" not in attribution["evidence"] + assert "turn_index:0" in attribution["evidence"] + assert len(projection["usage_rows"]) == 1 + + +def test_delayed_row_resolves_after_transcript_replay(): + case = _load_json(RECON / "cases.json")["late_row"] + partial = case["input_records"] + final_answer = _load_json(RECON / "cases.json")["ambiguous"]["input_records"][1] + turn_end = { + "source_id": (f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/4a386a37-ca7e-4ebc-a746-cdda20f2a4bb"), + "source_kind": "transcript", + "native_session_id": NATIVE_SESSION_ID, + "ts": "2026-09-10T17:08:25.626Z", + "observed_at": OBSERVED_AT, + "payload": { + "type": "assistant.turn_end", + "data": {"turnId": "1"}, + "id": "4a386a37-ca7e-4ebc-a746-cdda20f2a4bb", + "timestamp": "2026-09-10T17:08:25.626Z", + "parentId": "33cc6465-29e1-4a04-8bdb-00241474b4d2", + "schema_version": 1, + }, + "locator": { + "file": "events.jsonl", + "native_event_id": "4a386a37-ca7e-4ebc-a746-cdda20f2a4bb", + }, + } + full = partial + [final_answer, turn_end] + pending_projection, _ = build_projection(partial, {}) + resolved_projection, _ = build_projection(full, {}) + assert pending_projection["attributions"][0]["status"] == "pending" + assert resolved_projection["attributions"][0]["status"] == "matched" + assert resolved_projection["attributions"][0]["call_id"].endswith( + "33cc6465-29e1-4a04-8bdb-00241474b4d2" + ) + + +# --- conflicting joins --- + + +def test_two_usage_rows_claiming_one_call_become_conflicting(): + semantic = _semantic( + calls=[ + _call_candidate( + call_id=CALL_A, + tool_call_ids=["tool-a"], + ), + _call_candidate( + call_id=CALL_B, + tool_call_ids=[], + finish_evidence=[{"source_id": "finish-b"}], + start_ts=TS_CYCLE_1, + end_ts=TS_CYCLE_1, + ), + ] + ) + accounting = _accounting( + candidates=[ + _accounting_candidate(row_id=13, finish_reason="tool_calls", initiator="user"), + _accounting_candidate(row_id=14, finish_reason="tool_calls", initiator="user"), + ] + ) + attributions = join_usage(semantic, accounting) + assert all(item["status"] == "conflicting" for item in attributions) + assert all(item["call_id"] is None for item in attributions) + assert all(f"call_id:{CALL_A}" in item["evidence"] for item in attributions) + assert all( + not any(token.startswith("join_conflict:") for token in item["evidence"]) + for item in attributions + ) + + +# --- build_projection composition --- + + +def test_build_projection_attaches_accounting_calls_without_mutating_semantics( + cli_transcript_records: list[SourceRecord], +): + source_key = _source_key(cli_transcript_records) + records = cli_transcript_records + _six_call_records(source_key=source_key) + semantic_before, _ = build_semantics(records, {}) + projection, _state = build_projection(records, {}) + semantic_after, _ = build_semantics(records, {}) + + assert semantic_before["turns"] == semantic_after["turns"] + assert all(not turn.get("accounting_calls") for turn in semantic_before["turns"]) + + attached = _accounting_calls(projection["turns"]) + assert len(attached) == 6 + matched = [item for item in projection["attributions"] if item["status"] == "matched"] + by_turn = {turn["turn_id"]: turn for turn in _iter_turns(projection["turns"])} + for attribution in matched: + owner = next( + turn + for turn in _iter_turns(projection["turns"]) + if any( + item.get("call_id") == attribution["call_id"] + for item in turn.get("accounting_calls") or [] + ) + ) + if attribution["agent_id"] is None: + assert owner["turn_id"] == attribution["stored_turn_id"] + else: + assert owner["turn_id"] != attribution["stored_turn_id"] + assert owner in (by_turn[attribution["stored_turn_id"]].get("subagents") or []) + assert attribution["call_id"] in [ + item["call_id"] for item in owner.get("accounting_calls") or [] + ] + + +def test_matched_child_usage_attaches_to_nested_turn( + cli_transcript_records: list[SourceRecord], +): + source_key = _source_key(cli_transcript_records) + records = cli_transcript_records + _six_call_records(source_key=source_key) + projection, _ = build_projection(records, {}) + child = next(item for item in projection["attributions"] if item["agent_id"] == CHILD_AGENT_ID) + main = next(turn for turn in projection["turns"] if turn["turn_id"] == child["stored_turn_id"]) + nested = main["subagents"][0] + nested_ids = [ + item["call_id"] + for item in nested.get("accounting_calls") or [] + if item["attribution_status"] == "matched" + ] + main_ids = [ + item["call_id"] for item in main.get("accounting_calls") or [] if item.get("call_id") + ] + assert child["call_id"] in nested_ids + assert child["call_id"] not in main_ids + assert any(llm.get("call_id") == child["call_id"] for llm in nested.get("llm_calls") or []) + + +def test_retry_reordering_same_interaction_keeps_call_assignments(): + first = _call_candidate(call_id=CALL_A, tool_call_ids=["tool-a"], start_ts=TS_CYCLE_0) + second = _call_candidate( + call_id=CALL_B, + tool_call_ids=[], + finish_evidence=[{"source_id": "finish-b"}], + start_ts=TS_CYCLE_1, + end_ts=TS_CYCLE_1, + ) + usage_user = _accounting_candidate(row_id=13, finish_reason="tool_calls", initiator="user") + usage_agent = _accounting_candidate(row_id=14, finish_reason="stop", initiator="agent") + forward = join_usage( + _semantic(calls=[first, second]), + _accounting(candidates=[usage_user, usage_agent]), + ) + reversed_calls = join_usage( + _semantic(calls=[second, first]), + _accounting(candidates=[usage_agent, usage_user]), + ) + by_forward = {item["logical_call_id"].rsplit(":", 1)[-1]: item for item in forward} + by_reversed = {item["logical_call_id"].rsplit(":", 1)[-1]: item for item in reversed_calls} + assert by_forward["13"]["call_id"] == by_reversed["13"]["call_id"] == CALL_A + assert by_forward["14"]["call_id"] == by_reversed["14"]["call_id"] == CALL_B + assert "order:0" in by_forward["13"]["evidence"] + assert "order:1" in by_forward["14"]["evidence"] + assert "order:0" in by_reversed["13"]["evidence"] + assert "order:1" in by_reversed["14"]["evidence"] + + +def test_cyclic_child_does_not_consume_turn_index(): + child_a = _call_candidate( + call_id=f"copilot:call:{SOURCE_KEY}/{NATIVE_SESSION_ID}/cycle-a", + stored_turn_id=f"{TURN_ONE}:agent-a", + interaction_id="child-cycle", + agent_id="agent-a", + parent_tool_call_id="tool-b", + tool_call_ids=["tool-a"], + ) + child_b = _call_candidate( + call_id=f"copilot:call:{SOURCE_KEY}/{NATIVE_SESSION_ID}/cycle-b", + stored_turn_id=f"{TURN_ONE}:agent-b", + interaction_id="child-cycle", + agent_id="agent-b", + parent_tool_call_id="tool-a", + tool_call_ids=["tool-b"], + start_ts=TS_CYCLE_1, + ) + main = _call_candidate(call_id=CALL_A, tool_call_ids=["tool-main"]) + attributions = join_usage( + _semantic(calls=[child_a, child_b, main]), + _accounting(candidates=[_accounting_candidate(row_id=13, turn_index=0)]), + ) + assert attributions[0]["stored_turn_id"] == TURN_ONE + assert attributions[0]["call_id"] == CALL_A + assert attributions[0]["status"] == "matched" + + +def test_non_unique_parent_tool_does_not_consume_turn_index(): + child = _call_candidate( + call_id=f"copilot:call:{SOURCE_KEY}/{NATIVE_SESSION_ID}/orphan-child", + stored_turn_id=f"{TURN_ONE}:agent-a", + interaction_id="child-orphan", + agent_id="agent-a", + parent_tool_call_id="shared-tool", + tool_call_ids=["child-tool"], + ) + main = _call_candidate(call_id=CALL_A, tool_call_ids=["shared-tool"]) + other = _call_candidate( + call_id=CALL_B, + stored_turn_id=f"copilot:turn:{SOURCE_KEY}:{NATIVE_SESSION_ID}:{INTERACTION_TWO}", + interaction_id=INTERACTION_TWO, + tool_call_ids=["shared-tool"], + start_ts=TS_CYCLE_1, + ) + attributions = join_usage( + _semantic(calls=[child, main, other]), + _accounting(candidates=[_accounting_candidate(row_id=13, turn_index=0)]), + ) + assert attributions[0]["stored_turn_id"] == TURN_ONE + assert attributions[0]["call_id"] == CALL_A + assert attributions[0]["status"] == "matched" + + +def test_partial_turn_incremental_state_matches_full_archive(): + cases = _load_json(RECON / "cases.json") + partial = cases["partial_turn"]["input_records"] + later = cases["abort"]["input_records"] + full = partial + later + first, state = build_projection(partial, {}) + assert ( + "6d2b89fd-a653-430c-b532-b0936d72eb42|main" in state["semantic_state"]["open_interactions"] + ) + incremental, inc_state = build_projection(later, state) + from_flat, flat_state = build_projection(later, state["semantic_state"]) + complete, full_state = build_projection(full, {}) + assert incremental["turns"] == complete["turns"] == from_flat["turns"] + assert ( + inc_state["semantic_state"]["open_interactions"].keys() + == full_state["semantic_state"]["open_interactions"].keys() + == flat_state["semantic_state"]["open_interactions"].keys() + ) + assert ( + "6d2b89fd-a653-430c-b532-b0936d72eb42|main" + in inc_state["semantic_state"]["open_interactions"] + ) + assert first["turns"] == [] + + +def test_direct_match_does_not_emit_join_capability_gap( + cli_transcript_records: list[SourceRecord], +): + source_key = _source_key(cli_transcript_records) + records = cli_transcript_records + _six_call_records(source_key=source_key) + projection, _ = build_projection(records, {}) + join_gaps = [ + item + for item in projection["diagnostics"] + if item["code"] == "capability_gap" and item.get("details", {}).get("logical_call_id") + ] + assert join_gaps == [] + + +def test_accounting_rows_persist_when_attribution_is_ambiguous(): + user = { + "source_id": f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/amb-user", + "source_kind": "transcript", + "native_session_id": NATIVE_SESSION_ID, + "ts": "2026-09-10T17:08:22.203Z", + "observed_at": OBSERVED_AT, + "payload": { + "type": "user.message", + "data": { + "content": "two similar cycles", + "interactionId": INTERACTION_ONE, + "turnId": "0", + }, + "id": "amb-user", + "timestamp": "2026-09-10T17:08:22.203Z", + "schema_version": 1, + }, + "locator": {"file": "events.jsonl", "native_event_id": "amb-user"}, + } + first_msg = { + "source_id": f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/a4a17e63-7ba5-422f-8ee9-b495be417328", + "source_kind": "transcript", + "native_session_id": NATIVE_SESSION_ID, + "ts": TS_CYCLE_0, + "observed_at": OBSERVED_AT, + "payload": { + "type": "assistant.message", + "data": { + "content": "", + "model": "gpt-5.6-luna", + "interactionId": INTERACTION_ONE, + "turnId": "0", + "toolRequests": [{"toolCallId": "tool-a", "name": "view", "arguments": {}}], + }, + "id": "a4a17e63-7ba5-422f-8ee9-b495be417328", + "timestamp": TS_CYCLE_0, + "schema_version": 1, + }, + "locator": { + "file": "events.jsonl", + "native_event_id": "a4a17e63-7ba5-422f-8ee9-b495be417328", + }, + } + first_end = { + "source_id": f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/amb-end-0", + "source_kind": "transcript", + "native_session_id": NATIVE_SESSION_ID, + "ts": "2026-09-10T17:08:24.593Z", + "observed_at": OBSERVED_AT, + "payload": { + "type": "assistant.turn_end", + "data": {"turnId": "0"}, + "id": "amb-end-0", + "timestamp": "2026-09-10T17:08:24.593Z", + "schema_version": 1, + }, + "locator": {"file": "events.jsonl", "native_event_id": "amb-end-0"}, + } + second_msg = { + "source_id": f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/33cc6465-29e1-4a04-8bdb-00241474b4d2", + "source_kind": "transcript", + "native_session_id": NATIVE_SESSION_ID, + "ts": TS_CYCLE_1, + "observed_at": OBSERVED_AT, + "payload": { + "type": "assistant.message", + "data": { + "content": "", + "model": "gpt-5.6-luna", + "interactionId": INTERACTION_ONE, + "turnId": "1", + "toolRequests": [{"toolCallId": "tool-b", "name": "view", "arguments": {}}], + }, + "id": "33cc6465-29e1-4a04-8bdb-00241474b4d2", + "timestamp": TS_CYCLE_1, + "schema_version": 1, + }, + "locator": { + "file": "events.jsonl", + "native_event_id": "33cc6465-29e1-4a04-8bdb-00241474b4d2", + }, + } + second_end = { + "source_id": f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/amb-end-1", + "source_kind": "transcript", + "native_session_id": NATIVE_SESSION_ID, + "ts": "2026-09-10T17:08:25.700Z", + "observed_at": OBSERVED_AT, + "payload": { + "type": "assistant.turn_end", + "data": {"turnId": "1"}, + "id": "amb-end-1", + "timestamp": "2026-09-10T17:08:25.700Z", + "schema_version": 1, + }, + "locator": {"file": "events.jsonl", "native_event_id": "amb-end-1"}, + } + row = copy.deepcopy(_load_json(FIXTURES / "assistant-usage-events.json")[0]) + row["finish_reason"] = None + row["initiator"] = None + usage = _usage_record( + row, + content_revision="sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + ) + ambiguous, _ = build_projection([user, first_msg, first_end, second_msg, second_end, usage], {}) + matched, _ = build_projection([user, first_msg, first_end, usage], {}) + assert ambiguous["attributions"][0]["status"] == "ambiguous" + assert matched["attributions"][0]["status"] == "matched" + assert len(ambiguous["usage_rows"]) == len(matched["usage_rows"]) == 1 + assert _metric_totals(ambiguous["usage_rows"]) == _metric_totals(matched["usage_rows"]) + assert ambiguous["usage_rows"][0].input_tokens == 6452 + + +def test_late_row_projection_keeps_usage_rows_while_attribution_pending(): + case = _load_json(RECON / "cases.json")["late_row"] + projection, _ = build_projection(case["input_records"], {}) + assert projection["attributions"][0]["status"] == "pending" + assert len(projection["usage_rows"]) == 1 + assert projection["usage_rows"][0].input_tokens == 6587 + + +def test_unmatched_usage_with_known_turn_stays_on_main_without_call_id(): + abort = _load_json(RECON / "cases.json")["abort"]["input_records"] + row = copy.deepcopy(_load_json(FIXTURES / "assistant-usage-events.json")[0]) + row["model"] = "not-the-transcript-model" + usage = _usage_record( + row, + content_revision="sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + ) + projection, _ = build_projection(abort + [usage], {}) + assert projection["attributions"][0]["status"] == "pending" + assert projection["attributions"][0]["stored_turn_id"] is not None + assert projection["attributions"][0]["call_id"] is None + attached = _accounting_calls(projection["turns"]) + assert len(attached) == 1 + assert attached[0]["call_id"] is None + assert attached[0]["attribution_status"] == "pending" + owner = next(turn for turn in projection["turns"] if turn.get("accounting_calls")) + assert owner["turn_id"] == projection["attributions"][0]["stored_turn_id"] + + +def test_pending_usage_without_stored_turn_skips_accounting_calls(): + records = _six_call_records()[:1] + projection, _ = build_projection(records, {}) + assert projection["attributions"][0]["stored_turn_id"] is None + assert projection["attributions"][0]["status"] == "pending" + assert len(projection["usage_rows"]) == 1 + assert projection["turns"] == [] + assert _accounting_calls(projection["turns"]) == [] + pending = [item for item in projection["pending"] if item["kind"] == "unmatched_usage"] + assert len(pending) == 1 + + +def test_matched_inferred_usage_emits_diagnostic(cli_transcript_records: list[SourceRecord]): + source_key = _source_key(cli_transcript_records) + records = cli_transcript_records + _six_call_records(source_key=source_key) + projection, _ = build_projection(records, {}) + inferred = [item for item in projection["diagnostics"] if item["code"] == "inferred_join"] + assert len(inferred) == 6 + + +def test_unmatched_usage_emits_pending_item(): + case = _load_json(RECON / "cases.json")["late_row"] + projection, _ = build_projection(case["input_records"], {}) + pending = [item for item in projection["pending"] if item["kind"] == "unmatched_usage"] + assert len(pending) == 1 + assert pending[0]["reason"] == "usage attribution is pending" + assert pending[0]["id"].startswith("pending:usage:") + + +def test_source_correction_replaces_attribution_target_after_replay( + cli_transcript_records: list[SourceRecord], +): + source_key = _source_key(cli_transcript_records) + original = cli_transcript_records + _six_call_records(source_key=source_key) + first, state = build_projection(original, {}) + original_row = next(row for row in first["usage_rows"] if row.call_id.endswith(":13")) + original_join = next( + item for item in first["attributions"] if item["logical_call_id"] == original_row.call_id + ) + corrected = _rewrite_source_key( + copy.deepcopy(_load_json(RECON / "cases.json")["revision"]["input_records"][1]), + source_key, + ) + second, _ = build_projection(original + [corrected], state) + assert len(second["usage_rows"]) == 6 + replaced = next(row for row in second["usage_rows"] if row.call_id == original_row.call_id) + joined = next( + item for item in second["attributions"] if item["logical_call_id"] == original_row.call_id + ) + assert replaced.output_tokens == 999 + assert original_row.output_tokens != 999 + assert joined["call_id"] == original_join["call_id"] + assert joined["usage_source_id"] == corrected["source_id"] + assert joined["logical_call_id"] == original_join["logical_call_id"] + assert _metric_totals(second["usage_rows"])["output_tokens"] == ( + _metric_totals(first["usage_rows"])["output_tokens"] - original_row.output_tokens + 999 + ) + + +@pytest.fixture +def cli_transcript_records(tmp_path: Path) -> list[SourceRecord]: + return _drain_cli_transcript(tmp_path / "copilot-home") diff --git a/tests/platforms/copilot/test_capture.py b/tests/platforms/copilot/test_capture.py new file mode 100644 index 00000000..061141ab --- /dev/null +++ b/tests/platforms/copilot/test_capture.py @@ -0,0 +1,966 @@ +"""Behavioral tests for Copilot capture composition (sync, hooks, spool, archive).""" + +from __future__ import annotations + +import json +import shutil +import sqlite3 +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path +from typing import Any +from unittest.mock import patch + +import pytest + +import thirdeye.platforms.copilot.archive as archive_mod +import thirdeye.platforms.copilot.capture as capture_mod +import thirdeye.platforms.copilot.state as state_mod +from thirdeye.config import Config +from thirdeye.meta import read_meta +from thirdeye.paths import meta_path, session_dir +from thirdeye.platforms.copilot.archive import commit_batch, load_cursor +from thirdeye.platforms.copilot.capture import ( + capture_session, + iter_captured_records, + record_hook, + sync, +) +from thirdeye.platforms.copilot.constants import PLATFORM_NAME +from thirdeye.platforms.copilot.hook_payload import parse_hook +from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id +from thirdeye.platforms.copilot.spool import enqueue_hook, read_spool +from thirdeye.platforms.copilot.types import SourceBatch, SourcePaths, SourceRecord, SyncResult + +FIXTURES = Path(__file__).parent / "fixtures" +CLI_FIXTURE = FIXTURES +NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" +OBSERVED_AT = "2026-09-10T17:08:25.626Z" + + +def _load_json(path: Path) -> Any: + return json.loads(path.read_text(encoding="utf-8")) + + +def _record( + source_id: str, + *, + native_session_id: str = NATIVE_SESSION_ID, + source_kind: str = "transcript", +) -> SourceRecord: + return { + "source_id": source_id, + "source_kind": source_kind, + "native_session_id": native_session_id, + "ts": "2026-09-10T17:08:24.000Z", + "observed_at": "2026-09-10T17:08:25.000Z", + "payload": {"schema_version": 1, "type": "user.message"}, + "locator": {"file": "events.jsonl", "offset": 0}, + } + + +def _batch( + paths: SourcePaths, + records: list[SourceRecord], + *, + native_session_id: str = NATIVE_SESSION_ID, + next_cursor: dict[str, Any] | None = None, + base_cursor: dict[str, Any] | None = None, +) -> SourceBatch: + cursor: dict[str, Any] = dict(next_cursor or {"generation": 1}) + if base_cursor is not None: + cursor["base_cursor"] = base_cursor + return { + "source_key": paths["source_key"], + "native_session_id": native_session_id, + "cwd": "/proj", + "records": records, + "next_cursor": cursor, + "diagnostics": [], + } + + +def _write_transcript(home: Path, native_id: str, *, events_path: Path | None = None) -> None: + session_dir = home / "session-state" / native_id + session_dir.mkdir(parents=True, exist_ok=True) + source = events_path or (CLI_FIXTURE / "events.jsonl") + shutil.copy(source, session_dir / "events.jsonl") + (session_dir / "workspace.yaml").write_text("cwd: /sanitized/workspace\n", encoding="utf-8") + + +def _write_database(home: Path, *, session_id: str = NATIVE_SESSION_ID) -> None: + home.mkdir(parents=True, exist_ok=True) + database = home / "session-store.db" + connection = sqlite3.connect(database) + try: + connection.executescript( + """ + CREATE TABLE sessions (id TEXT PRIMARY KEY, cwd TEXT, created_at TEXT); + CREATE TABLE turns ( + id INTEGER PRIMARY KEY, + session_id TEXT NOT NULL, + turn_index INTEGER, + content TEXT, + updated_at TEXT + ); + CREATE TABLE assistant_usage_events ( + id INTEGER PRIMARY KEY, + session_id TEXT NOT NULL, + turn_index INTEGER, + agent_id TEXT, + parent_tool_call_id TEXT, + model TEXT, + input_tokens INTEGER, + output_tokens INTEGER, + cache_read_tokens INTEGER, + cache_write_tokens INTEGER, + reasoning_tokens INTEGER, + total_nano_aiu INTEGER, + request_multiplier REAL, + duration_ms INTEGER, + time_to_first_token_ms REAL, + output_ttft_ms REAL, + inter_token_latency_ms REAL, + initiator TEXT, + api_endpoint TEXT, + reasoning_effort TEXT, + finish_reason TEXT, + content_filter_triggered INTEGER, + token_details_json TEXT, + created_at TEXT + ); + """ + ) + connection.execute( + "INSERT INTO sessions (id, cwd, created_at) VALUES (?, ?, ?)", + (session_id, "/tmp/probe", "2026-09-10T17:08:00.000Z"), + ) + connection.execute( + "INSERT INTO turns (id, session_id, turn_index, content, updated_at) " + "VALUES (?, ?, ?, ?, ?)", + (1, session_id, 0, "turn-0", "2026-09-10T17:08:10.000Z"), + ) + connection.execute( + "INSERT INTO turns (id, session_id, turn_index, content, updated_at) " + "VALUES (?, ?, ?, ?, ?)", + (2, session_id, 1, "turn-1", "2026-09-10T17:08:11.000Z"), + ) + for row in _load_json(CLI_FIXTURE / "assistant-usage-events.json"): + columns = ", ".join(row) + placeholders = ", ".join("?" for _ in row) + connection.execute( + f"INSERT INTO assistant_usage_events ({columns}) VALUES ({placeholders})", + tuple(row.values()), + ) + connection.commit() + finally: + connection.close() + + +@pytest.fixture +def copilot_env(tmp_path: Path) -> tuple[Config, SourcePaths]: + home = tmp_path / "copilot-home" + home.mkdir() + config = Config(root=tmp_path / "thirdeye") + paths = resolve_sources(home) + return config, paths + + +def _hook_record(*, observation_id: str, session_id: str = NATIVE_SESSION_ID) -> SourceRecord: + return parse_hook( + "agentStop", + { + "sessionId": session_id, + "timestamp": 1789060105626, + "cwd": "/fixture/workspace", + "stopReason": "end_turn", + }, + {"env": {"WB_PLAN": "p"}}, + observed_at=OBSERVED_AT, + observation_id=observation_id, + ) + + +@contextmanager +def _fault_at(point: str) -> Iterator[None]: + def injector(name: str) -> None: + if name == point: + raise RuntimeError(f"injected fault at {point}") + + archive_mod._fault_injector = injector + state_mod._fault_injector = injector + try: + yield + finally: + archive_mod._fault_injector = None + state_mod._fault_injector = None + + +def _empty_result() -> SyncResult: + return { + "sessions": 0, + "records_written": 0, + "duplicate_records": 0, + "pending": 0, + "errors": 0, + } + + +def _composed_batch( + paths: SourcePaths, + *, + diagnostics: list[dict[str, Any]] | None = None, + records: list[SourceRecord] | None = None, + transcript_exhausted: bool = True, + database_exhausted: bool = True, +) -> SourceBatch: + return { + "source_key": paths["source_key"], + "native_session_id": NATIVE_SESSION_ID, + "cwd": "/proj", + "records": records or [_record("composed/1")], + "next_cursor": { + "transcript": {"byte_offset": 1}, + "database": {"database_offset": 1}, + "base_cursor": {}, + "transcript_exhausted": transcript_exhausted, + "database_exhausted": database_exhausted, + }, + "diagnostics": diagnostics or [], + } + + +# --- module exports --- + + +def test_iter_captured_records_is_reexported_from_archive( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + stored = stored_session_id(paths, NATIVE_SESSION_ID) + commit_batch(config, paths, _batch(paths, [_record("key/a/exported")])) + captured = list(iter_captured_records(config, stored)) + assert len(captured) == 1 + assert captured[0]["source_id"] == "key/a/exported" + + +def test_capture_module_has_no_usage_store_or_export_imports() -> None: + source = Path(capture_mod.__file__).read_text(encoding="utf-8") + assert "UsageStore" not in source + assert "usage_store" not in source + assert "logfire" not in source + assert "otel" not in source.lower() + + +# --- sync semantics --- + + +def test_sync_empty_discovery_is_successful_noop(copilot_env: tuple[Config, SourcePaths]) -> None: + config, paths = copilot_env + assert sync(config, paths) == { + "sessions": 0, + "records_written": 0, + "duplicate_records": 0, + "pending": 0, + "errors": 0, + } + + +def test_sync_missing_selected_session_returns_error( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + assert sync(config, paths, session_id="missing-session-id") == { + "sessions": 0, + "records_written": 0, + "duplicate_records": 0, + "pending": 0, + "errors": 1, + } + + +def test_sync_discovers_hook_only_spool_session(copilot_env: tuple[Config, SourcePaths]) -> None: + config, paths = copilot_env + hook_only = "hook-only-session-00000001" + record = _hook_record(observation_id="obs-hook-only", session_id=hook_only) + enqueue_hook(config, paths, record) + + result = sync(config, paths, session_id=hook_only) + # Missing transcript/database sources surface diagnostics without blocking the hook. + assert result["records_written"] >= 1 + assert result["errors"] >= 1 + assert result["pending"] >= 1 + stored = stored_session_id(paths, hook_only) + captured = list(iter_captured_records(config, stored)) + assert any(item["source_kind"] == "hook" for item in captured) + assert read_spool(config, paths, hook_only) == [] + + +# --- capture_session integration --- + + +def test_capture_session_commits_fixture_transcript_and_database( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + home = Path(paths["home"]) + _write_transcript(home, NATIVE_SESSION_ID) + _write_database(home, session_id=NATIVE_SESSION_ID) + + result = capture_session(config, paths, NATIVE_SESSION_ID) + assert result["errors"] == 0 + assert result["records_written"] > 0 + + stored = stored_session_id(paths, NATIVE_SESSION_ID) + captured = list(iter_captured_records(config, stored)) + kinds = {record["source_kind"] for record in captured} + assert "transcript" in kinds + assert "database" in kinds + transcript_count = sum(1 for record in captured if record["source_kind"] == "transcript") + assert transcript_count == 76 + + +def test_capture_session_prepends_spool_records_before_source_records( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + hook = _hook_record(observation_id="obs-order") + enqueue_hook(config, paths, hook) + order: list[str] = [] + + original_commit = capture_mod.commit_batch + + def tracking_commit(cfg: Config, p: SourcePaths, batch: SourceBatch) -> SyncResult: + order.extend(record["source_kind"] for record in batch["records"]) + return original_commit(cfg, p, batch) + + monkeypatch.setattr(capture_mod, "commit_batch", tracking_commit) + capture_session(config, paths, NATIVE_SESSION_ID) + + assert order[0] == "hook" + + +def test_capture_session_closes_when_spool_drains_start_then_end( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + names = iter(["a" * 32, "b" * 32]) + + class _OrderedUUID: + def __init__(self, value: str) -> None: + self.hex = value + + monkeypatch.setattr( + "thirdeye.platforms.copilot.spool.uuid.uuid4", + lambda: _OrderedUUID(next(names)), + ) + start = parse_hook( + "sessionStart", + { + "sessionId": NATIVE_SESSION_ID, + "timestamp": 1789060102204, + "cwd": "/fixture/workspace", + "source": "new", + }, + {}, + observed_at="2026-09-10T17:08:22.000Z", + observation_id="spool-start", + ) + end = parse_hook( + "sessionEnd", + { + "sessionId": NATIVE_SESSION_ID, + "timestamp": 1789060147875, + "cwd": "/fixture/workspace", + "reason": "user_exit", + }, + {}, + observed_at="2026-09-10T17:08:47.000Z", + observation_id="spool-end", + ) + enqueue_hook(config, paths, start) + enqueue_hook(config, paths, end) + + capture_session(config, paths, NATIVE_SESSION_ID) + + stored = stored_session_id(paths, NATIVE_SESSION_ID) + meta = read_meta(meta_path(session_dir(config.root, PLATFORM_NAME, stored))) + assert meta is not None + assert meta.status == "closed" + assert meta.ended_at is not None + + +def test_capture_session_acks_spool_after_successful_commit( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + record = _hook_record(observation_id="obs-ack") + enqueue_hook(config, paths, record) + assert read_spool(config, paths, NATIVE_SESSION_ID) + + capture_session(config, paths, NATIVE_SESSION_ID) + assert read_spool(config, paths, NATIVE_SESSION_ID) == [] + + +def test_capture_session_does_not_ack_spool_when_commit_reports_errors( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + record = _hook_record(observation_id="obs-no-ack") + enqueue_hook(config, paths, record) + + def failing_commit(_cfg: Config, _paths: SourcePaths, _batch: SourceBatch) -> SyncResult: + return { + "sessions": 1, + "records_written": 0, + "duplicate_records": 0, + "pending": 1, + "errors": 1, + } + + monkeypatch.setattr(capture_mod, "commit_batch", failing_commit) + capture_session(config, paths, NATIVE_SESSION_ID) + assert len(read_spool(config, paths, NATIVE_SESSION_ID)) == 1 + + +def test_capture_session_retries_once_after_stale_cursor_race( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + home = Path(paths["home"]) + _write_transcript(home, NATIVE_SESSION_ID) + _write_database(home, session_id=NATIVE_SESSION_ID) + + seed = capture_session(config, paths, NATIVE_SESSION_ID) + assert seed["errors"] == 0 + assert load_cursor(config, paths, NATIVE_SESSION_ID) != {} + + commit_calls: list[int] = [] + original_commit = capture_mod.commit_batch + + def tracking_commit(cfg: Config, p: SourcePaths, batch: SourceBatch) -> SyncResult: + commit_calls.append(len(batch["records"])) + return original_commit(cfg, p, batch) + + monkeypatch.setattr(capture_mod, "commit_batch", tracking_commit) + + original_load = capture_mod.load_cursor + + def staged_load(cfg: Config, p: SourcePaths, native_id: str) -> dict[str, Any]: + if not commit_calls: + return {} + return original_load(cfg, p, native_id) + + monkeypatch.setattr(capture_mod, "load_cursor", staged_load) + result = capture_session(config, paths, NATIVE_SESSION_ID) + + assert len(commit_calls) == 2 + assert result["errors"] == 0 + + +def test_sync_repeat_is_idempotent(copilot_env: tuple[Config, SourcePaths]) -> None: + config, paths = copilot_env + home = Path(paths["home"]) + _write_transcript(home, NATIVE_SESSION_ID) + _write_database(home, session_id=NATIVE_SESSION_ID) + stored = stored_session_id(paths, NATIVE_SESSION_ID) + + first = sync(config, paths, session_id=NATIVE_SESSION_ID) + count_after_first = len(list(iter_captured_records(config, stored))) + second = sync(config, paths, session_id=NATIVE_SESSION_ID) + count_after_second = len(list(iter_captured_records(config, stored))) + + assert first["records_written"] > 0 + assert first["errors"] == 0 + assert second["records_written"] == 0 + assert second["errors"] == 0 + assert count_after_second == count_after_first + + +def test_late_database_rows_are_ingested_after_initial_transcript_only_sync( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + home = Path(paths["home"]) + _write_transcript(home, NATIVE_SESSION_ID) + + first = sync(config, paths, session_id=NATIVE_SESSION_ID) + assert first["errors"] == 1 + assert first["pending"] == 1 + stored = stored_session_id(paths, NATIVE_SESSION_ID) + before = list(iter_captured_records(config, stored)) + assert before + assert not any(record["source_kind"] == "database" for record in before) + + _write_database(home, session_id=NATIVE_SESSION_ID) + second = sync(config, paths, session_id=NATIVE_SESSION_ID) + assert second["errors"] == 0 + assert second["records_written"] > 0 + + after = list(iter_captured_records(config, stored)) + assert any(record["source_kind"] == "database" for record in after) + + +def test_one_unavailable_source_does_not_block_the_other( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + home = Path(paths["home"]) + _write_transcript(home, NATIVE_SESSION_ID) + + def fail_database(_paths: SourcePaths, _native: str, _cursor: dict[str, Any]) -> Any: + raise OSError("database locked") + + with patch("thirdeye.platforms.copilot.sources.read_database", side_effect=fail_database): + result = capture_session(config, paths, NATIVE_SESSION_ID) + + assert result["records_written"] > 0 + assert result["errors"] >= 1 + stored = stored_session_id(paths, NATIVE_SESSION_ID) + assert any( + record["source_kind"] == "transcript" for record in iter_captured_records(config, stored) + ) + + +# --- record_hook --- + + +def test_record_hook_enqueues_then_captures(copilot_env: tuple[Config, SourcePaths]) -> None: + config, paths = copilot_env + home = Path(paths["home"]) + _write_transcript(home, NATIVE_SESSION_ID) + _write_database(home, session_id=NATIVE_SESSION_ID) + + result = record_hook( + config, + paths, + "agentStop", + { + "sessionId": NATIVE_SESSION_ID, + "timestamp": 1789060105626, + "cwd": "/fixture/workspace", + "stopReason": "end_turn", + }, + {"env": {"WB_PLAN": "probe"}}, + ) + + assert result["errors"] == 0 + assert result["records_written"] >= 1 + assert read_spool(config, paths, NATIVE_SESSION_ID) == [] + stored = stored_session_id(paths, NATIVE_SESSION_ID) + kinds = {record["source_kind"] for record in iter_captured_records(config, stored)} + assert "hook" in kinds + assert "transcript" in kinds + + +def test_record_hook_uses_caller_context_not_importer_environment( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + captured_context: dict[str, Any] = {} + + def capture_parse( + event: str, + payload: dict, + context: dict, + *, + observed_at: str, + observation_id: str, + ) -> SourceRecord: + captured_context.update(context) + return parse_hook( + event, payload, context, observed_at=observed_at, observation_id=observation_id + ) + + monkeypatch.setattr(capture_mod, "parse_hook", capture_parse) + monkeypatch.setattr(capture_mod, "capture_session", lambda *_args, **_kwargs: _empty_result()) + + record_hook( + config, + paths, + "sessionStart", + {"sessionId": NATIVE_SESSION_ID, "timestamp": 1, "cwd": "/x"}, + {"env": {"CUSTOM": "from-caller"}}, + ) + assert captured_context == {"env": {"CUSTOM": "from-caller"}} + + +def test_sync_full_fixture_writes_expected_record_volume( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + home = Path(paths["home"]) + _write_transcript(home, NATIVE_SESSION_ID) + _write_database(home, session_id=NATIVE_SESSION_ID) + + result = sync(config, paths, session_id=NATIVE_SESSION_ID) + assert result["errors"] == 0 + assert result["records_written"] > 80 + + stored = stored_session_id(paths, NATIVE_SESSION_ID) + captured = list(iter_captured_records(config, stored)) + assert len(captured) == result["records_written"] + + +def test_capture_session_reports_pending_when_transcript_exceeds_reader_limit( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + home = Path(paths["home"]) + session_dir = home / "session-state" / NATIVE_SESSION_ID + session_dir.mkdir(parents=True, exist_ok=True) + events = "".join(f'{{"id":"evt-{index}"}}\n' for index in range(1001)) + (session_dir / "events.jsonl").write_text(events, encoding="utf-8") + (session_dir / "workspace.yaml").write_text("cwd: /sanitized/workspace\n", encoding="utf-8") + _write_database(home, session_id=NATIVE_SESSION_ID) + + result = capture_session(config, paths, NATIVE_SESSION_ID) + stored = stored_session_id(paths, NATIVE_SESSION_ID) + captured = list(iter_captured_records(config, stored)) + assert result["pending"] >= 1 + assert len(captured) < 1001 + 20 + assert result["records_written"] == len(captured) + + +def test_sync_does_not_chase_growing_database_source( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + reads = {"n": 0} + + def growing_database( + _paths: SourcePaths, _native: str, cursor: dict[str, Any], **_kwargs: Any + ) -> Any: + reads["n"] += 1 + if reads["n"] > 80: + raise AssertionError("sync chased a growing database source") + offset = 0 + if isinstance(cursor, dict) and isinstance(cursor.get("database_offset"), int): + offset = cursor["database_offset"] + return { + "records": [ + _record(f"db/{offset + index}", source_kind="database") for index in range(1000) + ], + "next_cursor": {"database_generation": "gen-1", "database_offset": offset + 1000}, + "diagnostics": [], + "cwd": None, + "exhausted": False, + } + + def empty_transcript( + _paths: SourcePaths, _native: str, _cursor: dict[str, Any], **_kwargs: Any + ) -> Any: + return { + "records": [], + "next_cursor": {"byte_offset": 0, "snapshot_end": 0, "file_generation": "g"}, + "diagnostics": [], + "cwd": None, + "exhausted": True, + } + + monkeypatch.setattr("thirdeye.platforms.copilot.sources.read_database", growing_database) + monkeypatch.setattr("thirdeye.platforms.copilot.sources.read_transcript", empty_transcript) + monkeypatch.setattr( + "thirdeye.platforms.copilot.sources.discover_database_sessions", + lambda _paths: [NATIVE_SESSION_ID], + ) + monkeypatch.setattr( + "thirdeye.platforms.copilot.sources.discover_transcripts", lambda _paths: [] + ) + + result = sync(config, paths, session_id=NATIVE_SESSION_ID) + stored = stored_session_id(paths, NATIVE_SESSION_ID) + captured = [ + record + for record in iter_captured_records(config, stored) + if record["source_kind"] == "database" + ] + + assert reads["n"] <= 80 + assert result["sessions"] == 1 + assert 1000 <= len(captured) <= 40_000 + assert len(captured) == len({record["source_id"] for record in captured}) + + +def test_sync_drains_paginated_database_snapshot( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + home = Path(paths["home"]) + _write_transcript(home, NATIVE_SESSION_ID) + _write_database(home, session_id=NATIVE_SESSION_ID) + database = home / "session-store.db" + connection = sqlite3.connect(database) + try: + connection.executemany( + "INSERT INTO turns (session_id, turn_index, content, updated_at) VALUES (?, ?, ?, ?)", + [ + (NATIVE_SESSION_ID, index, f"turn-{index}", "2026-09-10T17:08:10.000Z") + for index in range(2, 1002) + ], + ) + connection.commit() + baseline = connection.execute( + "SELECT COUNT(*) FROM sessions WHERE id = ? UNION ALL " + "SELECT COUNT(*) FROM turns WHERE session_id = ? UNION ALL " + "SELECT COUNT(*) FROM assistant_usage_events WHERE session_id = ?", + (NATIVE_SESSION_ID, NATIVE_SESSION_ID, NATIVE_SESSION_ID), + ).fetchall() + expected = sum(row[0] for row in baseline) + finally: + connection.close() + + result = sync(config, paths, session_id=NATIVE_SESSION_ID) + stored = stored_session_id(paths, NATIVE_SESSION_ID) + captured = list(iter_captured_records(config, stored)) + database_records = [record for record in captured if record["source_kind"] == "database"] + + assert result["errors"] == 0 + assert result["pending"] == 0 + assert len(database_records) == expected + + +def test_sync_drains_paginated_invocation_snapshot( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + home = Path(paths["home"]) + session_dir = home / "session-state" / NATIVE_SESSION_ID + session_dir.mkdir(parents=True, exist_ok=True) + events = "".join(f'{{"id":"evt-{index}"}}\n' for index in range(1001)) + (session_dir / "events.jsonl").write_text(events, encoding="utf-8") + (session_dir / "workspace.yaml").write_text("cwd: /sanitized/workspace\n", encoding="utf-8") + _write_database(home, session_id=NATIVE_SESSION_ID) + + result = sync(config, paths, session_id=NATIVE_SESSION_ID) + stored = stored_session_id(paths, NATIVE_SESSION_ID) + captured = list(iter_captured_records(config, stored)) + transcript_events = [record for record in captured if record["source_kind"] == "transcript"] + assert result["pending"] == 0 + assert result["errors"] == 0 + assert len(transcript_events) == 1001 + assert result["sessions"] == 1 + + +def test_capture_session_pending_follows_retry_batch_when_source_becomes_available( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + unavailable = _composed_batch( + paths, + diagnostics=[ + { + "code": "copilot_database_missing", + "message": "Copilot session database is not present", + } + ], + ) + available = _composed_batch(paths, diagnostics=[], records=[_record("composed/retry")]) + reads = [unavailable, available] + commits = 0 + + def fake_read(_paths: SourcePaths, _native: str, _cursor: dict[str, Any]) -> SourceBatch: + return reads.pop(0) + + def fake_commit(_cfg: Config, _paths: SourcePaths, _batch: SourceBatch) -> SyncResult: + nonlocal commits + commits += 1 + if commits == 1: + return { + "sessions": 1, + "records_written": 2, + "duplicate_records": 0, + "pending": 1, + "errors": 1, + } + return { + "sessions": 1, + "records_written": 1, + "duplicate_records": 0, + "pending": 0, + "errors": 0, + } + + def fake_load(_cfg: Config, _paths: SourcePaths, _native: str) -> dict[str, Any]: + if commits == 0: + return {} + return {"transcript": {"byte_offset": 8}} + + monkeypatch.setattr(capture_mod, "read_batch", fake_read) + monkeypatch.setattr(capture_mod, "commit_batch", fake_commit) + monkeypatch.setattr(capture_mod, "load_cursor", fake_load) + + result = capture_session(config, paths, NATIVE_SESSION_ID) + assert commits == 2 + assert result["pending"] == 0 + assert result["errors"] == 0 + assert result["records_written"] == 3 + + +def test_capture_session_pending_follows_retry_batch_when_source_becomes_unavailable( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + available = _composed_batch(paths, diagnostics=[]) + unavailable = _composed_batch( + paths, + diagnostics=[ + { + "code": "copilot_database_missing", + "message": "Copilot session database is not present", + } + ], + ) + reads = [available, unavailable] + commits = 0 + + def fake_read(_paths: SourcePaths, _native: str, _cursor: dict[str, Any]) -> SourceBatch: + return reads.pop(0) + + def fake_commit(_cfg: Config, _paths: SourcePaths, _batch: SourceBatch) -> SyncResult: + nonlocal commits + commits += 1 + if commits == 1: + return { + "sessions": 1, + "records_written": 0, + "duplicate_records": 0, + "pending": 1, + "errors": 1, + } + return { + "sessions": 1, + "records_written": 1, + "duplicate_records": 0, + "pending": 0, + "errors": 0, + } + + def fake_load(_cfg: Config, _paths: SourcePaths, _native: str) -> dict[str, Any]: + if commits == 0: + return {} + return {"transcript": {"byte_offset": 8}} + + monkeypatch.setattr(capture_mod, "read_batch", fake_read) + monkeypatch.setattr(capture_mod, "commit_batch", fake_commit) + monkeypatch.setattr(capture_mod, "load_cursor", fake_load) + + result = capture_session(config, paths, NATIVE_SESSION_ID) + assert commits == 2 + assert result["pending"] >= 1 + assert result["errors"] >= 1 + assert result["records_written"] == 1 + + +def test_capture_session_keeps_journal_recovery_counts_after_stale_retry( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + home = Path(paths["home"]) + _write_transcript(home, NATIVE_SESSION_ID) + _write_database(home, session_id=NATIVE_SESSION_ID) + + seed = commit_batch( + config, paths, _batch(paths, [_record("key/a/seed")], next_cursor={"generation": 1}) + ) + assert seed["errors"] == 0 + + with _fault_at("after_journal"): + with pytest.raises(RuntimeError, match="injected fault"): + commit_batch( + config, + paths, + _batch( + paths, + [_record("key/a/journal-only")], + next_cursor={"generation": 2}, + base_cursor={"generation": 1}, + ), + ) + + commit_calls: list[SyncResult] = [] + original_commit = capture_mod.commit_batch + + def tracking_commit(cfg: Config, p: SourcePaths, batch: SourceBatch) -> SyncResult: + result = original_commit(cfg, p, batch) + commit_calls.append(result) + return result + + monkeypatch.setattr(capture_mod, "commit_batch", tracking_commit) + + original_load = capture_mod.load_cursor + + def staged_load(cfg: Config, p: SourcePaths, native_id: str) -> dict[str, Any]: + if not commit_calls: + return {"generation": 1} + return original_load(cfg, p, native_id) + + monkeypatch.setattr(capture_mod, "load_cursor", staged_load) + result = capture_session(config, paths, NATIVE_SESSION_ID) + + assert len(commit_calls) == 2 + assert commit_calls[0]["records_written"] >= 1 + assert result["records_written"] == ( + commit_calls[0]["records_written"] + commit_calls[1]["records_written"] + ) + assert result["duplicate_records"] == ( + commit_calls[0]["duplicate_records"] + commit_calls[1]["duplicate_records"] + ) + captured = list(iter_captured_records(config, stored_session_id(paths, NATIVE_SESSION_ID))) + assert any(record["source_id"] == "key/a/journal-only" for record in captured) + + +def test_sync_reports_discovery_failure_instead_of_empty_noop( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + + def boom(_paths: SourcePaths) -> list[str]: + raise OSError("database unreadable") + + with patch("thirdeye.platforms.copilot.sources.discover_database_sessions", boom): + result = sync(config, paths) + + assert result["sessions"] == 0 + assert result["records_written"] == 0 + assert result["errors"] >= 1 + assert result["pending"] >= 1 + + +def test_sync_selected_unreadable_database_is_not_reported_as_missing( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + Path(paths["database"]).write_text("this is not a sqlite database", encoding="utf-8") + + result = sync(config, paths, session_id=NATIVE_SESSION_ID) + assert result != { + "sessions": 0, + "records_written": 0, + "duplicate_records": 0, + "pending": 0, + "errors": 1, + } + assert result["errors"] >= 1 + assert result["pending"] >= 1 + + +def test_sync_incompatible_database_only_is_not_silent_noop( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + Path(paths["database"]).write_text("this is not a sqlite database", encoding="utf-8") + + result = sync(config, paths) + assert result["errors"] >= 1 + assert result["pending"] >= 1 + assert result["sessions"] == 0 diff --git a/tests/platforms/copilot/test_capture_reads.py b/tests/platforms/copilot/test_capture_reads.py new file mode 100644 index 00000000..4e2a16a0 --- /dev/null +++ b/tests/platforms/copilot/test_capture_reads.py @@ -0,0 +1,211 @@ +"""Compatibility coverage for generic CLI reads of raw Copilot evidence.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from click.testing import CliRunner + +from thirdeye.cli import main +from thirdeye.config import Config +from thirdeye.platforms.copilot.archive import commit_batch +from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id +from thirdeye.platforms.copilot.types import SourceBatch, SourcePaths, SourceRecord +from thirdeye.store import Store +from thirdeye.turns import session_turns + +NATIVE_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" +SOURCE_TS = "2026-09-10T17:08:24.506Z" +OBSERVED_AT = "2026-09-10T17:08:25.000Z" +TOOL_PATH = "/fixture/workspace/alpha.txt" + +PROMPT_PAYLOAD = { + "type": "user.message", + "id": "prompt-1", + "timestamp": SOURCE_TS, + "data": {"content": "Read alpha.txt and beta.txt with separate view calls."}, +} +TOOL_PAYLOAD = { + "type": "tool.execution_start", + "id": "tool-1", + "timestamp": SOURCE_TS, + "data": {"toolName": "view", "arguments": {"path": TOOL_PATH}}, +} +HOOK_PAYLOAD = { + "event": "userPromptSubmitted", + "hook_payload": { + "sessionId": NATIVE_ID, + "prompt": "Read only alpha.txt as an explore child.", + "agentId": "child-agent-id", + }, +} + + +def _record(kind: str, source_id: str, payload: dict[str, Any]) -> SourceRecord: + return { + "source_id": source_id, + "source_kind": kind, + "native_session_id": NATIVE_ID, + "ts": SOURCE_TS, + "observed_at": OBSERVED_AT, + "payload": payload, + "locator": {"file": "events.jsonl", "file_generation": "fixture-gen", "byte_offset": 42}, + } + + +def _assert_versioned_envelope( + event: dict[str, Any], *, source_kind: str, payload: dict[str, Any] +) -> None: + data = event["data"] + assert data["schema_version"] == 1 + record = data["source_record"] + assert record["source_kind"] == source_kind + assert record["payload"] == payload + assert "schema_version" not in record["payload"] + + +def _capture_synthetic_batch(config: Config, paths: SourcePaths) -> str: + """Archive labeled synthetic SourceRecords for generic read compatibility.""" + records = [ + _record("transcript", f"{paths['source_key']}/{NATIVE_ID}/prompt-1", PROMPT_PAYLOAD), + _record("transcript", f"{paths['source_key']}/{NATIVE_ID}/tool-1", TOOL_PAYLOAD), + _record("hook", f"hook/{NATIVE_ID}/child-prompt-1", HOOK_PAYLOAD), + ] + batch: SourceBatch = { + "source_key": paths["source_key"], + "native_session_id": NATIVE_ID, + "cwd": "/fixture/workspace", + "records": records, + "next_cursor": {"fixture": 1}, + "diagnostics": [], + } + commit_batch(config, paths, batch) + return stored_session_id(paths, NATIVE_ID) + + +def test_store_lists_and_retains_raw_source_identity(tmp_path: Path) -> None: + config = Config(root=tmp_path / "thirdeye") + paths = resolve_sources(tmp_path / "copilot-home") + stored_id = _capture_synthetic_batch(config, paths) + + sessions = list(Store(config).list_sessions(platform="copilot")) + assert [(session.session_id, session.cwd) for session in sessions] == [ + (stored_id, "/fixture/workspace") + ] + assert sessions[0].extra["copilot"]["native_session_id"] == NATIVE_ID + + events = list(Store(config).reader(stored_id).iter_events()) + assert [event["t"] for event in events] == [ + "copilot_transcript", + "copilot_transcript", + "copilot_hook", + ] + assert all(event["t"] not in {"user_message", "tool_call"} for event in events) + assert events[0]["ts"] == SOURCE_TS + _assert_versioned_envelope(events[0], source_kind="transcript", payload=PROMPT_PAYLOAD) + _assert_versioned_envelope(events[1], source_kind="transcript", payload=TOOL_PAYLOAD) + source_record = events[1]["data"]["source_record"] + assert source_record["source_id"] == f"{paths['source_key']}/{NATIVE_ID}/tool-1" + assert source_record["native_session_id"] == NATIVE_ID + assert source_record["locator"]["byte_offset"] == 42 + _assert_versioned_envelope(events[2], source_kind="hook", payload=HOOK_PAYLOAD) + assert ( + events[2]["data"]["source_record"]["payload"]["hook_payload"]["agentId"] == "child-agent-id" + ) + + +def test_generic_cli_reads_search_raw_copilot_content(tmp_path: Path) -> None: + config = Config(root=tmp_path / "thirdeye") + paths = resolve_sources(tmp_path / "copilot-home") + stored_id = _capture_synthetic_batch(config, paths) + runner = CliRunner() + env = {"THIRDEYE_HOME": str(config.root)} + + listed = runner.invoke(main, ["list", "--platform", "copilot"], env=env) + shown = runner.invoke(main, ["show", stored_id, "--json"], env=env) + events = runner.invoke(main, ["events", stored_id, "--json", "--no-findings"], env=env) + prompt_search = runner.invoke( + main, ["search", "separate view calls", "--platform", "copilot"], env=env + ) + tool_search = runner.invoke(main, ["search", TOOL_PATH, "--platform", "copilot"], env=env) + tailed = runner.invoke(main, ["tail", stored_id, "-n", "1", "--json"], env=env) + + assert listed.exit_code == shown.exit_code == events.exit_code == prompt_search.exit_code == 0 + assert tool_search.exit_code == tailed.exit_code == 0 + assert stored_id in listed.output + assert NATIVE_ID in listed.output + assert '"t":"copilot_transcript"' in shown.output + assert '"type":"tool.execution_start"' in events.output + assert '"schema_version":1' in events.output + assert '"source_kind":"transcript"' in events.output + assert "separate view calls" in prompt_search.output + assert TOOL_PATH in tool_search.output + assert '"t":"copilot_hook"' in tailed.output + + +def test_all_source_kinds_map_to_raw_event_types_without_projection(tmp_path: Path) -> None: + config = Config(root=tmp_path / "thirdeye") + paths = resolve_sources(tmp_path / "copilot-home") + transcript_payload = {"type": "user.message"} + database_payload = {"table": "assistant_usage_events", "tokens": 42} + hook_payload = {"event": "sessionStart", "hook_payload": {}} + metadata_payload = {"file": "workspace.yaml", "cwd": "/fixture/workspace"} + records = [ + _record("transcript", f"{paths['source_key']}/{NATIVE_ID}/tx", transcript_payload), + _record("database", f"{paths['source_key']}/{NATIVE_ID}/db", database_payload), + _record("hook", f"hook/{NATIVE_ID}/hk", hook_payload), + _record("metadata", f"{paths['source_key']}/{NATIVE_ID}/meta", metadata_payload), + ] + batch: SourceBatch = { + "source_key": paths["source_key"], + "native_session_id": NATIVE_ID, + "cwd": "/fixture/workspace", + "records": records, + "next_cursor": {"fixture": 1}, + "diagnostics": [], + } + commit_batch(config, paths, batch) + stored_id = stored_session_id(paths, NATIVE_ID) + + events = list(Store(config).reader(stored_id).iter_events()) + assert [event["t"] for event in events] == [ + "copilot_transcript", + "copilot_database", + "copilot_hook", + "copilot_metadata", + ] + _assert_versioned_envelope(events[0], source_kind="transcript", payload=transcript_payload) + _assert_versioned_envelope(events[1], source_kind="database", payload=database_payload) + _assert_versioned_envelope(events[2], source_kind="hook", payload=hook_payload) + _assert_versioned_envelope(events[3], source_kind="metadata", payload=metadata_payload) + assert all( + event["t"] not in {"user_message", "tool_call", "assistant_message"} for event in events + ) + + +def test_copilot_sessions_are_not_sliced_into_eval_turns(tmp_path: Path) -> None: + config = Config(root=tmp_path / "thirdeye") + paths = resolve_sources(tmp_path / "copilot-home") + stored_id = _capture_synthetic_batch(config, paths) + store = Store(config) + meta = store.get_meta(stored_id) + + assert session_turns(meta, store) == [] + + +def test_hook_prompt_content_is_searchable(tmp_path: Path) -> None: + config = Config(root=tmp_path / "thirdeye") + paths = resolve_sources(tmp_path / "copilot-home") + _capture_synthetic_batch(config, paths) + runner = CliRunner() + env = {"THIRDEYE_HOME": str(config.root)} + + hook_search = runner.invoke( + main, + ["search", "explore child", "--platform", "copilot"], + env=env, + ) + + assert hook_search.exit_code == 0 + assert "explore child" in hook_search.output diff --git a/tests/platforms/copilot/test_command.py b/tests/platforms/copilot/test_command.py new file mode 100644 index 00000000..078bbb26 --- /dev/null +++ b/tests/platforms/copilot/test_command.py @@ -0,0 +1,1246 @@ +"""CLI tests for thirdeye copilot sync/watch/status and package registration.""" + +from __future__ import annotations + +import json +import shutil +import sqlite3 +from pathlib import Path +from typing import Any + +import pytest +from click.testing import CliRunner + +from thirdeye.cli import main +from thirdeye.config import Config +from thirdeye.platforms.copilot.capture import iter_captured_records +from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id +from thirdeye.platforms.copilot.types import SourcePaths, SyncResult + +FIXTURES = Path(__file__).parent / "fixtures" +CLI_FIXTURE = FIXTURES +NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" +HOOK_BIN = "thirdeye-copilot-hook" +SECRET_PROMPT = "SECRET PROMPT BODY" + + +@pytest.fixture +def isolated_home(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + home = tmp_path / "thirdeye" + copilot_home = tmp_path / "copilot-default" + copilot_home.mkdir() + monkeypatch.setenv("THIRDEYE_HOME", str(home)) + monkeypatch.setenv("COPILOT_HOME", str(copilot_home)) + return home + + +def _counts_line(**overrides: int) -> str: + values = { + "sessions": 0, + "records_written": 0, + "duplicate_records": 0, + "pending": 0, + "errors": 0, + **overrides, + } + return ( + f"sessions={values['sessions']} " + f"records_written={values['records_written']} " + f"duplicate_records={values['duplicate_records']} " + f"pending={values['pending']} " + f"errors={values['errors']}" + ) + + +def _empty_result(**overrides: int) -> SyncResult: + result: SyncResult = { + "sessions": 0, + "records_written": 0, + "duplicate_records": 0, + "pending": 0, + "errors": 0, + } + result.update(overrides) # type: ignore[typeddict-item] + return result + + +def _write_transcript(home: Path, native_id: str) -> None: + session_dir = home / "session-state" / native_id + session_dir.mkdir(parents=True, exist_ok=True) + shutil.copy(CLI_FIXTURE / "events.jsonl", session_dir / "events.jsonl") + (session_dir / "workspace.yaml").write_text("cwd: /sanitized/workspace\n", encoding="utf-8") + + +def _write_database(home: Path, *, session_id: str = NATIVE_SESSION_ID) -> None: + home.mkdir(parents=True, exist_ok=True) + database = home / "session-store.db" + connection = sqlite3.connect(database) + try: + connection.executescript( + """ + CREATE TABLE sessions (id TEXT PRIMARY KEY, cwd TEXT, created_at TEXT); + CREATE TABLE turns ( + id INTEGER PRIMARY KEY, + session_id TEXT NOT NULL, + turn_index INTEGER, + content TEXT, + updated_at TEXT + ); + CREATE TABLE assistant_usage_events ( + id INTEGER PRIMARY KEY, + session_id TEXT NOT NULL, + turn_index INTEGER, + agent_id TEXT, + parent_tool_call_id TEXT, + model TEXT, + input_tokens INTEGER, + output_tokens INTEGER, + cache_read_tokens INTEGER, + cache_write_tokens INTEGER, + reasoning_tokens INTEGER, + total_nano_aiu INTEGER, + request_multiplier REAL, + duration_ms INTEGER, + time_to_first_token_ms REAL, + output_ttft_ms REAL, + inter_token_latency_ms REAL, + initiator TEXT, + api_endpoint TEXT, + reasoning_effort TEXT, + finish_reason TEXT, + content_filter_triggered INTEGER, + token_details_json TEXT, + created_at TEXT + ); + """ + ) + connection.execute( + "INSERT INTO sessions (id, cwd, created_at) VALUES (?, ?, ?)", + (session_id, "/tmp/probe", "2026-09-10T17:08:00.000Z"), + ) + connection.execute( + "INSERT INTO turns (id, session_id, turn_index, content, updated_at) " + "VALUES (?, ?, ?, ?, ?)", + (1, session_id, 0, "turn-0", "2026-09-10T17:08:10.000Z"), + ) + connection.execute( + "INSERT INTO turns (id, session_id, turn_index, content, updated_at) " + "VALUES (?, ?, ?, ?, ?)", + (2, session_id, 1, "turn-1", "2026-09-10T17:08:11.000Z"), + ) + usage_rows = CLI_FIXTURE / "assistant-usage-events.json" + if usage_rows.is_file(): + for row in json.loads(usage_rows.read_text(encoding="utf-8")): + columns = ", ".join(row) + placeholders = ", ".join("?" for _ in row) + connection.execute( + f"INSERT INTO assistant_usage_events ({columns}) VALUES ({placeholders})", + tuple(row.values()), + ) + connection.commit() + finally: + connection.close() + + +def _prompt_hook_record() -> dict[str, Any]: + return { + "source_id": f"hook/{NATIVE_SESSION_ID}/obs-1", + "source_kind": "hook", + "native_session_id": NATIVE_SESSION_ID, + "ts": "2026-09-10T17:08:01.000Z", + "observed_at": "2026-09-10T17:08:02.000Z", + "payload": { + "schema_version": 1, + "event": "userPromptSubmitted", + "hook_payload": { + "sessionId": NATIVE_SESSION_ID, + "prompt": SECRET_PROMPT, + }, + "context": {}, + }, + "locator": {"observation_id": "obs-1", "event": "userPromptSubmitted"}, + } + + +def _minimal_status( + *, configured: bool = False, errors: list[dict[str, Any]] | None = None +) -> dict: + return { + "paths": { + "home": "/tmp/copilot", + "source_key": "abc123", + "session_root": "/tmp/copilot/session-state", + "database": "/tmp/copilot/session-store.db", + }, + "installation": { + "configured": configured, + "hooks_file": "/tmp/copilot/hooks/thirdeye.json", + }, + "capabilities": { + "transcripts": { + "exists": True, + "readable": True, + "path": "/tmp/copilot/session-state", + }, + "database": { + "exists": True, + "readable": True, + "path": "/tmp/copilot/session-store.db", + }, + "database_wal": { + "exists": False, + "readable": False, + "path": "/tmp/copilot/session-store.db-wal", + }, + }, + "last_observed_hook": None, + "last_successful_import": None, + "sessions": [], + "pending": { + "spool_records": 0, + "spool_sessions": [], + "followup": 0, + "leases": 0, + "journals": 0, + }, + "errors": errors or [], + } + + +def _escaping_source_home(tmp_path: Path) -> Path: + home = tmp_path / "escaped-copilot" + home.mkdir() + outside = tmp_path / "outside-session-state" + outside.mkdir() + (home / "session-state").symlink_to(outside) + return home + + +# -- command registration ------------------------------------------------------ + + +def test_copilot_group_appears_in_main_help() -> None: + result = CliRunner().invoke(main, ["--help"]) + assert result.exit_code == 0 + assert "copilot" in result.output + + +def test_copilot_help_lists_subcommands() -> None: + result = CliRunner().invoke(main, ["copilot", "--help"]) + assert result.exit_code == 0, result.output + for subcommand in ("sync", "reconcile", "watch", "status"): + assert subcommand in result.output + + +def test_sync_help_documents_flags() -> None: + result = CliRunner().invoke(main, ["copilot", "sync", "--help"]) + assert result.exit_code == 0, result.output + assert "--session-id" in result.output + assert "--source-home" in result.output + assert "--export" in result.output + + +def test_reconcile_help_documents_flags() -> None: + result = CliRunner().invoke(main, ["copilot", "reconcile", "--help"]) + assert result.exit_code == 0, result.output + assert "--session-id" in result.output + assert "--rebuild" in result.output + assert "--export" in result.output + + +def test_watch_help_documents_interval_and_source_home() -> None: + result = CliRunner().invoke(main, ["copilot", "watch", "--help"]) + assert result.exit_code == 0, result.output + assert "--interval" in result.output + assert "--source-home" in result.output + + +def test_status_help_documents_source_home() -> None: + result = CliRunner().invoke(main, ["copilot", "status", "--help"]) + assert result.exit_code == 0, result.output + assert "--source-home" in result.output + + +def test_watch_and_status_help_have_no_export_flag() -> None: + runner = CliRunner() + for args in (["copilot", "watch", "--help"], ["copilot", "status", "--help"]): + result = runner.invoke(main, args) + assert result.exit_code == 0, result.output + assert "--export" not in result.output + + +def test_pyproject_registers_copilot_hook_entrypoint() -> None: + pyproject = Path(__file__).resolve().parents[3] / "pyproject.toml" + text = pyproject.read_text(encoding="utf-8") + assert f"{HOOK_BIN} =" in text + assert "thirdeye.platforms.copilot.hooks:main" in text + + +def test_copilot_hook_entrypoint_is_importable() -> None: + from importlib.metadata import entry_points + + scripts = entry_points(group="console_scripts") + hook = next((ep for ep in scripts if ep.name == HOOK_BIN), None) + assert hook is not None + module_path, _, attr = hook.value.partition(":") + module = __import__(module_path, fromlist=[attr]) + assert callable(getattr(module, attr)) + + +# -- sync ---------------------------------------------------------------------- + + +def test_sync_invokes_capture_sync(isolated_home: Path, monkeypatch: pytest.MonkeyPatch) -> None: + calls: list[tuple[Any, ...]] = [] + reconcile_calls: list[tuple[bool, bool]] = [] + + def fake_sync( + config: Config, paths: SourcePaths, *, session_id: str | None = None + ) -> SyncResult: + calls.append((config.root, paths["home"], session_id)) + return _empty_result(sessions=2, records_written=5) + + def fake_reconcile_archived( + _config: Config, + _paths: SourcePaths, + *, + export: bool = False, + include_history: bool = False, + ) -> dict[str, dict[str, int]]: + reconcile_calls.append((export, include_history)) + return {} + + monkeypatch.setattr("thirdeye.commands.copilot.capture_sync", fake_sync) + monkeypatch.setattr( + "thirdeye.commands.copilot.reconcile_archived_sessions", + fake_reconcile_archived, + ) + result = CliRunner().invoke(main, ["copilot", "sync"]) + assert result.exit_code == 0, result.output + assert len(calls) == 1 + assert calls[0][2] is None + assert reconcile_calls == [(False, False)] + assert _counts_line(sessions=2, records_written=5) in result.output + + +def test_sync_passes_session_id(isolated_home: Path, monkeypatch: pytest.MonkeyPatch) -> None: + captured: dict[str, str | None] = {"session_id": "unset"} + + def fake_sync( + _config: Config, _paths: SourcePaths, *, session_id: str | None = None + ) -> SyncResult: + captured["session_id"] = session_id + return _empty_result(sessions=1) + + monkeypatch.setattr("thirdeye.commands.copilot.capture_sync", fake_sync) + monkeypatch.setattr( + "thirdeye.commands.copilot.reconcile_session", + lambda *_args, **_kwargs: { + "events": 0, + "usage": 0, + "turns": 0, + "exports": 0, + "pending": 0, + "ambiguous": 0, + "conflicting": 0, + "errors": 0, + }, + ) + result = CliRunner().invoke(main, ["copilot", "sync", "--session-id", NATIVE_SESSION_ID]) + assert result.exit_code == 0, result.output + assert captured["session_id"] == NATIVE_SESSION_ID + + +def test_sync_resolves_explicit_source_home( + isolated_home: Path, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + source_home = tmp_path / "custom copilot home" + source_home.mkdir() + resolved: dict[str, str] = {} + + def fake_resolve(source_home_arg: Path | None = None) -> SourcePaths: + paths = resolve_sources(source_home_arg) + resolved["home"] = paths["home"] + return paths + + def fake_sync( + _config: Config, paths: SourcePaths, *, session_id: str | None = None + ) -> SyncResult: + resolved["sync_home"] = paths["home"] + return _empty_result() + + monkeypatch.setattr("thirdeye.commands.copilot.resolve_sources", fake_resolve) + monkeypatch.setattr("thirdeye.commands.copilot.capture_sync", fake_sync) + result = CliRunner().invoke(main, ["copilot", "sync", "--source-home", str(source_home)]) + assert result.exit_code == 0, result.output + assert Path(resolved["home"]) == source_home.resolve() + assert Path(resolved["sync_home"]) == source_home.resolve() + + +def test_sync_uses_copilot_home_when_source_home_omitted( + isolated_home: Path, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + env_home = tmp_path / "from-env-home" + env_home.mkdir() + monkeypatch.setenv("COPILOT_HOME", str(env_home)) + seen: dict[str, str] = {} + + def fake_sync( + _config: Config, paths: SourcePaths, *, session_id: str | None = None + ) -> SyncResult: + seen["home"] = paths["home"] + return _empty_result() + + monkeypatch.setattr("thirdeye.commands.copilot.capture_sync", fake_sync) + result = CliRunner().invoke(main, ["copilot", "sync"]) + assert result.exit_code == 0, result.output + assert Path(seen["home"]) == env_home.resolve() + + +def test_sync_source_home_overrides_copilot_home( + isolated_home: Path, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + env_home = tmp_path / "from-env-home" + explicit = tmp_path / "explicit-home" + env_home.mkdir() + explicit.mkdir() + monkeypatch.setenv("COPILOT_HOME", str(env_home)) + seen: dict[str, str] = {} + + def fake_sync( + _config: Config, paths: SourcePaths, *, session_id: str | None = None + ) -> SyncResult: + seen["home"] = paths["home"] + return _empty_result() + + monkeypatch.setattr("thirdeye.commands.copilot.capture_sync", fake_sync) + result = CliRunner().invoke(main, ["copilot", "sync", "--source-home", str(explicit)]) + assert result.exit_code == 0, result.output + assert Path(seen["home"]) == explicit.resolve() + + +def test_sync_empty_discovery_exits_zero(isolated_home: Path, tmp_path: Path) -> None: + empty_home = tmp_path / "empty-copilot" + empty_home.mkdir() + result = CliRunner().invoke(main, ["copilot", "sync", "--source-home", str(empty_home)]) + assert result.exit_code == 0, result.output + assert _counts_line() in result.output + + +def test_sync_missing_session_id_exits_nonzero( + isolated_home: Path, + tmp_path: Path, +) -> None: + empty_home = tmp_path / "empty-copilot" + empty_home.mkdir() + result = CliRunner().invoke( + main, + ["copilot", "sync", "--source-home", str(empty_home), "--session-id", "missing-session-id"], + ) + assert result.exit_code != 0, result.output + assert "No such command" not in result.output + assert "missing-session-id" in result.output + assert "errors=1" in result.output + assert "was not found" in result.output + + +def test_sync_session_with_capture_errors_exits_nonzero( + isolated_home: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + def fake_sync( + _config: Config, + _paths: SourcePaths, + *, + session_id: str | None = None, + ) -> SyncResult: + return _empty_result(errors=1) + + monkeypatch.setattr("thirdeye.commands.copilot.capture_sync", fake_sync) + result = CliRunner().invoke( + main, + ["copilot", "sync", "--session-id", NATIVE_SESSION_ID], + ) + assert result.exit_code != 0, result.output + assert NATIVE_SESSION_ID in result.output + assert "errors=1" in result.output + assert "was not found" in result.output + + +def test_sync_imported_session_with_diagnostics_exits_zero( + isolated_home: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + def fake_sync( + _config: Config, + _paths: SourcePaths, + *, + session_id: str | None = None, + ) -> SyncResult: + return _empty_result(sessions=1, records_written=12, errors=1) + + monkeypatch.setattr("thirdeye.commands.copilot.capture_sync", fake_sync) + monkeypatch.setattr( + "thirdeye.commands.copilot.reconcile_session", + lambda *_args, **_kwargs: { + "events": 0, + "usage": 0, + "turns": 0, + "exports": 0, + "pending": 0, + "ambiguous": 0, + "conflicting": 0, + "errors": 0, + }, + ) + result = CliRunner().invoke( + main, + ["copilot", "sync", "--session-id", NATIVE_SESSION_ID], + ) + assert result.exit_code == 0, result.output + assert _counts_line(sessions=1, records_written=12, errors=1) in result.output + assert "imported with 1 source diagnostics" in result.output + assert "was not found" not in result.output + + +def test_sync_prints_source_diagnostics_without_content( + isolated_home: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + def fake_sync( + _config: Config, _paths: SourcePaths, *, session_id: str | None = None + ) -> SyncResult: + return _empty_result(sessions=1, errors=1, pending=1) + + def fake_status(_config: Config, _paths: SourcePaths) -> dict: + status = _minimal_status( + errors=[ + { + "code": "transcript_invalid_json", + "message": "complete transcript line is not JSON", + "session": NATIVE_SESSION_ID, + "locator": {"file": "events.jsonl", "offset": 12}, + "payload": {"prompt": SECRET_PROMPT}, + } + ] + ) + return status + + monkeypatch.setattr("thirdeye.commands.copilot.capture_sync", fake_sync) + monkeypatch.setattr("thirdeye.commands.copilot.capture_status", fake_status) + monkeypatch.setattr( + "thirdeye.commands.copilot.reconcile_session", + lambda *_args, **_kwargs: { + "events": 0, + "usage": 0, + "turns": 0, + "exports": 0, + "pending": 0, + "ambiguous": 0, + "conflicting": 0, + "errors": 0, + }, + ) + result = CliRunner().invoke(main, ["copilot", "sync", "--session-id", NATIVE_SESSION_ID]) + assert result.exit_code == 0, result.output + assert "transcript_invalid_json" in result.output + assert NATIVE_SESSION_ID in result.output + assert "events.jsonl" in result.output + assert SECRET_PROMPT not in result.output + assert "thirdeye copilot status" in result.output + assert "thirdeye copilot watch" in result.output + + +def test_sync_capture_value_error_becomes_click_exception( + isolated_home: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + def fake_sync( + _config: Config, + _paths: SourcePaths, + *, + session_id: str | None = None, + ) -> SyncResult: + raise ValueError("invalid native session routing") + + monkeypatch.setattr("thirdeye.commands.copilot.capture_sync", fake_sync) + result = CliRunner().invoke(main, ["copilot", "sync"]) + assert result.exit_code != 0, result.output + assert "invalid native session routing" in result.output + assert "Traceback" not in result.output + + +def test_sync_prints_counts(isolated_home: Path, monkeypatch: pytest.MonkeyPatch) -> None: + def fake_sync( + _config: Config, _paths: SourcePaths, *, session_id: str | None = None + ) -> SyncResult: + return _empty_result( + sessions=1, records_written=12, duplicate_records=3, pending=2, errors=0 + ) + + monkeypatch.setattr("thirdeye.commands.copilot.capture_sync", fake_sync) + result = CliRunner().invoke(main, ["copilot", "sync"]) + assert result.exit_code == 0, result.output + assert ( + _counts_line(sessions=1, records_written=12, duplicate_records=3, pending=2) + in result.output + ) + assert "thirdeye copilot watch" in result.output + + +def test_sync_fixture_session_end_to_end(isolated_home: Path, tmp_path: Path) -> None: + source_home = tmp_path / "copilot-home" + source_home.mkdir() + _write_transcript(source_home, NATIVE_SESSION_ID) + _write_database(source_home, session_id=NATIVE_SESSION_ID) + + result = CliRunner().invoke(main, ["copilot", "sync", "--source-home", str(source_home)]) + assert result.exit_code == 0, result.output + + paths = resolve_sources(source_home) + stored = stored_session_id(paths, NATIVE_SESSION_ID) + captured = list(iter_captured_records(Config.load(), stored)) + kinds = {record["source_kind"] for record in captured} + assert "transcript" in kinds + assert "database" in kinds + assert sum(1 for record in captured if record["source_kind"] == "transcript") == 76 + + +def test_sync_selected_id_transcript_only_exits_zero(isolated_home: Path, tmp_path: Path) -> None: + source_home = tmp_path / "transcript-only" + source_home.mkdir() + _write_transcript(source_home, NATIVE_SESSION_ID) + + result = CliRunner().invoke( + main, + [ + "copilot", + "sync", + "--source-home", + str(source_home), + "--session-id", + NATIVE_SESSION_ID, + ], + ) + assert result.exit_code == 0, result.output + assert "was not found" not in result.output + assert "imported with" in result.output + assert "copilot_database_missing" in result.output + assert SECRET_PROMPT not in result.output + + +def test_sync_selected_id_malformed_line_exits_zero(isolated_home: Path, tmp_path: Path) -> None: + source_home = tmp_path / "malformed-transcript" + source_home.mkdir() + _write_transcript(source_home, NATIVE_SESSION_ID) + _write_database(source_home, session_id=NATIVE_SESSION_ID) + events = source_home / "session-state" / NATIVE_SESSION_ID / "events.jsonl" + with events.open("a", encoding="utf-8") as handle: + handle.write("{not-json\n") + + result = CliRunner().invoke( + main, + [ + "copilot", + "sync", + "--source-home", + str(source_home), + "--session-id", + NATIVE_SESSION_ID, + ], + ) + assert result.exit_code == 0, result.output + assert "was not found" not in result.output + assert "imported with" in result.output + assert "transcript_invalid_json" in result.output + + +def test_sync_rejects_native_id_with_path_separators(isolated_home: Path) -> None: + result = CliRunner().invoke(main, ["copilot", "sync", "--session-id", "../escape"]) + assert result.exit_code != 0, result.output + assert "No such command" not in result.output + + +def test_sync_path_escape_is_click_error(isolated_home: Path, tmp_path: Path) -> None: + home = _escaping_source_home(tmp_path) + result = CliRunner().invoke(main, ["copilot", "sync", "--source-home", str(home)]) + assert result.exit_code != 0, result.output + assert "Traceback" not in result.output + assert "session_root" in result.output + assert "escapes" in result.output + assert str(home) in result.output + + +def test_sync_export_requests_history_reconciliation( + isolated_home: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + reconcile_calls: list[tuple[bool, bool]] = [] + + def fake_sync( + _config: Config, _paths: SourcePaths, *, session_id: str | None = None + ) -> SyncResult: + return _empty_result(sessions=1) + + def fake_reconcile_archived( + _config: Config, + _paths: SourcePaths, + *, + export: bool = False, + include_history: bool = False, + ) -> dict[str, dict[str, int]]: + reconcile_calls.append((export, include_history)) + return { + "stored-1": { + "events": 3, + "usage": 2, + "turns": 1, + "exports": 0, + "pending": 0, + "ambiguous": 0, + "conflicting": 0, + "errors": 0, + } + } + + monkeypatch.setattr("thirdeye.commands.copilot.capture_sync", fake_sync) + monkeypatch.setattr( + "thirdeye.commands.copilot.reconcile_archived_sessions", + fake_reconcile_archived, + ) + result = CliRunner().invoke(main, ["copilot", "sync", "--export"]) + assert result.exit_code == 0, result.output + assert reconcile_calls == [(True, True)] + assert "reconcile events=3 usage=2 turns=1" in result.output + + +def test_reconcile_command_invokes_reconcile_archive( + isolated_home: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[tuple[str, bool, bool, bool]] = [] + + def fake_reconcile_archive( + _config: Config, + stored_session_id: str, + *, + rebuild: bool = False, + export: bool = False, + include_history: bool = False, + ) -> dict[str, int]: + calls.append((stored_session_id, rebuild, export, include_history)) + return { + "events": 4, + "usage": 6, + "turns": 2, + "exports": 0, + "pending": 1, + "ambiguous": 0, + "conflicting": 0, + "errors": 0, + } + + monkeypatch.setattr( + "thirdeye.commands.copilot.reconcile_stored_session", fake_reconcile_archive + ) + result = CliRunner().invoke( + main, + ["copilot", "reconcile", "--session-id", "copilot-abc-stored", "--rebuild", "--export"], + ) + assert result.exit_code == 0, result.output + assert calls == [("copilot-abc-stored", True, True, True)] + assert "reconcile events=4 usage=6 turns=2 exports_queued=0 pending=1" in result.output + + +def test_reconcile_command_exits_nonzero_on_errors( + isolated_home: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + "thirdeye.commands.copilot.reconcile_stored_session", + lambda *_args, **_kwargs: { + "events": 0, + "usage": 0, + "turns": 0, + "exports": 0, + "pending": 0, + "ambiguous": 0, + "conflicting": 0, + "errors": 1, + }, + ) + result = CliRunner().invoke( + main, + ["copilot", "reconcile", "--session-id", "copilot-abc-stored"], + ) + assert result.exit_code != 0, result.output + assert "completed with errors" in result.output + + +def test_status_prints_projection_and_export_summaries( + isolated_home: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + status = _minimal_status(configured=True) + status["sessions"] = [ + { + "stored_session_id": "stored-1", + "projection": {"pending": 2, "ambiguous": 1, "conflicting": 0}, + "export": {"activated": True, "queued": 3, "delivered": 1, "errors": 0}, + }, + { + "stored_session_id": "stored-2", + "projection": {"pending": 0, "ambiguous": 0, "conflicting": 1}, + "export": {"activated": False, "queued": 0, "delivered": 0, "errors": 2}, + }, + ] + monkeypatch.setattr("thirdeye.commands.copilot.capture_status", lambda _config, _paths: status) + result = CliRunner().invoke(main, ["copilot", "status"]) + assert result.exit_code == 0, result.output + assert "Projection: pending=2 ambiguous=1 conflicting=1" in result.output + assert "Export: activated=1 queued=3 delivered=1 errors=2" in result.output + + +def test_sync_session_id_reconciles_only_selected_session( + isolated_home: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + archived_calls: list[tuple[bool, bool]] = [] + session_calls: list[tuple[str, bool, bool]] = [] + + def fake_sync( + _config: Config, _paths: SourcePaths, *, session_id: str | None = None + ) -> SyncResult: + return _empty_result(sessions=1) + + def fake_archived( + _config: Config, + _paths: SourcePaths, + *, + export: bool = False, + include_history: bool = False, + ) -> dict[str, dict[str, int]]: + archived_calls.append((export, include_history)) + return { + "other-stored": { + "events": 9, + "usage": 0, + "turns": 0, + "exports": 1, + "pending": 0, + "ambiguous": 0, + "conflicting": 0, + "errors": 0, + } + } + + def fake_session( + _config: Config, + _paths: SourcePaths, + native_session_id: str, + *, + export: bool = False, + include_history: bool = False, + ) -> dict[str, int]: + session_calls.append((native_session_id, export, include_history)) + return { + "events": 1, + "usage": 0, + "turns": 1, + "exports": 0, + "pending": 0, + "ambiguous": 0, + "conflicting": 0, + "errors": 0, + } + + monkeypatch.setattr("thirdeye.commands.copilot.capture_sync", fake_sync) + monkeypatch.setattr("thirdeye.commands.copilot.reconcile_archived_sessions", fake_archived) + monkeypatch.setattr("thirdeye.commands.copilot.reconcile_session", fake_session) + result = CliRunner().invoke( + main, ["copilot", "sync", "--session-id", NATIVE_SESSION_ID, "--export"] + ) + assert result.exit_code == 0, result.output + assert archived_calls == [] + assert session_calls == [(NATIVE_SESSION_ID, True, True)] + assert "other-stored" not in result.output + assert "reconcile events=1" in result.output + + +def test_sync_missing_session_does_not_reconcile_unrelated_history( + isolated_home: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + archived_calls: list[object] = [] + session_calls: list[object] = [] + + def fake_sync( + _config: Config, _paths: SourcePaths, *, session_id: str | None = None + ) -> SyncResult: + return _empty_result(sessions=0, errors=1) + + monkeypatch.setattr("thirdeye.commands.copilot.capture_sync", fake_sync) + monkeypatch.setattr( + "thirdeye.commands.copilot.reconcile_archived_sessions", + lambda *_a, **_k: archived_calls.append(True) or {}, + ) + monkeypatch.setattr( + "thirdeye.commands.copilot.reconcile_session", + lambda *_a, **_k: session_calls.append(True) or {}, + ) + result = CliRunner().invoke( + main, ["copilot", "sync", "--session-id", NATIVE_SESSION_ID, "--export"] + ) + assert result.exit_code != 0, result.output + assert "was not found" in result.output + assert archived_calls == [] + assert session_calls == [] + + +def test_reconcile_without_session_id_reconciles_all_archives( + isolated_home: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[str] = [] + + def fake_reconcile( + _config: Config, + stored_session_id: str, + *, + rebuild: bool = False, + export: bool = False, + include_history: bool = False, + ) -> dict[str, int]: + calls.append(stored_session_id) + return { + "events": 1, + "usage": 0, + "turns": 1, + "exports": 0, + "pending": 0, + "ambiguous": 0, + "conflicting": 0, + "errors": 0, + } + + monkeypatch.setattr( + "thirdeye.commands.copilot.all_archived_session_ids", + lambda _config: ["copilot-aaa-one", "copilot-bbb-two"], + ) + monkeypatch.setattr("thirdeye.commands.copilot.reconcile_stored_session", fake_reconcile) + result = CliRunner().invoke(main, ["copilot", "reconcile", "--rebuild"]) + assert result.exit_code == 0, result.output + assert calls == ["copilot-aaa-one", "copilot-bbb-two"] + assert result.output.count("reconcile events=1") == 2 + + +def test_reconcile_rejects_path_escape_session_id(isolated_home: Path) -> None: + result = CliRunner().invoke(main, ["copilot", "reconcile", "--session-id", "../secret"]) + assert result.exit_code != 0, result.output + assert "Traceback" not in result.output + assert "path separator" in result.output or "stored session ID" in result.output + + +# -- watch --------------------------------------------------------------------- + + +def test_watch_rejects_interval_below_minimum(isolated_home: Path) -> None: + result = CliRunner().invoke(main, ["copilot", "watch", "--interval", "0.05"]) + assert result.exit_code != 0, result.output + assert "No such command" not in result.output + assert "0.1" in result.output or "interval" in result.output.lower() + + +def test_watch_rejects_non_finite_interval(isolated_home: Path) -> None: + result = CliRunner().invoke(main, ["copilot", "watch", "--interval", "inf"]) + assert result.exit_code != 0, result.output + assert "No such command" not in result.output + + +def test_watch_invokes_watch_module(isolated_home: Path, monkeypatch: pytest.MonkeyPatch) -> None: + calls: list[float] = [] + + def fake_watch(_config: Config, _paths: SourcePaths, *, interval: float = 1.0) -> None: + calls.append(interval) + + monkeypatch.setattr("thirdeye.commands.copilot.watch_loop", fake_watch) + result = CliRunner().invoke(main, ["copilot", "watch", "--interval", "2.5"]) + assert result.exit_code == 0, result.output + assert calls == [2.5] + + +def test_watch_prints_start_and_stop(isolated_home: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("thirdeye.commands.copilot.watch_loop", lambda *_args, **_kwargs: None) + result = CliRunner().invoke(main, ["copilot", "watch", "--interval", "1.5"]) + assert result.exit_code == 0, result.output + assert "1.5" in result.output + assert "queued locally" in result.output + assert "local-only" not in result.output + assert "Ctrl-C" in result.output + assert "Stopped" in result.output + + +def test_watch_keyboard_interrupt_prints_stop( + isolated_home: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + def fake_watch(_config: Config, _paths: SourcePaths, *, interval: float = 1.0) -> None: + raise KeyboardInterrupt + + monkeypatch.setattr("thirdeye.commands.copilot.watch_loop", fake_watch) + result = CliRunner().invoke(main, ["copilot", "watch"]) + assert result.exit_code == 0, result.output + assert "Stopped" in result.output + assert "Traceback" not in result.output + + +def test_watch_passes_source_home( + isolated_home: Path, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + source_home = tmp_path / "watch home" + source_home.mkdir() + seen: dict[str, str] = {} + + def fake_watch(_config: Config, paths: SourcePaths, *, interval: float = 1.0) -> None: + seen["home"] = paths["home"] + + monkeypatch.setattr("thirdeye.commands.copilot.watch_loop", fake_watch) + result = CliRunner().invoke( + main, + ["copilot", "watch", "--source-home", str(source_home), "--interval", "1"], + ) + assert result.exit_code == 0, result.output + assert Path(seen["home"]) == source_home.resolve() + assert str(source_home.resolve()) in result.output + + +def test_watch_path_escape_is_click_error(isolated_home: Path, tmp_path: Path) -> None: + home = _escaping_source_home(tmp_path) + result = CliRunner().invoke(main, ["copilot", "watch", "--source-home", str(home)]) + assert result.exit_code != 0, result.output + assert "Traceback" not in result.output + assert "session_root" in result.output + assert "escapes" in result.output + + +# -- status -------------------------------------------------------------------- + + +def test_status_invokes_capture_status( + isolated_home: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + called = {"count": 0} + + def fake_status(_config: Config, _paths: SourcePaths) -> dict: + called["count"] += 1 + return _minimal_status(configured=False) + + monkeypatch.setattr("thirdeye.commands.copilot.capture_status", fake_status) + result = CliRunner().invoke(main, ["copilot", "status"]) + assert result.exit_code == 0, result.output + assert called["count"] == 1 + + +def test_status_exits_zero_when_hooks_missing_but_sources_ok( + isolated_home: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + "thirdeye.commands.copilot.capture_status", + lambda _config, _paths: _minimal_status(configured=False, errors=[]), + ) + result = CliRunner().invoke(main, ["copilot", "status"]) + assert result.exit_code == 0, result.output + assert "Hooks: not configured" in result.output + + +def test_status_exits_nonzero_on_source_errors( + isolated_home: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + "thirdeye.commands.copilot.capture_status", + lambda _config, _paths: _minimal_status( + configured=True, + errors=[ + { + "kind": "source_unreadable", + "message": "database unreadable", + "session": NATIVE_SESSION_ID, + "path": "/tmp/copilot/session-store.db", + } + ], + ), + ) + result = CliRunner().invoke(main, ["copilot", "status"]) + assert result.exit_code != 0, result.output + assert "source_unreadable" in result.output + assert "database unreadable" in result.output + assert NATIVE_SESSION_ID in result.output + assert "/tmp/copilot/session-store.db" in result.output + + +def test_status_absence_codes_exit_zero( + isolated_home: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + "thirdeye.commands.copilot.capture_status", + lambda _config, _paths: _minimal_status( + errors=[ + { + "code": "copilot_database_missing", + "message": "database file is absent", + "session": NATIVE_SESSION_ID, + } + ] + ), + ) + result = CliRunner().invoke(main, ["copilot", "status"]) + assert result.exit_code == 0, result.output + assert "copilot_database_missing" in result.output + assert NATIVE_SESSION_ID in result.output + + +def test_status_retryable_codes_exit_zero( + isolated_home: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + "thirdeye.commands.copilot.capture_status", + lambda _config, _paths: _minimal_status( + errors=[ + { + "code": "copilot_database_busy", + "message": "database is busy", + "session": NATIVE_SESSION_ID, + } + ] + ), + ) + result = CliRunner().invoke(main, ["copilot", "status"]) + assert result.exit_code == 0, result.output + assert "copilot_database_busy" in result.output + + +def test_status_transcript_only_home_exits_zero(isolated_home: Path, tmp_path: Path) -> None: + source_home = tmp_path / "transcript-only-status" + source_home.mkdir() + _write_transcript(source_home, NATIVE_SESSION_ID) + sync = CliRunner().invoke(main, ["copilot", "sync", "--source-home", str(source_home)]) + assert sync.exit_code == 0, sync.output + result = CliRunner().invoke(main, ["copilot", "status", "--source-home", str(source_home)]) + assert result.exit_code == 0, result.output + assert "copilot_database_missing" in result.output + + +def test_status_does_not_print_hook_prompt( + isolated_home: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + status = _minimal_status(configured=False) + status["last_observed_hook"] = _prompt_hook_record() + monkeypatch.setattr("thirdeye.commands.copilot.capture_status", lambda _config, _paths: status) + result = CliRunner().invoke(main, ["copilot", "status"]) + assert result.exit_code == 0, result.output + assert SECRET_PROMPT not in result.output + assert "hook_payload" not in result.output + assert "userPromptSubmitted" in result.output + assert NATIVE_SESSION_ID in result.output + assert "2026-09-10T17:08:02.000Z" in result.output + + +def test_status_error_lines_omit_payload( + isolated_home: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + "thirdeye.commands.copilot.capture_status", + lambda _config, _paths: _minimal_status( + configured=True, + errors=[ + { + "kind": "source_unreadable", + "message": "PermissionError", + "session": "sess-1", + "path": "/secret/db", + "payload": {"prompt": SECRET_PROMPT}, + } + ], + ), + ) + result = CliRunner().invoke(main, ["copilot", "status"]) + assert result.exit_code != 0, result.output + assert "sess-1" in result.output + assert "/secret/db" in result.output + assert SECRET_PROMPT not in result.output + assert "hook_payload" not in result.output + + +def test_status_prints_string_errors( + isolated_home: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + "thirdeye.commands.copilot.capture_status", + lambda _config, _paths: _minimal_status(errors=["legacy string error"]), # type: ignore[list-item] + ) + result = CliRunner().invoke(main, ["copilot", "status"]) + assert result.exit_code != 0, result.output + assert "legacy string error" in result.output + + +def test_status_prints_paths_capabilities_and_guidance( + isolated_home: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + "thirdeye.commands.copilot.capture_status", + lambda _config, _paths: _minimal_status(configured=True), + ) + result = CliRunner().invoke(main, ["copilot", "status"]) + assert result.exit_code == 0, result.output + assert "Copilot home: /tmp/copilot" in result.output + assert "Hooks: configured" in result.output + assert "Restart Copilot CLI" not in result.output + assert "Transcripts:" in result.output + assert "Database:" in result.output + assert "Database WAL:" in result.output + assert "exists=True" in result.output + assert "readable=True" in result.output + assert "trusted folder" in result.output + + +def test_status_resolves_source_home_with_spaces( + isolated_home: Path, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + source_home = tmp_path / "copilot home with spaces" + source_home.mkdir() + seen: dict[str, str] = {} + + def fake_status(_config: Config, paths: SourcePaths) -> dict: + seen["home"] = paths["home"] + return _minimal_status() + + monkeypatch.setattr("thirdeye.commands.copilot.capture_status", fake_status) + result = CliRunner().invoke(main, ["copilot", "status", "--source-home", str(source_home)]) + assert result.exit_code == 0, result.output + assert Path(seen["home"]) == source_home.resolve() + + +def test_status_path_escape_is_click_error(isolated_home: Path, tmp_path: Path) -> None: + home = _escaping_source_home(tmp_path) + result = CliRunner().invoke(main, ["copilot", "status", "--source-home", str(home)]) + assert result.exit_code != 0, result.output + assert "Traceback" not in result.output + assert "session_root" in result.output + assert "escapes" in result.output diff --git a/tests/platforms/copilot/test_contracts.py b/tests/platforms/copilot/test_contracts.py new file mode 100644 index 00000000..45990c3c --- /dev/null +++ b/tests/platforms/copilot/test_contracts.py @@ -0,0 +1,509 @@ +"""Frozen contract tests for Copilot V1 capture types and identity.""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +from pathlib import Path +from typing import Any + +import pytest + +import thirdeye.platforms.copilot as copilot_pkg +from thirdeye.platforms.copilot.constants import ( + CLI_HOOK_EVENT_ALIASES, + CLI_HOOK_EVENTS, + COPILOT_HOME_ENV, + DISPLAY_NAME, + EVENT_ENVELOPE_VERSION, + HOOKS_DIRECTORY_NAME, + OWNED_HOOK_FILENAME, + PLATFORM_NAME, + SCHEMA_VERSION, + SOURCE_RECORD_SCHEMA_VERSION, + SOURCE_SCHEMA_VERSION, +) +from thirdeye.platforms.copilot.identity import ( + SOURCE_KEY_PREFIX_LEN, + resolve_sources, + source_keys_share_stored_prefix, + stored_session_id, + validate_native_id, +) +from thirdeye.platforms.copilot.types import ( + SCHEMA_VERSION as TYPES_SCHEMA_VERSION, +) +from thirdeye.platforms.copilot.types import ( + SourceBatch, + SourcePaths, + SourceRecord, + SourceSlice, + SyncResult, +) + +FIXTURES = Path(__file__).parent / "fixtures" +CLI_FIXTURE = FIXTURES +V1_CASES = FIXTURES / "cases" + +NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" +CHILD_AGENT_ID = "bf8cb9f3-2097-4db0-a3c8-78a2653b2106" + + +def _load_json(path: Path) -> Any: + return json.loads(path.read_text(encoding="utf-8")) + + +def _round_trip(value: dict[str, Any]) -> dict[str, Any]: + return json.loads(json.dumps(value)) + + +# --- constants --- + + +def test_platform_identity_constants(): + assert PLATFORM_NAME == "copilot" + assert DISPLAY_NAME == "GitHub Copilot CLI" + + +def test_schema_version_is_frozen_at_one(): + for version in ( + SOURCE_SCHEMA_VERSION, + SCHEMA_VERSION, + SOURCE_RECORD_SCHEMA_VERSION, + EVENT_ENVELOPE_VERSION, + TYPES_SCHEMA_VERSION, + ): + assert version == 1 + + +def test_constants_module_has_no_resolved_paths(): + import thirdeye.platforms.copilot.constants as constants + + for name in dir(constants): + if name.startswith("_"): + continue + value = getattr(constants, name) + if isinstance(value, str) and ("/" in value or "\\" in value): + pytest.fail(f"constants.{name} looks like a resolved path: {value!r}") + + +def test_hook_install_vocabulary(): + assert HOOKS_DIRECTORY_NAME == "hooks" + assert OWNED_HOOK_FILENAME == "thirdeye.json" + assert COPILOT_HOME_ENV == "COPILOT_HOME" + + +def test_cli_hook_aliases_cover_observed_events(): + expected = { + "sessionStart", + "userPromptSubmitted", + "preToolUse", + "postToolUse", + "agentStop", + "subagentStart", + "subagentStop", + "sessionEnd", + } + assert expected <= set(CLI_HOOK_EVENT_ALIASES) + assert CLI_HOOK_EVENTS == tuple(CLI_HOOK_EVENT_ALIASES) + + +def test_hook_aliases_map_to_snake_case(): + for camel, snake in CLI_HOOK_EVENT_ALIASES.items(): + assert camel[0].islower() + assert snake == re.sub(r"(? Any: + return json.loads(path.read_text(encoding="utf-8")) + + +def _create_standard_schema(connection: sqlite3.Connection) -> None: + connection.executescript( + """ + CREATE TABLE sessions ( + id TEXT PRIMARY KEY, + cwd TEXT, + created_at TEXT + ); + CREATE TABLE turns ( + id INTEGER PRIMARY KEY, + session_id TEXT NOT NULL, + turn_index INTEGER, + content TEXT, + updated_at TEXT + ); + CREATE TABLE assistant_usage_events ( + id INTEGER PRIMARY KEY, + session_id TEXT NOT NULL, + turn_index INTEGER, + agent_id TEXT, + parent_tool_call_id TEXT, + model TEXT, + input_tokens INTEGER, + output_tokens INTEGER, + cache_read_tokens INTEGER, + cache_write_tokens INTEGER, + reasoning_tokens INTEGER, + total_nano_aiu INTEGER, + request_multiplier REAL, + duration_ms INTEGER, + time_to_first_token_ms REAL, + output_ttft_ms REAL, + inter_token_latency_ms REAL, + initiator TEXT, + api_endpoint TEXT, + reasoning_effort TEXT, + finish_reason TEXT, + content_filter_triggered INTEGER, + token_details_json TEXT, + created_at TEXT + ); + """ + ) + + +def _seed_session( + connection: sqlite3.Connection, + session_id: str, + *, + cwd: str = "/tmp/workspace", +) -> None: + connection.execute( + "INSERT INTO sessions (id, cwd, created_at) VALUES (?, ?, ?)", + (session_id, cwd, "2026-09-10T17:08:00.000Z"), + ) + + +def _write_database( + home: Path, + *, + session_id: str = "session-a", + cwd: str = "/tmp/workspace", + turns: list[tuple[int, str]] | None = None, + usage_rows: list[dict[str, Any]] | None = None, + extra_sessions: list[str] | None = None, + schema_sql: str | None = None, + journal_mode: str = "WAL", + seed_session: bool = True, +) -> Path: + home.mkdir(parents=True, exist_ok=True) + database = home / "session-store.db" + connection = sqlite3.connect(database) + try: + if schema_sql is not None: + connection.executescript(schema_sql) + else: + _create_standard_schema(connection) + if journal_mode: + connection.execute(f"PRAGMA journal_mode={journal_mode}") + if seed_session: + _seed_session(connection, session_id, cwd=cwd) + for other in extra_sessions or (): + _seed_session(connection, other, cwd=f"/tmp/{other}") + for turn_id, content in turns or (): + connection.execute( + "INSERT INTO turns (id, session_id, turn_index, content, updated_at) " + "VALUES (?, ?, ?, ?, ?)", + (turn_id, session_id, turn_id, content, "2026-09-10T17:08:10.000Z"), + ) + for row in usage_rows or (): + columns = ", ".join(row) + placeholders = ", ".join("?" for _ in row) + connection.execute( + f"INSERT INTO assistant_usage_events ({columns}) VALUES ({placeholders})", + tuple(row.values()), + ) + connection.commit() + finally: + connection.close() + return database + + +def _paths(home: Path) -> dict[str, str]: + return resolve_sources(home) + + +def _records_for_table(records: list[dict[str, Any]], table: str) -> list[dict[str, Any]]: + return [record for record in records if record["payload"]["table"] == table] + + +def _collect_all( + paths: dict[str, str], + native_id: str, + *, + max_records: int = 1000, +) -> list[dict[str, Any]]: + cursor: dict[str, Any] = {} + collected: list[dict[str, Any]] = [] + for _ in range(10_000): + slice_ = read_database(paths, native_id, cursor, max_records=max_records) + collected.extend(slice_["records"]) + if slice_["exhausted"]: + return collected + cursor = slice_["next_cursor"] + raise AssertionError("database reader did not exhaust within 10000 bounded reads") + + +# --- module boundaries --- + + +def test_database_module_has_no_forbidden_imports(): + import thirdeye.platforms.copilot.database as database + + source = Path(database.__file__).read_text(encoding="utf-8") + assert "UsageStore" not in source + assert "usage_store" not in source + assert "from thirdeye.store" not in source + assert "import thirdeye.store" not in source + assert "transcript" not in source + + +# --- discovery --- + + +def test_discover_database_sessions_returns_sorted_unique_ids(tmp_path: Path): + home = tmp_path / "copilot" + _write_database( + home, + session_id="session-b", + extra_sessions=["session-a", "session-c"], + ) + paths = _paths(home) + assert discover_database_sessions(paths) == ["session-a", "session-b", "session-c"] + + +def test_discover_database_sessions_empty_when_database_missing(tmp_path: Path): + home = tmp_path / "copilot" + home.mkdir() + assert discover_database_sessions(_paths(home)) == [] + + +def test_discover_database_sessions_empty_when_sessions_table_missing(tmp_path: Path): + home = tmp_path / "copilot" + home.mkdir() + database = home / "session-store.db" + connection = sqlite3.connect(database) + connection.execute("CREATE TABLE turns (id INTEGER PRIMARY KEY, session_id TEXT)") + connection.commit() + connection.close() + assert discover_database_sessions(_paths(home)) == [] + + +# --- missing / empty sources --- + + +def test_read_database_missing_file_reports_diagnostic(tmp_path: Path): + home = tmp_path / "copilot" + home.mkdir() + paths = _paths(home) + slice_ = read_database(paths, "session-a", {}) + assert slice_["records"] == [] + assert slice_["exhausted"] is True + assert slice_["cwd"] is None + assert slice_["diagnostics"] == [ + { + "code": "copilot_database_missing", + "message": "Copilot session database is not present", + "path": paths["database"], + } + ] + + +def test_read_database_non_file_path_reports_unreadable(tmp_path: Path): + home = tmp_path / "copilot" + home.mkdir() + database = home / "session-store.db" + database.mkdir() + paths = _paths(home) + slice_ = read_database(paths, "session-a", {}) + assert slice_["records"] == [] + assert slice_["diagnostics"][0]["code"] == "copilot_database_unreadable" + + +def test_read_database_empty_session_returns_only_session_row(tmp_path: Path): + home = tmp_path / "copilot" + _write_database(home, session_id="session-a") + slice_ = read_database(_paths(home), "session-a", {}) + assert len(slice_["records"]) == 1 + assert slice_["records"][0]["payload"]["table"] == "sessions" + assert slice_["exhausted"] is True + assert slice_["cwd"] == "/tmp/workspace" + + +def test_read_database_unknown_session_is_empty_exhausted(tmp_path: Path): + home = tmp_path / "copilot" + _write_database(home, session_id="session-a") + slice_ = read_database(_paths(home), "missing-session", {}) + assert slice_["records"] == [] + assert slice_["exhausted"] is True + assert slice_["cwd"] is None + + +# --- all three tables --- + + +def test_read_database_reads_sessions_turns_and_usage_events(tmp_path: Path): + home = tmp_path / "copilot" + usage_row = { + "id": 13, + "session_id": "session-a", + "turn_index": 0, + "agent_id": None, + "parent_tool_call_id": None, + "model": "gpt-5.6-luna", + "input_tokens": 10, + "output_tokens": 5, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "reasoning_tokens": 0, + "total_nano_aiu": 1000, + "request_multiplier": 1.0, + "duration_ms": 100, + "time_to_first_token_ms": 50.0, + "output_ttft_ms": 50.0, + "inter_token_latency_ms": None, + "initiator": "user", + "api_endpoint": "ws:/responses", + "reasoning_effort": "medium", + "finish_reason": "stop", + "content_filter_triggered": 0, + "token_details_json": "[]", + "created_at": "2026-09-10T17:08:24.498Z", + } + _write_database( + home, + session_id="session-a", + turns=[(1, "first turn"), (2, "second turn")], + usage_rows=[usage_row], + ) + records = _collect_all(_paths(home), "session-a") + tables = {record["payload"]["table"] for record in records} + assert tables == {"assistant_usage_events", "sessions", "turns"} + assert len(_records_for_table(records, "turns")) == 2 + assert len(_records_for_table(records, "assistant_usage_events")) == 1 + + +def test_cli_fixture_usage_rows_are_readable_from_database(tmp_path: Path): + home = tmp_path / "copilot" + usage_rows = _load_json(CLI_FIXTURE / "assistant-usage-events.json") + _write_database( + home, + session_id=NATIVE_SESSION_ID, + cwd="/tmp/probe", + turns=[(0, "turn-0"), (1, "turn-1")], + usage_rows=usage_rows, + ) + records = _collect_all(_paths(home), NATIVE_SESSION_ID) + usage_records = _records_for_table(records, "assistant_usage_events") + assert len(usage_records) == 6 + assert {record["payload"]["row"]["id"] for record in usage_records} == { + 13, + 14, + 15, + 16, + 17, + 18, + } + assert usage_records[0]["source_kind"] == "database" + assert usage_records[0]["ts"] == "2026-09-10T17:08:24.498Z" + assert usage_records[0]["native_session_id"] == NATIVE_SESSION_ID + + +# --- source record shape --- + + +def test_source_record_includes_revision_and_generation(tmp_path: Path): + home = tmp_path / "copilot" + _write_database(home, session_id="session-a", turns=[(7, "first")]) + record = read_database(_paths(home), "session-a", {})["records"][0] + assert record["source_id"].startswith(f"copilot-db:{_paths(home)['source_key']}:session-a:") + assert record["locator"]["table"] == "sessions" + assert record["locator"]["content_revision"].startswith("sha256:") + assert record["locator"]["generation"].startswith("sha256:") + assert record["payload"]["row"]["id"] == "session-a" + + +# --- WAL / live changes --- + + +def test_reads_uncheckpointed_wal_commits(tmp_path: Path): + home = tmp_path / "copilot" + database = _write_database(home, session_id="session-a", turns=[(1, "checkpointed")]) + writer = sqlite3.connect(database) + try: + writer.execute("PRAGMA journal_mode=WAL") + writer.execute("PRAGMA wal_autocheckpoint=0") + writer.execute( + "INSERT INTO turns (id, session_id, turn_index, content, updated_at) " + "VALUES (?, ?, ?, ?, ?)", + (2, "session-a", 2, "wal-only", "2026-09-10T17:08:20.000Z"), + ) + writer.execute( + "INSERT INTO assistant_usage_events (id, session_id, turn_index, model, created_at) " + "VALUES (?, ?, ?, ?, ?)", + (99, "session-a", 2, "gpt-test", "2026-09-10T17:08:21.000Z"), + ) + writer.commit() + wal_path = Path(f"{database}-wal") + assert wal_path.is_file() + assert wal_path.stat().st_size > 0 + + records = _collect_all(_paths(home), "session-a") + turn_contents = { + record["payload"]["row"]["content"] + for record in records + if record["payload"]["table"] == "turns" + } + assert turn_contents == {"checkpointed", "wal-only"} + assert any( + record["payload"]["table"] == "assistant_usage_events" + and record["payload"]["row"]["id"] == 99 + for record in records + ) + assert wal_path.is_file() + assert wal_path.stat().st_size > 0 + finally: + writer.close() + + +def test_late_database_rows_visible_without_transcript_changes(tmp_path: Path): + home = tmp_path / "copilot" + database = _write_database(home, session_id="session-a", turns=[(1, "initial")]) + first = _collect_all(_paths(home), "session-a") + + connection = sqlite3.connect(database) + connection.execute( + "INSERT INTO assistant_usage_events (id, session_id, turn_index, model, created_at) " + "VALUES (?, ?, ?, ?, ?)", + (42, "session-a", 1, "gpt-late", "2026-09-10T17:09:00.000Z"), + ) + connection.commit() + connection.close() + + second = _collect_all(_paths(home), "session-a") + assert len(second) == len(first) + 1 + assert any( + record["payload"]["table"] == "assistant_usage_events" + and record["payload"]["row"]["model"] == "gpt-late" + for record in second + ) + + +# --- revisions and row reuse --- + + +def test_updated_turn_produces_new_content_revision(tmp_path: Path): + home = tmp_path / "copilot" + database = _write_database(home, session_id="session-a", turns=[(7, "first")]) + paths = _paths(home) + + before = _records_for_table(_collect_all(paths, "session-a"), "turns")[0] + connection = sqlite3.connect(database) + connection.execute( + "UPDATE turns SET content = ?, updated_at = ? WHERE id = ?", + ("updated", "2026-09-10T17:10:00.000Z", 7), + ) + connection.commit() + connection.close() + + after = _records_for_table(_collect_all(paths, "session-a"), "turns")[0] + assert before["locator"]["primary_key"] == after["locator"]["primary_key"] == 7 + assert before["locator"]["content_revision"] != after["locator"]["content_revision"] + assert before["source_id"] != after["source_id"] + assert after["payload"]["row"]["content"] == "updated" + + +def test_unchanged_resnapshot_keeps_stable_source_id(tmp_path: Path): + home = tmp_path / "copilot" + paths = _paths(home) + _write_database(home, session_id="session-a", turns=[(7, "stable")]) + first = read_database(paths, "session-a", {})["records"] + second = read_database(paths, "session-a", {})["records"] + turn_first = _records_for_table(first, "turns")[0] + turn_second = _records_for_table(second, "turns")[0] + assert turn_first["source_id"] == turn_second["source_id"] + assert turn_first["locator"]["content_revision"] == turn_second["locator"]["content_revision"] + + +def test_database_replacement_changes_generation_and_resets_cursor(tmp_path: Path): + home = tmp_path / "copilot" + database = _write_database(home, session_id="session-a", turns=[(1, "a"), (2, "b")]) + paths = _paths(home) + + first = read_database(paths, "session-a", {}, max_records=1) + old_generation = first["next_cursor"]["database_generation"] + assert first["exhausted"] is False + + replacement_home = tmp_path / "replacement-copilot" + replacement = _write_database( + replacement_home, + session_id="session-a", + turns=[(1, "replacement")], + journal_mode="DELETE", + ) + os.replace(replacement, database) + + after_replace = read_database(paths, "session-a", first["next_cursor"], max_records=10) + assert after_replace["next_cursor"]["database_generation"] != old_generation + assert after_replace["next_cursor"]["database_offset"] == len(after_replace["records"]) + assert { + record["payload"]["row"]["content"] + for record in _records_for_table(after_replace["records"], "turns") + } == {"replacement"} + + +# --- pagination --- + + +def test_read_database_paginates_with_cursor(tmp_path: Path): + home = tmp_path / "copilot" + _write_database( + home, + session_id="session-a", + turns=[(index, f"turn-{index}") for index in range(1, 6)], + journal_mode="DELETE", + ) + paths = _paths(home) + + page_one = read_database(paths, "session-a", {}, max_records=2) + assert len(page_one["records"]) == 2 + assert page_one["exhausted"] is False + assert page_one["next_cursor"]["database_offset"] == 2 + + page_two = read_database(paths, "session-a", page_one["next_cursor"], max_records=2) + assert len(page_two["records"]) == 2 + assert page_two["exhausted"] is False + + remaining = read_database(paths, "session-a", page_two["next_cursor"], max_records=10) + assert len(remaining["records"]) == 2 # two remaining turns after five total rows + assert remaining["exhausted"] is True + + +def test_stale_cursor_resets_when_generation_changes(tmp_path: Path): + home = tmp_path / "copilot" + database = _write_database(home, session_id="session-a", turns=[(1, "one"), (2, "two")]) + paths = _paths(home) + first = read_database(paths, "session-a", {}, max_records=1) + stale_cursor = { + "database_generation": "sha256:deadbeef", + "database_offset": 99, + } + + os.remove(database) + _write_database(home, session_id="session-a", turns=[(1, "fresh")]) + + slice_ = read_database(paths, "session-a", stale_cursor, max_records=10) + assert slice_["next_cursor"]["database_offset"] == len(slice_["records"]) + + +# --- schema diagnostics --- + + +def test_missing_table_reports_diagnostic_but_reads_available_tables(tmp_path: Path): + home = tmp_path / "copilot" + home.mkdir() + database = home / "session-store.db" + connection = sqlite3.connect(database) + connection.executescript( + """ + CREATE TABLE sessions (id TEXT PRIMARY KEY, cwd TEXT); + CREATE TABLE turns (id INTEGER PRIMARY KEY, session_id TEXT, content TEXT); + """ + ) + connection.execute("INSERT INTO sessions VALUES ('session-a', '/tmp')") + connection.execute("INSERT INTO turns VALUES (1, 'session-a', 'only-turn')") + connection.commit() + connection.close() + + slice_ = read_database(_paths(home), "session-a", {}) + codes = {diag["code"] for diag in slice_["diagnostics"]} + assert "copilot_database_table_missing" in codes + tables = {record["payload"]["table"] for record in slice_["records"]} + assert tables == {"sessions", "turns"} + + +def test_missing_session_column_reports_diagnostic(tmp_path: Path): + home = tmp_path / "copilot" + schema = """ + CREATE TABLE sessions (id TEXT PRIMARY KEY, cwd TEXT); + CREATE TABLE turns (id INTEGER PRIMARY KEY, content TEXT); + CREATE TABLE assistant_usage_events (id INTEGER PRIMARY KEY, model TEXT); + """ + _write_database(home, session_id="session-a", schema_sql=schema, seed_session=False) + connection = sqlite3.connect(home / "session-store.db") + connection.execute("INSERT INTO sessions VALUES ('session-a', '/tmp')") + connection.commit() + connection.close() + slice_ = read_database(_paths(home), "session-a", {}) + codes = {diag["code"] for diag in slice_["diagnostics"]} + assert "copilot_database_missing_session_column" in codes + assert {record["payload"]["table"] for record in slice_["records"]} == {"sessions"} + + +def test_missing_primary_key_reports_diagnostic(tmp_path: Path): + home = tmp_path / "copilot" + schema = """ + CREATE TABLE sessions (session_id TEXT, cwd TEXT); + CREATE TABLE turns (session_id TEXT, content TEXT); + CREATE TABLE assistant_usage_events (session_id TEXT, model TEXT); + """ + _write_database(home, session_id="session-a", schema_sql=schema, seed_session=False) + connection = sqlite3.connect(home / "session-store.db") + connection.execute("INSERT INTO sessions VALUES ('session-a', '/tmp')") + connection.commit() + connection.close() + slice_ = read_database(_paths(home), "session-a", {}) + codes = {diag["code"] for diag in slice_["diagnostics"]} + assert "copilot_database_missing_primary_key" in codes + assert slice_["records"] == [] + + +def test_optional_columns_are_preserved_when_present(tmp_path: Path): + home = tmp_path / "copilot" + schema = """ + CREATE TABLE sessions (id TEXT PRIMARY KEY, cwd TEXT, extra_flag INTEGER); + CREATE TABLE turns (id INTEGER PRIMARY KEY, session_id TEXT, content TEXT); + CREATE TABLE assistant_usage_events (id INTEGER PRIMARY KEY, session_id TEXT, model TEXT); + """ + _write_database(home, session_id="session-a", schema_sql=schema, seed_session=False) + connection = sqlite3.connect(home / "session-store.db") + connection.execute("INSERT INTO sessions (id, cwd, extra_flag) VALUES ('session-a', '/tmp', 1)") + connection.commit() + connection.close() + + session_record = _records_for_table(_collect_all(_paths(home), "session-a"), "sessions")[0] + assert session_record["payload"]["row"]["extra_flag"] == 1 + + +# --- locking --- + + +def test_busy_database_returns_actionable_diagnostic(tmp_path: Path): + home = tmp_path / "copilot" + database = _write_database(home, session_id="session-a", turns=[(1, "locked")]) + paths = _paths(home) + hold = threading.Event() + release = threading.Event() + error: list[str] = [] + + def hold_exclusive_lock() -> None: + connection = sqlite3.connect(database, timeout=30) + try: + connection.execute("BEGIN EXCLUSIVE") + hold.set() + release.wait(timeout=5) + finally: + connection.close() + + thread = threading.Thread(target=hold_exclusive_lock, daemon=True) + thread.start() + assert hold.wait(timeout=2) + + slice_ = read_database(paths, "session-a", {}) + release.set() + thread.join(timeout=2) + + if any(diag["code"] == "copilot_database_busy" for diag in slice_["diagnostics"]): + assert slice_["records"] == [] + assert slice_["exhausted"] is True + else: + # Some platforms may not block read-only URIs on EXCLUSIVE; accept successful read. + assert slice_["records"] + + +# --- validation --- + + +def test_read_database_rejects_invalid_native_id(tmp_path: Path): + home = tmp_path / "copilot" + _write_database(home, session_id="session-a") + with pytest.raises(ValueError, match="native session ID"): + read_database(_paths(home), "../escape", {}) + + +def test_read_database_rejects_invalid_max_records(tmp_path: Path): + home = tmp_path / "copilot" + _write_database(home, session_id="session-a") + with pytest.raises(ValueError, match="max_records"): + read_database(_paths(home), "session-a", {}, max_records=0) + + +# --- special values --- + + +def test_bytes_column_is_base64_encoded_in_payload(tmp_path: Path): + home = tmp_path / "copilot" + schema = """ + CREATE TABLE sessions (id TEXT PRIMARY KEY, cwd TEXT); + CREATE TABLE turns (id INTEGER PRIMARY KEY, session_id TEXT, blob_data BLOB); + CREATE TABLE assistant_usage_events (id INTEGER PRIMARY KEY, session_id TEXT); + """ + _write_database(home, session_id="session-a", schema_sql=schema, seed_session=False) + connection = sqlite3.connect(home / "session-store.db") + connection.execute("INSERT INTO sessions VALUES ('session-a', '/tmp')") + connection.execute( + "INSERT INTO turns (id, session_id, blob_data) VALUES (?, ?, ?)", + (1, "session-a", b"\x00\xff"), + ) + connection.commit() + connection.close() + + turn = _records_for_table(_collect_all(_paths(home), "session-a"), "turns")[0] + assert turn["payload"]["row"]["blob_data"] == { + "encoding": "base64", + "data": "AP8=", + } + + +# --- review fixes: discovery, pragma PK, cursor, diagnostics, connections --- + + +def test_discover_database_sessions_unions_ids_from_all_allowed_tables(tmp_path: Path): + home = tmp_path / "copilot" + home.mkdir() + database = home / "session-store.db" + connection = sqlite3.connect(database) + try: + connection.executescript( + """ + CREATE TABLE sessions (id TEXT PRIMARY KEY, cwd TEXT); + CREATE TABLE turns ( + id INTEGER PRIMARY KEY, + session_id TEXT, + user_message TEXT + ); + CREATE TABLE assistant_usage_events ( + id INTEGER PRIMARY KEY, + session_id TEXT, + model TEXT + ); + """ + ) + connection.execute("INSERT INTO sessions VALUES ('in-sessions', '/tmp')") + connection.execute("INSERT INTO turns VALUES (1, 'in-turns-only', 'hello')") + connection.execute("INSERT INTO assistant_usage_events VALUES (1, 'in-usage-only', 'gpt')") + connection.commit() + finally: + connection.close() + + assert discover_database_sessions(_paths(home)) == [ + "in-sessions", + "in-turns-only", + "in-usage-only", + ] + + +def test_discover_database_sessions_without_sessions_table_uses_other_tables( + tmp_path: Path, +): + home = tmp_path / "copilot" + home.mkdir() + database = home / "session-store.db" + connection = sqlite3.connect(database) + try: + connection.executescript( + """ + CREATE TABLE turns ( + id INTEGER PRIMARY KEY, + session_id TEXT, + user_message TEXT + ); + CREATE TABLE assistant_usage_events ( + id INTEGER PRIMARY KEY, + session_id TEXT, + model TEXT + ); + """ + ) + connection.execute("INSERT INTO turns VALUES (1, 'from-turns', 'hello')") + connection.execute("INSERT INTO assistant_usage_events VALUES (1, 'from-usage', 'gpt')") + connection.commit() + finally: + connection.close() + + assert discover_database_sessions(_paths(home)) == ["from-turns", "from-usage"] + + +def test_observed_cli_schema_preserves_session_and_turn_columns(tmp_path: Path): + home = tmp_path / "copilot" + schema = """ + CREATE TABLE sessions ( + id TEXT PRIMARY KEY, + cwd TEXT, + repository TEXT, + host_type TEXT, + branch TEXT, + summary TEXT, + created_at TEXT, + updated_at TEXT + ); + CREATE TABLE turns ( + id INTEGER PRIMARY KEY, + session_id TEXT NOT NULL, + turn_index INTEGER NOT NULL, + user_message TEXT, + assistant_response TEXT, + timestamp TEXT + ); + CREATE TABLE assistant_usage_events ( + id INTEGER PRIMARY KEY, + session_id TEXT NOT NULL, + turn_index INTEGER, + model TEXT, + created_at TEXT + ); + """ + _write_database(home, session_id="session-a", schema_sql=schema, seed_session=False) + database = home / "session-store.db" + connection = sqlite3.connect(database) + try: + connection.execute( + "INSERT INTO sessions " + "(id, cwd, repository, host_type, branch, summary, created_at, updated_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + ( + "session-a", + "/tmp/probe", + "github.com/example/repo", + "github", + "main", + "sum two files", + "2026-09-10T17:08:00.000Z", + "2026-09-10T17:08:50.000Z", + ), + ) + connection.execute( + "INSERT INTO turns " + "(id, session_id, turn_index, user_message, assistant_response, timestamp) " + "VALUES (?, ?, ?, ?, ?, ?)", + ( + 1, + "session-a", + 0, + "read alpha and beta", + "42", + "2026-09-10T17:08:10.000Z", + ), + ) + connection.execute( + "INSERT INTO assistant_usage_events " + "(id, session_id, turn_index, model, created_at) VALUES (?, ?, ?, ?, ?)", + (13, "session-a", 0, "gpt-5.6-luna", "2026-09-10T17:08:24.498Z"), + ) + connection.commit() + finally: + connection.close() + + records = _collect_all(_paths(home), "session-a") + session = _records_for_table(records, "sessions")[0] + turn = _records_for_table(records, "turns")[0] + usage = _records_for_table(records, "assistant_usage_events")[0] + assert session["payload"]["row"]["repository"] == "github.com/example/repo" + assert session["payload"]["row"]["cwd"] == "/tmp/probe" + assert turn["payload"]["row"]["user_message"] == "read alpha and beta" + assert turn["payload"]["row"]["assistant_response"] == "42" + assert turn["locator"]["primary_key"] == 1 + assert usage["payload"]["row"]["model"] == "gpt-5.6-luna" + + +def test_primary_key_uses_pragma_pk_when_column_is_not_named_id(tmp_path: Path): + home = tmp_path / "copilot" + schema = """ + CREATE TABLE sessions (session_pk TEXT PRIMARY KEY, cwd TEXT); + CREATE TABLE turns ( + turn_pk INTEGER PRIMARY KEY, + session_id TEXT, + user_message TEXT + ); + CREATE TABLE assistant_usage_events ( + usage_pk INTEGER PRIMARY KEY, + session_id TEXT, + model TEXT + ); + """ + _write_database(home, session_id="session-a", schema_sql=schema, seed_session=False) + connection = sqlite3.connect(home / "session-store.db") + try: + connection.execute("INSERT INTO sessions VALUES ('session-a', '/tmp')") + connection.execute("INSERT INTO turns VALUES (9, 'session-a', 'hello')") + connection.execute("INSERT INTO assistant_usage_events VALUES (3, 'session-a', 'gpt')") + connection.commit() + finally: + connection.close() + + records = _collect_all(_paths(home), "session-a") + session = _records_for_table(records, "sessions")[0] + turn = _records_for_table(records, "turns")[0] + usage = _records_for_table(records, "assistant_usage_events")[0] + assert session["locator"]["primary_key"] == "session-a" + assert turn["locator"]["primary_key"] == 9 + assert usage["locator"]["primary_key"] == 3 + assert "session-a" in session["source_id"] + + +def test_composite_primary_key_is_row_identity(tmp_path: Path): + home = tmp_path / "copilot" + schema = """ + CREATE TABLE sessions (id TEXT PRIMARY KEY, cwd TEXT); + CREATE TABLE turns ( + session_id TEXT NOT NULL, + turn_index INTEGER NOT NULL, + user_message TEXT, + assistant_response TEXT, + PRIMARY KEY (session_id, turn_index) + ); + CREATE TABLE assistant_usage_events ( + id INTEGER PRIMARY KEY, + session_id TEXT, + model TEXT + ); + """ + _write_database(home, session_id="session-a", schema_sql=schema, seed_session=False) + connection = sqlite3.connect(home / "session-store.db") + try: + connection.execute("INSERT INTO sessions VALUES ('session-a', '/tmp')") + connection.execute("INSERT INTO turns VALUES ('session-a', 0, 'first', 'reply-one')") + connection.execute("INSERT INTO turns VALUES ('session-a', 1, 'second', 'reply-two')") + connection.commit() + finally: + connection.close() + + turns = _records_for_table(_collect_all(_paths(home), "session-a"), "turns") + assert [record["locator"]["primary_key"] for record in turns] == [ + {"session_id": "session-a", "turn_index": 0}, + {"session_id": "session-a", "turn_index": 1}, + ] + assert turns[0]["source_id"] != turns[1]["source_id"] + + +def test_unrelated_wal_write_does_not_reset_pagination(tmp_path: Path): + home = tmp_path / "copilot" + database = _write_database( + home, + session_id="session-a", + turns=[(index, f"turn-{index}") for index in range(1, 6)], + extra_sessions=["session-b"], + ) + paths = _paths(home) + page_one = read_database(paths, "session-a", {}, max_records=2) + assert page_one["exhausted"] is False + first_ids = [record["source_id"] for record in page_one["records"]] + + writer = sqlite3.connect(database) + try: + writer.execute("PRAGMA journal_mode=WAL") + writer.execute("PRAGMA wal_autocheckpoint=0") + writer.execute( + "INSERT INTO turns (id, session_id, turn_index, content, updated_at) " + "VALUES (?, ?, ?, ?, ?)", + (50, "session-b", 1, "other-session", "2026-09-10T17:11:00.000Z"), + ) + writer.commit() + wal_path = Path(f"{database}-wal") + assert wal_path.is_file() + assert wal_path.stat().st_size > 0 + + page_two = read_database(paths, "session-a", page_one["next_cursor"], max_records=2) + second_ids = [record["source_id"] for record in page_two["records"]] + assert page_two["records"] + assert second_ids != first_ids + assert ( + page_two["next_cursor"]["database_generation"] + == page_one["next_cursor"]["database_generation"] + ) + assert page_two["next_cursor"]["database_offset"] == 4 + finally: + writer.close() + + +def test_same_session_writes_do_not_starve_later_rows(tmp_path: Path): + home = tmp_path / "copilot" + database = _write_database( + home, + session_id="session-a", + turns=[(index, f"turn-{index}") for index in range(1, 6)], + ) + paths = _paths(home) + page_one = read_database(paths, "session-a", {}, max_records=2) + assert page_one["exhausted"] is False + first_ids = [record["source_id"] for record in page_one["records"]] + + writer = sqlite3.connect(database) + try: + writer.execute("PRAGMA journal_mode=WAL") + writer.execute("PRAGMA wal_autocheckpoint=0") + writer.execute( + "UPDATE turns SET content = ?, updated_at = ? WHERE id = ?", + ("updated-turn-1", "2026-09-10T17:13:00.000Z", 1), + ) + writer.execute( + "INSERT INTO turns (id, session_id, turn_index, content, updated_at) " + "VALUES (?, ?, ?, ?, ?)", + (99, "session-a", 99, "late-same-session", "2026-09-10T17:13:01.000Z"), + ) + writer.execute( + "INSERT INTO assistant_usage_events (id, session_id, turn_index, model, created_at) " + "VALUES (?, ?, ?, ?, ?)", + (50, "session-a", 99, "gpt-live", "2026-09-10T17:13:02.000Z"), + ) + writer.commit() + wal_path = Path(f"{database}-wal") + assert wal_path.is_file() + assert wal_path.stat().st_size > 0 + + page_two = read_database(paths, "session-a", page_one["next_cursor"], max_records=2) + second_ids = [record["source_id"] for record in page_two["records"]] + assert page_two["records"] + assert second_ids != first_ids + assert ( + page_two["next_cursor"]["database_generation"] + == page_one["next_cursor"]["database_generation"] + ) + assert page_two["next_cursor"]["database_offset"] == 4 + + collected = list(page_one["records"]) + list(page_two["records"]) + cursor = page_two["next_cursor"] + for _ in range(20): + slice_ = read_database(paths, "session-a", cursor, max_records=2) + collected.extend(slice_["records"]) + cursor = slice_["next_cursor"] + if slice_["exhausted"]: + break + else: + raise AssertionError("pagination did not exhaust after same-session writes") + + turn_contents = { + record["payload"]["row"]["content"] + for record in collected + if record["payload"]["table"] == "turns" + } + assert "turn-5" in turn_contents + assert "late-same-session" in turn_contents + assert any( + record["payload"]["table"] == "assistant_usage_events" + and record["payload"]["row"]["model"] == "gpt-live" + for record in collected + ) + finally: + writer.close() + + +def test_repeated_same_session_inserts_still_reach_later_rows(tmp_path: Path): + home = tmp_path / "copilot" + database = _write_database( + home, + session_id="session-a", + turns=[(index, f"turn-{index}") for index in range(1, 9)], + ) + paths = _paths(home) + cursor: dict[str, Any] = {} + collected: list[dict[str, Any]] = [] + first_page_ids: list[str] | None = None + writer = sqlite3.connect(database) + try: + writer.execute("PRAGMA journal_mode=WAL") + writer.execute("PRAGMA wal_autocheckpoint=0") + for index in range(20): + slice_ = read_database(paths, "session-a", cursor, max_records=2) + page_ids = [record["source_id"] for record in slice_["records"]] + if first_page_ids is None: + first_page_ids = page_ids + else: + assert page_ids != first_page_ids + collected.extend(slice_["records"]) + writer.execute( + "INSERT INTO assistant_usage_events " + "(id, session_id, turn_index, model, created_at) VALUES (?, ?, ?, ?, ?)", + ( + 100 + index, + "session-a", + index, + f"gpt-live-{index}", + "2026-09-10T17:14:00.000Z", + ), + ) + writer.commit() + if slice_["exhausted"]: + break + cursor = slice_["next_cursor"] + else: + raise AssertionError("continuous same-session inserts starved later rows") + finally: + writer.close() + + assert first_page_ids is not None + turn_contents = { + record["payload"]["row"]["content"] + for record in collected + if record["payload"]["table"] == "turns" + } + assert "turn-8" in turn_contents + assert any(record["payload"]["table"] == "assistant_usage_events" for record in collected) + + +def test_same_row_id_different_content_is_new_revision(tmp_path: Path): + home = tmp_path / "copilot" + database = _write_database(home, session_id="session-a", turns=[(7, "first")]) + paths = _paths(home) + before = _records_for_table(_collect_all(paths, "session-a"), "turns")[0] + + connection = sqlite3.connect(database) + try: + connection.execute("DELETE FROM turns WHERE id = 7") + connection.execute( + "INSERT INTO turns (id, session_id, turn_index, content, updated_at) " + "VALUES (?, ?, ?, ?, ?)", + (7, "session-a", 7, "reused", "2026-09-10T17:12:00.000Z"), + ) + connection.commit() + finally: + connection.close() + + after = _records_for_table(_collect_all(paths, "session-a"), "turns")[0] + assert before["locator"]["primary_key"] == after["locator"]["primary_key"] == 7 + assert before["locator"]["content_revision"] != after["locator"]["content_revision"] + assert before["source_id"] != after["source_id"] + assert after["payload"]["row"]["content"] == "reused" + + +def test_malformed_database_is_incompatible_not_busy(tmp_path: Path): + home = tmp_path / "copilot" + home.mkdir() + database = home / "session-store.db" + database.write_bytes(b"this is not a sqlite database") + slice_ = read_database(_paths(home), "session-a", {}) + assert slice_["records"] == [] + assert slice_["exhausted"] is True + assert slice_["diagnostics"][0]["code"] == "copilot_database_incompatible" + assert "busy" not in slice_["diagnostics"][0]["code"] + + +def test_read_and_discover_close_sqlite_connections(tmp_path: Path): + home = tmp_path / "copilot" + _write_database(home, session_id="session-a", turns=[(1, "closed")]) + paths = _paths(home) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always", ResourceWarning) + discover_database_sessions(paths) + read_database(paths, "session-a", {}) + gc.collect() + leaked = [item for item in caught if "unclosed database" in str(item.message)] + assert leaked == [] diff --git a/tests/platforms/copilot/test_events.py b/tests/platforms/copilot/test_events.py new file mode 100644 index 00000000..d2a94c74 --- /dev/null +++ b/tests/platforms/copilot/test_events.py @@ -0,0 +1,493 @@ +"""Behavioral tests for Copilot semantic event normalization.""" + +from __future__ import annotations + +import ast +import json +from pathlib import Path +from typing import Any + +import pytest + +from thirdeye.platforms.copilot.events import ( + agent_id, + interaction_id, + normalize_record, + normalize_records, + record_data, + record_type, + tool_call_id, +) +from thirdeye.platforms.copilot.types import SourceRecord + +FIXTURES = Path(__file__).parent / "fixtures" +RECON_CASES = FIXTURES / "reconciliation-cases" +NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" +SOURCE_KEY = "a" * 64 + + +def _load_json(path: Path) -> Any: + return json.loads(path.read_text(encoding="utf-8")) + + +def _transcript_record( + *, + source_id: str, + native_type: str, + data: dict[str, Any], + ts: str = "2026-09-10T17:08:24.503Z", + agent: str | None = None, +) -> SourceRecord: + payload: dict[str, Any] = { + "type": native_type, + "data": data, + "id": source_id.rsplit("/", 1)[-1], + "timestamp": ts, + "schema_version": 1, + } + if agent is not None: + payload["agentId"] = agent + return { + "source_id": source_id, + "source_kind": "transcript", + "native_session_id": NATIVE_SESSION_ID, + "ts": ts, + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": payload, + "locator": {"file": "events.jsonl", "native_event_id": payload["id"]}, + } + + +def _hook_record(*, source_id: str, event: str, hook_payload: dict[str, Any]) -> SourceRecord: + return { + "source_id": source_id, + "source_kind": "hook", + "native_session_id": NATIVE_SESSION_ID, + "ts": "2026-09-10T17:08:24.500Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "schema_version": 1, + "event": event, + "hook_payload": hook_payload, + "context": {}, + }, + "locator": {"observation_id": source_id.rsplit("/", 1)[-1], "event": event}, + } + + +def _event_kinds(record: SourceRecord) -> list[str]: + return [event["kind"] for event in normalize_record(record)] + + +# --- record accessors --- + + +def test_record_type_uses_event_field_for_hooks(): + record = _hook_record( + source_id=f"hook/{NATIVE_SESSION_ID}/obs-1", + event="preToolUse", + hook_payload={"toolName": "view"}, + ) + assert record_type(record) == "preToolUse" + + +def test_record_data_reads_hook_payload(): + record = _hook_record( + source_id=f"hook/{NATIVE_SESSION_ID}/obs-1", + event="permissionRequest", + hook_payload={"toolName": "view", "toolArgs": {"path": "/tmp/a.txt"}}, + ) + assert record_data(record)["toolName"] == "view" + + +def test_identity_helpers_read_transcript_fields(): + record = _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/msg-1", + native_type="assistant.message", + data={ + "interactionId": "interaction-a", + "turnId": "0", + "toolRequests": [{"toolCallId": "call_abc", "name": "view", "arguments": {}}], + }, + agent="agent-child", + ) + execution = _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/tool-1", + native_type="tool.execution_start", + data={"toolCallId": "call_abc", "toolName": "view", "arguments": {}}, + ) + assert interaction_id(record) == "interaction-a" + assert agent_id(record) == "agent-child" + assert tool_call_id(execution) == "call_abc" + + +# --- user and assistant messages --- + + +def test_user_message_normalizes_delivery_and_source_metadata(): + record = _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/user-1", + native_type="user.message", + data={ + "content": "hello", + "interactionId": "ix-1", + "turnId": "0", + "delivery": "idle", + "source": "agent-parent", + }, + ) + events = normalize_record(record) + assert len(events) == 1 + event = events[0] + assert event["kind"] == "user_prompt" + assert event["source_references"][0]["role"] == "user_prompt" + assert event["attributes"]["delivery"] == "idle" + assert event["attributes"]["source"] == "agent-parent" + assert event["attributes"]["interaction_id"] == "ix-1" + + +def test_assistant_message_emits_tool_requests_with_stable_ids(): + source_id = f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/assistant-1" + record = _transcript_record( + source_id=source_id, + native_type="assistant.message", + data={ + "content": "", + "model": "gpt-5.6-luna", + "interactionId": "ix-1", + "turnId": "0", + "toolRequests": [ + { + "toolCallId": "call_alpha", + "name": "view", + "arguments": {"path": "/fixture/workspace/alpha.txt"}, + "intentionSummary": "view alpha", + }, + { + "toolCallId": "call_beta", + "name": "view", + "arguments": {"path": "/fixture/workspace/beta.txt"}, + }, + ], + }, + ) + events = normalize_record(record) + assert [event["kind"] for event in events] == [ + "assistant_message", + "tool_request", + "tool_request", + ] + tool_events = events[1:] + assert tool_events[0]["id"] == f"copilot:event:{source_id}:tool:call_alpha" + assert tool_events[1]["id"] == f"copilot:event:{source_id}:tool:call_beta" + assert tool_events[0]["attributes"]["arguments"] == {"path": "/fixture/workspace/alpha.txt"} + assert tool_events[0]["attributes"]["intention_summary"] == "view alpha" + + +# --- tool execution --- + + +def test_tool_execution_complete_and_failure_kinds(): + success = _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/tool-ok", + native_type="tool.execution_complete", + data={"toolCallId": "call_ok", "success": True, "result": "17"}, + ) + failure = _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/tool-bad", + native_type="tool.execution_complete", + data={"toolCallId": "call_bad", "success": False, "result": "denied"}, + ) + assert _event_kinds(success) == ["tool_execution_complete"] + assert _event_kinds(failure) == ["tool_execution_failure"] + assert normalize_record(failure)[0]["attributes"]["result"] == "denied" + + +def test_tool_execution_start_preserves_arguments(): + record = _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/tool-start", + native_type="tool.execution_start", + data={ + "toolCallId": "call_start", + "toolName": "view", + "arguments": {"path": "/fixture/workspace/alpha.txt"}, + }, + ) + event = normalize_record(record)[0] + assert event["kind"] == "tool_execution_start" + assert event["attributes"]["name"] == "view" + assert event["attributes"]["arguments"] == {"path": "/fixture/workspace/alpha.txt"} + + +# --- permission, compaction, lifecycle --- + + +@pytest.mark.parametrize( + ("case_name", "expected_kind", "expected_classification", "observed"), + [ + ("permission", "permission_request", "main", False), + ("compaction", "compaction", "checkpoint", False), + ("auxiliary_title_generation", "auxiliary_model_call", "title_generation", False), + ], +) +def test_synthetic_reconciliation_case_event_normalization( + case_name: str, + expected_kind: str, + expected_classification: str, + observed: bool, +) -> None: + case = _load_json(RECON_CASES / "cases.json")[case_name] + assert case["observed"] is observed + events = normalize_records(case["input_records"]) + matching = [event for event in events if event["kind"] == expected_kind] + assert matching, f"expected {expected_kind} in synthetic case {case_name}" + assert matching[0]["classification"] == expected_classification + expected_events = case["expected"].get("events") + if expected_events: + by_id = {event["id"]: event for event in events} + for expected in expected_events: + actual = by_id[expected["id"]] + assert actual["kind"] == expected["kind"] + assert actual["classification"] == expected["classification"] + + +def test_permission_hook_maps_without_becoming_tool_execution_role(): + case = _load_json(RECON_CASES / "cases.json")["permission"] + event = normalize_record(case["input_records"][0])[0] + assert event["kind"] == "permission_request" + assert event["source_references"][0]["role"] == "permission_request" + + +def test_compaction_checkpoint_classification(): + case = _load_json(RECON_CASES / "cases.json")["compaction"] + transcript = case["input_records"][0] + event = normalize_record(transcript)[0] + assert event["kind"] == "compaction" + assert event["classification"] == "checkpoint" + assert event["source_references"][0]["role"] == "checkpoint" + + +def test_session_shutdown_is_shutdown_validation(): + record = _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/shutdown", + native_type="session.shutdown", + data={"totalNanoAiu": 123}, + ) + event = normalize_record(record)[0] + assert event["kind"] == "session_shutdown" + assert event["classification"] == "shutdown_validation" + + +def test_subagent_events_use_nested_child_role(): + record = _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/sub-start", + native_type="subagent.started", + data={"toolCallId": "call_parent_task", "agentName": "explore"}, + agent="bf8cb9f3-2097-4db0-a3c8-78a2653b2106", + ) + event = normalize_record(record)[0] + assert event["kind"] == "subagent_started" + assert event["source_references"][0]["role"] == "nested_child" + assert event["attributes"]["parent_tool_call_id"] == "call_parent_task" + + +def test_unknown_record_without_native_type_is_preserved() -> None: + record = _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/typeless", + native_type="user.message", + data={"content": "hello"}, + ) + record["payload"].pop("type") + event = normalize_record(record)[0] + assert event["kind"] == "unknown" + assert "native_type" not in event["attributes"] + assert event["attributes"]["raw_payload"]["data"]["content"] == "hello" + assert event["source_references"][0]["source_kind"] == "transcript" + + +def test_prompt_transformation_transcript_event() -> None: + record = _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/prompt-xf", + native_type="prompt.transformation", + data={"interactionId": "ix-1", "prompt": "expanded"}, + ) + event = normalize_record(record)[0] + assert event["kind"] == "prompt_transformation" + assert event["attributes"]["prompt"] == "expanded" + + +# --- unknown and hook observations --- + + +def test_unknown_native_type_is_preserved(): + record = _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/future", + native_type="future.event.v2", + data={"feature": "beta"}, + ) + event = normalize_record(record)[0] + assert event["kind"] == "unknown" + assert event["attributes"]["native_type"] == "future.event.v2" + assert event["attributes"]["raw_payload"]["type"] == "future.event.v2" + + +def test_hook_pretooluse_stays_hook_observation_not_transcript_execution(): + record = _hook_record( + source_id=f"hook/{NATIVE_SESSION_ID}/obs-pre", + event="preToolUse", + hook_payload={"toolName": "view", "toolArgs": {"path": "/tmp/a.txt"}}, + ) + event = normalize_record(record)[0] + assert event["kind"] in {"notification", "unknown"} + assert event["kind"] != "tool_execution_start" + assert event["source_references"][0]["role"] == "hook" + assert event["attributes"]["native_type"] == "preToolUse" + + +def test_external_tool_hooks_are_not_tool_execution_events() -> None: + records = [ + _hook_record( + source_id=f"hook/{NATIVE_SESSION_ID}/obs-pre", + event="preToolUse", + hook_payload={"toolName": "view"}, + ), + _hook_record( + source_id=f"hook/{NATIVE_SESSION_ID}/obs-post", + event="postToolUse", + hook_payload={"toolName": "view"}, + ), + _hook_record( + source_id=f"hook/{NATIVE_SESSION_ID}/obs-fail", + event="postToolUseFailure", + hook_payload={"toolName": "view"}, + ), + ] + kinds = [normalize_record(record)[0]["kind"] for record in records] + assert kinds == ["notification", "notification", "notification"] or all( + kind == "unknown" for kind in kinds + ) + for kind in kinds: + assert kind not in { + "tool_execution_start", + "tool_execution_complete", + "tool_execution_failure", + } + + +def test_normalize_records_replays_in_order_without_deduplication(): + records = [ + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/a", + native_type="user.message", + data={"content": "one", "interactionId": "ix", "turnId": "0"}, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/b", + native_type="user.message", + data={"content": "two", "interactionId": "ix", "turnId": "0"}, + ), + ] + events = normalize_records(records) + assert len(events) == 2 + assert events[0]["attributes"]["interaction_id"] == "ix" + assert events[1]["source_ids"] == [records[1]["source_id"]] + + +def test_events_module_does_not_import_usage_or_export() -> None: + source = Path(__import__("thirdeye.platforms.copilot.events", fromlist=["__file__"]).__file__) + tree = ast.parse(source.read_text(encoding="utf-8")) + imported: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imported.update(alias.name.split(".")[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module: + imported.add(node.module.split(".")[0]) + imported.add(node.module) + forbidden = {"usage", "attribution", "projection_store", "export_state"} + assert not (imported & forbidden) + assert "thirdeye.platforms.copilot.usage" not in imported + + +def test_database_records_are_skipped_not_unknown() -> None: + record: SourceRecord = { + "source_id": f"copilot-db:{SOURCE_KEY}:{NATIVE_SESSION_ID}:assistant_usage_events:13:sha256:abc", + "source_kind": "database", + "native_session_id": NATIVE_SESSION_ID, + "ts": "2026-09-10T17:08:24.498Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": {"table": "assistant_usage_events", "row": {"id": 13, "model": "gpt-5.6-luna"}}, + "locator": {"table": "assistant_usage_events", "primary_key": 13}, + } + assert normalize_record(record) == [] + case = _load_json(RECON_CASES / "cases.json")["compaction"] + events = normalize_records(case["input_records"]) + assert [event["kind"] for event in events] == ["compaction"] + + +def test_assistant_turn_markers_are_cycle_events_not_unknown() -> None: + start = _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/turn-start", + native_type="assistant.turn_start", + data={"turnId": "0", "interactionId": "ix-1"}, + ) + end = _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/turn-end", + native_type="assistant.turn_end", + data={"turnId": "0"}, + ) + assert _event_kinds(start) == ["assistant_turn_start"] + assert _event_kinds(end) == ["assistant_turn_end"] + assert normalize_record(end)[0]["source_references"][0]["role"] == "finish" + + +def test_session_lifecycle_and_subagent_configured_are_not_unknown() -> None: + model_change = _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/model-change", + native_type="session.model_change", + data={"newModel": "auto"}, + ) + auto_mode = _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/auto-mode", + native_type="session.auto_mode_resolved", + data={"chosenModel": "gpt-5.6-luna"}, + ) + configured = _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/sub-configured", + native_type="subagent.configured", + data={"model": "gpt-5.6-luna"}, + agent="child-agent", + ) + assert _event_kinds(model_change) == ["notification"] + assert _event_kinds(auto_mode) == ["notification"] + configured_event = normalize_record(configured)[0] + assert configured_event["kind"] in {"notification", "unknown"} + assert configured_event["kind"] != "subagent_started" + assert configured_event["attributes"]["native_type"] == "subagent.configured" + + +def test_agent_stop_hook_is_not_session_end() -> None: + record = _hook_record( + source_id=f"hook/{NATIVE_SESSION_ID}/obs-stop", + event="agentStop", + hook_payload={"sessionId": NATIVE_SESSION_ID, "stopReason": "end_turn"}, + ) + event = normalize_record(record)[0] + assert event["kind"] == "agent_stop" + assert event["source_references"][0]["role"] == "finish" + + +def test_model_error_stays_auxiliary_and_tool_error_keeps_call_id() -> None: + model_error = _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/model-error", + native_type="model.error", + data={"model": "gpt-4o-mini", "purpose": "session_title"}, + ) + tool_error = _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/tool-error", + native_type="tool.execution_error", + data={"toolCallId": "call_bad", "result": "denied"}, + ) + assert _event_kinds(model_error) == ["auxiliary_model_call"] + assert normalize_record(model_error)[0]["classification"] == "title_generation" + assert _event_kinds(tool_error) == ["tool_execution_failure"] + assert normalize_record(tool_error)[0]["attributes"]["tool_call_id"] == "call_bad" diff --git a/tests/platforms/copilot/test_export.py b/tests/platforms/copilot/test_export.py new file mode 100644 index 00000000..eaa78ac4 --- /dev/null +++ b/tests/platforms/copilot/test_export.py @@ -0,0 +1,2053 @@ +"""Behavioral tests for Copilot export eligibility, placement ledger, and queue_exports.""" + +from __future__ import annotations + +import json +import shutil +from pathlib import Path +from typing import Any + +import pytest + +from thirdeye import otel_export +from thirdeye.config import Config, LogfireSettings +from thirdeye.meta import read_meta, write_meta +from thirdeye.paths import meta_path, session_dir +from thirdeye.platforms.copilot import export_state as copilot_export_state +from thirdeye.platforms.copilot import export_transport +from thirdeye.platforms.copilot.archive import commit_batch +from thirdeye.platforms.copilot.constants import PLATFORM_NAME +from thirdeye.platforms.copilot.export import queue_exports +from thirdeye.platforms.copilot.export_state import ( + empty_export_state, + export_state_path, + initialize_eligibility, + load_export_state, + mark_placement_delivered, + record_placement, + update_export_state, + usage_digest, +) +from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id +from thirdeye.platforms.copilot.projection import build_projection +from thirdeye.platforms.copilot.transcript import read_transcript +from thirdeye.platforms.copilot.types import Projection, SourceBatch, SourcePaths, SourceRecord +from thirdeye.usage.types import UsageRow + +FIXTURES = Path(__file__).parent / "fixtures" +RECON = FIXTURES / "reconciliation-cases" +NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" +SOURCE_KEY = "a" * 64 +GENERATION = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" +OBSERVED_AT = "2026-09-10T17:09:00.000Z" +TURN_ONE = f"copilot:turn:{SOURCE_KEY}:{NATIVE_SESSION_ID}:interaction-main-1" +CALL_MATCHED = f"copilot:call:{SOURCE_KEY}/{NATIVE_SESSION_ID}/a4a17e63-7ba5-422f-8ee9-b495be417328" +ACCOUNTING_MATCHED = ( + f"copilot:usage:{SOURCE_KEY}:assistant_usage_events:" + f"sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:13" +) +ACCOUNTING_UNMATCHED = ( + f"copilot:usage:{SOURCE_KEY}:assistant_usage_events:" + f"sha256%3Abbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:14" +) + + +def _load_json(path: Path) -> Any: + return json.loads(path.read_text(encoding="utf-8")) + + +def _record(source_id: str, *, native_session_id: str = NATIVE_SESSION_ID) -> SourceRecord: + return { + "source_id": source_id, + "source_kind": "transcript", + "native_session_id": native_session_id, + "ts": "2026-09-10T17:08:24.000Z", + "observed_at": "2026-09-10T17:08:25.000Z", + "payload": {"schema_version": 1, "type": "user.message"}, + "locator": {"file": "events.jsonl", "offset": 0}, + } + + +def _batch(paths: SourcePaths, records: list[SourceRecord]) -> SourceBatch: + return { + "source_key": paths["source_key"], + "native_session_id": NATIVE_SESSION_ID, + "cwd": "/proj", + "records": records, + "next_cursor": {"generation": 1}, + "diagnostics": [], + } + + +def _usage_row(**overrides: Any) -> UsageRow: + fields = dict( + session_id="stored-session", + seq=0, + call_id=ACCOUNTING_MATCHED, + ts="2026-09-10T17:08:24.498Z", + platform=PLATFORM_NAME, + provider_name="unknown", + response_model="gpt-5.6-luna", + input_tokens=6452, + output_tokens=107, + cache_creation_input_tokens=6449, + ) + fields.update(overrides) + return UsageRow(**fields) + + +def _main_turn( + *, + turn_id: str = TURN_ONE, + status: str = "completed", + accounting_calls: list[dict[str, Any]] | None = None, + llm_calls: list[dict[str, Any]] | None = None, + subagents: list[dict[str, Any]] | None = None, +) -> dict[str, Any]: + return { + "turn_id": turn_id, + "start_ts": "2026-09-10T17:08:24.000Z", + "end_ts": "2026-09-10T17:08:28.000Z", + "input_message": "hello", + "output_message": "done", + "status": status, + "llm_calls": llm_calls + if llm_calls is not None + else [ + { + "call_id": CALL_MATCHED, + "provider": "unknown", + "model": "gpt-5.6-luna", + "start_ts": "2026-09-10T17:08:24.503Z", + "end_ts": "2026-09-10T17:08:24.593Z", + "input_messages": [], + "output_messages": [], + "usage": {}, + "tool_calls": [], + } + ], + "permission_requests": [], + "subagents": subagents or [], + "attributes": {"interaction_id": "interaction-main-1"}, + "accounting_calls": accounting_calls or [], + } + + +def _accounting_call( + *, + accounting_id: str = ACCOUNTING_MATCHED, + attribution_status: str = "matched", + call_id: str | None = CALL_MATCHED, + usage: dict[str, Any] | None = None, +) -> dict[str, Any]: + return { + "accounting_id": accounting_id, + "usage": usage or _usage_row(call_id=accounting_id).to_dict(), + "attribution_status": attribution_status, + "agent_id": None, + "call_id": call_id, + "attributes": {"copilot.logical_call_id": accounting_id}, + } + + +def _attribution( + *, + logical_call_id: str, + status: str = "matched", + stored_turn_id: str | None = TURN_ONE, + call_id: str | None = None, + usage_source_id: str = "db-row-1", +) -> dict[str, Any]: + return { + "usage_source_id": usage_source_id, + "logical_call_id": logical_call_id, + "stored_turn_id": stored_turn_id, + "agent_id": None, + "call_id": call_id, + "status": status, + "join_kind": "inferred", + "evidence": ["join_kind:inferred"], + } + + +def _projection(**overrides: Any) -> Projection: + base: Projection = { + "normalized_events": [], + "turns": [], + "usage_rows": [], + "attributions": [], + "pending": [], + "diagnostics": [], + } + base.update(overrides) + return base + + +@pytest.fixture +def config(tmp_path: Path) -> Config: + return Config(root=tmp_path / "thirdeye") + + +@pytest.fixture +def enabled_config(tmp_path: Path) -> Config: + return Config( + root=tmp_path / "thirdeye", + logfire=LogfireSettings(enabled=True, token="fake-token"), + ) + + +@pytest.fixture +def paths(tmp_path: Path) -> SourcePaths: + home = tmp_path / "copilot-home" + home.mkdir() + return resolve_sources(home) + + +def _seed_session(config: Config, paths: SourcePaths) -> str: + commit_batch(config, paths, _batch(paths, [_record("key/a/event-1")])) + return stored_session_id(paths, NATIVE_SESSION_ID) + + +def _directory(config: Config, stored: str) -> Path: + return session_dir(config.root, PLATFORM_NAME, stored) + + +def _close_session(config: Config, stored: str) -> None: + path = meta_path(_directory(config, stored)) + meta = read_meta(path) + assert meta is not None + meta.status = "closed" + meta.ended_at = "2026-09-10T17:09:01.000Z" + write_meta(path, meta) + + +@pytest.fixture +def export_calls(monkeypatch: pytest.MonkeyPatch) -> dict[str, list[Any]]: + calls: dict[str, list[Any]] = { + "turn": [], + "turn_accounting": [], + "session_accounting": [], + } + + def _turn(*args: Any, **kwargs: Any) -> bool: + calls["turn"].append(args[4]) + return True + + def _turn_accounting(*args: Any, **kwargs: Any) -> bool: + calls["turn_accounting"].append(args[5]) + return True + + def _session_accounting(*args: Any, **kwargs: Any) -> bool: + calls["session_accounting"].append(args[4]) + return True + + monkeypatch.setattr(export_transport, "queue_turn", _turn) + monkeypatch.setattr(export_transport, "queue_turn_accounting", _turn_accounting) + monkeypatch.setattr(export_transport, "queue_session_accounting", _session_accounting) + return calls + + +class TestExportState: + def test_pending_unowned_accounting_defaults_on_legacy_state( + self, config: Config, paths: SourcePaths + ) -> None: + stored = _seed_session(config, paths) + export_state_path(_directory(config, stored)).write_text( + json.dumps({"schema_version": 1, "activated": True}), + encoding="utf-8", + ) + + state = load_export_state(config, stored) + + assert state["pending_unowned_accounting"] == [] + + def test_pending_unowned_accounting_normalizes_and_replaces( + self, config: Config, paths: SourcePaths + ) -> None: + stored = _seed_session(config, paths) + saved = update_export_state( + config, + stored, + lambda state: { + **state, + "pending_unowned_accounting": ["usage-b", "usage-a", "usage-a", None], + }, + ) + saved["placements"]["usage-a"] = {"destination": "chat-span"} + + updated = copilot_export_state.set_pending_unowned_accounting(saved, ["usage-c"]) + + assert saved["pending_unowned_accounting"] == ["usage-a", "usage-b"] + assert updated["pending_unowned_accounting"] == ["usage-c"] + assert updated["placements"] == saved["placements"] + + def test_initialize_eligibility_first_activation_excludes_terminal_history(self) -> None: + state = initialize_eligibility( + empty_export_state(), + terminal_turn_ids=["turn-a", "turn-b"], + accounting_ids=["acct-1"], + include_history=False, + ) + assert state["activated"] is True + assert state["excluded_turn_ids"] == ["turn-a", "turn-b"] + assert state["excluded_accounting_ids"] == ["acct-1"] + + def test_initialize_eligibility_include_history_keeps_terminal_eligible(self) -> None: + state = initialize_eligibility( + empty_export_state(), + terminal_turn_ids=["turn-a"], + accounting_ids=["acct-1"], + include_history=True, + ) + assert state["activated"] is True + assert state["excluded_turn_ids"] == [] + assert state["excluded_accounting_ids"] == [] + + def test_initialize_eligibility_later_include_history_opts_in(self) -> None: + first = initialize_eligibility( + empty_export_state(), + terminal_turn_ids=["turn-a"], + accounting_ids=["acct-1"], + include_history=False, + ) + second = initialize_eligibility( + first, + terminal_turn_ids=["turn-a"], + accounting_ids=["acct-1"], + include_history=True, + ) + assert second["excluded_turn_ids"] == [] + assert second["excluded_accounting_ids"] == [] + + def test_record_placement_is_idempotent_for_same_decision(self) -> None: + usage = _usage_row().to_dict() + state, entry, accepted = record_placement( + empty_export_state(), + accounting_id="acct-1", + destination="chat-span", + span_id="call-1", + usage=usage, + ) + assert accepted is True + assert entry is not None + again, same_entry, same_accepted = record_placement( + state, + accounting_id="acct-1", + destination="chat-span", + span_id="call-1", + usage=usage, + ) + assert same_accepted is True + assert same_entry == entry + + def test_record_placement_replaces_undelivered_placement_on_changed_destination( + self, + ) -> None: + """Nothing was ever confirmed delivered, so a corrected destination + just replaces the queued-but-unconfirmed placement outright.""" + usage = _usage_row().to_dict() + state, _, accepted = record_placement( + empty_export_state(), + accounting_id="acct-1", + destination="turn-accounting-span", + span_id="span-a", + usage=usage, + ) + assert accepted is True + replaced, entry, accepted_again = record_placement( + state, + accounting_id="acct-1", + destination="chat-span", + span_id="call-1", + usage=usage, + ) + assert accepted_again is True + assert entry is not None + assert entry["destination"] == "chat-span" + assert replaced["placements"]["acct-1"]["destination"] == "chat-span" + assert "acct-1" not in replaced["conflicts"] + + def test_record_placement_replaces_undelivered_placement_on_usage_correction( + self, + ) -> None: + usage = _usage_row().to_dict() + state, _, accepted = record_placement( + empty_export_state(), + accounting_id="acct-1", + destination="chat-span", + span_id="call-1", + usage=usage, + ) + assert accepted is True + corrected = dict(usage) + corrected["output_tokens"] = 999 + replaced, entry, accepted_again = record_placement( + state, + accounting_id="acct-1", + destination="chat-span", + span_id="call-1", + usage=corrected, + ) + assert accepted_again is True + assert entry is not None + assert entry["usage_digest"] == usage_digest(corrected) + assert "acct-1" not in replaced["conflicts"] + + def test_record_placement_conflicts_on_changed_destination_after_delivery(self) -> None: + usage = _usage_row().to_dict() + state, _, accepted = record_placement( + empty_export_state(), + accounting_id="acct-1", + destination="turn-accounting-span", + span_id="span-a", + usage=usage, + delivered=True, + ) + assert accepted is True + conflicted, existing, rejected = record_placement( + state, + accounting_id="acct-1", + destination="chat-span", + span_id="call-1", + usage=usage, + ) + assert rejected is False + assert existing is not None + assert existing["destination"] == "turn-accounting-span" + assert "acct-1" in conflicted["conflicts"] + + def test_record_placement_conflicts_on_usage_correction_after_delivery(self) -> None: + usage = _usage_row().to_dict() + state, _, accepted = record_placement( + empty_export_state(), + accounting_id="acct-1", + destination="chat-span", + span_id="call-1", + usage=usage, + delivered=True, + ) + assert accepted is True + corrected = dict(usage) + corrected["output_tokens"] = 999 + conflicted, _, rejected = record_placement( + state, + accounting_id="acct-1", + destination="chat-span", + span_id="call-1", + usage=corrected, + ) + assert rejected is False + assert "acct-1" in conflicted["conflicts"] + assert conflicted["conflicts"]["acct-1"]["candidate"]["usage_digest"] == usage_digest( + corrected + ) + + def test_record_placement_self_heals_emitted_flag_when_delivery_confirmed(self) -> None: + """Same destination/span/usage as before, but the caller now has a + fresh durable-claim read showing delivery succeeded: the ledger's own + ``emitted`` flag was never told directly, so this is the only place + it catches up.""" + usage = _usage_row().to_dict() + state, entry, _ = record_placement( + empty_export_state(), + accounting_id="acct-1", + destination="chat-span", + span_id="call-1", + usage=usage, + ) + assert entry is not None + assert entry["emitted"] is False + healed, healed_entry, accepted = record_placement( + state, + accounting_id="acct-1", + destination="chat-span", + span_id="call-1", + usage=usage, + delivered=True, + ) + assert accepted is True + assert healed_entry is not None + assert healed_entry["emitted"] is True + assert healed["placements"]["acct-1"]["emitted"] is True + + def test_record_placement_conflicts_when_old_job_is_claimed_in_flight(self) -> None: + """The old job's own worker could deliver at any moment -- relocating + while it is ``"claimed"`` is proven neither safe nor already known to + be delivered, so it must be quarantined rather than guessed either + way, exactly like a confirmed-delivered correction.""" + usage = _usage_row().to_dict() + state, _, accepted = record_placement( + empty_export_state(), + accounting_id="acct-1", + destination="turn-accounting-span", + span_id="span-a", + usage=usage, + ) + assert accepted is True + conflicted, existing, rejected = record_placement( + state, + accounting_id="acct-1", + destination="chat-span", + span_id="call-1", + usage=usage, + old_job_state="claimed", + ) + assert rejected is False + assert existing is not None + assert existing["destination"] == "turn-accounting-span" + assert "acct-1" in conflicted["conflicts"] + assert "in flight" in conflicted["conflicts"]["acct-1"]["reason"] + + def test_record_placement_replaces_when_old_job_proven_inert(self) -> None: + """Unlike ``"claimed"``, a ``None`` (never queued), ``"queued"`` + (untouched by any worker), or ``"failed"`` (permanently gave up, will + never be retried by the transport) old job state proves nothing is + in flight, so relocating is exactly as safe as it always was for an + undelivered placement.""" + usage = _usage_row().to_dict() + for old_state in (None, "queued", "failed"): + state, _, accepted = record_placement( + empty_export_state(), + accounting_id="acct-1", + destination="turn-accounting-span", + span_id="span-a", + usage=usage, + ) + assert accepted is True + replaced, entry, accepted_again = record_placement( + state, + accounting_id="acct-1", + destination="chat-span", + span_id="call-1", + usage=usage, + old_job_state=old_state, + ) + assert accepted_again is True, old_state + assert entry is not None + assert entry["destination"] == "chat-span" + assert "acct-1" not in replaced["conflicts"] + + +class TestQueueExports: + def test_unknown_session_raises(self, enabled_config: Config) -> None: + projection = _projection() + with pytest.raises(ValueError, match="unknown Copilot session"): + queue_exports(enabled_config, "missing-session", projection) + + def test_unconfigured_returns_zero_but_activates( + self, config: Config, paths: SourcePaths + ) -> None: + stored = _seed_session(config, paths) + projection = _projection( + turns=[_main_turn(accounting_calls=[_accounting_call()])], + usage_rows=[_usage_row()], + attributions=[_attribution(logical_call_id=ACCOUNTING_MATCHED)], + ) + queued = queue_exports(config, stored, projection) + assert queued == 0 + state = load_export_state(config, stored) + assert state["activated"] is True + assert TURN_ONE in state["excluded_turn_ids"] + assert ACCOUNTING_MATCHED in state["excluded_accounting_ids"] + + def test_unsupported_ledger_schema_version_fails_closed( + self, config: Config, paths: SourcePaths + ) -> None: + """A missing ledger file legitimately starts from empty state, but a + present file with an unrecognized ``schema_version`` must never be + silently treated as empty -- that would forget every recorded + placement and risk emitting tokens at a second location.""" + stored = _seed_session(config, paths) + directory = _directory(config, stored) + directory.mkdir(parents=True, exist_ok=True) + export_state_path(directory).write_text( + json.dumps({"schema_version": 999, "placements": {"acct-1": {"emitted": True}}}), + encoding="utf-8", + ) + with pytest.raises(ValueError, match="unsupported Copilot export ledger schema_version"): + load_export_state(config, stored) + projection = _projection(turns=[_main_turn()]) + with pytest.raises(ValueError, match="unsupported Copilot export ledger schema_version"): + queue_exports(config, stored, projection) + + def test_first_activation_without_history_queues_nothing( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + ) -> None: + stored = _seed_session(enabled_config, paths) + projection = _projection( + turns=[ + _main_turn( + accounting_calls=[ + _accounting_call(), + _accounting_call( + accounting_id=ACCOUNTING_UNMATCHED, + attribution_status="ambiguous", + call_id=None, + usage=_usage_row(call_id=ACCOUNTING_UNMATCHED).to_dict(), + ), + ] + ) + ], + usage_rows=[ + _usage_row(), + _usage_row(call_id=ACCOUNTING_UNMATCHED, input_tokens=6587, output_tokens=5), + ], + attributions=[ + _attribution(logical_call_id=ACCOUNTING_MATCHED), + _attribution( + logical_call_id=ACCOUNTING_UNMATCHED, + status="ambiguous", + call_id=None, + ), + ], + ) + queued = queue_exports(enabled_config, stored, projection) + assert queued == 0 + assert export_calls["turn"] == [] + assert export_calls["turn_accounting"] == [] + assert export_calls["session_accounting"] == [] + + def test_include_history_queues_terminal_turn_and_matched_accounting( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + ) -> None: + stored = _seed_session(enabled_config, paths) + projection = _projection( + turns=[_main_turn(accounting_calls=[_accounting_call()])], + usage_rows=[_usage_row()], + attributions=[_attribution(logical_call_id=ACCOUNTING_MATCHED)], + ) + queued = queue_exports(enabled_config, stored, projection, include_history=True) + assert queued == 1 + assert len(export_calls["turn"]) == 1 + turn = export_calls["turn"][0] + assert turn["turn_span_id"] + assert len(turn["accounting_calls"]) == 1 + assert turn["accounting_calls"][0]["accounting_id"] == ACCOUNTING_MATCHED + assert export_calls["turn_accounting"] == [] + state = load_export_state(enabled_config, stored) + placement = state["placements"][ACCOUNTING_MATCHED] + assert placement["destination"] == "chat-span" + assert placement["span_id"] == CALL_MATCHED + + def test_ambiguous_terminal_usage_queues_turn_accounting_job( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + ) -> None: + stored = _seed_session(enabled_config, paths) + projection = _projection( + turns=[ + _main_turn( + accounting_calls=[ + _accounting_call( + accounting_id=ACCOUNTING_UNMATCHED, + attribution_status="ambiguous", + call_id=None, + usage=_usage_row(call_id=ACCOUNTING_UNMATCHED).to_dict(), + ) + ] + ) + ], + usage_rows=[ + _usage_row(call_id=ACCOUNTING_UNMATCHED, input_tokens=6587, output_tokens=5) + ], + attributions=[ + _attribution( + logical_call_id=ACCOUNTING_UNMATCHED, + status="ambiguous", + call_id=None, + ) + ], + ) + queued = queue_exports(enabled_config, stored, projection, include_history=True) + assert queued == 2 + assert len(export_calls["turn_accounting"]) == 1 + assert export_calls["turn_accounting"][0]["accounting_id"] == ACCOUNTING_UNMATCHED + assert len(export_calls["turn"]) == 1 + assert export_calls["turn"][0]["accounting_calls"] == [] + state = load_export_state(enabled_config, stored) + assert state["placements"][ACCOUNTING_UNMATCHED]["destination"] == "turn-accounting-span" + + def test_session_accounting_for_unattached_ambiguous_usage( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + ) -> None: + stored = _seed_session(enabled_config, paths) + usage = _usage_row(call_id=ACCOUNTING_UNMATCHED, input_tokens=6587, output_tokens=5) + projection = _projection( + turns=[_main_turn()], + usage_rows=[usage], + attributions=[ + _attribution( + logical_call_id=ACCOUNTING_UNMATCHED, + status="ambiguous", + stored_turn_id=None, + call_id=None, + ) + ], + ) + _close_session(enabled_config, stored) + queued = queue_exports(enabled_config, stored, projection, include_history=True) + assert queued == 2 + assert len(export_calls["session_accounting"]) == 1 + assert export_calls["session_accounting"][0]["accounting_id"] == ACCOUNTING_UNMATCHED + state = load_export_state(enabled_config, stored) + assert state["placements"][ACCOUNTING_UNMATCHED]["destination"] == "session-accounting-span" + + def test_ownership_barrier_defers_until_matched_call_reconstructs( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + ) -> None: + stored = _seed_session(enabled_config, paths) + queue_exports(enabled_config, stored, _projection()) + incomplete = _projection( + turns=[_main_turn(llm_calls=[])], + usage_rows=[_usage_row()], + attributions=[ + _attribution( + logical_call_id=ACCOUNTING_MATCHED, + call_id=CALL_MATCHED, + ) + ], + ) + + first_queued = queue_exports(enabled_config, stored, incomplete) + + first_state = load_export_state(enabled_config, stored) + assert first_queued == 0 + assert export_calls["turn"] == [] + assert export_calls["turn_accounting"] == [] + assert export_calls["session_accounting"] == [] + assert ACCOUNTING_MATCHED not in first_state["placements"] + assert first_state["pending_unowned_accounting"] == [ACCOUNTING_MATCHED] + + complete = _projection( + turns=[_main_turn(accounting_calls=[_accounting_call()])], + usage_rows=[_usage_row()], + attributions=[ + _attribution( + logical_call_id=ACCOUNTING_MATCHED, + call_id=CALL_MATCHED, + ) + ], + ) + + second_queued = queue_exports(enabled_config, stored, complete) + + second_state = load_export_state(enabled_config, stored) + assert second_queued == 1 + assert len(export_calls["turn"]) == 1 + assert export_calls["turn"][0]["accounting_calls"][0]["accounting_id"] == ( + ACCOUNTING_MATCHED + ) + assert second_state["placements"][ACCOUNTING_MATCHED]["destination"] == "chat-span" + assert second_state["pending_unowned_accounting"] == [] + assert ACCOUNTING_MATCHED not in second_state["conflicts"] + + def test_open_ownerless_accounting_defers_until_session_closes( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + ) -> None: + stored = _seed_session(enabled_config, paths) + usage = _usage_row(call_id=ACCOUNTING_UNMATCHED) + projection = _projection( + usage_rows=[usage], + attributions=[ + _attribution( + logical_call_id=ACCOUNTING_UNMATCHED, + status="ambiguous", + stored_turn_id=None, + call_id=None, + ) + ], + ) + + open_queued = queue_exports(enabled_config, stored, projection) + + open_state = load_export_state(enabled_config, stored) + assert open_queued == 0 + assert export_calls["session_accounting"] == [] + assert ACCOUNTING_UNMATCHED not in open_state["excluded_accounting_ids"] + assert open_state["pending_unowned_accounting"] == [ACCOUNTING_UNMATCHED] + + _close_session(enabled_config, stored) + closed_queued = queue_exports(enabled_config, stored, projection) + + closed_state = load_export_state(enabled_config, stored) + assert closed_queued == 1 + assert len(export_calls["session_accounting"]) == 1 + assert closed_state["placements"][ACCOUNTING_UNMATCHED]["destination"] == ( + "session-accounting-span" + ) + assert closed_state["pending_unowned_accounting"] == [] + + def test_closed_session_flushes_incomplete_owned_accounting_under_turn( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + ) -> None: + stored = _seed_session(enabled_config, paths) + queue_exports(enabled_config, stored, _projection()) + _close_session(enabled_config, stored) + incomplete = _projection( + turns=[_main_turn(llm_calls=[])], + usage_rows=[_usage_row()], + attributions=[ + _attribution( + logical_call_id=ACCOUNTING_MATCHED, + call_id=CALL_MATCHED, + ) + ], + ) + + queued = queue_exports(enabled_config, stored, incomplete) + + state = load_export_state(enabled_config, stored) + assert queued == 2 + assert len(export_calls["turn"]) == 1 + assert len(export_calls["turn_accounting"]) == 1 + assert export_calls["session_accounting"] == [] + assert state["placements"][ACCOUNTING_MATCHED]["destination"] == "turn-accounting-span" + assert state["pending_unowned_accounting"] == [] + + def test_pending_and_conflicting_attributions_are_not_exported( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + ) -> None: + stored = _seed_session(enabled_config, paths) + pending_usage = _usage_row(call_id="pending-call", input_tokens=100, output_tokens=1) + conflicting_usage = _usage_row(call_id="conflict-call", input_tokens=200, output_tokens=2) + projection = _projection( + turns=[ + _main_turn( + accounting_calls=[ + _accounting_call( + accounting_id="pending-call", + attribution_status="pending", + call_id=None, + usage=pending_usage.to_dict(), + ) + ] + ) + ], + usage_rows=[pending_usage, conflicting_usage], + attributions=[ + _attribution(logical_call_id="pending-call", status="pending", call_id=None), + _attribution( + logical_call_id="conflict-call", + status="conflicting", + stored_turn_id=None, + call_id=None, + ), + ], + ) + queued = queue_exports(enabled_config, stored, projection, include_history=True) + assert queued == 1 + assert export_calls["turn_accounting"] == [] + assert export_calls["session_accounting"] == [] + state = load_export_state(enabled_config, stored) + assert "pending-call" not in state["placements"] + assert "conflict-call" not in state["placements"] + + def _ambiguous_then_matched_projections(self) -> tuple[Projection, Projection]: + ambiguous_projection = _projection( + turns=[ + _main_turn( + accounting_calls=[ + _accounting_call( + accounting_id=ACCOUNTING_UNMATCHED, + attribution_status="ambiguous", + call_id=None, + usage=_usage_row(call_id=ACCOUNTING_UNMATCHED).to_dict(), + ) + ] + ) + ], + usage_rows=[ + _usage_row(call_id=ACCOUNTING_UNMATCHED, input_tokens=6587, output_tokens=5) + ], + attributions=[ + _attribution( + logical_call_id=ACCOUNTING_UNMATCHED, + status="ambiguous", + call_id=None, + ) + ], + ) + matched_projection = _projection( + turns=[ + _main_turn( + accounting_calls=[ + _accounting_call( + accounting_id=ACCOUNTING_UNMATCHED, + attribution_status="matched", + call_id=CALL_MATCHED, + usage=_usage_row(call_id=ACCOUNTING_UNMATCHED).to_dict(), + ) + ] + ) + ], + usage_rows=[ + _usage_row(call_id=ACCOUNTING_UNMATCHED, input_tokens=6587, output_tokens=5) + ], + attributions=[_attribution(logical_call_id=ACCOUNTING_UNMATCHED, call_id=CALL_MATCHED)], + ) + return ambiguous_projection, matched_projection + + def test_pre_delivery_correction_relocates_to_chat_without_conflict( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + ) -> None: + """Nothing was ever confirmed delivered for the fallback placement + (no real worker ran), so improved local matching is still free to + relocate the tokens onto the now-known chat span.""" + stored = _seed_session(enabled_config, paths) + ambiguous_projection, matched_projection = self._ambiguous_then_matched_projections() + queue_exports(enabled_config, stored, ambiguous_projection, include_history=True) + export_calls["turn"].clear() + export_calls["turn_accounting"].clear() + + queue_exports(enabled_config, stored, matched_projection, include_history=True) + + state = load_export_state(enabled_config, stored) + placement = state["placements"][ACCOUNTING_UNMATCHED] + assert placement["destination"] == "chat-span" + assert placement["span_id"] == CALL_MATCHED + assert ACCOUNTING_UNMATCHED not in state["conflicts"] + turn = export_calls["turn"][0] + assert turn["accounting_calls"][0]["accounting_id"] == ACCOUNTING_UNMATCHED + + def test_fallback_placement_prevents_later_chat_relocation_after_confirmed_delivery( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + ) -> None: + """Once the durable transport claim shows the fallback span was + actually flushed, later local matching can no longer move those + tokens onto the chat span — that would double the emitted total.""" + stored = _seed_session(enabled_config, paths) + ambiguous_projection, matched_projection = self._ambiguous_then_matched_projections() + queue_exports(enabled_config, stored, ambiguous_projection, include_history=True) + + directory = _directory(enabled_config, stored) + export_transport._mark_delivered(directory, ACCOUNTING_UNMATCHED) + + export_calls["turn"].clear() + export_calls["turn_accounting"].clear() + + queue_exports(enabled_config, stored, matched_projection, include_history=True) + + state = load_export_state(enabled_config, stored) + placement = state["placements"][ACCOUNTING_UNMATCHED] + assert placement["destination"] == "turn-accounting-span" + assert placement["emitted"] is True + assert ACCOUNTING_UNMATCHED in state["conflicts"] + assert export_calls["turn_accounting"] == [] + turn = export_calls["turn"][0] + assert turn["accounting_calls"] == [] + + def test_confirmed_turn_delivery_routes_new_match_to_fallback( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + ) -> None: + """Even with no prior accounting placement at all, a chat span that + the durable turn claim shows was already flushed is immutable: a + newly-matched usage row must land on a turn-accounting fallback + span, never on that already-sent chat span.""" + stored = _seed_session(enabled_config, paths) + turn_only_projection = _projection(turns=[_main_turn()]) + queue_exports(enabled_config, stored, turn_only_projection, include_history=True) + + directory = _directory(enabled_config, stored) + claim_path = otel_export._turn_claim_path(directory, TURN_ONE) + claim_path.parent.mkdir(parents=True, exist_ok=True) + claim_path.write_text("sent", encoding="utf-8", newline="\n") + + export_calls["turn"].clear() + _, matched_projection = self._ambiguous_then_matched_projections() + queue_exports(enabled_config, stored, matched_projection, include_history=True) + + state = load_export_state(enabled_config, stored) + placement = state["placements"][ACCOUNTING_UNMATCHED] + assert placement["destination"] == "turn-accounting-span" + assert len(export_calls["turn_accounting"]) == 1 + turn = export_calls["turn"][0] + assert turn["accounting_calls"] == [] + + def test_turn_claim_confirms_previously_embedded_chat_accounting( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + ) -> None: + """The turn acknowledgement is sufficient for accounting that the + Copilot ledger already placed inside that turn's chat span. This + survives a crash before the shared exporter writes its secondary + per-accounting acknowledgement.""" + stored = _seed_session(enabled_config, paths) + _, matched_projection = self._ambiguous_then_matched_projections() + queue_exports(enabled_config, stored, matched_projection, include_history=True) + + directory = _directory(enabled_config, stored) + claim_path = otel_export._turn_claim_path(directory, TURN_ONE) + claim_path.parent.mkdir(parents=True, exist_ok=True) + claim_path.write_text("sent", encoding="utf-8", newline="\n") + export_calls["turn"].clear() + export_calls["turn_accounting"].clear() + + queue_exports(enabled_config, stored, matched_projection, include_history=True) + + placement = load_export_state(enabled_config, stored)["placements"][ACCOUNTING_UNMATCHED] + assert placement["destination"] == "chat-span" + assert placement["emitted"] is True + assert export_calls["turn_accounting"] == [] + + def test_emitted_placement_skips_requeue( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + ) -> None: + stored = _seed_session(enabled_config, paths) + + def _mark_delivered(state: dict[str, Any]) -> dict[str, Any]: + placed, _, _ = record_placement( + state, + accounting_id=ACCOUNTING_UNMATCHED, + destination="turn-accounting-span", + span_id=f"accounting:{stored}:{TURN_ONE}:{ACCOUNTING_UNMATCHED}", + usage=_usage_row(call_id=ACCOUNTING_UNMATCHED).to_dict(), + ) + return mark_placement_delivered(placed, ACCOUNTING_UNMATCHED) + + update_export_state(enabled_config, stored, _mark_delivered) + update_export_state( + enabled_config, + stored, + lambda state: initialize_eligibility( + state, + terminal_turn_ids=[TURN_ONE], + accounting_ids=[ACCOUNTING_UNMATCHED], + include_history=True, + ), + ) + + projection = _projection( + turns=[ + _main_turn( + accounting_calls=[ + _accounting_call( + accounting_id=ACCOUNTING_UNMATCHED, + attribution_status="ambiguous", + call_id=None, + usage=_usage_row(call_id=ACCOUNTING_UNMATCHED).to_dict(), + ) + ] + ) + ], + usage_rows=[ + _usage_row(call_id=ACCOUNTING_UNMATCHED, input_tokens=6587, output_tokens=5) + ], + attributions=[ + _attribution( + logical_call_id=ACCOUNTING_UNMATCHED, + status="ambiguous", + call_id=None, + ) + ], + ) + queued = queue_exports(enabled_config, stored, projection, include_history=True) + assert queued == 1 + assert export_calls["turn_accounting"] == [] + assert len(export_calls["turn"]) == 1 + + def test_late_terminal_turn_becomes_eligible_without_history_opt_in( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + ) -> None: + stored = _seed_session(enabled_config, paths) + first_turn = _main_turn(turn_id="turn-first") + queue_exports( + enabled_config, + stored, + _projection(turns=[first_turn], usage_rows=[], attributions=[]), + ) + export_calls["turn"].clear() + + second_turn = _main_turn(turn_id="turn-second") + queued = queue_exports( + enabled_config, + stored, + _projection(turns=[first_turn, second_turn], usage_rows=[], attributions=[]), + ) + assert queued == 1 + assert export_calls["turn"][0]["turn_id"] == "turn-second" + + def test_accounting_job_failure_records_error( + self, + enabled_config: Config, + paths: SourcePaths, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + stored = _seed_session(enabled_config, paths) + monkeypatch.setattr(export_transport, "queue_turn", lambda *args, **kwargs: True) + monkeypatch.setattr( + export_transport, "queue_turn_accounting", lambda *args, **kwargs: False + ) + monkeypatch.setattr( + export_transport, "queue_session_accounting", lambda *args, **kwargs: False + ) + + projection = _projection( + turns=[ + _main_turn( + accounting_calls=[ + _accounting_call( + accounting_id=ACCOUNTING_UNMATCHED, + attribution_status="ambiguous", + call_id=None, + usage=_usage_row(call_id=ACCOUNTING_UNMATCHED).to_dict(), + ) + ] + ) + ], + usage_rows=[ + _usage_row(call_id=ACCOUNTING_UNMATCHED, input_tokens=6587, output_tokens=5) + ], + attributions=[ + _attribution( + logical_call_id=ACCOUNTING_UNMATCHED, + status="ambiguous", + call_id=None, + ) + ], + ) + queued = queue_exports(enabled_config, stored, projection, include_history=True) + assert queued == 1 + state = load_export_state(enabled_config, stored) + assert state["placements"][ACCOUNTING_UNMATCHED]["last_error"] == ( + "accounting job was not queued" + ) + + def test_turn_job_failure_is_not_counted_and_is_recorded( + self, + enabled_config: Config, + paths: SourcePaths, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """A Copilot turn spawn/write failure must not be counted as a + successful queue.""" + stored = _seed_session(enabled_config, paths) + monkeypatch.setattr(export_transport, "queue_turn", lambda *args, **kwargs: False) + + projection = _projection(turns=[_main_turn()]) + queued = queue_exports(enabled_config, stored, projection, include_history=True) + assert queued == 0 + state = load_export_state(enabled_config, stored) + assert state["turn_errors"][TURN_ONE] == "turn export job was not queued" + + def test_errors_clear_once_a_retry_succeeds( + self, + enabled_config: Config, + paths: SourcePaths, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """A stale `last_error`/turn error from an earlier failed attempt must + not linger once a later reconciliation successfully queues the job.""" + stored = _seed_session(enabled_config, paths) + monkeypatch.setattr(export_transport, "queue_turn", lambda *args, **kwargs: False) + monkeypatch.setattr( + export_transport, "queue_turn_accounting", lambda *args, **kwargs: False + ) + + projection = _projection( + turns=[ + _main_turn( + accounting_calls=[ + _accounting_call( + accounting_id=ACCOUNTING_UNMATCHED, + attribution_status="ambiguous", + call_id=None, + usage=_usage_row(call_id=ACCOUNTING_UNMATCHED).to_dict(), + ) + ] + ) + ], + usage_rows=[ + _usage_row(call_id=ACCOUNTING_UNMATCHED, input_tokens=6587, output_tokens=5) + ], + attributions=[ + _attribution(logical_call_id=ACCOUNTING_UNMATCHED, status="ambiguous", call_id=None) + ], + ) + queue_exports(enabled_config, stored, projection, include_history=True) + state = load_export_state(enabled_config, stored) + assert state["turn_errors"][TURN_ONE] == "turn export job was not queued" + assert state["placements"][ACCOUNTING_UNMATCHED]["last_error"] == ( + "accounting job was not queued" + ) + + monkeypatch.setattr(export_transport, "queue_turn", lambda *args, **kwargs: True) + monkeypatch.setattr(export_transport, "queue_turn_accounting", lambda *args, **kwargs: True) + queue_exports(enabled_config, stored, projection, include_history=True) + + state = load_export_state(enabled_config, stored) + assert TURN_ONE not in state["turn_errors"] + assert state["placements"][ACCOUNTING_UNMATCHED]["last_error"] is None + + def test_restart_preserves_ledger_and_boundary( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + ) -> None: + stored = _seed_session(enabled_config, paths) + projection = _projection( + turns=[_main_turn(accounting_calls=[_accounting_call()])], + usage_rows=[_usage_row()], + attributions=[_attribution(logical_call_id=ACCOUNTING_MATCHED)], + ) + queue_exports(enabled_config, stored, projection, include_history=True) + ledger_path = export_state_path(_directory(enabled_config, stored)) + saved = json.loads(ledger_path.read_text(encoding="utf-8")) + + export_calls["turn"].clear() + reloaded = Config( + root=enabled_config.root, + logfire=LogfireSettings(enabled=True, token="fake-token"), + ) + queued = queue_exports(reloaded, stored, projection, include_history=True) + assert queued == 1 + assert ( + json.loads(ledger_path.read_text(encoding="utf-8"))["placements"] == saved["placements"] + ) + + def test_non_terminal_turns_are_not_exported( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + ) -> None: + stored = _seed_session(enabled_config, paths) + projection = _projection( + turns=[_main_turn(status="in_progress", accounting_calls=[_accounting_call()])], + usage_rows=[_usage_row()], + attributions=[_attribution(logical_call_id=ACCOUNTING_MATCHED)], + ) + queued = queue_exports(enabled_config, stored, projection, include_history=True) + assert queued == 0 + assert export_calls["turn"] == [] + + def test_open_interaction_accounting_becomes_eligible_once_turn_completes( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + ) -> None: + """First activation must only wall off *already terminal* history. + + A usage row owned by an interaction that is still open at first + activation is not history yet — it must stay eligible once that + interaction later completes, without an explicit ``--export``. + """ + stored = _seed_session(enabled_config, paths) + open_projection = _projection( + turns=[ + _main_turn( + status="in_progress", + accounting_calls=[_accounting_call()], + ) + ], + usage_rows=[_usage_row()], + attributions=[_attribution(logical_call_id=ACCOUNTING_MATCHED)], + ) + queued = queue_exports(enabled_config, stored, open_projection) + assert queued == 0 + + completed_projection = _projection( + turns=[_main_turn(status="completed", accounting_calls=[_accounting_call()])], + usage_rows=[_usage_row()], + attributions=[_attribution(logical_call_id=ACCOUNTING_MATCHED)], + ) + queued = queue_exports(enabled_config, stored, completed_projection) + assert queued == 1 + state = load_export_state(enabled_config, stored) + assert ACCOUNTING_MATCHED in state["placements"] + assert state["placements"][ACCOUNTING_MATCHED]["destination"] == "chat-span" + + def test_pending_attribution_on_open_turn_is_eligible_once_resolved( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + ) -> None: + """A pending (unresolved) attribution owned by a still-open turn at + first activation must not be forever excluded once the interaction + finishes and the attribution later resolves to ambiguous/matched.""" + stored = _seed_session(enabled_config, paths) + open_projection = _projection( + turns=[ + _main_turn( + status="in_progress", + accounting_calls=[ + _accounting_call( + accounting_id=ACCOUNTING_UNMATCHED, + attribution_status="pending", + call_id=None, + usage=_usage_row(call_id=ACCOUNTING_UNMATCHED).to_dict(), + ) + ], + ) + ], + usage_rows=[_usage_row(call_id=ACCOUNTING_UNMATCHED)], + attributions=[ + _attribution(logical_call_id=ACCOUNTING_UNMATCHED, status="pending", call_id=None) + ], + ) + queue_exports(enabled_config, stored, open_projection) + + resolved_projection = _projection( + turns=[ + _main_turn( + status="completed", + accounting_calls=[ + _accounting_call( + accounting_id=ACCOUNTING_UNMATCHED, + attribution_status="ambiguous", + call_id=None, + usage=_usage_row(call_id=ACCOUNTING_UNMATCHED).to_dict(), + ) + ], + ) + ], + usage_rows=[_usage_row(call_id=ACCOUNTING_UNMATCHED)], + attributions=[ + _attribution(logical_call_id=ACCOUNTING_UNMATCHED, status="ambiguous", call_id=None) + ], + ) + queued = queue_exports(enabled_config, stored, resolved_projection) + assert queued == 2 + state = load_export_state(enabled_config, stored) + assert ACCOUNTING_UNMATCHED in state["placements"] + + def test_child_turn_accounting_stays_excluded_with_its_main_interaction( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + ) -> None: + """A nested child (subagent) turn is never exported as its own job, + so its own id is never a member of the boundary lists built from main + turns. Eligibility for its accounting must resolve to the owning main + interaction, not the child's own id -- otherwise usage discovered + later for a child of an already-excluded historical main turn would + wrongly become eligible on its own.""" + stored = _seed_session(enabled_config, paths) + child_turn_id = f"{TURN_ONE}:agent-1" + first_projection = _projection( + turns=[_main_turn(subagents=[_main_turn(turn_id=child_turn_id)])], + ) + queued = queue_exports(enabled_config, stored, first_projection) + assert queued == 0 + state = load_export_state(enabled_config, stored) + assert TURN_ONE in state["excluded_turn_ids"] + assert child_turn_id not in state["excluded_turn_ids"] + + late_child_accounting_projection = _projection( + turns=[ + _main_turn( + subagents=[ + _main_turn( + turn_id=child_turn_id, + accounting_calls=[_accounting_call()], + ) + ] + ) + ], + usage_rows=[_usage_row()], + attributions=[ + _attribution(logical_call_id=ACCOUNTING_MATCHED, stored_turn_id=child_turn_id) + ], + ) + queued = queue_exports(enabled_config, stored, late_child_accounting_projection) + assert queued == 0 + assert export_calls["turn_accounting"] == [] + state = load_export_state(enabled_config, stored) + assert ACCOUNTING_MATCHED not in state["placements"] + + def test_pending_ownerless_usage_is_eligible_once_it_resolves_to_a_completed_turn( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + ) -> None: + """At first activation a usage row's attribution may still be + ``"pending"`` with no ``stored_turn_id`` at all -- unresolved, not + confirmed ownerless. That must not be locked into the historical + boundary the way a *resolved* ownerless usage row would be: once it + later resolves to a real, completed owning turn, it must still be + exportable, exactly like any other accounting attached late to an + interaction that was open (or simply not yet observed) at + activation.""" + stored = _seed_session(enabled_config, paths) + pending_projection = _projection( + usage_rows=[_usage_row()], + attributions=[ + _attribution( + logical_call_id=ACCOUNTING_MATCHED, + status="pending", + stored_turn_id=None, + call_id=None, + ) + ], + ) + queued = queue_exports(enabled_config, stored, pending_projection) + assert queued == 0 + state = load_export_state(enabled_config, stored) + assert ACCOUNTING_MATCHED not in state["excluded_accounting_ids"] + + resolved_projection = _projection( + turns=[_main_turn(accounting_calls=[_accounting_call()])], + usage_rows=[_usage_row()], + attributions=[_attribution(logical_call_id=ACCOUNTING_MATCHED)], + ) + queued = queue_exports(enabled_config, stored, resolved_projection) + assert queued == 1 # the turn job; chat-span placement has no separate job + state = load_export_state(enabled_config, stored) + assert ACCOUNTING_MATCHED in state["placements"] + assert state["placements"][ACCOUNTING_MATCHED]["destination"] == "chat-span" + + def test_relocation_cancels_stale_queued_job_at_old_span( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + ) -> None: + """A prior reconciliation placed unmatched usage on a turn-accounting + fallback span and dispatched its deterministic job, which is still + sitting on disk untouched (``"queued"``) because no worker has + claimed it yet. When a later reconciliation discovers a real match + and relocates the tokens to the chat span, the stale fallback job + must be cancelled -- otherwise the already-dispatched worker could + still pick it up later and deliver the same tokens a second time.""" + stored = _seed_session(enabled_config, paths) + old_span_id = f"accounting:{stored}:{TURN_ONE}:{ACCOUNTING_UNMATCHED}" + job_path = export_transport.job_path(enabled_config.root, old_span_id) + job_path.parent.mkdir(parents=True, exist_ok=True) + job_path.write_text(json.dumps({"state": "queued", "attempt": 0}), encoding="utf-8") + + def _seed(state: dict[str, Any]) -> dict[str, Any]: + placed, _, _ = record_placement( + state, + accounting_id=ACCOUNTING_UNMATCHED, + destination="turn-accounting-span", + span_id=old_span_id, + usage=_usage_row(call_id=ACCOUNTING_UNMATCHED).to_dict(), + ) + return initialize_eligibility( + placed, + terminal_turn_ids=[TURN_ONE], + accounting_ids=[], + include_history=True, + ) + + update_export_state(enabled_config, stored, _seed) + assert job_path.exists() + + _, matched_projection = self._ambiguous_then_matched_projections() + queue_exports(enabled_config, stored, matched_projection, include_history=True) + + assert not job_path.exists() + state = load_export_state(enabled_config, stored) + placement = state["placements"][ACCOUNTING_UNMATCHED] + assert placement["destination"] == "chat-span" + assert ACCOUNTING_UNMATCHED not in state["conflicts"] + + def test_relocation_is_quarantined_while_old_job_is_claimed( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + ) -> None: + """Unlike the merely-``"queued"`` case, a job a worker has already + ``"claimed"`` could deliver at any moment. Relocating anyway risks a + duplicate if it does; leaving the old placement in place and + recording it as an inconclusive conflict is the only safe response, + the same way a confirmed-delivered correction is quarantined rather + than guessed.""" + stored = _seed_session(enabled_config, paths) + old_span_id = f"accounting:{stored}:{TURN_ONE}:{ACCOUNTING_UNMATCHED}" + job_path = export_transport.job_path(enabled_config.root, old_span_id) + job_path.parent.mkdir(parents=True, exist_ok=True) + job_path.write_text(json.dumps({"state": "claimed", "attempt": 0}), encoding="utf-8") + claim_path = export_transport.claim_path(job_path) + claim_path.write_text("worker-42", encoding="utf-8") + + def _seed(state: dict[str, Any]) -> dict[str, Any]: + placed, _, _ = record_placement( + state, + accounting_id=ACCOUNTING_UNMATCHED, + destination="turn-accounting-span", + span_id=old_span_id, + usage=_usage_row(call_id=ACCOUNTING_UNMATCHED).to_dict(), + ) + return initialize_eligibility( + placed, + terminal_turn_ids=[TURN_ONE], + accounting_ids=[], + include_history=True, + ) + + update_export_state(enabled_config, stored, _seed) + + _, matched_projection = self._ambiguous_then_matched_projections() + queue_exports(enabled_config, stored, matched_projection, include_history=True) + + assert job_path.exists() + assert claim_path.exists() + assert json.loads(job_path.read_text(encoding="utf-8"))["state"] == "claimed" + state = load_export_state(enabled_config, stored) + placement = state["placements"][ACCOUNTING_UNMATCHED] + assert placement["destination"] == "turn-accounting-span" + assert ACCOUNTING_UNMATCHED in state["conflicts"] + assert "in flight" in state["conflicts"][ACCOUNTING_UNMATCHED]["reason"] + + def test_relocation_is_quarantined_when_worker_claims_during_cancellation( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + monkeypatch: pytest.MonkeyPatch, + ) -> None: + stored = _seed_session(enabled_config, paths) + old_span_id = f"accounting:{stored}:{TURN_ONE}:{ACCOUNTING_UNMATCHED}" + + def _seed(state: dict[str, Any]) -> dict[str, Any]: + placed, _, _ = record_placement( + state, + accounting_id=ACCOUNTING_UNMATCHED, + destination="turn-accounting-span", + span_id=old_span_id, + usage=_usage_row(call_id=ACCOUNTING_UNMATCHED).to_dict(), + ) + return initialize_eligibility( + placed, + terminal_turn_ids=[TURN_ONE], + accounting_ids=[], + include_history=True, + ) + + update_export_state(enabled_config, stored, _seed) + monkeypatch.setattr( + export_transport, + "cancel", + lambda *args: {"state": "claimed", "attempt": 0, "last_error": None}, + ) + + _, matched_projection = self._ambiguous_then_matched_projections() + queue_exports(enabled_config, stored, matched_projection, include_history=True) + + state = load_export_state(enabled_config, stored) + assert state["placements"][ACCOUNTING_UNMATCHED]["span_id"] == old_span_id + assert ACCOUNTING_UNMATCHED in state["conflicts"] + + def test_permanently_failed_accounting_job_is_reported_and_not_cleared( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + ) -> None: + """The generic transport gives up on a deterministic accounting job + after exhausting its retries and marks it permanently ``"failed"`` on + disk (see ``otel_worker._claim_job``) -- it will never retry that job + again on its own. A local write+dispatch reporting success only means + the job file exists and a worker was spawned at some point; it must + not be conflated with actual delivery health, or a permanently stuck + job would silently look fine and have its error cleared forever.""" + stored = _seed_session(enabled_config, paths) + projection = _projection( + turns=[ + _main_turn( + accounting_calls=[ + _accounting_call( + accounting_id=ACCOUNTING_UNMATCHED, + attribution_status="ambiguous", + call_id=None, + usage=_usage_row(call_id=ACCOUNTING_UNMATCHED).to_dict(), + ) + ] + ) + ], + usage_rows=[ + _usage_row(call_id=ACCOUNTING_UNMATCHED, input_tokens=6587, output_tokens=5) + ], + attributions=[ + _attribution(logical_call_id=ACCOUNTING_UNMATCHED, status="ambiguous", call_id=None) + ], + ) + span_id = f"accounting:{stored}:{TURN_ONE}:{ACCOUNTING_UNMATCHED}" + job_path = export_transport.job_path(enabled_config.root, span_id) + job_path.parent.mkdir(parents=True, exist_ok=True) + job_path.write_text( + json.dumps( + {"state": "failed", "attempt": 5, "last_error": "TimeoutError: collector stalled"} + ), + encoding="utf-8", + ) + + queued = queue_exports(enabled_config, stored, projection, include_history=True) + assert ( + queued == 1 + ) # only the turn job; the permanently-failed accounting job does not count + state = load_export_state(enabled_config, stored) + placement = state["placements"][ACCOUNTING_UNMATCHED] + assert placement["job_state"] == "failed" + assert placement["job_attempt"] == 5 + assert placement["last_error"] == "TimeoutError: collector stalled" + + @pytest.mark.parametrize( + ("job_state", "last_error"), + [ + ("queued", None), + ("claimed", None), + ("retrying", "ConnectionError: collector unavailable"), + ], + ) + def test_accounting_worker_lifecycle_is_reflected_in_ledger( + self, + enabled_config: Config, + paths: SourcePaths, + export_calls: dict[str, list[Any]], + monkeypatch: pytest.MonkeyPatch, + job_state: str, + last_error: str | None, + ) -> None: + stored = _seed_session(enabled_config, paths) + projection = _projection( + turns=[ + _main_turn( + accounting_calls=[ + _accounting_call( + accounting_id=ACCOUNTING_UNMATCHED, + attribution_status="ambiguous", + call_id=None, + usage=_usage_row(call_id=ACCOUNTING_UNMATCHED).to_dict(), + ) + ] + ) + ], + usage_rows=[_usage_row(call_id=ACCOUNTING_UNMATCHED)], + attributions=[ + _attribution(logical_call_id=ACCOUNTING_UNMATCHED, status="ambiguous", call_id=None) + ], + ) + monkeypatch.setattr( + export_transport, + "status", + lambda *args: {"state": job_state, "attempt": 2, "last_error": last_error}, + ) + + queue_exports(enabled_config, stored, projection, include_history=True) + + placement = load_export_state(enabled_config, stored)["placements"][ACCOUNTING_UNMATCHED] + assert placement["job_state"] == job_state + assert placement["job_attempt"] == 2 + assert placement["last_error"] == last_error + + +class TestWorkerConfirmedDelivery: + """`queue_exports` against a real (locally flushed, never remote) Logfire + instance and the real detached worker, to verify the durable delivery + claim actually prevents a second emission across a simulated restart.""" + + @pytest.fixture(autouse=True) + def _reset_otel_state(self): + pytest.importorskip("logfire") + from thirdeye import otel_export as _otel_export + + _otel_export._state["attempted"] = False + _otel_export._state["instance"] = None + _otel_export._state["id_generator"] = None + yield + _otel_export._state["attempted"] = False + _otel_export._state["instance"] = None + _otel_export._state["id_generator"] = None + + @pytest.fixture + def exporter(self): + from logfire.testing import TestExporter + + return TestExporter() + + @pytest.fixture + def wired_instance(self, exporter, monkeypatch: pytest.MonkeyPatch): + import logfire + from opentelemetry.sdk.trace.export import SimpleSpanProcessor + + instance = logfire.configure( + send_to_logfire=False, + console=False, + additional_span_processors=[SimpleSpanProcessor(exporter)], + advanced=logfire.AdvancedOptions(id_generator=otel_export._id_generator()), + ) + monkeypatch.setattr(otel_export, "_get_instance", lambda config, platform: instance) + return instance + + @pytest.fixture(autouse=True) + def _synchronous_worker(self, monkeypatch: pytest.MonkeyPatch, enabled_config: Config): + """Run the detached worker in-process instead of spawning a real + child, same as the generic transport's own worker tests do.""" + from thirdeye import otel_worker + + monkeypatch.setattr(Config, "load", lambda: enabled_config) + + def _run(job_path: Path) -> None: + otel_worker.main([str(job_path)]) + + monkeypatch.setattr(otel_export, "_spawn", _run) + + def _run_accounting(job_path: Path) -> None: + export_transport.main([str(job_path)]) + + monkeypatch.setattr(export_transport, "_spawn", _run_accounting) + + def test_confirmed_accounting_delivery_survives_restart_without_double_emission( + self, + enabled_config: Config, + paths: SourcePaths, + wired_instance, + exporter, + ) -> None: + stored = _seed_session(enabled_config, paths) + projection = _projection( + turns=[ + _main_turn( + accounting_calls=[ + _accounting_call( + accounting_id=ACCOUNTING_UNMATCHED, + attribution_status="ambiguous", + call_id=None, + usage=_usage_row(call_id=ACCOUNTING_UNMATCHED).to_dict(), + ) + ] + ) + ], + usage_rows=[ + _usage_row(call_id=ACCOUNTING_UNMATCHED, input_tokens=6587, output_tokens=5) + ], + attributions=[ + _attribution(logical_call_id=ACCOUNTING_UNMATCHED, status="ambiguous", call_id=None) + ], + ) + + queue_exports(enabled_config, stored, projection, include_history=True) + directory = _directory(enabled_config, stored) + assert export_transport.delivery_sent(directory, ACCOUNTING_UNMATCHED) is True + first_accounting_spans = [ + span for span in exporter.exported_spans_as_dict() if span["name"] == "accounting" + ] + assert len(first_accounting_spans) == 1 + + # Simulate a restart: fresh Config/instance state, same durable + # ledger and durable transport claim on disk. + reloaded = Config( + root=enabled_config.root, + logfire=LogfireSettings(enabled=True, token="fake-token"), + ) + queued = queue_exports(reloaded, stored, projection, include_history=True) + assert queued == 1 # the turn job re-queues; harmless, first-wins claim there too + + second_accounting_spans = [ + span for span in exporter.exported_spans_as_dict() if span["name"] == "accounting" + ] + assert len(second_accounting_spans) == 1 # unchanged: no second accounting emission + state = load_export_state(enabled_config, stored) + assert state["placements"][ACCOUNTING_UNMATCHED]["emitted"] is True + + def test_chat_embedded_accounting_survives_restart_without_relocation_or_duplicate( + self, + enabled_config: Config, + paths: SourcePaths, + wired_instance, + exporter, + ) -> None: + """Matched usage placed on the chat span is delivered as part of the + turn's own job -- there is no separate fallback accounting job for + it. Copilot reconciles the durable turn claim with its chat placement; + otherwise a later pass could relocate the accounting to a fallback + span and duplicate tokens already flushed on the chat span.""" + stored = _seed_session(enabled_config, paths) + projection = _projection( + turns=[_main_turn(accounting_calls=[_accounting_call()])], + usage_rows=[_usage_row()], + attributions=[_attribution(logical_call_id=ACCOUNTING_MATCHED, call_id=CALL_MATCHED)], + ) + + queue_exports(enabled_config, stored, projection, include_history=True) + directory = _directory(enabled_config, stored) + assert otel_export.turn_export_sent(directory, TURN_ONE) is True + chat_spans = [ + span for span in exporter.exported_spans_as_dict() if span["name"].startswith("chat") + ] + assert len(chat_spans) == 1 + accounting_spans = [ + span for span in exporter.exported_spans_as_dict() if span["name"] == "accounting" + ] + assert len(accounting_spans) == 0 + + # Simulate a restart: fresh Config/instance state, same durable + # ledger, turn claim, and accounting delivery claim on disk. + reloaded = Config( + root=enabled_config.root, + logfire=LogfireSettings(enabled=True, token="fake-token"), + ) + queue_exports(reloaded, stored, projection, include_history=True) + + chat_spans_after = [ + span for span in exporter.exported_spans_as_dict() if span["name"].startswith("chat") + ] + accounting_spans_after = [ + span for span in exporter.exported_spans_as_dict() if span["name"] == "accounting" + ] + assert len(chat_spans_after) == 1 # unchanged: the turn's own claim is first-wins + assert len(accounting_spans_after) == 0 # never relocated to a fallback span + state = load_export_state(enabled_config, stored) + placement = state["placements"][ACCOUNTING_MATCHED] + assert placement["destination"] == "chat-span" + assert placement["emitted"] is True + assert ACCOUNTING_MATCHED not in state["conflicts"] + + def test_undelivered_same_span_token_correction_rewrites_job_and_delivers_once( + self, + enabled_config: Config, + paths: SourcePaths, + wired_instance, + exporter, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + stored = _seed_session(enabled_config, paths) + delayed: list[Path] = [] + + def _hold(job_path: Path) -> None: + delayed.append(job_path) + + monkeypatch.setattr(export_transport, "_spawn", _hold) + monkeypatch.setattr(otel_export, "_spawn", lambda job_path: None) + + first = _projection( + turns=[ + _main_turn( + accounting_calls=[ + _accounting_call( + accounting_id=ACCOUNTING_UNMATCHED, + attribution_status="ambiguous", + call_id=None, + usage=_usage_row( + call_id=ACCOUNTING_UNMATCHED, input_tokens=6587, output_tokens=5 + ).to_dict(), + ) + ] + ) + ], + usage_rows=[ + _usage_row(call_id=ACCOUNTING_UNMATCHED, input_tokens=6587, output_tokens=5) + ], + attributions=[ + _attribution(logical_call_id=ACCOUNTING_UNMATCHED, status="ambiguous", call_id=None) + ], + ) + queue_exports(enabled_config, stored, first, include_history=True) + span_id = f"accounting:{stored}:{TURN_ONE}:{ACCOUNTING_UNMATCHED}" + job_path = export_transport.job_path(enabled_config.root, span_id) + assert job_path.exists() + first_job = json.loads(job_path.read_text(encoding="utf-8")) + assert first_job["usage"]["gen_ai.usage.output_tokens"] == 5 + + corrected_usage = _usage_row( + call_id=ACCOUNTING_UNMATCHED, input_tokens=6587, output_tokens=999 + ).to_dict() + second = _projection( + turns=[ + _main_turn( + accounting_calls=[ + _accounting_call( + accounting_id=ACCOUNTING_UNMATCHED, + attribution_status="ambiguous", + call_id=None, + usage=corrected_usage, + ) + ] + ) + ], + usage_rows=[ + _usage_row(call_id=ACCOUNTING_UNMATCHED, input_tokens=6587, output_tokens=999) + ], + attributions=[ + _attribution(logical_call_id=ACCOUNTING_UNMATCHED, status="ambiguous", call_id=None) + ], + ) + queue_exports(enabled_config, stored, second, include_history=True) + rewritten = json.loads(job_path.read_text(encoding="utf-8")) + assert rewritten["usage"]["gen_ai.usage.output_tokens"] == 999 + assert rewritten["state"] == "queued" + + export_transport.main([str(job_path)]) + accounting_spans = [ + span for span in exporter.exported_spans_as_dict() if span["name"] == "accounting" + ] + assert len(accounting_spans) == 1 + assert accounting_spans[0]["attributes"]["gen_ai.usage.output_tokens"] == 999 + directory = _directory(enabled_config, stored) + assert export_transport.delivery_sent(directory, ACCOUNTING_UNMATCHED) is True + state = load_export_state(enabled_config, stored) + assert ACCOUNTING_UNMATCHED not in state["conflicts"] + + def test_observed_six_call_queue_exports_labels_native_billing_not_usd( + self, + enabled_config: Config, + wired_instance, + exporter, + tmp_path: Path, + ) -> None: + home = tmp_path / "copilot-home" + home.mkdir() + paths = resolve_sources(home) + commit_batch(enabled_config, paths, _batch(paths, _drain_cli_transcript(home))) + stored = stored_session_id(paths, NATIVE_SESSION_ID) + source_key = _source_key(_drain_cli_transcript(home)) + records = _drain_cli_transcript(home) + _six_call_records(source_key=source_key) + projection, _ = build_projection(records, {}) + + queue_exports(enabled_config, stored, projection, include_history=True) + + billed = [ + span + for span in exporter.exported_spans_as_dict() + if span["attributes"].get("thirdeye.accounting.billing.kind") == "copilot-native-unit" + ] + assert billed + for span in billed: + assert span["attributes"]["thirdeye.accounting.billing.kind"] == "copilot-native-unit" + assert "operation.cost" not in span["attributes"] + assert span["attributes"]["copilot.billing.nano_aiu"] + + +def _drain_cli_transcript(home: Path) -> list[SourceRecord]: + session_dir = home / "session-state" / NATIVE_SESSION_ID + session_dir.mkdir(parents=True, exist_ok=True) + shutil.copy(FIXTURES / "events.jsonl", session_dir / "events.jsonl") + (session_dir / "workspace.yaml").write_text("cwd: /sanitized/workspace\n", encoding="utf-8") + paths = resolve_sources(home) + cursor: dict[str, Any] = {} + records: list[SourceRecord] = [] + while True: + slice_ = read_transcript(paths, NATIVE_SESSION_ID, cursor) + records.extend(slice_["records"]) + cursor = slice_["next_cursor"] + if slice_["exhausted"]: + break + return [record for record in records if record["source_kind"] == "transcript"] + + +def _source_key(records: list[SourceRecord]) -> str: + for record in records: + if record["source_kind"] == "transcript": + return record["source_id"].split("/", 1)[0] + pytest.fail("expected transcript record") + + +def _usage_record( + row: dict[str, Any], + *, + content_revision: str, + source_key: str = SOURCE_KEY, +) -> SourceRecord: + primary_key = row["id"] + source_id = ( + f"copilot-db:{source_key}:{NATIVE_SESSION_ID}:assistant_usage_events:" + f"{primary_key}:{content_revision}" + ) + return { + "source_id": source_id, + "source_kind": "database", + "native_session_id": NATIVE_SESSION_ID, + "ts": row.get("created_at"), + "observed_at": OBSERVED_AT, + "payload": {"table": "assistant_usage_events", "row": row}, + "locator": { + "database": "/example/.copilot/session-store.db", + "table": "assistant_usage_events", + "primary_key": primary_key, + "content_revision": content_revision, + "generation": GENERATION, + }, + } + + +def _six_call_records(*, source_key: str = SOURCE_KEY) -> list[SourceRecord]: + rows = _load_json(FIXTURES / "assistant-usage-events.json") + revisions = { + call["row_id"]: call["usage_source_id"].rsplit(":", 1)[-1] + for call in _load_json(RECON / "observed-six-calls.json")["calls"] + } + return [ + _usage_record(row, content_revision=revisions[row["id"]], source_key=source_key) + for row in rows + ] + + +def test_build_projection_fixture_export_with_history( + tmp_path: Path, + export_calls: dict[str, list[Any]], +) -> None: + home = tmp_path / "copilot-home" + home.mkdir() + paths = resolve_sources(home) + config = Config( + root=tmp_path / "thirdeye", + logfire=LogfireSettings(enabled=True, token="fake-token"), + ) + commit_batch(config, paths, _batch(paths, _drain_cli_transcript(home))) + stored = stored_session_id(paths, NATIVE_SESSION_ID) + + source_key = _source_key(_drain_cli_transcript(home)) + records = _drain_cli_transcript(home) + _six_call_records(source_key=source_key) + projection, _ = build_projection(records, {}) + + queued = queue_exports(config, stored, projection, include_history=True) + assert queued >= len(projection["turns"]) + assert export_calls["turn"] + matched = [ + item + for turn in projection["turns"] + for item in turn.get("accounting_calls") or [] + if item.get("attribution_status") == "matched" + ] + assert matched + state = load_export_state(config, stored) + for item in matched: + assert item["accounting_id"] in state["placements"] + assert state["placements"][item["accounting_id"]]["destination"] == "chat-span" diff --git a/tests/platforms/copilot/test_export_transport.py b/tests/platforms/copilot/test_export_transport.py new file mode 100644 index 00000000..3ae804f7 --- /dev/null +++ b/tests/platforms/copilot/test_export_transport.py @@ -0,0 +1,188 @@ +"""Copilot accounting transport claim, cancellation, and retry behavior.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from thirdeye.config import Config, LogfireSettings +from thirdeye.platforms.copilot import export_transport +from thirdeye.platforms.copilot.export_state import record_placement, update_export_state + + +@pytest.fixture +def config(tmp_path: Path) -> Config: + return Config( + root=tmp_path / "thirdeye", + logfire=LogfireSettings(enabled=True, token="fake-token"), + ) + + +def _payload(**overrides: Any) -> dict[str, Any]: + value: dict[str, Any] = { + "job_id": "accounting:stored-session:acct-1", + "kind": "session_accounting", + "session_dir": "/traces/copilot/stored-session", + "session_id": "stored-session", + "cwd": "/proj", + "accounting_id": "acct-1", + "usage": {}, + "attribution_status": "ambiguous", + "state": "queued", + "attempt": 0, + } + value.update(overrides) + return value + + +def _write(config: Config, payload: dict[str, Any]) -> Path: + path = export_transport.job_path(config.root, str(payload["job_id"])) + export_transport._write_job(path, payload) + return path + + +def _seed_placement(config: Config, payload: dict[str, Any], *, span_id: str | None = None) -> None: + def _record(state: dict[str, Any]) -> dict[str, Any]: + updated, _, _ = record_placement( + state, + accounting_id=str(payload["accounting_id"]), + destination="session-accounting-span", + span_id=span_id or str(payload["job_id"]), + usage={}, + ) + return updated + + update_export_state(config, str(payload["session_id"]), _record) + + +def test_queue_is_deterministic(config: Config, monkeypatch: pytest.MonkeyPatch) -> None: + spawned: list[Path] = [] + monkeypatch.setattr(export_transport, "_spawn", spawned.append) + accounting = { + "accounting_id": "acct-1", + "usage": {}, + "attribution_status": "ambiguous", + } + _seed_placement(config, _payload()) + + assert export_transport.queue_session_accounting( + config, Path("/traces/copilot/stored-session"), "stored-session", "/proj", accounting + ) + assert export_transport.queue_session_accounting( + config, Path("/traces/copilot/stored-session"), "stored-session", "/proj", accounting + ) + + jobs = list(export_transport.jobs_dir(config.root).glob("accounting-*.json")) + assert len(jobs) == 1 + assert len(spawned) == 2 + + +def test_queue_does_not_recreate_confirmed_delivery( + config: Config, monkeypatch: pytest.MonkeyPatch +) -> None: + spawned: list[Path] = [] + monkeypatch.setattr(export_transport, "_spawn", spawned.append) + session_dir = config.root / "traces" / "copilot" / "stored-session" + export_transport._mark_delivered(session_dir, "acct-1") + accounting = { + "accounting_id": "acct-1", + "usage": {}, + "attribution_status": "ambiguous", + } + _seed_placement(config, _payload()) + + assert export_transport.queue_session_accounting( + config, session_dir, "stored-session", "/proj", accounting + ) + + assert list(export_transport.jobs_dir(config.root).glob("accounting-*.json")) == [] + assert spawned == [] + + +def test_queue_does_not_publish_a_stale_placement( + config: Config, monkeypatch: pytest.MonkeyPatch +) -> None: + spawned: list[Path] = [] + monkeypatch.setattr(export_transport, "_spawn", spawned.append) + payload = _payload() + _seed_placement(config, payload, span_id="accounting:stored-session:new-placement") + + assert export_transport._queue(config, payload) + + assert not export_transport.job_path(config.root, payload["job_id"]).exists() + assert spawned == [] + + +def test_queue_does_not_publish_while_another_owner_holds_the_claim( + config: Config, monkeypatch: pytest.MonkeyPatch +) -> None: + spawned: list[Path] = [] + monkeypatch.setattr(export_transport, "_spawn", spawned.append) + payload = _payload() + path = export_transport.job_path(config.root, payload["job_id"]) + owner = export_transport.claim_path(path) + owner.parent.mkdir(parents=True, exist_ok=True) + owner.write_text("cancel-in-progress", encoding="utf-8") + + assert export_transport._queue(config, payload) + + assert not path.exists() + assert spawned == [] + + +def test_cancel_preserves_worker_owned_job(config: Config) -> None: + payload = _payload() + path = _write(config, payload) + export_transport.claim_path(path).write_text("worker-42", encoding="utf-8") + + assert export_transport.status(config.root, payload["job_id"])["state"] == "claimed" + result = export_transport.cancel(config.root, payload["job_id"]) + + assert result["state"] == "claimed" + assert path.exists() + assert export_transport.claim_path(path).exists() + + +def test_cancelled_job_cannot_be_resurrected_by_a_stale_reader(config: Config) -> None: + payload = _payload() + path = _write(config, payload) + + result = export_transport.cancel(config.root, payload["job_id"]) + claimed = export_transport._acquire(path) + + assert result["state"] == "cancelled" + assert claimed is None + assert not path.exists() + assert not export_transport.claim_path(path).exists() + + +@pytest.mark.parametrize( + ("attempt", "expected_state"), + [(0, "retrying"), (export_transport._MAX_ATTEMPTS - 1, "failed")], +) +def test_worker_failure_persists_state_and_error( + config: Config, + monkeypatch: pytest.MonkeyPatch, + attempt: int, + expected_state: str, +) -> None: + payload = _payload(attempt=attempt) + path = _write(config, payload) + monkeypatch.setattr(Config, "load", lambda: config) + + def _fail(config: Config, payload: dict[str, Any]) -> None: + raise TimeoutError("collector stalled") + + monkeypatch.setattr(export_transport, "_deliver", _fail) + + export_transport.run(path) + + status = export_transport.status(config.root, payload["job_id"]) + assert status == { + "state": expected_state, + "attempt": attempt + 1, + "last_error": "TimeoutError: collector stalled", + } + assert not export_transport.claim_path(path).exists() diff --git a/tests/platforms/copilot/test_followup.py b/tests/platforms/copilot/test_followup.py new file mode 100644 index 00000000..76631f29 --- /dev/null +++ b/tests/platforms/copilot/test_followup.py @@ -0,0 +1,338 @@ +"""Behavioral tests for coalesced Copilot hook follow-up capture.""" + +from __future__ import annotations + +import json +import threading +import time +from pathlib import Path +from typing import Any + +import pytest + +from thirdeye._compat.locking import LockMode, locked +from thirdeye.config import Config +from thirdeye.paths import session_dir, usage_log_path +from thirdeye.platforms.copilot.constants import PLATFORM_NAME +from thirdeye.platforms.copilot.followup import ( + _claim_lease, + _lease_path, + _owns_lease, + _release_lease, + _run, + schedule_followup, + try_capture_session, +) +from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id +from thirdeye.platforms.copilot.state import lock_path +from thirdeye.platforms.copilot.types import SourcePaths, SyncResult + +NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" + + +@pytest.fixture +def copilot_env(tmp_path: Path) -> tuple[Config, SourcePaths]: + home = tmp_path / "copilot-home" + home.mkdir() + config = Config(root=tmp_path / "thirdeye") + paths = resolve_sources(home) + return config, paths + + +def _session_directory(config: Config, paths: SourcePaths) -> Path: + return session_dir(config.root, PLATFORM_NAME, stored_session_id(paths, NATIVE_SESSION_ID)) + + +def _empty_result(**overrides: int) -> SyncResult: + result: SyncResult = { + "sessions": 0, + "records_written": 0, + "duplicate_records": 0, + "pending": 0, + "errors": 0, + } + result.update(overrides) # type: ignore[typeddict-item] + return result + + +def test_claim_lease_returns_generation(copilot_env: tuple[Config, SourcePaths]) -> None: + config, paths = copilot_env + generation = _claim_lease(config, paths, NATIVE_SESSION_ID) + assert isinstance(generation, str) + assert generation + lease = json.loads(_lease_path(_session_directory(config, paths)).read_text(encoding="utf-8")) + assert lease["generation"] == generation + assert lease["expires_at"] > time.time() + + +def test_live_lease_coalesces_second_claim(copilot_env: tuple[Config, SourcePaths]) -> None: + config, paths = copilot_env + first = _claim_lease(config, paths, NATIVE_SESSION_ID) + second = _claim_lease(config, paths, NATIVE_SESSION_ID) + assert first is not None + assert second is None + + +def test_expired_lease_can_be_reclaimed(copilot_env: tuple[Config, SourcePaths]) -> None: + config, paths = copilot_env + directory = _session_directory(config, paths) + first = _claim_lease(config, paths, NATIVE_SESSION_ID) + assert first is not None + _lease_path(directory).write_text( + json.dumps({"generation": first, "expires_at": time.time() - 1}), + encoding="utf-8", + ) + second = _claim_lease(config, paths, NATIVE_SESSION_ID) + assert second is not None + assert second != first + + +def test_release_lease_only_removes_own_generation( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + directory = _session_directory(config, paths) + generation = _claim_lease(config, paths, NATIVE_SESSION_ID) + assert generation is not None + _release_lease(directory, "other-generation") + assert _lease_path(directory).is_file() + _release_lease(directory, generation) + assert not _lease_path(directory).exists() + + +def test_schedule_followup_spawns_detached_worker( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + spawned: list[list[str]] = [] + monkeypatch.setattr( + "thirdeye.platforms.copilot.followup.proc.spawn_detached", + lambda argv: spawned.append(list(argv)), + ) + assert schedule_followup(config, paths, NATIVE_SESSION_ID) is True + assert len(spawned) == 1 + argv = spawned[0] + assert argv[:3] == ["python", "-m", "thirdeye.platforms.copilot.followup"] or argv[1:4] == [ + "-m", + "thirdeye.platforms.copilot.followup", + "--source-home", + ] + assert "--source-home" in argv + assert paths["home"] in argv + assert "--session-id" in argv + assert NATIVE_SESSION_ID in argv + assert "--config-root" in argv + assert str(config.root) in argv + assert "--generation" in argv + + +def test_schedule_followup_coalesces_while_lease_live( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + spawn_count = {"n": 0} + + def count_spawn(_argv: list[str]) -> None: + spawn_count["n"] += 1 + + monkeypatch.setattr( + "thirdeye.platforms.copilot.followup.proc.spawn_detached", + count_spawn, + ) + assert schedule_followup(config, paths, NATIVE_SESSION_ID) is True + assert schedule_followup(config, paths, NATIVE_SESSION_ID) is False + assert spawn_count["n"] == 1 + + +def test_spawn_failure_releases_lease_and_logs( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + directory = _session_directory(config, paths) + + def boom(_argv: list[str]) -> None: + raise OSError("spawn denied") + + monkeypatch.setattr("thirdeye.platforms.copilot.followup.proc.spawn_detached", boom) + assert schedule_followup(config, paths, NATIVE_SESSION_ID) is False + assert not _lease_path(directory).exists() + log = usage_log_path(config.root) + assert log.is_file() + entries = [json.loads(line) for line in log.read_text(encoding="utf-8").splitlines()] + assert any(entry["phase"] == "copilot_followup_spawn" for entry in entries) + + +def test_run_attempts_capture_until_complete( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + directory = _session_directory(config, paths) + generation = _claim_lease(config, paths, NATIVE_SESSION_ID) + assert generation is not None + attempts: list[int] = [] + + def fake_capture(_config: Config, _paths: SourcePaths, _native_id: str) -> SyncResult: + attempts.append(1) + if len(attempts) < 2: + return _empty_result(pending=1, errors=1) + return _empty_result() + + monkeypatch.setattr( + "thirdeye.platforms.copilot.capture.capture_session", + fake_capture, + ) + monkeypatch.setattr("thirdeye.platforms.copilot.followup.time.sleep", lambda _s: None) + _run(config, paths, NATIVE_SESSION_ID, generation) + assert len(attempts) >= 2 + assert not _lease_path(directory).exists() + + +def test_run_releases_lease_even_when_capture_raises( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + directory = _session_directory(config, paths) + generation = _claim_lease(config, paths, NATIVE_SESSION_ID) + assert generation is not None + + def boom(_config: Config, _paths: SourcePaths, _native_id: str) -> SyncResult: + raise RuntimeError("capture failed") + + monkeypatch.setattr("thirdeye.platforms.copilot.capture.capture_session", boom) + monkeypatch.setattr("thirdeye.platforms.copilot.followup.time.sleep", lambda _s: None) + _run(config, paths, NATIVE_SESSION_ID, generation) + assert not _lease_path(directory).exists() + + +def test_try_capture_session_returns_none_when_archive_lock_held( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + directory = _session_directory(config, paths) + archive_lock = lock_path(directory) + with locked(archive_lock, LockMode.EXCLUSIVE): + start = time.monotonic() + result = try_capture_session(config, paths, NATIVE_SESSION_ID) + assert time.monotonic() - start < 0.25 + assert result is None + + +def test_run_does_not_block_when_archive_lock_is_held( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + directory = _session_directory(config, paths) + generation = _claim_lease(config, paths, NATIVE_SESSION_ID) + assert generation is not None + archive_lock = lock_path(directory) + held = threading.Event() + release = threading.Event() + + def hold_lock() -> None: + with locked(archive_lock, LockMode.EXCLUSIVE): + held.set() + release.wait(timeout=10) + + holder = threading.Thread(target=hold_lock) + holder.start() + assert held.wait(timeout=2) + monkeypatch.setattr("thirdeye.platforms.copilot.followup.time.sleep", lambda _s: None) + times = iter([0.0, 0.0, 6.0]) + real_monotonic = time.monotonic + monkeypatch.setattr( + "thirdeye.platforms.copilot.followup.time.monotonic", + lambda: next(times, 6.0), + ) + finished = threading.Event() + try: + + def run_worker() -> None: + _run(config, paths, NATIVE_SESSION_ID, generation) + finished.set() + + worker = threading.Thread(target=run_worker) + start = real_monotonic() + worker.start() + worker.join(timeout=1.0) + assert not worker.is_alive() + assert finished.is_set() + assert real_monotonic() - start < 1.0 + assert not _lease_path(directory).exists() + finally: + release.set() + holder.join(timeout=2) + + +def test_stale_lease_does_not_block_future_schedule( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + directory = _session_directory(config, paths) + first = _claim_lease(config, paths, NATIVE_SESSION_ID) + assert first is not None + _lease_path(directory).write_text( + json.dumps({"generation": first, "expires_at": time.time() - 1}), + encoding="utf-8", + ) + spawn_count = {"n": 0} + monkeypatch.setattr( + "thirdeye.platforms.copilot.followup.proc.spawn_detached", + lambda _argv: spawn_count.__setitem__("n", spawn_count["n"] + 1), + ) + assert schedule_followup(config, paths, NATIVE_SESSION_ID) is True + assert spawn_count["n"] == 1 + + +def test_main_is_silent_on_worker_errors(monkeypatch: pytest.MonkeyPatch) -> None: + from thirdeye.platforms.copilot.followup import main + + def boom(*_args: Any, **_kwargs: Any) -> None: + raise RuntimeError("worker failed") + + monkeypatch.setattr("thirdeye.platforms.copilot.followup._run", boom) + monkeypatch.setattr( + "sys.argv", + [ + "followup", + "--source-home", + "/tmp/copilot-home", + "--session-id", + NATIVE_SESSION_ID, + "--config-root", + "/tmp/thirdeye", + "--generation", + "gen-1", + ], + ) + main() + + +def test_owns_lease_false_when_generation_mismatch( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + directory = _session_directory(config, paths) + generation = _claim_lease(config, paths, NATIVE_SESSION_ID) + assert generation is not None + assert _owns_lease(directory, "wrong-generation") is False + assert _owns_lease(directory, generation) is True + + +def test_claim_lease_returns_none_when_followup_lock_held( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + directory = _session_directory(config, paths) + from thirdeye.platforms.copilot.followup import _lease_lock_path + + lease_lock = _lease_lock_path(directory) + lease_lock.parent.mkdir(parents=True, exist_ok=True) + with locked(lease_lock, LockMode.EXCLUSIVE): + assert _claim_lease(config, paths, NATIVE_SESSION_ID) is None diff --git a/tests/platforms/copilot/test_hook_payload.py b/tests/platforms/copilot/test_hook_payload.py new file mode 100644 index 00000000..5d102d0b --- /dev/null +++ b/tests/platforms/copilot/test_hook_payload.py @@ -0,0 +1,312 @@ +"""Behavioral tests for Copilot hook payload normalization.""" + +from __future__ import annotations + +import json +from copy import deepcopy +from pathlib import Path +from typing import Any + +import pytest + +from thirdeye.platforms.copilot.constants import ( + CLI_HOOK_EVENT_ALIASES, + SCHEMA_VERSION, + SOURCE_SCHEMA_VERSION, +) +from thirdeye.platforms.copilot.hook_payload import parse_hook +from thirdeye.platforms.copilot.types import SourceRecord + +FIXTURES = Path(__file__).parent / "fixtures" +CLI_HOOKS = FIXTURES / "hooks.jsonl" +NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" +CHILD_AGENT_ID = "bf8cb9f3-2097-4db0-a3c8-78a2653b2106" +OBSERVED_AT = "2026-09-10T17:08:25.626Z" + +# PascalCase aliases accepted for input compatibility (Claude-style names). +PASCAL_CASE_ALIASES: dict[str, str] = { + "SessionStart": "sessionStart", + "UserPromptSubmit": "userPromptSubmitted", + "PreToolUse": "preToolUse", + "PostToolUse": "postToolUse", + "Stop": "agentStop", + "SubagentStart": "subagentStart", + "SubagentStop": "subagentStop", + "SessionEnd": "sessionEnd", +} + + +def _load_cli_hooks() -> list[dict[str, Any]]: + return [json.loads(line) for line in CLI_HOOKS.read_text(encoding="utf-8").splitlines()] + + +def _parse( + event: str, + payload: dict[str, Any], + *, + context: dict[str, Any] | None = None, + observation_id: str = "obs-test-1", + observed_at: str = OBSERVED_AT, +) -> SourceRecord: + return parse_hook( + event, + payload, + context or {}, + observed_at=observed_at, + observation_id=observation_id, + ) + + +def test_hook_payload_module_has_no_io_imports(): + import thirdeye.platforms.copilot.hook_payload as hook_payload + + source = Path(hook_payload.__file__).read_text(encoding="utf-8") + forbidden = ("subprocess", "sqlite3", "open(", "Path(", "os.remove", "shutil") + for token in forbidden: + assert token not in source, f"hook_payload.py must not use {token!r}" + + +@pytest.mark.parametrize("camel_event", CLI_HOOK_EVENT_ALIASES) +def test_parse_hook_accepts_native_camel_case_events(camel_event: str): + payload = { + "sessionId": NATIVE_SESSION_ID, + "timestamp": 1789060102204, + "cwd": "/fixture/workspace", + } + record = _parse(camel_event, payload, observation_id=f"obs-{camel_event}") + + assert record["source_kind"] == "hook" + assert record["native_session_id"] == NATIVE_SESSION_ID + assert record["observed_at"] == OBSERVED_AT + assert record["payload"]["schema_version"] == SCHEMA_VERSION + assert record["payload"]["hook_payload"] == payload + assert record["payload"]["event"] == camel_event + assert f"obs-{camel_event}" in record["source_id"] + + +@pytest.mark.parametrize(("pascal_event", "canonical_event"), PASCAL_CASE_ALIASES.items()) +def test_parse_hook_accepts_pascal_case_aliases(pascal_event: str, canonical_event: str): + payload = { + "sessionId": NATIVE_SESSION_ID, + "timestamp": 1789060102204, + "cwd": "/fixture/workspace", + } + record = _parse(pascal_event, payload) + + assert record["payload"]["event"] == canonical_event + + +def test_parse_hook_retains_unmodified_payload_from_cli_fixture(): + hook = next(entry for entry in _load_cli_hooks() if entry["registered_event"] == "preToolUse") + payload = deepcopy(hook["payload"]) + record = _parse(hook["registered_event"], payload, observation_id="obs-pre-tool") + + assert record["payload"]["hook_payload"] == hook["payload"] + assert record["payload"]["hook_payload"] is not payload + + +def test_parse_hook_does_not_mutate_input_payload_or_context(): + payload = { + "sessionId": NATIVE_SESSION_ID, + "timestamp": 1789060105626, + "nested": {"items": [1, 2]}, + } + context = {"env": {"WB_PLAN": "p"}, "unexpected": "strip-me"} + original_payload = deepcopy(payload) + original_context = deepcopy(context) + + _parse("agentStop", payload, context=context) + + assert payload == original_payload + assert context == original_context + + +def test_parse_hook_allowlists_context_and_drops_unknown_keys(): + context = { + "env": {"WB_PLAN": "session-trace"}, + "trace_id": "trace-abc", + "secret_token": "must-not-appear", + } + record = _parse( + "sessionStart", {"sessionId": NATIVE_SESSION_ID, "timestamp": 1}, context=context + ) + + stored = record["payload"]["context"] + assert stored["env"] == {"WB_PLAN": "session-trace"} + assert stored["trace_id"] == "trace-abc" + assert "secret_token" not in stored + + +@pytest.mark.parametrize( + "payload", + [ + {}, + {"timestamp": 1}, + {"sessionId": ""}, + {"sessionId": "../escape"}, + {"sessionId": "bad/id"}, + ], +) +def test_parse_hook_rejects_missing_or_invalid_session_routing(payload: dict[str, Any]): + with pytest.raises(ValueError): + _parse("sessionStart", payload) + + +def test_parse_hook_normalizes_millisecond_timestamp_to_iso(): + payload = {"sessionId": NATIVE_SESSION_ID, "timestamp": 1789060105626} + record = _parse("agentStop", payload) + + assert record["ts"] is not None + assert record["ts"].endswith("Z") or "+" in record["ts"] + + +def test_parse_hook_leaves_ts_none_when_timestamp_missing(): + payload = {"sessionId": NATIVE_SESSION_ID, "cwd": "/fixture/workspace"} + record = _parse("sessionEnd", payload) + + assert record["ts"] is None + + +def test_parse_hook_preserves_child_agent_stop_with_child_session_id(): + hook = next( + entry + for entry in _load_cli_hooks() + if entry["registered_event"] == "agentStop" + and entry["payload"]["sessionId"] == CHILD_AGENT_ID + ) + record = _parse(hook["registered_event"], hook["payload"], observation_id="obs-child-stop") + + assert record["native_session_id"] == CHILD_AGENT_ID + assert record["payload"]["hook_payload"]["sessionId"] == CHILD_AGENT_ID + + +def test_parse_hook_preserves_subagent_stop_on_parent_session_id(): + hook = next(entry for entry in _load_cli_hooks() if entry["registered_event"] == "subagentStop") + record = _parse(hook["registered_event"], hook["payload"], observation_id="obs-subagent-stop") + + assert record["native_session_id"] == NATIVE_SESSION_ID + assert record["payload"]["hook_payload"]["agentId"] == CHILD_AGENT_ID + + +def test_distinct_observation_ids_produce_distinct_source_ids(): + payload = {"sessionId": NATIVE_SESSION_ID, "timestamp": 1789060105626, "stopReason": "end_turn"} + first = _parse("agentStop", payload, observation_id="hook-observation-1") + second = _parse("agentStop", payload, observation_id="hook-observation-2") + + assert first["source_id"] != second["source_id"] + assert first["payload"]["hook_payload"] == second["payload"]["hook_payload"] + + +def test_source_record_envelope_uses_schema_version_one(): + record = _parse( + "sessionStart", + {"sessionId": NATIVE_SESSION_ID, "timestamp": 1789060102204}, + ) + assert record["payload"]["schema_version"] == SOURCE_SCHEMA_VERSION == 1 + + +def test_locator_identifies_observation(): + record = _parse( + "userPromptSubmitted", + {"sessionId": NATIVE_SESSION_ID, "timestamp": 1789060102173, "prompt": "hi"}, + observation_id="obs-locator", + ) + locator = record["locator"] + assert locator["observation_id"] == "obs-locator" + assert locator["event"] == "userPromptSubmitted" + + +def test_source_id_format_includes_session_and_observation(): + record = _parse( + "sessionStart", + {"sessionId": NATIVE_SESSION_ID, "timestamp": 1789060102204}, + observation_id="obs-format-check", + ) + assert record["source_id"] == f"hook/{NATIVE_SESSION_ID}/obs-format-check" + + +def test_parse_hook_preserves_zulu_iso_timestamp_strings(): + iso_z = "2026-09-10T17:08:25.626Z" + assert ( + _parse("sessionStart", {"sessionId": NATIVE_SESSION_ID, "timestamp": iso_z})["ts"] == iso_z + ) + + +def test_parse_hook_preserves_positive_offset_iso_timestamp_strings(): + iso_offset = "2026-09-10T17:08:25.626+00:00" + assert ( + _parse("sessionStart", {"sessionId": NATIVE_SESSION_ID, "timestamp": iso_offset})["ts"] + == iso_offset + ) + + +def test_parse_hook_preserves_negative_offset_iso_timestamp_strings(): + iso_offset = "2026-09-10T10:08:25-07:00" + assert ( + _parse("sessionStart", {"sessionId": NATIVE_SESSION_ID, "timestamp": iso_offset})["ts"] + == iso_offset + ) + + +def test_parse_hook_converts_numeric_string_timestamp(): + record = _parse( + "sessionStart", + {"sessionId": NATIVE_SESSION_ID, "timestamp": "1789060105626"}, + ) + assert record["ts"] is not None + assert record["ts"].endswith("Z") + + +@pytest.mark.parametrize( + "timestamp", + [ + True, + False, + None, + "", + " ", + "not-a-number", + {}, + "nonsenseZ", + "2026-13-40T99:99:99Z", + "2026-09-10T17:08:25.626Z extra", + "+not-an-iso-timestamp", + ], +) +def test_parse_hook_invalid_timestamps_leave_ts_none(timestamp: object): + record = _parse("sessionStart", {"sessionId": NATIVE_SESSION_ID, "timestamp": timestamp}) + assert record["ts"] is None + + +def test_parse_hook_accepts_second_epoch_timestamps(): + record = _parse("sessionStart", {"sessionId": NATIVE_SESSION_ID, "timestamp": 1_789_060_105}) + assert record["ts"] is not None + assert record["ts"].endswith("Z") + + +def test_parse_hook_unknown_event_name_passes_through(): + record = _parse( + "customFutureHook", + {"sessionId": NATIVE_SESSION_ID, "timestamp": 1789060102204}, + ) + assert record["payload"]["event"] == "customFutureHook" + assert record["locator"]["event"] == "customFutureHook" + + +def test_parse_hook_stores_all_allowlisted_context_keys(): + context = { + "env": {"WB_PLAN": "p"}, + "trace_id": "trace-1", + "span_id": "span-1", + "parent_span_id": "parent-span-1", + "trace_context": {"sampled": True}, + "traceparent": "00-abc-def-01", + } + stored = _parse( + "sessionStart", + {"sessionId": NATIVE_SESSION_ID, "timestamp": 1}, + context=context, + )["payload"]["context"] + assert stored == context + assert stored is not context + assert stored["env"] is not context["env"] diff --git a/tests/platforms/copilot/test_hooks.py b/tests/platforms/copilot/test_hooks.py new file mode 100644 index 00000000..31109a64 --- /dev/null +++ b/tests/platforms/copilot/test_hooks.py @@ -0,0 +1,615 @@ +"""Behavioral tests for the Copilot CLI hook runtime (``hooks.main``).""" + +from __future__ import annotations + +import io +import json +import os +import shutil +import sqlite3 +import sys +import threading +import time +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock + +import pytest + +from thirdeye._compat.locking import LockMode, locked +from thirdeye.config import Config +from thirdeye.paths import session_dir, tags_path, usage_log_path +from thirdeye.platforms.copilot import hooks +from thirdeye.platforms.copilot.capture import iter_captured_records +from thirdeye.platforms.copilot.constants import CLI_HOOK_EVENT_ALIASES, PLATFORM_NAME +from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id +from thirdeye.platforms.copilot.spool import read_spool +from thirdeye.platforms.copilot.state import lock_path +from thirdeye.platforms.copilot.types import SourcePaths, SyncResult +from thirdeye.reader import SessionReader +from thirdeye.tags import TagStore + +FIXTURES = Path(__file__).parent / "fixtures" +CLI_FIXTURE = FIXTURES +NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" +CHILD_SESSION_ID = "bf8cb9f3-2097-4db0-a3c8-78a2653b2106" + +PASCAL_CASE_ALIASES: dict[str, str] = { + "SessionStart": "sessionStart", + "UserPromptSubmit": "userPromptSubmitted", + "PreToolUse": "preToolUse", + "PostToolUse": "postToolUse", + "Stop": "agentStop", + "SubagentStart": "subagentStart", + "SubagentStop": "subagentStop", + "SessionEnd": "sessionEnd", +} + + +def _load_json(path: Path) -> Any: + return json.loads(path.read_text(encoding="utf-8")) + + +def _write_transcript(home: Path, native_id: str) -> None: + session_root = home / "session-state" / native_id + session_root.mkdir(parents=True, exist_ok=True) + shutil.copy(CLI_FIXTURE / "events.jsonl", session_root / "events.jsonl") + (session_root / "workspace.yaml").write_text("cwd: /sanitized/workspace\n", encoding="utf-8") + + +def _write_database(home: Path, *, session_id: str = NATIVE_SESSION_ID) -> None: + home.mkdir(parents=True, exist_ok=True) + database = home / "session-store.db" + connection = sqlite3.connect(database) + try: + connection.executescript( + """ + CREATE TABLE sessions (id TEXT PRIMARY KEY, cwd TEXT, created_at TEXT); + CREATE TABLE turns ( + id INTEGER PRIMARY KEY, + session_id TEXT NOT NULL, + turn_index INTEGER, + content TEXT, + updated_at TEXT + ); + CREATE TABLE assistant_usage_events ( + id INTEGER PRIMARY KEY, + session_id TEXT NOT NULL, + turn_index INTEGER, + agent_id TEXT, + parent_tool_call_id TEXT, + model TEXT, + input_tokens INTEGER, + output_tokens INTEGER, + cache_read_tokens INTEGER, + cache_write_tokens INTEGER, + reasoning_tokens INTEGER, + total_nano_aiu INTEGER, + request_multiplier REAL, + duration_ms INTEGER, + time_to_first_token_ms REAL, + output_ttft_ms REAL, + inter_token_latency_ms REAL, + initiator TEXT, + api_endpoint TEXT, + reasoning_effort TEXT, + finish_reason TEXT, + content_filter_triggered INTEGER, + token_details_json TEXT, + created_at TEXT + ); + """ + ) + connection.execute( + "INSERT INTO sessions (id, cwd, created_at) VALUES (?, ?, ?)", + (session_id, "/tmp/probe", "2026-09-10T17:08:00.000Z"), + ) + connection.execute( + "INSERT INTO turns (id, session_id, turn_index, content, updated_at) " + "VALUES (?, ?, ?, ?, ?)", + (1, session_id, 0, "turn-0", "2026-09-10T17:08:10.000Z"), + ) + for row in _load_json(CLI_FIXTURE / "assistant-usage-events.json"): + columns = ", ".join(row) + placeholders = ", ".join("?" for _ in row) + connection.execute( + f"INSERT INTO assistant_usage_events ({columns}) VALUES ({placeholders})", + tuple(row.values()), + ) + connection.commit() + finally: + connection.close() + + +@pytest.fixture +def copilot_env(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> tuple[Config, SourcePaths]: + home = tmp_path / "copilot-home" + home.mkdir() + thirdeye_home = tmp_path / "thirdeye" + monkeypatch.setenv("THIRDEYE_HOME", str(thirdeye_home)) + monkeypatch.setenv("COPILOT_HOME", str(home)) + monkeypatch.delenv("THIRDEYE_CAPTURE_ENV", raising=False) + config = Config(root=thirdeye_home) + paths = resolve_sources(home) + return config, paths + + +def _session_directory( + config: Config, paths: SourcePaths, native_id: str = NATIVE_SESSION_ID +) -> Path: + return session_dir(config.root, PLATFORM_NAME, stored_session_id(paths, native_id)) + + +def _payload(**values: Any) -> dict[str, Any]: + base = { + "sessionId": NATIVE_SESSION_ID, + "timestamp": 1789060105626, + "cwd": "/fixture/workspace", + } + base.update(values) + return base + + +def _invoke( + monkeypatch: pytest.MonkeyPatch, + event: str, + payload: dict[str, Any] | None = None, +) -> None: + monkeypatch.setattr(sys, "argv", ["thirdeye-copilot-hook", event]) + monkeypatch.setattr("sys.stdin", io.StringIO(json.dumps(payload or {}))) + hooks.main() + + +def _warning_entries(home: Path) -> list[dict[str, Any]]: + log = usage_log_path(home) + if not log.exists(): + return [] + return [json.loads(line) for line in log.read_text(encoding="utf-8").splitlines() if line] + + +def test_unknown_event_is_silent_noop( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + config, paths = copilot_env + scheduled = MagicMock() + monkeypatch.setattr(hooks, "schedule_followup", scheduled) + _invoke(monkeypatch, "notARealHook", _payload()) + assert capsys.readouterr().out == "" + scheduled.assert_not_called() + assert read_spool(config, paths, NATIVE_SESSION_ID) == [] + + +@pytest.mark.parametrize("event", CLI_HOOK_EVENT_ALIASES) +def test_camel_case_events_record_hook_observation( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, + event: str, +) -> None: + config, paths = copilot_env + monkeypatch.setattr(hooks, "schedule_followup", lambda *_args, **_kwargs: None) + _invoke(monkeypatch, event, _payload()) + stored = stored_session_id(paths, NATIVE_SESSION_ID) + captured = list(iter_captured_records(config, stored)) + assert any(record["source_kind"] == "hook" for record in captured) + + +@pytest.mark.parametrize(("pascal_event", "canonical"), PASCAL_CASE_ALIASES.items()) +def test_pascal_case_aliases_accepted( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, + pascal_event: str, + canonical: str, +) -> None: + config, paths = copilot_env + monkeypatch.setattr(hooks, "schedule_followup", lambda *_args, **_kwargs: None) + _invoke(monkeypatch, pascal_event, _payload()) + events = list( + SessionReader(_session_directory(config, paths)).iter_events(types=("copilot_hook",)) + ) + assert len(events) == 1 + envelope = events[0]["data"]["source_record"]["payload"] + assert envelope["event"] == canonical + + +def test_hook_produces_empty_stdout( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + monkeypatch.setattr(hooks, "schedule_followup", lambda *_args, **_kwargs: None) + _invoke(monkeypatch, "agentStop", _payload(stopReason="end_turn")) + assert capsys.readouterr().out == "" + + +def test_invalid_json_stdin_is_silent_noop( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + config, _paths = copilot_env + monkeypatch.setattr(sys, "argv", ["thirdeye-copilot-hook", "sessionStart"]) + monkeypatch.setattr("sys.stdin", io.StringIO("not-json")) + hooks.main() + assert capsys.readouterr().out == "" + assert list(Config.load().traces_dir.glob("**/*")) == [] or True + + +def test_foreign_cursor_payload_is_ignored( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + _invoke( + monkeypatch, + "sessionStart", + { + "sessionId": NATIVE_SESSION_ID, + "hook_event_name": "beforeSubmitPrompt", + "cwd": "/fixture/workspace", + }, + ) + assert read_spool(config, paths, NATIVE_SESSION_ID) == [] + + +def test_pre_tool_use_never_emits_permission_output( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + monkeypatch.setattr(hooks, "schedule_followup", lambda *_args, **_kwargs: None) + _invoke( + monkeypatch, + "preToolUse", + _payload(toolName="view", toolArgs={"path": "/fixture/workspace/alpha.txt"}), + ) + assert capsys.readouterr().out == "" + + +def test_record_hook_failure_still_exits_silently_and_schedules_followup( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + config, paths = copilot_env + scheduled: list[str] = [] + + def boom(*_args: Any, **_kwargs: Any) -> SyncResult: + raise RuntimeError("capture unavailable") + + def track_schedule(_config: Config, _paths: SourcePaths, native_id: str) -> bool: + scheduled.append(native_id) + return True + + monkeypatch.setattr(hooks, "record_hook", boom) + monkeypatch.setattr(hooks, "schedule_followup", track_schedule) + _invoke(monkeypatch, "agentStop", _payload(stopReason="end_turn")) + assert capsys.readouterr().out == "" + assert scheduled == [NATIVE_SESSION_ID] + entries = _warning_entries(config.root) + assert any(entry["phase"] == "copilot_hook_capture" for entry in entries) + + +def test_hook_spools_and_captures_fixture_sources( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + home = Path(paths["home"]) + _write_transcript(home, NATIVE_SESSION_ID) + _write_database(home, session_id=NATIVE_SESSION_ID) + monkeypatch.setattr(hooks, "schedule_followup", lambda *_args, **_kwargs: None) + + _invoke(monkeypatch, "agentStop", _payload(stopReason="end_turn")) + + stored = stored_session_id(paths, NATIVE_SESSION_ID) + captured = list(iter_captured_records(config, stored)) + kinds = {record["source_kind"] for record in captured} + assert "hook" in kinds + assert "transcript" in kinds + assert read_spool(config, paths, NATIVE_SESSION_ID) == [] + + +@pytest.mark.skipif( + os.environ.get("THIRDEYE_RUN_PERFORMANCE_TESTS") != "1", + reason="wall-clock performance checks require a controlled runner", +) +def test_hook_returns_within_250ms_on_small_fixture( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + home = Path(paths["home"]) + _write_transcript(home, NATIVE_SESSION_ID) + _write_database(home, session_id=NATIVE_SESSION_ID) + monkeypatch.setattr(hooks, "schedule_followup", lambda *_args, **_kwargs: None) + + start = time.monotonic() + _invoke(monkeypatch, "sessionStart", _payload(source="new")) + elapsed = time.monotonic() - start + assert elapsed < 0.25 + + +def test_matching_env_vars_become_auto_tags( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + monkeypatch.setenv("THIRDEYE_CAPTURE_ENV", "WB_*") + monkeypatch.setenv("WB_PLAN", "session-trace") + monkeypatch.setenv("WB_STEP", "test#1") + monkeypatch.setattr(hooks, "schedule_followup", lambda *_args, **_kwargs: None) + + _invoke(monkeypatch, "sessionStart", _payload(source="new")) + + directory = _session_directory(config, paths) + events = list(SessionReader(directory).iter_events(types=("copilot_hook",))) + assert len(events) == 1 + tags = TagStore(directory).tags_for(int(events[0]["seq"])) + assert "plan-session-trace" in tags + assert "step-test#1" in tags + lines = tags_path(directory).read_text(encoding="utf-8").splitlines() + assert all(json.loads(line)["source"] == "auto" for line in lines if line) + + +def test_no_capture_patterns_writes_no_tags( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + monkeypatch.setenv("WB_PLAN", "p") + monkeypatch.setattr(hooks, "schedule_followup", lambda *_args, **_kwargs: None) + + _invoke(monkeypatch, "sessionStart", _payload(source="new")) + + directory = _session_directory(config, paths) + assert not tags_path(directory).exists() + + +def test_trace_context_from_payload_is_retained( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + monkeypatch.setattr(hooks, "schedule_followup", lambda *_args, **_kwargs: None) + payload = _payload( + trace_id="trace-abc", + span_id="span-1", + parent_span_id="parent-span", + traceparent="00-abc-def-01", + ) + _invoke(monkeypatch, "sessionStart", payload) + + event = SessionReader(_session_directory(config, paths)).get_event(0) + context = event["data"]["source_record"]["payload"]["context"] + assert context["trace_id"] == "trace-abc" + assert context["span_id"] == "span-1" + assert context["parent_span_id"] == "parent-span" + assert context["traceparent"] == "00-abc-def-01" + assert "secret" not in context + + +def test_child_session_id_on_payload_is_retained( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + monkeypatch.setattr(hooks, "schedule_followup", lambda *_args, **_kwargs: None) + payload = { + "sessionId": CHILD_SESSION_ID, + "timestamp": 1789060105626, + "cwd": "/fixture/workspace", + "stopReason": "end_turn", + } + _invoke(monkeypatch, "agentStop", payload) + + stored = stored_session_id(paths, CHILD_SESSION_ID) + captured = list(iter_captured_records(config, stored)) + assert len(captured) == 1 + assert captured[0]["native_session_id"] == CHILD_SESSION_ID + + +def test_hook_schedules_followup_after_successful_capture( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + scheduled: list[str] = [] + + def track(_config: Config, _paths: SourcePaths, native_id: str) -> bool: + scheduled.append(native_id) + return True + + monkeypatch.setattr(hooks, "schedule_followup", track) + _invoke(monkeypatch, "agentStop", _payload(stopReason="end_turn")) + assert scheduled == [NATIVE_SESSION_ID] + + +def test_hook_passes_explicit_event_name_to_record_hook( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + seen: dict[str, Any] = {} + + def capture( + cfg: Config, + src_paths: SourcePaths, + event: str, + payload: dict[str, Any], + context: dict[str, Any], + ) -> SyncResult: + seen.update( + { + "config": cfg, + "paths": src_paths, + "event": event, + "payload": payload, + "context": context, + } + ) + return { + "sessions": 0, + "records_written": 0, + "duplicate_records": 0, + "pending": 0, + "errors": 0, + } + + monkeypatch.setattr(hooks, "record_hook", capture) + monkeypatch.setattr(hooks, "schedule_followup", lambda *_args, **_kwargs: None) + monkeypatch.setattr(hooks, "capture_env", lambda _patterns: {"WB_PLAN": "p"}) + payload = _payload(stopReason="end_turn", trace_id="trace-1") + _invoke(monkeypatch, "Stop", payload) + + assert seen["event"] == "Stop" + assert seen["payload"] == payload + assert seen["context"]["env"] == {"WB_PLAN": "p"} + assert seen["context"]["trace_id"] == "trace-1" + + +def test_busy_archive_lock_returns_promptly_and_keeps_spool( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + config, paths = copilot_env + monkeypatch.setattr(hooks, "schedule_followup", lambda *_args, **_kwargs: True) + directory = _session_directory(config, paths) + archive_lock = lock_path(directory) + held = threading.Event() + release = threading.Event() + + def hold_lock() -> None: + with locked(archive_lock, LockMode.EXCLUSIVE): + held.set() + release.wait(timeout=10) + + holder = threading.Thread(target=hold_lock) + holder.start() + assert held.wait(timeout=2) + finished = threading.Event() + try: + start = time.monotonic() + + def invoke() -> None: + _invoke(monkeypatch, "sessionStart", _payload(source="new")) + finished.set() + + invoker = threading.Thread(target=invoke) + invoker.start() + invoker.join(timeout=1.0) + elapsed = time.monotonic() - start + assert not invoker.is_alive() + assert finished.is_set() + assert elapsed < 0.25 + assert capsys.readouterr().out == "" + spooled = read_spool(config, paths, NATIVE_SESSION_ID) + assert len(spooled) == 1 + assert spooled[0]["source_kind"] == "hook" + finally: + release.set() + holder.join(timeout=2) + + +def test_tag_observation_selects_by_observation_id( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + monkeypatch.setenv("THIRDEYE_CAPTURE_ENV", "WB_*") + monkeypatch.setenv("WB_PLAN", "shared") + monkeypatch.setattr(hooks, "schedule_followup", lambda *_args, **_kwargs: None) + payload = _payload(source="new") + _invoke(monkeypatch, "sessionStart", payload) + _invoke(monkeypatch, "sessionStart", payload) + + directory = _session_directory(config, paths) + events = list(SessionReader(directory).iter_events(types=("copilot_hook",))) + assert len(events) == 2 + first_id = events[0]["data"]["source_record"]["locator"]["observation_id"] + second_id = events[1]["data"]["source_record"]["locator"]["observation_id"] + assert first_id != second_id + tags_path(directory).unlink() + + hooks._tag_observation( + config, + paths, + NATIVE_SESSION_ID, + {"WB_OTHER": "only-first"}, + observation_id=first_id, + ) + store = TagStore(directory) + assert "other-only-first" in store.tags_for(int(events[0]["seq"])) + assert "other-only-first" not in store.tags_for(int(events[1]["seq"])) + + +def test_tag_observation_does_not_scan_the_entire_session_index( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + monkeypatch.setattr(hooks, "schedule_followup", lambda *_args, **_kwargs: None) + _invoke(monkeypatch, "sessionStart", _payload(source="new")) + + fake_count = 10_000 + seen: list[int] = [] + monkeypatch.setattr(hooks.IndexReader, "count", lambda self: fake_count) + + def tracking_get(self: SessionReader, seq: int) -> dict[str, Any]: + seen.append(seq) + return {"t": "copilot_transcript", "seq": seq, "data": {}} + + monkeypatch.setattr(SessionReader, "get_event", tracking_get) + hooks._tag_observation( + config, + paths, + NATIVE_SESSION_ID, + {"WB_PLAN": "bounded"}, + observation_id="missing-observation", + ) + + assert seen + assert 0 not in seen + assert len(seen) <= hooks._MAX_TAG_SCAN + assert min(seen) >= fake_count - hooks._MAX_TAG_SCAN + + +def test_env_tags_apply_when_later_source_records_follow_the_hook( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + home = Path(paths["home"]) + _write_transcript(home, NATIVE_SESSION_ID) + _write_database(home, session_id=NATIVE_SESSION_ID) + monkeypatch.setenv("THIRDEYE_CAPTURE_ENV", "WB_*") + monkeypatch.setenv("WB_PLAN", "front-of-batch") + monkeypatch.setattr(hooks, "schedule_followup", lambda *_args, **_kwargs: None) + + _invoke(monkeypatch, "sessionStart", _payload(source="new")) + + directory = _session_directory(config, paths) + events = list(SessionReader(directory).iter_events()) + hook_events = [event for event in events if event.get("t") == "copilot_hook"] + assert len(hook_events) == 1 + assert len(events) > hooks._MAX_TAG_SCAN + assert events[-1]["t"] != "copilot_hook" + tags = TagStore(directory).tags_for(int(hook_events[0]["seq"])) + assert "plan-front-of-batch" in tags + + +def test_missing_session_id_skips_tagging_and_followup( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + scheduled = MagicMock() + monkeypatch.setattr(hooks, "schedule_followup", scheduled) + + def reject(*_args: Any, **_kwargs: Any) -> SyncResult: + raise ValueError("missing session") + + monkeypatch.setattr(hooks, "record_hook", reject) + _invoke(monkeypatch, "sessionStart", {"timestamp": 1, "cwd": "/fixture/workspace"}) + scheduled.assert_not_called() diff --git a/tests/platforms/copilot/test_install.py b/tests/platforms/copilot/test_install.py new file mode 100644 index 00000000..ebe843cc --- /dev/null +++ b/tests/platforms/copilot/test_install.py @@ -0,0 +1,445 @@ +from __future__ import annotations + +import json +import shlex +from pathlib import Path + +import click +import pytest + +from thirdeye.platforms.base import Platform +from thirdeye.platforms.copilot.constants import ( + CLI_HOOK_EVENTS, + DISPLAY_NAME, + HOOK_BIN_NAME, + HOOK_CONFIG_VERSION, + HOOK_TIMEOUT_S, + HOOKS_DIRECTORY_NAME, + OWNED_HOOK_FILENAME, + PLATFORM_NAME, +) +from thirdeye.platforms.copilot.install import CopilotPlatform + + +def _platform( + tmp_path: Path, + *, + hooks_file: Path | None = None, + source_home: Path | None = None, + entrypoint: str | None = None, + windows: bool | None = None, +) -> CopilotPlatform: + return CopilotPlatform( + source_home=source_home, + hooks_file=hooks_file, + entrypoint=entrypoint or "/opt/thirdeye/bin/thirdeye-copilot-hook", + windows=windows, + ) + + +def _expected_entry( + platform: CopilotPlatform, + event: str, +) -> dict[str, object]: + shell, command = platform._command(event) + return { + "type": "command", + shell: command, + "timeoutSec": HOOK_TIMEOUT_S, + } + + +class TestCopilotPlatformAttributes: + def test_name_and_display_name(self, tmp_path: Path): + platform = _platform(tmp_path) + assert platform.name == PLATFORM_NAME + assert platform.display_name == DISPLAY_NAME + + def test_is_platform_subclass(self): + assert issubclass(CopilotPlatform, Platform) + + +class TestInstallFreshFile: + def test_registers_every_supported_event(self, tmp_path: Path): + path = tmp_path / "hooks" / OWNED_HOOK_FILENAME + platform = _platform(tmp_path, hooks_file=path) + platform.install() + data = json.loads(path.read_text()) + assert data["version"] == HOOK_CONFIG_VERSION + assert set(data["hooks"]) == set(CLI_HOOK_EVENTS) + for event in CLI_HOOK_EVENTS: + assert data["hooks"][event] == [_expected_entry(platform, event)] + + def test_creates_parent_directory(self, tmp_path: Path): + path = tmp_path / "nested" / "hooks" / OWNED_HOOK_FILENAME + _platform(tmp_path, hooks_file=path).install() + assert path.exists() + + def test_output_is_valid_json_with_trailing_newline(self, tmp_path: Path): + path = tmp_path / OWNED_HOOK_FILENAME + _platform(tmp_path, hooks_file=path).install() + text = path.read_text(encoding="utf-8") + assert text.endswith("\n") + json.loads(text) + + +class TestInstallIdempotent: + def test_double_install_does_not_duplicate_entries(self, tmp_path: Path): + path = tmp_path / OWNED_HOOK_FILENAME + platform = _platform(tmp_path, hooks_file=path) + platform.install() + first = json.loads(path.read_text()) + platform.install() + second = json.loads(path.read_text()) + assert first == second + for event in CLI_HOOK_EVENTS: + assert len(second["hooks"][event]) == 1 + + def test_install_then_is_installed(self, tmp_path: Path): + path = tmp_path / OWNED_HOOK_FILENAME + platform = _platform(tmp_path, hooks_file=path) + assert platform.is_installed() is False + platform.install() + assert platform.is_installed() is True + + +class TestInstallMergeAndUpgrade: + def test_preserves_foreign_hooks_and_extra_fields(self, tmp_path: Path): + path = tmp_path / OWNED_HOOK_FILENAME + foreign = { + "type": "command", + "bash": "/opt/foreign-hook sessionStart", + "timeoutSec": 17, + } + path.write_text( + json.dumps( + { + "version": HOOK_CONFIG_VERSION, + "theme": "dark", + "hooks": {"sessionStart": [foreign]}, + } + ) + ) + platform = _platform(tmp_path, hooks_file=path) + platform.install() + data = json.loads(path.read_text()) + assert data["theme"] == "dark" + assert foreign in data["hooks"]["sessionStart"] + assert _expected_entry(platform, "sessionStart") in data["hooks"]["sessionStart"] + + def test_replaces_stale_owned_command_without_duplicating(self, tmp_path: Path): + path = tmp_path / OWNED_HOOK_FILENAME + stale = { + "type": "command", + "bash": "/old/path/thirdeye-copilot-hook sessionStart", + "timeoutSec": HOOK_TIMEOUT_S, + } + path.write_text( + json.dumps({"version": HOOK_CONFIG_VERSION, "hooks": {"sessionStart": [stale]}}) + ) + platform = _platform( + tmp_path, + hooks_file=path, + entrypoint="/new/path/thirdeye-copilot-hook", + windows=False, + ) + platform.install() + data = json.loads(path.read_text()) + commands = [entry.get("bash") for entry in data["hooks"]["sessionStart"]] + assert commands.count(_expected_entry(platform, "sessionStart")["bash"]) == 1 + assert stale["bash"] not in commands + + def test_upgrades_partial_install_to_full_event_set(self, tmp_path: Path): + path = tmp_path / OWNED_HOOK_FILENAME + platform = _platform(tmp_path, hooks_file=path) + partial = { + "version": HOOK_CONFIG_VERSION, + "hooks": { + "sessionStart": [_expected_entry(platform, "sessionStart")], + "sessionEnd": [_expected_entry(platform, "sessionEnd")], + }, + } + path.write_text(json.dumps(partial)) + platform.install() + data = json.loads(path.read_text()) + assert set(data["hooks"]) == set(CLI_HOOK_EVENTS) + + +class TestUninstall: + def test_removes_only_owned_entries_and_deletes_empty_file(self, tmp_path: Path): + path = tmp_path / OWNED_HOOK_FILENAME + platform = _platform(tmp_path, hooks_file=path) + platform.install() + platform.uninstall() + assert not path.exists() + + def test_uninstall_preserves_foreign_hooks_and_extra_fields(self, tmp_path: Path): + path = tmp_path / OWNED_HOOK_FILENAME + foreign = { + "type": "command", + "bash": "/opt/foreign-hook agentStop", + "timeoutSec": 12, + } + path.write_text( + json.dumps( + { + "version": HOOK_CONFIG_VERSION, + "notes": "keep me", + "hooks": {"agentStop": [foreign]}, + } + ) + ) + platform = _platform(tmp_path, hooks_file=path) + platform.install() + platform.uninstall() + data = json.loads(path.read_text()) + assert data["notes"] == "keep me" + assert data["hooks"] == {"agentStop": [foreign]} + + def test_uninstall_on_missing_file_is_noop(self, tmp_path: Path): + path = tmp_path / OWNED_HOOK_FILENAME + _platform(tmp_path, hooks_file=path).uninstall() + + +class TestShellQuoting: + def test_bash_quotes_paths_with_spaces(self, tmp_path: Path): + entrypoint = "/opt/my tools/thirdeye-copilot-hook" + platform = _platform(tmp_path, entrypoint=entrypoint, windows=False) + _, command = platform._command("sessionStart") + parts = shlex.split(command, posix=True) + assert parts == [entrypoint, "sessionStart"] + + def test_powershell_quotes_paths_with_spaces(self, tmp_path: Path): + entrypoint = r"C:\Users\First Last\tools\thirdeye-copilot-hook.exe" + platform = _platform(tmp_path, entrypoint=entrypoint, windows=True) + shell, command = platform._command("userPromptSubmitted") + assert shell == "powershell" + assert command.startswith("& ") + assert entrypoint in command + assert "'userPromptSubmitted'" in command + + def test_install_writes_bash_commands_on_posix(self, tmp_path: Path): + path = tmp_path / OWNED_HOOK_FILENAME + entrypoint = "/opt/my tools/thirdeye-copilot-hook" + platform = _platform(tmp_path, hooks_file=path, entrypoint=entrypoint, windows=False) + platform.install() + entry = json.loads(path.read_text())["hooks"]["preToolUse"][0] + assert "bash" in entry + assert "powershell" not in entry + assert shlex.split(entry["bash"], posix=True) == [entrypoint, "preToolUse"] + + def test_install_writes_powershell_commands_on_windows(self, tmp_path: Path): + path = tmp_path / OWNED_HOOK_FILENAME + entrypoint = r"C:\Program Files\Thirdeye\thirdeye-copilot-hook.exe" + platform = _platform(tmp_path, hooks_file=path, entrypoint=entrypoint, windows=True) + platform.install() + entry = json.loads(path.read_text())["hooks"]["postToolUse"][0] + assert entry["powershell"].startswith("& ") + assert entrypoint in entry["powershell"] + + +class TestMalformedConfig: + @pytest.mark.parametrize( + "payload, needle", + [ + ("{not json", "not valid JSON"), + (json.dumps([]), "must be a JSON object"), + (json.dumps({"version": 2, "hooks": {}}), "expected version"), + (json.dumps({"version": 1, "hooks": []}), "'hooks' must be an object"), + ( + json.dumps({"version": 1, "hooks": {"sessionStart": "nope"}}), + "hooks.sessionStart must be a list", + ), + ( + json.dumps({"version": 1, "hooks": {"customFutureEvent": "not-a-list"}}), + "hooks.customFutureEvent must be a list", + ), + ], + ) + def test_install_refuses_malformed_document_and_preserves_bytes( + self, + tmp_path: Path, + payload: str, + needle: str, + ): + path = tmp_path / OWNED_HOOK_FILENAME + path.write_text(payload) + before = path.read_bytes() + with pytest.raises(click.ClickException) as exc_info: + _platform(tmp_path, hooks_file=path).install() + assert needle in str(exc_info.value) + assert path.read_bytes() == before + + def test_is_installed_returns_false_for_malformed_document(self, tmp_path: Path): + path = tmp_path / OWNED_HOOK_FILENAME + path.write_text("{broken") + platform = _platform(tmp_path, hooks_file=path) + assert platform.is_installed() is False + + def test_uninstall_refuses_malformed_unknown_event_and_preserves_bytes(self, tmp_path: Path): + path = tmp_path / OWNED_HOOK_FILENAME + payload = json.dumps({"version": 1, "hooks": {"customFutureEvent": "not-a-list"}}) + path.write_text(payload) + before = path.read_bytes() + with pytest.raises(click.ClickException) as exc_info: + _platform(tmp_path, hooks_file=path).uninstall() + assert "hooks.customFutureEvent must be a list" in str(exc_info.value) + assert path.read_bytes() == before + + +class TestSourceHomeScope: + def test_resolves_hooks_file_under_source_home(self, tmp_path: Path): + source_home = tmp_path / "custom-copilot-home" + platform = CopilotPlatform( + source_home=source_home, + entrypoint="/opt/bin/thirdeye-copilot-hook", + windows=False, + ) + expected = source_home / HOOKS_DIRECTORY_NAME / OWNED_HOOK_FILENAME + assert platform.hooks_file == expected.resolve() + platform.install() + assert expected.exists() + + def test_explicit_hooks_file_overrides_source_home(self, tmp_path: Path): + override = tmp_path / "override" / OWNED_HOOK_FILENAME + platform = CopilotPlatform( + source_home=tmp_path / "ignored-home", + hooks_file=override, + entrypoint="/opt/bin/thirdeye-copilot-hook", + windows=False, + ) + assert platform.hooks_file == override + platform.install() + assert override.exists() + assert not (tmp_path / "ignored-home").exists() + + +class TestInstallStateChecks: + def test_requires_every_event_to_be_installed(self, tmp_path: Path): + path = tmp_path / OWNED_HOOK_FILENAME + platform = _platform(tmp_path, hooks_file=path) + platform.install() + data = json.loads(path.read_text()) + data["hooks"].pop(CLI_HOOK_EVENTS[0]) + path.write_text(json.dumps(data)) + assert platform.is_installed() is False + + def test_is_installed_false_when_timeout_differs(self, tmp_path: Path): + path = tmp_path / OWNED_HOOK_FILENAME + platform = _platform(tmp_path, hooks_file=path) + platform.install() + data = json.loads(path.read_text()) + data["hooks"]["notification"][0]["timeoutSec"] = 99 + path.write_text(json.dumps(data)) + assert platform.is_installed() is False + + +class TestWindowsOwnershipRecognition: + def test_replaces_stale_powershell_owned_command(self, tmp_path: Path): + path = tmp_path / OWNED_HOOK_FILENAME + stale = { + "type": "command", + "powershell": "& 'C:\\Old Path\\thirdeye-copilot-hook.exe' 'sessionStart'", + "timeoutSec": HOOK_TIMEOUT_S, + } + path.write_text( + json.dumps({"version": HOOK_CONFIG_VERSION, "hooks": {"sessionStart": [stale]}}) + ) + entrypoint = r"C:\New Path\thirdeye-copilot-hook.exe" + platform = _platform(tmp_path, hooks_file=path, entrypoint=entrypoint, windows=True) + platform.install() + data = json.loads(path.read_text()) + assert data["hooks"]["sessionStart"] == [_expected_entry(platform, "sessionStart")] + + def test_recognizes_double_quoted_powershell_executable(self, tmp_path: Path): + path = tmp_path / OWNED_HOOK_FILENAME + owned = { + "type": "command", + "powershell": '& "C:\\Tools\\thirdeye-copilot-hook.exe" "agentStop"', + "timeoutSec": HOOK_TIMEOUT_S, + } + path.write_text( + json.dumps({"version": HOOK_CONFIG_VERSION, "hooks": {"agentStop": [owned]}}) + ) + platform = _platform( + tmp_path, + hooks_file=path, + entrypoint=r"C:\Tools\thirdeye-copilot-hook.exe", + windows=True, + ) + platform.install() + data = json.loads(path.read_text()) + assert len(data["hooks"]["agentStop"]) == 1 + assert platform.is_installed() is True + + +class TestUnknownEvents: + def test_preserves_well_formed_unknown_events_across_install_and_uninstall( + self, tmp_path: Path + ): + path = tmp_path / OWNED_HOOK_FILENAME + foreign = { + "type": "command", + "bash": "/opt/foreign-hook customFutureEvent", + "timeoutSec": 9, + } + path.write_text( + json.dumps( + { + "version": HOOK_CONFIG_VERSION, + "hooks": {"customFutureEvent": [foreign]}, + } + ) + ) + platform = _platform(tmp_path, hooks_file=path, windows=False) + platform.install() + installed = json.loads(path.read_text()) + assert installed["hooks"]["customFutureEvent"] == [foreign] + platform.uninstall() + remaining = json.loads(path.read_text()) + assert remaining["hooks"] == {"customFutureEvent": [foreign]} + assert "sessionStart" not in remaining["hooks"] + + +class TestEntrypointRequired: + def test_install_errors_when_dispatcher_is_missing(self, tmp_path: Path, monkeypatch): + monkeypatch.setattr( + "thirdeye.platforms.copilot.install.shutil.which", + lambda _name: None, + ) + path = tmp_path / OWNED_HOOK_FILENAME + platform = CopilotPlatform(hooks_file=path, windows=False) + with pytest.raises(click.ClickException) as exc_info: + platform.install() + message = str(exc_info.value) + assert HOOK_BIN_NAME in message + assert "PATH" in message + assert not path.exists() + + def test_injected_entrypoint_does_not_require_path_lookup(self, tmp_path: Path, monkeypatch): + monkeypatch.setattr( + "thirdeye.platforms.copilot.install.shutil.which", + lambda _name: None, + ) + path = tmp_path / OWNED_HOOK_FILENAME + _platform(tmp_path, hooks_file=path).install() + assert path.exists() + + +class TestInstallGuidance: + def test_reports_scope_restart_status_and_watch_fallback( + self, tmp_path: Path, capsys: pytest.CaptureFixture[str] + ): + source_home = tmp_path / "custom-copilot-home" + platform = CopilotPlatform( + source_home=source_home, + entrypoint="/opt/bin/thirdeye-copilot-hook", + windows=False, + ) + platform.install() + output = capsys.readouterr().out + assert str(platform.hooks_file) in output + assert f"Scope: Copilot home {source_home.resolve()}" in output + assert "Restart Copilot CLI" in output + assert "thirdeye copilot status" in output + assert "thirdeye copilot watch" in output diff --git a/tests/platforms/copilot/test_migration.py b/tests/platforms/copilot/test_migration.py new file mode 100644 index 00000000..6ecee742 --- /dev/null +++ b/tests/platforms/copilot/test_migration.py @@ -0,0 +1,566 @@ +"""Migration, journal recovery, rebuild equivalence, and competing projection commits.""" + +from __future__ import annotations + +import json +import subprocess +import sys +import time +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path +from typing import Any + +import pytest + +import thirdeye.platforms.copilot.projection_state as projection_state_mod +from thirdeye._compat import fsops +from thirdeye.config import Config +from thirdeye.paths import session_dir, usage_jsonl_path +from thirdeye.platforms.copilot.archive import commit_batch +from thirdeye.platforms.copilot.constants import PLATFORM_NAME +from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id +from thirdeye.platforms.copilot.projection_state import ( + empty_projection_state, + projection_journal_path, + projection_state_path, + publish_projection_document, + read_projection_document, +) +from thirdeye.platforms.copilot.projection_store import ( + commit_projection, + load_projection_state, + read_projected_turns, + reset_projection_state, +) +from thirdeye.platforms.copilot.types import ( + PROJECTION_SCHEMA_VERSION, + Projection, + SourceBatch, + SourcePaths, + SourceRecord, +) +from thirdeye.usage.index import UsageIndex +from thirdeye.usage.read import iter_calls +from thirdeye.usage.types import UsageRow + +NATIVE_ID = "session-migrate" +INTERACTION_ID = "interaction-migrate-1" + + +def _record(source_id: str, *, ts: str = "2026-09-10T17:08:24.000Z") -> SourceRecord: + return { + "source_id": source_id, + "source_kind": "transcript", + "native_session_id": NATIVE_ID, + "ts": ts, + "observed_at": "2026-09-10T17:08:25.000Z", + "payload": {"schema_version": 1, "type": "user.message"}, + "locator": {"file": "events.jsonl", "offset": 0}, + } + + +def _batch(paths: SourcePaths, records: list[SourceRecord]) -> SourceBatch: + return { + "source_key": paths["source_key"], + "native_session_id": NATIVE_ID, + "cwd": "/proj", + "records": records, + "next_cursor": {"generation": 1}, + "diagnostics": [], + } + + +def _usage_row(*, call_id: str, input_tokens: int, session_id: str) -> UsageRow: + return UsageRow( + session_id=session_id, + seq=0, + call_id=call_id, + ts="2026-09-10T17:08:30.000Z", + platform=PLATFORM_NAME, + provider_name="openai", + response_model="gpt-5.6-luna", + input_tokens=input_tokens, + output_tokens=10, + ) + + +def _main_turn(*, source_ids: list[str]) -> dict[str, Any]: + return { + "turn_id": "copilot:turn:key:session:interaction-migrate-1", + "start_ts": "2026-09-10T17:08:24.000Z", + "end_ts": "2026-09-10T17:08:28.000Z", + "input_message": "hello", + "output_message": "done", + "status": "completed", + "llm_calls": [], + "permission_requests": [], + "subagents": [], + "attributes": {"interaction_id": INTERACTION_ID}, + "source_ids": source_ids, + } + + +def _normalized_event(event_id: str, *, source_ids: list[str]) -> dict[str, Any]: + return { + "id": event_id, + "kind": "user_prompt", + "classification": "main", + "initiator": "user", + "source_ids": source_ids, + "attributes": {"interaction_id": INTERACTION_ID}, + } + + +def _projection(**overrides: Any) -> Projection: + base: Projection = { + "normalized_events": [], + "turns": [], + "usage_rows": [], + "attributions": [], + "pending": [], + "diagnostics": [], + } + base.update(overrides) + return base + + +@contextmanager +def fault_at(point: str) -> Iterator[list[str]]: + seen: list[str] = [] + + def injector(name: str) -> None: + seen.append(name) + if name == point: + raise RuntimeError(f"injected fault at {point}") + + projection_state_mod._fault_injector = injector + try: + yield seen + finally: + projection_state_mod._fault_injector = None + + +@pytest.fixture +def config(tmp_path: Path) -> Config: + return Config(root=tmp_path / "thirdeye") + + +@pytest.fixture +def paths(tmp_path: Path) -> SourcePaths: + home = tmp_path / "copilot-home" + home.mkdir() + return resolve_sources(home) + + +def _stored(_config: Config, paths: SourcePaths) -> str: + return stored_session_id(paths, NATIVE_ID) + + +def _directory(config: Config, stored: str) -> Path: + return session_dir(config.root, PLATFORM_NAME, stored) + + +def _seed_v1_archive(config: Config, paths: SourcePaths) -> str: + commit_batch( + config, + paths, + _batch( + paths, [_record("key/a/v1-one"), _record("key/a/v1-two", ts="2026-09-10T17:08:26.000Z")] + ), + ) + return _stored(config, paths) + + +def _sample_projection(stored: str) -> Projection: + return _projection( + normalized_events=[ + _normalized_event("evt-one", source_ids=["key/a/v1-one"]), + _normalized_event("evt-two", source_ids=["key/a/v1-two"]), + ], + turns=[_main_turn(source_ids=["key/a/v1-one", "key/a/v1-two"])], + usage_rows=[_usage_row(call_id="usage-src-1", input_tokens=111, session_id=stored)], + attributions=[ + { + "usage_source_id": "usage-src-1", + "logical_call_id": "logical-usage-1", + "stored_turn_id": "copilot:turn:key:session:interaction-migrate-1", + "agent_id": None, + "call_id": None, + "status": "matched", + "join_kind": "direct", + "evidence": [], + } + ], + ) + + +def test_v1_archive_first_projection_commit_is_upgrade_safe( + config: Config, paths: SourcePaths +) -> None: + stored = _seed_v1_archive(config, paths) + directory = _directory(config, stored) + + assert not projection_state_path(directory).exists() + counts = commit_projection(config, stored, _sample_projection(stored), empty_projection_state()) + + assert counts["events"] == 2 + assert counts["turns"] == 1 + assert counts["usage"] == 1 + assert projection_state_path(directory).is_file() + assert len(read_projected_turns(config, stored)) == 1 + + +@pytest.mark.parametrize( + "fault_point", + [ + "after_projection_journal", + "after_projection_state", + "after_projection_journal_clear", + ], +) +def test_projection_journal_crash_recovers_on_next_read( + config: Config, paths: SourcePaths, fault_point: str +) -> None: + stored = _seed_v1_archive(config, paths) + directory = _directory(config, stored) + projection = _sample_projection(stored) + + with fault_at(fault_point): + with pytest.raises(RuntimeError, match="injected fault"): + commit_projection(config, stored, projection, empty_projection_state()) + + if fault_point == "after_projection_journal": + assert projection_journal_path(directory).is_file() + + state = load_projection_state(config, stored) + assert state["index_totals"]["events"] == 2 + assert not projection_journal_path(directory).exists() + turns = read_projected_turns(config, stored) + assert len(turns) == 1 + assert len(turns[0]["events"]) == 2 + + +def test_orphan_journal_recovers_without_new_commit(config: Config, paths: SourcePaths) -> None: + stored = _seed_v1_archive(config, paths) + directory = _directory(config, stored) + document = read_projection_document(directory) + document["indexes"]["events"] = { + "evt-orphan": _normalized_event("evt-orphan", source_ids=["key/a/v1-one"]) + } + publish_projection_document(directory, document) + + with fault_at("after_projection_journal"): + with pytest.raises(RuntimeError): + publish_projection_document(directory, document) + + assert projection_journal_path(directory).is_file() + recovered = read_projection_document(directory) + assert "evt-orphan" in recovered["indexes"]["events"] + assert not projection_journal_path(directory).exists() + + +def test_load_projection_state_recovers_missing_usage_sidecar( + config: Config, paths: SourcePaths +) -> None: + stored = _seed_v1_archive(config, paths) + commit_projection(config, stored, _sample_projection(stored), empty_projection_state()) + directory = _directory(config, stored) + fsops.unlink(usage_jsonl_path(directory), missing_ok=True) + assert not usage_jsonl_path(directory).exists() + + load_projection_state(config, stored) + + rows = list(iter_calls(directory)) + assert len(rows) == 1 + assert rows[0].input_tokens == 111 + + +def _indexed_usage(config: Config, stored: str) -> list[tuple[str, int]]: + index = UsageIndex(config.root) + connection = index.connect() + try: + index.refresh(connection) + rows = connection.execute( + "SELECT call_id, gen_ai_usage_input_tokens FROM usage WHERE session_id = ? " + "ORDER BY call_id", + (stored,), + ).fetchall() + finally: + connection.close() + return [(str(call_id), int(tokens)) for call_id, tokens in rows] + + +def test_load_clears_stale_usage_index_after_sidecar_unlink_crash( + config: Config, paths: SourcePaths, monkeypatch: pytest.MonkeyPatch +) -> None: + stored = _seed_v1_archive(config, paths) + commit_projection(config, stored, _sample_projection(stored), empty_projection_state()) + assert _indexed_usage(config, stored) == [("logical-usage-1", 111)] + + import thirdeye.platforms.copilot.projection_store as store_mod + + monkeypatch.setattr(store_mod, "_invalidate_usage_index", lambda *_args, **_kwargs: None) + reset_projection_state(config, stored) + directory = _directory(config, stored) + assert not usage_jsonl_path(directory).exists() + assert _indexed_usage(config, stored) == [("logical-usage-1", 111)] + + monkeypatch.undo() + load_projection_state(config, stored) + assert _indexed_usage(config, stored) == [] + assert list(iter_calls(directory)) == [] + + +def test_rebuild_after_reset_matches_original_projection( + config: Config, paths: SourcePaths +) -> None: + stored = _seed_v1_archive(config, paths) + projection = _sample_projection(stored) + commit_projection(config, stored, projection, empty_projection_state()) + + before_turns = read_projected_turns(config, stored) + before_state = load_projection_state(config, stored) + + reset_projection_state(config, stored) + commit_projection(config, stored, projection, empty_projection_state()) + + after_turns = read_projected_turns(config, stored) + after_state = load_projection_state(config, stored) + + assert after_turns == before_turns + assert after_state["index_totals"] == before_state["index_totals"] + + +def test_competing_projection_commits_merge_indexes(config: Config, paths: SourcePaths) -> None: + stored = _seed_v1_archive(config, paths) + start_signal = config.root.parent / "start-projection-writers" + script = r""" +from pathlib import Path +import sys +import time + +from thirdeye.config import Config +from thirdeye.platforms.copilot.projection_state import empty_projection_state +from thirdeye.platforms.copilot.projection_store import commit_projection +from thirdeye.usage.types import UsageRow + +root = Path(sys.argv[1]) +stored = sys.argv[2] +writer_id = sys.argv[3] +start_signal = Path(sys.argv[4]) +while not start_signal.exists(): + time.sleep(0.001) + +source_id = f"key/a/v1-{'one' if writer_id == 'a' else 'two'}" +event_id = f"evt-{writer_id}" +logical_id = f"logical-{writer_id}" +usage_source = f"usage-{writer_id}" +config = Config(root=root) +row = UsageRow( + session_id=stored, + seq=0, + call_id=usage_source, + ts="2026-09-10T17:08:30.000Z", + platform="copilot", + provider_name="openai", + response_model="gpt-5.6-luna", + input_tokens=50 if writer_id == "a" else 75, + output_tokens=10, +) +commit_projection( + config, + stored, + { + "normalized_events": [ + { + "id": event_id, + "kind": "user_prompt", + "classification": "main", + "initiator": "user", + "source_ids": [source_id], + "attributes": {"interaction_id": "interaction-migrate-1"}, + } + ], + "turns": [], + "usage_rows": [row], + "attributions": [ + { + "usage_source_id": usage_source, + "logical_call_id": logical_id, + "stored_turn_id": None, + "agent_id": None, + "call_id": None, + "status": "pending", + "join_kind": None, + "evidence": [], + } + ], + "pending": [], + "diagnostics": [], + }, + empty_projection_state(), +) +""" + processes = [ + subprocess.Popen( + [sys.executable, "-c", script, str(config.root), stored, writer_id, str(start_signal)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + encoding="utf-8", + ) + for writer_id in ("a", "b") + ] + start_signal.touch() + deadline = time.monotonic() + 30 + failures: list[str] = [] + for process in processes: + stdout, stderr = process.communicate(timeout=max(0.1, deadline - time.monotonic())) + if process.returncode != 0: + failures.append(f"exit={process.returncode} stdout={stdout!r} stderr={stderr!r}") + assert not failures, failures + + state = load_projection_state(config, stored) + assert state["index_totals"]["events"] == 2 + assert state["index_totals"]["usage"] == 2 + + document = json.loads(projection_state_path(_directory(config, stored)).read_text()) + assert set(document["indexes"]["events"]) == {"evt-a", "evt-b"} + assert set(document["indexes"]["usage"]) == {"logical-a", "logical-b"} + + +def test_replay_same_event_id_replaces_without_duplicating( + config: Config, paths: SourcePaths +) -> None: + stored = _seed_v1_archive(config, paths) + first = _projection( + normalized_events=[ + { + **_normalized_event("evt-stable", source_ids=["key/a/v1-one"]), + "attributes": {"interaction_id": INTERACTION_ID, "note": "first"}, + } + ], + ) + second = _projection( + normalized_events=[ + { + **_normalized_event("evt-stable", source_ids=["key/a/v1-one"]), + "attributes": {"interaction_id": INTERACTION_ID, "note": "second"}, + } + ], + ) + + commit_projection(config, stored, first, empty_projection_state()) + commit_projection(config, stored, second, empty_projection_state()) + + document = json.loads(projection_state_path(_directory(config, stored)).read_text()) + events = document["indexes"]["events"] + assert len(events) == 1 + assert events["evt-stable"]["attributes"]["note"] == "second" + + +def test_usage_sidecar_rewrite_after_state_publication_crash( + config: Config, paths: SourcePaths, monkeypatch: pytest.MonkeyPatch +) -> None: + stored = _seed_v1_archive(config, paths) + directory = _directory(config, stored) + writes: list[int] = [] + + import thirdeye.platforms.copilot.projection_store as store_mod + + original_write_usage = store_mod._write_usage_index + + def counting_write_usage(path: Path, usage_index: dict[str, Any]) -> None: + writes.append(len(usage_index)) + if len(writes) == 2: + raise RuntimeError("crash while rewriting usage sidecar") + original_write_usage(path, usage_index) + + monkeypatch.setattr(store_mod, "_write_usage_index", counting_write_usage) + + commit_projection(config, stored, _sample_projection(stored), empty_projection_state()) + + with pytest.raises(RuntimeError, match="crash while rewriting usage sidecar"): + commit_projection( + config, + stored, + _projection( + usage_rows=[_usage_row(call_id="usage-src-2", input_tokens=222, session_id=stored)], + attributions=[ + { + "usage_source_id": "usage-src-2", + "logical_call_id": "logical-usage-1", + "stored_turn_id": None, + "agent_id": None, + "call_id": None, + "status": "matched", + "join_kind": "direct", + "evidence": [], + } + ], + ), + empty_projection_state(), + ) + + load_projection_state(config, stored) + rows = list(iter_calls(directory)) + assert len(rows) == 1 + assert rows[0].input_tokens == 222 + + +def test_stale_projection_schema_is_discarded_for_rebuild( + config: Config, paths: SourcePaths +) -> None: + stored = _seed_v1_archive(config, paths) + directory = _directory(config, stored) + stale = { + "schema_version": PROJECTION_SCHEMA_VERSION - 1, + "state": { + "projection_schema_version": PROJECTION_SCHEMA_VERSION - 1, + "archive_source_ids": ["stale"], + "semantic_state": {"open_interactions": {"old": {}}}, + "accounting_state": {"logical_calls": {}}, + "projection_revision": "stale", + }, + "indexes": {"events": {"old": {"id": "old"}}}, + } + projection_state_path(directory).write_text(json.dumps(stale), encoding="utf-8") + + state = load_projection_state(config, stored) + assert state["archive_source_ids"] == [] + assert state["semantic_state"]["open_interactions"] == {} + assert not projection_state_path(directory).exists() + assert read_projected_turns(config, stored) == [] + + +def test_stale_journal_schema_is_discarded_without_raising( + config: Config, paths: SourcePaths +) -> None: + stored = _seed_v1_archive(config, paths) + directory = _directory(config, stored) + journal = { + "schema_version": 0, + "document": { + "schema_version": 1, + "state": {"projection_schema_version": 1}, + "indexes": {}, + }, + } + projection_journal_path(directory).write_text(json.dumps(journal) + "\n", encoding="utf-8") + + state = load_projection_state(config, stored) + assert state["projection_schema_version"] == PROJECTION_SCHEMA_VERSION + assert not projection_journal_path(directory).exists() + + +def test_corrupt_journal_is_discarded_and_snapshot_used(config: Config, paths: SourcePaths) -> None: + stored = _seed_v1_archive(config, paths) + commit_projection(config, stored, _sample_projection(stored), empty_projection_state()) + directory = _directory(config, stored) + projection_journal_path(directory).write_text("{not-json", encoding="utf-8") + + state = load_projection_state(config, stored) + assert state["index_totals"]["events"] == 2 + assert not projection_journal_path(directory).exists() + assert len(read_projected_turns(config, stored)) == 1 diff --git a/tests/platforms/copilot/test_projection_store.py b/tests/platforms/copilot/test_projection_store.py new file mode 100644 index 00000000..68b5b65f --- /dev/null +++ b/tests/platforms/copilot/test_projection_store.py @@ -0,0 +1,962 @@ +"""Behavioral tests for Copilot V2 projection commit, load, and turn reads.""" + +from __future__ import annotations + +import json +import shutil +from pathlib import Path +from typing import Any + +import pytest + +from thirdeye.config import Config +from thirdeye.paths import session_dir, usage_db_path, usage_jsonl_path +from thirdeye.platforms.copilot.archive import commit_batch +from thirdeye.platforms.copilot.constants import PLATFORM_NAME, SOURCE_SCHEMA_VERSION +from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id +from thirdeye.platforms.copilot.projection_state import ( + empty_projection_state, + projection_journal_path, + projection_state_path, +) +from thirdeye.platforms.copilot.projection_store import ( + commit_projection, + load_projection_state, + read_projected_turns, + reset_projection_state, +) +from thirdeye.platforms.copilot.types import Projection, SourceBatch, SourcePaths, SourceRecord +from thirdeye.reader import SessionReader +from thirdeye.usage.index import UsageIndex +from thirdeye.usage.read import iter_calls +from thirdeye.usage.types import UsageRow + +NATIVE_ID = "session-proj" +INTERACTION_ID = "interaction-main-1" + + +def _record( + source_id: str, + *, + native_session_id: str = NATIVE_ID, + source_kind: str = "transcript", + ts: str = "2026-09-10T17:08:24.000Z", +) -> SourceRecord: + return { + "source_id": source_id, + "source_kind": source_kind, + "native_session_id": native_session_id, + "ts": ts, + "observed_at": "2026-09-10T17:08:25.000Z", + "payload": {"schema_version": 1, "type": "user.message"}, + "locator": {"file": "events.jsonl", "offset": 0}, + } + + +def _batch( + paths: SourcePaths, + records: list[SourceRecord], + *, + next_cursor: dict[str, Any] | None = None, +) -> SourceBatch: + return { + "source_key": paths["source_key"], + "native_session_id": NATIVE_ID, + "cwd": "/proj", + "records": records, + "next_cursor": dict(next_cursor or {"generation": 1}), + "diagnostics": [], + } + + +def _usage_row( + *, + call_id: str, + input_tokens: int = 100, + output_tokens: int = 10, + session_id: str = "stored-session", + seq: int = 0, +) -> UsageRow: + return UsageRow( + session_id=session_id, + seq=seq, + call_id=call_id, + ts="2026-09-10T17:08:30.000Z", + platform=PLATFORM_NAME, + provider_name="openai", + response_model="gpt-5.6-luna", + input_tokens=input_tokens, + output_tokens=output_tokens, + ) + + +def _main_turn( + *, + turn_id: str = "copilot:turn:key:session:interaction-main-1", + interaction_id: str = INTERACTION_ID, + start_ts: str = "2026-09-10T17:08:24.000Z", + end_ts: str = "2026-09-10T17:08:28.000Z", + source_ids: list[str] | None = None, + agent_id: str | None = None, +) -> dict[str, Any]: + attributes: dict[str, Any] = {"interaction_id": interaction_id} + if agent_id is not None: + attributes["agent_id"] = agent_id + turn: dict[str, Any] = { + "turn_id": turn_id, + "start_ts": start_ts, + "end_ts": end_ts, + "input_message": "hello", + "output_message": "done", + "status": "completed", + "llm_calls": [], + "permission_requests": [], + "subagents": [], + "attributes": attributes, + } + if source_ids: + turn["source_ids"] = source_ids + return turn + + +def _normalized_event( + event_id: str, + *, + interaction_id: str = INTERACTION_ID, + source_ids: list[str], + stored_turn_id: str | None = None, +) -> dict[str, Any]: + attributes: dict[str, Any] = {"interaction_id": interaction_id} + if stored_turn_id is not None: + attributes["stored_turn_id"] = stored_turn_id + return { + "id": event_id, + "kind": "user_prompt", + "classification": "main", + "initiator": "user", + "source_ids": source_ids, + "source_references": [ + {"source_id": source_id, "source_kind": "transcript", "role": "user_prompt"} + for source_id in source_ids + ], + "attributes": attributes, + } + + +def _attribution( + *, + logical_call_id: str, + usage_source_id: str, + status: str = "matched", +) -> dict[str, Any]: + return { + "usage_source_id": usage_source_id, + "logical_call_id": logical_call_id, + "stored_turn_id": "copilot:turn:key:session:interaction-main-1", + "agent_id": None, + "call_id": None, + "status": status, + "join_kind": "direct", + "evidence": ["direct:usage_source_id"], + } + + +def _projection(**overrides: Any) -> Projection: + base: Projection = { + "normalized_events": [], + "turns": [], + "usage_rows": [], + "attributions": [], + "pending": [], + "diagnostics": [], + } + base.update(overrides) + return base + + +@pytest.fixture +def config(tmp_path: Path) -> Config: + return Config(root=tmp_path / "thirdeye") + + +@pytest.fixture +def paths(tmp_path: Path) -> SourcePaths: + home = tmp_path / "copilot-home" + home.mkdir() + return resolve_sources(home) + + +def _stored(_config: Config, paths: SourcePaths) -> str: + return stored_session_id(paths, NATIVE_ID) + + +def _directory(config: Config, stored: str) -> Path: + return session_dir(config.root, PLATFORM_NAME, stored) + + +def _seed_archive(config: Config, paths: SourcePaths, records: list[SourceRecord]) -> str: + commit_batch(config, paths, _batch(paths, records)) + return _stored(config, paths) + + +def test_commit_projection_persists_indexes_and_usage_sidecar( + config: Config, paths: SourcePaths +) -> None: + stored = _seed_archive( + config, + paths, + [_record("key/a/event-1"), _record("key/a/event-2", ts="2026-09-10T17:08:26.000Z")], + ) + directory = _directory(config, stored) + + projection = _projection( + normalized_events=[ + _normalized_event("copilot:event:key/a/event-1", source_ids=["key/a/event-1"]), + _normalized_event("copilot:event:key/a/event-2", source_ids=["key/a/event-2"]), + ], + turns=[_main_turn(source_ids=["key/a/event-1", "key/a/event-2"])], + usage_rows=[_usage_row(call_id="src-usage-1", session_id=stored)], + attributions=[_attribution(logical_call_id="logical-1", usage_source_id="src-usage-1")], + pending=[{"id": "pending-1", "kind": "open_interaction", "message": "unfinished"}], + diagnostics=[ + { + "code": "missing_capability", + "severity": "info", + "message": "no direct join id", + "source_ids": [], + "details": {}, + } + ], + ) + next_state = empty_projection_state() + next_state["archive_source_ids"] = ["key/a/event-1", "key/a/event-2"] + + counts = commit_projection(config, stored, projection, next_state) + + assert counts == { + "events": 2, + "turns": 1, + "usage": 1, + "attributions": 1, + "pending": 1, + "diagnostics": 1, + } + assert projection_state_path(directory).is_file() + assert not projection_journal_path(directory).exists() + + usage_rows = list(iter_calls(directory)) + assert len(usage_rows) == 1 + assert usage_rows[0].call_id == "logical-1" + + state = load_projection_state(config, stored) + assert state["archive_source_ids"] == ["key/a/event-1", "key/a/event-2"] + assert state["index_totals"]["events"] == 2 + assert state["projection_revision"].startswith("sha256:") + + +def test_read_projected_turns_join_archived_store_events( + config: Config, paths: SourcePaths +) -> None: + stored = _seed_archive( + config, + paths, + [_record("key/a/user"), _record("key/a/assistant", ts="2026-09-10T17:08:26.000Z")], + ) + turn_id = "copilot:turn:key:session:interaction-main-1" + + commit_projection( + config, + stored, + _projection( + normalized_events=[ + _normalized_event("evt-user", source_ids=["key/a/user"]), + _normalized_event("evt-assistant", source_ids=["key/a/assistant"]), + ], + turns=[_main_turn(turn_id=turn_id)], + ), + empty_projection_state(), + ) + + turns = read_projected_turns(config, stored) + assert len(turns) == 1 + turn = turns[0] + assert turn["turn_id"] == turn_id + assert turn["session_id"] == stored + assert turn["platform"] == PLATFORM_NAME + assert turn["cwd"] == "/proj" + assert [event["data"]["source_record"]["source_id"] for event in turn["events"]] == [ + "key/a/user", + "key/a/assistant", + ] + assert turn["start_seq"] == 0 + assert turn["end_seq"] == 1 + + +def test_child_agent_top_level_turns_are_excluded(config: Config, paths: SourcePaths) -> None: + stored = _seed_archive(config, paths, [_record("key/a/main")]) + main_turn = _main_turn() + child_turn = _main_turn( + turn_id="copilot:turn:key:session:child", + interaction_id="child-interaction", + agent_id="subagent-123", + ) + + commit_projection( + config, + stored, + _projection( + normalized_events=[_normalized_event("evt-main", source_ids=["key/a/main"])], + turns=[main_turn, child_turn], + ), + empty_projection_state(), + ) + + turns = read_projected_turns(config, stored) + assert [turn["turn_id"] for turn in turns] == [main_turn["turn_id"]] + + +def test_nested_child_archive_events_stay_on_main_turn(config: Config, paths: SourcePaths) -> None: + stored = _seed_archive( + config, + paths, + [_record("key/a/main"), _record("key/a/child", ts="2026-09-10T17:08:26.000Z")], + ) + main_turn = _main_turn(source_ids=["key/a/main"]) + main_turn["subagents"] = [ + { + "turn_id": "copilot:turn:key:session:child", + "start_ts": "2026-09-10T17:08:26.000Z", + "end_ts": "2026-09-10T17:08:27.000Z", + "input_message": "explore", + "output_message": "found", + "status": "completed", + "llm_calls": [], + "permission_requests": [], + "subagents": [], + "attributes": {"interaction_id": INTERACTION_ID, "agent_id": "subagent-123"}, + "source_ids": ["key/a/child"], + } + ] + child_event = _normalized_event("evt-child", source_ids=["key/a/child"]) + child_event["attributes"]["agent_id"] = "subagent-123" + + commit_projection( + config, + stored, + _projection( + normalized_events=[ + _normalized_event("evt-main", source_ids=["key/a/main"]), + child_event, + ], + turns=[main_turn], + ), + empty_projection_state(), + ) + + turns = read_projected_turns(config, stored) + assert len(turns) == 1 + assert [event["data"]["source_record"]["source_id"] for event in turns[0]["events"]] == [ + "key/a/main", + "key/a/child", + ] + + +def test_usage_revision_replaces_under_stable_logical_identity( + config: Config, paths: SourcePaths +) -> None: + stored = _seed_archive(config, paths, [_record("key/a/usage")]) + + first = _projection( + usage_rows=[_usage_row(call_id="src-rev-1", input_tokens=100, session_id=stored)], + attributions=[_attribution(logical_call_id="logical-1", usage_source_id="src-rev-1")], + ) + commit_projection(config, stored, first, empty_projection_state()) + + second = _projection( + usage_rows=[_usage_row(call_id="src-rev-2", input_tokens=150, session_id=stored, seq=1)], + attributions=[_attribution(logical_call_id="logical-1", usage_source_id="src-rev-2")], + ) + counts = commit_projection(config, stored, second, empty_projection_state()) + + assert counts["usage"] == 1 + directory = _directory(config, stored) + rows = list(iter_calls(directory)) + assert len(rows) == 1 + assert rows[0].input_tokens == 150 + assert rows[0].call_id == "logical-1" + + sidecar_lines = usage_jsonl_path(directory).read_text(encoding="utf-8").splitlines() + assert len(sidecar_lines) == 1 + + +def test_next_state_is_authoritative_and_closes_interactions( + config: Config, paths: SourcePaths +) -> None: + stored = _seed_archive(config, paths, [_record("key/a/one"), _record("key/a/two")]) + + first_state = empty_projection_state() + first_state["archive_source_ids"] = ["key/a/one"] + first_state["semantic_state"] = { + "open_interactions": { + INTERACTION_ID: { + "interaction_id": INTERACTION_ID, + "agent_id": None, + "stored_turn_id": "turn-open", + "source_ids": ["key/a/one"], + "last_event_source_id": "key/a/one", + "start_ts": "2026-09-10T17:08:24.000Z", + "pending_tool_call_ids": [], + } + } + } + commit_projection( + config, + stored, + _projection( + normalized_events=[_normalized_event("evt-1", source_ids=["key/a/one"])], + ), + first_state, + ) + + second_state = empty_projection_state() + second_state["archive_source_ids"] = ["key/a/one", "key/a/two"] + second_state["semantic_state"] = {"open_interactions": {}} + second_state["accounting_state"] = { + "logical_calls": { + "logical-1": { + "logical_call_id": "logical-1", + "generation": "gen-a", + "content_revision": "rev-a", + "metrics_digest": "sha256:abc", + "usage_source_id": "src-rev-1", + } + } + } + commit_projection( + config, + stored, + _projection( + normalized_events=[_normalized_event("evt-2", source_ids=["key/a/two"])], + ), + second_state, + ) + + replaced = load_projection_state(config, stored) + assert replaced["archive_source_ids"] == ["key/a/one", "key/a/two"] + assert replaced["semantic_state"]["open_interactions"] == {} + assert "logical-1" in replaced["accounting_state"]["logical_calls"] + + document_events = json.loads(projection_state_path(_directory(config, stored)).read_text())[ + "indexes" + ]["events"] + assert set(document_events) == {"evt-1", "evt-2"} + + +def test_load_projection_state_returns_defensive_copy(config: Config, paths: SourcePaths) -> None: + stored = _seed_archive(config, paths, [_record("key/a/event")]) + state = empty_projection_state() + state["archive_source_ids"] = ["key/a/event"] + commit_projection(config, stored, _projection(), state) + + loaded = load_projection_state(config, stored) + loaded["archive_source_ids"].append("mutated") + again = load_projection_state(config, stored) + assert again["archive_source_ids"] == ["key/a/event"] + + +def test_commit_projection_requires_usage_row_instances(config: Config, paths: SourcePaths) -> None: + stored = _seed_archive(config, paths, [_record("key/a/event")]) + bad: Projection = _projection(usage_rows=[{"call_id": "not-a-row"}]) # type: ignore[list-item] + with pytest.raises(TypeError, match="UsageRow"): + commit_projection(config, stored, bad, empty_projection_state()) + + +def test_projection_reads_do_not_require_live_copilot_sources( + config: Config, paths: SourcePaths, tmp_path: Path +) -> None: + stored = _seed_archive( + config, + paths, + [_record("key/a/user"), _record("key/a/assistant", ts="2026-09-10T17:08:26.000Z")], + ) + commit_projection( + config, + stored, + _projection( + normalized_events=[ + _normalized_event("evt-user", source_ids=["key/a/user"]), + _normalized_event("evt-assistant", source_ids=["key/a/assistant"]), + ], + turns=[_main_turn()], + ), + empty_projection_state(), + ) + + copilot_home = tmp_path / "copilot-home" + session_root = copilot_home / "session-state" / NATIVE_ID + session_root.mkdir(parents=True) + (session_root / "events.jsonl").write_text("{}\n", encoding="utf-8") + (copilot_home / "session-store.db").write_bytes(b"sqlite") + assert any(copilot_home.iterdir()) + shutil.rmtree(session_root) + (copilot_home / "session-store.db").unlink() + assert not session_root.exists() + assert not (copilot_home / "session-store.db").exists() + + turns = read_projected_turns(config, stored) + assert len(turns) == 1 + assert len(turns[0]["events"]) == 2 + archived = list(SessionReader(_directory(config, stored)).iter_events()) + assert len(archived) == 2 + + +def test_unfinished_projection_pending_does_not_block_later_capture( + config: Config, paths: SourcePaths +) -> None: + stored = _seed_archive(config, paths, [_record("key/a/first")]) + commit_projection( + config, + stored, + _projection( + pending=[{"id": "pending-open", "kind": "open_interaction", "message": "still open"}], + diagnostics=[ + { + "code": "missing_capability", + "severity": "warning", + "message": "partial semantics", + "source_ids": ["key/a/first"], + "details": {}, + } + ], + ), + empty_projection_state(), + ) + + commit_batch( + config, paths, _batch(paths, [_record("key/a/second")], next_cursor={"generation": 2}) + ) + + commit_projection( + config, + stored, + _projection( + normalized_events=[_normalized_event("evt-second", source_ids=["key/a/second"])], + ), + empty_projection_state(), + ) + + state = load_projection_state(config, stored) + assert state["index_totals"]["events"] == 1 + assert state["index_totals"]["pending"] == 0 + assert state["index_totals"]["diagnostics"] == 0 + captured = { + event["data"]["source_record"]["source_id"] + for event in SessionReader(_directory(config, stored)).iter_events( + types={"copilot_transcript"} + ) + } + assert captured == {"key/a/first", "key/a/second"} + + +def test_reset_projection_state_removes_only_derived_files( + config: Config, paths: SourcePaths +) -> None: + stored = _seed_archive(config, paths, [_record("key/a/persist")]) + commit_projection( + config, + stored, + _projection( + usage_rows=[_usage_row(call_id="usage-1", session_id=stored)], + normalized_events=[_normalized_event("evt", source_ids=["key/a/persist"])], + turns=[_main_turn(source_ids=["key/a/persist"])], + ), + empty_projection_state(), + ) + directory = _directory(config, stored) + + reset_projection_state(config, stored) + + assert not projection_state_path(directory).exists() + assert not usage_jsonl_path(directory).exists() + archived = list(SessionReader(directory).iter_events()) + assert len(archived) == 1 + assert archived[0]["data"]["schema_version"] == SOURCE_SCHEMA_VERSION + + +def test_reset_projection_state_leaves_export_ledger_untouched( + config: Config, paths: SourcePaths +) -> None: + stored = _seed_archive(config, paths, [_record("key/a/persist")]) + commit_projection( + config, + stored, + _projection( + usage_rows=[_usage_row(call_id="usage-1", session_id=stored)], + turns=[_main_turn(source_ids=["key/a/persist"])], + ), + empty_projection_state(), + ) + directory = _directory(config, stored) + ledger = directory / "copilot.export.ledger.json" + payload = '{"schema_version":1,"jobs":[]}\n' + ledger.write_text(payload, encoding="utf-8") + + reset_projection_state(config, stored) + + assert ledger.is_file() + assert ledger.read_text(encoding="utf-8") == payload + assert not projection_state_path(directory).exists() + assert not usage_jsonl_path(directory).exists() + + +def _indexed_usage(config: Config, stored: str) -> list[tuple[str, int, int]]: + index = UsageIndex(config.root) + connection = index.connect() + try: + index.refresh(connection) + rows = connection.execute( + "SELECT call_id, gen_ai_usage_input_tokens, seq FROM usage WHERE session_id = ? " + "ORDER BY call_id", + (stored,), + ).fetchall() + finally: + connection.close() + return [(str(call_id), int(tokens), int(seq)) for call_id, tokens, seq in rows] + + +def test_incremental_turn_commit_retains_prior_partition_evidence( + config: Config, paths: SourcePaths +) -> None: + stored = _seed_archive( + config, + paths, + [_record("key/a/e1"), _record("key/a/e2", ts="2026-09-10T17:08:26.000Z")], + ) + turn = _main_turn(source_ids=["key/a/e1"]) + commit_projection( + config, + stored, + _projection( + normalized_events=[_normalized_event("evt-1", source_ids=["key/a/e1"])], + turns=[turn], + ), + empty_projection_state(), + ) + first = read_projected_turns(config, stored) + assert [event["data"]["source_record"]["source_id"] for event in first[0]["events"]] == [ + "key/a/e1" + ] + + later = _main_turn(source_ids=["key/a/e2"]) + commit_projection( + config, + stored, + _projection( + normalized_events=[_normalized_event("evt-2", source_ids=["key/a/e2"])], + turns=[later], + ), + empty_projection_state(), + ) + turns = read_projected_turns(config, stored) + assert len(turns) == 1 + assert [event["data"]["source_record"]["source_id"] for event in turns[0]["events"]] == [ + "key/a/e1", + "key/a/e2", + ] + assert turns[0]["start_seq"] == 0 + assert turns[0]["end_seq"] == 1 + + +def test_derived_state_does_not_duplicate_raw_archive_payloads( + config: Config, paths: SourcePaths +) -> None: + stored = _seed_archive( + config, + paths, + [_record("key/a/user"), _record("key/a/assistant", ts="2026-09-10T17:08:26.000Z")], + ) + commit_projection( + config, + stored, + _projection( + normalized_events=[ + _normalized_event("evt-user", source_ids=["key/a/user"]), + _normalized_event("evt-assistant", source_ids=["key/a/assistant"]), + ], + turns=[_main_turn(source_ids=["key/a/user", "key/a/assistant"])], + ), + empty_projection_state(), + ) + document = json.loads(projection_state_path(_directory(config, stored)).read_text()) + dumped = json.dumps(document) + assert "user.message" not in dumped + assert document["indexes"]["turns"] + turn_record = next(iter(document["indexes"]["turns"].values())) + assert "events" not in turn_record + assert turn_record["span"]["input_message"] == "hello" + assert set(turn_record["source_ids"]) == {"key/a/user", "key/a/assistant"} + + +def test_unresolved_turn_evidence_does_not_invent_store_events( + config: Config, paths: SourcePaths +) -> None: + stored = _seed_archive(config, paths, [_record("key/a/other")]) + commit_projection( + config, + stored, + _projection(turns=[_main_turn()]), + empty_projection_state(), + ) + turns = read_projected_turns(config, stored) + assert len(turns) == 1 + assert turns[0]["events"] == [] + assert turns[0]["start_seq"] is None + assert turns[0]["end_seq"] is None + + +def test_in_progress_turns_are_omitted_from_projected_reads( + config: Config, paths: SourcePaths +) -> None: + stored = _seed_archive(config, paths, [_record("key/a/open")]) + open_turn = _main_turn(source_ids=["key/a/open"]) + open_turn["status"] = "in_progress" + commit_projection( + config, + stored, + _projection( + normalized_events=[_normalized_event("evt-open", source_ids=["key/a/open"])], + turns=[open_turn], + ), + empty_projection_state(), + ) + assert read_projected_turns(config, stored) == [] + + +def test_commit_projection_rejects_unknown_session(config: Config) -> None: + with pytest.raises(ValueError, match="unknown Copilot session"): + commit_projection(config, "ghost-session", _projection(), empty_projection_state()) + assert not (config.root / "traces" / PLATFORM_NAME / "ghost-session").exists() + + +def test_read_and_reset_unknown_session_do_not_create_paths(config: Config) -> None: + ghost = "ghost-session" + state = load_projection_state(config, ghost) + assert state["archive_source_ids"] == [] + assert state["semantic_state"]["open_interactions"] == {} + assert read_projected_turns(config, ghost) == [] + reset_projection_state(config, ghost) + assert not (config.root / "traces" / PLATFORM_NAME / ghost).exists() + assert not usage_db_path(config.root).exists() + + +def test_commit_projection_rejects_usage_row_for_another_session( + config: Config, paths: SourcePaths +) -> None: + stored = _seed_archive(config, paths, [_record("key/a/event")]) + with pytest.raises(ValueError, match="session_id"): + commit_projection( + config, + stored, + _projection(usage_rows=[_usage_row(call_id="logical-1", session_id="other")]), + empty_projection_state(), + ) + + +def test_usage_seq_is_stamped_from_archived_revision(config: Config, paths: SourcePaths) -> None: + stored = _seed_archive( + config, + paths, + [ + _record("key/a/prompt"), + _record("db-src-1", source_kind="database", ts="2026-09-10T17:08:30.000Z"), + ], + ) + next_state = empty_projection_state() + next_state["accounting_state"] = { + "logical_calls": { + "logical-1": { + "logical_call_id": "logical-1", + "generation": "gen-a", + "content_revision": "rev-a", + "metrics_digest": "sha256:abc", + "usage_source_id": "db-src-1", + } + } + } + commit_projection( + config, + stored, + _projection( + usage_rows=[_usage_row(call_id="db-src-1", session_id=stored)], + attributions=[_attribution(logical_call_id="logical-1", usage_source_id="db-src-1")], + ), + next_state, + ) + rows = list(iter_calls(_directory(config, stored))) + assert len(rows) == 1 + assert rows[0].seq == 1 + assert rows[0].call_id == "logical-1" + assert _indexed_usage(config, stored) == [("logical-1", 100, 1)] + + +def test_usage_correction_is_visible_through_usage_index( + config: Config, paths: SourcePaths +) -> None: + stored = _seed_archive(config, paths, [_record("key/a/usage")]) + first = _projection( + usage_rows=[_usage_row(call_id="logical-1", input_tokens=100, session_id=stored)], + ) + commit_projection(config, stored, first, empty_projection_state()) + assert _indexed_usage(config, stored) == [("logical-1", 100, 0)] + + second = _projection( + usage_rows=[_usage_row(call_id="logical-1", input_tokens=150, session_id=stored, seq=1)], + ) + commit_projection(config, stored, second, empty_projection_state()) + assert _indexed_usage(config, stored) == [("logical-1", 150, 0)] + + +def test_delayed_attribution_rekeys_source_id_without_double_counting( + config: Config, paths: SourcePaths +) -> None: + stored = _seed_archive(config, paths, [_record("key/a/usage")]) + first_state = empty_projection_state() + first_state["accounting_state"] = { + "logical_calls": { + "copilot:usage:logical-1": { + "logical_call_id": "copilot:usage:logical-1", + "generation": "gen-a", + "content_revision": "rev-1", + "metrics_digest": "sha256:abc", + "usage_source_id": "db-src-rev1", + } + } + } + commit_projection( + config, + stored, + _projection( + usage_rows=[_usage_row(call_id="db-src-rev1", input_tokens=100, session_id=stored)] + ), + first_state, + ) + + second_state = empty_projection_state() + second_state["accounting_state"] = { + "logical_calls": { + "copilot:usage:logical-1": { + "logical_call_id": "copilot:usage:logical-1", + "generation": "gen-a", + "content_revision": "rev-2", + "metrics_digest": "sha256:def", + "usage_source_id": "db-src-rev2", + } + } + } + commit_projection( + config, + stored, + _projection( + usage_rows=[_usage_row(call_id="db-src-rev2", input_tokens=150, session_id=stored)], + attributions=[ + _attribution( + logical_call_id="copilot:usage:logical-1", usage_source_id="db-src-rev2" + ) + ], + ), + second_state, + ) + + rows = list(iter_calls(_directory(config, stored))) + assert [(row.call_id, row.input_tokens) for row in rows] == [("copilot:usage:logical-1", 150)] + assert _indexed_usage(config, stored) == [("copilot:usage:logical-1", 150, 0)] + document = json.loads(projection_state_path(_directory(config, stored)).read_text()) + assert set(document["indexes"]["usage"]) == {"copilot:usage:logical-1"} + + +def test_identity_only_commit_rekeys_existing_usage_without_a_row( + config: Config, paths: SourcePaths +) -> None: + stored = _seed_archive(config, paths, [_record("key/a/usage")]) + commit_projection( + config, + stored, + _projection( + usage_rows=[_usage_row(call_id="db-src-rev1", input_tokens=100, session_id=stored)] + ), + empty_projection_state(), + ) + directory = _directory(config, stored) + assert [(row.call_id, row.input_tokens) for row in iter_calls(directory)] == [ + ("db-src-rev1", 100) + ] + + later_state = empty_projection_state() + later_state["accounting_state"] = { + "logical_calls": { + "copilot:usage:logical-1": { + "logical_call_id": "copilot:usage:logical-1", + "generation": "gen-a", + "content_revision": "rev-1", + "metrics_digest": "sha256:abc", + "usage_source_id": "db-src-rev1", + } + } + } + counts = commit_projection( + config, + stored, + _projection( + attributions=[ + _attribution( + logical_call_id="copilot:usage:logical-1", usage_source_id="db-src-rev1" + ) + ] + ), + later_state, + ) + + assert counts["usage"] == 1 + rows = list(iter_calls(directory)) + assert [(row.call_id, row.input_tokens) for row in rows] == [("copilot:usage:logical-1", 100)] + assert _indexed_usage(config, stored) == [("copilot:usage:logical-1", 100, 0)] + document = json.loads(projection_state_path(directory).read_text()) + assert set(document["indexes"]["usage"]) == {"copilot:usage:logical-1"} + + +def test_load_repairs_stale_usage_index_when_sidecar_already_matches( + config: Config, paths: SourcePaths, monkeypatch: pytest.MonkeyPatch +) -> None: + stored = _seed_archive(config, paths, [_record("key/a/usage")]) + commit_projection( + config, + stored, + _projection( + usage_rows=[_usage_row(call_id="logical-1", input_tokens=100, session_id=stored)] + ), + empty_projection_state(), + ) + assert _indexed_usage(config, stored) == [("logical-1", 100, 0)] + + import thirdeye.platforms.copilot.projection_store as store_mod + + monkeypatch.setattr(store_mod, "_invalidate_usage_index", lambda *_args, **_kwargs: None) + commit_projection( + config, + stored, + _projection( + usage_rows=[_usage_row(call_id="logical-1", input_tokens=150, session_id=stored)] + ), + empty_projection_state(), + ) + + directory = _directory(config, stored) + sidecar_rows = [ + json.loads(line) for line in usage_jsonl_path(directory).read_text().splitlines() if line + ] + assert sidecar_rows[0]["gen_ai.usage.input_tokens"] == 150 + assert _indexed_usage(config, stored) == [("logical-1", 100, 0)] + + monkeypatch.undo() + load_projection_state(config, stored) + assert _indexed_usage(config, stored) == [("logical-1", 150, 0)] diff --git a/tests/platforms/copilot/test_reconcile.py b/tests/platforms/copilot/test_reconcile.py new file mode 100644 index 00000000..38153d1f --- /dev/null +++ b/tests/platforms/copilot/test_reconcile.py @@ -0,0 +1,788 @@ +"""Behavioral tests for archive-only Copilot reconciliation.""" + +from __future__ import annotations + +import copy +import json +import shutil +from pathlib import Path +from typing import Any + +import pytest + +from thirdeye.config import Config +from thirdeye.paths import session_dir +from thirdeye.platforms.copilot import projection_store +from thirdeye.platforms.copilot.archive import commit_batch, iter_captured_records +from thirdeye.platforms.copilot.constants import PLATFORM_NAME +from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id +from thirdeye.platforms.copilot.projection import build_projection as _real_build_projection +from thirdeye.platforms.copilot.projection_state import projection_state_path +from thirdeye.platforms.copilot.projection_store import ( + ProjectionConflictError, + commit_projection, + load_projection_state, + read_projected_turns, +) +from thirdeye.platforms.copilot.reconcile import reconcile_archive +from thirdeye.platforms.copilot.transcript import read_transcript +from thirdeye.platforms.copilot.types import SourceBatch, SourcePaths, SourceRecord +from thirdeye.reader import SessionReader + +FIXTURES = Path(__file__).parent / "fixtures" +RECON = FIXTURES / "reconciliation-cases" +NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" +FULL_NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd7" +SOURCE_KEY = "a" * 64 +GENERATION = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" +OBSERVED_AT = "2026-09-10T17:09:00.000Z" +RESULT_KEYS = ( + "events", + "usage", + "turns", + "exports", + "pending", + "ambiguous", + "conflicting", + "errors", +) + + +def _load_json(path: Path) -> Any: + return json.loads(path.read_text(encoding="utf-8")) + + +def _directory(config: Config, stored: str) -> Path: + return session_dir(config.root, PLATFORM_NAME, stored) + + +def _document(config: Config, stored: str) -> dict[str, Any]: + return json.loads(projection_state_path(_directory(config, stored)).read_text()) + + +def _normalized_index_values( + config: Config, stored: str, native_session_id: str, index_name: str +) -> list[Any]: + index = _document(config, stored)["indexes"][index_name] + normalized = [_rewrite_native_id(item, native_session_id) for item in index.values()] + return sorted(normalized, key=lambda item: json.dumps(item, sort_keys=True, default=str)) + + +def _drain_cli_transcript( + home: Path, *, native_session_id: str = NATIVE_SESSION_ID +) -> list[SourceRecord]: + session_dir = home / "session-state" / native_session_id + session_dir.mkdir(parents=True, exist_ok=True) + shutil.copy(FIXTURES / "events.jsonl", session_dir / "events.jsonl") + (session_dir / "workspace.yaml").write_text("cwd: /sanitized/workspace\n", encoding="utf-8") + paths = resolve_sources(home) + cursor: dict[str, Any] = {} + records: list[SourceRecord] = [] + while True: + slice_ = read_transcript(paths, native_session_id, cursor) + records.extend(slice_["records"]) + cursor = slice_["next_cursor"] + if slice_["exhausted"]: + break + return [record for record in records if record["source_kind"] == "transcript"] + + +def _source_key(records: list[SourceRecord]) -> str: + for record in records: + if record["source_kind"] == "transcript": + return record["source_id"].split("/", 1)[0] + pytest.fail("expected at least one transcript record") + + +def _usage_record( + row: dict[str, Any], + *, + content_revision: str, + generation: str = GENERATION, + source_key: str = SOURCE_KEY, + native_session_id: str = NATIVE_SESSION_ID, + observed_at: str = OBSERVED_AT, +) -> SourceRecord: + primary_key = row["id"] + source_id = ( + f"copilot-db:{source_key}:{native_session_id}:assistant_usage_events:" + f"{primary_key}:{content_revision}" + ) + return { + "source_id": source_id, + "source_kind": "database", + "native_session_id": native_session_id, + "ts": row.get("created_at"), + "observed_at": observed_at, + "payload": {"table": "assistant_usage_events", "row": row}, + "locator": { + "database": "/example/.copilot/session-store.db", + "table": "assistant_usage_events", + "primary_key": primary_key, + "content_revision": content_revision, + "generation": generation, + }, + } + + +def _six_call_records( + *, source_key: str = SOURCE_KEY, native_session_id: str = NATIVE_SESSION_ID +) -> list[SourceRecord]: + rows = _load_json(FIXTURES / "assistant-usage-events.json") + revisions = { + call["row_id"]: call["usage_source_id"].rsplit(":", 1)[-1] + for call in _load_json(RECON / "observed-six-calls.json")["calls"] + } + return [ + _usage_record( + row, + content_revision=revisions[row["id"]], + source_key=source_key, + native_session_id=native_session_id, + ) + for row in rows + ] + + +def _batch( + paths: SourcePaths, + records: list[SourceRecord], + *, + native_session_id: str = NATIVE_SESSION_ID, +) -> SourceBatch: + return { + "source_key": paths["source_key"], + "native_session_id": native_session_id, + "cwd": "/proj", + "records": records, + "next_cursor": {"generation": 1}, + "diagnostics": [], + } + + +def _seed_archive( + config: Config, + paths: SourcePaths, + records: list[SourceRecord], + *, + native_session_id: str = NATIVE_SESSION_ID, +) -> str: + commit_batch(config, paths, _batch(paths, records, native_session_id=native_session_id)) + return stored_session_id(paths, native_session_id) + + +def _substitute_source_key(value: str, source_key: str) -> str: + return value.replace(SOURCE_KEY, source_key) + + +def _rewrite_source_key(value: Any, source_key: str) -> Any: + if isinstance(value, str): + return _substitute_source_key(value, source_key) + if isinstance(value, list): + return [_rewrite_source_key(item, source_key) for item in value] + if isinstance(value, dict): + return {key: _rewrite_source_key(item, source_key) for key, item in value.items()} + return value + + +_VOLATILE_KEYS = frozenset({"observed_at", "locator"}) + + +def _rewrite_native_id(value: Any, native_session_id: str, placeholder: str = "NATIVE") -> Any: + """Normalize identity fields that legitimately differ between two + + independently seeded stored sessions built from the same fixture content: + the native session ID baked into IDs/paths, each session's own wall-clock + capture timestamp, and each archived copy's own filesystem locator (byte + offsets/inode generation are per-file-instance, not semantic content). + """ + if isinstance(value, str): + for identity in {native_session_id, NATIVE_SESSION_ID, FULL_NATIVE_SESSION_ID}: + value = value.replace(identity, placeholder) + return value + if isinstance(value, list): + return [_rewrite_native_id(item, native_session_id, placeholder) for item in value] + if isinstance(value, dict): + return { + key: _rewrite_native_id(item, native_session_id, placeholder) + for key, item in value.items() + if key not in _VOLATILE_KEYS + } + return value + + +def _append_archive( + config: Config, + paths: SourcePaths, + records: list[SourceRecord], + *, + generation: int, + native_session_id: str = NATIVE_SESSION_ID, +) -> None: + commit_batch( + config, + paths, + { + **_batch(paths, records, native_session_id=native_session_id), + "next_cursor": {"generation": generation}, + }, + ) + + +@pytest.fixture +def config(tmp_path: Path) -> Config: + return Config(root=tmp_path / "thirdeye") + + +@pytest.fixture +def copilot_home(tmp_path: Path) -> Path: + home = tmp_path / "copilot-home" + home.mkdir() + return home + + +@pytest.fixture +def paths(copilot_home: Path) -> SourcePaths: + return resolve_sources(copilot_home) + + +@pytest.fixture +def cli_transcript_records(copilot_home: Path) -> list[SourceRecord]: + return _drain_cli_transcript(copilot_home) + + +def _full_corpus_records( + cli_transcript_records: list[SourceRecord], *, native_session_id: str = NATIVE_SESSION_ID +) -> list[SourceRecord]: + source_key = _source_key(cli_transcript_records) + return cli_transcript_records + _six_call_records( + source_key=source_key, native_session_id=native_session_id + ) + + +def _seed_full_corpus( + config: Config, + paths: SourcePaths, + cli_transcript_records: list[SourceRecord], + *, + native_session_id: str = NATIVE_SESSION_ID, +) -> str: + return _seed_archive( + config, + paths, + _full_corpus_records(cli_transcript_records, native_session_id=native_session_id), + native_session_id=native_session_id, + ) + + +def test_reconcile_archive_projects_six_call_corpus( + config: Config, + paths: SourcePaths, + cli_transcript_records: list[SourceRecord], +) -> None: + stored = _seed_full_corpus(config, paths, cli_transcript_records) + + result = reconcile_archive(config, stored) + + assert set(result.keys()) == set(RESULT_KEYS) + assert result["exports"] == 0 + assert result["errors"] == 0 + assert result["usage"] == 6 + assert result["turns"] == 2 + assert result["events"] > 0 + assert result["ambiguous"] == 0 + assert result["conflicting"] == 0 + turns = read_projected_turns(config, stored) + assert len(turns) == 2 + state = load_projection_state(config, stored) + assert len(state["archive_source_ids"]) == len(list(iter_captured_records(config, stored))) + + +def test_reconcile_archive_is_idempotent( + config: Config, + paths: SourcePaths, + cli_transcript_records: list[SourceRecord], +) -> None: + stored = _seed_full_corpus(config, paths, cli_transcript_records) + + first = reconcile_archive(config, stored) + first_turns = read_projected_turns(config, stored) + first_state = load_projection_state(config, stored) + first_document = _document(config, stored) + + second = reconcile_archive(config, stored) + second_turns = read_projected_turns(config, stored) + second_state = load_projection_state(config, stored) + second_document = _document(config, stored) + + assert first == second + assert first_turns == second_turns + # commit_sequence is a storage-owned counter that advances on every + # successful commit_projection call, whether or not its content changed + # -- it must not be idempotent, unlike everything else in builder state. + assert second_state["commit_sequence"] == first_state["commit_sequence"] + 1 + assert {k: v for k, v in first_state.items() if k != "commit_sequence"} == { + k: v for k, v in second_state.items() if k != "commit_sequence" + } + for turn in second_turns: + seqs = [event.get("seq") for event in turn.get("events") or []] + assert len(seqs) == len(set(seqs)), "re-reconciling must not duplicate turn events" + # Full raw indexes (including each turn's nested accounting_calls, which + # read_projected_turns intentionally does not surface) must be byte-for- + # byte identical across the two re-derivations: re-running reconcile + # must not accumulate duplicate accounting calls or usage/attribution + # entries inside any index. + normalized_first = {k: v for k, v in first_document["state"].items() if k != "commit_sequence"} + normalized_second = { + k: v for k, v in second_document["state"].items() if k != "commit_sequence" + } + assert normalized_first == normalized_second + assert first_document["indexes"] == second_document["indexes"] + for record in second_document["indexes"]["turns"].values(): + calls = record["span"].get("accounting_calls", []) + call_ids = [call["accounting_id"] for call in calls] + assert len(call_ids) == len(set(call_ids)) + + +def test_rebuild_is_idempotent_and_matches_initial_reconcile( + config: Config, + paths: SourcePaths, + cli_transcript_records: list[SourceRecord], +) -> None: + stored = _seed_full_corpus(config, paths, cli_transcript_records) + initial = reconcile_archive(config, stored) + first_rebuild = reconcile_archive(config, stored, rebuild=True) + second_rebuild = reconcile_archive(config, stored, rebuild=True) + + assert first_rebuild == second_rebuild + assert first_rebuild["events"] == initial["events"] + assert first_rebuild["usage"] == initial["usage"] + assert first_rebuild["turns"] == initial["turns"] + assert first_rebuild["exports"] == 0 + + +def test_incremental_reconcile_matches_full_replay( + config: Config, + paths: SourcePaths, + copilot_home: Path, + cli_transcript_records: list[SourceRecord], +) -> None: + source_key = _source_key(cli_transcript_records) + usage = _six_call_records(source_key=source_key) + partial = cli_transcript_records[: len(cli_transcript_records) // 2] + remainder = cli_transcript_records[len(cli_transcript_records) // 2 :] + + stored_incremental = _seed_archive(config, paths, partial) + reconcile_archive(config, stored_incremental) + _append_archive(config, paths, remainder + usage, generation=2) + incremental = reconcile_archive(config, stored_incremental) + incremental_turns = read_projected_turns(config, stored_incremental) + + # A genuinely independent stored session -- same source home, but a + # different native session ID -- committed in one shot from the same + # fixture content. Reusing the incremental session's own ID here would + # make "full" just another reconcile of the archive the incremental case + # already fully populated, which proves nothing about replay equivalence. + full_transcript = _drain_cli_transcript(copilot_home, native_session_id=FULL_NATIVE_SESSION_ID) + stored_full = _seed_full_corpus( + config, paths, full_transcript, native_session_id=FULL_NATIVE_SESSION_ID + ) + full = reconcile_archive(config, stored_full) + full_turns = read_projected_turns(config, stored_full) + incremental_document = _document(config, stored_incremental) + full_document = _document(config, stored_full) + + assert incremental == full + normalized_incremental = _rewrite_native_id(incremental_turns, NATIVE_SESSION_ID) + normalized_full = _rewrite_native_id(full_turns, FULL_NATIVE_SESSION_ID) + assert normalized_incremental == normalized_full + + # Builder state must agree independently of the storage-owned commit + # counter and identity-dependent document digest. + incremental_state = copy.deepcopy(incremental_document["state"]) + full_state = copy.deepcopy(full_document["state"]) + for state in (incremental_state, full_state): + state.pop("commit_sequence", None) + state.pop("projection_revision", None) + assert _rewrite_native_id(incremental_state, NATIVE_SESSION_ID) == _rewrite_native_id( + full_state, FULL_NATIVE_SESSION_ID + ) + + # Index keys can fall back to a content digest computed over pre- + # normalization payloads (see _index_key), so two sessions built from the + # same content under different native IDs are not guaranteed to share + # digest-fallback keys even though their *content* is equivalent. + # Comparing normalized values as an order-independent multiset avoids + # that false negative while still catching a real divergence (a usage + # row, attribution, pending item, or diagnostic present under one path + # and not the other, or duplicated under either). + # Comparing every index's normalized values includes raw stored turns and + # their nested accounting_calls, plus events and the usage identity map. + for name in incremental_document["indexes"]: + incremental_values = _normalized_index_values( + config, stored_incremental, NATIVE_SESSION_ID, name + ) + full_values = _normalized_index_values(config, stored_full, FULL_NATIVE_SESSION_ID, name) + assert incremental_values == full_values, f"{name} index diverged between replay paths" + + +def test_reconcile_default_does_not_export( + config: Config, + paths: SourcePaths, + cli_transcript_records: list[SourceRecord], + monkeypatch: pytest.MonkeyPatch, +) -> None: + stored = _seed_full_corpus(config, paths, cli_transcript_records) + + def explode(*_args: Any, **_kwargs: Any) -> Any: + raise AssertionError("reconcile_archive must not import export assembly by default") + + # Patch the name bound inside reconcile.py itself: `import_module` there + # is `from importlib import import_module`, a local reference that a + # patch on `importlib.import_module` would not intercept. + monkeypatch.setattr("thirdeye.platforms.copilot.reconcile.import_module", explode) + + result = reconcile_archive(config, stored) + + assert result["exports"] == 0 + assert result["errors"] == 0 + + +def test_export_failure_does_not_roll_back_projection( + config: Config, + paths: SourcePaths, + cli_transcript_records: list[SourceRecord], + monkeypatch: pytest.MonkeyPatch, +) -> None: + stored = _seed_full_corpus(config, paths, cli_transcript_records) + + def boom(*_args: Any, **_kwargs: Any) -> int: + raise RuntimeError("export unavailable") + + monkeypatch.setattr( + "thirdeye.platforms.copilot.reconcile.queue_exports", + boom, + ) + + result = reconcile_archive(config, stored, export=True) + + assert result["turns"] == 2 + assert result["usage"] == 6 + assert result["exports"] == 0 + assert result["errors"] == 1 + assert len(read_projected_turns(config, stored)) == 2 + + +def test_projection_failure_preserves_prior_derived_state( + config: Config, + paths: SourcePaths, + cli_transcript_records: list[SourceRecord], + monkeypatch: pytest.MonkeyPatch, +) -> None: + stored = _seed_full_corpus(config, paths, cli_transcript_records) + baseline = reconcile_archive(config, stored) + prior_turns = read_projected_turns(config, stored) + + def explode(*_args: Any, **_kwargs: Any) -> tuple[Any, dict[str, Any]]: + raise RuntimeError("projection failed") + + monkeypatch.setattr("thirdeye.platforms.copilot.reconcile.build_projection", explode) + + result = reconcile_archive(config, stored) + + assert result["errors"] == 1 + assert result["events"] == baseline["events"] + assert result["usage"] == baseline["usage"] + assert result["turns"] == baseline["turns"] + assert read_projected_turns(config, stored) == prior_turns + + +def test_build_failure_during_rebuild_preserves_prior_projection( + config: Config, + paths: SourcePaths, + cli_transcript_records: list[SourceRecord], + monkeypatch: pytest.MonkeyPatch, +) -> None: + stored = _seed_full_corpus(config, paths, cli_transcript_records) + reconcile_archive(config, stored) + prior_turns = read_projected_turns(config, stored) + + def explode(*_args: Any, **_kwargs: Any) -> tuple[Any, dict[str, Any]]: + raise RuntimeError("projection failed") + + monkeypatch.setattr("thirdeye.platforms.copilot.reconcile.build_projection", explode) + + result = reconcile_archive(config, stored, rebuild=True) + + assert result["errors"] == 1 + assert read_projected_turns(config, stored) == prior_turns + + +def test_commit_failure_during_rebuild_preserves_prior_projection( + config: Config, + paths: SourcePaths, + cli_transcript_records: list[SourceRecord], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A rebuild that fails inside commit_projection (after build succeeds) + + must not lose the previous projection. This is the atomicity gap a + delete-then-commit rebuild would have: this test forces the failure to + happen after a valid projection has already been built, exercising the + commit step itself rather than the build step. + """ + stored = _seed_full_corpus(config, paths, cli_transcript_records) + baseline = reconcile_archive(config, stored) + prior_turns = read_projected_turns(config, stored) + prior_state = load_projection_state(config, stored) + + def build_with_invalid_usage_row( + records: list[SourceRecord], prior_state: dict[str, Any] + ) -> tuple[Any, dict[str, Any]]: + projection, next_state = _real_build_projection(records, prior_state) + broken = dict(projection) + broken["usage_rows"] = [*projection["usage_rows"], {"not": "a UsageRow instance"}] + return broken, next_state + + monkeypatch.setattr( + "thirdeye.platforms.copilot.reconcile.build_projection", build_with_invalid_usage_row + ) + + result = reconcile_archive(config, stored, rebuild=True) + + assert result["errors"] == baseline["errors"] + 1 + assert result["turns"] == baseline["turns"] + assert result["usage"] == baseline["usage"] + assert read_projected_turns(config, stored) == prior_turns + assert load_projection_state(config, stored) == prior_state + + +def test_commit_projection_rejects_a_stale_base_commit_sequence( + config: Config, + paths: SourcePaths, + cli_transcript_records: list[SourceRecord], +) -> None: + """A commit built from state read before a concurrent writer committed + + must be refused rather than silently overwriting that newer commit. + This exercises commit_projection's optimistic-concurrency check + directly: build a projection from state observed at commit_sequence 1, + let another writer advance the stored session to commit_sequence 2, then + attempt to commit the stale one with its now-outdated base sequence. + """ + stored = _seed_full_corpus(config, paths, cli_transcript_records) + reconcile_archive(config, stored) + stale_state = load_projection_state(config, stored) + assert stale_state["commit_sequence"] == 1 + records = list(iter_captured_records(config, stored)) + stale_projection, stale_next_state = _real_build_projection(records, stale_state) + stale_next_state["archive_source_ids"] = [record["source_id"] for record in records] + + reconcile_archive(config, stored, rebuild=True) + newer_turns = read_projected_turns(config, stored) + newer_state = load_projection_state(config, stored) + assert newer_state["commit_sequence"] == 2 + + with pytest.raises(ProjectionConflictError): + commit_projection( + config, + stored, + stale_projection, + stale_next_state, + base_commit_sequence=stale_state["commit_sequence"], + ) + + assert read_projected_turns(config, stored) == newer_turns + assert load_projection_state(config, stored) == newer_state + + +def test_reconcile_reports_error_when_projection_advances_concurrently( + config: Config, + paths: SourcePaths, + cli_transcript_records: list[SourceRecord], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """reconcile_archive itself must surface a concurrent-writer conflict + + as an error rather than corrupt state, even though its own load, build, + and commit are not one held lock. A second writer (simulated here by + recursively calling reconcile_archive from inside a patched + commit_projection, guarded so it only races once) finishes a full + rebuild between this call's load and its commit; the racing call's own + stale commit must then be refused, and the session must be left exactly + as the interleaved rebuild left it. + """ + stored = _seed_full_corpus(config, paths, cli_transcript_records) + reconcile_archive(config, stored) + baseline_turns = read_projected_turns(config, stored) + + real_commit = projection_store.commit_projection + raced = {"done": False} + + def racing_commit( + cfg: Config, sid: str, projection: Any, next_state: dict[str, Any], **kwargs: Any + ) -> dict[str, int]: + if not raced["done"]: + raced["done"] = True + reconcile_archive(cfg, sid, rebuild=True) + return real_commit(cfg, sid, projection, next_state, **kwargs) + + monkeypatch.setattr("thirdeye.platforms.copilot.reconcile.commit_projection", racing_commit) + + result = reconcile_archive(config, stored) + + assert result["errors"] == 1 + assert read_projected_turns(config, stored) == baseline_turns + + +def test_usage_sidecar_publish_failure_reports_the_committed_document( + config: Config, + paths: SourcePaths, + cli_transcript_records: list[SourceRecord], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A post-validation I/O failure while publishing the usage-sidecar + + mirror is a distinct failure mode from a validation failure: by the time + it can happen, the projection document has already durably published + (see commit_projection's docstring), so the counts reconcile_archive + reports must reflect that new document -- not the prior one -- with the + error counted on top. The next reconcile call must self-heal the + sidecar without reprocessing anything new. + """ + stored = _seed_full_corpus(config, paths, cli_transcript_records) + baseline = reconcile_archive(config, stored) + prior_turns = read_projected_turns(config, stored) + baseline_sequence = load_projection_state(config, stored)["commit_sequence"] + + original_publish_usage = projection_store._publish_usage + + def selective_boom( + cfg: Config, sid: str, directory: Path, usage_index: dict[str, Any], *, force: bool = False + ) -> None: + # load_projection_state's self-heal always passes force=True; only + # commit_projection's own (non-forced) publish should fail here, so + # this reaches the specific "document committed, sidecar mirror + # failed" state the docstring describes rather than failing before + # commit_projection is ever entered. + if force: + original_publish_usage(cfg, sid, directory, usage_index, force=force) + return + raise OSError("disk full while rewriting usage sidecar") + + monkeypatch.setattr( + "thirdeye.platforms.copilot.projection_store._publish_usage", selective_boom + ) + + result = reconcile_archive(config, stored, rebuild=True) + + assert result["errors"] == baseline["errors"] + 1 + assert result["turns"] == baseline["turns"] + assert result["usage"] == baseline["usage"] + # The document committed despite the sidecar failure -- proven by the + # storage-owned commit_sequence advancing even though this call reported + # an error -- so turns already reflect the (content-equivalent, since + # this rebuilds the same archive) new projection rather than being stuck + # on the old one. Read the raw document rather than load_projection_state + # here: that call would itself retry the still-patched, still-failing + # sidecar publish as part of its own self-heal. + assert _document(config, stored)["state"]["commit_sequence"] == baseline_sequence + 1 + assert read_projected_turns(config, stored) == prior_turns + + monkeypatch.undo() + healed = reconcile_archive(config, stored) + assert healed["errors"] == 0 + assert read_projected_turns(config, stored) == prior_turns + + +def test_reconcile_unknown_session_reports_error_without_creating_paths( + config: Config, +) -> None: + ghost = "copilot-ghost-session" + + result = reconcile_archive(config, ghost) + + assert result["errors"] == 1 + assert result["exports"] == 0 + assert all(result[key] == 0 for key in RESULT_KEYS if key not in {"errors", "exports"}) + assert not (config.root / "traces" / "copilot" / ghost).exists() + + +def test_reconcile_works_without_live_copilot_source_files( + config: Config, + copilot_home: Path, + cli_transcript_records: list[SourceRecord], +) -> None: + paths = resolve_sources(copilot_home) + stored = _seed_full_corpus(config, paths, cli_transcript_records) + shutil.rmtree(copilot_home) + + result = reconcile_archive(config, stored, rebuild=True) + + assert result["errors"] == 0 + assert result["turns"] == 2 + assert result["usage"] == 6 + assert len(list(iter_captured_records(config, stored))) > 0 + + +def test_rebuild_preserves_immutable_archive_records( + config: Config, + paths: SourcePaths, + cli_transcript_records: list[SourceRecord], +) -> None: + stored = _seed_full_corpus(config, paths, cli_transcript_records) + before = { + event["data"]["source_record"]["source_id"] + for event in SessionReader(config.root / "traces" / "copilot" / stored).iter_events() + } + + reconcile_archive(config, stored, rebuild=True) + + after = { + event["data"]["source_record"]["source_id"] + for event in SessionReader(config.root / "traces" / "copilot" / stored).iter_events() + } + assert before == after + + +def test_late_row_archive_reconcile_resolves_after_full_replay( + config: Config, + paths: SourcePaths, +) -> None: + source_key = paths["source_key"] + case = _load_json(RECON / "cases.json")["late_row"] + partial = _rewrite_source_key(case["input_records"], source_key) + final_answer = _rewrite_source_key( + _load_json(RECON / "cases.json")["ambiguous"]["input_records"][1], + source_key, + ) + turn_end = copy.deepcopy(final_answer) + turn_end.update( + { + "source_id": (f"{source_key}/{NATIVE_SESSION_ID}/4a386a37-ca7e-4ebc-a746-cdda20f2a4bb"), + "payload": { + "type": "assistant.turn_end", + "data": {"turnId": "1"}, + "id": "4a386a37-ca7e-4ebc-a746-cdda20f2a4bb", + "timestamp": "2026-09-10T17:08:25.626Z", + "parentId": "33cc6465-29e1-4a04-8bdb-00241474b4d2", + "schema_version": 1, + }, + } + ) + full = partial + [final_answer, turn_end] + + stored = _seed_archive(config, paths, partial) + pending = reconcile_archive(config, stored) + assert pending["errors"] == 0 + assert pending["usage"] == 1 + assert pending["pending"] >= 1 + + _append_archive(config, paths, [final_answer, turn_end], generation=2) + resolved = reconcile_archive(config, stored) + + assert resolved["errors"] == 0 + assert resolved["usage"] == 1 + assert resolved["pending"] < pending["pending"] + state = load_projection_state(config, stored) + assert len(state["archive_source_ids"]) == len(full) diff --git a/tests/platforms/copilot/test_recovery.py b/tests/platforms/copilot/test_recovery.py new file mode 100644 index 00000000..f5f15da6 --- /dev/null +++ b/tests/platforms/copilot/test_recovery.py @@ -0,0 +1,471 @@ +"""Crash recovery, journal replay, and stale-cursor contention for Copilot archive.""" + +from __future__ import annotations + +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path +from typing import Any + +import pytest + +import thirdeye.platforms.copilot.archive as archive_mod +import thirdeye.platforms.copilot.state as state_mod +from thirdeye._compat import fsops +from thirdeye.config import Config +from thirdeye.meta import read_meta +from thirdeye.paths import meta_path +from thirdeye.platforms.copilot.archive import commit_batch, iter_captured_records, load_cursor +from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id +from thirdeye.platforms.copilot.state import journal_path, read_json, state_path +from thirdeye.platforms.copilot.types import SourceBatch, SourcePaths, SourceRecord + +NATIVE_ID = "session-a" + + +def _record( + source_id: str, + *, + native_session_id: str = NATIVE_ID, + source_kind: str = "transcript", + payload: dict[str, Any] | None = None, +) -> SourceRecord: + return { + "source_id": source_id, + "source_kind": source_kind, + "native_session_id": native_session_id, + "ts": "2026-09-10T17:08:24.000Z", + "observed_at": "2026-09-10T17:08:25.000Z", + "payload": payload or {"schema_version": 1, "type": "user.message"}, + "locator": {"file": "events.jsonl", "offset": 0}, + } + + +def _batch( + paths: SourcePaths, + records: list[SourceRecord], + *, + next_cursor: dict[str, Any], + base_cursor: dict[str, Any] | None = None, +) -> SourceBatch: + cursor = dict(next_cursor) + if base_cursor is not None: + cursor["base_cursor"] = base_cursor + return { + "source_key": paths["source_key"], + "native_session_id": NATIVE_ID, + "cwd": "/proj", + "records": records, + "next_cursor": cursor, + "diagnostics": [], + } + + +@contextmanager +def fault_at(point: str) -> Iterator[None]: + seen: list[str] = [] + + def injector(name: str) -> None: + seen.append(name) + if name == point: + raise RuntimeError(f"injected fault at {point}") + + archive_mod._fault_injector = injector + state_mod._fault_injector = injector + try: + yield seen + finally: + archive_mod._fault_injector = None + state_mod._fault_injector = None + + +@pytest.fixture +def config(tmp_path) -> Config: + return Config(root=tmp_path / "thirdeye") + + +@pytest.fixture +def paths(tmp_path) -> SourcePaths: + home = tmp_path / "copilot-home" + home.mkdir() + return resolve_sources(home) + + +def _session_dir(config: Config, paths: SourcePaths): + from thirdeye.paths import session_dir + from thirdeye.platforms.copilot.constants import PLATFORM_NAME + + return session_dir(config.root, PLATFORM_NAME, stored_session_id(paths, NATIVE_ID)) + + +@pytest.mark.parametrize( + "fault_point", + [ + "after_journal", + "after_append", + "after_checkpoint", + "after_journal_clear", + ], +) +def test_crash_at_journal_boundary_recovers_on_next_commit( + config: Config, + paths: SourcePaths, + fault_point: str, +) -> None: + directory = _session_dir(config, paths) + records = [_record("key/a/recover-1"), _record("key/a/recover-2")] + + with fault_at(fault_point): + with pytest.raises(RuntimeError, match="injected fault"): + commit_batch( + config, + paths, + _batch(paths, records, next_cursor={"generation": 1}), + ) + + if fault_point == "after_journal": + assert journal_path(directory).is_file() + + if fault_point in {"after_append", "after_checkpoint", "after_journal_clear"}: + captured_before = list(iter_captured_records(config, stored_session_id(paths, NATIVE_ID))) + assert len(captured_before) == 2 + + recovery = commit_batch( + config, + paths, + _batch( + paths, + [_record("key/a/recover-3")], + next_cursor={"generation": 2}, + base_cursor={"generation": 1}, + ), + ) + + assert recovery["records_written"] >= 1 + assert not journal_path(directory).exists() + cursor = load_cursor(config, paths, NATIVE_ID) + assert cursor == {"generation": 2} + captured = list(iter_captured_records(config, stored_session_id(paths, NATIVE_ID))) + assert {r["source_id"] for r in captured} >= { + "key/a/recover-1", + "key/a/recover-2", + "key/a/recover-3", + } + + +def test_recovery_after_journal_only_replays_uncommitted_records( + config: Config, + paths: SourcePaths, +) -> None: + directory = _session_dir(config, paths) + records = [_record("key/a/journal-only")] + + with fault_at("after_journal"): + with pytest.raises(RuntimeError): + commit_batch(config, paths, _batch(paths, records, next_cursor={"generation": 1})) + + assert journal_path(directory).is_file() + result = commit_batch( + config, + paths, + _batch(paths, [], next_cursor={"generation": 1}, base_cursor={}), + ) + assert result["records_written"] == 1 + assert result["duplicate_records"] == 0 + assert ( + list(iter_captured_records(config, stored_session_id(paths, NATIVE_ID)))[0]["source_id"] + == "key/a/journal-only" + ) + + +def test_stale_cursor_rejects_competing_batch(config: Config, paths: SourcePaths) -> None: + first = commit_batch( + config, + paths, + _batch(paths, [_record("key/a/first")], next_cursor={"generation": 1}), + ) + assert first["errors"] == 0 + + stale = commit_batch( + config, + paths, + _batch( + paths, + [_record("key/a/stale")], + next_cursor={"generation": 99}, + base_cursor={}, + ), + ) + assert stale == { + "sessions": 1, + "records_written": 0, + "duplicate_records": 0, + "pending": 1, + "errors": 1, + } + assert load_cursor(config, paths, NATIVE_ID) == {"generation": 1} + captured = list(iter_captured_records(config, stored_session_id(paths, NATIVE_ID))) + assert {r["source_id"] for r in captured} == {"key/a/first"} + + state = read_json(state_path(_session_dir(config, paths))) + assert state is not None + assert any(item.get("kind") == "stale_cursor" for item in state["health"]["diagnostics"]) + + +def test_two_competing_batches_second_wins_first_becomes_stale( + config: Config, + paths: SourcePaths, +) -> None: + batch_a = _batch( + paths, + [_record("key/a/contender-a")], + next_cursor={"generation": 1}, + base_cursor={}, + ) + batch_b = _batch( + paths, + [_record("key/a/contender-b")], + next_cursor={"generation": 1}, + base_cursor={}, + ) + + winner = commit_batch(config, paths, batch_b) + assert winner["records_written"] == 1 + + loser = commit_batch(config, paths, batch_a) + assert loser["errors"] == 1 + assert loser["pending"] == 1 + assert loser["records_written"] == 0 + + captured = list(iter_captured_records(config, stored_session_id(paths, NATIVE_ID))) + assert {r["source_id"] for r in captured} == {"key/a/contender-b"} + + +def test_recovery_replay_is_idempotent_when_events_already_committed( + config: Config, + paths: SourcePaths, +) -> None: + directory = _session_dir(config, paths) + records = [_record("key/a/idempotent")] + + with fault_at("after_append"): + with pytest.raises(RuntimeError): + commit_batch(config, paths, _batch(paths, records, next_cursor={"generation": 1})) + + assert journal_path(directory).is_file() + first_recovery = commit_batch( + config, + paths, + _batch(paths, [], next_cursor={"generation": 1}, base_cursor={}), + ) + assert first_recovery["records_written"] == 0 + assert first_recovery["duplicate_records"] == 1 + assert not journal_path(directory).exists() + + second_recovery = commit_batch( + config, + paths, + _batch( + paths, + [_record("key/a/next")], + next_cursor={"generation": 2}, + base_cursor={"generation": 1}, + ), + ) + assert second_recovery["records_written"] == 1 + assert second_recovery["duplicate_records"] == 0 + + +def _session_end_record(source_id: str = "key/a/close") -> SourceRecord: + return _record( + source_id, + source_kind="hook", + payload={"event": "sessionEnd", "context": {}}, + ) + + +def _resume_record(source_id: str = "key/a/resume") -> SourceRecord: + return _record( + source_id, + source_kind="hook", + payload={"event": "resume", "context": {}}, + ) + + +@pytest.mark.parametrize( + "fault_point", + [ + "after_journal", + "after_append", + "after_lifecycle", + "after_checkpoint", + "after_journal_clear", + ], +) +def test_session_end_survives_crash_at_every_boundary( + config: Config, + paths: SourcePaths, + fault_point: str, +) -> None: + directory = _session_dir(config, paths) + with fault_at(fault_point): + with pytest.raises(RuntimeError, match="injected fault"): + commit_batch( + config, + paths, + _batch(paths, [_session_end_record()], next_cursor={"generation": 1}), + ) + + cursor = load_cursor(config, paths, NATIVE_ID) + assert cursor == {"generation": 1} + assert not journal_path(directory).exists() + meta = read_meta(meta_path(directory)) + assert meta is not None + assert meta.status == "closed" + assert meta.ended_at is not None + + +@pytest.mark.parametrize( + "fault_point", + [ + "after_journal", + "after_append", + "after_lifecycle", + "after_checkpoint", + "after_journal_clear", + ], +) +def test_resume_survives_crash_at_every_boundary( + config: Config, + paths: SourcePaths, + fault_point: str, +) -> None: + directory = _session_dir(config, paths) + commit_batch( + config, + paths, + _batch(paths, [_session_end_record()], next_cursor={"generation": 1}), + ) + closed = read_meta(meta_path(directory)) + assert closed is not None + assert closed.status == "closed" + + with fault_at(fault_point): + with pytest.raises(RuntimeError, match="injected fault"): + commit_batch( + config, + paths, + _batch( + paths, + [_resume_record()], + next_cursor={"generation": 2}, + base_cursor={"generation": 1}, + ), + ) + + cursor = load_cursor(config, paths, NATIVE_ID) + assert cursor == {"generation": 2} + assert not journal_path(directory).exists() + meta = read_meta(meta_path(directory)) + assert meta is not None + assert meta.status == "open" + assert meta.ended_at is None + + +def test_load_cursor_completes_recovery_without_new_batch( + config: Config, + paths: SourcePaths, +) -> None: + directory = _session_dir(config, paths) + with fault_at("after_journal"): + with pytest.raises(RuntimeError): + commit_batch( + config, + paths, + _batch(paths, [_record("key/a/journal-only")], next_cursor={"generation": 1}), + ) + + assert journal_path(directory).is_file() + cursor = load_cursor(config, paths, NATIVE_ID) + assert cursor == {"generation": 1} + assert not journal_path(directory).exists() + captured = list(iter_captured_records(config, stored_session_id(paths, NATIVE_ID))) + assert [record["source_id"] for record in captured] == ["key/a/journal-only"] + + +def test_iter_captured_records_completes_recovery_without_new_batch( + config: Config, + paths: SourcePaths, +) -> None: + directory = _session_dir(config, paths) + with fault_at("after_journal"): + with pytest.raises(RuntimeError): + commit_batch( + config, + paths, + _batch(paths, [_record("key/a/via-iter")], next_cursor={"generation": 1}), + ) + + assert journal_path(directory).is_file() + captured = list(iter_captured_records(config, stored_session_id(paths, NATIVE_ID))) + assert [record["source_id"] for record in captured] == ["key/a/via-iter"] + assert load_cursor(config, paths, NATIVE_ID) == {"generation": 1} + assert not journal_path(directory).exists() + + +def test_state_publication_fsyncs_directory_after_replace_and_unlink( + config: Config, + paths: SourcePaths, + monkeypatch: pytest.MonkeyPatch, +) -> None: + order: list[str] = [] + real_replace = fsops.replace + real_unlink = fsops.unlink + real_sync = fsops.sync_directory + + def replace(src: Path | str, dst: Path | str) -> None: + order.append(f"replace:{Path(dst).name}") + real_replace(src, dst) + + def unlink(path: Path, *, missing_ok: bool = False) -> None: + order.append(f"unlink:{path.name}") + real_unlink(path, missing_ok=missing_ok) + + def sync_directory(path: Path) -> None: + order.append(f"dirsync:{path.name}") + real_sync(path) + + monkeypatch.setattr(fsops, "replace", replace) + monkeypatch.setattr(fsops, "unlink", unlink) + monkeypatch.setattr(fsops, "sync_directory", sync_directory) + + commit_batch( + config, + paths, + _batch(paths, [_record("key/a/durable")], next_cursor={"generation": 1}), + ) + + pairs = list(zip(order, order[1:], strict=False)) + assert ("replace:copilot.journal.json", f"dirsync:{_session_dir(config, paths).name}") in pairs + assert ("replace:copilot.state.json", f"dirsync:{_session_dir(config, paths).name}") in pairs + assert ("unlink:copilot.journal.json", f"dirsync:{_session_dir(config, paths).name}") in pairs + + +def test_crash_between_replace_and_directory_sync_still_publishes_state( + config: Config, + paths: SourcePaths, +) -> None: + directory = _session_dir(config, paths) + with fault_at("after_state_replace"): + with pytest.raises(RuntimeError, match="injected fault"): + commit_batch( + config, + paths, + _batch(paths, [_record("key/a/dirsync")], next_cursor={"generation": 1}), + ) + + assert state_path(directory).is_file() + cursor = load_cursor(config, paths, NATIVE_ID) + assert cursor == {"generation": 1} + captured = list(iter_captured_records(config, stored_session_id(paths, NATIVE_ID))) + assert [record["source_id"] for record in captured] == ["key/a/dirsync"] diff --git a/tests/platforms/copilot/test_runtime.py b/tests/platforms/copilot/test_runtime.py new file mode 100644 index 00000000..88b3e3c2 --- /dev/null +++ b/tests/platforms/copilot/test_runtime.py @@ -0,0 +1,631 @@ +"""Integration tests for Copilot runtime reconciliation wiring.""" + +from __future__ import annotations + +import json +import shutil +import sys +from pathlib import Path +from typing import Any + +import pytest + +from thirdeye.config import Config, LogfireSettings +from thirdeye.paths import otel_jobs_dir, session_dir +from thirdeye.platforms.copilot import watch as watch_mod +from thirdeye.platforms.copilot.archive import commit_batch +from thirdeye.platforms.copilot.constants import PLATFORM_NAME +from thirdeye.platforms.copilot.export_state import ( + load_export_state, + record_placement, + update_export_state, +) +from thirdeye.platforms.copilot.export_transport import delivery_claim_path, job_path +from thirdeye.platforms.copilot.followup import _claim_lease, _run +from thirdeye.platforms.copilot.followup import main as followup_main +from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id +from thirdeye.platforms.copilot.projection_store import read_projection_status +from thirdeye.platforms.copilot.runtime import ( + all_archived_session_ids, + archived_session_ids, + exports_configured, + load_runtime_status, + reconcile_archived_sessions, + reconcile_session, +) +from thirdeye.platforms.copilot.state import state_path +from thirdeye.platforms.copilot.status import capture_status +from thirdeye.platforms.copilot.types import SourceBatch, SourcePaths, SourceRecord, SyncResult +from thirdeye.usage.errlog import log_capture_error + +FIXTURES = Path(__file__).parent / "fixtures" +NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" +OTHER_SESSION_ID = "a1b2c3d4-e5f6-7890-abcd-ef1234567890" + + +@pytest.fixture +def copilot_env(tmp_path: Path) -> tuple[Config, SourcePaths]: + home = tmp_path / "copilot-home" + home.mkdir() + config = Config(root=tmp_path / "thirdeye") + paths = resolve_sources(home) + return config, paths + + +def _record(source_id: str, *, native_session_id: str = NATIVE_SESSION_ID) -> SourceRecord: + return { + "source_id": source_id, + "source_kind": "transcript", + "native_session_id": native_session_id, + "ts": "2026-09-10T17:08:24.000Z", + "observed_at": "2026-09-10T17:08:25.000Z", + "payload": {"schema_version": 1, "type": "user.message"}, + "locator": {"file": "events.jsonl", "offset": 0}, + } + + +def _batch( + paths: SourcePaths, + records: list[SourceRecord], + *, + native_session_id: str = NATIVE_SESSION_ID, +) -> SourceBatch: + return { + "source_key": paths["source_key"], + "native_session_id": native_session_id, + "cwd": "/proj", + "records": records, + "next_cursor": {"generation": 1}, + "diagnostics": [], + } + + +def _seed_archive( + config: Config, + paths: SourcePaths, + *, + native_session_id: str = NATIVE_SESSION_ID, +) -> str: + commit_batch( + config, + paths, + _batch( + paths, + [_record("runtime/archived", native_session_id=native_session_id)], + native_session_id=native_session_id, + ), + ) + return stored_session_id(paths, native_session_id) + + +def _empty_result(**overrides: int) -> SyncResult: + result: SyncResult = { + "sessions": 0, + "records_written": 0, + "duplicate_records": 0, + "pending": 0, + "errors": 0, + } + result.update(overrides) # type: ignore[typeddict-item] + return result + + +def test_exports_configured_requires_enabled_logfire_with_token() -> None: + assert exports_configured(Config(root=Path("/tmp/thirdeye"))) is False + assert ( + exports_configured( + Config( + root=Path("/tmp/thirdeye"), + logfire=LogfireSettings(enabled=True, token="fake-token"), + ) + ) + is True + ) + assert ( + exports_configured( + Config( + root=Path("/tmp/thirdeye"), + logfire=LogfireSettings(enabled=True, token=""), + ) + ) + is False + ) + + +def test_archived_session_ids_lists_sessions_for_source_home( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + stored = _seed_archive(config, paths) + _seed_archive(config, paths, native_session_id=OTHER_SESSION_ID) + + assert archived_session_ids(config, paths) == sorted( + [stored, stored_session_id(paths, OTHER_SESSION_ID)] + ) + + +def test_archived_session_ids_ignores_other_source_keys( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + stored = _seed_archive(config, paths) + directory = session_dir(config.root, PLATFORM_NAME, stored) + state = json.loads(state_path(directory).read_text(encoding="utf-8")) + state["source_key"] = "x" * 64 + state_path(directory).write_text(json.dumps(state) + "\n", encoding="utf-8") + + assert archived_session_ids(config, paths) == [] + + +def test_all_archived_session_ids_lists_every_copilot_archive( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + first = _seed_archive(config, paths) + second = _seed_archive(config, paths, native_session_id=OTHER_SESSION_ID) + assert all_archived_session_ids(config) == sorted([first, second]) + + +def test_reconcile_session_maps_native_id_to_stored_session( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + stored = _seed_archive(config, paths) + calls: list[tuple[str, bool, bool]] = [] + + def fake_reconcile_archive( + _config: Config, + stored_session_id: str, + *, + rebuild: bool = False, + export: bool = False, + include_history: bool = False, + ) -> dict[str, int]: + calls.append((stored_session_id, export, include_history)) + return { + "events": 1, + "usage": 0, + "turns": 1, + "exports": 0, + "pending": 0, + "ambiguous": 0, + "conflicting": 0, + "errors": 0, + } + + monkeypatch.setattr( + "thirdeye.platforms.copilot.runtime.reconcile_archive", fake_reconcile_archive + ) + result = reconcile_session(config, paths, NATIVE_SESSION_ID, export=True, include_history=True) + + assert result["turns"] == 1 + assert calls == [(stored, True, True)] + + +def test_reconcile_session_activates_eligibility_without_logfire( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + stored = _seed_archive(config, paths) + + result = reconcile_session(config, paths, NATIVE_SESSION_ID, export=True) + + assert result["exports"] == 0 + state = load_export_state(config, stored) + assert state["activated"] is True + restarted = load_export_state(config, stored) + assert restarted["activated"] is True + + +def test_reconcile_archived_sessions_replays_all_retained_sessions( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + first = _seed_archive(config, paths) + second = _seed_archive(config, paths, native_session_id=OTHER_SESSION_ID) + seen: list[str] = [] + + def fake_reconcile_archive( + _config: Config, + stored_session_id: str, + *, + rebuild: bool = False, + export: bool = False, + include_history: bool = False, + ) -> dict[str, int]: + seen.append(stored_session_id) + return { + "events": 0, + "usage": 0, + "turns": 0, + "exports": 0, + "pending": 0, + "ambiguous": 0, + "conflicting": 0, + "errors": 0, + } + + monkeypatch.setattr( + "thirdeye.platforms.copilot.runtime.reconcile_archive", fake_reconcile_archive + ) + results = reconcile_archived_sessions(config, paths) + + assert set(seen) == {first, second} + assert set(results) == {first, second} + + +def test_reconcile_archived_sessions_works_after_source_removal( + copilot_env: tuple[Config, SourcePaths], + tmp_path: Path, +) -> None: + config, paths = copilot_env + home = Path(paths["home"]) + session_dir_path = home / "session-state" / NATIVE_SESSION_ID + session_dir_path.mkdir(parents=True) + shutil.copy(FIXTURES / "events.jsonl", session_dir_path / "events.jsonl") + (session_dir_path / "workspace.yaml").write_text( + "cwd: /sanitized/workspace\n", encoding="utf-8" + ) + + from thirdeye.platforms.copilot.capture import sync + + sync(config, paths, session_id=NATIVE_SESSION_ID) + stored = stored_session_id(paths, NATIVE_SESSION_ID) + shutil.rmtree(home / "session-state") + (home / "session-store.db").unlink(missing_ok=True) + + result = reconcile_archived_sessions(config, paths)[stored] + + assert result["errors"] == 0 + assert read_projection_status(config, stored)["errors"] == 0 + + +def test_followup_reconciles_after_successful_capture( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + generation = _claim_lease(config, paths, NATIVE_SESSION_ID) + assert generation is not None + reconcile_calls: list[tuple[str, bool]] = [] + + def fake_capture(_config: Config, _paths: SourcePaths, _native_id: str) -> SyncResult: + return _empty_result(sessions=1, records_written=1) + + def fake_reconcile_session( + _config: Config, + _paths: SourcePaths, + native_session_id: str, + *, + export: bool = False, + include_history: bool = False, + ) -> dict[str, int]: + reconcile_calls.append((native_session_id, export)) + return { + "events": 0, + "usage": 0, + "turns": 0, + "exports": 0, + "pending": 0, + "ambiguous": 0, + "conflicting": 0, + "errors": 0, + } + + monkeypatch.setattr("thirdeye.platforms.copilot.capture.capture_session", fake_capture) + monkeypatch.setattr( + "thirdeye.platforms.copilot.runtime.reconcile_session", + fake_reconcile_session, + ) + _run(config, paths, NATIVE_SESSION_ID, generation) + + assert reconcile_calls == [(NATIVE_SESSION_ID, True)] + + +def test_followup_reconcile_failure_does_not_block_capture_completion( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + from thirdeye.platforms.copilot.followup import _lease_path + + generation = _claim_lease(config, paths, NATIVE_SESSION_ID) + assert generation is not None + directory = session_dir(config.root, PLATFORM_NAME, stored_session_id(paths, NATIVE_SESSION_ID)) + + def fake_capture(_config: Config, _paths: SourcePaths, _native_id: str) -> SyncResult: + return _empty_result(sessions=1) + + def boom(*_args: Any, **_kwargs: Any) -> dict[str, int]: + raise RuntimeError("projection failed") + + monkeypatch.setattr("thirdeye.platforms.copilot.capture.capture_session", fake_capture) + monkeypatch.setattr("thirdeye.platforms.copilot.runtime.reconcile_session", boom) + _run(config, paths, NATIVE_SESSION_ID, generation) + + assert not _lease_path(directory).exists() + log = config.root / "logs" / "usage-errors.jsonl" + assert log.is_file() + entries = [json.loads(line) for line in log.read_text(encoding="utf-8").splitlines()] + assert any(entry.get("phase") == "copilot_followup_reconcile" for entry in entries) + + +def test_followup_logs_returned_reconcile_errors( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + stored = _seed_archive(config, paths) + generation = _claim_lease(config, paths, NATIVE_SESSION_ID) + assert generation is not None + + def fake_capture(_config: Config, _paths: SourcePaths, _native_id: str) -> SyncResult: + return _empty_result(sessions=1) + + def fake_reconcile_archive(*_args: Any, **_kwargs: Any) -> dict[str, int]: + return { + "events": 0, + "usage": 0, + "turns": 0, + "exports": 0, + "pending": 0, + "ambiguous": 0, + "conflicting": 0, + "errors": 2, + } + + monkeypatch.setattr("thirdeye.platforms.copilot.capture.capture_session", fake_capture) + monkeypatch.setattr( + "thirdeye.platforms.copilot.runtime.reconcile_archive", fake_reconcile_archive + ) + _run(config, paths, NATIVE_SESSION_ID, generation) + + status = load_runtime_status(config, stored) + assert status["last_error"]["errors"] == 2 + log = config.root / "logs" / "usage-errors.jsonl" + entries = [json.loads(line) for line in log.read_text(encoding="utf-8").splitlines()] + assert any(entry.get("phase") == "copilot_reconcile" for entry in entries) + + +def test_followup_entrypoint_loads_persisted_logfire_settings( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + config.write_logfire_settings(LogfireSettings(enabled=True, token="live-token")) + seen: list[Config] = [] + + def fake_run(loaded: Config, _paths: SourcePaths, _native_id: str, _generation: str) -> None: + seen.append(loaded) + + monkeypatch.setattr("thirdeye.platforms.copilot.followup._run", fake_run) + monkeypatch.setattr( + sys, + "argv", + [ + "followup", + "--source-home", + paths["home"], + "--session-id", + NATIVE_SESSION_ID, + "--config-root", + str(config.root), + "--generation", + "deadbeef", + ], + ) + followup_main() + + assert len(seen) == 1 + assert seen[0].logfire.enabled is True + assert seen[0].logfire.token == "live-token" + + +def test_watch_activation_reconciles_archived_sessions( + monkeypatch: pytest.MonkeyPatch, + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + activation: list[bool] = [] + per_session: list[str] = [] + + def fake_sync( + _config: Config, _paths: SourcePaths, *, session_id: str | None = None + ) -> SyncResult: + return _empty_result() + + def fake_reconcile_archived( + _config: Config, + _paths: SourcePaths, + *, + export: bool = False, + include_history: bool = False, + ) -> dict[str, dict[str, int]]: + activation.append(export) + return {} + + def fake_reconcile_one( + _config: Config, + _paths: SourcePaths, + native_session_id: str, + *, + export: bool = False, + include_history: bool = False, + ) -> dict[str, int]: + per_session.append(native_session_id) + return { + "events": 0, + "usage": 0, + "turns": 0, + "exports": 0, + "pending": 0, + "ambiguous": 0, + "conflicting": 0, + "errors": 0, + } + + monkeypatch.setattr(watch_mod, "sync", fake_sync) + monkeypatch.setattr(watch_mod, "reconcile_archived_sessions", fake_reconcile_archived) + monkeypatch.setattr(watch_mod, "reconcile_session", fake_reconcile_one) + monkeypatch.setattr( + watch_mod, "_SLEEP", lambda _interval: (_ for _ in ()).throw(KeyboardInterrupt) + ) + + watch_mod.watch(config, paths, interval=0.1) + + assert activation == [True] + assert per_session == [] + + +def test_status_counts_delivery_claim_as_delivered_not_queued( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + stored = _seed_archive(config, paths) + directory = session_dir(config.root, PLATFORM_NAME, stored) + + def _record(state: dict[str, Any]) -> dict[str, Any]: + updated, _, _ = record_placement( + state, + accounting_id="acct-1", + destination="session-accounting-span", + span_id="accounting:stored:acct-1", + usage={}, + delivered=False, + ) + return updated + + update_export_state(config, stored, _record) + claim = delivery_claim_path(directory, "acct-1") + claim.parent.mkdir(parents=True, exist_ok=True) + claim.write_text("sent", encoding="utf-8") + + status = capture_status(config, paths) + export = status["sessions"][0]["export"] + assert export["delivered"] >= 1 + assert export["queued"] == 0 + + +def test_status_counts_failed_accounting_job_as_error_backlog( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + stored = _seed_archive(config, paths) + span_id = f"accounting:{stored}:acct-fail" + path = job_path(config.root, span_id) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps( + { + "job_id": span_id, + "kind": "session_accounting", + "session_id": stored, + "accounting_id": "acct-fail", + "state": "failed", + "attempt": 5, + "last_error": "TimeoutError: collector stalled", + } + ), + encoding="utf-8", + ) + + status = capture_status(config, paths) + export = status["sessions"][0]["export"] + assert export["errors"] >= 1 + assert export["queued"] >= 1 + assert export["delivered"] == 0 + + +def test_status_counts_turn_job_and_sent_claim( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + stored = _seed_archive(config, paths) + directory = session_dir(config.root, PLATFORM_NAME, stored) + jobs = otel_jobs_dir(config.root) + jobs.mkdir(parents=True, exist_ok=True) + (jobs / "turn-queued.json").write_text( + json.dumps({"kind": "turn", "session_id": stored, "job_id": "turn-queued"}), + encoding="utf-8", + ) + sent_dir = directory / "otel-turns-sent" + sent_dir.mkdir(parents=True, exist_ok=True) + (sent_dir / "delivered.json").write_text("sent", encoding="utf-8") + + status = capture_status(config, paths) + export = status["sessions"][0]["export"] + assert export["queued"] >= 1 + assert export["delivered"] >= 1 + + +def test_status_counts_failed_turn_export_from_worker_log( + copilot_env: tuple[Config, SourcePaths], +) -> None: + """A failed whole-turn export deletes the job before delivery. + + The only remaining evidence is usage-errors.jsonl. Status must still + report backlog and errors without a surviving job or sent claim. + """ + + config, paths = copilot_env + stored = _seed_archive(config, paths) + log_capture_error( + thirdeye_home=config.root, + phase="otel_worker_export_failed", + level="error", + platform=PLATFORM_NAME, + session_id=stored, + error=RuntimeError("logfire flush failed"), + message="kind=turn", + ) + log_capture_error( + thirdeye_home=config.root, + phase="otel_worker_export_failed", + level="error", + platform=PLATFORM_NAME, + session_id="copilot-other-session", + error=RuntimeError("other session"), + message="kind=turn", + ) + + jobs = otel_jobs_dir(config.root) + assert not jobs.exists() or not any(path.suffix == ".json" for path in jobs.iterdir()) + assert not (session_dir(config.root, PLATFORM_NAME, stored) / "otel-turns-sent").exists() + + status = capture_status(config, paths) + export = status["sessions"][0]["export"] + assert export["errors"] == 1 + assert export["queued"] == 1 + assert export["delivered"] == 0 + + +def test_status_exposes_persisted_reconcile_last_error( + copilot_env: tuple[Config, SourcePaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, paths = copilot_env + stored = _seed_archive(config, paths) + + monkeypatch.setattr( + "thirdeye.platforms.copilot.runtime.reconcile_archive", + lambda *_args, **_kwargs: { + "events": 0, + "usage": 0, + "turns": 0, + "exports": 0, + "pending": 0, + "ambiguous": 0, + "conflicting": 0, + "errors": 1, + }, + ) + reconcile_session(config, paths, NATIVE_SESSION_ID) + + status = capture_status(config, paths) + export = status["sessions"][0]["export"] + assert export["errors"] >= 1 + assert export["last_error"]["errors"] == 1 + assert any(error.get("kind") == "copilot_reconcile_error" for error in status["errors"]) + assert stored in {session["stored_session_id"] for session in status["sessions"]} diff --git a/tests/platforms/copilot/test_sources.py b/tests/platforms/copilot/test_sources.py new file mode 100644 index 00000000..ab50ea08 --- /dev/null +++ b/tests/platforms/copilot/test_sources.py @@ -0,0 +1,401 @@ +"""Behavioral tests for Copilot source discovery and batch composition.""" + +from __future__ import annotations + +import json +import sqlite3 +from pathlib import Path +from typing import Any +from unittest.mock import patch + +from thirdeye.platforms.copilot.database import discover_database_sessions +from thirdeye.platforms.copilot.identity import resolve_sources +from thirdeye.platforms.copilot.sources import discover_sessions, read_batch +from thirdeye.platforms.copilot.sources import resolve_sources as reexported_resolve +from thirdeye.platforms.copilot.transcript import discover_transcripts +from thirdeye.platforms.copilot.types import SourcePaths, SourceRecord, SourceSlice + +FIXTURES = Path(__file__).parent / "fixtures" +V1_SLICE = FIXTURES / "cases" / "source-slice.json" +NATIVE_ID = "session-a" + + +def _paths(home: Path) -> SourcePaths: + return resolve_sources(home) + + +def _write_transcript( + home: Path, native_id: str, *, events: str = '{"id":"1"}\n', cwd: str | None = None +) -> None: + session_dir = home / "session-state" / native_id + session_dir.mkdir(parents=True, exist_ok=True) + (session_dir / "events.jsonl").write_text(events, encoding="utf-8") + if cwd is not None: + (session_dir / "workspace.yaml").write_text(f"cwd: {cwd}\n", encoding="utf-8") + + +def _write_database( + home: Path, + *, + session_id: str = NATIVE_ID, + cwd: str = "/db/workspace", + extra_sessions: list[str] | None = None, +) -> None: + home.mkdir(parents=True, exist_ok=True) + database = home / "session-store.db" + connection = sqlite3.connect(database) + try: + connection.executescript( + """ + CREATE TABLE sessions (id TEXT PRIMARY KEY, cwd TEXT, created_at TEXT); + CREATE TABLE turns ( + id INTEGER PRIMARY KEY, + session_id TEXT NOT NULL, + turn_index INTEGER, + content TEXT, + updated_at TEXT + ); + CREATE TABLE assistant_usage_events ( + id INTEGER PRIMARY KEY, + session_id TEXT NOT NULL, + turn_index INTEGER, + created_at TEXT + ); + """ + ) + connection.execute( + "INSERT INTO sessions (id, cwd, created_at) VALUES (?, ?, ?)", + (session_id, cwd, "2026-09-10T17:08:00.000Z"), + ) + connection.execute( + "INSERT INTO turns (id, session_id, turn_index, content, updated_at) " + "VALUES (?, ?, ?, ?, ?)", + (1, session_id, 1, "turn", "2026-09-10T17:08:10.000Z"), + ) + for other in extra_sessions or (): + connection.execute( + "INSERT INTO sessions (id, cwd, created_at) VALUES (?, ?, ?)", + (other, f"/tmp/{other}", "2026-09-10T17:08:00.000Z"), + ) + connection.commit() + finally: + connection.close() + + +def _record(source_id: str, *, source_kind: str = "transcript") -> SourceRecord: + return { + "source_id": source_id, + "source_kind": source_kind, + "native_session_id": NATIVE_ID, + "ts": "2026-09-10T17:08:24.000Z", + "observed_at": "2026-09-10T17:08:25.000Z", + "payload": {"schema_version": 1, "type": "user.message"}, + "locator": {"file": "events.jsonl", "offset": 0}, + } + + +def _slice( + *, + records: list[SourceRecord] | None = None, + next_cursor: dict[str, Any] | None = None, + diagnostics: list[dict[str, Any]] | None = None, + cwd: str | None = "/fixture/workspace", + exhausted: bool = False, +) -> SourceSlice: + return { + "records": records or [], + "next_cursor": next_cursor or {"byte_offset": 1}, + "diagnostics": diagnostics or [], + "cwd": cwd, + "exhausted": exhausted, + } + + +# --- re-exports and discovery --- + + +def test_resolve_sources_is_reexported_from_identity(tmp_path: Path) -> None: + home = tmp_path / "copilot" + home.mkdir() + assert reexported_resolve(home) == resolve_sources(home) + + +def test_discover_sessions_unions_transcript_and_database_sorted(tmp_path: Path) -> None: + home = tmp_path / "copilot" + _write_transcript(home, "session-b") + _write_transcript(home, "session-a") + _write_database(home, session_id="session-c", extra_sessions=["session-d"]) + + assert discover_sessions(_paths(home)) == ["session-a", "session-b", "session-c", "session-d"] + + +def test_discover_sessions_returns_transcript_only_when_database_missing(tmp_path: Path) -> None: + home = tmp_path / "copilot" + _write_transcript(home, "session-z") + assert discover_sessions(_paths(home)) == ["session-z"] + + +def test_discover_sessions_continues_when_one_discoverer_raises(tmp_path: Path) -> None: + home = tmp_path / "copilot" + _write_transcript(home, "session-a") + + def boom(_paths: SourcePaths) -> list[str]: + raise OSError("database unreadable") + + with patch("thirdeye.platforms.copilot.sources.discover_database_sessions", boom): + assert discover_sessions(_paths(home)) == ["session-a"] + + +def test_discover_sessions_continues_when_transcript_discover_raises(tmp_path: Path) -> None: + home = tmp_path / "copilot" + _write_database(home, session_id="session-db") + + def boom(_paths: SourcePaths) -> list[str]: + raise ValueError("transcript layout invalid") + + with patch("thirdeye.platforms.copilot.sources.discover_transcripts", boom): + assert discover_sessions(_paths(home)) == ["session-db"] + + +# --- read_batch composition --- + + +def test_read_batch_merges_transcript_and_database_records(tmp_path: Path) -> None: + home = tmp_path / "copilot" + _write_transcript(home, NATIVE_ID, events='{"id":"evt-1"}\n') + _write_database(home, session_id=NATIVE_ID) + paths = _paths(home) + + batch = read_batch(paths, NATIVE_ID, {}) + kinds = {record["source_kind"] for record in batch["records"]} + assert "transcript" in kinds + assert "database" in kinds + assert batch["source_key"] == paths["source_key"] + assert batch["native_session_id"] == NATIVE_ID + + +def test_read_batch_namespaces_reader_cursors_independently() -> None: + transcript_cursor = {"byte_offset": 64, "file_generation": "gen-1"} + database_cursor = {"row_id": 7, "table": "turns"} + transcript = _slice( + records=[_record("t/1")], next_cursor=transcript_cursor, cwd="/transcript/cwd" + ) + database = _slice( + records=[_record("d/1", source_kind="database")], + next_cursor=database_cursor, + cwd="/database/cwd", + ) + + with ( + patch("thirdeye.platforms.copilot.sources.read_transcript", return_value=transcript), + patch("thirdeye.platforms.copilot.sources.read_database", return_value=database), + ): + batch = read_batch(_paths(Path("/tmp/unused")), NATIVE_ID, {"transcript": {"stale": True}}) + + assert batch["next_cursor"]["transcript"] == transcript_cursor + assert batch["next_cursor"]["database"] == database_cursor + assert batch["next_cursor"]["base_cursor"] == {"transcript": {"stale": True}} + assert batch["next_cursor"]["transcript_exhausted"] is False + assert batch["next_cursor"]["database_exhausted"] is False + + +def test_read_batch_passes_namespaced_cursors_to_each_reader(tmp_path: Path) -> None: + home = tmp_path / "copilot" + home.mkdir() + paths = _paths(home) + incoming = { + "transcript": {"byte_offset": 10}, + "database": {"row_id": 3}, + "base_cursor": {"ignored": True}, + } + seen: dict[str, dict[str, Any]] = {} + + def capture_transcript( + _paths: SourcePaths, _native: str, cursor: dict[str, Any] + ) -> SourceSlice: + seen["transcript"] = cursor + return _slice(exhausted=True) + + def capture_database(_paths: SourcePaths, _native: str, cursor: dict[str, Any]) -> SourceSlice: + seen["database"] = cursor + return _slice(exhausted=True) + + with ( + patch("thirdeye.platforms.copilot.sources.read_transcript", side_effect=capture_transcript), + patch("thirdeye.platforms.copilot.sources.read_database", side_effect=capture_database), + ): + read_batch(paths, NATIVE_ID, incoming) + + assert seen["transcript"] == {"byte_offset": 10} + assert seen["database"] == {"row_id": 3} + + +def test_read_batch_surfaces_transcript_failure_as_diagnostic_while_database_progresses( + tmp_path: Path, +) -> None: + home = tmp_path / "copilot" + _write_database(home, session_id=NATIVE_ID) + paths = _paths(home) + + def fail_transcript(_paths: SourcePaths, _native: str, _cursor: dict[str, Any]) -> SourceSlice: + raise OSError("events.jsonl locked") + + with patch("thirdeye.platforms.copilot.sources.read_transcript", side_effect=fail_transcript): + batch = read_batch(paths, NATIVE_ID, {}) + + assert any(item["code"] == "copilot_transcript_read_failed" for item in batch["diagnostics"]) + assert any(record["source_kind"] == "database" for record in batch["records"]) + + +def test_read_batch_surfaces_database_failure_as_diagnostic_while_transcript_progresses( + tmp_path: Path, +) -> None: + home = tmp_path / "copilot" + _write_transcript(home, NATIVE_ID) + paths = _paths(home) + + def fail_database(_paths: SourcePaths, _native: str, _cursor: dict[str, Any]) -> SourceSlice: + raise ValueError("database schema mismatch") + + with patch("thirdeye.platforms.copilot.sources.read_database", side_effect=fail_database): + batch = read_batch(paths, NATIVE_ID, {}) + + assert any(item["code"] == "copilot_database_read_failed" for item in batch["diagnostics"]) + assert any(record["source_kind"] == "transcript" for record in batch["records"]) + + +def test_read_batch_prefers_transcript_cwd_over_database() -> None: + transcript = _slice(cwd="/from/transcript") + database = _slice(cwd="/from/database", records=[_record("d/1", source_kind="database")]) + + with ( + patch("thirdeye.platforms.copilot.sources.read_transcript", return_value=transcript), + patch("thirdeye.platforms.copilot.sources.read_database", return_value=database), + ): + batch = read_batch(_paths(Path("/tmp/unused")), NATIVE_ID, {}) + + assert batch["cwd"] == "/from/transcript" + + +def test_read_batch_falls_back_to_database_cwd_when_transcript_missing() -> None: + transcript = _slice(cwd=None) + database = _slice(cwd="/from/database", records=[_record("d/1", source_kind="database")]) + + with ( + patch("thirdeye.platforms.copilot.sources.read_transcript", return_value=transcript), + patch("thirdeye.platforms.copilot.sources.read_database", return_value=database), + ): + batch = read_batch(_paths(Path("/tmp/unused")), NATIVE_ID, {}) + + assert batch["cwd"] == "/from/database" + + +def test_read_batch_fixture_slice_shape_is_compatible() -> None: + """The v1 source-slice fixture must compose into a SourceBatch boundary.""" + fixture = json.loads(V1_SLICE.read_text(encoding="utf-8")) + transcript = _slice( + records=fixture["records"], + next_cursor=fixture["next_cursor"], + diagnostics=fixture["diagnostics"], + cwd=fixture["cwd"], + exhausted=fixture["exhausted"], + ) + database = _slice(exhausted=True) + + with ( + patch("thirdeye.platforms.copilot.sources.read_transcript", return_value=transcript), + patch("thirdeye.platforms.copilot.sources.read_database", return_value=database), + ): + batch = read_batch(_paths(Path("/tmp/unused")), NATIVE_ID, {}) + + assert len(batch["records"]) == 1 + assert batch["records"][0]["source_kind"] == "transcript" + assert batch["diagnostics"] == [] + + +def test_read_batch_preserves_independent_exhaustion_flags() -> None: + transcript = _slice(records=[_record("t/1")], exhausted=False) + database = _slice( + records=[_record("d/1", source_kind="database")], + exhausted=True, + cwd="/database/cwd", + ) + + with ( + patch("thirdeye.platforms.copilot.sources.read_transcript", return_value=transcript), + patch("thirdeye.platforms.copilot.sources.read_database", return_value=database), + ): + batch = read_batch(_paths(Path("/tmp/unused")), NATIVE_ID, {}) + + assert batch["next_cursor"]["transcript_exhausted"] is False + assert batch["next_cursor"]["database_exhausted"] is True + + +def test_read_batch_marks_failed_reader_slice_as_not_exhausted(tmp_path: Path) -> None: + home = tmp_path / "copilot" + _write_transcript(home, NATIVE_ID) + paths = _paths(home) + + def fail_database(_paths: SourcePaths, _native: str, _cursor: dict[str, Any]) -> SourceSlice: + raise OSError("database locked") + + with patch("thirdeye.platforms.copilot.sources.read_database", side_effect=fail_database): + batch = read_batch(paths, NATIVE_ID, {}) + + assert batch["next_cursor"]["database_exhausted"] is False + assert batch["next_cursor"]["transcript_exhausted"] is True + + +def test_discover_sessions_delegates_to_underlying_readers(tmp_path: Path) -> None: + home = tmp_path / "copilot" + _write_transcript(home, "only-transcript") + paths = _paths(home) + assert discover_transcripts(paths) == ["only-transcript"] + assert discover_database_sessions(paths) == [] + assert discover_sessions(paths) == ["only-transcript"] + + +def test_read_batch_freezes_database_snapshot_end_against_later_inserts(tmp_path: Path) -> None: + home = tmp_path / "copilot" + home.mkdir() + paths = _paths(home) + reads = {"n": 0} + + def growing_database(_paths: SourcePaths, _native: str, cursor: dict[str, Any]) -> SourceSlice: + reads["n"] += 1 + if reads["n"] > 80: + raise AssertionError("read_batch chased a growing database source") + offset = 0 + if isinstance(cursor, dict) and isinstance(cursor.get("database_offset"), int): + offset = cursor["database_offset"] + return _slice( + records=[ + _record(f"d/{offset + index}", source_kind="database") for index in range(1000) + ], + next_cursor={"database_generation": "gen-1", "database_offset": offset + 1000}, + cwd=None, + exhausted=False, + ) + + transcript = _slice(exhausted=True, cwd=None, next_cursor={"byte_offset": 0, "snapshot_end": 0}) + with ( + patch("thirdeye.platforms.copilot.sources.read_transcript", return_value=transcript), + patch("thirdeye.platforms.copilot.sources.read_database", side_effect=growing_database), + ): + first = read_batch(paths, NATIVE_ID, {}) + snapshot_end = first["next_cursor"]["database"]["snapshot_end"] + cursor = first["next_cursor"] + pages = 1 + records = list(first["records"]) + while not cursor.get("database_exhausted") and pages < 100: + batch = read_batch(paths, NATIVE_ID, cursor) + records.extend(batch["records"]) + cursor = batch["next_cursor"] + pages += 1 + assert cursor["database"]["snapshot_end"] == snapshot_end + + assert isinstance(snapshot_end, int) + assert cursor.get("database_exhausted") is True + assert cursor["database"]["database_offset"] == snapshot_end + assert len([item for item in records if item["source_kind"] == "database"]) == snapshot_end + assert reads["n"] <= 80 diff --git a/tests/platforms/copilot/test_spool.py b/tests/platforms/copilot/test_spool.py new file mode 100644 index 00000000..e6764496 --- /dev/null +++ b/tests/platforms/copilot/test_spool.py @@ -0,0 +1,325 @@ +"""Behavioral tests for durable Copilot hook observation spooling.""" + +from __future__ import annotations + +import json +import subprocess +import sys +import threading +from pathlib import Path +from typing import Any + +import pytest + +from thirdeye.config import Config +from thirdeye.platforms.copilot.hook_payload import parse_hook +from thirdeye.platforms.copilot.identity import resolve_sources +from thirdeye.platforms.copilot.spool import ack_spool, enqueue_hook, read_spool +from thirdeye.platforms.copilot.types import SourcePaths, SourceRecord + +NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" +OBSERVED_AT = "2026-09-10T17:08:25.626Z" +_CONCURRENT_WORKERS = 8 +_CONCURRENT_RECORDS = 16 + + +@pytest.fixture +def copilot_env(tmp_path: Path) -> tuple[Config, SourcePaths]: + copilot_home = tmp_path / "copilot-home" + copilot_home.mkdir() + config = Config(root=tmp_path / "thirdeye") + paths = resolve_sources(copilot_home) + return config, paths + + +def _hook_record( + *, + observation_id: str, + event: str = "agentStop", + session_id: str = NATIVE_SESSION_ID, + stop_reason: str = "end_turn", +) -> SourceRecord: + return parse_hook( + event, + { + "sessionId": session_id, + "timestamp": 1789060105626, + "cwd": "/fixture/workspace", + "stopReason": stop_reason, + }, + {"env": {"WB_PLAN": "p"}}, + observed_at=OBSERVED_AT, + observation_id=observation_id, + ) + + +def test_enqueue_returns_spool_path_and_read_returns_complete_record( + copilot_env: tuple[Config, SourcePaths], +): + config, paths = copilot_env + record = _hook_record(observation_id="obs-enqueue-1") + + spool_path = enqueue_hook(config, paths, record) + assert Path(spool_path).is_file() + + records = read_spool(config, paths, NATIVE_SESSION_ID) + assert len(records) == 1 + assert records[0]["source_id"] == record["source_id"] + assert records[0]["payload"]["hook_payload"]["stopReason"] == "end_turn" + + +def test_same_content_distinct_observation_ids_remain_separate( + copilot_env: tuple[Config, SourcePaths], +): + config, paths = copilot_env + first = _hook_record(observation_id="hook-observation-1") + second = _hook_record(observation_id="hook-observation-2") + + enqueue_hook(config, paths, first) + enqueue_hook(config, paths, second) + + records = read_spool(config, paths, NATIVE_SESSION_ID) + assert len(records) == 2 + assert {record["source_id"] for record in records} == {first["source_id"], second["source_id"]} + + +def test_ack_spool_removes_only_committed_ids(copilot_env: tuple[Config, SourcePaths]): + config, paths = copilot_env + keep = _hook_record(observation_id="obs-keep") + drop = _hook_record(observation_id="obs-drop") + enqueue_hook(config, paths, keep) + enqueue_hook(config, paths, drop) + + ack_spool(config, paths, [drop["source_id"]]) + + remaining = read_spool(config, paths, NATIVE_SESSION_ID) + assert len(remaining) == 1 + assert remaining[0]["source_id"] == keep["source_id"] + + +def test_ack_spool_unknown_ids_are_no_ops(copilot_env: tuple[Config, SourcePaths]): + config, paths = copilot_env + record = _hook_record(observation_id="obs-stable") + enqueue_hook(config, paths, record) + + ack_spool(config, paths, ["nonexistent-source-id"]) + ack_spool(config, paths, []) + + remaining = read_spool(config, paths, NATIVE_SESSION_ID) + assert len(remaining) == 1 + assert remaining[0]["source_id"] == record["source_id"] + + +def test_malformed_spool_neighbor_does_not_discard_valid_records( + copilot_env: tuple[Config, SourcePaths], +): + config, paths = copilot_env + valid = _hook_record(observation_id="obs-valid-neighbor") + spool_path = Path(enqueue_hook(config, paths, valid)) + + (spool_path.parent / "000000-malformed.json").write_text("{not json", encoding="utf-8") + + records = read_spool(config, paths, NATIVE_SESSION_ID) + assert len(records) == 1 + assert records[0]["source_id"] == valid["source_id"] + + +def test_concurrent_enqueue_preserves_all_records(tmp_path: Path): + copilot_home = tmp_path / "copilot-home" + copilot_home.mkdir() + config = Config(root=tmp_path / "thirdeye") + paths = resolve_sources(copilot_home) + + barrier = threading.Barrier(_CONCURRENT_WORKERS) + errors: list[str] = [] + + def worker(worker_id: int) -> None: + try: + barrier.wait(timeout=5) + for index in range(_CONCURRENT_RECORDS): + record = _hook_record(observation_id=f"obs-{worker_id}-{index}") + enqueue_hook(config, paths, record) + except Exception as exc: # pragma: no cover - surfaced via errors list + errors.append(str(exc)) + + threads = [ + threading.Thread(target=worker, args=(worker_id,)) + for worker_id in range(_CONCURRENT_WORKERS) + ] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=30) + + assert not errors + records = read_spool(config, paths, NATIVE_SESSION_ID) + assert len(records) == _CONCURRENT_WORKERS * _CONCURRENT_RECORDS + + +def test_spool_survives_subprocess_isolation(tmp_path: Path): + copilot_home = tmp_path / "copilot-home" + copilot_home.mkdir() + config_root = tmp_path / "thirdeye" + script = f""" +from pathlib import Path + +from thirdeye.config import Config +from thirdeye.platforms.copilot.hook_payload import parse_hook +from thirdeye.platforms.copilot.identity import resolve_sources +from thirdeye.platforms.copilot.spool import enqueue_hook, read_spool + +config = Config(root=Path({str(config_root)!r})) +paths = resolve_sources(Path({str(copilot_home)!r})) +record = parse_hook( + "sessionStart", + {{"sessionId": {NATIVE_SESSION_ID!r}, "timestamp": 1789060102204}}, + {{}}, + observed_at={OBSERVED_AT!r}, + observation_id="obs-subprocess", +) +enqueue_hook(config, paths, record) +assert len(read_spool(config, paths, {NATIVE_SESSION_ID!r})) == 1 +""" + result = subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + text=True, + encoding="utf-8", + check=False, + ) + assert result.returncode == 0, result.stderr + + config = Config(root=config_root) + paths = resolve_sources(copilot_home) + records = read_spool(config, paths, NATIVE_SESSION_ID) + assert len(records) == 1 + assert records[0]["payload"]["hook_payload"]["sessionId"] == NATIVE_SESSION_ID + + +def test_read_spool_returns_empty_for_missing_session(copilot_env: tuple[Config, SourcePaths]): + config, paths = copilot_env + assert read_spool(config, paths, NATIVE_SESSION_ID) == [] + + +def test_malformed_spool_writes_diagnostic_sidecar(copilot_env: tuple[Config, SourcePaths]): + config, paths = copilot_env + valid = _hook_record(observation_id="obs-diag-valid") + spool_path = Path(enqueue_hook(config, paths, valid)) + malformed = spool_path.parent / "000000-malformed.json" + malformed.write_text("{not json", encoding="utf-8") + + records = read_spool(config, paths, NATIVE_SESSION_ID) + assert len(records) == 1 + diag_path = malformed.with_name(f"{malformed.name}.diag") + assert diag_path.is_file() + assert "JSONDecodeError" in diag_path.read_text(encoding="utf-8") + + +def test_malformed_spool_missing_source_id_is_skipped(copilot_env: tuple[Config, SourcePaths]): + config, paths = copilot_env + valid = _hook_record(observation_id="obs-valid-shape") + spool_dir = Path(enqueue_hook(config, paths, valid)).parent + malformed = spool_dir / "000001-missing-source-id.json" + malformed.write_text('{"source_kind": "hook"}', encoding="utf-8") + + records = read_spool(config, paths, NATIVE_SESSION_ID) + assert len(records) == 1 + assert records[0]["source_id"] == valid["source_id"] + diag_path = malformed.with_name(f"{malformed.name}.diag") + assert diag_path.is_file() + assert "SourceRecord" in diag_path.read_text(encoding="utf-8") + + +def test_read_spool_skips_source_id_only_object(copilot_env: tuple[Config, SourcePaths]): + config, paths = copilot_env + valid = _hook_record(observation_id="obs-complete-neighbor") + spool_dir = Path(enqueue_hook(config, paths, valid)).parent + malformed = spool_dir / "000003-source-id-only.json" + malformed.write_text('{"source_id": "hook/x/obs"}', encoding="utf-8") + + records = read_spool(config, paths, NATIVE_SESSION_ID) + assert len(records) == 1 + assert records[0]["source_id"] == valid["source_id"] + diag_path = malformed.with_name(f"{malformed.name}.diag") + assert diag_path.is_file() + assert "SourceRecord" in diag_path.read_text(encoding="utf-8") + + +@pytest.mark.parametrize( + "corrupt", + [ + {"payload": None}, + {"payload": []}, + {"locator": "not-a-dict"}, + {"ts": 1789060105626}, + {"observed_at": None}, + {"source_kind": "transcript"}, + {"native_session_id": "bf8cb9f3-2097-4db0-a3c8-78a2653b2106"}, + {"source_id": ""}, + ], +) +def test_read_spool_skips_records_with_corrupt_or_missing_required_fields( + copilot_env: tuple[Config, SourcePaths], + corrupt: dict[str, Any], +): + config, paths = copilot_env + valid = _hook_record(observation_id="obs-required-fields") + spool_dir = Path(enqueue_hook(config, paths, valid)).parent + incomplete = dict(valid) + incomplete.update(corrupt) + malformed = spool_dir / "000004-incomplete.json" + malformed.write_text(json.dumps(incomplete), encoding="utf-8") + + records = read_spool(config, paths, NATIVE_SESSION_ID) + assert len(records) == 1 + assert records[0]["source_id"] == valid["source_id"] + diag_path = malformed.with_name(f"{malformed.name}.diag") + assert diag_path.is_file() + assert diag_path.read_text(encoding="utf-8").strip() + + +def test_ack_spool_finds_records_across_sessions(copilot_env: tuple[Config, SourcePaths]): + config, paths = copilot_env + child_id = "bf8cb9f3-2097-4db0-a3c8-78a2653b2106" + parent = _hook_record(observation_id="obs-parent-ack", session_id=NATIVE_SESSION_ID) + child = _hook_record(observation_id="obs-child-ack", session_id=child_id) + enqueue_hook(config, paths, parent) + enqueue_hook(config, paths, child) + + ack_spool(config, paths, [child["source_id"]]) + + assert len(read_spool(config, paths, NATIVE_SESSION_ID)) == 1 + assert read_spool(config, paths, child_id) == [] + + +def test_ack_spool_leaves_malformed_files_in_place(copilot_env: tuple[Config, SourcePaths]): + config, paths = copilot_env + valid = _hook_record(observation_id="obs-ack-malformed-neighbor") + spool_dir = Path(enqueue_hook(config, paths, valid)).parent + malformed = spool_dir / "000002-bad-for-ack.json" + malformed.write_text("{bad", encoding="utf-8") + + ack_spool(config, paths, [valid["source_id"]]) + + assert malformed.is_file() + assert read_spool(config, paths, NATIVE_SESSION_ID) == [] + + +def test_child_session_hooks_spool_under_native_session_id(tmp_path: Path): + child_id = "bf8cb9f3-2097-4db0-a3c8-78a2653b2106" + copilot_home = tmp_path / "copilot-home" + copilot_home.mkdir() + config = Config(root=tmp_path / "thirdeye") + paths = resolve_sources(copilot_home) + + parent = _hook_record(observation_id="obs-parent", session_id=NATIVE_SESSION_ID) + child = _hook_record(observation_id="obs-child", session_id=child_id) + enqueue_hook(config, paths, parent) + enqueue_hook(config, paths, child) + + parent_records = read_spool(config, paths, NATIVE_SESSION_ID) + child_records = read_spool(config, paths, child_id) + assert len(parent_records) == 1 + assert len(child_records) == 1 + assert parent_records[0]["native_session_id"] == NATIVE_SESSION_ID + assert child_records[0]["native_session_id"] == child_id diff --git a/tests/platforms/copilot/test_status.py b/tests/platforms/copilot/test_status.py new file mode 100644 index 00000000..a3dd7458 --- /dev/null +++ b/tests/platforms/copilot/test_status.py @@ -0,0 +1,400 @@ +"""Behavioral tests for Copilot local capture status reporting.""" + +from __future__ import annotations + +import json +import shutil +import sqlite3 +import time +from pathlib import Path + +import pytest + +import thirdeye.platforms.copilot.status as status_mod +from thirdeye.config import Config +from thirdeye.paths import session_dir +from thirdeye.platforms.copilot.archive import commit_batch +from thirdeye.platforms.copilot.constants import ( + FOLLOWUP_LEASE_FILENAME, + OWNED_HOOK_FILENAME, + PLATFORM_NAME, +) +from thirdeye.platforms.copilot.hook_payload import parse_hook +from thirdeye.platforms.copilot.identity import ( + SOURCE_KEY_PREFIX_LEN, + resolve_sources, + stored_session_id, +) +from thirdeye.platforms.copilot.install import CopilotPlatform +from thirdeye.platforms.copilot.spool import enqueue_hook +from thirdeye.platforms.copilot.state import ( + journal_path, + read_json, + state_path, + write_journal, + write_state, +) +from thirdeye.platforms.copilot.status import capture_status +from thirdeye.platforms.copilot.types import SourceBatch, SourcePaths, SourceRecord + +FIXTURES = Path(__file__).parent / "fixtures" +CLI_FIXTURE = FIXTURES +NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" +OBSERVED_AT_EARLY = "2026-09-10T17:08:20.000Z" +OBSERVED_AT_LATE = "2026-09-10T17:08:30.000Z" + + +@pytest.fixture +def copilot_env(tmp_path: Path) -> tuple[Config, SourcePaths]: + home = tmp_path / "copilot-home" + home.mkdir() + config = Config(root=tmp_path / "thirdeye") + paths = resolve_sources(home) + return config, paths + + +def _record( + source_id: str, + *, + source_kind: str = "transcript", + observed_at: str = OBSERVED_AT_EARLY, + native_session_id: str = NATIVE_SESSION_ID, +) -> SourceRecord: + return { + "source_id": source_id, + "source_kind": source_kind, + "native_session_id": native_session_id, + "ts": "2026-09-10T17:08:24.000Z", + "observed_at": observed_at, + "payload": {"schema_version": 1, "type": "user.message"}, + "locator": {"file": "events.jsonl", "offset": 0}, + } + + +def _batch( + paths: SourcePaths, + records: list[SourceRecord], + *, + native_session_id: str = NATIVE_SESSION_ID, +) -> SourceBatch: + return { + "source_key": paths["source_key"], + "native_session_id": native_session_id, + "cwd": "/proj", + "records": records, + "next_cursor": {"generation": 1}, + "diagnostics": [], + } + + +def _write_lease(directory: Path, *, expires_at: float) -> None: + (directory / FOLLOWUP_LEASE_FILENAME).write_text( + json.dumps({"generation": "lease-test", "expires_at": expires_at}) + "\n", + encoding="utf-8", + ) + + +def _hook_record(*, observation_id: str, observed_at: str = OBSERVED_AT_LATE) -> SourceRecord: + return parse_hook( + "agentStop", + { + "sessionId": NATIVE_SESSION_ID, + "timestamp": 1789060105626, + "cwd": "/fixture/workspace", + "stopReason": "end_turn", + }, + {"env": {"WB_PLAN": "p"}}, + observed_at=observed_at, + observation_id=observation_id, + ) + + +def _write_transcript(home: Path, native_id: str) -> None: + session_dir = home / "session-state" / native_id + session_dir.mkdir(parents=True, exist_ok=True) + shutil.copy(CLI_FIXTURE / "events.jsonl", session_dir / "events.jsonl") + + +def _write_database(home: Path, *, session_id: str = NATIVE_SESSION_ID) -> None: + database = home / "session-store.db" + connection = sqlite3.connect(database) + try: + connection.execute("PRAGMA journal_mode=WAL") + connection.executescript( + "CREATE TABLE sessions (id TEXT PRIMARY KEY, cwd TEXT, created_at TEXT);" + ) + connection.execute( + "INSERT INTO sessions (id, cwd, created_at) VALUES (?, ?, ?)", + (session_id, "/tmp/probe", "2026-09-10T17:08:00.000Z"), + ) + connection.commit() + finally: + connection.close() + + +def test_status_module_has_no_command_or_hook_runtime_imports() -> None: + source = Path(status_mod.__file__).read_text(encoding="utf-8") + assert "thirdeye.commands" not in source + assert "hook_lifecycle" not in source + assert "from .followup" not in source + + +def test_capture_status_reports_empty_installation_and_capabilities( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + status = capture_status(config, paths) + + assert status["paths"] == dict(paths) + assert status["installation"]["configured"] is False + assert status["installation"]["hooks_file"].endswith(OWNED_HOOK_FILENAME) + assert status["capabilities"]["transcripts"]["exists"] is False + assert status["capabilities"]["database"]["exists"] is False + assert status["capabilities"]["database_wal"]["exists"] is False + assert status["last_observed_hook"] is None + assert status["last_successful_import"] is None + assert status["sessions"] == [] + assert status["pending"] == { + "spool_records": 0, + "spool_sessions": [], + "followup": 0, + "leases": 0, + "journals": 0, + } + assert status["errors"] == [] + + +def test_capture_status_reports_readable_source_capabilities( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + home = Path(paths["home"]) + _write_transcript(home, NATIVE_SESSION_ID) + _write_database(home, session_id=NATIVE_SESSION_ID) + + status = capture_status(config, paths) + + assert status["capabilities"]["transcripts"]["readable"] is True + assert status["capabilities"]["database"]["readable"] is True + assert status["errors"] == [] + + +def test_capture_status_reports_installed_hooks( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + # Install and inspect with the same runtime entrypoint resolution. + platform = CopilotPlatform(source_home=Path(paths["home"])) + platform.install() + + status = capture_status(config, paths) + + assert status["installation"]["configured"] is True + assert status["errors"] == [] + + +def test_capture_status_reports_archived_session_progress( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + commit_batch(config, paths, _batch(paths, [_record("status/archived-1")])) + + status = capture_status(config, paths) + stored = stored_session_id(paths, NATIVE_SESSION_ID) + + assert len(status["sessions"]) == 1 + session = status["sessions"][0] + assert session["stored_session_id"] == stored + assert session["native_session_id"] == NATIVE_SESSION_ID + assert session["journal_pending"] is False + assert isinstance(status["last_successful_import"], str) + + +def test_capture_status_reports_pending_spool_and_latest_hook( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + record = _hook_record(observation_id="status-spool-hook") + enqueue_hook(config, paths, record) + + status = capture_status(config, paths) + + assert status["pending"]["spool_records"] == 1 + assert status["pending"]["spool_sessions"] == [NATIVE_SESSION_ID] + assert status["last_observed_hook"] is not None + assert status["last_observed_hook"]["source_id"] == record["source_id"] + + +def test_capture_status_prefers_latest_hook_observed_at( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + archived = _record("status/archived-hook", source_kind="hook", observed_at=OBSERVED_AT_EARLY) + commit_batch(config, paths, _batch(paths, [archived])) + spooled = _hook_record(observation_id="status-newer-hook", observed_at=OBSERVED_AT_LATE) + enqueue_hook(config, paths, spooled) + + status = capture_status(config, paths) + + assert status["last_observed_hook"] is not None + assert status["last_observed_hook"]["source_id"] == spooled["source_id"] + + +def test_capture_status_reports_followup_leases_and_journal( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + expired_id = "expired-lease-session" + commit_batch(config, paths, _batch(paths, [_record("status/pending-state")])) + commit_batch( + config, + paths, + _batch( + paths, + [_record("status/expired-lease", native_session_id=expired_id)], + native_session_id=expired_id, + ), + ) + live_dir = session_dir(config.root, PLATFORM_NAME, stored_session_id(paths, NATIVE_SESSION_ID)) + expired_dir = session_dir(config.root, PLATFORM_NAME, stored_session_id(paths, expired_id)) + now = time.time() + _write_lease(live_dir, expires_at=now + 60) + _write_lease(expired_dir, expires_at=now - 60) + write_journal(live_dir, {"pending": True}) + + status = capture_status(config, paths) + + assert status["pending"]["followup"] == 1 + assert status["pending"]["leases"] == 1 + assert status["pending"]["journals"] == 1 + assert journal_path(live_dir).is_file() + + +def test_capture_status_reports_invalid_archive_state( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + commit_batch(config, paths, _batch(paths, [_record("status/invalid-state")])) + directory = session_dir(config.root, PLATFORM_NAME, stored_session_id(paths, NATIVE_SESSION_ID)) + state_path(directory).write_text("{not valid json", encoding="utf-8") + + status = capture_status(config, paths) + + assert any(error.get("kind") == "invalid_archive_state" for error in status["errors"]) + + +def test_capture_status_tolerates_missing_optional_followup_state( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + commit_batch(config, paths, _batch(paths, [_record("status/minimal-state")])) + directory = session_dir(config.root, PLATFORM_NAME, stored_session_id(paths, NATIVE_SESSION_ID)) + state_path(directory).write_text( + json.dumps( + { + "schema_version": 1, + "source_key": paths["source_key"], + "source_home": paths["home"], + "native_session_id": NATIVE_SESSION_ID, + "cursor": {}, + "health": {"diagnostics": [], "last_successful_import": None}, + } + ) + + "\n", + encoding="utf-8", + ) + + status = capture_status(config, paths) + + assert status["pending"]["followup"] == 0 + assert status["pending"]["leases"] == 0 + assert status["errors"] == [] + + +def test_capture_status_reports_source_key_prefix_collision( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + commit_batch(config, paths, _batch(paths, [_record("status/collision")])) + directory = session_dir(config.root, PLATFORM_NAME, stored_session_id(paths, NATIVE_SESSION_ID)) + state = read_json(state_path(directory)) + assert state is not None + original = state["source_key"] + replacement = "b" if original[SOURCE_KEY_PREFIX_LEN] != "b" else "a" + colliding = ( + original[:SOURCE_KEY_PREFIX_LEN] + replacement + original[SOURCE_KEY_PREFIX_LEN + 1 :] + ) + assert colliding != original + state["source_key"] = colliding + write_state(directory, state) + + status = capture_status(config, paths) + + assert status["sessions"] == [] + assert any(error.get("kind") == "source_key_collision" for error in status["errors"]) + + +def test_capture_status_reports_unusable_database( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + Path(paths["database"]).write_text("not a sqlite database\n", encoding="utf-8") + + status = capture_status(config, paths) + + assert status["capabilities"]["database"]["exists"] is True + assert status["capabilities"]["database"]["readable"] is False + assert any(error.get("kind") == "source_unreadable" for error in status["errors"]) + + +def test_capture_status_reports_unusable_transcript_root( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + Path(paths["session_root"]).write_text("not a directory\n", encoding="utf-8") + + status = capture_status(config, paths) + + assert status["capabilities"]["transcripts"]["exists"] is True + assert status["capabilities"]["transcripts"]["readable"] is False + assert any(error.get("kind") == "source_unreadable" for error in status["errors"]) + + +def test_capture_status_reports_unreadable_transcript_directory( + monkeypatch: pytest.MonkeyPatch, + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + session_root = Path(paths["session_root"]) + session_root.mkdir(parents=True) + original_iterdir = Path.iterdir + + def fake_iterdir(self: Path): + if self == session_root: + raise PermissionError("denied") + return original_iterdir(self) + + monkeypatch.setattr(Path, "iterdir", fake_iterdir) + + status = capture_status(config, paths) + + assert status["capabilities"]["transcripts"]["exists"] is True + assert status["capabilities"]["transcripts"]["readable"] is False + assert any(error.get("kind") == "source_unreadable" for error in status["errors"]) + + +def test_capture_status_reports_malformed_spool_without_payloads( + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + spool_dir = Path(config.root) / "spool" / "copilot" / paths["source_key"] / NATIVE_SESSION_ID + spool_dir.mkdir(parents=True) + (spool_dir / "broken.json").write_text("{not valid json\n", encoding="utf-8") + + status = capture_status(config, paths) + + assert status["pending"]["spool_records"] >= 1 + assert NATIVE_SESSION_ID in status["pending"]["spool_sessions"] + assert any(error.get("kind") == "spool_unreadable" for error in status["errors"]) + serialized = json.dumps(status["errors"]) + assert "prompt" not in serialized.lower() diff --git a/tests/platforms/copilot/test_tracing.py b/tests/platforms/copilot/test_tracing.py new file mode 100644 index 00000000..faf1acb8 --- /dev/null +++ b/tests/platforms/copilot/test_tracing.py @@ -0,0 +1,1227 @@ +"""Behavioral tests for Copilot semantic projection (build_semantics).""" + +from __future__ import annotations + +import ast +import json +import shutil +from collections import Counter +from pathlib import Path +from typing import Any + +import pytest + +from thirdeye.config import Config +from thirdeye.platforms.copilot.archive import commit_batch +from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id +from thirdeye.platforms.copilot.reconcile import reconcile_archive +from thirdeye.platforms.copilot.tracing import build_semantics +from thirdeye.platforms.copilot.transcript import read_transcript +from thirdeye.platforms.copilot.turns import build_turns +from thirdeye.platforms.copilot.types import SourceRecord + +FIXTURES = Path(__file__).parent / "fixtures" +RECON_CASES = FIXTURES / "reconciliation-cases" +NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" +CHILD_AGENT_ID = "bf8cb9f3-2097-4db0-a3c8-78a2653b2106" +SOURCE_KEY = "a" * 64 + + +def _load_json(path: Path) -> Any: + return json.loads(path.read_text(encoding="utf-8")) + + +def _drain_cli_transcript(home: Path) -> list[SourceRecord]: + session_dir = home / "session-state" / NATIVE_SESSION_ID + session_dir.mkdir(parents=True, exist_ok=True) + shutil.copy(FIXTURES / "events.jsonl", session_dir / "events.jsonl") + (session_dir / "workspace.yaml").write_text("cwd: /sanitized/workspace\n", encoding="utf-8") + paths = resolve_sources(home) + cursor: dict[str, Any] = {} + records: list[SourceRecord] = [] + while True: + slice_ = read_transcript(paths, NATIVE_SESSION_ID, cursor) + records.extend(slice_["records"]) + cursor = slice_["next_cursor"] + if slice_["exhausted"]: + break + return [record for record in records if record["source_kind"] == "transcript"] + + +@pytest.fixture +def cli_transcript_records(tmp_path: Path) -> list[SourceRecord]: + return _drain_cli_transcript(tmp_path / "copilot-home") + + +def _transcript_record( + *, + source_id: str, + native_type: str, + data: dict[str, Any], + ts: str = "2026-09-10T17:08:24.503Z", + agent: str | None = None, +) -> SourceRecord: + payload: dict[str, Any] = { + "type": native_type, + "data": data, + "id": source_id.rsplit("/", 1)[-1], + "timestamp": ts, + "schema_version": 1, + } + if agent is not None: + payload["agentId"] = agent + return { + "source_id": source_id, + "source_kind": "transcript", + "native_session_id": NATIVE_SESSION_ID, + "ts": ts, + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": payload, + "locator": {"file": "events.jsonl", "native_event_id": payload["id"]}, + } + + +# --- observed CLI fixture --- + + +def test_cli_fixture_reconstructs_two_main_interactions_and_explore_child( + cli_transcript_records: list[SourceRecord], +) -> None: + projection, state = build_semantics(cli_transcript_records, {}) + turns = projection["turns"] + + assert len(turns) == 2 + assert [turn["output_message"] for turn in turns] == ["42", "42"] + assert [turn["attributes"]["interaction_id"] for turn in turns] == [ + "6d2b89fd-a653-430c-b532-b0936d72eb42", + "793d3703-6f4a-4814-8877-34a7325848ce", + ] + + child_turn = turns[1]["subagents"][0] + assert child_turn["output_message"] == "42" + assert child_turn["attributes"]["agent_id"] == CHILD_AGENT_ID + assert child_turn["attributes"]["parent_tool_call_id"] == "call_qx4FH5DADTeT1qVLb37HNpBk" + assert child_turn["attributes"]["interaction_id"] == "7c0fa097-c0e2-48da-b2b6-fcfc1ad83a6b" + + assert projection["pending"] == [] + assert state["open_interactions"] == {} + + +def test_cli_fixture_counts_five_tools_and_six_call_candidates( + cli_transcript_records: list[SourceRecord], +) -> None: + projection, _ = build_semantics(cli_transcript_records, {}) + tool_requests = [event for event in projection["events"] if event["kind"] == "tool_request"] + tool_starts = [ + event for event in projection["events"] if event["kind"] == "tool_execution_start" + ] + tool_completes = [ + event for event in projection["events"] if event["kind"] == "tool_execution_complete" + ] + + assert len(tool_requests) == 5 + assert len(tool_starts) == 5 + assert len(tool_completes) == 5 + assert len(projection["call_candidates"]) == 6 + + tool_ids = sorted(event["attributes"]["tool_call_id"] for event in tool_requests) + assert tool_ids == [ + "call_Jeh7IbrUHaq4jVdxtyQCVrns", + "call_YSSva4HCniiETlxdGGjcrHbh", + "call_ayHplfzxjRFMTCpmTKEFhCSJ", + "call_qx4FH5DADTeT1qVLb37HNpBk", + "call_zZncCGtp1twgcL2eoFUNwInh", + ] + + +def test_cli_fixture_replay_is_deterministic(cli_transcript_records: list[SourceRecord]) -> None: + first, first_state = build_semantics(cli_transcript_records, {}) + second, _ = build_semantics(cli_transcript_records, first_state) + third, _ = build_semantics(cli_transcript_records, {}) + + assert first["events"] == third["events"] + assert first["turns"] == third["turns"] + assert first["call_candidates"] == third["call_candidates"] + assert second["events"] == third["events"] + + +def _assert_expected_projection(projection: dict[str, Any], expected: dict[str, Any]) -> None: + for key in ("events", "turns", "call_candidates", "pending", "diagnostics"): + if key not in expected: + continue + actual = projection[key] + wanted = expected[key] + if key == "events": + by_id = {event["id"]: event for event in actual} + for item in wanted: + got = by_id[item["id"]] + assert got["kind"] == item["kind"] + assert got["classification"] == item["classification"] + assert got["ts"] == item["ts"] + assert got["source_ids"] == item["source_ids"] + for attr_key, attr_value in item.get("attributes", {}).items(): + assert got["attributes"].get(attr_key) == attr_value + continue + if key == "call_candidates": + by_id = {item["call_id"]: item for item in actual} + for item in wanted: + got = by_id[item["call_id"]] + for field, value in item.items(): + assert got[field] == value, field + continue + assert actual == wanted + + +def test_build_semantics_does_not_import_usage_or_attribution_modules() -> None: + forbidden = {"usage", "attribution", "projection_store", "export_state"} + for module_name in ("tracing", "turns", "events"): + source = Path( + __import__(f"thirdeye.platforms.copilot.{module_name}", fromlist=["__file__"]).__file__ + ) + tree = ast.parse(source.read_text(encoding="utf-8")) + imported: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imported.update(alias.name.split(".", 1)[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module: + imported.add(node.module.split(".", 1)[0]) + imported.add(node.module) + assert not (imported & forbidden), module_name + + +# --- interaction grouping and turn completion --- + + +def test_tool_cycle_does_not_complete_user_turn_without_final_answer() -> None: + case = _load_json(RECON_CASES / "semantic-projection.json") + projection, state = build_semantics(case["input_records"], {}) + _assert_expected_projection(projection, case["expected"]) + + pending_kinds = Counter(item["kind"] for item in projection["pending"]) + assert pending_kinds["open_interaction"] == 1 + assert pending_kinds["incomplete_tool_pair"] == 2 + assert "missing_identity" not in pending_kinds + + open_key = "6d2b89fd-a653-430c-b532-b0936d72eb42|main" + assert open_key in state["open_interactions"] + assert state["open_interactions"][open_key]["pending_tool_call_ids"] == [ + "call_YSSva4HCniiETlxdGGjcrHbh", + "call_ayHplfzxjRFMTCpmTKEFhCSJ", + ] + candidate = projection["call_candidates"][0] + assert candidate["end_ts"] == "2026-09-10T17:08:24.593Z" + assert candidate["finish_evidence"][0]["kind"] == "assistant_turn_end" + + +def test_partial_user_prompt_stays_open_with_pending_item() -> None: + case = _load_json(RECON_CASES / "cases.json")["partial_turn"] + projection, state = build_semantics(case["input_records"], {}) + + assert projection["turns"] == [] + assert any(item["kind"] == "open_interaction" for item in projection["pending"]) + assert "6d2b89fd-a653-430c-b532-b0936d72eb42|main" in state["open_interactions"] + + +def test_missing_interaction_id_creates_pending_not_fabricated_turn() -> None: + record = _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/user-no-ix", + native_type="user.message", + data={"content": "orphan prompt", "turnId": "0"}, + ) + _, _, pending, _, _ = build_turns([record]) + + assert any(item["kind"] == "missing_identity" for item in pending) + assert all(item["kind"] != "open_interaction" for item in pending) + + +def test_incomplete_tool_pair_when_execution_lacks_request() -> None: + execution = _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/exec-orphan", + native_type="tool.execution_start", + data={"toolCallId": "call_orphan", "toolName": "view", "arguments": {}}, + ) + _, _, pending, _, _ = build_turns([execution]) + + assert pending == [ + { + "id": "pending:tool:call_orphan", + "kind": "incomplete_tool_pair", + "reason": "tool execution start has no requesting assistant message", + "source_ids": [execution["source_id"]], + "evidence": ["tool_call_id:call_orphan"], + } + ] + + +def test_identical_concurrent_tools_pair_by_tool_call_id() -> None: + case = _load_json(RECON_CASES / "cases.json")["identical_concurrent_tools"] + projection, _ = build_semantics(case["input_records"], {}) + expected = case["expected"] + + start_events = [ + event for event in projection["events"] if event["kind"] == "tool_execution_start" + ] + assert sorted(event["id"] for event in start_events) == sorted(expected["event_ids"]) + assert sorted(event["attributes"]["tool_call_id"] for event in start_events) == sorted( + expected["tool_call_ids"] + ) + + candidate = projection["call_candidates"][0] + assert candidate["tool_call_ids"] == expected["tool_call_ids"] + assert len({event["attributes"]["tool_call_id"] for event in start_events}) == 2 + + +def test_identical_concurrent_identical_args_pair_by_tool_call_id() -> None: + case = _load_json(RECON_CASES / "cases.json")["identical_concurrent_identical_args"] + assert case["observed"] is False + projection, _ = build_semantics(case["input_records"], {}) + expected = case["expected"] + + start_events = [ + event for event in projection["events"] if event["kind"] == "tool_execution_start" + ] + assert sorted(event["id"] for event in start_events) == sorted(expected["event_ids"]) + assert sorted(event["attributes"]["tool_call_id"] for event in start_events) == sorted( + expected["tool_call_ids"] + ) + arguments = [event["attributes"].get("arguments") for event in start_events] + assert arguments[0] == arguments[1] + assert {event["attributes"].get("name") for event in start_events} == {"view"} + candidate = projection["call_candidates"][0] + assert candidate["tool_call_ids"] == expected["tool_call_ids"] + assert len({event["attributes"]["tool_call_id"] for event in start_events}) == 2 + + +# --- nested child ownership --- + + +def test_nested_child_partial_records_do_not_create_main_turn_from_hook_session_id() -> None: + case = _load_json(RECON_CASES / "cases.json")["nested_child"] + projection, _ = build_semantics(case["input_records"], {}) + + assert projection["turns"] == [] + assert {event["kind"] for event in projection["events"]} == { + "subagent_started", + "user_prompt", + "prompt_transformation", + } + + hook_only = [case["input_records"][2]] + hook_projection, _ = build_semantics(hook_only, {}) + assert hook_projection["turns"] == [] + + +def test_child_turn_nests_under_parent_when_full_fixture_replayed( + cli_transcript_records: list[SourceRecord], +) -> None: + projection, _ = build_semantics(cli_transcript_records, {}) + parent = next( + turn + for turn in projection["turns"] + if turn["attributes"]["interaction_id"] == "793d3703-6f4a-4814-8877-34a7325848ce" + ) + assert len(parent["subagents"]) == 1 + child = parent["subagents"][0] + assert child["attributes"]["agent_id"] == CHILD_AGENT_ID + assert child["attributes"]["parent_tool_call_id"] == "call_qx4FH5DADTeT1qVLb37HNpBk" + assert all(turn["attributes"].get("agent_id") is None for turn in projection["turns"]) + + +def test_prior_open_interaction_state_is_retained_when_replay_still_open() -> None: + case = _load_json(RECON_CASES / "cases.json")["partial_turn"] + _, state = build_semantics(case["input_records"], {}) + prior_item = dict(state["open_interactions"]["6d2b89fd-a653-430c-b532-b0936d72eb42|main"]) + prior_item["note"] = "retained-from-incremental-caller" + + _, merged_state = build_semantics( + case["input_records"], {"open_interactions": state["open_interactions"]} + ) + retained = merged_state["open_interactions"]["6d2b89fd-a653-430c-b532-b0936d72eb42|main"] + assert retained["interaction_id"] == prior_item["interaction_id"] + assert retained["stored_turn_id"] == prior_item["stored_turn_id"] + + # Simulate a key only present in prior state (unfinished partition from earlier chunk). + orphan_key = "orphan|main" + prior = { + "open_interactions": { + orphan_key: { + "interaction_id": "orphan-interaction", + "agent_id": None, + "stored_turn_id": f"copilot:turn:{SOURCE_KEY}:{NATIVE_SESSION_ID}:orphan-interaction", + "source_ids": ["kept/source"], + "last_event_source_id": "kept/source", + "start_ts": "2026-09-10T17:08:00.000Z", + "pending_tool_call_ids": [], + } + } + } + _, merged = build_semantics(case["input_records"], prior) + assert orphan_key in merged["open_interactions"] + + +def test_prior_state_does_not_resurrect_completed_interaction( + cli_transcript_records: list[SourceRecord], +) -> None: + completed_key = "6d2b89fd-a653-430c-b532-b0936d72eb42|main" + interaction_id = completed_key.split("|", 1)[0] + user = next( + record + for record in cli_transcript_records + if record["payload"].get("type") == "user.message" + and record["payload"]["data"].get("interactionId") == interaction_id + ) + prior = { + "open_interactions": { + completed_key: { + "interaction_id": interaction_id, + "agent_id": None, + "stored_turn_id": ( + f"copilot:turn:{SOURCE_KEY}:{NATIVE_SESSION_ID}:{interaction_id}" + ), + "source_ids": [user["source_id"]], + "last_event_source_id": user["source_id"], + "start_ts": user.get("ts"), + "pending_tool_call_ids": [], + } + } + } + full, merged = build_semantics(cli_transcript_records, prior) + assert completed_key not in merged["open_interactions"] + assert any(turn["attributes"]["interaction_id"] == interaction_id for turn in full["turns"]) + + +# --- semantic projection contract slice --- + + +def test_semantic_projection_fixture_normalizes_expected_events() -> None: + case = _load_json(RECON_CASES / "semantic-projection.json") + projection, _ = build_semantics(case["input_records"], {}) + _assert_expected_projection(projection, case["expected"]) + turn_end_id = ( + "copilot:event:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/" + "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6/0080e44c-ad62-4288-b2b2-061ec2b73d80" + ) + by_id = {event["id"]: event for event in projection["events"]} + assert by_id[turn_end_id]["kind"] == "assistant_turn_end" + + +def test_auxiliary_model_events_do_not_create_call_candidates() -> None: + case = _load_json(RECON_CASES / "cases.json")["auxiliary_title_generation"] + assert case["observed"] is False + projection, _ = build_semantics(case["input_records"], {}) + _assert_expected_projection(projection, case["expected"]) + assert projection["turns"] == [] + + +def test_retry_case_is_database_only_and_emits_no_semantic_events() -> None: + """The shared retry fixture is accounting-only; semantics has nothing to reconstruct.""" + case = _load_json(RECON_CASES / "cases.json")["retry"] + assert case["observed"] is False + assert all(record["source_kind"] == "database" for record in case["input_records"]) + projection, _ = build_semantics(case["input_records"], {}) + assert projection["events"] == [] + assert projection["turns"] == [] + assert projection["call_candidates"] == [] + assert projection["pending"] == [] + + +def test_observed_versus_synthetic_cases_run_through_build_semantics(tmp_path: Path) -> None: + cases = _load_json(RECON_CASES / "cases.json") + observed = {name for name, case in cases.items() if case["observed"]} + synthetic = {name for name, case in cases.items() if not case["observed"]} + assert observed == {"identical_concurrent_tools", "nested_child"} + assert "retry" in synthetic + assert "permission" in synthetic + assert "partial_turn" in synthetic + assert "abort" in synthetic + assert "unknown_version" in synthetic + assert "identical_concurrent_identical_args" in synthetic + for name in ( + "identical_concurrent_tools", + "permission", + "partial_turn", + "abort", + "unknown_version", + "identical_concurrent_identical_args", + ): + projection, _ = build_semantics(cases[name]["input_records"], {}) + assert "events" in projection + _assert_expected_projection(projection, cases[name]["expected"]) + nested = build_semantics(cases["nested_child"]["input_records"], {})[0] + assert nested["turns"] == [] + started = [event for event in nested["events"] if event["kind"] == "subagent_started"] + assert started[0]["attributes"]["parent_tool_call_id"] == "call_qx4FH5DADTeT1qVLb37HNpBk" + + unknown = cases["unknown_version"] + unknown_projection, _ = build_semantics(unknown["input_records"], {}) + assert unknown_projection["turns"] == [] + unknown_event = next( + event for event in unknown_projection["events"] if event["kind"] == "unknown" + ) + assert "raw_payload" in unknown_event["attributes"] + assert unknown_event["attributes"]["raw_payload"]["schema_version"] == 2 + assert any( + item["kind"] == "missing_source_capability" for item in unknown_projection["pending"] + ) + + config = Config(root=tmp_path / "thirdeye") + home = tmp_path / "copilot-home" + home.mkdir() + paths = resolve_sources(home) + native_session_id = unknown["input_records"][0]["native_session_id"] + commit_batch( + config, + paths, + { + "source_key": paths["source_key"], + "native_session_id": native_session_id, + "cwd": "/proj", + "records": unknown["input_records"], + "next_cursor": {"generation": 1}, + "diagnostics": [], + }, + ) + stored = stored_session_id(paths, native_session_id) + result = reconcile_archive(config, stored) + assert result["turns"] == 0 + assert result["pending"] >= 1 + + +def test_completed_child_without_parent_link_is_pending_not_dropped() -> None: + child = "child-agent-1" + records = [ + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/child-user", + native_type="user.message", + data={"content": "explore", "interactionId": "child-ix", "turnId": "0"}, + agent=child, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/child-final", + native_type="assistant.message", + data={ + "content": "42", + "model": "gpt-5.6-luna", + "interactionId": "child-ix", + "turnId": "0", + "phase": "final_answer", + "toolRequests": [], + }, + agent=child, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/child-end", + native_type="assistant.turn_end", + data={"turnId": "0"}, + agent=child, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/child-stop", + native_type="subagent.completed", + data={"agentName": "explore"}, + agent=child, + ), + ] + projection, _ = build_semantics(records, {}) + assert projection["turns"] == [] + assert any(item["kind"] == "missing_identity" for item in projection["pending"]) + assert any(item["code"] == "capability_gap" for item in projection["diagnostics"]) + assert projection["call_candidates"][0]["stored_turn_id"] is None + + +def test_abort_emits_interrupted_turn() -> None: + records = [ + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/abort-user", + native_type="user.message", + data={"content": "hello", "interactionId": "ix-abort", "turnId": "0"}, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/abort-start", + native_type="assistant.turn_start", + data={"turnId": "0", "interactionId": "ix-abort"}, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/abort-msg", + native_type="assistant.message", + data={ + "content": "partial", + "model": "gpt-5.6-luna", + "interactionId": "ix-abort", + "turnId": "0", + "toolRequests": [], + }, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/abort-event", + native_type="assistant.abort", + data={"turnId": "0", "interactionId": "ix-abort"}, + ), + ] + projection, state = build_semantics(records, {}) + assert len(projection["turns"]) == 1 + assert projection["turns"][0]["status"] == "interrupted" + assert projection["turns"][0]["output_message"] == "partial" + assert state["open_interactions"] == {} + + +def test_intermediate_assistant_output_does_not_close_user_turn() -> None: + records = [ + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/mid-user", + native_type="user.message", + data={"content": "keep going", "interactionId": "ix-mid", "turnId": "0"}, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/mid-msg", + native_type="assistant.message", + data={ + "content": "working...", + "model": "gpt-5.6-luna", + "interactionId": "ix-mid", + "turnId": "0", + "toolRequests": [], + }, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/mid-end", + native_type="assistant.turn_end", + data={"turnId": "0"}, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/mid-tool-msg", + native_type="assistant.message", + data={ + "content": "", + "model": "gpt-5.6-luna", + "interactionId": "ix-mid", + "turnId": "1", + "toolRequests": [{"toolCallId": "call_later", "name": "view", "arguments": {}}], + }, + ), + ] + projection, state = build_semantics(records, {}) + assert projection["turns"] == [] + assert "ix-mid|main" in state["open_interactions"] + + +def test_turn_id_includes_agent_to_avoid_collision() -> None: + shared = "shared-interaction" + records = [ + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/main-user", + native_type="user.message", + data={"content": "parent", "interactionId": shared, "turnId": "0"}, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/child-user", + native_type="user.message", + data={"content": "child", "interactionId": shared, "turnId": "0"}, + agent="agent-child", + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/main-final", + native_type="assistant.message", + data={ + "content": "done", + "model": "gpt-5.6-luna", + "interactionId": shared, + "turnId": "0", + "phase": "final_answer", + "toolRequests": [], + }, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/child-final", + native_type="assistant.message", + data={ + "content": "done", + "model": "gpt-5.6-luna", + "interactionId": shared, + "turnId": "0", + "phase": "final_answer", + "toolRequests": [], + "parentToolCallId": "call_parent", + }, + agent="agent-child", + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/main-end", + native_type="assistant.turn_end", + data={"turnId": "0"}, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/child-end", + native_type="assistant.turn_end", + data={"turnId": "0"}, + agent="agent-child", + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/shutdown", + native_type="session.shutdown", + data={}, + ), + ] + main_id = f"copilot:turn:{SOURCE_KEY}:{NATIVE_SESSION_ID}:{shared}" + child_id = f"{main_id}:agent-child" + assert main_id != child_id + projection, _ = build_semantics(records, {}) + assert projection["turns"][0]["turn_id"] == main_id + stored = {candidate["stored_turn_id"] for candidate in projection["call_candidates"]} + assert main_id in stored + assert child_id not in stored + + +def test_permission_request_attaches_to_completed_turn() -> None: + records = [ + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/perm-user", + native_type="user.message", + data={"content": "read it", "interactionId": "ix-perm", "turnId": "0"}, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/perm-req", + native_type="permission.request", + data={ + "toolName": "view", + "toolArgs": {"path": "/tmp/a.txt"}, + "interactionId": "ix-perm", + "turnId": "0", + }, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/perm-dec", + native_type="permission.decision", + data={"toolName": "view", "decision": "allow", "interactionId": "ix-perm"}, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/perm-final", + native_type="assistant.message", + data={ + "content": "ok", + "model": "gpt-5.6-luna", + "interactionId": "ix-perm", + "turnId": "0", + "phase": "final_answer", + "toolRequests": [], + }, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/perm-end", + native_type="assistant.turn_end", + data={"turnId": "0"}, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/perm-shutdown", + native_type="session.shutdown", + data={}, + ), + ] + projection, _ = build_semantics(records, {}) + assert len(projection["turns"]) == 1 + requests = projection["turns"][0]["permission_requests"] + assert len(requests) == 1 + assert requests[0]["tool_name"] == "view" + assert requests[0]["attributes"]["decision"] == "allow" + + +def test_semantic_retry_after_error_stays_same_interaction() -> None: + records = [ + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/retry-user", + native_type="user.message", + data={"content": "try again", "interactionId": "ix-retry", "turnId": "0"}, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/retry-err-msg", + native_type="assistant.message", + data={ + "content": "", + "model": "gpt-5.6-luna", + "interactionId": "ix-retry", + "turnId": "0", + "toolRequests": [], + }, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/retry-error", + native_type="assistant.error", + data={"turnId": "0", "interactionId": "ix-retry", "message": "timeout"}, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/retry-final", + native_type="assistant.message", + data={ + "content": "42", + "model": "gpt-5.6-luna", + "interactionId": "ix-retry", + "turnId": "1", + "phase": "final_answer", + "toolRequests": [], + }, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/retry-end", + native_type="assistant.turn_end", + data={"turnId": "1"}, + ), + ] + projection, _ = build_semantics(records, {}) + assert len(projection["turns"]) == 1 + assert projection["turns"][0]["status"] == "completed" + assert projection["turns"][0]["output_message"] == "42" + assert len(projection["call_candidates"]) == 2 + assert {candidate["interaction_id"] for candidate in projection["call_candidates"]} == { + "ix-retry" + } + + +def test_turn_end_attaches_finish_to_matching_native_turn_not_last_call() -> None: + records = [ + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/ov-user", + native_type="user.message", + data={"content": "two cycles", "interactionId": "ix-overlap", "turnId": "0"}, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/ov-call-a", + native_type="assistant.message", + data={ + "content": "first", + "model": "gpt-5.6-luna", + "interactionId": "ix-overlap", + "turnId": "0", + "toolRequests": [], + }, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/ov-call-b", + native_type="assistant.message", + data={ + "content": "second", + "model": "gpt-5.6-luna", + "interactionId": "ix-overlap", + "turnId": "1", + "phase": "final_answer", + "toolRequests": [], + }, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/ov-end-0", + native_type="assistant.turn_end", + data={"turnId": "0"}, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/ov-end-1", + native_type="assistant.turn_end", + data={"turnId": "1"}, + ), + ] + projection, _ = build_semantics(records, {}) + by_id = {candidate["call_id"]: candidate for candidate in projection["call_candidates"]} + first = by_id[f"copilot:call:{SOURCE_KEY}/{NATIVE_SESSION_ID}/ov-call-a"] + second = by_id[f"copilot:call:{SOURCE_KEY}/{NATIVE_SESSION_ID}/ov-call-b"] + assert first["finish_evidence"][0]["source_id"].endswith("/ov-end-0") + assert second["finish_evidence"][0]["source_id"].endswith("/ov-end-1") + assert ( + first["end_ts"] != second["end_ts"] or first["finish_evidence"] != second["finish_evidence"] + ) + + +def test_unmatched_turn_end_is_missing_identity_not_incomplete_tool() -> None: + record = _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/orphan-end", + native_type="assistant.turn_end", + data={"turnId": "99"}, + ) + _, _, pending, _, _ = build_turns([record]) + assert pending + assert pending[0]["kind"] == "missing_identity" + assert all(item["kind"] != "incomplete_tool_pair" for item in pending) + + +def test_incomplete_tools_stay_pending_on_completed_turn() -> None: + records = [ + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/inc-user", + native_type="user.message", + data={"content": "read then answer", "interactionId": "ix-inc", "turnId": "0"}, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/inc-tools", + native_type="assistant.message", + data={ + "content": "", + "model": "gpt-5.6-luna", + "interactionId": "ix-inc", + "turnId": "0", + "toolRequests": [{"toolCallId": "call_missing", "name": "view", "arguments": {}}], + }, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/inc-final", + native_type="assistant.message", + data={ + "content": "42", + "model": "gpt-5.6-luna", + "interactionId": "ix-inc", + "turnId": "1", + "phase": "final_answer", + "toolRequests": [], + }, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/inc-end", + native_type="assistant.turn_end", + data={"turnId": "1"}, + ), + ] + projection, _ = build_semantics(records, {}) + assert len(projection["turns"]) == 1 + assert any( + item["kind"] == "incomplete_tool_pair" and "call_missing" in item["id"] + for item in projection["pending"] + ) + + +def test_followup_user_message_does_not_mark_previous_turn_completed() -> None: + records = [ + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/prev-user", + native_type="user.message", + data={"content": "first", "interactionId": "ix-old", "turnId": "0"}, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/prev-tools", + native_type="assistant.message", + data={ + "content": "", + "model": "gpt-5.6-luna", + "interactionId": "ix-old", + "turnId": "0", + "toolRequests": [{"toolCallId": "call_out", "name": "view", "arguments": {}}], + }, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/next-user", + native_type="user.message", + data={"content": "second", "interactionId": "ix-new", "turnId": "0"}, + ), + ] + projection, state = build_semantics(records, {}) + old_turns = [ + turn for turn in projection["turns"] if turn["attributes"]["interaction_id"] == "ix-old" + ] + assert old_turns == [] + assert "ix-old|main" in state["open_interactions"] + assert any( + item["kind"] == "open_interaction" and "ix-old" in item["evidence"][0] + for item in projection["pending"] + ) + + +def test_completed_child_is_retained_when_later_partition_only_has_parent() -> None: + child = "agent-child" + parent_tool = "call_parent_task" + parent_open = [ + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/p-user", + native_type="user.message", + data={"content": "delegate", "interactionId": "ix-parent", "turnId": "0"}, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/p-task", + native_type="assistant.message", + data={ + "content": "", + "model": "gpt-5.6-luna", + "interactionId": "ix-parent", + "turnId": "0", + "toolRequests": [{"toolCallId": parent_tool, "name": "task", "arguments": {}}], + }, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/p-task-end", + native_type="assistant.turn_end", + data={"turnId": "0"}, + ), + ] + child_complete = [ + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/c-start", + native_type="subagent.started", + data={"toolCallId": parent_tool, "agentName": "explore"}, + agent=child, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/c-user", + native_type="user.message", + data={"content": "explore", "interactionId": "ix-child", "turnId": "0"}, + agent=child, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/c-final", + native_type="assistant.message", + data={ + "content": "42", + "model": "gpt-5.6-luna", + "interactionId": "ix-child", + "turnId": "0", + "phase": "final_answer", + "toolRequests": [], + "parentToolCallId": parent_tool, + }, + agent=child, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/c-end", + native_type="assistant.turn_end", + data={"turnId": "0"}, + agent=child, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/c-stop", + native_type="subagent.completed", + data={"toolCallId": parent_tool, "agentName": "explore"}, + agent=child, + ), + ] + parent_close = [ + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/p-final", + native_type="assistant.message", + data={ + "content": "42", + "model": "gpt-5.6-luna", + "interactionId": "ix-parent", + "turnId": "1", + "phase": "final_answer", + "toolRequests": [], + }, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/p-final-end", + native_type="assistant.turn_end", + data={"turnId": "1"}, + ), + ] + _, state = build_semantics([*parent_open, *child_complete], {}) + open_parent = state["open_interactions"]["ix-parent|main"] + assert open_parent.get("nested_children") + + projection, _ = build_semantics([*parent_open, *parent_close], state) + assert len(projection["turns"]) == 1 + assert len(projection["turns"][0]["subagents"]) == 1 + assert projection["turns"][0]["subagents"][0]["output_message"] == "42" + + full, _ = build_semantics([*parent_open, *child_complete, *parent_close], {}) + assert len(full["turns"][0]["subagents"]) == 1 + + +def test_reasoning_summary_is_readable_span_content() -> None: + records = [ + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/rs-user", + native_type="user.message", + data={"content": "why", "interactionId": "ix-rs", "turnId": "0"}, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/rs-msg", + native_type="assistant.message", + data={ + "content": "42", + "model": "gpt-5.6-luna", + "interactionId": "ix-rs", + "turnId": "0", + "phase": "final_answer", + "reasoningSummary": "added the two file values", + "intentionSummary": "tool intent must not become reasoning", + "toolRequests": [], + }, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/rs-end", + native_type="assistant.turn_end", + data={"turnId": "0"}, + ), + ] + projection, _ = build_semantics(records, {}) + parts = projection["turns"][0]["llm_calls"][0]["output_messages"][0]["parts"] + reasoning = [part for part in parts if part["type"] == "reasoning"] + assert reasoning == [{"type": "reasoning", "content": "added the two file values"}] + assert all(part["content"] != "tool intent must not become reasoning" for part in parts) + + +def test_failed_tool_execution_pairs_through_build_turns() -> None: + records = [ + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/fail-user", + native_type="user.message", + data={"content": "read it", "interactionId": "ix-fail", "turnId": "0"}, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/fail-req", + native_type="assistant.message", + data={ + "content": "", + "model": "gpt-5.6-luna", + "interactionId": "ix-fail", + "turnId": "0", + "toolRequests": [ + {"toolCallId": "call_denied", "name": "view", "arguments": {"path": "/tmp/x"}} + ], + }, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/fail-result", + native_type="tool.execution_complete", + data={"toolCallId": "call_denied", "success": False, "result": "denied"}, + ts="2026-09-10T17:08:24.700Z", + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/fail-final", + native_type="assistant.message", + data={ + "content": "could not read", + "model": "gpt-5.6-luna", + "interactionId": "ix-fail", + "turnId": "1", + "phase": "final_answer", + "toolRequests": [], + }, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/fail-end", + native_type="assistant.turn_end", + data={"turnId": "1"}, + ), + ] + projection, _ = build_semantics(records, {}) + tools = projection["turns"][0]["llm_calls"][0]["tool_calls"] + assert len(tools) == 1 + assert tools[0]["tool_call_id"] == "call_denied" + assert tools[0]["attributes"]["success"] is False + assert tools[0]["attributes"]["result"] == "denied" + assert tools[0]["end_ts"] == "2026-09-10T17:08:24.700Z" + assert not any(item["kind"] == "incomplete_tool_pair" for item in projection["pending"]) + + +def test_assistant_markers_without_interaction_id_are_pending() -> None: + start = _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/noix-start", + native_type="assistant.turn_start", + data={"turnId": "0"}, + ) + message = _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/noix-msg", + native_type="assistant.message", + data={"content": "hello", "model": "gpt-5.6-luna", "turnId": "0", "toolRequests": []}, + ) + _, _, pending, _, _ = build_turns([start, message]) + kinds = [item["kind"] for item in pending] + assert kinds.count("missing_identity") >= 2 + assert all(item["kind"] != "open_interaction" for item in pending) + + +def test_mixed_hook_and_transcript_does_not_double_count_tools() -> None: + records: list[SourceRecord] = [ + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/mix-user", + native_type="user.message", + data={"content": "read", "interactionId": "ix-mix", "turnId": "0"}, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/mix-req", + native_type="assistant.message", + data={ + "content": "", + "model": "gpt-5.6-luna", + "interactionId": "ix-mix", + "turnId": "0", + "toolRequests": [{"toolCallId": "call_mix", "name": "view", "arguments": {}}], + }, + ), + { + "source_id": f"hook/{NATIVE_SESSION_ID}/obs-pre-mix", + "source_kind": "hook", + "native_session_id": NATIVE_SESSION_ID, + "ts": "2026-09-10T17:08:24.500Z", + "observed_at": "2026-09-10T17:09:00.000Z", + "payload": { + "schema_version": 1, + "event": "preToolUse", + "hook_payload": {"toolName": "view", "toolArgs": {}}, + "context": {}, + }, + "locator": {"observation_id": "obs-pre-mix", "event": "preToolUse"}, + }, + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/mix-start", + native_type="tool.execution_start", + data={"toolCallId": "call_mix", "toolName": "view", "arguments": {}}, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/mix-done", + native_type="tool.execution_complete", + data={"toolCallId": "call_mix", "success": True, "result": "17"}, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/mix-final", + native_type="assistant.message", + data={ + "content": "17", + "model": "gpt-5.6-luna", + "interactionId": "ix-mix", + "turnId": "1", + "phase": "final_answer", + "toolRequests": [], + }, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/mix-end", + native_type="assistant.turn_end", + data={"turnId": "1"}, + ), + ] + projection, _ = build_semantics(records, {}) + starts = [event for event in projection["events"] if event["kind"] == "tool_execution_start"] + assert len(starts) == 1 + hook_obs = [ + event + for event in projection["events"] + if event["source_ids"] == [f"hook/{NATIVE_SESSION_ID}/obs-pre-mix"] + ] + assert hook_obs[0]["kind"] in {"notification", "unknown"} + + +def test_cli_fixture_has_one_subagent_started( + cli_transcript_records: list[SourceRecord], +) -> None: + projection, _ = build_semantics(cli_transcript_records, {}) + started = [event for event in projection["events"] if event["kind"] == "subagent_started"] + assert len(started) == 1 + configured = [ + event + for event in projection["events"] + if (event.get("attributes") or {}).get("native_type") == "subagent.configured" + ] + assert configured + assert configured[0]["kind"] != "subagent_started" + + +def test_prompt_transformation_flows_through_build_semantics() -> None: + records = [ + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/xf-user", + native_type="user.message", + data={"content": "hi", "interactionId": "ix-xf", "turnId": "0"}, + ), + _transcript_record( + source_id=f"{SOURCE_KEY}/{NATIVE_SESSION_ID}/xf-prompt", + native_type="prompt.transformation", + data={"interactionId": "ix-xf", "prompt": "expanded hi"}, + ), + ] + projection, _ = build_semantics(records, {}) + kinds = [event["kind"] for event in projection["events"]] + assert "prompt_transformation" in kinds diff --git a/tests/platforms/copilot/test_transcript.py b/tests/platforms/copilot/test_transcript.py new file mode 100644 index 00000000..f9f3f1f5 --- /dev/null +++ b/tests/platforms/copilot/test_transcript.py @@ -0,0 +1,682 @@ +"""Behavioral tests for the Copilot CLI transcript reader.""" + +from __future__ import annotations + +import json +import shutil +from pathlib import Path +from typing import Any + +import pytest + +from thirdeye.platforms.copilot.constants import SOURCE_SCHEMA_VERSION +from thirdeye.platforms.copilot.identity import resolve_sources +from thirdeye.platforms.copilot.transcript import discover_transcripts, read_transcript +from thirdeye.platforms.copilot.types import SourcePaths, SourceRecord, SourceSlice + +FIXTURES = Path(__file__).parent / "fixtures" +V1_CASES = FIXTURES / "cases" +CLI_FIXTURE = FIXTURES +NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" + + +def _session_paths(home: Path) -> SourcePaths: + return resolve_sources(home) + + +def _write_session( + home: Path, + native_id: str, + *, + events: str | bytes | None = None, + workspace: str | None = None, +) -> Path: + session_dir = home / "session-state" / native_id + session_dir.mkdir(parents=True, exist_ok=True) + if events is not None: + (session_dir / "events.jsonl").write_bytes( + events if isinstance(events, bytes) else events.encode("utf-8") + ) + if workspace is not None: + (session_dir / "workspace.yaml").write_text(workspace, encoding="utf-8") + return session_dir + + +def _transcript_records(slice_: SourceSlice) -> list[SourceRecord]: + return [record for record in slice_["records"] if record["source_kind"] == "transcript"] + + +def _metadata_records(slice_: SourceSlice) -> list[SourceRecord]: + return [record for record in slice_["records"] if record["source_kind"] == "metadata"] + + +def _diagnostic_codes(slice_: SourceSlice) -> set[str]: + return {item["code"] for item in slice_["diagnostics"]} + + +def _drain_transcript( + paths: SourcePaths, + native_id: str, + *, + max_records: int = 1000, + max_bytes: int = 4_194_304, +) -> tuple[list[SourceRecord], list[dict[str, Any]], SourceSlice]: + cursor: dict[str, Any] = {} + records: list[SourceRecord] = [] + diagnostics: list[dict[str, Any]] = [] + last: SourceSlice | None = None + while True: + last = read_transcript( + paths, + native_id, + cursor, + max_records=max_records, + max_bytes=max_bytes, + ) + records.extend(last["records"]) + diagnostics.extend(last["diagnostics"]) + cursor = last["next_cursor"] + if last["exhausted"]: + break + assert last is not None + return records, diagnostics, last + + +# --- discovery --- + + +def test_discover_transcripts_finds_sessions_with_events(tmp_path: Path): + home = tmp_path / "copilot" + _write_session(home, "session-b", events='{"id":"1"}\n') + _write_session(home, "session-a", events='{"id":"2"}\n') + (home / "session-state" / "no-events").mkdir(parents=True) + + assert discover_transcripts(_session_paths(home)) == ["session-a", "session-b"] + + +def test_discover_transcripts_ignores_unsafe_or_outside_entries(tmp_path: Path): + home = tmp_path / "copilot" + root = home / "session-state" + root.mkdir(parents=True) + _write_session(home, "valid", events='{"id":"1"}\n') + (root / "bad/id").mkdir(parents=True) + (root / "bad/id" / "events.jsonl").write_text('{"id":"x"}\n', encoding="utf-8") + (root / "..").resolve() # ensure traversal candidate exists on POSIX + + assert discover_transcripts(_session_paths(home)) == ["valid"] + + +def test_discover_transcripts_returns_empty_when_root_missing(tmp_path: Path): + home = tmp_path / "missing-home" + assert discover_transcripts(_session_paths(home)) == [] + + +def test_read_transcript_rejects_traversal_native_id(tmp_path: Path): + home = tmp_path / "copilot" + _write_session(home, "valid", events='{"id":"1"}\n') + paths = _session_paths(home) + with pytest.raises(ValueError, match="path separator"): + read_transcript(paths, "../valid", {}) + + +# --- happy path and metadata --- + + +def test_read_transcript_preserves_unknown_fields_and_native_event_id(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + fixture = (V1_CASES / "unknown-event-fields.jsonl").read_text(encoding="utf-8") + _write_session(home, native, events=fixture, workspace="cwd: /fixture/workspace\n") + paths = _session_paths(home) + + slice_ = read_transcript(paths, native, {}) + transcript = _transcript_records(slice_)[0] + metadata = _metadata_records(slice_)[0] + + assert transcript["source_id"] == f"{paths['source_key']}/{native}/unknown-1" + assert transcript["ts"] == "2026-09-10T17:08:24.000Z" + assert transcript["payload"]["schema_version"] == SOURCE_SCHEMA_VERSION + assert transcript["payload"]["top_level_future"] == "retain" + assert transcript["payload"]["data"]["future_field"]["nested"] == [ + 1, + True, + {"opaque": "retain"}, + ] + assert transcript["locator"]["native_event_id"] == "unknown-1" + assert metadata["payload"]["data"]["cwd"] == "/fixture/workspace" + assert slice_["cwd"] == "/fixture/workspace" + + +def test_read_transcript_uses_workspace_path_fallback(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + _write_session( + home, + native, + events='{"id":"1","type":"user.message"}\n', + workspace="workspacePath: /from/workspacePath\n", + ) + slice_ = read_transcript(_session_paths(home), native, {}) + assert slice_["cwd"] == "/from/workspacePath" + + +def test_read_transcript_missing_event_id_uses_lower_confidence_locator(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + fixture = (V1_CASES / "missing-event-id.jsonl").read_text(encoding="utf-8") + _write_session(home, native, events=fixture) + paths = _session_paths(home) + + record = _transcript_records(read_transcript(paths, native, {}))[0] + assert record["source_id"].startswith(f"{paths['source_key']}/{native}/") + assert "native_event_id" not in record["locator"] + assert record["locator"]["identity_confidence"] == "lower" + assert "content_digest" in record["locator"] + assert record["locator"]["byte_offset"] == 0 + + +# --- malformed and invalid complete lines --- + + +def test_read_transcript_invalid_utf8_complete_line_is_retained(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + bad = b"\xffnot-utf8\n" + _write_session(home, native, events=bad) + slice_ = read_transcript(_session_paths(home), native, {}) + + record = _transcript_records(slice_)[0] + assert record["payload"]["malformed"] == "invalid_utf8" + assert "raw_bytes_base64" in record["payload"] + assert "transcript_invalid_utf8" in _diagnostic_codes(slice_) + assert slice_["exhausted"] is True + + +def test_read_transcript_invalid_json_complete_line_is_retained(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + _write_session(home, native, events="not json at all\n") + slice_ = read_transcript(_session_paths(home), native, {}) + + record = _transcript_records(slice_)[0] + assert record["payload"]["malformed"] == "invalid_json" + assert record["payload"]["raw_line"] == "not json at all" + assert "transcript_invalid_json" in _diagnostic_codes(slice_) + + +def test_read_transcript_non_object_json_complete_line_is_retained(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + _write_session(home, native, events='["array"]\n') + slice_ = read_transcript(_session_paths(home), native, {}) + + record = _transcript_records(slice_)[0] + assert record["payload"]["malformed"] == "non_object_json" + assert record["payload"]["raw_value"] == ["array"] + assert "transcript_non_object" in _diagnostic_codes(slice_) + + +def test_read_transcript_invalid_timestamp_emits_diagnostic_but_keeps_record(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + _write_session(home, native, events='{"id":"1","timestamp":"not-a-date"}\n') + slice_ = read_transcript(_session_paths(home), native, {}) + + record = _transcript_records(slice_)[0] + assert record["ts"] is None + assert record["payload"]["timestamp"] == "not-a-date" + assert "transcript_invalid_timestamp" in _diagnostic_codes(slice_) + + +# --- partial trailing lines --- + + +def test_read_transcript_trailing_json_fixture_treats_unterminated_line_as_malformed( + tmp_path: Path, +): + """The v1 trailing-json fixture ends with a newline; it is a complete physical line.""" + + home = tmp_path / "copilot" + native = "session-a" + fixture = (V1_CASES / "trailing-json.jsonl").read_bytes() + _write_session(home, native, events=fixture) + paths = _session_paths(home) + + slice_ = read_transcript(paths, native, {}) + records = _transcript_records(slice_) + assert len(records) == 2 + assert records[0]["payload"]["id"] == "complete-1" + assert records[1]["payload"]["malformed"] == "invalid_json" + assert "transcript_invalid_json" in _diagnostic_codes(slice_) + assert slice_["exhausted"] is True + + +def test_read_transcript_defers_physical_line_without_newline(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + complete = b'{"id":"complete-1","type":"known"}\n' + partial = b'{"id":"incomplete-2","type":"unterminated"' + _write_session(home, native, events=complete + partial) + paths = _session_paths(home) + + first = read_transcript(paths, native, {}) + assert len(_transcript_records(first)) == 1 + assert first["next_cursor"]["byte_offset"] == len(complete) + assert first["exhausted"] is False + + +def test_read_transcript_replacement_replays_deferred_partial_line(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + complete = b'{"id":"complete-1","type":"known"}\n' + partial = b'{"id":"incomplete-2","type":"unterminated"' + finished = complete + b'{"id":"incomplete-2","type":"now-complete"}\n' + path = _write_session(home, native, events=complete + partial) + paths = _session_paths(home) + + first = read_transcript(paths, native, {}) + event_path = path / "events.jsonl" + replacement = path / "replacement-events.jsonl" + replacement.write_bytes(finished) + replacement.replace(event_path) + + second = read_transcript(paths, native, first["next_cursor"]) + assert "transcript_replaced" in _diagnostic_codes(second) + ids = [record["payload"].get("id") for record in _transcript_records(second)] + assert ids == ["complete-1", "incomplete-2"] + assert second["exhausted"] is True + + +def test_read_transcript_defers_trailing_incomplete_utf8(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + raw = bytes.fromhex((V1_CASES / "trailing-utf8.hex").read_text(encoding="utf-8").strip()) + _write_session(home, native, events=raw) + paths = _session_paths(home) + + first = read_transcript(paths, native, {}) + assert len(_transcript_records(first)) == 1 + assert first["exhausted"] is False + assert first["next_cursor"]["byte_offset"] == raw.index(b"\n") + 1 + + +def test_read_transcript_completes_deferred_utf8_after_replacement(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + raw = bytes.fromhex((V1_CASES / "trailing-utf8.hex").read_text(encoding="utf-8").strip()) + path = _write_session(home, native, events=raw) + paths = _session_paths(home) + + first = read_transcript(paths, native, {}) + assert len(_transcript_records(first)) == 1 + assert first["exhausted"] is False + + completed = raw + b"\xac\n" + event_path = path / "events.jsonl" + replacement = path / "replacement-events.jsonl" + replacement.write_bytes(completed) + replacement.replace(event_path) + + second = read_transcript(paths, native, first["next_cursor"]) + assert "transcript_replaced" in _diagnostic_codes(second) + transcript = _transcript_records(second) + assert len(transcript) == 2 + assert transcript[1]["payload"]["malformed"] == "invalid_json" + assert "transcript_invalid_json" in _diagnostic_codes(second) + + +# --- bounds and snapshot-end behavior --- + + +def test_read_transcript_respects_max_records_and_max_bytes(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + lines = [json.dumps({"id": f"event-{index}", "payload": "x" * 40}) for index in range(4)] + _write_session(home, native, events="\n".join(lines) + "\n") + paths = _session_paths(home) + + by_count = read_transcript(paths, native, {}, max_records=2) + assert len(_transcript_records(by_count)) == 2 + assert by_count["exhausted"] is False + + by_bytes = read_transcript(paths, native, {}, max_records=100, max_bytes=120) + assert len(_transcript_records(by_bytes)) == 1 + assert by_bytes["exhausted"] is False + + +def test_read_transcript_accepts_one_oversize_complete_record(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + big = json.dumps({"id": "big", "payload": "x" * 256}) + _write_session(home, native, events=f"{big}\n") + slice_ = read_transcript(_session_paths(home), native, {}, max_bytes=32) + + assert len(_transcript_records(slice_)) == 1 + assert "transcript_record_oversize" in _diagnostic_codes(slice_) + assert slice_["exhausted"] is True + + +def test_read_transcript_snapshot_end_drains_initial_file_without_later_appends(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + path = _write_session( + home, + native, + events='{"id":"first"}\n{"id":"second"}\n', + ) + paths = _session_paths(home) + + first = read_transcript(paths, native, {}, max_records=1) + assert [record["payload"]["id"] for record in _transcript_records(first)] == ["first"] + snapshot_end = first["next_cursor"]["snapshot_end"] + + (path / "events.jsonl").open("a", encoding="utf-8").write('{"id":"third"}\n') + + second = read_transcript(paths, native, first["next_cursor"], max_records=10) + assert second["next_cursor"]["snapshot_end"] == snapshot_end + assert [record["payload"]["id"] for record in _transcript_records(second)] == ["second"] + assert second["exhausted"] is True + + third = read_transcript(paths, native, second["next_cursor"]) + assert [record["payload"]["id"] for record in _transcript_records(third)] == ["third"] + assert third["exhausted"] is True + + +def test_read_transcript_opens_new_snapshot_for_appended_partial_line(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + path = _write_session(home, native, events='{"id":"first"}\n') + paths = _session_paths(home) + first = read_transcript(paths, native, {}) + assert first["exhausted"] is True + + with (path / "events.jsonl").open("ab") as stream: + stream.write(b'{"id":"partial-without-newline"') + + second = read_transcript(paths, native, first["next_cursor"]) + assert _transcript_records(second) == [] + assert second["next_cursor"]["byte_offset"] == first["next_cursor"]["byte_offset"] + assert second["exhausted"] is False + + +def test_read_transcript_appended_newline_completes_deferred_partial_line(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + path = _write_session(home, native, events='{"id":"first"}\n') + paths = _session_paths(home) + first = read_transcript(paths, native, {}) + assert first["exhausted"] is True + + with (path / "events.jsonl").open("ab") as stream: + stream.write(b'{"id":"second","type":"appended"') + + second = read_transcript(paths, native, first["next_cursor"]) + assert _transcript_records(second) == [] + assert second["exhausted"] is False + + with (path / "events.jsonl").open("ab") as stream: + stream.write(b"}\n") + + third = read_transcript(paths, native, second["next_cursor"]) + assert [record["payload"]["id"] for record in _transcript_records(third)] == ["second"] + assert third["exhausted"] is True + + +def test_read_transcript_reads_appended_complete_line_in_fresh_snapshot(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + path = _write_session(home, native, events='{"id":"first"}\n') + paths = _session_paths(home) + first = read_transcript(paths, native, {}) + assert first["exhausted"] is True + + with (path / "events.jsonl").open("ab") as stream: + stream.write(b'{"id":"second"}\n') + + second = read_transcript(paths, native, first["next_cursor"]) + assert [record["payload"]["id"] for record in _transcript_records(second)] == ["second"] + assert second["exhausted"] is True + + +# --- replacement, truncation, and cursor recovery --- + + +def test_read_transcript_replays_after_file_replacement(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + path = _write_session(home, native, events='{"id":"old"}\n') + paths = _session_paths(home) + + first = read_transcript(paths, native, {}) + assert first["exhausted"] is True + + event_path = path / "events.jsonl" + replacement = path / "replacement-events.jsonl" + replacement_bytes = b'{"id":"new"}\n' + replacement.write_bytes(replacement_bytes) + replacement.replace(event_path) + + second = read_transcript(paths, native, first["next_cursor"]) + assert "transcript_replaced" in _diagnostic_codes(second) + assert [record["payload"]["id"] for record in _transcript_records(second)] == ["new"] + assert second["next_cursor"]["byte_offset"] == len(replacement_bytes) + + +def test_read_transcript_replays_after_truncation(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + path = _write_session( + home, + native, + events='{"id":"first"}\n{"id":"second"}\n', + ) + paths = _session_paths(home) + first = read_transcript(paths, native, {}, max_records=1) + assert first["next_cursor"]["byte_offset"] > 0 + + event_path = path / "events.jsonl" + event_path.write_text('{"id":"shorter"}\n', encoding="utf-8") + + second = read_transcript(paths, native, first["next_cursor"]) + assert "transcript_replaced" in _diagnostic_codes(second) + assert [record["payload"]["id"] for record in _transcript_records(second)] == ["shorter"] + + +def test_read_transcript_invalid_cursor_replays_from_zero(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + _write_session(home, native, events='{"id":"one"}\n') + paths = _session_paths(home) + + slice_ = read_transcript(paths, native, {"byte_offset": -5}) + assert "transcript_cursor_invalid" in _diagnostic_codes(slice_) + assert [record["payload"]["id"] for record in _transcript_records(slice_)] == ["one"] + + +def test_read_transcript_repeatable_source_ids_after_replacement(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + line = '{"id":"stable","type":"user.message"}\n' + path = _write_session(home, native, events=line) + paths = _session_paths(home) + + first_id = _transcript_records(read_transcript(paths, native, {}))[0]["source_id"] + (path / "events.jsonl").write_text(line, encoding="utf-8") + second_id = _transcript_records(read_transcript(paths, native, {}))[0]["source_id"] + assert first_id == second_id + + +def test_read_transcript_inplace_rewrite_before_continuity_window_replays(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + original = b'{"id":"orig"}\n' + rewritten = b'{"id":"edit"}\n' + assert len(original) == len(rewritten) + padding = b"".join( + json.dumps({"id": f"pad-{index:03d}", "body": "x" * 64}).encode() + b"\n" + for index in range(80) + ) + path = _write_session(home, native, events=original + padding) + paths = _session_paths(home) + + first = read_transcript(paths, native, {}) + assert first["next_cursor"]["byte_offset"] > 4096 + assert [record["payload"]["id"] for record in _transcript_records(first)[:1]] == ["orig"] + + with (path / "events.jsonl").open("r+b") as stream: + stream.seek(0) + stream.write(rewritten) + + second = read_transcript(paths, native, first["next_cursor"]) + assert "transcript_replaced" in _diagnostic_codes(second) + assert [record["payload"]["id"] for record in _transcript_records(second)[:1]] == ["edit"] + + +# --- unavailable source and workspace errors --- + + +def test_read_transcript_missing_events_is_not_exhausted(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + (home / "session-state" / native).mkdir(parents=True) + slice_ = read_transcript(_session_paths(home), native, {}) + + assert slice_["records"] == [] + assert slice_["exhausted"] is False + assert "transcript_unavailable" in _diagnostic_codes(slice_) + + +def _symlink_or_skip(link: Path, target: Path) -> None: + try: + link.symlink_to(target) + except OSError as exc: + pytest.skip(f"symlinks unavailable: {exc}") + + +def test_read_transcript_rejects_events_symlink_outside_session(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + session_dir = _write_session(home, native) + outside = tmp_path / "outside-events.jsonl" + outside.write_text('{"id":"stolen"}\n', encoding="utf-8") + _symlink_or_skip(session_dir / "events.jsonl", outside) + + paths = _session_paths(home) + slice_ = read_transcript(paths, native, {}) + assert _transcript_records(slice_) == [] + assert "stolen" not in json.dumps(slice_["records"]) + assert slice_["exhausted"] is False + assert "transcript_path_escaped" in _diagnostic_codes(slice_) + assert discover_transcripts(paths) == [] + + +def test_read_transcript_rejects_workspace_symlink_outside_session(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + session_dir = _write_session(home, native, events='{"id":"1"}\n') + outside = tmp_path / "outside-workspace.yaml" + outside.write_text("cwd: /stolen/workspace\n", encoding="utf-8") + _symlink_or_skip(session_dir / "workspace.yaml", outside) + + slice_ = read_transcript(_session_paths(home), native, {}) + assert [record["payload"]["id"] for record in _transcript_records(slice_)] == ["1"] + assert _metadata_records(slice_) == [] + assert slice_["cwd"] is None + assert "/stolen/workspace" not in json.dumps(slice_["records"]) + assert "workspace_path_escaped" in _diagnostic_codes(slice_) + + +def test_read_transcript_invalid_workspace_emits_diagnostic_without_blocking_events(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + _write_session( + home, + native, + events='{"id":"1"}\n', + workspace="!!: not: valid: yaml: [\n", + ) + slice_ = read_transcript(_session_paths(home), native, {}) + + assert len(_transcript_records(slice_)) == 1 + assert "workspace_metadata_invalid" in _diagnostic_codes(slice_) + assert slice_["cwd"] is None + + +def test_read_transcript_rejects_negative_limits(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + _write_session(home, native, events='{"id":"1"}\n') + paths = _session_paths(home) + with pytest.raises(ValueError, match="max_records and max_bytes"): + read_transcript(paths, native, {}, max_records=-1) + + +# --- observed cli fixture via injected home layout --- + + +def test_read_transcript_cli_fixture_preserves_seventy_six_events(tmp_path: Path): + home = tmp_path / "copilot" + native = NATIVE_SESSION_ID + session_dir = _write_session(home, native) + shutil.copy(CLI_FIXTURE / "events.jsonl", session_dir / "events.jsonl") + (session_dir / "workspace.yaml").write_text( + "cwd: /sanitized/workspace\nrepo: probe\n", + encoding="utf-8", + ) + paths = _session_paths(home) + + records, diagnostics, last = _drain_transcript(paths, native) + transcript = [record for record in records if record["source_kind"] == "transcript"] + metadata = [record for record in records if record["source_kind"] == "metadata"] + + assert len(transcript) == 76 + assert len(metadata) == 1 + assert last["cwd"] == "/sanitized/workspace" + assert diagnostics == [] or all( + item["code"] != "transcript_invalid_json" for item in diagnostics + ) + + user_messages = [ + record for record in transcript if record["payload"].get("type") == "user.message" + ] + assert len(user_messages) == 3 + assert all( + record["payload"]["schema_version"] == SOURCE_SCHEMA_VERSION for record in transcript + ) + + +def test_read_transcript_cursor_advances_by_byte_offsets(tmp_path: Path): + home = tmp_path / "copilot" + native = "session-a" + first_line = '{"id":"first"}\n' + second_line = '{"id":"second"}\n' + _write_session(home, native, events=first_line + second_line) + paths = _session_paths(home) + + first = read_transcript(paths, native, {}, max_records=1) + assert first["next_cursor"]["byte_offset"] == len(first_line) + assert "continuity_start" in first["next_cursor"] + assert "continuity_digest" in first["next_cursor"] + + second = read_transcript(paths, native, first["next_cursor"], max_records=1) + assert second["next_cursor"]["byte_offset"] == len(first_line) + len(second_line) + + +def test_read_transcript_module_has_no_forbidden_imports(): + source = Path( + __import__("thirdeye.platforms.copilot.transcript", fromlist=["__file__"]).__file__ + ) + imports = [ + line.strip() + for line in source.read_text(encoding="utf-8").splitlines() + if line.startswith(("import ", "from ")) + ] + joined = "\n".join(imports) + for forbidden in ( + "thirdeye.store", + "sqlite3", + "hook_payload", + "platforms.copilot.archive", + "platforms.copilot.database", + ): + assert forbidden not in joined diff --git a/tests/platforms/copilot/test_usage.py b/tests/platforms/copilot/test_usage.py new file mode 100644 index 00000000..f7c56c3d --- /dev/null +++ b/tests/platforms/copilot/test_usage.py @@ -0,0 +1,830 @@ +"""Behavioral tests for Copilot database usage normalization.""" + +from __future__ import annotations + +import copy +import json +from pathlib import Path +from typing import Any + +from thirdeye.platforms.copilot.identity import resolve_sources +from thirdeye.platforms.copilot.types import SourceRecord +from thirdeye.platforms.copilot.usage import build_accounting +from thirdeye.usage.types import UsageRow + +FIXTURES = Path(__file__).parent / "fixtures" +RECONCILIATION = FIXTURES / "reconciliation-cases" + +NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" +SOURCE_KEY = "a" * 64 +GENERATION = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" +CHILD_AGENT_ID = "bf8cb9f3-2097-4db0-a3c8-78a2653b2106" +OBSERVED_AT = "2026-09-10T17:09:00.000Z" + +SHUTDOWN_TOTALS = { + "input_tokens": 35396, + "output_tokens": 328, + "reasoning_tokens": 55, + "cache_read_tokens": 23948, + "cache_write_tokens": 11430, + "total_nano_aiu": 373366000, +} + +MAIN_AGENT_TOTALS = { + "input_tokens": 26416, + "output_tokens": 229, + "cache_read_tokens": 19522, + "cache_write_tokens": 6882, + "reasoning_tokens": 39, +} + +CHILD_AGENT_TOTALS = { + "input_tokens": 8980, + "output_tokens": 99, + "cache_read_tokens": 4426, + "cache_write_tokens": 4548, + "reasoning_tokens": 16, +} + + +def _load_json(path: Path) -> Any: + return json.loads(path.read_text(encoding="utf-8")) + + +def _diagnostic_codes(projection: dict[str, Any]) -> set[str]: + return {item["code"] for item in projection["diagnostics"]} + + +def _usage_records(records: list[SourceRecord]) -> list[SourceRecord]: + return [ + record + for record in records + if record.get("source_kind") == "database" + and isinstance(record.get("payload"), dict) + and record["payload"].get("table") == "assistant_usage_events" + ] + + +def _usage_record( + row: dict[str, Any], + *, + content_revision: str, + generation: str = GENERATION, + source_key: str = SOURCE_KEY, + observed_at: str = OBSERVED_AT, +) -> SourceRecord: + primary_key = row["id"] + source_id = ( + f"copilot-db:{source_key}:{NATIVE_SESSION_ID}:assistant_usage_events:" + f"{primary_key}:{content_revision}" + ) + return { + "source_id": source_id, + "source_kind": "database", + "native_session_id": NATIVE_SESSION_ID, + "ts": row.get("created_at"), + "observed_at": observed_at, + "payload": {"table": "assistant_usage_events", "row": row}, + "locator": { + "database": "/example/.copilot/session-store.db", + "table": "assistant_usage_events", + "primary_key": primary_key, + "content_revision": content_revision, + "generation": generation, + }, + } + + +def _shutdown_record(*, source_id: str = "transcript/shutdown-1") -> SourceRecord: + usage = _load_json(FIXTURES / "usage.json") + return { + "source_id": source_id, + "source_kind": "transcript", + "native_session_id": NATIVE_SESSION_ID, + "ts": "2026-09-10T17:08:50.000Z", + "observed_at": OBSERVED_AT, + "payload": {"type": "session.shutdown", "data": usage}, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 0, + "byte_length": 100, + "native_event_id": "shutdown-1", + }, + } + + +def _checkpoint_record(*, source_id: str = "transcript/checkpoint-1") -> SourceRecord: + return { + "source_id": source_id, + "source_kind": "transcript", + "native_session_id": NATIVE_SESSION_ID, + "ts": "2026-09-10T17:08:25.700Z", + "observed_at": OBSERVED_AT, + "payload": { + "type": "session.usage_checkpoint", + "data": { + "inputTokens": 26416, + "outputTokens": 229, + "cacheReadTokens": 19522, + "cacheWriteTokens": 6882, + "reasoningTokens": 39, + "totalNanoAiu": 238814000, + }, + }, + "locator": { + "file": "events.jsonl", + "file_generation": "1-abc", + "byte_offset": 0, + "byte_length": 100, + "native_event_id": "checkpoint-1", + }, + } + + +def _six_call_records() -> list[SourceRecord]: + rows = _load_json(FIXTURES / "assistant-usage-events.json") + revisions = { + call["row_id"]: call["usage_source_id"].rsplit(":", 1)[-1] + for call in _load_json(RECONCILIATION / "observed-six-calls.json")["calls"] + } + return [_usage_record(row, content_revision=revisions[row["id"]]) for row in rows] + + +def _metric_totals(usage_rows: list[UsageRow]) -> dict[str, int]: + totals = { + "input_tokens": 0, + "output_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "reasoning_tokens": 0, + } + for row in usage_rows: + totals["input_tokens"] += row.input_tokens + totals["output_tokens"] += row.output_tokens + totals["cache_read_tokens"] += row.cache_read_input_tokens or 0 + totals["cache_write_tokens"] += row.cache_creation_input_tokens or 0 + totals["reasoning_tokens"] += row.reasoning_output_tokens or 0 + return totals + + +def _nano_aiu_total(candidates: list[dict[str, Any]]) -> int: + return sum( + candidate["supplemental_metrics"]["total_nano_aiu"] + for candidate in candidates + if "total_nano_aiu" in candidate["supplemental_metrics"] + ) + + +# --- contract fixture --- + + +def test_accounting_projection_fixture_matches_build_accounting(): + case = _load_json(RECONCILIATION / "accounting-projection.json") + projection, state = build_accounting(case["input_records"], {}) + expected = case["expected"] + + assert [row.to_dict() for row in projection["usage_rows"]] == expected["usage_rows"] + assert projection["candidates"] == expected["candidates"] + assert projection["diagnostics"] == expected["diagnostics"] + assert set(state["logical_calls"]) == {expected["candidates"][0]["logical_call_id"]} + + +# --- observed six-call corpus --- + + +def test_six_calls_are_accounted_once_from_cli_rows(): + records = _six_call_records() + projection, _state = build_accounting(records, {}) + + assert len(projection["usage_rows"]) == 6 + assert len(projection["candidates"]) == 6 + assert len({row.call_id for row in projection["usage_rows"]}) == 6 + assert len({candidate["logical_call_id"] for candidate in projection["candidates"]}) == 6 + + +def test_six_call_totals_match_observed_shutdown_fixture(): + records = _six_call_records() + projection, _state = build_accounting(records, {}) + totals = _metric_totals(projection["usage_rows"]) + + assert totals["input_tokens"] == SHUTDOWN_TOTALS["input_tokens"] + assert totals["output_tokens"] == SHUTDOWN_TOTALS["output_tokens"] + assert totals["cache_read_tokens"] == SHUTDOWN_TOTALS["cache_read_tokens"] + assert totals["cache_write_tokens"] == SHUTDOWN_TOTALS["cache_write_tokens"] + assert totals["reasoning_tokens"] == SHUTDOWN_TOTALS["reasoning_tokens"] + assert _nano_aiu_total(projection["candidates"]) == SHUTDOWN_TOTALS["total_nano_aiu"] + + +def test_per_agent_totals_match_shutdown_agent_metrics(): + records = _six_call_records() + projection, _state = build_accounting(records, {}) + by_call = {candidate["logical_call_id"]: candidate for candidate in projection["candidates"]} + grouped: dict[str | None, list[UsageRow]] = {} + for usage_row in projection["usage_rows"]: + grouped.setdefault(by_call[usage_row.call_id]["agent_id"], []).append(usage_row) + + assert _metric_totals(grouped[None]) == MAIN_AGENT_TOTALS + assert _metric_totals(grouped[CHILD_AGENT_ID]) == CHILD_AGENT_TOTALS + + +def test_candidates_preserve_agent_parent_and_turn_index_evidence(): + records = _six_call_records() + projection, _state = build_accounting(records, {}) + by_row_id = { + candidate["revision"]["primary_key"]: candidate for candidate in projection["candidates"] + } + + main_turn_zero = [by_row_id[str(row_id)] for row_id in (13, 14)] + assert all(item["turn_index"] == 0 for item in main_turn_zero) + assert all(item["agent_id"] is None for item in main_turn_zero) + + child_calls = [by_row_id[str(row_id)] for row_id in (16, 17)] + assert all(item["agent_id"] == CHILD_AGENT_ID for item in child_calls) + assert all( + item["parent_tool_call_id"] == "call_qx4FH5DADTeT1qVLb37HNpBk" for item in child_calls + ) + + +def test_shutdown_validation_passes_when_totals_match(): + records = _six_call_records() + [_shutdown_record()] + projection, _state = build_accounting(records, {}) + assert "shutdown_total_mismatch" not in _diagnostic_codes(projection) + + +def test_database_reader_records_normalize_to_same_six_calls(tmp_path: Path): + from tests.platforms.copilot.test_database import _collect_all, _write_database + + usage_rows = _load_json(FIXTURES / "assistant-usage-events.json") + home = tmp_path / "copilot" + _write_database( + home, + session_id=NATIVE_SESSION_ID, + cwd="/tmp/probe", + usage_rows=usage_rows, + ) + db_records = _usage_records(_collect_all(resolve_sources(home), NATIVE_SESSION_ID)) + synthetic_records = _six_call_records() + + sample = db_records[0] + source_key = sample["source_id"].split(":")[1] + generation = sample["locator"]["generation"] + aligned = [ + _usage_record( + record["payload"]["row"], + content_revision=record["locator"]["content_revision"], + generation=generation, + source_key=source_key, + ) + for record in synthetic_records + ] + + db_projection, _ = build_accounting(db_records, {}) + synthetic_projection, _ = build_accounting(aligned, {}) + + assert [row.call_id for row in db_projection["usage_rows"]] == [ + row.call_id for row in synthetic_projection["usage_rows"] + ] + assert [candidate["logical_call_id"] for candidate in db_projection["candidates"]] == [ + candidate["logical_call_id"] for candidate in synthetic_projection["candidates"] + ] + assert _metric_totals(db_projection["usage_rows"]) == _metric_totals( + synthetic_projection["usage_rows"] + ) + + +# --- provider handling --- + + +def test_unknown_provider_maps_to_unknown_in_usage_row(): + row = copy.deepcopy(_load_json(FIXTURES / "assistant-usage-events.json")[0]) + record = _usage_record( + row, + content_revision="sha256:68bf2ca8903d9bdfe15a9d61144ba8b9b0e352678680e4490f2259bd2f468f47", + ) + projection, _state = build_accounting([record], {}) + + assert projection["candidates"][0]["provider"] is None + assert projection["usage_rows"][0].provider_name == "unknown" + unknown = [item for item in projection["diagnostics"] if item["code"] == "unknown_provider"] + assert len(unknown) == 1 + assert record["source_id"] in unknown[0]["source_ids"] + + +def test_explicit_provider_is_preserved(): + row = copy.deepcopy(_load_json(FIXTURES / "assistant-usage-events.json")[0]) + row["provider"] = "openai" + record = _usage_record(row, content_revision="sha256:provider-rev") + projection, _state = build_accounting([record], {}) + + assert projection["candidates"][0]["provider"] == "openai" + assert projection["usage_rows"][0].provider_name == "openai" + assert "unknown_provider" not in _diagnostic_codes(projection) + + +def test_provider_name_column_is_accepted(): + row = copy.deepcopy(_load_json(FIXTURES / "assistant-usage-events.json")[0]) + row["provider_name"] = "anthropic" + record = _usage_record(row, content_revision="sha256:provider-name-rev") + projection, _state = build_accounting([record], {}) + + assert projection["candidates"][0]["provider"] == "anthropic" + assert projection["usage_rows"][0].provider_name == "anthropic" + + +# --- absent vs zero --- + + +def test_absent_cache_and_reasoning_tokens_stay_none_in_usage_row(): + row = copy.deepcopy(_load_json(FIXTURES / "assistant-usage-events.json")[0]) + row.pop("cache_read_tokens", None) + row.pop("cache_write_tokens", None) + row.pop("reasoning_tokens", None) + record = _usage_record(row, content_revision="sha256:absent-cache-rev") + projection, _state = build_accounting([record], {}) + + usage_row = projection["usage_rows"][0] + assert usage_row.cache_read_input_tokens is None + assert usage_row.cache_creation_input_tokens is None + assert usage_row.reasoning_output_tokens is None + serialized = usage_row.to_dict() + assert "gen_ai.usage.cache_read.input_tokens" not in serialized + assert "gen_ai.usage.cache_creation.input_tokens" not in serialized + assert "gen_ai.usage.reasoning.output_tokens" not in serialized + + +# --- missing / partial rows --- + + +def test_missing_required_fields_keep_candidate_without_usage_row(): + row = copy.deepcopy(_load_json(FIXTURES / "assistant-usage-events.json")[0]) + row.pop("output_tokens") + record = _usage_record(row, content_revision="sha256:missing-output-rev") + projection, _state = build_accounting([record], {}) + + assert len(projection["candidates"]) == 1 + assert projection["usage_rows"] == [] + missing = [item for item in projection["diagnostics"] if item["code"] == "missing_usage_fields"] + assert missing + assert "output_tokens" in missing[0]["details"]["missing_fields"] + supplemental = projection["candidates"][0]["supplemental_metrics"] + assert "output_tokens" not in supplemental + assert supplemental["input_tokens"] == row["input_tokens"] + + +def test_missing_timestamp_diagnostic_and_no_usage_row(): + row = copy.deepcopy(_load_json(FIXTURES / "assistant-usage-events.json")[0]) + row.pop("created_at") + record = _usage_record(row, content_revision="sha256:missing-ts-rev") + record["ts"] = None + projection, _state = build_accounting([record], {}) + + assert projection["usage_rows"] == [] + missing = projection["diagnostics"][0] + assert missing["code"] == "missing_usage_fields" + assert "timestamp" in missing["details"]["missing_fields"] + + +# --- revisions and conflicts --- + + +def test_later_revision_replaces_earlier_for_same_logical_call(): + row = copy.deepcopy(_load_json(FIXTURES / "assistant-usage-events.json")[0]) + first = _usage_record(row, content_revision="sha256:first-revision") + updated = copy.deepcopy(row) + updated["output_tokens"] = 999 + second = _usage_record(updated, content_revision="sha256:second-revision") + projection, state = build_accounting([first, second], {}) + + assert len(projection["usage_rows"]) == 1 + assert projection["usage_rows"][0].output_tokens == 999 + assert projection["candidates"][0]["usage_source_id"] == second["source_id"] + assert projection["candidates"][0]["source_ids"] == [first["source_id"], second["source_id"]] + assert len(state["logical_calls"]) == 1 + stored = state["logical_calls"][projection["candidates"][0]["logical_call_id"]] + assert stored["metrics_digest"].startswith("sha256:") + assert stored["metrics_digest"] != "" + + +def test_later_revision_in_new_partition_keeps_prior_source_ids(): + row = copy.deepcopy(_load_json(FIXTURES / "assistant-usage-events.json")[0]) + first = _usage_record(row, content_revision="sha256:first-revision") + updated = copy.deepcopy(row) + updated["output_tokens"] = 999 + second = _usage_record(updated, content_revision="sha256:second-revision") + _, state = build_accounting([first], {}) + projection, next_state = build_accounting([second], state) + + assert projection["candidates"][0]["source_ids"] == [first["source_id"], second["source_id"]] + stored = next_state["logical_calls"][projection["candidates"][0]["logical_call_id"]] + assert stored["source_ids"] == [first["source_id"], second["source_id"]] + assert projection["usage_rows"][0].output_tokens == 999 + + +def test_prior_usage_source_id_seeds_when_source_ids_absent(): + row = copy.deepcopy(_load_json(FIXTURES / "assistant-usage-events.json")[0]) + first = _usage_record(row, content_revision="sha256:first-revision") + _, state = build_accounting([first], {}) + logical_id = next(iter(state["logical_calls"])) + del state["logical_calls"][logical_id]["source_ids"] + updated = copy.deepcopy(row) + updated["output_tokens"] = 999 + second = _usage_record(updated, content_revision="sha256:second-revision") + projection, _next_state = build_accounting([second], state) + + assert projection["candidates"][0]["source_ids"] == [first["source_id"], second["source_id"]] + + +def test_reused_row_id_across_generations_emits_warning(): + row_a = copy.deepcopy(_load_json(FIXTURES / "assistant-usage-events.json")[0]) + row_b = copy.deepcopy(row_a) + row_b["output_tokens"] = 50 + first = _usage_record( + row_a, + content_revision="sha256:gen-a-rev", + generation="sha256:generation-a", + ) + second = _usage_record( + row_b, + content_revision="sha256:gen-b-rev", + generation="sha256:generation-b", + ) + projection, state = build_accounting([first, second], {}) + + assert "usage_row_id_reuse" in _diagnostic_codes(projection) + reuse = next(item for item in projection["diagnostics"] if item["code"] == "usage_row_id_reuse") + assert reuse["severity"] == "error" + assert first["source_id"] in reuse["source_ids"] + assert second["source_id"] in reuse["source_ids"] + assert projection["usage_rows"] == [] + assert len(projection["candidates"]) == 2 + assert {candidate["usage_source_id"] for candidate in projection["candidates"]} == { + first["source_id"], + second["source_id"], + } + assert len(state["logical_calls"]) == 2 + assert all(call["quarantined"] is True for call in state["logical_calls"].values()) + + +def test_incompatible_metrics_quarantine_logical_call(): + row = copy.deepcopy(_load_json(FIXTURES / "assistant-usage-events.json")[0]) + row["cache_read_tokens"] = row["input_tokens"] + 1 + record = _usage_record(row, content_revision="sha256:incompatible-rev") + projection, state = build_accounting([record], {}) + + assert projection["usage_rows"] == [] + assert len(projection["candidates"]) == 1 + assert projection["candidates"][0]["usage_source_id"] == record["source_id"] + assert "usage_revision_conflict" in _diagnostic_codes(projection) + conflict = next( + item for item in projection["diagnostics"] if item["code"] == "usage_revision_conflict" + ) + assert record["source_id"] in conflict["source_ids"] + assert state["logical_calls"] + + +def test_cache_read_plus_write_exceeding_input_quarantines(): + row = copy.deepcopy(_load_json(FIXTURES / "assistant-usage-events.json")[0]) + row["input_tokens"] = 100 + row["cache_read_tokens"] = 80 + row["cache_write_tokens"] = 80 + record = _usage_record(row, content_revision="sha256:cache-sum-rev") + projection, state = build_accounting([record], {}) + + assert projection["usage_rows"] == [] + assert len(projection["candidates"]) == 1 + assert "usage_revision_conflict" in _diagnostic_codes(projection) + logical_id = projection["candidates"][0]["logical_call_id"] + assert state["logical_calls"][logical_id]["quarantined"] is True + + +# --- checkpoint / shutdown --- + + +def test_checkpoint_snapshot_is_not_additive(): + records = _six_call_records() + [_checkpoint_record()] + projection, _state = build_accounting(records, {}) + + assert len(projection["usage_rows"]) == 6 + assert "checkpoint_not_additive" in _diagnostic_codes(projection) + + +def test_shutdown_mismatch_emits_diagnostic(): + usage = _load_json(FIXTURES / "usage.json") + usage["modelMetrics"]["gpt-5.6-luna"]["usage"]["inputTokens"] = 1 + shutdown = _shutdown_record() + shutdown["payload"]["data"] = usage + projection, _state = build_accounting(_six_call_records() + [shutdown], {}) + + assert "shutdown_total_mismatch" in _diagnostic_codes(projection) + mismatch = next( + item for item in projection["diagnostics"] if item["code"] == "shutdown_total_mismatch" + ) + assert mismatch["details"]["expected_input_tokens"] == 1 + assert mismatch["details"]["accounted_input_tokens"] == SHUTDOWN_TOTALS["input_tokens"] + + +def test_shutdown_float_nano_aiu_is_used_for_validation(): + usage = _load_json(FIXTURES / "usage.json") + usage["modelMetrics"]["gpt-5.6-luna"]["totalNanoAiu"] = 1.0 + shutdown = _shutdown_record() + shutdown["payload"]["data"] = usage + projection, _state = build_accounting(_six_call_records() + [shutdown], {}) + + mismatch = next( + item for item in projection["diagnostics"] if item["code"] == "shutdown_total_mismatch" + ) + assert mismatch["details"]["expected_total_nano_aiu"] == 1 + assert "capability_gap" not in _diagnostic_codes(projection) + + +def test_shutdown_agent_metrics_mismatch_is_reported(): + usage = _load_json(FIXTURES / "usage.json") + usage["agentMetrics"]["main"]["modelMetrics"]["gpt-5.6-luna"]["usage"]["inputTokens"] = 1 + shutdown = _shutdown_record() + shutdown["payload"]["data"] = usage + projection, _state = build_accounting(_six_call_records() + [shutdown], {}) + + mismatches = [ + item for item in projection["diagnostics"] if item["code"] == "shutdown_total_mismatch" + ] + assert any( + item["details"].get("agent_id") == "main" + and item["details"].get("expected_input_tokens") == 1 + for item in mismatches + ) + + +def test_unusable_shutdown_emits_capability_gap_instead_of_silent_skip(): + shutdown = _shutdown_record() + shutdown["payload"]["data"] = { + "modelMetrics": {"gpt-5.6-luna": {"usage": {"inputTokens": "not-a-number"}}} + } + projection, _state = build_accounting(_six_call_records() + [shutdown], {}) + + gaps = [item for item in projection["diagnostics"] if item["code"] == "capability_gap"] + assert gaps + assert shutdown["source_id"] in gaps[0]["source_ids"] + assert "shutdown_total_mismatch" not in _diagnostic_codes(projection) + + +def test_shutdown_data_not_a_dict_emits_capability_gap(): + shutdown = _shutdown_record() + shutdown["payload"]["data"] = "not-a-dict" + projection, _state = build_accounting(_six_call_records() + [shutdown], {}) + + gaps = [item for item in projection["diagnostics"] if item["code"] == "capability_gap"] + assert any(item["details"].get("reason") == "missing_shutdown_data" for item in gaps) + assert shutdown["source_id"] in gaps[0]["source_ids"] + assert "shutdown_total_mismatch" not in _diagnostic_codes(projection) + + +def test_unparseable_agent_metrics_emits_capability_gap(): + usage = _load_json(FIXTURES / "usage.json") + usage["agentMetrics"] = {"main": "not-a-dict"} + shutdown = _shutdown_record() + shutdown["payload"]["data"] = usage + projection, _state = build_accounting(_six_call_records() + [shutdown], {}) + + gaps = [item for item in projection["diagnostics"] if item["code"] == "capability_gap"] + assert any( + item["details"].get("reason") == "unparseable_agent_metrics" + and shutdown["source_id"] in item["source_ids"] + for item in gaps + ) + + +def test_mixed_archive_does_not_charge_sessions_turns_or_title_calls(): + title = _load_json(RECONCILIATION / "cases.json")["auxiliary_title_generation"][ + "input_records" + ][0] + sessions_record: SourceRecord = { + "source_id": ( + f"copilot-db:{SOURCE_KEY}:{NATIVE_SESSION_ID}:sessions:1:sha256:sessions-rev" + ), + "source_kind": "database", + "native_session_id": NATIVE_SESSION_ID, + "ts": None, + "observed_at": OBSERVED_AT, + "payload": { + "schema_version": 1, + "table": "sessions", + "row": {"id": NATIVE_SESSION_ID}, + }, + "locator": { + "database": "/example/.copilot/session-store.db", + "table": "sessions", + "primary_key": NATIVE_SESSION_ID, + "content_revision": "sha256:sessions-rev", + "generation": GENERATION, + }, + } + records = _six_call_records() + [sessions_record, title] + projection, _state = build_accounting(records, {}) + totals = _metric_totals(projection["usage_rows"]) + + assert len(projection["usage_rows"]) == 6 + assert len(projection["candidates"]) == 6 + assert totals["input_tokens"] == SHUTDOWN_TOTALS["input_tokens"] + assert totals["output_tokens"] == SHUTDOWN_TOTALS["output_tokens"] + assert totals["cache_read_tokens"] == SHUTDOWN_TOTALS["cache_read_tokens"] + assert totals["cache_write_tokens"] == SHUTDOWN_TOTALS["cache_write_tokens"] + assert totals["reasoning_tokens"] == SHUTDOWN_TOTALS["reasoning_tokens"] + assert _nano_aiu_total(projection["candidates"]) == SHUTDOWN_TOTALS["total_nano_aiu"] + + +# --- supplemental metrics --- + + +def test_supplemental_metrics_preserve_nano_aiu_separately(): + records = _six_call_records() + projection, _state = build_accounting(records, {}) + candidate = projection["candidates"][0] + + assert "total_nano_aiu" in candidate["supplemental_metrics"] + assert candidate["supplemental_metrics"]["total_nano_aiu"] == 174125000 + usage_row = projection["usage_rows"][0].to_dict() + assert "total_nano_aiu" not in usage_row + + +# --- incremental replay --- + + +def test_incremental_replay_matches_full_archive(): + records = _six_call_records() + full_projection, full_state = build_accounting(records, {}) + + state: dict[str, Any] = {} + incremental_projection = None + for index in range(1, len(records) + 1): + incremental_projection, state = build_accounting(records[:index], state) + + assert incremental_projection is not None + assert [row.to_dict() for row in incremental_projection["usage_rows"]] == [ + row.to_dict() for row in full_projection["usage_rows"] + ] + assert incremental_projection["candidates"] == full_projection["candidates"] + assert state["logical_calls"] == full_state["logical_calls"] + + +def test_late_arrival_adds_new_call_without_rerunning_agent(): + first_batch = _six_call_records()[:3] + late_row = copy.deepcopy(_load_json(FIXTURES / "assistant-usage-events.json")[3]) + late_record = _usage_record( + late_row, + content_revision="sha256:late-arrival-rev", + ) + + first_projection, state = build_accounting(first_batch, {}) + second_projection, _state = build_accounting(first_batch + [late_record], state) + + assert len(first_projection["usage_rows"]) == 3 + assert len(second_projection["usage_rows"]) == 4 + assert second_projection["candidates"][-1]["agent_id"] == CHILD_AGENT_ID + + +# --- prior state --- + + +def test_prior_state_logical_calls_are_preserved_for_unseen_ids(): + records = _six_call_records()[:1] + _, state = build_accounting(records, {}) + inherited = copy.deepcopy(state) + + projection, next_state = build_accounting([], inherited) + assert projection["usage_rows"] == [] + assert projection["candidates"] == [] + assert next_state["logical_calls"] == inherited["logical_calls"] + + +def test_nested_accounting_state_preserves_unseen_logical_calls(): + records = _six_call_records()[:1] + _, state = build_accounting(records, {}) + nested = {"accounting_state": copy.deepcopy(state)} + + projection, next_state = build_accounting([], nested) + assert projection["usage_rows"] == [] + assert next_state["logical_calls"] == state["logical_calls"] + + +def test_incomplete_locator_emits_capability_gap_and_keeps_the_source_id(): + row = copy.deepcopy(_load_json(FIXTURES / "assistant-usage-events.json")[0]) + record = _usage_record(row, content_revision="sha256:missing-generation-rev") + del record["locator"]["generation"] + projection, _state = build_accounting([record], {}) + + assert projection["usage_rows"] == [] + assert projection["candidates"] == [] + gap = next(item for item in projection["diagnostics"] if item["code"] == "capability_gap") + assert record["source_id"] in gap["source_ids"] + + +def test_invalid_source_identity_emits_capability_gap(): + row = copy.deepcopy(_load_json(FIXTURES / "assistant-usage-events.json")[0]) + record = _usage_record(row, content_revision="sha256:bad-source-rev") + record["source_id"] = "not-a-copilot-db-identity" + projection, _state = build_accounting([record], {}) + + assert projection["usage_rows"] == [] + assert projection["candidates"] == [] + gap = next(item for item in projection["diagnostics"] if item["code"] == "capability_gap") + assert record["source_id"] in gap["source_ids"] + + +def test_truncated_source_key_emits_capability_gap(): + row = copy.deepcopy(_load_json(FIXTURES / "assistant-usage-events.json")[0]) + record = _usage_record(row, content_revision="sha256:short-key-rev", source_key="abc") + projection, _state = build_accounting([record], {}) + + assert projection["usage_rows"] == [] + gap = next(item for item in projection["diagnostics"] if item["code"] == "capability_gap") + assert record["source_id"] in gap["source_ids"] + assert gap["details"]["reason"] == "source_id is not a 64-character copilot-db identity" + + +def test_later_incompatible_revision_conflicts_using_prior_metrics_digest(): + row = copy.deepcopy(_load_json(FIXTURES / "assistant-usage-events.json")[0]) + first = _usage_record(row, content_revision="sha256:first-digest-rev") + _, state = build_accounting([first], {}) + logical_id = next(iter(state["logical_calls"])) + prior_digest = state["logical_calls"][logical_id]["metrics_digest"] + + bad = copy.deepcopy(row) + bad["cache_read_tokens"] = bad["input_tokens"] + 1 + second = _usage_record(bad, content_revision="sha256:second-digest-rev") + projection, _next_state = build_accounting([second], state) + + conflict = next( + item for item in projection["diagnostics"] if item["code"] == "usage_revision_conflict" + ) + assert conflict["details"]["prior_metrics_digest"] == prior_digest + assert conflict["details"]["metrics_digest"] != prior_digest + assert projection["usage_rows"] == [] + assert len(projection["candidates"]) == 1 + assert projection["candidates"][0]["source_ids"] == [first["source_id"], second["source_id"]] + assert first["source_id"] in conflict["source_ids"] + assert second["source_id"] in conflict["source_ids"] + + +def test_disjoint_partition_does_not_false_mismatch_shutdown(): + records = _six_call_records() + _first, state = build_accounting(records[:3], {}) + second, _next_state = build_accounting(records[3:] + [_shutdown_record()], state) + + assert "shutdown_total_mismatch" not in _diagnostic_codes(second) + assert len(second["usage_rows"]) == 3 + + +def test_row_id_reuse_is_detected_across_partitions(): + row_a = copy.deepcopy(_load_json(FIXTURES / "assistant-usage-events.json")[0]) + row_b = copy.deepcopy(row_a) + row_b["output_tokens"] = 50 + first = _usage_record( + row_a, + content_revision="sha256:gen-a-rev", + generation="sha256:generation-a", + ) + second = _usage_record( + row_b, + content_revision="sha256:gen-b-rev", + generation="sha256:generation-b", + ) + _first_projection, state = build_accounting([first], {}) + projection, next_state = build_accounting([second], state) + + assert len(_first_projection["usage_rows"]) == 1 + assert "usage_row_id_reuse" in _diagnostic_codes(projection) + reuse = next(item for item in projection["diagnostics"] if item["code"] == "usage_row_id_reuse") + assert reuse["severity"] == "error" + assert first["source_id"] in reuse["source_ids"] + assert second["source_id"] in reuse["source_ids"] + assert projection["usage_rows"] == [] + assert all(call["quarantined"] is True for call in next_state["logical_calls"].values()) + + +def test_integer_valued_float_row_metrics_do_not_false_mismatch_shutdown(): + records = _six_call_records() + for record in records: + row = record["payload"]["row"] + assert isinstance(row, dict) + for field in ( + "input_tokens", + "output_tokens", + "cache_read_tokens", + "cache_write_tokens", + "reasoning_tokens", + "total_nano_aiu", + ): + if isinstance(row.get(field), int): + row[field] = float(row[field]) + projection, state = build_accounting(records + [_shutdown_record()], {}) + totals = _metric_totals(projection["usage_rows"]) + + assert len(projection["usage_rows"]) == 6 + assert totals["input_tokens"] == SHUTDOWN_TOTALS["input_tokens"] + assert totals["output_tokens"] == SHUTDOWN_TOTALS["output_tokens"] + assert totals["cache_read_tokens"] == SHUTDOWN_TOTALS["cache_read_tokens"] + assert totals["cache_write_tokens"] == SHUTDOWN_TOTALS["cache_write_tokens"] + assert totals["reasoning_tokens"] == SHUTDOWN_TOTALS["reasoning_tokens"] + assert state["accounted_metrics"]["total_nano_aiu"] == SHUTDOWN_TOTALS["total_nano_aiu"] + assert "shutdown_total_mismatch" not in _diagnostic_codes(projection) diff --git a/tests/platforms/copilot/test_watch.py b/tests/platforms/copilot/test_watch.py new file mode 100644 index 00000000..b6810522 --- /dev/null +++ b/tests/platforms/copilot/test_watch.py @@ -0,0 +1,619 @@ +"""Behavioral tests for Copilot foreground watch polling.""" + +from __future__ import annotations + +import shutil +import sqlite3 +from pathlib import Path +from typing import Any + +import pytest + +import thirdeye.platforms.copilot.watch as watch_mod +from thirdeye.config import Config +from thirdeye.platforms.copilot.hook_payload import parse_hook +from thirdeye.platforms.copilot.identity import resolve_sources +from thirdeye.platforms.copilot.spool import enqueue_hook +from thirdeye.platforms.copilot.types import SourcePaths, SyncResult +from thirdeye.platforms.copilot.watch import _changed_sessions, watch + +FIXTURES = Path(__file__).parent / "fixtures" +CLI_FIXTURE = FIXTURES +NATIVE_SESSION_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" +OTHER_SESSION_ID = "a1b2c3d4-e5f6-7890-abcd-ef1234567890" +OBSERVED_AT = "2026-09-10T17:08:25.626Z" + + +@pytest.fixture +def copilot_env(tmp_path: Path) -> tuple[Config, SourcePaths]: + home = tmp_path / "copilot-home" + home.mkdir() + config = Config(root=tmp_path / "thirdeye") + paths = resolve_sources(home) + return config, paths + + +def _empty_result(**overrides: int) -> SyncResult: + result: SyncResult = { + "sessions": 0, + "records_written": 0, + "duplicate_records": 0, + "pending": 0, + "errors": 0, + } + result.update(overrides) # type: ignore[typeddict-item] + return result + + +def _stub_reconcile(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(watch_mod, "reconcile_archived_sessions", lambda *_args, **_kwargs: {}) + monkeypatch.setattr( + watch_mod, + "reconcile_session", + lambda *_args, **_kwargs: { + "events": 0, + "usage": 0, + "turns": 0, + "exports": 0, + "pending": 0, + "ambiguous": 0, + "conflicting": 0, + "errors": 0, + }, + ) + + +def _write_transcript(home: Path, native_id: str, *, events_path: Path | None = None) -> Path: + session_dir = home / "session-state" / native_id + session_dir.mkdir(parents=True, exist_ok=True) + source = events_path or (CLI_FIXTURE / "events.jsonl") + destination = session_dir / "events.jsonl" + shutil.copy(source, destination) + (session_dir / "workspace.yaml").write_text("cwd: /sanitized/workspace\n", encoding="utf-8") + return destination + + +def _write_database(home: Path, *, session_id: str = NATIVE_SESSION_ID) -> Path: + home.mkdir(parents=True, exist_ok=True) + database = home / "session-store.db" + connection = sqlite3.connect(database) + try: + connection.execute("PRAGMA journal_mode=WAL") + connection.executescript( + """ + CREATE TABLE sessions (id TEXT PRIMARY KEY, cwd TEXT, created_at TEXT); + CREATE TABLE turns ( + id INTEGER PRIMARY KEY, + session_id TEXT NOT NULL, + turn_index INTEGER, + content TEXT, + updated_at TEXT + ); + """ + ) + connection.execute( + "INSERT INTO sessions (id, cwd, created_at) VALUES (?, ?, ?)", + (session_id, "/tmp/probe", "2026-09-10T17:08:00.000Z"), + ) + connection.execute( + "INSERT INTO turns (id, session_id, turn_index, content, updated_at) " + "VALUES (?, ?, ?, ?, ?)", + (1, session_id, 0, "turn-0", "2026-09-10T17:08:10.000Z"), + ) + connection.commit() + finally: + connection.close() + return database + + +def _hook_record(*, observation_id: str, session_id: str = NATIVE_SESSION_ID) -> dict[str, Any]: + return parse_hook( + "agentStop", + { + "sessionId": session_id, + "timestamp": 1789060105626, + "cwd": "/fixture/workspace", + "stopReason": "end_turn", + }, + {"env": {"WB_PLAN": "p"}}, + observed_at=OBSERVED_AT, + observation_id=observation_id, + ) + + +def _snapshot( + *, + sessions: set[str] | None = None, + database_sessions: set[str] | None = None, + transcripts: dict[str, object] | None = None, + database: tuple[object, object, object] | None = None, + spool: dict[str, tuple[tuple[str, object], ...]] | None = None, +) -> dict[str, Any]: + session_set = sessions or set() + return { + "sessions": session_set, + "database_sessions": database_sessions or set(), + "transcripts": transcripts or {}, + "database": database or (None, None, None), + "spool": spool or {}, + } + + +def test_watch_module_has_no_command_or_hook_runtime_imports() -> None: + source = Path(watch_mod.__file__).read_text(encoding="utf-8") + assert "thirdeye.commands" not in source + assert "hook_lifecycle" not in source + assert "followup" not in source + + +@pytest.mark.parametrize( + "interval", + [0.09, -1.0, float("inf"), float("nan"), True, "1"], +) +def test_watch_rejects_invalid_interval(interval: object) -> None: + config = Config(root=Path("/tmp/thirdeye")) + paths = resolve_sources(Path("/tmp/copilot")) + with pytest.raises(ValueError, match="interval must be a finite number"): + watch(config, paths, interval=interval) # type: ignore[arg-type] + + +def test_changed_sessions_detects_new_session() -> None: + before = _snapshot(sessions=set()) + after = _snapshot(sessions={NATIVE_SESSION_ID}) + assert _changed_sessions(before, after) == {NATIVE_SESSION_ID} + + +def test_changed_sessions_detects_transcript_stamp_change() -> None: + stamp_a = (1, 2, 100, 200) + stamp_b = (1, 2, 150, 200) + before = _snapshot( + sessions={NATIVE_SESSION_ID}, + transcripts={NATIVE_SESSION_ID: stamp_a}, + ) + after = _snapshot( + sessions={NATIVE_SESSION_ID}, + transcripts={NATIVE_SESSION_ID: stamp_b}, + ) + assert _changed_sessions(before, after) == {NATIVE_SESSION_ID} + + +def test_changed_sessions_detects_spool_stamp_change() -> None: + before = _snapshot( + sessions={NATIVE_SESSION_ID}, + spool={NATIVE_SESSION_ID: (("a.json", (1, 2, 3, 4)),)}, + ) + after = _snapshot( + sessions={NATIVE_SESSION_ID}, + spool={NATIVE_SESSION_ID: (("a.json", (1, 2, 5, 4)),)}, + ) + assert _changed_sessions(before, after) == {NATIVE_SESSION_ID} + + +def test_changed_sessions_database_change_includes_prior_database_sessions() -> None: + before = _snapshot( + sessions={NATIVE_SESSION_ID, OTHER_SESSION_ID}, + database_sessions={NATIVE_SESSION_ID}, + database=((1, 2, 3, 4), None, None), + ) + after = _snapshot( + sessions={NATIVE_SESSION_ID, OTHER_SESSION_ID}, + database_sessions={OTHER_SESSION_ID}, + database=((1, 2, 3, 4), (5, 6, 7, 8), None), + ) + changed = _changed_sessions(before, after) + assert NATIVE_SESSION_ID in changed + assert OTHER_SESSION_ID in changed + + +def test_watch_performs_initial_full_sync( + monkeypatch: pytest.MonkeyPatch, copilot_env: tuple[Config, SourcePaths] +) -> None: + config, paths = copilot_env + calls: list[str | None] = [] + + def tracking_sync(cfg: Config, p: SourcePaths, *, session_id: str | None = None) -> SyncResult: + calls.append(session_id) + return _empty_result() + + def stop_immediately(_interval: float) -> None: + raise KeyboardInterrupt + + monkeypatch.setattr(watch_mod, "sync", tracking_sync) + _stub_reconcile(monkeypatch) + monkeypatch.setattr(watch_mod, "_SLEEP", stop_immediately) + + watch(config, paths, interval=0.1) + + assert calls == [None] + + +def test_watch_skips_per_session_sync_when_sources_are_quiet( + monkeypatch: pytest.MonkeyPatch, + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + home = Path(paths["home"]) + _write_transcript(home, NATIVE_SESSION_ID) + calls: list[str | None] = [] + cycle = {"count": 0} + + def tracking_sync(cfg: Config, p: SourcePaths, *, session_id: str | None = None) -> SyncResult: + calls.append(session_id) + return _empty_result() + + def sleep_then_interrupt(_interval: float) -> None: + cycle["count"] += 1 + if cycle["count"] >= 2: + raise KeyboardInterrupt + + monkeypatch.setattr(watch_mod, "sync", tracking_sync) + _stub_reconcile(monkeypatch) + monkeypatch.setattr(watch_mod, "_SLEEP", sleep_then_interrupt) + + watch(config, paths, interval=0.1) + + assert calls == [None] + + +def test_watch_syncs_only_changed_transcript_session( + monkeypatch: pytest.MonkeyPatch, + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + home = Path(paths["home"]) + _write_transcript(home, NATIVE_SESSION_ID) + _write_transcript(home, OTHER_SESSION_ID) + events_path = home / "session-state" / NATIVE_SESSION_ID / "events.jsonl" + calls: list[str | None] = [] + reconciled: list[str] = [] + cycle = {"count": 0} + + def tracking_sync(cfg: Config, p: SourcePaths, *, session_id: str | None = None) -> SyncResult: + calls.append(session_id) + return _empty_result() + + def track_reconcile( + _config: Config, + _paths: SourcePaths, + native_session_id: str, + *, + export: bool = False, + include_history: bool = False, + ) -> dict[str, int]: + reconciled.append(native_session_id) + return { + "events": 0, + "usage": 0, + "turns": 0, + "exports": 0, + "pending": 0, + "ambiguous": 0, + "conflicting": 0, + "errors": 0, + } + + def append_during_poll(_interval: float) -> None: + cycle["count"] += 1 + if cycle["count"] == 1: + with events_path.open("a", encoding="utf-8") as stream: + stream.write('{"type":"synthetic.append"}\n') + else: + raise KeyboardInterrupt + + monkeypatch.setattr(watch_mod, "sync", tracking_sync) + monkeypatch.setattr(watch_mod, "reconcile_archived_sessions", lambda *_args, **_kwargs: {}) + monkeypatch.setattr(watch_mod, "reconcile_session", track_reconcile) + monkeypatch.setattr(watch_mod, "_SLEEP", append_during_poll) + + watch(config, paths, interval=0.1) + + assert calls[0] is None + assert calls.count(NATIVE_SESSION_ID) == 1 + assert OTHER_SESSION_ID not in calls + assert reconciled == [NATIVE_SESSION_ID] + + +def test_watch_detects_database_wal_change( + monkeypatch: pytest.MonkeyPatch, + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + home = Path(paths["home"]) + database = _write_database(home, session_id=NATIVE_SESSION_ID) + calls: list[str | None] = [] + cycle = {"count": 0} + + def tracking_sync(cfg: Config, p: SourcePaths, *, session_id: str | None = None) -> SyncResult: + calls.append(session_id) + return _empty_result() + + def mutate_database(_interval: float) -> None: + cycle["count"] += 1 + if cycle["count"] == 1: + connection = sqlite3.connect(database) + try: + connection.execute( + "INSERT INTO turns (id, session_id, turn_index, content, updated_at) " + "VALUES (?, ?, ?, ?, ?)", + (2, NATIVE_SESSION_ID, 1, "late-row", "2026-09-10T17:08:12.000Z"), + ) + connection.commit() + finally: + connection.close() + else: + raise KeyboardInterrupt + + monkeypatch.setattr(watch_mod, "sync", tracking_sync) + _stub_reconcile(monkeypatch) + monkeypatch.setattr(watch_mod, "_SLEEP", mutate_database) + + watch(config, paths, interval=0.1) + + assert calls[0] is None + assert calls.count(NATIVE_SESSION_ID) >= 1 + + +def test_watch_detects_spool_change( + monkeypatch: pytest.MonkeyPatch, + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + calls: list[str | None] = [] + cycle = {"count": 0} + + def tracking_sync(cfg: Config, p: SourcePaths, *, session_id: str | None = None) -> SyncResult: + calls.append(session_id) + return _empty_result() + + def enqueue_during_poll(_interval: float) -> None: + cycle["count"] += 1 + if cycle["count"] == 1: + enqueue_hook(config, paths, _hook_record(observation_id="obs-watch-spool")) + else: + raise KeyboardInterrupt + + monkeypatch.setattr(watch_mod, "sync", tracking_sync) + _stub_reconcile(monkeypatch) + monkeypatch.setattr(watch_mod, "_SLEEP", enqueue_during_poll) + + watch(config, paths, interval=0.1) + + assert calls[0] is None + assert calls.count(NATIVE_SESSION_ID) == 1 + + +def test_watch_retries_sessions_with_pending_or_errors( + monkeypatch: pytest.MonkeyPatch, + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + home = Path(paths["home"]) + _write_transcript(home, NATIVE_SESSION_ID) + attempts: dict[str, int] = {} + cycle = {"count": 0} + + def flaky_sync(cfg: Config, p: SourcePaths, *, session_id: str | None = None) -> SyncResult: + if session_id is None: + return _empty_result(pending=1) + attempts[session_id] = attempts.get(session_id, 0) + 1 + if attempts[session_id] == 1: + return _empty_result(pending=1) + return _empty_result() + + def three_poll_cycles(_interval: float) -> None: + cycle["count"] += 1 + if cycle["count"] >= 3: + raise KeyboardInterrupt + + monkeypatch.setattr(watch_mod, "sync", flaky_sync) + _stub_reconcile(monkeypatch) + monkeypatch.setattr(watch_mod, "_SLEEP", three_poll_cycles) + + watch(config, paths, interval=0.1) + + assert attempts.get(NATIVE_SESSION_ID, 0) >= 2 + + +def test_watch_exits_cleanly_on_keyboard_interrupt( + monkeypatch: pytest.MonkeyPatch, + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + + monkeypatch.setattr(watch_mod, "sync", lambda *args, **kwargs: _empty_result()) + _stub_reconcile(monkeypatch) + monkeypatch.setattr( + watch_mod, "_SLEEP", lambda _interval: (_ for _ in ()).throw(KeyboardInterrupt) + ) + + watch(config, paths, interval=0.1) + + +def test_watch_integration_captures_transcript_append( + monkeypatch: pytest.MonkeyPatch, + copilot_env: tuple[Config, SourcePaths], +) -> None: + from thirdeye.platforms.copilot.capture import iter_captured_records + from thirdeye.platforms.copilot.identity import stored_session_id + + config, paths = copilot_env + home = Path(paths["home"]) + events_path = _write_transcript(home, NATIVE_SESSION_ID) + stored = stored_session_id(paths, NATIVE_SESSION_ID) + cycle = {"count": 0} + + def append_then_stop(_interval: float) -> None: + cycle["count"] += 1 + if cycle["count"] == 1: + with events_path.open("a", encoding="utf-8") as stream: + stream.write('{"type":"synthetic.append"}\n') + else: + raise KeyboardInterrupt + + monkeypatch.setattr(watch_mod, "_SLEEP", append_then_stop) + + before = len(list(iter_captured_records(config, stored))) + watch(config, paths, interval=0.1) + after = len(list(iter_captured_records(config, stored))) + + assert after > before + + +def _unlink_database(database: Path) -> None: + database.unlink() + for suffix in ("-wal", "-shm"): + database.with_name(database.name + suffix).unlink(missing_ok=True) + + +def test_changed_sessions_database_deletion_includes_prior_ids() -> None: + before = _snapshot( + sessions={NATIVE_SESSION_ID}, + database_sessions={NATIVE_SESSION_ID}, + database=((1, 2, 3, 4), None, None), + ) + after = _snapshot(sessions=set(), database_sessions=set(), database=(None, None, None)) + assert _changed_sessions(before, after) == {NATIVE_SESSION_ID} + + +def test_watch_does_not_retry_deleted_database_session( + monkeypatch: pytest.MonkeyPatch, + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + home = Path(paths["home"]) + database = _write_database(home, session_id=NATIVE_SESSION_ID) + calls: list[str | None] = [] + cycle = {"count": 0} + + def tracking_sync(cfg: Config, p: SourcePaths, *, session_id: str | None = None) -> SyncResult: + calls.append(session_id) + if session_id is None: + return _empty_result() + if not Path(paths["database"]).is_file(): + return _empty_result(errors=1) + return _empty_result() + + def delete_then_poll(_interval: float) -> None: + cycle["count"] += 1 + if cycle["count"] == 1: + _unlink_database(database) + elif cycle["count"] >= 4: + raise KeyboardInterrupt + + monkeypatch.setattr(watch_mod, "sync", tracking_sync) + _stub_reconcile(monkeypatch) + monkeypatch.setattr(watch_mod, "_SLEEP", delete_then_poll) + + watch(config, paths, interval=0.1) + + assert calls[0] is None + assert calls.count(NATIVE_SESSION_ID) == 1 + + +def test_watch_syncs_recreated_database_after_deletion( + monkeypatch: pytest.MonkeyPatch, + copilot_env: tuple[Config, SourcePaths], +) -> None: + config, paths = copilot_env + home = Path(paths["home"]) + database = _write_database(home, session_id=NATIVE_SESSION_ID) + calls: list[str | None] = [] + cycle = {"count": 0} + + def tracking_sync(cfg: Config, p: SourcePaths, *, session_id: str | None = None) -> SyncResult: + calls.append(session_id) + return _empty_result() + + def delete_then_recreate(_interval: float) -> None: + cycle["count"] += 1 + if cycle["count"] == 1: + _unlink_database(database) + elif cycle["count"] == 2: + _write_database(home, session_id=NATIVE_SESSION_ID) + else: + raise KeyboardInterrupt + + monkeypatch.setattr(watch_mod, "sync", tracking_sync) + _stub_reconcile(monkeypatch) + monkeypatch.setattr(watch_mod, "_SLEEP", delete_then_recreate) + + watch(config, paths, interval=0.1) + + assert calls[0] is None + assert calls.count(NATIVE_SESSION_ID) >= 2 + + +def test_watch_restart_then_captures_later_append( + monkeypatch: pytest.MonkeyPatch, + copilot_env: tuple[Config, SourcePaths], +) -> None: + from thirdeye.platforms.copilot.capture import iter_captured_records + from thirdeye.platforms.copilot.identity import stored_session_id + + config, paths = copilot_env + home = Path(paths["home"]) + events_path = _write_transcript(home, NATIVE_SESSION_ID) + stored = stored_session_id(paths, NATIVE_SESSION_ID) + + monkeypatch.setattr( + watch_mod, "_SLEEP", lambda _interval: (_ for _ in ()).throw(KeyboardInterrupt) + ) + watch(config, paths, interval=0.1) + first = len(list(iter_captured_records(config, stored))) + assert first > 0 + + cycle = {"count": 0} + + def append_then_stop(_interval: float) -> None: + cycle["count"] += 1 + if cycle["count"] == 1: + with events_path.open("a", encoding="utf-8") as stream: + stream.write('{"type":"synthetic.restart-append"}\n') + else: + raise KeyboardInterrupt + + monkeypatch.setattr(watch_mod, "_SLEEP", append_then_stop) + watch(config, paths, interval=0.1) + second = len(list(iter_captured_records(config, stored))) + assert second > first + + +def test_watch_integration_captures_late_database_rows_then_survives_deletion( + monkeypatch: pytest.MonkeyPatch, + copilot_env: tuple[Config, SourcePaths], +) -> None: + from thirdeye.platforms.copilot.capture import iter_captured_records + from thirdeye.platforms.copilot.identity import stored_session_id + + config, paths = copilot_env + home = Path(paths["home"]) + database = _write_database(home, session_id=NATIVE_SESSION_ID) + stored = stored_session_id(paths, NATIVE_SESSION_ID) + cycle = {"count": 0} + + def late_row_then_delete(_interval: float) -> None: + cycle["count"] += 1 + if cycle["count"] == 1: + connection = sqlite3.connect(database) + try: + connection.execute( + "INSERT INTO turns (id, session_id, turn_index, content, updated_at) " + "VALUES (?, ?, ?, ?, ?)", + (2, NATIVE_SESSION_ID, 1, "late-row", "2026-09-10T17:08:12.000Z"), + ) + connection.commit() + finally: + connection.close() + elif cycle["count"] == 2: + _unlink_database(database) + elif cycle["count"] >= 4: + raise KeyboardInterrupt + + monkeypatch.setattr(watch_mod, "_SLEEP", late_row_then_delete) + watch(config, paths, interval=0.1) + + payloads = [record["payload"] for record in iter_captured_records(config, stored)] + assert any( + isinstance(payload, dict) and payload.get("row", {}).get("content") == "late-row" + for payload in payloads + ) diff --git a/tests/shared/copilot_projection_fixtures.py b/tests/shared/copilot_projection_fixtures.py new file mode 100644 index 00000000..bb579358 --- /dev/null +++ b/tests/shared/copilot_projection_fixtures.py @@ -0,0 +1,373 @@ +"""Shared helpers to seed Copilot V2 projections for turn and usage view tests.""" + +from __future__ import annotations + +import json +import shutil +from pathlib import Path +from typing import Any + +from thirdeye.config import Config +from thirdeye.platforms.copilot.archive import commit_batch +from thirdeye.platforms.copilot.constants import PLATFORM_NAME +from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id +from thirdeye.platforms.copilot.projection import build_projection +from thirdeye.platforms.copilot.projection_state import empty_projection_state +from thirdeye.platforms.copilot.projection_store import commit_projection +from thirdeye.platforms.copilot.transcript import read_transcript +from thirdeye.platforms.copilot.types import Projection, SourceBatch, SourceRecord +from thirdeye.usage.types import UsageRow + +NATIVE_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" +INTERACTION_ONE = "6d2b89fd-a653-430c-b532-b0936d72eb42" +INTERACTION_TWO = "7e3c90ae-b764-541d-c643-c1047e83fc53" +INTERACTION_THREE = "8f4d01bf-c875-652e-d754-d2158f94ad64" +TURN_ONE_ID = "copilot:turn:fixture:session:6d2b89fd-a653-430c-b532-b0936d72eb42" +TURN_TWO_ID = "copilot:turn:fixture:session:7e3c90ae-b764-541d-c643-c1047e83fc53" +TURN_THREE_ID = "copilot:turn:fixture:session:8f4d01bf-c875-652e-d754-d2158f94ad64" +USAGE_MODEL_ONE = "gpt-5.6-luna" +USAGE_MODEL_TWO = "gpt-4.1-copilot-sentinel" +USAGE_TOKENS_ONE = 1111 +USAGE_TOKENS_TWO = 2222 +TURN_RECORD_KEYS = ( + "id", + "turn_id", + "session_id", + "platform", + "cwd", + "start_seq", + "end_seq", + "start_ts", + "end_ts", + "events", +) + + +def _transcript_record( + paths_source_key: str, + event_id: str, + *, + content: str, + ts: str, + interaction_id: str, + message_type: str = "user.message", + agent_id: str | None = None, +) -> SourceRecord: + payload: dict[str, Any] = { + "type": message_type, + "id": event_id, + "timestamp": ts, + "data": { + "content": content, + "interactionId": interaction_id, + "turnId": "0", + }, + } + if agent_id is not None: + payload["agentId"] = agent_id + return { + "source_id": f"{paths_source_key}/{NATIVE_ID}/{event_id}", + "source_kind": "transcript", + "native_session_id": NATIVE_ID, + "ts": ts, + "observed_at": ts, + "payload": payload, + "locator": {"file": "events.jsonl", "file_generation": "fixture-gen", "byte_offset": 0}, + } + + +def _main_turn( + *, + turn_id: str, + interaction_id: str, + source_ids: list[str], + start_ts: str, + end_ts: str, + status: str = "completed", + subagents: list[dict[str, Any]] | None = None, +) -> dict[str, Any]: + return { + "turn_id": turn_id, + "start_ts": start_ts, + "end_ts": end_ts, + "input_message": "prompt", + "output_message": "done" if status == "completed" else "", + "status": status, + "llm_calls": [], + "permission_requests": [], + "subagents": subagents or [], + "attributes": {"interaction_id": interaction_id}, + "source_ids": source_ids, + } + + +def _usage_row( + *, + stored_id: str, + call_id: str, + input_tokens: int, + response_model: str, + ts: str, +) -> UsageRow: + return UsageRow( + session_id=stored_id, + seq=0, + call_id=call_id, + ts=ts, + platform=PLATFORM_NAME, + provider_name="unknown", + response_model=response_model, + input_tokens=input_tokens, + output_tokens=10, + ) + + +def _attribution( + *, logical_call_id: str, usage_source_id: str, stored_turn_id: str +) -> dict[str, Any]: + return { + "usage_source_id": usage_source_id, + "logical_call_id": logical_call_id, + "stored_turn_id": stored_turn_id, + "agent_id": None, + "call_id": None, + "status": "matched", + "join_kind": "direct", + "evidence": ["direct:usage_source_id"], + } + + +def seed_two_main_interaction_projection(config: Config, tmp_path: Path) -> str: + """Archive evidence and commit a projection with two completed main user turns.""" + paths = resolve_sources(tmp_path / "copilot-home") + source_key = paths["source_key"] + ts1 = "2026-09-10T17:08:22.203Z" + ts2 = "2026-09-10T17:08:24.503Z" + ts3 = "2026-09-10T17:08:25.503Z" + ts4 = "2026-09-10T17:08:40.000Z" + ts5 = "2026-09-10T17:08:42.000Z" + ts6 = "2026-09-10T17:08:50.000Z" + user1 = "user-one" + asst1 = "assistant-one" + child1 = "child-one" + user2 = "user-two" + asst2 = "assistant-two" + user3 = "user-three" + records = [ + _transcript_record( + source_key, + user1, + content="Read alpha.txt and beta.txt with separate view calls in parallel.", + ts=ts1, + interaction_id=INTERACTION_ONE, + ), + _transcript_record( + source_key, + asst1, + content="I'll read both files now.", + ts=ts2, + interaction_id=INTERACTION_ONE, + message_type="assistant.message", + ), + _transcript_record( + source_key, + child1, + content="Explore child reading alpha.txt for the parent task.", + ts=ts3, + interaction_id=INTERACTION_ONE, + message_type="assistant.message", + agent_id="child-agent-id", + ), + _transcript_record( + source_key, + user2, + content="Report the final sum only.", + ts=ts4, + interaction_id=INTERACTION_TWO, + ), + _transcript_record( + source_key, + asst2, + content="The final sum is 100.", + ts=ts5, + interaction_id=INTERACTION_TWO, + message_type="assistant.message", + ), + _transcript_record( + source_key, + user3, + content="Keep going on the still-open follow-up.", + ts=ts6, + interaction_id=INTERACTION_THREE, + ), + ] + batch: SourceBatch = { + "source_key": source_key, + "native_session_id": NATIVE_ID, + "cwd": "/fixture/workspace", + "records": records, + "next_cursor": {"fixture": 1}, + "diagnostics": [], + } + commit_batch(config, paths, batch) + stored_id = stored_session_id(paths, NATIVE_ID) + + turn_one_sources = [ + f"{source_key}/{NATIVE_ID}/{user1}", + f"{source_key}/{NATIVE_ID}/{asst1}", + f"{source_key}/{NATIVE_ID}/{child1}", + ] + turn_two_sources = [ + f"{source_key}/{NATIVE_ID}/{user2}", + f"{source_key}/{NATIVE_ID}/{asst2}", + ] + turn_three_sources = [f"{source_key}/{NATIVE_ID}/{user3}"] + turn_one = _main_turn( + turn_id=TURN_ONE_ID, + interaction_id=INTERACTION_ONE, + source_ids=turn_one_sources, + start_ts=ts1, + end_ts=ts3, + subagents=[ + { + "turn_id": "copilot:turn:fixture:session:child", + "start_ts": ts3, + "end_ts": ts3, + "input_message": "explore", + "output_message": "done", + "status": "completed", + "llm_calls": [], + "permission_requests": [], + "subagents": [], + "attributes": { + "interaction_id": INTERACTION_ONE, + "agent_id": "child-agent-id", + }, + "source_ids": [f"{source_key}/{NATIVE_ID}/{child1}"], + } + ], + ) + turn_two = _main_turn( + turn_id=TURN_TWO_ID, + interaction_id=INTERACTION_TWO, + source_ids=turn_two_sources, + start_ts=ts4, + end_ts=ts5, + ) + turn_three = _main_turn( + turn_id=TURN_THREE_ID, + interaction_id=INTERACTION_THREE, + source_ids=turn_three_sources, + start_ts=ts6, + end_ts=ts6, + status="in_progress", + ) + usage_one = _usage_row( + stored_id=stored_id, + call_id="usage-source-one", + input_tokens=USAGE_TOKENS_ONE, + response_model=USAGE_MODEL_ONE, + ts=ts2, + ) + usage_two = _usage_row( + stored_id=stored_id, + call_id="usage-source-two", + input_tokens=USAGE_TOKENS_TWO, + response_model=USAGE_MODEL_TWO, + ts=ts5, + ) + projection: Projection = { + "normalized_events": [], + "turns": [turn_one, turn_two, turn_three], + "usage_rows": [usage_one, usage_two], + "attributions": [ + _attribution( + logical_call_id="logical-call-one", + usage_source_id="usage-source-one", + stored_turn_id=TURN_ONE_ID, + ), + _attribution( + logical_call_id="logical-call-two", + usage_source_id="usage-source-two", + stored_turn_id=TURN_TWO_ID, + ), + ], + "pending": [], + "diagnostics": [], + } + commit_projection(config, stored_id, projection, empty_projection_state()) + return stored_id + + +_COPILOT_FIXTURES = Path(__file__).resolve().parents[1] / "platforms" / "copilot" / "fixtures" +_OBSERVED_AT = "2026-09-10T17:09:00.000Z" + + +def seed_observed_six_call_projection(config: Config, tmp_path: Path) -> str: + """Archive the observed CLI corpus and commit its six-call projection.""" + home = tmp_path / "copilot-home" + home.mkdir(parents=True, exist_ok=True) + session_path = home / "session-state" / NATIVE_ID + session_path.mkdir(parents=True, exist_ok=True) + shutil.copy(_COPILOT_FIXTURES / "events.jsonl", session_path / "events.jsonl") + (session_path / "workspace.yaml").write_text("cwd: /sanitized/workspace\n", encoding="utf-8") + paths = resolve_sources(home) + cursor: dict[str, Any] = {} + transcript: list[SourceRecord] = [] + while True: + slice_ = read_transcript(paths, NATIVE_ID, cursor) + transcript.extend(slice_["records"]) + cursor = slice_["next_cursor"] + if slice_["exhausted"]: + break + source_key = next( + record["source_id"].split("/", 1)[0] + for record in transcript + if record["source_kind"] == "transcript" + ) + rows = json.loads((_COPILOT_FIXTURES / "assistant-usage-events.json").read_text()) + revisions = { + call["row_id"]: call["usage_source_id"].rsplit(":", 1)[-1] + for call in json.loads( + (_COPILOT_FIXTURES / "reconciliation-cases" / "observed-six-calls.json").read_text() + )["calls"] + } + usage_records: list[SourceRecord] = [] + for row in rows: + content_revision = revisions[row["id"]] + usage_records.append( + { + "source_id": ( + f"copilot-db:{source_key}:{NATIVE_ID}:assistant_usage_events:" + f"{row['id']}:{content_revision}" + ), + "source_kind": "database", + "native_session_id": NATIVE_ID, + "ts": row.get("created_at"), + "observed_at": _OBSERVED_AT, + "payload": {"table": "assistant_usage_events", "row": row}, + "locator": { + "database": "/example/.copilot/session-store.db", + "table": "assistant_usage_events", + "primary_key": row["id"], + "content_revision": content_revision, + "generation": ( + "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + ), + }, + } + ) + records = transcript + usage_records + batch: SourceBatch = { + "source_key": paths["source_key"], + "native_session_id": NATIVE_ID, + "cwd": "/fixture/workspace", + "records": records, + "next_cursor": {"generation": 1}, + "diagnostics": [], + } + commit_batch(config, paths, batch) + stored_id = stored_session_id(paths, NATIVE_ID) + projection, next_state = build_projection(records, {}) + commit_projection(config, stored_id, projection, next_state) + return stored_id diff --git a/tests/shared/test_accounting_export.py b/tests/shared/test_accounting_export.py new file mode 100644 index 00000000..2482dd76 --- /dev/null +++ b/tests/shared/test_accounting_export.py @@ -0,0 +1,829 @@ +from __future__ import annotations + +import hashlib +import json +import time +from pathlib import Path +from typing import Any + +import pytest + +from thirdeye import otel_export, otel_worker +from thirdeye.config import Config, LogfireSettings +from thirdeye.paths import otel_jobs_dir +from thirdeye.span_ids import chat_span_id +from thirdeye.usage.types import UsageRow + +pytest.importorskip("logfire") + +from logfire.testing import TestExporter # noqa: E402 +from opentelemetry.sdk.trace.export import SimpleSpanProcessor # noqa: E402 + +_FIXTURES = ( + Path(__file__).resolve().parents[1] + / "platforms" + / "copilot" + / "fixtures" + / "reconciliation-cases" +) +_TRANSPORT = json.loads((_FIXTURES / "transport.json").read_text(encoding="utf-8")) + + +@pytest.fixture(autouse=True) +def _reset_state(): + otel_export._state["attempted"] = False + otel_export._state["instance"] = None + otel_export._state["id_generator"] = None + yield + otel_export._state["attempted"] = False + otel_export._state["instance"] = None + otel_export._state["id_generator"] = None + + +@pytest.fixture +def exporter(): + return TestExporter() + + +@pytest.fixture +def wired_instance(exporter, monkeypatch: pytest.MonkeyPatch): + import logfire + + instance = logfire.configure( + send_to_logfire=False, + console=False, + additional_span_processors=[SimpleSpanProcessor(exporter)], + advanced=logfire.AdvancedOptions(id_generator=otel_export._id_generator()), + ) + monkeypatch.setattr(otel_export, "_get_instance", lambda config, platform: instance) + return instance + + +@pytest.fixture +def enabled_config(tmp_path: Path) -> Config: + return Config( + root=tmp_path, + logfire=LogfireSettings(enabled=True, token="fake-token"), + ) + + +def _turn(**overrides: Any) -> dict[str, Any]: + defaults: dict[str, Any] = dict( + turn_id="turn_1", + start_ts="2026-01-01T00:00:00.000Z", + end_ts="2026-01-01T00:00:05.000Z", + input_message="hi", + output_message="hello", + status="completed", + llm_calls=[], + permission_requests=[], + subagents=[], + attributes={}, + ) + defaults.update(overrides) + return defaults + + +def _llm_call(**overrides: Any) -> dict[str, Any]: + defaults: dict[str, Any] = dict( + call_id="call_1", + provider="anthropic", + model="claude-sonnet-5", + start_ts="2026-01-01T00:00:01.000Z", + end_ts="2026-01-01T00:00:02.000Z", + input_messages=[{"role": "user", "parts": [{"type": "text", "content": "hi"}]}], + output_messages=[{"role": "assistant", "parts": [{"type": "text", "content": "hello"}]}], + usage={"input_tokens": 100, "output_tokens": 50}, + tool_calls=[], + ) + defaults.update(overrides) + return defaults + + +def _usage_row(**overrides: Any) -> dict[str, Any]: + fields = dict( + session_id="s1", + seq=0, + call_id="usage-1", + ts="2026-01-01T00:00:01.500Z", + platform="copilot", + provider_name="unknown", + response_model="gpt-test", + input_tokens=6452, + output_tokens=107, + cache_creation_input_tokens=6449, + ) + fields.update(overrides) + return UsageRow(**fields).to_dict() + + +def _accounting_call(**overrides: Any) -> dict[str, Any]: + defaults: dict[str, Any] = dict( + accounting_id="acct-1", + usage=_usage_row(), + attribution_status="matched", + agent_id=None, + call_id="call_1", + attributes={"accounting.destination": "chat-span"}, + ) + defaults.update(overrides) + return defaults + + +def _error_log_entries(home: Path) -> list[dict]: + log = home / "logs" / "usage-errors.jsonl" + if not log.exists(): + return [] + return [json.loads(line) for line in log.read_text().splitlines() if line] + + +def _accounting_job_path(root: Path, job_id: str) -> Path: + digest = hashlib.sha256(job_id.encode("utf-8")).hexdigest() + return otel_jobs_dir(root) / f"accounting-{digest}.json" + + +def _write_queued_accounting_job(root: Path, **fields: Any) -> Path: + job_id = str(fields["job_id"]) + job_path = _accounting_job_path(root, job_id) + job_path.parent.mkdir(parents=True, exist_ok=True) + payload = {"state": "queued", "attempt": 0, **fields} + job_path.write_text(json.dumps(payload), encoding="utf-8") + return job_path + + +class TestAccountingAttributes: + def test_projects_usage_and_metadata(self): + accounting = _accounting_call( + accounting_id="acct-42", + attribution_status="pending", + agent_id="agent-9", + attributes={"copilot.logical_call_id": "logical-1"}, + ) + attrs = otel_export._accounting_attributes(accounting) + + assert attrs["thirdeye.accounting.id"] == "acct-42" + assert attrs["thirdeye.accounting.attribution_status"] == "pending" + assert attrs["thirdeye.accounting.agent_id"] == "agent-9" + assert attrs["gen_ai.usage.input_tokens"] == 6452 + assert attrs["copilot.logical_call_id"] == "logical-1" + assert json.loads(attrs["thirdeye.accounting.usage"])["call_id"] == "usage-1" + + def test_labels_copilot_native_billing_distinct_from_usd(self): + accounting = _accounting_call( + attributes={"copilot.billing.nano_aiu": 12, "operation.cost": 0.05}, + ) + attrs = otel_export._accounting_attributes(accounting) + + assert attrs["thirdeye.accounting.billing.kind"] == "copilot-native-unit" + assert attrs["copilot.billing.nano_aiu"] == 12 + + def test_usd_only_cost_does_not_set_native_billing_kind(self): + accounting = _accounting_call(attributes={"operation.cost": 0.05}) + attrs = otel_export._accounting_attributes(accounting) + + assert attrs["operation.cost"] == 0.05 + assert "thirdeye.accounting.billing.kind" not in attrs + + +class TestOrdinaryTurnCompatibility: + def test_turn_without_accounting_calls_is_unchanged( + self, tmp_path: Path, enabled_config: Config, wired_instance, exporter + ): + call = _llm_call() + turn = _turn(llm_calls=[call]) + session_dir = tmp_path / "traces" / "claude" / "s1" + + otel_export._export_turn_inner( + config=enabled_config, + session_dir_=session_dir, + session_id="s1", + platform="claude", + cwd="/proj", + turn=turn, + ) + + spans = exporter.exported_spans_as_dict() + chat_spans = [span for span in spans if span["name"].startswith("chat")] + accounting_spans = [span for span in spans if span["name"] == "accounting"] + + assert len(chat_spans) == 1 + assert accounting_spans == [] + assert chat_spans[0]["attributes"]["gen_ai.usage.input_tokens"] == 100 + assert "thirdeye.accounting.id" not in chat_spans[0]["attributes"] + + +class TestNestedTurnAccounting: + def test_matched_accounting_merges_onto_chat_span_only( + self, tmp_path: Path, enabled_config: Config, wired_instance, exporter + ): + session_id = "copilot-session" + platform = "copilot" + call_id = "copilot:call:matched" + call = _llm_call(call_id=call_id, usage={}) + accounting = _accounting_call( + call_id=call_id, + usage=_usage_row( + call_id="usage-matched", + input_tokens=6452, + output_tokens=107, + ), + ) + turn = _turn(llm_calls=[call], accounting_calls=[accounting]) + + otel_export._export_turn_inner( + config=enabled_config, + session_dir_=tmp_path / "traces" / platform / session_id, + session_id=session_id, + platform=platform, + cwd="/proj", + turn=turn, + ) + + spans = exporter.exported_spans_as_dict() + chat_span = next(span for span in spans if span["name"].startswith("chat")) + accounting_spans = [span for span in spans if span["name"] == "accounting"] + + assert accounting_spans == [] + assert chat_span["context"]["span_id"] == chat_span_id(platform, session_id, call_id) + assert chat_span["attributes"]["gen_ai.usage.input_tokens"] == 6452 + assert chat_span["attributes"]["thirdeye.accounting.id"] == "acct-1" + schema = json.loads(chat_span["attributes"]["logfire.json_schema"]) + assert "gen_ai.input.messages" in schema["properties"] + assert "gen_ai.output.messages" in schema["properties"] + + def test_matched_copilot_billing_labels_chat_span( + self, tmp_path: Path, enabled_config: Config, wired_instance, exporter + ): + session_id = "copilot-session" + platform = "copilot" + call_id = "copilot:call:billed" + call = _llm_call(call_id=call_id, usage={}) + accounting = _accounting_call( + call_id=call_id, + usage=_usage_row(call_id="usage-billed", input_tokens=6452, output_tokens=107), + attributes={"copilot.billing.nano_aiu": 12}, + ) + turn = _turn(llm_calls=[call], accounting_calls=[accounting]) + + otel_export._export_turn_inner( + config=enabled_config, + session_dir_=tmp_path / "traces" / platform / session_id, + session_id=session_id, + platform=platform, + cwd="/proj", + turn=turn, + ) + + chat_span = next( + span for span in exporter.exported_spans_as_dict() if span["name"].startswith("chat") + ) + assert chat_span["attributes"]["thirdeye.accounting.billing.kind"] == "copilot-native-unit" + assert chat_span["attributes"]["copilot.billing.nano_aiu"] == 12 + assert chat_span["attributes"]["gen_ai.usage.input_tokens"] == 6452 + + def test_unknown_call_id_emits_turn_owned_span_not_chat( + self, tmp_path: Path, enabled_config: Config, wired_instance, exporter + ): + session_id = "copilot-session" + platform = "copilot" + turn_id = "turn-unknown-call" + call = _llm_call(call_id="known-call") + accounting = _accounting_call( + accounting_id="acct-unknown-call", + call_id="not-in-llm-calls", + attribution_status="pending", + usage=_usage_row(call_id="usage-unknown", input_tokens=10, output_tokens=2), + attributes={"accounting.destination": "turn-accounting-span"}, + ) + turn = _turn(turn_id=turn_id, llm_calls=[call], accounting_calls=[accounting]) + + otel_export._export_turn_inner( + config=enabled_config, + session_dir_=tmp_path / "traces" / platform / session_id, + session_id=session_id, + platform=platform, + cwd="/proj", + turn=turn, + ) + + spans = exporter.exported_spans_as_dict() + chat_spans = [span for span in spans if span["name"].startswith("chat")] + accounting_spans = [span for span in spans if span["name"] == "accounting"] + turn_span = next(span for span in spans if span["name"] == "invoke_agent") + + assert len(chat_spans) == 1 + assert chat_spans[0]["attributes"].get("thirdeye.accounting.id") is None + assert len(accounting_spans) == 1 + assert accounting_spans[0]["parent"]["span_id"] == turn_span["context"]["span_id"] + assert accounting_spans[0]["attributes"]["thirdeye.accounting.id"] == "acct-unknown-call" + + def test_unmatched_accounting_emits_turn_owned_span_with_deterministic_id( + self, tmp_path: Path, enabled_config: Config, wired_instance, exporter + ): + session_id = "copilot-session" + platform = "copilot" + turn_id = "turn-unmatched" + call = _llm_call(call_id="known-call", usage={}) + unmatched = _accounting_call( + accounting_id="acct-unmatched", + call_id=None, + attribution_status="pending", + usage=_usage_row(call_id="usage-unmatched", input_tokens=6587, output_tokens=5), + attributes={"accounting.destination": "turn-accounting-span"}, + ) + turn = _turn(turn_id=turn_id, llm_calls=[call], accounting_calls=[unmatched]) + + otel_export._export_turn_inner( + config=enabled_config, + session_dir_=tmp_path / "traces" / platform / session_id, + session_id=session_id, + platform=platform, + cwd="/proj", + turn=turn, + ) + + spans = exporter.exported_spans_as_dict() + chat_span = next(span for span in spans if span["name"].startswith("chat")) + accounting_span = next(span for span in spans if span["name"] == "accounting") + turn_span = next(span for span in spans if span["name"] == "invoke_agent") + + assert chat_span["attributes"].get("thirdeye.accounting.id") is None + assert accounting_span["parent"]["span_id"] == turn_span["context"]["span_id"] + assert accounting_span["context"]["span_id"] == otel_export._accounting_span_id( + platform, session_id, "acct-unmatched", turn_id + ) + assert accounting_span["attributes"]["gen_ai.usage.input_tokens"] == 6587 + assert accounting_span["attributes"]["thirdeye.accounting.attribution_status"] == "pending" + + def test_fixture_turn_with_matched_and_unmatched_accounting( + self, tmp_path: Path, enabled_config: Config, wired_instance, exporter + ): + turn = dict(_TRANSPORT["turn_span_with_accounting_calls"]) + session_id = turn["accounting_calls"][0]["usage"]["session_id"] + platform = "copilot" + + otel_export._export_turn_inner( + config=enabled_config, + session_dir_=tmp_path / "traces" / platform / session_id, + session_id=session_id, + platform=platform, + cwd="/proj", + turn=turn, + ) + + spans = exporter.exported_spans_as_dict() + chat_spans = [span for span in spans if span["name"].startswith("chat")] + accounting_spans = [span for span in spans if span["name"] == "accounting"] + + assert len(chat_spans) == 1 + assert len(accounting_spans) == 1 + matched_id = turn["accounting_calls"][0]["accounting_id"] + unmatched_id = turn["accounting_calls"][1]["accounting_id"] + assert chat_spans[0]["attributes"]["thirdeye.accounting.id"] == matched_id + assert accounting_spans[0]["attributes"]["thirdeye.accounting.id"] == unmatched_id + + +class TestSessionAccountingExport: + def test_queues_deterministic_job_without_duplicates( + self, enabled_config: Config, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ): + spawned: list[Path] = [] + monkeypatch.setattr(otel_export, "_spawn", spawned.append) + session_dir = tmp_path / "traces" / "copilot" / "s1" + accounting = { + "accounting_id": "acct-session", + "usage": _usage_row(session_id="s1"), + "attribution_status": "pending", + "agent_id": None, + "attributes": {"accounting.destination": "session-accounting-span"}, + } + + first = otel_export.export_session_accounting( + enabled_config, session_dir, "s1", "copilot", "/proj", accounting + ) + second = otel_export.export_session_accounting( + enabled_config, session_dir, "s1", "copilot", "/proj", accounting + ) + + assert first is True + assert second is True + jobs = list(otel_jobs_dir(enabled_config.root).glob("accounting-*.json")) + assert len(jobs) == 1 + payload = json.loads(jobs[0].read_text(encoding="utf-8")) + assert payload["kind"] == "session_accounting" + assert payload["destination"] == "session-accounting-span" + assert payload["state"] == "queued" + assert payload["job_id"] == "accounting:s1:acct-session" + assert payload["span_id"] == payload["job_id"] + assert len(spawned) == 2 + + def test_disabled_config_does_not_queue(self, tmp_path: Path): + config = Config(root=tmp_path, logfire=LogfireSettings(enabled=False, token="")) + session_dir = tmp_path / "traces" / "copilot" / "s1" + accounting = { + "accounting_id": "acct-session", + "usage": _usage_row(), + "attribution_status": "pending", + } + assert ( + otel_export.export_session_accounting( + config, session_dir, "s1", "copilot", "/proj", accounting + ) + is False + ) + assert list(otel_jobs_dir(config.root).glob("accounting-*.json")) == [] + + def test_worker_round_trips_session_accounting_job( + self, + enabled_config: Config, + wired_instance, + exporter, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ): + monkeypatch.setattr(Config, "load", lambda: enabled_config) + session_id = "s1" + platform = "copilot" + session_dir = tmp_path / "traces" / platform / session_id + accounting_id = "copilot-usage-7f3a" + job_id = f"accounting:{session_id}:{accounting_id}" + digest = hashlib.sha256(job_id.encode("utf-8")).hexdigest() + job_path = otel_jobs_dir(enabled_config.root) / f"accounting-{digest}.json" + job_path.parent.mkdir(parents=True, exist_ok=True) + job_path.write_text( + json.dumps( + { + "job_id": job_id, + "kind": "session_accounting", + "session_dir": str(session_dir), + "session_id": session_id, + "platform": platform, + "cwd": "/proj", + "accounting_id": accounting_id, + "destination": "session-accounting-span", + "usage": _usage_row(session_id=session_id), + "attribution_status": "pending", + "agent_id": None, + "attributes": {}, + "span_id": job_id, + "state": "queued", + "attempt": 0, + } + ), + encoding="utf-8", + ) + + otel_worker.main([str(job_path)]) + + assert not job_path.exists() + spans = exporter.exported_spans_as_dict() + accounting_span = next(span for span in spans if span["name"] == "accounting") + session_span = next(span for span in spans if span["name"] == "session") + + assert accounting_span["parent"]["span_id"] == session_span["context"]["span_id"] + assert accounting_span["attributes"]["thirdeye.accounting.id"] == accounting_id + assert accounting_span["attributes"]["thirdeye.platform"] == platform + assert accounting_span["attributes"]["thirdeye.cwd"] == "/proj" + assert accounting_span["context"]["span_id"] == otel_export._accounting_span_id( + platform, session_id, accounting_id + ) + + +class TestTurnAccountingExport: + def test_queues_deterministic_job_without_duplicates( + self, enabled_config: Config, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ): + spawned: list[Path] = [] + monkeypatch.setattr(otel_export, "_spawn", spawned.append) + session_dir = tmp_path / "traces" / "copilot" / "s1" + accounting = { + "accounting_id": "acct-turn", + "usage": _usage_row(session_id="s1"), + "attribution_status": "pending", + "agent_id": None, + "attributes": {"accounting.destination": "turn-accounting-span"}, + } + + first = otel_export.export_turn_accounting( + enabled_config, session_dir, "s1", "copilot", "/proj", "turn_1", accounting + ) + second = otel_export.export_turn_accounting( + enabled_config, session_dir, "s1", "copilot", "/proj", "turn_1", accounting + ) + + assert first is True + assert second is True + jobs = list(otel_jobs_dir(enabled_config.root).glob("accounting-*.json")) + assert len(jobs) == 1 + payload = json.loads(jobs[0].read_text(encoding="utf-8")) + assert payload["kind"] == "turn_accounting" + assert payload["destination"] == "turn-accounting-span" + assert payload["state"] == "queued" + assert payload["job_id"] == "accounting:s1:turn_1:acct-turn" + assert payload["span_id"] == payload["job_id"] + assert payload["turn_id"] == "turn_1" + assert len(spawned) == 2 + + def test_worker_round_trips_turn_accounting_job( + self, + enabled_config: Config, + wired_instance, + exporter, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ): + monkeypatch.setattr(Config, "load", lambda: enabled_config) + session_id = "s1" + platform = "copilot" + turn_id = "turn_1" + session_dir = tmp_path / "traces" / platform / session_id + accounting_id = "acct-delayed" + job_id = f"accounting:{session_id}:{turn_id}:{accounting_id}" + turn_span_id = 4242 + job_path = _write_queued_accounting_job( + enabled_config.root, + job_id=job_id, + kind="turn_accounting", + session_dir=str(session_dir), + session_id=session_id, + platform=platform, + cwd="/proj", + turn_id=turn_id, + turn_span_id=str(turn_span_id), + accounting_id=accounting_id, + destination="turn-accounting-span", + usage=_usage_row(session_id=session_id), + attribution_status="pending", + agent_id=None, + attributes={"accounting.destination": "turn-accounting-span"}, + span_id=job_id, + ) + + otel_worker.main([str(job_path)]) + + assert not job_path.exists() + spans = exporter.exported_spans_as_dict() + accounting_span = next(span for span in spans if span["name"] == "accounting") + invented_turns = [span for span in spans if span["name"] == "invoke_agent"] + invented_chats = [span for span in spans if span["name"].startswith("chat")] + + assert invented_turns == [] + assert invented_chats == [] + assert accounting_span["parent"]["span_id"] == turn_span_id + assert accounting_span["attributes"]["thirdeye.accounting.id"] == accounting_id + assert accounting_span["attributes"]["thirdeye.turn.id"] == turn_id + assert accounting_span["attributes"]["thirdeye.platform"] == platform + assert accounting_span["context"]["span_id"] == otel_export._accounting_span_id( + platform, session_id, accounting_id, turn_id + ) + + +class TestWorkerClaimRecovery: + def test_stale_claim_is_recovered_and_export_retried( + self, + enabled_config: Config, + wired_instance, + exporter, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ): + monkeypatch.setattr(Config, "load", lambda: enabled_config) + session_id = "s1" + platform = "copilot" + session_dir = tmp_path / "traces" / platform / session_id + job_id = "accounting:s1:acct-retry" + digest = hashlib.sha256(job_id.encode("utf-8")).hexdigest() + job_path = otel_jobs_dir(enabled_config.root) / f"accounting-{digest}.json" + job_path.parent.mkdir(parents=True, exist_ok=True) + job_path.write_text( + json.dumps( + { + "job_id": job_id, + "kind": "session_accounting", + "session_dir": str(session_dir), + "session_id": session_id, + "platform": platform, + "cwd": "/proj", + "accounting_id": "acct-retry", + "destination": "session-accounting-span", + "usage": _usage_row(session_id=session_id), + "attribution_status": "pending", + "state": "claimed", + "attempt": 0, + } + ), + encoding="utf-8", + ) + claim_path = otel_worker._job_claim_path(job_path) + claim_path.write_text("99999", encoding="utf-8") + stale_at = time.time() - otel_worker._JOB_CLAIM_STALE_S - 5 + import os + + os.utime(claim_path, (stale_at, stale_at)) + + otel_worker.main([str(job_path)]) + + assert not job_path.exists() + assert not claim_path.exists() + assert any(span["name"] == "accounting" for span in exporter.exported_spans_as_dict()) + + def test_export_failure_retains_job_and_increments_attempt( + self, enabled_config: Config, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ): + monkeypatch.setattr(Config, "load", lambda: enabled_config) + session_id = "s1" + platform = "copilot" + session_dir = tmp_path / "traces" / platform / session_id + job_id = "accounting:s1:acct-fail" + digest = hashlib.sha256(job_id.encode("utf-8")).hexdigest() + job_path = otel_jobs_dir(enabled_config.root) / f"accounting-{digest}.json" + job_path.parent.mkdir(parents=True, exist_ok=True) + job_path.write_text( + json.dumps( + { + "job_id": job_id, + "kind": "session_accounting", + "session_dir": str(session_dir), + "session_id": session_id, + "platform": platform, + "cwd": "/proj", + "accounting_id": "acct-fail", + "destination": "session-accounting-span", + "usage": _usage_row(session_id=session_id), + "attribution_status": "pending", + "state": "queued", + "attempt": 0, + } + ), + encoding="utf-8", + ) + + def _boom(**kwargs): + raise RuntimeError("flush failed") + + monkeypatch.setattr(otel_export, "_export_session_accounting_inner", _boom) + otel_worker.main([str(job_path)]) + + assert job_path.exists() + payload = json.loads(job_path.read_text(encoding="utf-8")) + assert payload["state"] == "queued" + assert payload["attempt"] == 1 + assert not otel_worker._job_claim_path(job_path).exists() + entries = _error_log_entries(enabled_config.root) + assert any("kind=session_accounting" in entry["message"] for entry in entries) + + def test_fresh_claim_blocks_concurrent_worker( + self, enabled_config: Config, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ): + job_path = tmp_path / "accounting-job.json" + payload = {"state": "queued", "attempt": 0} + job_path.write_text(json.dumps(payload), encoding="utf-8") + claim_path = otel_worker._job_claim_path(job_path) + claim_path.write_text("42", encoding="utf-8") + monkeypatch.setattr( + time, + "time", + lambda: claim_path.stat().st_mtime + 1, + ) + + claimed = otel_worker._claim_job(job_path, payload) + + assert claimed is None + + def test_success_writes_emitted_before_unlinking( + self, + enabled_config: Config, + wired_instance, + exporter, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ): + monkeypatch.setattr(Config, "load", lambda: enabled_config) + states: list[str] = [] + original = otel_worker._write_job_state + + def _capture(job_path: Path, payload: dict[str, Any]) -> None: + states.append(str(payload.get("state"))) + original(job_path, payload) + + monkeypatch.setattr(otel_worker, "_write_job_state", _capture) + job_path = _write_queued_accounting_job( + enabled_config.root, + job_id="accounting:s1:acct-emitted", + kind="session_accounting", + session_dir=str(tmp_path / "traces" / "copilot" / "s1"), + session_id="s1", + platform="copilot", + cwd="/proj", + accounting_id="acct-emitted", + destination="session-accounting-span", + usage=_usage_row(session_id="s1"), + attribution_status="pending", + span_id="accounting:s1:acct-emitted", + ) + + otel_worker.main([str(job_path)]) + + assert "emitted" in states + assert states[-1] == "emitted" + assert not job_path.exists() + assert any(span["name"] == "accounting" for span in exporter.exported_spans_as_dict()) + + def test_missing_logfire_instance_retains_accounting_job( + self, enabled_config: Config, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ): + monkeypatch.setattr(Config, "load", lambda: enabled_config) + monkeypatch.setattr(otel_export, "_get_instance", lambda config, platform: None) + job_path = _write_queued_accounting_job( + enabled_config.root, + job_id="accounting:s1:acct-noop", + kind="session_accounting", + session_dir=str(tmp_path / "traces" / "copilot" / "s1"), + session_id="s1", + platform="copilot", + cwd="/proj", + accounting_id="acct-noop", + destination="session-accounting-span", + usage=_usage_row(session_id="s1"), + attribution_status="pending", + span_id="accounting:s1:acct-noop", + ) + + otel_worker.main([str(job_path)]) + + assert job_path.exists() + payload = json.loads(job_path.read_text(encoding="utf-8")) + assert payload["state"] == "queued" + assert payload["attempt"] == 1 + assert not otel_worker._job_claim_path(job_path).exists() + + def test_worker_dispatches_turn_accounting_kind( + self, enabled_config: Config, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ): + monkeypatch.setattr(Config, "load", lambda: enabled_config) + calls: list[dict[str, Any]] = [] + monkeypatch.setattr( + otel_export, + "_export_turn_accounting_inner", + lambda **kwargs: calls.append(kwargs), + raising=False, + ) + job_path = _write_queued_accounting_job( + enabled_config.root, + job_id="accounting:s1:turn_1:acct-dispatch", + kind="turn_accounting", + session_dir=str(tmp_path / "traces" / "copilot" / "s1"), + session_id="s1", + platform="copilot", + cwd="/proj", + turn_id="turn_1", + accounting_id="acct-dispatch", + destination="turn-accounting-span", + usage=_usage_row(session_id="s1"), + attribution_status="pending", + span_id="accounting:s1:turn_1:acct-dispatch", + ) + + otel_worker.main([str(job_path)]) + + assert len(calls) == 1 + assert calls[0]["session_id"] == "s1" + assert calls[0]["turn_id"] == "turn_1" + assert calls[0]["accounting"]["accounting_id"] == "acct-dispatch" + assert not job_path.exists() + + def test_exhausted_retries_mark_job_failed( + self, enabled_config: Config, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ): + monkeypatch.setattr(Config, "load", lambda: enabled_config) + job_path = _write_queued_accounting_job( + enabled_config.root, + job_id="accounting:s1:acct-exhausted", + kind="session_accounting", + session_dir=str(tmp_path / "traces" / "copilot" / "s1"), + session_id="s1", + platform="copilot", + cwd="/proj", + accounting_id="acct-exhausted", + destination="session-accounting-span", + usage=_usage_row(session_id="s1"), + attribution_status="pending", + span_id="accounting:s1:acct-exhausted", + attempt=otel_worker._JOB_MAX_ATTEMPTS - 1, + ) + + def _boom(**kwargs): + raise RuntimeError("flush failed") + + monkeypatch.setattr(otel_export, "_export_session_accounting_inner", _boom) + otel_worker.main([str(job_path)]) + + assert job_path.exists() + payload = json.loads(job_path.read_text(encoding="utf-8")) + assert payload["state"] == "failed" + assert payload["attempt"] == otel_worker._JOB_MAX_ATTEMPTS + + otel_worker.main([str(job_path)]) + still = json.loads(job_path.read_text(encoding="utf-8")) + assert still["state"] == "failed" + assert still["attempt"] == otel_worker._JOB_MAX_ATTEMPTS diff --git a/tests/shared/test_add_command.py b/tests/shared/test_add_command.py index 83bac55e..1831f0fc 100644 --- a/tests/shared/test_add_command.py +++ b/tests/shared/test_add_command.py @@ -9,6 +9,7 @@ from thirdeye.commands.add import PLATFORMS, find_orphaned_hooks from thirdeye.platforms.claude.install import ClaudePlatform from thirdeye.platforms.codex.install import CodexPlatform +from thirdeye.platforms.copilot.install import CopilotPlatform from thirdeye.platforms.cursor.install import CursorPlatform # -- command registration ------------------------------------------------------ @@ -50,6 +51,18 @@ def test_remove_help_mentions_codex(): assert "--codex" in r.output +def test_add_help_mentions_copilot(): + r = CliRunner().invoke(main, ["add", "--help"]) + assert r.exit_code == 0 + assert "--copilot" in r.output + + +def test_remove_help_mentions_copilot(): + r = CliRunner().invoke(main, ["remove", "--help"]) + assert r.exit_code == 0 + assert "--copilot" in r.output + + # -- platform flag required ---------------------------------------------------- @@ -59,12 +72,26 @@ def test_add_requires_platform(): assert "platform" in r.output.lower() +def test_add_requires_platform_lists_all_supported_flags(): + r = CliRunner().invoke(main, ["add"]) + assert r.exit_code != 0 + for flag in ("--claude", "--codex", "--cursor", "--copilot"): + assert flag in r.output + + def test_remove_requires_platform(): r = CliRunner().invoke(main, ["remove"]) assert r.exit_code != 0 assert "platform" in r.output.lower() +def test_remove_requires_platform_lists_all_supported_flags(): + r = CliRunner().invoke(main, ["remove"]) + assert r.exit_code != 0 + for flag in ("--claude", "--codex", "--cursor", "--copilot"): + assert flag in r.output + + # -- install (add --claude) ---------------------------------------------------- @@ -214,7 +241,7 @@ def test_ingest_still_works(tmp_path: Path): def test_platforms_dict_is_exactly_supported_platforms(): - assert set(PLATFORMS) == {"claude", "codex", "cursor"} + assert set(PLATFORMS) == {"claude", "codex", "cursor", "copilot"} def test_platforms_dict_has_claude(): @@ -232,6 +259,11 @@ def test_platforms_dict_has_cursor(): assert PLATFORMS["cursor"] is CursorPlatform +def test_platforms_dict_has_copilot(): + assert "copilot" in PLATFORMS + assert PLATFORMS["copilot"] is CopilotPlatform + + def test_platform_flag_value_maps_to_platforms_key(): for key, cls in PLATFORMS.items(): instance = cls() @@ -375,6 +407,48 @@ def test_list_shows_supported_platforms(monkeypatch): assert "codex" in r.output assert "gemini" not in r.output assert "cursor" in r.output + assert "copilot" in r.output + + +# -- install (add --copilot) --------------------------------------------------- + + +def test_add_copilot_calls_install(monkeypatch): + from unittest.mock import MagicMock + + mock_platform = MagicMock() + mock_platform.display_name = "GitHub Copilot CLI" + mock_cls = MagicMock(return_value=mock_platform) + + monkeypatch.setitem(PLATFORMS, "copilot", mock_cls) + r = CliRunner().invoke(main, ["add", "--copilot"]) + assert r.exit_code == 0, r.output + mock_cls.assert_called_once() + mock_platform.install.assert_called_once() + + +def test_add_copilot_writes_hooks(tmp_path: Path, monkeypatch): + hooks_file = tmp_path / "hooks" / "thirdeye.json" + platform = CopilotPlatform(hooks_file=hooks_file, entrypoint="/opt/bin/thirdeye-copilot-hook") + monkeypatch.setitem(PLATFORMS, "copilot", lambda: platform) + r = CliRunner().invoke(main, ["add", "--copilot"]) + assert r.exit_code == 0, r.output + assert hooks_file.exists() + assert "Installed tracing for GitHub Copilot CLI" in r.output + + +def test_remove_copilot_calls_uninstall(monkeypatch): + from unittest.mock import MagicMock + + mock_platform = MagicMock() + mock_platform.display_name = "GitHub Copilot CLI" + mock_cls = MagicMock(return_value=mock_platform) + + monkeypatch.setitem(PLATFORMS, "copilot", mock_cls) + r = CliRunner().invoke(main, ["remove", "--copilot"]) + assert r.exit_code == 0, r.output + mock_cls.assert_called_once() + mock_platform.uninstall.assert_called_once() # -- find_orphaned_hooks ------------------------------------------------------- diff --git a/tests/shared/test_provenance.py b/tests/shared/test_provenance.py index 5a007e94..a0a32ff7 100644 --- a/tests/shared/test_provenance.py +++ b/tests/shared/test_provenance.py @@ -209,3 +209,55 @@ def test_classification_does_not_mutate_payload(payload: dict[str, Any], expecte foreign_payload_reason(payload, expected) assert payload == original + + +# --- copilot-specific provenance (positive evidence, fail-open elsewhere) --- + + +@pytest.mark.parametrize( + "event_name", + [ + "sessionStart", + "userPromptSubmitted", + "preToolUse", + "postToolUse", + "agentStop", + "subagentStart", + "subagentStop", + "sessionEnd", + ], +) +def test_copilot_accepts_native_camel_case_events(event_name: str): + assert foreign_payload_reason({"hook_event_name": event_name}, "copilot") is None + + +@pytest.mark.parametrize("event_name", _PASCAL_CASE_EVENTS) +def test_copilot_accepts_pascal_case_aliases(event_name: str): + assert foreign_payload_reason({"hook_event_name": event_name}, "copilot") is None + + +@pytest.mark.parametrize("marker", ["cursor_version", "composer_mode"]) +@pytest.mark.parametrize("value", [None, False, ""]) +def test_cursor_markers_are_foreign_for_copilot(marker: str, value: object): + reason = foreign_payload_reason({marker: value}, "copilot") + + assert isinstance(reason, str) + assert reason + assert marker in reason + + +def test_copilot_does_not_change_claude_codex_cursor_regressions(): + assert foreign_payload_reason({"hook_event_name": "SessionStart"}, "claude") is None + assert foreign_payload_reason({"hook_event_name": "beforeSubmitPrompt"}, "claude") is not None + assert foreign_payload_reason({"hook_event_name": "SessionStart"}, "cursor") is not None + assert foreign_payload_reason({"hook_event_name": "beforeSubmitPrompt"}, "cursor") is None + + +def test_copilot_rejects_cursor_only_camel_case_events(): + reason = foreign_payload_reason({"hook_event_name": "beforeSubmitPrompt"}, "copilot") + assert isinstance(reason, str) + assert "beforeSubmitPrompt" in reason + + +def test_copilot_is_known_platform(): + assert foreign_payload_reason({}, "copilot") is None diff --git a/tests/shared/test_setup_command.py b/tests/shared/test_setup_command.py index b076e8e9..542fcda9 100644 --- a/tests/shared/test_setup_command.py +++ b/tests/shared/test_setup_command.py @@ -45,7 +45,7 @@ def test_setup_appears_in_help() -> None: def test_setup_multiselects_agents_skills_and_logfire( monkeypatch: pytest.MonkeyPatch, ) -> None: - platforms = {name: _fake_platform(name) for name in ("claude", "codex", "cursor")} + platforms = {name: _fake_platform(name) for name in ("claude", "codex", "cursor", "copilot")} _fake_resolver(monkeypatch, platforms) monkeypatch.setattr("thirdeye.commands.setup.logfire_cmd.is_available", lambda: True) monkeypatch.setattr( @@ -73,7 +73,7 @@ def test_setup_multiselects_agents_skills_and_logfire( def test_setup_can_skip_agent_and_skill_multiselects( monkeypatch: pytest.MonkeyPatch, ) -> None: - platforms = {name: _fake_platform(name) for name in ("claude", "codex", "cursor")} + platforms = {name: _fake_platform(name) for name in ("claude", "codex", "cursor", "copilot")} _fake_resolver(monkeypatch, platforms) monkeypatch.setattr("thirdeye.commands.setup.logfire_cmd.is_available", lambda: False) @@ -89,7 +89,7 @@ def test_setup_can_skip_agent_and_skill_multiselects( def test_setup_skips_entire_logfire_step_when_extra_is_missing( monkeypatch: pytest.MonkeyPatch, ) -> None: - platforms = {name: _fake_platform(name) for name in ("claude", "codex", "cursor")} + platforms = {name: _fake_platform(name) for name in ("claude", "codex", "cursor", "copilot")} _fake_resolver(monkeypatch, platforms) monkeypatch.setattr("thirdeye.commands.setup.logfire_cmd.is_available", lambda: False) @@ -104,7 +104,8 @@ def test_setup_does_not_ask_about_already_configured_agents( monkeypatch: pytest.MonkeyPatch, ) -> None: platforms = { - name: _fake_platform(name, installed=True) for name in ("claude", "codex", "cursor") + name: _fake_platform(name, installed=True) + for name in ("claude", "codex", "cursor", "copilot") } _fake_resolver(monkeypatch, platforms) monkeypatch.setattr("thirdeye.commands.setup._install_new_skills", lambda _: "up to date") @@ -125,6 +126,7 @@ def test_setup_installs_only_new_skills(monkeypatch: pytest.MonkeyPatch) -> None "claude": _fake_platform("claude", installed=True), "codex": _fake_platform("codex"), "cursor": _fake_platform("cursor"), + "copilot": _fake_platform("copilot"), } _fake_resolver(monkeypatch, platforms) monkeypatch.setattr("thirdeye.commands.setup.logfire_cmd.is_available", lambda: False) @@ -151,6 +153,7 @@ def test_setup_offers_to_replace_a_foreign_codex_notifier( "claude": _fake_platform("claude"), "codex": codex, "cursor": _fake_platform("cursor"), + "copilot": _fake_platform("copilot"), } def resolve(name: str, **kwargs: object) -> object: @@ -183,6 +186,7 @@ def test_setup_preserves_foreign_codex_notifier_when_declined( "claude": _fake_platform("claude"), "codex": CodexPlatform(config_file=config_file, hooks_file=hooks_file), "cursor": _fake_platform("cursor"), + "copilot": _fake_platform("copilot"), } _fake_resolver(monkeypatch, platforms) monkeypatch.setattr("thirdeye.commands.setup.logfire_cmd.is_available", lambda: False) @@ -199,7 +203,8 @@ def test_setup_keeps_existing_logfire_token_by_default( monkeypatch: pytest.MonkeyPatch, ) -> None: platforms = { - name: _fake_platform(name, installed=True) for name in ("claude", "codex", "cursor") + name: _fake_platform(name, installed=True) + for name in ("claude", "codex", "cursor", "copilot") } _fake_resolver(monkeypatch, platforms) monkeypatch.setattr("thirdeye.commands.setup._install_new_skills", lambda _: "up to date") @@ -216,7 +221,8 @@ def test_setup_keeps_existing_logfire_token_by_default( def test_setup_can_replace_existing_logfire_token(monkeypatch: pytest.MonkeyPatch) -> None: platforms = { - name: _fake_platform(name, installed=True) for name in ("claude", "codex", "cursor") + name: _fake_platform(name, installed=True) + for name in ("claude", "codex", "cursor", "copilot") } _fake_resolver(monkeypatch, platforms) monkeypatch.setattr("thirdeye.commands.setup._install_new_skills", lambda _: "up to date") @@ -233,7 +239,7 @@ def test_setup_can_replace_existing_logfire_token(monkeypatch: pytest.MonkeyPatc def test_setup_surfaces_logfire_auth_failure(monkeypatch: pytest.MonkeyPatch) -> None: - platforms = {name: _fake_platform(name) for name in ("claude", "codex", "cursor")} + platforms = {name: _fake_platform(name) for name in ("claude", "codex", "cursor", "copilot")} _fake_resolver(monkeypatch, platforms) monkeypatch.setattr("thirdeye.commands.setup.logfire_cmd.is_available", lambda: True) monkeypatch.setattr( @@ -252,7 +258,8 @@ def test_setup_can_enable_an_existing_disabled_logfire_token( monkeypatch: pytest.MonkeyPatch, ) -> None: platforms = { - name: _fake_platform(name, installed=True) for name in ("claude", "codex", "cursor") + name: _fake_platform(name, installed=True) + for name in ("claude", "codex", "cursor", "copilot") } _fake_resolver(monkeypatch, platforms) monkeypatch.setattr("thirdeye.commands.setup._install_new_skills", lambda _: "up to date") @@ -284,3 +291,17 @@ def test_skill_targets_follow_configured_agents(platforms: list[str], targets: l from thirdeye.commands.setup import _skill_targets assert _skill_targets(platforms) == targets + + +def test_setup_can_install_copilot_tracing(monkeypatch: pytest.MonkeyPatch) -> None: + platforms = {name: _fake_platform(name) for name in ("claude", "codex", "cursor", "copilot")} + platforms["copilot"].display_name = "GitHub Copilot CLI" + _fake_resolver(monkeypatch, platforms) + monkeypatch.setattr("thirdeye.commands.setup.logfire_cmd.is_available", lambda: False) + + result = CliRunner().invoke(main, ["setup"], input="4\nnone\n") + + assert result.exit_code == 0, result.output + platforms["copilot"].install.assert_called_once() + platforms["claude"].install.assert_not_called() + assert "Installed tracing for GitHub Copilot CLI" in result.output diff --git a/tests/shared/test_turns.py b/tests/shared/test_turns.py index a81d7a41..b61cdaad 100644 --- a/tests/shared/test_turns.py +++ b/tests/shared/test_turns.py @@ -1,5 +1,12 @@ from __future__ import annotations +from tests.shared.copilot_projection_fixtures import ( + TURN_ONE_ID, + TURN_RECORD_KEYS, + TURN_THREE_ID, + TURN_TWO_ID, + seed_two_main_interaction_projection, +) from thirdeye.config import Config from thirdeye.store import Store from thirdeye.turns import filter_turns, session_turns @@ -60,6 +67,44 @@ def test_turn_query_searches_every_session_and_ands_terms_within_turn(tmp_path): assert [turn["session_id"] for turn in matches] == ["first-session"] +def test_copilot_session_turns_use_projected_main_interactions_not_child_slices(tmp_path): + config = Config(root=tmp_path / "thirdeye") + store = Store(config) + stored_id = seed_two_main_interaction_projection(config, tmp_path) + meta = store.get_meta(stored_id) + + turns = session_turns(meta, store) + + assert [turn["turn_id"] for turn in turns] == [TURN_ONE_ID, TURN_TWO_ID] + assert TURN_THREE_ID not in [turn["turn_id"] for turn in turns] + for turn in turns: + assert [key for key in TURN_RECORD_KEYS if key not in turn] == [] + assert turn["id"] == f"{stored_id}:{turn['turn_id']}" + assert turn["session_id"] == stored_id + assert turn["platform"] == "copilot" + assert turn["cwd"] == "/fixture/workspace" + assert turn["start_seq"] is not None + assert turn["end_seq"] is not None + assert turn["start_ts"] + assert turn["end_ts"] + assert isinstance(turn["events"], list) + assert turns[0]["start_seq"] == 0 + assert turns[0]["end_seq"] == 2 + assert turns[1]["start_seq"] == 3 + assert turns[1]["end_seq"] == 4 + assert len(turns[0]["events"]) == 3 + assert len(turns[1]["events"]) == 2 + assert ( + "alpha.txt" in turns[0]["events"][0]["data"]["source_record"]["payload"]["data"]["content"] + ) + assert ( + "final sum" in turns[1]["events"][0]["data"]["source_record"]["payload"]["data"]["content"] + ) + assert filter_turns([meta], store, query="alpha.txt") == [turns[0]] + assert filter_turns([meta], store, query="final sum") == [turns[1]] + assert filter_turns([meta], store, query="alpha.txt,final sum") == [] + + def test_turn_query_terms_cannot_match_across_different_turns(tmp_path): store = Store(Config(root=tmp_path)) with store.open_session("session", platform="claude", cwd="/project") as writer: diff --git a/tests/web/test_copilot_capture_views.py b/tests/web/test_copilot_capture_views.py new file mode 100644 index 00000000..800ea500 --- /dev/null +++ b/tests/web/test_copilot_capture_views.py @@ -0,0 +1,374 @@ +"""Copilot web views: raw archive evidence and projected turn/usage surfaces.""" + +from __future__ import annotations + +import sys +from pathlib import Path +from types import ModuleType + +import pytest + +from tests.shared.copilot_projection_fixtures import ( + TURN_ONE_ID, + TURN_THREE_ID, + TURN_TWO_ID, + USAGE_MODEL_ONE, + USAGE_MODEL_TWO, + USAGE_TOKENS_ONE, + USAGE_TOKENS_TWO, + seed_observed_six_call_projection, + seed_two_main_interaction_projection, +) +from thirdeye.config import LogfireSettings +from thirdeye.platforms.copilot.archive import commit_batch +from thirdeye.platforms.copilot.identity import resolve_sources, stored_session_id +from thirdeye.platforms.copilot.types import SourceBatch, SourceRecord +from thirdeye.turns import session_turns + +pytest.importorskip("starlette") + +NATIVE_ID = "5a7e8e11-4a6b-49ff-a33e-95d411c4cdd6" +SOURCE_TS = "2026-09-10T17:08:44.557Z" +USAGE_MODEL = "gpt-4.1-copilot-sentinel" +USAGE_INPUT_TOKENS = 424242 + + +def _capture_synthetic_batch(web_config, tmp_path: Path) -> str: + """Archive labeled synthetic child evidence; no turn projection is involved.""" + paths = resolve_sources(tmp_path / "copilot-home") + records: list[SourceRecord] = [ + { + "source_id": f"{paths['source_key']}/{NATIVE_ID}/child-message", + "source_kind": "transcript", + "native_session_id": NATIVE_ID, + "ts": SOURCE_TS, + "observed_at": "2026-09-10T17:08:45.000Z", + "payload": { + "type": "user.message", + "id": "child-message", + "timestamp": SOURCE_TS, + "agentId": "child-agent-id", + "data": {"content": "Read alpha.txt and beta.txt as the explore child."}, + }, + "locator": { + "file": "events.jsonl", + "file_generation": "fixture-gen", + "byte_offset": 512, + }, + }, + { + "source_id": f"hook/{NATIVE_ID}/child-stop", + "source_kind": "hook", + "native_session_id": NATIVE_ID, + "ts": SOURCE_TS, + "observed_at": "2026-09-10T17:08:45.001Z", + "payload": { + "event": "agentStop", + "hook_payload": { + "sessionId": NATIVE_ID, + "agentId": "child-agent-id", + "response": "42", + }, + }, + "locator": {"observation_id": "child-stop", "event": "agentStop"}, + }, + ] + batch: SourceBatch = { + "source_key": paths["source_key"], + "native_session_id": NATIVE_ID, + "cwd": "/fixture/workspace", + "records": records, + "next_cursor": {"fixture": 1}, + "diagnostics": [], + } + commit_batch(web_config, paths, batch) + return stored_session_id(paths, NATIVE_ID) + + +def test_generic_event_views_show_raw_child_and_hook_evidence( + client, web_config, tmp_path: Path +) -> None: + stored_id = _capture_synthetic_batch(web_config, tmp_path) + + session = client.get(f"/sessions/{stored_id}") + tree = client.get(f"/sessions/{stored_id}/tree") + detail = client.get(f"/sessions/{stored_id}/events/1") + search = client.get("/search?q=alpha.txt&platform=copilot") + + assert ( + session.status_code == tree.status_code == detail.status_code == search.status_code == 200 + ) + assert b"copilot" in session.content + assert b"copilot_transcript" in tree.content + assert b"copilot_hook" in tree.content + assert b"user_message" not in tree.content + assert b"tool_call" not in tree.content + assert b"child-agent-id" in detail.content + assert NATIVE_ID.encode() in detail.content + assert SOURCE_TS.encode() in detail.content + assert b"child-message" in detail.content or b"child-stop" in detail.content + assert b'"schema_version": 1' in detail.content + assert b'"source_kind": "hook"' in detail.content + assert stored_id.encode() in search.content + assert b"alpha.txt" in search.content + + +def test_copilot_database_events_render_in_generic_tree(client, web_config, tmp_path: Path) -> None: + paths = resolve_sources(tmp_path / "copilot-home") + records: list[SourceRecord] = [ + { + "source_id": f"{paths['source_key']}/{NATIVE_ID}/usage-row", + "source_kind": "database", + "native_session_id": NATIVE_ID, + "ts": SOURCE_TS, + "observed_at": "2026-09-10T17:08:45.000Z", + "payload": { + "table": "assistant_usage_events", + "model": "gpt-4.1", + "input_tokens": 100, + }, + "locator": {"table": "assistant_usage_events", "rowid": 7}, + }, + { + "source_id": f"{paths['source_key']}/{NATIVE_ID}/workspace-meta", + "source_kind": "metadata", + "native_session_id": NATIVE_ID, + "ts": None, + "observed_at": "2026-09-10T17:08:45.001Z", + "payload": { + "file": "workspace.yaml", + "cwd": "/fixture/workspace", + }, + "locator": {"file": "workspace.yaml"}, + }, + ] + batch: SourceBatch = { + "source_key": paths["source_key"], + "native_session_id": NATIVE_ID, + "cwd": "/fixture/workspace", + "records": records, + "next_cursor": {"fixture": 1}, + "diagnostics": [], + } + commit_batch(web_config, paths, batch) + stored_id = stored_session_id(paths, NATIVE_ID) + + tree = client.get(f"/sessions/{stored_id}/tree") + detail_db = client.get(f"/sessions/{stored_id}/events/0") + detail_meta = client.get(f"/sessions/{stored_id}/events/1") + + assert tree.status_code == detail_db.status_code == detail_meta.status_code == 200 + assert b"copilot_database" in tree.content + assert b"copilot_metadata" in tree.content + assert b"assistant_usage_events" in detail_db.content + assert b'"schema_version": 1' in detail_db.content + assert b'"source_kind": "database"' in detail_db.content + assert b"workspace.yaml" in detail_meta.content + assert b'"source_kind": "metadata"' in detail_meta.content + assert b"user_message" not in tree.content + + +def test_copilot_sessions_are_excluded_from_index_turn_query( + client, web_config, tmp_path: Path +) -> None: + stored_id = _capture_synthetic_batch(web_config, tmp_path) + store = client.app.state.store + meta = store.get_meta(stored_id) + + assert session_turns(meta, store) == [] + + without_turn_filter = client.get("/?platform=copilot&since=2020-01-01") + with_turn_query = client.get("/?platform=copilot&since=2020-01-01&turn_query=alpha.txt") + + assert without_turn_filter.status_code == with_turn_query.status_code == 200 + assert stored_id.encode() in without_turn_filter.content + assert stored_id.encode() not in with_turn_query.content + + +def test_copilot_projected_turns_participate_in_index_turn_query( + client, web_config, tmp_path: Path +) -> None: + stored_id = seed_two_main_interaction_projection(web_config, tmp_path) + store = client.app.state.store + meta = store.get_meta(stored_id) + turns = session_turns(meta, store) + + assert [turn["turn_id"] for turn in turns] == [TURN_ONE_ID, TURN_TWO_ID] + + first_turn_query = client.get("/?platform=copilot&since=2020-01-01&turn_query=alpha.txt") + second_turn_query = client.get("/?platform=copilot&since=2020-01-01&turn_query=final%20sum") + cross_turn_query = client.get( + "/?platform=copilot&since=2020-01-01&turn_query=alpha.txt,final%20sum" + ) + + assert first_turn_query.status_code == second_turn_query.status_code == 200 + assert stored_id.encode() in first_turn_query.content + assert stored_id.encode() in second_turn_query.content + assert stored_id.encode() not in cross_turn_query.content + + +def test_copilot_session_usage_page_shows_projected_usage_rows( + client, web_config, tmp_path: Path +) -> None: + stored_id = seed_two_main_interaction_projection(web_config, tmp_path) + + usage = client.get(f"/sessions/{stored_id}/usage") + + assert usage.status_code == 200 + body = usage.text + tbody = body.split("", 1)[1].split("", 1)[0] + assert tbody.count("") == 2 + assert USAGE_MODEL_ONE in tbody + assert USAGE_MODEL_TWO in tbody + assert str(USAGE_TOKENS_ONE) in tbody + assert str(USAGE_TOKENS_TWO) in tbody + assert str(USAGE_TOKENS_TWO * 2) not in body + + +def test_copilot_session_usage_page_labels_native_billing_and_latency( + client, web_config, tmp_path: Path +) -> None: + stored_id = seed_observed_six_call_projection(web_config, tmp_path) + + usage = client.get(f"/sessions/{stored_id}/usage") + + assert usage.status_code == 200 + body = usage.text + assert "copilot native billing (nano-AIU)" in body + assert "duration_ms" in body + assert "TTFT (ms)" in body + assert "174125000" in body + tbody = body.split("", 1)[1].split("", 1)[0] + assert tbody.count("") == 6 + assert "operation.cost" not in body + assert "$" not in tbody + + +def test_copilot_projected_session_search_tag_and_eval_routes( + client, app, web_config, tmp_path: Path, monkeypatch +) -> None: + stored_id = seed_two_main_interaction_projection(web_config, tmp_path) + app.state.config = app.state.config.write_logfire_settings( + LogfireSettings(api_key="dataset-key") + ) + added: list[dict] = [] + + class Client: + def __init__(self, api_key): + pass + + def __enter__(self): + return self + + def __exit__(self, *args): + return None + + def create_dataset(self, **kwargs): + pass + + def add_cases(self, name, *, cases): + added.extend(cases) + + package = ModuleType("logfire") + experimental = ModuleType("logfire.experimental") + api_client = ModuleType("logfire.experimental.api_client") + api_client.LogfireAPIClient = Client + monkeypatch.setitem(sys.modules, "logfire", package) + monkeypatch.setitem(sys.modules, "logfire.experimental", experimental) + monkeypatch.setitem(sys.modules, "logfire.experimental.api_client", api_client) + + session = client.get(f"/sessions/{stored_id}") + tree = client.get(f"/sessions/{stored_id}/tree") + detail = client.get(f"/sessions/{stored_id}/events/0") + search = client.get("/search?q=alpha.txt&platform=copilot") + tagged = client.post(f"/sessions/{stored_id}/events/0/tags", data={"tag": "review"}) + tagged_index = client.get("/?tag=review&since=2020-01-01") + untagged_index = client.get("/?tag=missing-tag&since=2020-01-01") + export = client.post( + "/sessions/logfire-dataset", + data={ + "dataset_name": "copilot-turns", + "dataset_scope": "turn", + "platform": "copilot", + "since": "2020-01-01", + }, + ) + + assert session.status_code == tree.status_code == search.status_code == 200 + assert detail.status_code == 200 + assert b"copilot" in session.content + assert b"/fixture/workspace" in session.content + assert b"copilot_transcript" in tree.content + assert b"user_message" not in tree.content + assert b"tool_call" not in tree.content + assert b"alpha.txt" in detail.content + assert b'"schema_version": 1' in detail.content + assert stored_id.encode() in search.content + assert b"alpha.txt" in search.content + assert tagged.status_code == 200 + assert b"review" in tagged.content + assert tagged_index.status_code == untagged_index.status_code == 200 + assert stored_id.encode() in tagged_index.content + assert stored_id.encode() not in untagged_index.content + + assert export.status_code == 200 + assert "Sent 2 turns" in export.text + names = [case["name"] for case in added] + assert names == [ + f"{stored_id}:{TURN_ONE_ID}", + f"{stored_id}:{TURN_TWO_ID}", + ] + assert TURN_THREE_ID not in "".join(names) + assert not any(case["name"].endswith(":child") for case in added) + first, second = added + assert "turn" in first["inputs"] and "turn" in second["inputs"] + assert len(first["inputs"]["turn"]["events"]) == 3 + assert len(second["inputs"]["turn"]["events"]) == 2 + event_types = {event.get("t") for case in added for event in case["inputs"]["turn"]["events"]} + assert event_types <= { + "copilot_transcript", + "copilot_database", + "copilot_hook", + "copilot_metadata", + } + assert "user_message" not in event_types + assert "tool_call" not in event_types + + +def test_copilot_session_usage_page_has_no_token_rows_without_projection( + client, web_config, tmp_path: Path +) -> None: + paths = resolve_sources(tmp_path / "copilot-home") + records: list[SourceRecord] = [ + { + "source_id": f"{paths['source_key']}/{NATIVE_ID}/usage-row", + "source_kind": "database", + "native_session_id": NATIVE_ID, + "ts": SOURCE_TS, + "observed_at": "2026-09-10T17:08:45.000Z", + "payload": { + "table": "assistant_usage_events", + "model": USAGE_MODEL, + "input_tokens": USAGE_INPUT_TOKENS, + }, + "locator": {"table": "assistant_usage_events", "rowid": 7}, + }, + ] + batch: SourceBatch = { + "source_key": paths["source_key"], + "native_session_id": NATIVE_ID, + "cwd": "/fixture/workspace", + "records": records, + "next_cursor": {"fixture": 1}, + "diagnostics": [], + } + commit_batch(web_config, paths, batch) + stored_id = stored_session_id(paths, NATIVE_ID) + + usage = client.get(f"/sessions/{stored_id}/usage") + + assert usage.status_code == 200 + assert b"usage" in usage.content + assert USAGE_MODEL.encode() not in usage.content + assert b"input_tokens" not in usage.content + assert str(USAGE_INPUT_TOKENS).encode() not in usage.content diff --git a/tests/web/test_routes_usage.py b/tests/web/test_routes_usage.py index 7d2e7557..c6e27006 100644 --- a/tests/web/test_routes_usage.py +++ b/tests/web/test_routes_usage.py @@ -79,17 +79,36 @@ def test_session_usage_404(client): assert r.status_code == 404 -def test_global_platform_filter_only_claude_and_codex(client): - """The platform filter offers only claude and codex — gemini is gone.""" +def test_global_platform_filter_includes_copilot(client): + """The platform filter lists supported agents, including copilot.""" r = client.get("/usage") assert r.status_code == 200 body = r.text assert 'value="claude"' in body assert 'value="codex"' in body + assert 'value="cursor"' in body + assert 'value="copilot"' in body assert 'value="gemini"' not in body assert ">gemini<" not in body +def test_global_usage_platform_filter_shows_copilot_rows(client, web_config, tmp_path): + from tests.shared.copilot_projection_fixtures import ( + USAGE_TOKENS_ONE, + USAGE_TOKENS_TWO, + seed_two_main_interaction_projection, + ) + + seed_two_main_interaction_projection(web_config, tmp_path) + r = client.get("/usage?platform=copilot&since=2026-09-01&until=2026-09-30") + assert r.status_code == 200 + assert 'value="copilot"' in r.text + assert "selected>copilot<" in r.text.replace("\n", "") + assert str(USAGE_TOKENS_ONE + USAGE_TOKENS_TWO) in r.text + assert str(USAGE_TOKENS_TWO * 2) not in r.text + assert "copilot native billing (nano-AIU)" in r.text + + def test_session_usage_renders_per_call_rows_with_model(client, web_config): """Per-call rows show the response_model in a model column.""" sd = _make_session( diff --git a/uv.lock b/uv.lock index 68856f5f..aa2c6bd2 100644 --- a/uv.lock +++ b/uv.lock @@ -341,6 +341,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/33/6b/e0547afaf41bf2c42e52430072fa5658766e3d65bd4b03a563d1b6336f57/distlib-0.4.0-py2.py3-none-any.whl", hash = "sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16", size = 469047, upload-time = "2025-07-17T16:51:58.613Z" }, ] +[[package]] +name = "execnet" +version = "2.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/89/780e11f9588d9e7128a3f87788354c7946a9cbb1401ad38a48c4db9a4f07/execnet-2.1.2.tar.gz", hash = "sha256:63d83bfdd9a23e35b9c6a3261412324f964c2ec8dcd8d3c6916ee9373e0befcd", size = 166622, upload-time = "2025-11-12T09:56:37.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708, upload-time = "2025-11-12T09:56:36.333Z" }, +] + [[package]] name = "executing" version = "2.2.1" @@ -1081,6 +1090,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" }, ] +[[package]] +name = "pytest-xdist" +version = "3.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "execnet" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/b4/439b179d1ff526791eb921115fca8e44e596a13efeda518b9d845a619450/pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1", size = 88069, upload-time = "2025-07-01T13:30:59.346Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396, upload-time = "2025-07-01T13:30:56.632Z" }, +] + [[package]] name = "python-discovery" version = "1.2.2" @@ -1242,6 +1264,7 @@ dev = [ { name = "pre-commit" }, { name = "pytest" }, { name = "pytest-cov" }, + { name = "pytest-xdist" }, { name = "ruff" }, ] logfire = [ @@ -1268,6 +1291,7 @@ requires-dist = [ { name = "pyaml", specifier = ">=23.0" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0" }, { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=5.0" }, + { name = "pytest-xdist", marker = "extra == 'dev'", specifier = ">=3.5" }, { name = "python-multipart", marker = "extra == 'ui'", specifier = ">=0.0.20" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.7" }, { name = "starlette", marker = "extra == 'ui'", specifier = ">=0.36" },