From 0a8eb643ec9e52349b9772a89478c0d768bcf7a6 Mon Sep 17 00:00:00 2001 From: Sinity Date: Wed, 19 Aug 2026 02:16:02 +0200 Subject: [PATCH 01/10] chore(devtools): remove catalog entries for tranche-1 fossil commands Removes CommandSpec entries for claim_vs_evidence, turso_probe, proof_world_real_slice, help_latency_probe, temporal_read_profile, and temporal_archive_aggregates ahead of deleting their implementations. resume_ranking_eval was never catalog-registered. Ref .agent/campaigns/2026-08-overhaul/ws-c-devtools-pruning.md --- devtools/command_catalog.py | 98 ------------------------------------- 1 file changed, 98 deletions(-) diff --git a/devtools/command_catalog.py b/devtools/command_catalog.py index 6cf37775b4..7bdc343c97 100644 --- a/devtools/command_catalog.py +++ b/devtools/command_catalog.py @@ -560,25 +560,6 @@ def to_dict(self) -> dict[str, object]: "devtools workspace bead-reimport-guard export /tmp/issues-snapshot.jsonl", ), ), - CommandSpec( - "demo real-slice-screen", - "workspace", - "Read-only extraction + privacy screening of a candidate real-archive session slice.", - "devtools.proof_world_real_slice", - use_when=( - "Assembling a candidate real-archive slice for the shared demo proof world " - "(polylogue-212.11): pulls sessions read-only via the Polylogue API, flattens them " - "to text, and screens for secret/credential and PII-adjacent patterns before any " - "operator decides to fold the slice into a shared fixture. Never mutates the source " - "archive and never writes into polylogue/scenarios/ on its own." - ), - examples=( - "devtools demo real-slice-screen --archive-root /realm/state/polylogue " - "--session claude-code-session:: --out .agent/scratch/real-slice", - "devtools demo real-slice-screen --archive-root /realm/state/polylogue " - "--refs-file refs.txt --out .agent/scratch/real-slice", - ), - ), CommandSpec( "workspace dev-loop", "workspace", @@ -1099,36 +1080,6 @@ def to_dict(self) -> dict[str, object]: "--backup-manifest /realm/staging/polylogue-backup/manifest.json", ), ), - CommandSpec( - "workspace temporal-read-profile", - "workspace", - "Measure read --view temporal phase timings on the active archive.", - "devtools.temporal_read_profile", - use_when=( - "Profile the shared temporal read-view builder before tuning query, projection, or rendering paths. " - "The command emits phase timings plus the temporal window summary and can write the report as a " - "dogfood/demo artifact." - ), - examples=( - "devtools workspace temporal-read-profile --query repo:polylogue --limit 1 --json", - "devtools workspace temporal-read-profile --query 'repo:polylogue devloop' --limit 3 --out .local/temporal-profile.json", - ), - ), - CommandSpec( - "workspace temporal-archive-aggregates", - "workspace", - "Build run-projection aggregate artifacts from the active archive.", - "devtools.temporal_archive_aggregates", - use_when=( - "Refresh private longitudinal run/observed-event/context-snapshot demo artifacts from " - "the canonical archive through one reusable command instead of copying raw sqlite3 " - "queries into README files." - ), - examples=( - "devtools workspace temporal-archive-aggregates --json", - "devtools workspace temporal-archive-aggregates --out-dir .local/temporal-archive-aggregates", - ), - ), CommandSpec( "workspace lineage-validation", "workspace", @@ -1160,21 +1111,6 @@ def to_dict(self) -> dict[str, object]: "devtools workspace affordance-usage --out-dir .local/evidence/agent-affordance-usage", ), ), - CommandSpec( - "workspace claim-vs-evidence", - "workspace", - "Analyze structured failures and the assistant behavior that followed.", - "devtools.claim_vs_evidence", - use_when=( - "Measure bounded, origin-stratified follow-up behavior from structural tool-result failures; " - "the report preserves ambiguous outcomes, calibration, sensitivity windows, and separate " - "usage/cost lanes instead of treating prose as the failure oracle." - ), - examples=( - "devtools workspace claim-vs-evidence --json", - "devtools workspace claim-vs-evidence --limit 5000 --out-dir .local/evidence/claim-vs-evidence", - ), - ), CommandSpec( "workspace degraded-archive-proof", "workspace", @@ -1233,24 +1169,6 @@ def to_dict(self) -> dict[str, object]: "devtools bench slo --skip-benchmarks --json", ), ), - CommandSpec( - "bench help-latency", - "benchmarking", - "Check `--help` wall-clock latency against the interactive-tier cold-CLI budget (polylogue-20d.2).", - "devtools.help_latency_probe", - use_when=( - "Catch CLI import-tax regressions continuously. Runs `polylogue --help` for a curated " - "set of root and nested subcommands as fresh subprocesses and compares the minimum wall time " - "against the 700ms cold-CLI budget from the 20d.14 interactive SLO tier. Fails when any " - "'required' target exceeds budget; 'informational' targets (currently `ops maintenance`, " - "known slow pending a lazy-import refactor) are reported but never block." - ), - examples=( - "devtools bench help-latency", - "devtools bench help-latency --json", - "devtools bench help-latency --repeats 5 --out .local/help-latency.json", - ), - ), CommandSpec( "lab policy schema-versioning", "verification lab", @@ -1345,22 +1263,6 @@ def to_dict(self) -> dict[str, object]: "devtools lab probe pipeline --input-mode archive-subset --capture-regression live-parse-drift", ), ), - CommandSpec( - "lab probe turso", - "verification lab", - "Probe Turso Database compatibility against Polylogue storage assumptions.", - "devtools.turso_probe", - use_when=( - "Collect executable evidence before changing production storage backends: " - "Python binding availability, generated-column support, FTS compatibility, MVCC, CDC, " - "vector functions, ATTACH, and WAL pragma behavior." - ), - examples=( - "devtools lab probe turso --json", - "devtools lab probe turso --check", - "devtools lab probe turso --tursodb /nix/store/.../bin/tursodb --json", - ), - ), CommandSpec( "bench memory", "benchmarking", From 90f9569837d2466e793302bdd791b3c1ed5d7873 Mon Sep 17 00:00:00 2001 From: Sinity Date: Wed, 19 Aug 2026 02:16:42 +0200 Subject: [PATCH 02/10] chore(devtools): delete claim_vs_evidence harness (closed campaign) The polylogue-sru campaign closed with terminal artifacts 2026-07-06; this CLI re-proved a claim that has already been made and recorded. PublicClaimProjection (the durable production primitive it wrote through) remains untouched in polylogue/insights/measurement/. Trims the finding doc's now-dead reproduction/regeneration sections to reflect the retired harness while keeping the finding's substantive text as a frozen historical record. Ref .agent/campaigns/2026-08-overhaul/ws-c-devtools-pruning.md --- devtools/claim_vs_evidence.py | 1745 ----------------- devtools/claim_vs_evidence_evidence.py | 348 ---- docs/findings/claim-vs-evidence.md | 65 +- tests/unit/devtools/test_claim_vs_evidence.py | 850 -------- .../test_claim_vs_evidence_evidence.py | 305 --- 5 files changed, 23 insertions(+), 3290 deletions(-) delete mode 100644 devtools/claim_vs_evidence.py delete mode 100644 devtools/claim_vs_evidence_evidence.py delete mode 100644 tests/unit/devtools/test_claim_vs_evidence.py delete mode 100644 tests/unit/devtools/test_claim_vs_evidence_evidence.py diff --git a/devtools/claim_vs_evidence.py b/devtools/claim_vs_evidence.py deleted file mode 100644 index 6a2beca24a..0000000000 --- a/devtools/claim_vs_evidence.py +++ /dev/null @@ -1,1745 +0,0 @@ -"""Build a focused claim-vs-evidence report from structured failures.""" - -from __future__ import annotations - -import argparse -import csv -import json -import random -import sys -from collections.abc import Iterable, Mapping -from contextlib import closing -from datetime import UTC, datetime -from pathlib import Path -from sqlite3 import Connection -from typing import Any - -from polylogue.archive.actions.followup import classify_failed_followup_evidence -from polylogue.config import Config, active_archive_root, get_config -from polylogue.storage.index_generation import RebuildLease -from polylogue.storage.sqlite.connection_profile import open_readonly_connection - -_WORDLESS_CONTINUATION_TEXT_CHAR_LIMIT = 40 -_COUNT_KEYS = ( - "failed_outcomes", - "acknowledged", - "silent_proceed", - "ambiguous", - "ambiguous_wordless_continuation", - "ambiguous_prose_no_marker", -) - -# This is a methodology split, not a truth claim about any single tool result. -# Read/search tools often fail as part of ordinary path discovery; shell/build/ -# edit tools are closer to consequential failures for a coding-agent workflow. -_BENIGN_RECOVERY_TOOLS = frozenset({"glob", "grep", "ls", "read"}) -_CONSEQUENTIAL_TOOLS = frozenset( - { - "bash", - "edit", - "multi_edit", - "notebook_edit", - "patch", - "run_command", - "shell", - "write", - } -) -_CALIBRATION_LABELS = ("acknowledged", "silent_proceed", "ambiguous") -_CALIBRATION_SAMPLE_FILE = "ack-marker-calibration.sample.csv" -_CALIBRATION_LABELS_FILE = "ack-marker-calibration.labels.csv" -_PUBLIC_SUMMARY_FILE = "public-summary.json" -_PUBLIC_REPRODUCTION_FILE = "PUBLIC_REPRODUCTION.md" -_COLD_READER_GATE_FILE = "COLD_READER_GATE.md" -_DEFAULT_N_MIN = 30 -_CALIBRATION_FIELDS = ( - "sample_id", - "human_label", - "classification", - "classification_reason", - "matched_marker", - "origin", - "model_name", - "tool_name", - "handler_class", - "session_ref", - "tool_result_message_ref", - "next_message_ref", - "next_text_preview", - "next3_classification", - "next3_matched_marker", - "next3_text_preview", -) - - -def _parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser( - prog="devtools workspace claim-vs-evidence", - description="Build a focused report over structured tool failures and the next assistant turn.", - ) - parser.add_argument("--archive-root", type=Path, default=None, help="Override the active archive root.") - parser.add_argument("--out-dir", type=Path, default=None, help="Write report.json, summary.json, and README.md.") - parser.add_argument("--limit", type=int, default=5000, help="Maximum structured failed outcomes to classify.") - parser.add_argument("--sample-limit", type=int, default=30, help="Maximum evidence samples per class.") - parser.add_argument( - "--n-min", - type=int, - default=_DEFAULT_N_MIN, - help="Minimum failures required before a split-cell rate is supported.", - ) - parser.add_argument( - "--calibration-size", - type=int, - default=50, - help="Deterministic stratified marker-calibration sample size to write.", - ) - parser.add_argument( - "--calibration-seed", - type=int, - default=20260703, - help="RNG seed for marker-calibration sample selection.", - ) - parser.add_argument( - "--calibration-labels", - type=Path, - default=None, - help="Optional CSV of human labels. Defaults to ack-marker-calibration.labels.csv in --out-dir when present.", - ) - parser.add_argument( - "--materialize-evidence", - action="store_true", - help=( - "Register this run's structured-failure selection, matched rows, and headline numbers " - "as durable query/result-set/finding evidence in the archive's user tier (polylogue-rxdo.13). " - "Off by default; report generation stays read-only unless explicitly requested, and the " - "materializing form requires exclusive offline writer ownership." - ), - ) - parser.add_argument("--json", action="store_true", help="Emit JSON report to stdout.") - return parser - - -def _config_with_archive_root(config: Config, archive_root: Path | None) -> Config: - if archive_root is None: - return config - resolved = archive_root.expanduser().resolve() - return Config( - archive_root=resolved, - render_root=config.render_root, - sources=config.sources, - db_path=resolved / "index.db", - drive_config=config.drive_config, - index_config=config.index_config, - ) - - -def _report_config(args: argparse.Namespace) -> Config: - """Resolve one file-set authority for both report reads and optional writes.""" - return _config_with_archive_root(get_config(), args.archive_root) - - -def _user_version(conn: Connection) -> int: - row = conn.execute("PRAGMA user_version").fetchone() - return int(row[0]) if row else 0 - - -def _rows(conn: Connection, sql: str, params: Iterable[object] = ()) -> list[dict[str, object]]: - cursor = conn.execute(sql, tuple(params)) - columns = [str(description[0]) for description in cursor.description or ()] - return [dict(zip(columns, row, strict=True)) for row in cursor.fetchall()] - - -def _scalar_int(conn: Connection, sql: str, params: Iterable[object] = ()) -> int: - row = conn.execute(sql, tuple(params)).fetchone() - return int(row[0]) if row is not None and row[0] is not None else 0 - - -def _table_exists(conn: Connection, name: str) -> bool: - return conn.execute("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?", (name,)).fetchone() is not None - - -def _economy_rows( - conn: Connection, - *, - session_ids: tuple[str, ...], - silent_by_origin: Mapping[str, int], -) -> dict[str, list[dict[str, object]]]: - """Return explicitly separate token and money lanes for the sampled harness. - - ``session_model_usage`` is the only rollup source. It is itself populated - from provider usage events with Codex's inclusive input/cache and - output/reasoning fields split into disjoint billable lanes; profile text - token columns are deliberately never consulted here. - """ - if not session_ids or not _table_exists(conn, "session_model_usage"): - return {"by_model": [], "by_origin": []} - placeholders = ", ".join("?" for _ in session_ids) - model_rows = _rows( - conn, - f""" - SELECT - s.origin, - u.model_name, - COUNT(*) AS session_model_rows, - SUM(u.input_tokens) AS input_tokens, - SUM(u.output_tokens) AS output_tokens, - SUM(u.cache_read_tokens) AS cache_read_tokens, - SUM(u.cache_write_tokens) AS cache_write_tokens, - SUM(CASE WHEN u.cost_provenance IN ('priced', 'estimated') THEN COALESCE(u.cost_usd, 0) ELSE 0 END) - AS catalog_cost_usd, - SUM(CASE WHEN u.cost_provenance = 'origin_reported' THEN COALESCE(u.cost_usd, 0) ELSE 0 END) - AS provider_reported_cost_usd - FROM session_model_usage AS u - JOIN sessions AS s ON s.session_id = u.session_id - WHERE u.session_id IN ({placeholders}) - GROUP BY s.origin, u.model_name - ORDER BY s.origin, u.model_name - """, - session_ids, - ) - event_rollups: dict[tuple[str, str], tuple[int, int | None]] = {} - if _table_exists(conn, "session_provider_usage_events"): - for row in _rows( - conn, - f""" - SELECT s.origin, COALESCE(NULLIF(e.model_name, ''), 'unknown') AS model_name, - COUNT(*) AS api_call_count, - MAX(CASE WHEN e.provider_event_type = 'message_usage' THEN 1 ELSE 0 END) - AS has_reasoning_delta, - SUM(CASE WHEN e.provider_event_type = 'message_usage' - THEN COALESCE(e.last_reasoning_output_tokens, 0) ELSE 0 END) - AS reasoning_tokens - FROM session_provider_usage_events AS e - JOIN sessions AS s ON s.session_id = e.session_id - WHERE e.session_id IN ({placeholders}) - GROUP BY s.origin, COALESCE(NULLIF(e.model_name, ''), 'unknown') - """, - session_ids, - ): - has_reasoning_delta = _object_int(row["has_reasoning_delta"]) == 1 - event_rollups[(str(row["origin"]), str(row["model_name"]))] = ( - _object_int(row["api_call_count"]), - _object_int(row["reasoning_tokens"]) if has_reasoning_delta else None, - ) - - def decorate(row: dict[str, object], *, model_name: str | None) -> dict[str, object]: - origin = str(row["origin"]) - input_tokens = _object_int(row["input_tokens"]) - cache_read_tokens = _object_int(row["cache_read_tokens"]) - catalog_cost = _object_float(row["catalog_cost_usd"]) - provider_cost = _object_float(row["provider_reported_cost_usd"]) - silent = silent_by_origin.get(origin, 0) - if model_name is not None: - api_call_count, reasoning_tokens = event_rollups.get((origin, model_name), (0, None)) - else: - origin_events = [event for (event_origin, _model), event in event_rollups.items() if event_origin == origin] - api_call_count = sum(event[0] for event in origin_events) - reasoning_values = [event[1] for event in origin_events if event[1] is not None] - reasoning_tokens = sum(reasoning_values) if reasoning_values else None - return { - "origin": origin, - "model_name": model_name, - "session_model_rows": _object_int(row["session_model_rows"]), - "api_call_count": api_call_count, - "input_tokens": input_tokens, - "output_tokens": _object_int(row["output_tokens"]), - "cache_read_tokens": cache_read_tokens, - "cache_write_tokens": _object_int(row["cache_write_tokens"]), - "reasoning_tokens": reasoning_tokens, - "reasoning_token_note": ( - "provider message_usage deltas only; cumulative token_count events are excluded" - if reasoning_tokens is not None - else "no provider message_usage reasoning delta in the sampled harness" - ), - "cache_read_share": cache_read_tokens / (input_tokens + cache_read_tokens) - if input_tokens + cache_read_tokens - else None, - "catalog_cost_usd": catalog_cost, - "provider_reported_cost_usd": provider_cost, - "silent_proceed_outcomes": silent, - "catalog_usd_per_silent_proceed": catalog_cost / silent if silent else None, - "provider_reported_usd_per_silent_proceed": provider_cost / silent if silent else None, - "token_source": "session_model_usage (provider-usage materialization; disjoint cache lanes)", - } - - by_model = [decorate(row, model_name=str(row["model_name"])) for row in model_rows] - by_origin_source: dict[str, dict[str, object]] = {} - for row in model_rows: - origin = str(row["origin"]) - aggregate = by_origin_source.setdefault( - origin, - { - "origin": origin, - "session_model_rows": 0, - "input_tokens": 0, - "output_tokens": 0, - "cache_read_tokens": 0, - "cache_write_tokens": 0, - "catalog_cost_usd": 0.0, - "provider_reported_cost_usd": 0.0, - }, - ) - for key in ("session_model_rows", "input_tokens", "output_tokens", "cache_read_tokens", "cache_write_tokens"): - aggregate[key] = _object_int(aggregate[key]) + _object_int(row[key]) - for key in ("catalog_cost_usd", "provider_reported_cost_usd"): - aggregate[key] = _object_float(aggregate[key]) + _object_float(row[key]) - return { - "by_model": by_model, - "by_origin": [decorate(row, model_name=None) for _, row in sorted(by_origin_source.items())], - } - - -def _object_int(value: object) -> int: - if value is None: - return 0 - return int(str(value)) - - -def _object_float(value: object) -> float: - return 0.0 if value is None else float(str(value)) - - -def _object_str_list(value: object) -> list[str]: - if isinstance(value, list | tuple): - return [str(item) for item in value] - return [] - - -def _ranked(mapping: dict[str, dict[str, int]], *, n_min: int) -> list[dict[str, object]]: - rows: list[dict[str, object]] = [] - for name, counts in mapping.items(): - failed = counts["failed_outcomes"] - silent = counts["silent_proceed"] - classified = counts["acknowledged"] + silent - supported = failed >= n_min - classified_supported = classified >= n_min - rows.append( - { - "name": name, - **counts, - "classified_outcomes": classified, - "n_min": n_min, - "coverage_status": "supported" if supported else "insufficient_n", - "publication_status": "supported" if supported else "not_supported", - "classified_coverage_status": "supported" if classified_supported else "insufficient_n", - "classified_publication_status": "supported" if classified_supported else "not_supported", - "silent_rate_lower_bound": (silent / failed) if supported else None, - "silent_rate_among_classified": (silent / classified) if classified_supported else None, - } - ) - - def sort_key(row: dict[str, object]) -> tuple[int, str]: - failed = row["failed_outcomes"] - failed_count = failed if isinstance(failed, int) else int(str(failed)) - return (-failed_count, str(row["name"])) - - return sorted(rows, key=sort_key) - - -def _empty_counts() -> dict[str, int]: - return dict.fromkeys(_COUNT_KEYS, 0) - - -def _empty_window_counts() -> dict[str, int]: - return { - "failed_outcomes": 0, - "acknowledged": 0, - "silent_proceed": 0, - "ambiguous": 0, - } - - -def _rate(numerator: int, denominator: int) -> float | None: - if denominator <= 0: - return None - return numerator / denominator - - -def _refine_classification_reason(classification_evidence: Mapping[str, object], row: dict[str, object]) -> str: - reason = str(classification_evidence["reason"]) - if classification_evidence["classification"] != "ambiguous": - return reason - if reason == "missing_next_assistant_message": - return reason - has_tool_use = bool(_object_int(row["next_has_tool_use"])) - pre_tool_text_chars = _object_int(row["next_pre_tool_text_chars"]) - if has_tool_use and pre_tool_text_chars <= _WORDLESS_CONTINUATION_TEXT_CHAR_LIMIT: - return "wordless_tool_continuation" - return "prose_no_marker" - - -def _ambiguous_counter_key(classification_reason: str) -> str | None: - if classification_reason == "wordless_tool_continuation": - return "ambiguous_wordless_continuation" - if classification_reason == "prose_no_marker": - return "ambiguous_prose_no_marker" - return None - - -def _handler_class(tool_name: str) -> str: - normalized = tool_name.strip().lower().replace("-", "_").replace(" ", "_") - if normalized in _BENIGN_RECOVERY_TOOLS: - return "benign_recovery" - if normalized in _CONSEQUENTIAL_TOOLS: - return "consequential" - return "other" - - -def _next_message_details( - conn: Connection, message_ids: Iterable[object], *, chunk_size: int = 500 -) -> dict[str, dict[str, object]]: - ids = [str(message_id) for message_id in message_ids if message_id] - details: dict[str, dict[str, Any]] = {} - for start in range(0, len(ids), chunk_size): - chunk = ids[start : start + chunk_size] - placeholders = ",".join("?" for _ in chunk) - rows = _rows( - conn, - f""" - SELECT - message_id, - position, - block_type, - COALESCE(text, '') AS text - FROM blocks - WHERE message_id IN ({placeholders}) - ORDER BY message_id, position - """, - chunk, - ) - for row in rows: - message_id = str(row["message_id"]) - detail = details.setdefault( - message_id, - { - "next_has_tool_use": 0, - "first_tool_use_position": None, - "next_pre_tool_text_chars": 0, - "text_parts": [], - }, - ) - block_type = str(row["block_type"]) - position = _object_int(row["position"]) - if block_type == "tool_use": - detail["next_has_tool_use"] = 1 - first_tool_use_position = detail["first_tool_use_position"] - if first_tool_use_position is None or position < _object_int(first_tool_use_position): - detail["first_tool_use_position"] = position - elif block_type == "text": - text = str(row["text"] or "") - text_parts = detail["text_parts"] - assert isinstance(text_parts, list) - text_parts.append(text) - first_tool_use_position = detail["first_tool_use_position"] - if first_tool_use_position is None or position < _object_int(first_tool_use_position): - detail["next_pre_tool_text_chars"] = max( - _object_int(detail["next_pre_tool_text_chars"]), - len(text.strip()), - ) - return { - message_id: { - "next_text": "\n".join(str(part) for part in detail["text_parts"])[:1200], - "next_has_tool_use": _object_int(detail["next_has_tool_use"]), - "next_pre_tool_text_chars": _object_int(detail["next_pre_tool_text_chars"]), - } - for message_id, detail in details.items() - } - - -def _failure_outcome_rows(conn: Connection, *, limit: int, origin: str | None) -> list[dict[str, object]]: - origin_predicate = "AND s.origin = ?" if origin is not None else "" - params: tuple[object, ...] = (origin, origin, origin, limit) if origin is not None else (limit,) - return _rows( - conn, - f""" - WITH failed AS ( - SELECT - r.session_id, - r.message_id AS tool_result_message_id, - r.block_id AS tool_result_block_id, - r.tool_id AS tool_result_tool_id, - s.origin, - r.tool_result_is_error AS is_error, - r.tool_result_exit_code AS exit_code, - r.message_id AS order_message_id - FROM blocks AS r INDEXED BY idx_blocks_tool_result_outcome - JOIN sessions AS s ON s.session_id = r.session_id - WHERE r.block_type = 'tool_result' - {origin_predicate} - AND r.tool_result_is_error = 1 - UNION ALL - SELECT - r.session_id, - r.message_id AS tool_result_message_id, - r.block_id AS tool_result_block_id, - r.tool_id AS tool_result_tool_id, - s.origin, - r.tool_result_is_error AS is_error, - r.tool_result_exit_code AS exit_code, - r.message_id AS order_message_id - FROM blocks AS r INDEXED BY idx_blocks_tool_result_outcome - JOIN sessions AS s ON s.session_id = r.session_id - WHERE r.block_type = 'tool_result' - {origin_predicate} - AND r.tool_result_exit_code IS NOT NULL - AND r.tool_result_exit_code != 0 - AND r.tool_result_is_error = 0 - UNION ALL - SELECT - r.session_id, - r.message_id AS tool_result_message_id, - r.block_id AS tool_result_block_id, - r.tool_id AS tool_result_tool_id, - s.origin, - r.tool_result_is_error AS is_error, - r.tool_result_exit_code AS exit_code, - r.message_id AS order_message_id - FROM blocks AS r INDEXED BY idx_blocks_tool_result_outcome - JOIN sessions AS s ON s.session_id = r.session_id - WHERE r.block_type = 'tool_result' - {origin_predicate} - AND r.tool_result_exit_code IS NOT NULL - AND r.tool_result_exit_code != 0 - AND r.tool_result_is_error IS NULL - ) - SELECT * - FROM failed - ORDER BY session_id, tool_result_tool_id, order_message_id, tool_result_block_id - LIMIT ? - """, - params, - ) - - -def _paired_failure_rows( - conn: Connection, - failure_rows: list[dict[str, object]], - *, - chunk_size: int = 250, -) -> list[dict[str, object]]: - paired_rows: list[dict[str, object]] = [] - for start in range(0, len(failure_rows), chunk_size): - chunk = failure_rows[start : start + chunk_size] - placeholders = ",".join("(?, ?, ?, ?, ?, ?, ?, ?, ?)" for _ in chunk) - params: list[object] = [] - for offset, row in enumerate(chunk, start=start): - params.extend( - [ - offset, - row["session_id"], - row["tool_result_message_id"], - row["tool_result_block_id"], - row["tool_result_tool_id"], - row["origin"], - row["is_error"], - row["exit_code"], - row["order_message_id"], - ] - ) - paired_rows.extend( - _rows( - conn, - f""" - WITH wanted( - sort_index, - session_id, - tool_result_message_id, - tool_result_block_id, - tool_result_tool_id, - origin, - is_error, - exit_code, - order_message_id - ) AS ( - VALUES {placeholders} - ), - paired AS ( - SELECT - w.sort_index, - w.session_id, - a.message_id, - w.tool_result_message_id, - w.tool_result_block_id, - w.tool_result_tool_id, - a.tool_name, - a.tool_command, - w.origin, - w.is_error, - w.exit_code, - m.model_name AS tool_message_model, - rm.position AS result_position, - ( - SELECT nm.message_id - FROM messages AS nm - WHERE nm.session_id = w.session_id - AND nm.material_origin = 'assistant_authored' - AND nm.position > rm.position - AND nm.position < COALESCE( - ( - SELECT MIN(next_human.position) - FROM messages AS next_human - WHERE next_human.session_id = w.session_id - AND next_human.material_origin = 'human_authored' - AND next_human.position > rm.position - ), - 9223372036854775807 - ) - ORDER BY nm.position, nm.variant_index, nm.message_id - LIMIT 1 - ) AS next_message_id - FROM wanted AS w - JOIN actions AS a - ON a.session_id = w.session_id - AND a.tool_result_block_id = w.tool_result_block_id - JOIN messages AS m ON m.message_id = a.message_id - JOIN messages AS rm ON rm.message_id = w.tool_result_message_id - ) - SELECT - p.session_id, - p.message_id, - p.tool_result_message_id, - p.tool_result_block_id, - p.tool_result_tool_id, - p.tool_name, - p.tool_command, - p.origin, - p.is_error, - p.exit_code, - p.result_position, - COALESCE(nm.model_name, p.tool_message_model, '') AS model_name, - p.next_message_id - FROM paired AS p - LEFT JOIN messages AS nm ON nm.message_id = p.next_message_id - ORDER BY p.sort_index - """, - params, - ) - ) - return paired_rows - - -def _assistant_window_details( - conn: Connection, - rows: list[dict[str, object]], - *, - window_size: int = 3, - chunk_size: int = 250, -) -> dict[int, dict[str, object]]: - details: dict[int, dict[str, Any]] = {} - for start in range(0, len(rows), chunk_size): - chunk = rows[start : start + chunk_size] - placeholders = ",".join("(?, ?, ?)" for _ in chunk) - params: list[object] = [] - for index, row in enumerate(chunk, start=start): - params.extend([index, row["session_id"], row["result_position"]]) - result_rows = _rows( - conn, - f""" - WITH wanted(sort_index, session_id, result_position) AS ( - VALUES {placeholders} - ), - bounded AS ( - SELECT - w.*, - ( - SELECT MIN(next_user.position) - FROM messages AS next_user - WHERE next_user.session_id = w.session_id - AND next_user.material_origin = 'human_authored' - AND next_user.position > w.result_position - ) AS next_user_position - FROM wanted AS w - ), - assistant_window AS ( - SELECT - b.sort_index, - m.message_id, - m.position, - m.variant_index, - DENSE_RANK() OVER ( - PARTITION BY b.sort_index - ORDER BY m.position - ) AS assistant_rank - FROM bounded AS b - JOIN messages AS m - ON m.session_id = b.session_id - AND m.material_origin = 'assistant_authored' - AND m.position > b.result_position - AND ( - b.next_user_position IS NULL - OR m.position < b.next_user_position - ) - ), - top_window AS ( - SELECT * - FROM assistant_window - WHERE assistant_rank <= ? - ) - SELECT - w.sort_index, - w.message_id, - w.assistant_rank, - b.position AS block_position, - b.block_type, - COALESCE(b.text, '') AS text - FROM top_window AS w - LEFT JOIN blocks AS b ON b.message_id = w.message_id - ORDER BY w.sort_index, w.assistant_rank, w.variant_index, w.message_id, b.position - """, - [*params, window_size], - ) - for result_row in result_rows: - sort_index = _object_int(result_row["sort_index"]) - detail = details.setdefault( - sort_index, - { - "message_ids": [], - "text_parts": [], - }, - ) - message_ids = detail["message_ids"] - text_parts = detail["text_parts"] - assert isinstance(message_ids, list) - assert isinstance(text_parts, list) - message_id = str(result_row["message_id"]) - if message_id not in message_ids: - message_ids.append(message_id) - if str(result_row["block_type"]) == "text": - text_parts.append(str(result_row["text"] or "")) - return { - index: { - "message_ids": detail["message_ids"], - "text": "\n".join(str(part) for part in detail["text_parts"])[:2400], - } - for index, detail in details.items() - } - - -def _structured_failure_rows(conn: Connection, *, limit: int, origin: str | None = None) -> list[dict[str, object]]: - candidate_limit = max(limit * 2, limit + 100) - rows = _paired_failure_rows(conn, _failure_outcome_rows(conn, limit=candidate_limit, origin=origin))[:limit] - details = _next_message_details(conn, (row.get("next_message_id") for row in rows)) - window_details = _assistant_window_details(conn, rows) - for index, row in enumerate(rows): - detail = details.get(str(row.get("next_message_id") or ""), {}) - row["next_text"] = detail.get("next_text", "") - row["next_has_tool_use"] = detail.get("next_has_tool_use", 0) - row["next_pre_tool_text_chars"] = detail.get("next_pre_tool_text_chars", 0) - window_detail = window_details.get(index, {}) - row["next3_message_ids"] = window_detail.get("message_ids", []) - row["next3_text"] = window_detail.get("text", "") - return rows - - -def _unpaired_structured_failure_count(conn: Connection) -> int: - return _scalar_int( - conn, - """ - SELECT COUNT(*) - FROM ( - SELECT r.session_id, r.tool_id - FROM blocks AS r INDEXED BY idx_blocks_tool_result_outcome - WHERE r.block_type = 'tool_result' - AND r.tool_result_is_error = 1 - UNION ALL - SELECT r.session_id, r.tool_id - FROM blocks AS r INDEXED BY idx_blocks_tool_result_outcome - WHERE r.block_type = 'tool_result' - AND r.tool_result_exit_code IS NOT NULL - AND r.tool_result_exit_code != 0 - AND r.tool_result_is_error = 0 - UNION ALL - SELECT r.session_id, r.tool_id - FROM blocks AS r INDEXED BY idx_blocks_tool_result_outcome - WHERE r.block_type = 'tool_result' - AND r.tool_result_exit_code IS NOT NULL - AND r.tool_result_exit_code != 0 - AND r.tool_result_is_error IS NULL - ) AS r - WHERE NOT EXISTS ( - SELECT 1 - FROM blocks AS u INDEXED BY idx_blocks_tool_id - WHERE u.tool_id = r.tool_id - AND u.session_id = r.session_id - AND u.block_type = 'tool_use' - ) - """, - ) - - -def _structured_failure_origin_counts(conn: Connection) -> list[dict[str, object]]: - return _rows( - conn, - """ - WITH failed AS ( - SELECT r.session_id - FROM blocks AS r INDEXED BY idx_blocks_tool_result_outcome - WHERE r.block_type = 'tool_result' - AND r.tool_result_is_error = 1 - UNION ALL - SELECT r.session_id - FROM blocks AS r INDEXED BY idx_blocks_tool_result_outcome - WHERE r.block_type = 'tool_result' - AND r.tool_result_exit_code IS NOT NULL - AND r.tool_result_exit_code != 0 - AND r.tool_result_is_error = 0 - UNION ALL - SELECT r.session_id - FROM blocks AS r INDEXED BY idx_blocks_tool_result_outcome - WHERE r.block_type = 'tool_result' - AND r.tool_result_exit_code IS NOT NULL - AND r.tool_result_exit_code != 0 - AND r.tool_result_is_error IS NULL - ) - SELECT s.origin, COUNT(*) AS failed_outcomes - FROM failed AS r - JOIN sessions AS s ON s.session_id = r.session_id - GROUP BY s.origin - ORDER BY failed_outcomes DESC, s.origin - """, - ) - - -def _origin_sample_limits(total_by_origin: list[dict[str, object]], limit: int) -> list[dict[str, object]]: - origins = [(str(row["origin"]), _object_int(row["failed_outcomes"])) for row in total_by_origin] - origins = [(origin, count) for origin, count in origins if count > 0] - total = sum(count for _, count in origins) - if total <= limit: - return [ - {"origin": origin, "total_structured_failures": count, "requested_limit": count} - for origin, count in origins - ] - if not origins: - return [] - if limit < len(origins): - return [ - {"origin": origin, "total_structured_failures": count, "requested_limit": 1} - for origin, count in origins[:limit] - ] - - allocation = {origin: 1 for origin, _ in origins} - remaining = limit - len(origins) - capacities = {origin: count - 1 for origin, count in origins} - capacity_total = sum(capacities.values()) - remainders: list[tuple[float, int, str]] = [] - if capacity_total: - for index, (origin, _count) in enumerate(origins): - exact = remaining * (capacities[origin] / capacity_total) - extra = min(capacities[origin], int(exact)) - allocation[origin] += extra - remainders.append((exact - extra, -index, origin)) - assigned = sum(allocation.values()) - for _remainder, _negative_index, origin in sorted(remainders, reverse=True): - if assigned >= limit: - break - if allocation[origin] < dict(origins)[origin]: - allocation[origin] += 1 - assigned += 1 - - return [ - {"origin": origin, "total_structured_failures": count, "requested_limit": allocation[origin]} - for origin, count in origins - if allocation[origin] > 0 - ] - - -def _calibration_sort_key(sample: Mapping[str, object]) -> tuple[str, str, str]: - return ( - str(sample["classification"]), - str(sample["session_ref"]), - str(sample["tool_result_message_ref"]), - ) - - -def _calibration_sample( - samples: list[dict[str, object]], - *, - size: int, - seed: int, -) -> list[dict[str, object]]: - if size <= 0: - return [] - by_class: dict[str, list[dict[str, object]]] = {label: [] for label in _CALIBRATION_LABELS} - for sample in samples: - classification = str(sample["classification"]) - if classification in by_class: - by_class[classification].append(sample) - rng = random.Random(seed) - selected: list[dict[str, object]] = [] - target_per_class = max(1, size // len(_CALIBRATION_LABELS)) - for label in _CALIBRATION_LABELS: - bucket = sorted(by_class[label], key=_calibration_sort_key) - take = min(target_per_class, len(bucket)) - if take: - selected.extend(rng.sample(bucket, take)) - if len(selected) < size: - selected_ids = { - ( - str(sample["session_ref"]), - str(sample["tool_result_message_ref"]), - ) - for sample in selected - } - remainder = [ - sample - for sample in sorted(samples, key=_calibration_sort_key) - if ( - str(sample["session_ref"]), - str(sample["tool_result_message_ref"]), - ) - not in selected_ids - ] - selected.extend(rng.sample(remainder, min(size - len(selected), len(remainder)))) - return sorted(selected[:size], key=_calibration_sort_key) - - -def _calibration_row(sample: Mapping[str, object], index: int, *, human_label: str = "") -> dict[str, object]: - row = {field: sample.get(field, "") for field in _CALIBRATION_FIELDS} - row["sample_id"] = f"cal-{index:03d}" - row["human_label"] = human_label - return row - - -def _write_csv(path: Path, rows: list[dict[str, object]]) -> None: - with path.open("w", encoding="utf-8", newline="") as handle: - writer = csv.DictWriter(handle, fieldnames=list(_CALIBRATION_FIELDS), extrasaction="ignore") - writer.writeheader() - writer.writerows(rows) - - -def _read_calibration_labels(path: Path) -> list[dict[str, str]]: - with path.open(encoding="utf-8", newline="") as handle: - return [ - {str(key): str(value or "") for key, value in row.items()} - for row in csv.DictReader(handle) - if row.get("human_label") - ] - - -def _calibration_metrics(label_rows: list[dict[str, str]], *, labels_path: Path | None) -> dict[str, object]: - confusion: dict[str, dict[str, int]] = { - human: dict.fromkeys(_CALIBRATION_LABELS, 0) for human in _CALIBRATION_LABELS - } - invalid_rows = 0 - for row in label_rows: - human = row.get("human_label", "").strip() - predicted = row.get("classification", "").strip() - if human not in confusion or predicted not in _CALIBRATION_LABELS: - invalid_rows += 1 - continue - confusion[human][predicted] += 1 - acknowledged_tp = confusion["acknowledged"]["acknowledged"] - predicted_acknowledged = sum(confusion[human]["acknowledged"] for human in _CALIBRATION_LABELS) - human_acknowledged = sum(confusion["acknowledged"].values()) - usable_rows = sum(sum(row.values()) for row in confusion.values()) - return { - "labels_path": str(labels_path) if labels_path is not None else None, - "labeled_rows": usable_rows, - "invalid_rows": invalid_rows, - "labels": list(_CALIBRATION_LABELS), - "confusion_matrix": confusion, - "ack_marker_precision": _rate(acknowledged_tp, predicted_acknowledged), - "ack_marker_recall": _rate(acknowledged_tp, human_acknowledged), - "ack_marker_true_positive": acknowledged_tp, - "ack_marker_predicted_positive": predicted_acknowledged, - "ack_marker_actual_positive": human_acknowledged, - } - - -def _calibration_frame_coverage(label_rows: list[dict[str, str]], samples: list[dict[str, object]]) -> dict[str, int]: - """State whether labels calibrate this report's actual current frame.""" - sampled_refs = {str(sample["tool_result_message_ref"]) for sample in samples} - labeled_refs = {str(row.get("tool_result_message_ref", "")) for row in label_rows} - matched = sampled_refs & labeled_refs - return { - "labeled_sample_refs": len(labeled_refs), - "current_sample_refs": len(sampled_refs), - "labels_in_current_sample": len(matched), - "labels_outside_current_sample": len(labeled_refs - sampled_refs), - } - - -def _calibration_labels_path(args: argparse.Namespace) -> Path | None: - calibration_labels = args.calibration_labels - if isinstance(calibration_labels, Path): - return calibration_labels - out_dir = args.out_dir - if not isinstance(out_dir, Path): - return None - candidate = out_dir / _CALIBRATION_LABELS_FILE - return candidate if candidate.exists() else None - - -def build_report(args: argparse.Namespace, *, config: Config | None = None) -> dict[str, Any]: - if args.limit < 1: - raise ValueError("--limit must be positive") - if args.sample_limit < 1: - raise ValueError("--sample-limit must be positive") - if args.n_min < 1: - raise ValueError("--n-min must be positive") - if args.calibration_size < 0: - raise ValueError("--calibration-size must be non-negative") - config = config or _report_config(args) - file_set_root = active_archive_root(config) - index_db = config.db_path - conn = open_readonly_connection(index_db) - try: - total_by_origin = _structured_failure_origin_counts(conn) - total_structured_failures = sum(_object_int(row["failed_outcomes"]) for row in total_by_origin) - unpaired_structured_failures = _unpaired_structured_failure_count(conn) - origin_limits = _origin_sample_limits(total_by_origin, args.limit) - rows = [] - sampled_by_origin: list[dict[str, object]] = [] - for origin_limit in origin_limits: - origin = str(origin_limit["origin"]) - requested_limit = _object_int(origin_limit["requested_limit"]) - origin_rows = _structured_failure_rows(conn, limit=requested_limit, origin=origin) - rows.extend(origin_rows) - sampled_by_origin.append( - { - **origin_limit, - "inspected_structured_failures": len(origin_rows), - } - ) - schema_version = _user_version(conn) - finally: - conn.close() - - totals = _empty_counts() - window3_totals = _empty_window_counts() - by_tool: dict[str, dict[str, int]] = {} - by_model: dict[str, dict[str, int]] = {} - by_origin: dict[str, dict[str, int]] = {} - by_handler_class: dict[str, dict[str, int]] = {} - samples_by_classification: dict[str, list[dict[str, object]]] = { - "acknowledged": [], - "silent_proceed": [], - "ambiguous": [], - } - samples_by_origin_classification: dict[str, dict[str, list[dict[str, object]]]] = {} - calibration_candidates: list[dict[str, object]] = [] - for row in rows: - classification_evidence = classify_failed_followup_evidence( - str(row["next_text"]) if row["next_text"] is not None else None - ) - classification = str(classification_evidence["classification"]) - classification_reason = _refine_classification_reason(classification_evidence, row) - next3_message_ids = _object_str_list(row["next3_message_ids"]) - next3_text = str(row["next3_text"] or "") - window3_evidence = classify_failed_followup_evidence(next3_text if next3_message_ids else None) - window3_classification = str(window3_evidence["classification"]) - tool = str(row["tool_name"] or "unknown") - handler_class = _handler_class(tool) - model = str(row["model_name"] or "unknown") - origin = str(row["origin"] or "unknown") - next_text = str(row["next_text"] or "") - sample = { - "classification": classification, - "classification_reason": classification_reason, - "matched_marker": classification_evidence["matched_marker"], - "session_ref": f"session:{row['session_id']}", - "tool_message_ref": f"message:{row['message_id']}", - "tool_result_message_ref": f"message:{row['tool_result_message_id']}", - "tool_result_block_ref": f"block:{row['tool_result_block_id']}", - "tool_result_tool_id": row["tool_result_tool_id"], - "next_message_ref": f"message:{row['next_message_id']}" if row["next_message_id"] else None, - "tool_name": tool, - "handler_class": handler_class, - "model_name": model, - "origin": origin, - "exit_code": row["exit_code"], - "is_error": row["is_error"], - "tool_command_preview": str(row["tool_command"] or "")[:160], - "next_text_preview": next_text[:500], - "next_has_tool_use": bool(_object_int(row["next_has_tool_use"])), - "next_pre_tool_text_chars": _object_int(row["next_pre_tool_text_chars"]), - "next3_message_refs": [f"message:{message_id}" for message_id in next3_message_ids], - "next3_classification": window3_classification, - "next3_classification_reason": window3_evidence["reason"], - "next3_matched_marker": window3_evidence["matched_marker"], - "next3_text_preview": next3_text[:500], - } - totals["failed_outcomes"] += 1 - totals[classification] += 1 - window3_totals["failed_outcomes"] += 1 - window3_totals[window3_classification] += 1 - ambiguous_counter_key = _ambiguous_counter_key(classification_reason) - if ambiguous_counter_key is not None: - totals[ambiguous_counter_key] += 1 - by_tool.setdefault(tool, _empty_counts()) - by_model.setdefault(model, _empty_counts()) - by_origin.setdefault(origin, _empty_counts()) - by_handler_class.setdefault(handler_class, _empty_counts()) - by_tool[tool]["failed_outcomes"] += 1 - by_tool[tool][classification] += 1 - if ambiguous_counter_key is not None: - by_tool[tool][ambiguous_counter_key] += 1 - by_model[model]["failed_outcomes"] += 1 - by_model[model][classification] += 1 - if ambiguous_counter_key is not None: - by_model[model][ambiguous_counter_key] += 1 - by_origin[origin]["failed_outcomes"] += 1 - by_origin[origin][classification] += 1 - if ambiguous_counter_key is not None: - by_origin[origin][ambiguous_counter_key] += 1 - by_handler_class[handler_class]["failed_outcomes"] += 1 - by_handler_class[handler_class][classification] += 1 - if ambiguous_counter_key is not None: - by_handler_class[handler_class][ambiguous_counter_key] += 1 - bucket = samples_by_classification[classification] - if len(bucket) < args.sample_limit: - bucket.append(sample) - origin_buckets = samples_by_origin_classification.setdefault( - origin, - { - "acknowledged": [], - "silent_proceed": [], - "ambiguous": [], - }, - ) - origin_bucket = origin_buckets[classification] - if len(origin_bucket) < args.sample_limit: - origin_bucket.append(sample) - calibration_candidates.append(sample) - totals["classified_outcomes"] = totals["acknowledged"] + totals["silent_proceed"] - window3_totals["classified_outcomes"] = window3_totals["acknowledged"] + window3_totals["silent_proceed"] - silent = totals["silent_proceed"] - failed = totals["failed_outcomes"] - classified = totals["classified_outcomes"] - aggregate_supported = failed >= args.n_min - window3_silent = window3_totals["silent_proceed"] - window3_classified = window3_totals["classified_outcomes"] - calibration_sample = _calibration_sample( - calibration_candidates, - size=args.calibration_size, - seed=args.calibration_seed, - ) - calibration_labels_path = _calibration_labels_path(args) - calibration_label_rows = ( - _read_calibration_labels(calibration_labels_path) - if calibration_labels_path is not None and calibration_labels_path.exists() - else [] - ) - calibration = { - "sample_size_requested": args.calibration_size, - "sample_size": len(calibration_sample), - "sample_seed": args.calibration_seed, - "sample_file": _CALIBRATION_SAMPLE_FILE if args.out_dir is not None else None, - "labels_file": _CALIBRATION_LABELS_FILE if args.out_dir is not None else None, - "metrics": _calibration_metrics(calibration_label_rows, labels_path=calibration_labels_path), - "frame_coverage": _calibration_frame_coverage(calibration_label_rows, calibration_sample), - } - silent_by_origin = {origin: counts["silent_proceed"] for origin, counts in by_origin.items()} - sampled_session_ids = tuple(sorted({str(row["session_id"]) for row in rows})) - with closing(open_readonly_connection(index_db)) as economy_conn: - economy = _economy_rows( - economy_conn, - session_ids=sampled_session_ids, - silent_by_origin=silent_by_origin, - ) - report: dict[str, Any] = { - "report_version": 1, - "captured_at": datetime.now(UTC).isoformat(), - "command": "devtools workspace claim-vs-evidence", - "archive_root": str(file_set_root), - "index_db": str(index_db), - "index_schema_version": schema_version, - "limit": args.limit, - "sample_frame": { - "total_structured_failures": total_structured_failures, - "unpaired_structured_failures": unpaired_structured_failures, - "inspected_structured_failures": len(rows), - "limit": args.limit, - "time_window": "entire archive (no since/until filter)", - "complete_failure_frame": len(rows) >= total_structured_failures, - "selection_strategy": ( - "origin-stratified bounded sample; at least one row per origin when limit allows, " - "then proportional fill by origin failure count; each origin candidate frame is bounded " - "before pairing to tool-use rows" - ), - "selection_order": "origin, session_id, tool_id, tool_result_message_id, tool_result_block_id", - "failure_predicate": "tool_result_is_error = 1 OR tool_result_exit_code != 0", - "classification_scope": "immediately following assistant message only", - "sensitivity_scope": "next 3 assistant messages after the failed result, stopping before the next user message", - "n_min": args.n_min, - "thin_cell_policy": ( - "Split cells below n_min are retained for coverage accounting but publish no rates: " - "coverage_status=insufficient_n and publication_status=not_supported; " - "classified-denominator rates independently require classified_outcomes >= n_min." - ), - "total_by_origin": total_by_origin, - "sampled_by_origin": sampled_by_origin, - }, - "definition": ( - "Structured failures are normalized tool-result outcomes with is_error=1 or non-zero exit_code. " - "The immediately following assistant message is classified only for explicit failure " - "acknowledgment markers; this is not an LLM judgment or prose-mined outcome." - ), - "totals": totals, - "window3_totals": window3_totals, - "rates": { - "coverage_status": "supported" if aggregate_supported else "insufficient_n", - "publication_status": "supported" if aggregate_supported else "not_supported", - "classified_coverage_status": "supported" if classified >= args.n_min else "insufficient_n", - "classified_publication_status": "supported" if classified >= args.n_min else "not_supported", - "window3_classified_coverage_status": "supported" if window3_classified >= args.n_min else "insufficient_n", - "window3_classified_publication_status": "supported" - if window3_classified >= args.n_min - else "not_supported", - "n_min": args.n_min, - "silent_rate_lower_bound": (silent / failed) if aggregate_supported else None, - "silent_rate_among_classified": (silent / classified) if classified >= args.n_min else None, - "window3_silent_rate_lower_bound": (window3_silent / failed) if aggregate_supported else None, - "window3_silent_rate_among_classified": ( - (window3_silent / window3_classified) if window3_classified >= args.n_min else None - ), - "ack_later_within_3": max(0, window3_totals["acknowledged"] - totals["acknowledged"]), - }, - "by_tool": _ranked(by_tool, n_min=args.n_min), - "by_model": _ranked(by_model, n_min=args.n_min), - "by_origin": _ranked(by_origin, n_min=args.n_min), - "by_handler_class": _ranked(by_handler_class, n_min=args.n_min), - "economy": { - **economy, - "scope": "sessions represented in the paired structured-failure harness", - "sampled_session_count": len(sampled_session_ids), - }, - "handler_class_definition": { - "benign_recovery": sorted(_BENIGN_RECOVERY_TOOLS), - "consequential": sorted(_CONSEQUENTIAL_TOOLS), - "other": "Any tool name outside the explicit benign/consequential methodology sets.", - }, - "evidence": { - "member_refs": sorted(f"block:{row['tool_result_block_id']}" for row in rows), - }, - "calibration": calibration, - "calibration_sample": [ - _calibration_row(sample, index) for index, sample in enumerate(calibration_sample, start=1) - ], - "samples_by_classification": samples_by_classification, - "samples_by_origin_classification": samples_by_origin_classification, - } - if args.out_dir is not None: - _write_artifacts(args.out_dir, report) - return report - - -def _write_json(path: Path, payload: object) -> None: - path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") - - -def _format_rate_percent(value: int | float | str | None) -> str: - if value is None: - return "not enough labels" - return f"{float(value):.1%}" - - -def _format_dollars(value: object) -> str: - return "not computable" if value is None else f"${_object_float(value):.2f}" - - -def _public_summary(report: dict[str, Any]) -> dict[str, Any]: - totals = report["totals"] - frame = report["sample_frame"] - rates = report["rates"] - calibration_metrics = report["calibration"]["metrics"] - return { - "artifact": "claim-vs-evidence-public-summary", - "generated_at": report["captured_at"], - "archive_root": report["archive_root"], - "index_schema_version": report["index_schema_version"], - "claim": ( - "Polylogue can ground a failure-follow-up finding in normalized tool-result outcomes, " - "state the bounded sample frame, and publish aggregate rates only when the observable follow-up " - "classification has enough coverage." - ), - "non_claim": ( - "The live aggregate is not reproducible without the private archive; the deterministic demo archive " - "reproduces the method and artifact shape, not the private corpus rates. A missing acknowledgement " - "does not establish bad recovery behavior, and protocol-only reasoning is deliberately ambiguous." - ), - "proofs": [ - { - "name": "structured_failure_frame", - "total_structured_failures": frame["total_structured_failures"], - "inspected_structured_failures": frame["inspected_structured_failures"], - "unpaired_structured_failures": frame["unpaired_structured_failures"], - "selection_strategy": frame["selection_strategy"], - "selection_order": frame["selection_order"], - "n_min": frame["n_min"], - "thin_cell_policy": frame["thin_cell_policy"], - }, - { - "name": "next_turn_classification", - "acknowledged": totals["acknowledged"], - "silent_proceed": totals["silent_proceed"], - "ambiguous": totals["ambiguous"], - "n_min": rates["n_min"], - "silent_rate_lower_bound": rates["silent_rate_lower_bound"], - "silent_rate_lower_bound_coverage_status": rates["coverage_status"], - "silent_rate_lower_bound_publication_status": rates["publication_status"], - "silent_rate_among_classified": rates["silent_rate_among_classified"], - "silent_rate_among_classified_coverage_status": rates["classified_coverage_status"], - "silent_rate_among_classified_publication_status": rates["classified_publication_status"], - }, - { - "name": "sensitivity_and_calibration", - "ack_later_within_3": rates["ack_later_within_3"], - "window3_silent_rate_lower_bound": rates["window3_silent_rate_lower_bound"], - "window3_silent_rate_lower_bound_coverage_status": rates["coverage_status"], - "window3_silent_rate_lower_bound_publication_status": rates["publication_status"], - "window3_silent_rate_among_classified": rates["window3_silent_rate_among_classified"], - "window3_silent_rate_among_classified_coverage_status": rates["window3_classified_coverage_status"], - "window3_silent_rate_among_classified_publication_status": rates[ - "window3_classified_publication_status" - ], - "calibration_labeled_rows": calibration_metrics["labeled_rows"], - "ack_marker_precision": calibration_metrics["ack_marker_precision"], - "ack_marker_recall": calibration_metrics["ack_marker_recall"], - }, - ], - "caveats": [ - "Private live-archive counts are aggregate-only in this public summary.", - "Deterministic demo reproduction validates the method and renderer, not the private rate estimates.", - "The classifier is an explicit marker detector; ambiguous rows remain in the denominator.", - "A wordless retry or a recovery that fixes the problem can be operationally appropriate; this metric does not judge it.", - "The report is bounded by --limit unless the limit exceeds the full structured-failure frame.", - "Split cells below n_min are coverage-only and explicitly not supported for rate publication.", - ], - "reproduction": { - "demo_archive_root": "/realm/tmp/polylogue-claim-vs-evidence-demo", - "commands": [ - "export POLYLOGUE_ARCHIVE_ROOT=/realm/tmp/polylogue-claim-vs-evidence-demo", - 'polylogue demo seed --root "$POLYLOGUE_ARCHIVE_ROOT" --force --with-overlays --format json', - 'polylogue demo verify --root "$POLYLOGUE_ARCHIVE_ROOT" --require-overlays --format json', - ("polylogue --plain --format json actions where is_error:true \\| group by followup_class \\| count"), - "polylogue --plain --format json actions where followup_class:silent_proceed", - ( - "devtools workspace claim-vs-evidence " - '--archive-root "$POLYLOGUE_ARCHIVE_ROOT" ' - "--limit 5000 --out-dir /realm/tmp/polylogue-claim-vs-evidence-repro --json" - ), - ], - "shared_queries": [ - "actions where is_error:true | group by followup_class | count", - "actions where followup_class:silent_proceed", - ], - }, - } - - -def _write_public_reproduction(path: Path, report: dict[str, Any]) -> None: - summary = _public_summary(report) - proof_by_name = {str(item["name"]): item for item in summary["proofs"]} - frame = proof_by_name["structured_failure_frame"] - classification = proof_by_name["next_turn_classification"] - sensitivity = proof_by_name["sensitivity_and_calibration"] - commands = "\n".join(summary["reproduction"]["commands"]) - path.write_text( - "\n".join( - [ - "# Claim-vs-Evidence Public Reproduction", - "", - "This packet is the public-safe wrapper for the live claim-vs-evidence demo.", - "It contains aggregate live-archive findings and a private-data-free reproduction", - "path over Polylogue's deterministic demo archive.", - "", - "## What A Reader Can Claim", - "", - str(summary["claim"]), - "", - "## What A Reader Cannot Claim", - "", - str(summary["non_claim"]), - "", - "## Live Aggregate Evidence", - "", - f"- archive root: `{report['archive_root']}`", - f"- index schema: v{report['index_schema_version']}", - f"- total structured failures: {int(frame['total_structured_failures']):,}", - f"- inspected structured failures: {int(frame['inspected_structured_failures']):,}", - f"- unpaired structured failures: {int(frame['unpaired_structured_failures']):,}", - f"- acknowledged next turn: {int(classification['acknowledged']):,}", - f"- silent-proceed next turn: {int(classification['silent_proceed']):,}", - f"- ambiguous next turn: {int(classification['ambiguous']):,}", - ( - f"- silent lower bound: {_format_rate_percent(classification['silent_rate_lower_bound'])} " - f"({classification['silent_rate_lower_bound_coverage_status']} / " - f"{classification['silent_rate_lower_bound_publication_status']})" - ), - ( - f"- silent among classified: {_format_rate_percent(classification['silent_rate_among_classified'])} " - f"({classification['silent_rate_among_classified_coverage_status']} / " - f"{classification['silent_rate_among_classified_publication_status']})" - ), - ( - f"- next-3 silent lower bound: " - f"{_format_rate_percent(sensitivity['window3_silent_rate_lower_bound'])} " - f"({sensitivity['window3_silent_rate_lower_bound_coverage_status']} / " - f"{sensitivity['window3_silent_rate_lower_bound_publication_status']})" - ), - ( - f"- next-3 silent among classified: " - f"{_format_rate_percent(sensitivity['window3_silent_rate_among_classified'])} " - f"({sensitivity['window3_silent_rate_among_classified_coverage_status']} / " - f"{sensitivity['window3_silent_rate_among_classified_publication_status']})" - ), - f"- calibration labeled rows: {int(sensitivity['calibration_labeled_rows']):,}", - f"- acknowledged-marker precision: {_format_rate_percent(sensitivity['ack_marker_precision'])}", - f"- acknowledged-marker recall: {_format_rate_percent(sensitivity['ack_marker_recall'])}", - "", - "## Shared Query Form", - "", - "The core next-turn counts are ordinary action-unit queries, not report-private SQL:", - "", - "```text", - *[str(query) for query in summary["reproduction"]["shared_queries"]], - "```", - "", - "## Reproduce The Method Without Private Data", - "", - "```bash", - commands, - "```", - "", - "The reproduction output should contain the same artifact family:", - "`claim-vs-evidence.report.json`, `summary.json`, `README.md`,", - f"`{_PUBLIC_SUMMARY_FILE}`, and this public reproduction contract.", - "Counts will differ because the deterministic demo archive is synthetic.", - "", - "## Caveats", - "", - *[f"- {item}" for item in summary["caveats"]], - "", - ] - ), - encoding="utf-8", - ) - - -def _write_cold_reader_gate(path: Path, report: dict[str, Any]) -> None: - summary = _public_summary(report) - path.write_text( - "\n".join( - [ - "# Cold-Reader Gate", - "", - "Give a fresh reader only this directory and ask:", - "", - "```text", - "Using only the files in this directory, state what the claim-vs-evidence", - "artifact proves, what it does not prove, what sample frame it used,", - "how to reproduce the method without private data, and the most important", - "caveats before quoting any rate.", - "```", - "", - "## Expected Passing Answer", - "", - "- Names structured tool-result outcomes as the failure evidence anchor.", - "- States that the live rate is aggregate private-archive evidence, not a seeded-corpus rate.", - "- Includes archive root, index schema, total failures, inspected failures, and unpaired failures.", - "- Reports next-turn silent lower bound, next-3 sensitivity, and calibration precision/recall.", - "- Explains that the deterministic demo archive reproduces the method and artifact shape only.", - "- Mentions ambiguous rows remain in the denominator and the classifier is marker-based.", - "", - "## Current Gate Evidence", - "", - f"- public summary: `{_PUBLIC_SUMMARY_FILE}`", - f"- public reproduction: `{_PUBLIC_REPRODUCTION_FILE}`", - f"- aggregate live archive root: `{summary['archive_root']}`", - f"- aggregate index schema: v{summary['index_schema_version']}", - "- status: ready for an external cold read; no private transcript previews are required.", - "", - ] - ), - encoding="utf-8", - ) - - -def _write_artifacts(out_dir: Path, report: dict[str, Any]) -> None: - out_dir.mkdir(parents=True, exist_ok=True) - _write_json(out_dir / "claim-vs-evidence.report.json", report) - _write_csv(out_dir / _CALIBRATION_SAMPLE_FILE, report["calibration_sample"]) - _write_json(out_dir / _PUBLIC_SUMMARY_FILE, _public_summary(report)) - _write_public_reproduction(out_dir / _PUBLIC_REPRODUCTION_FILE, report) - _write_cold_reader_gate(out_dir / _COLD_READER_GATE_FILE, report) - totals = report["totals"] - window3_totals = report["window3_totals"] - rates = report["rates"] - calibration = report["calibration"] - calibration_metrics = calibration["metrics"] - summary = { - "artifact": "claim-vs-evidence", - "updated_at": report["captured_at"], - "archive_root": report["archive_root"], - "index_schema_version": report["index_schema_version"], - "claim": ( - "Polylogue can produce a bounded claim-vs-evidence report by anchoring on structured " - "tool failures and classifying the immediately following assistant turn for explicit " - "failure acknowledgment." - ), - "non_claim": ( - "This is not a whole-archive rate unless --limit exceeds all structured failures, and it is " - "not an LLM judgment of intent or utility. Ambiguous follow-ups remain in the denominator." - ), - "proof_report": { - "failed_outcomes": totals["failed_outcomes"], - "total_structured_failures": report["sample_frame"]["total_structured_failures"], - "unpaired_structured_failures": report["sample_frame"]["unpaired_structured_failures"], - "complete_failure_frame": report["sample_frame"]["complete_failure_frame"], - "acknowledged": totals["acknowledged"], - "silent_proceed": totals["silent_proceed"], - "ambiguous": totals["ambiguous"], - "acknowledged_within_3": window3_totals["acknowledged"], - "silent_proceed_within_3": window3_totals["silent_proceed"], - "ambiguous_within_3": window3_totals["ambiguous"], - "ack_later_within_3": rates["ack_later_within_3"], - "ambiguous_wordless_continuation": totals["ambiguous_wordless_continuation"], - "ambiguous_prose_no_marker": totals["ambiguous_prose_no_marker"], - "silent_rate_lower_bound": rates["silent_rate_lower_bound"], - "silent_rate_lower_bound_coverage_status": rates["coverage_status"], - "silent_rate_lower_bound_publication_status": rates["publication_status"], - "silent_rate_among_classified": rates["silent_rate_among_classified"], - "silent_rate_among_classified_coverage_status": rates["classified_coverage_status"], - "silent_rate_among_classified_publication_status": rates["classified_publication_status"], - "window3_silent_rate_lower_bound": rates["window3_silent_rate_lower_bound"], - "window3_silent_rate_lower_bound_coverage_status": rates["coverage_status"], - "window3_silent_rate_lower_bound_publication_status": rates["publication_status"], - "window3_silent_rate_among_classified": rates["window3_silent_rate_among_classified"], - "window3_silent_rate_among_classified_coverage_status": rates["window3_classified_coverage_status"], - "window3_silent_rate_among_classified_publication_status": rates["window3_classified_publication_status"], - "n_min": rates["n_min"], - "by_handler_class": report["by_handler_class"], - "handler_class_definition": report["handler_class_definition"], - "limit": report["limit"], - "time_window": report["sample_frame"]["time_window"], - "sensitivity_scope": report["sample_frame"]["sensitivity_scope"], - "sampled_by_origin": report["sample_frame"]["sampled_by_origin"], - "calibration": { - "sample_size": calibration["sample_size"], - "sample_seed": calibration["sample_seed"], - "labeled_rows": calibration_metrics["labeled_rows"], - "ack_marker_precision": calibration_metrics["ack_marker_precision"], - "ack_marker_recall": calibration_metrics["ack_marker_recall"], - }, - }, - "caveats": [ - "The report is bounded by --limit for fast active-archive regeneration.", - "The headline classification inspects only the next assistant message; the next-3 window is a sensitivity row.", - "Marker calibration is based on the committed label CSV when present; unlabeled sample rows do not count.", - "Structured failure truth comes from normalized action result is_error/exit_code fields, not assistant prose.", - "Failed tool results without a paired tool-use row are reported as unpaired coverage gaps, not classified rows.", - ], - "source_files": [ - "claim-vs-evidence.report.json", - _PUBLIC_SUMMARY_FILE, - _PUBLIC_REPRODUCTION_FILE, - _COLD_READER_GATE_FILE, - _CALIBRATION_SAMPLE_FILE, - _CALIBRATION_LABELS_FILE, - ], - } - _write_json(out_dir / "summary.json", summary) - _write_readme(out_dir / "README.md", report) - - -def _write_readme(path: Path, report: dict[str, Any]) -> None: - totals = report["totals"] - window3_totals = report["window3_totals"] - rates = report["rates"] - frame = report["sample_frame"] - calibration = report["calibration"] - calibration_metrics = calibration["metrics"] - precision = calibration_metrics["ack_marker_precision"] - recall = calibration_metrics["ack_marker_recall"] - sampled_by_origin = [ - ( - f"- {row['origin']}: inspected {int(row['inspected_structured_failures']):,} / " - f"{int(row['total_structured_failures']):,} structured failures " - f"(requested {int(row['requested_limit']):,})" - ) - for row in frame["sampled_by_origin"] - ] - handler_class_rows = [ - ( - f"- {row['name']}: failed {int(row['failed_outcomes']):,}; " - f"silent {int(row['silent_proceed']):,}; " - f"ambiguous {int(row['ambiguous']):,}; " - + ( - f"silent lower bound {float(row['silent_rate_lower_bound']):.1%}" - if row["silent_rate_lower_bound"] is not None - else f"not supported (n < {int(row['n_min'])})" - ) - ) - for row in report["by_handler_class"] - ] - lines = [ - "# Claim-vs-Evidence Failure Follow-Up", - "", - f"Generated: {report['captured_at']}", - f"Archive root: `{report['archive_root']}`", - f"Index schema: v{report['index_schema_version']}", - "", - "## What This Proves", - "", - "This demo anchors on structured tool-result evidence and asks what the next assistant", - "turn did with that failure. It does not infer truth from assistant prose: the failure", - "predicate is `is_error=1` or a non-zero `exit_code` on normalized `actions` rows.", - "`silent-proceed` is only an observable absence of an explicit acknowledgement marker in a visible", - "next assistant message. It is not a judgment that recovery was wrong, unhelpful, or unsuccessful.", - "", - "## Current Bounded Result", - "", - f"- time window: {frame['time_window']}", - f"- total structured failures in frame: {frame['total_structured_failures']:,}", - f"- unpaired structured failures outside classifiable frame: {frame['unpaired_structured_failures']:,}", - f"- failed structured outcomes inspected: {totals['failed_outcomes']:,}", - f"- complete failure frame: {frame['complete_failure_frame']}", - f"- acknowledged: {totals['acknowledged']:,}", - f"- silent-proceed: {totals['silent_proceed']:,}", - f"- ambiguous: {totals['ambiguous']:,}", - f"- ambiguous wordless tool continuations: {totals['ambiguous_wordless_continuation']:,}", - f"- ambiguous prose without markers: {totals['ambiguous_prose_no_marker']:,}", - ( - f"- silent lower bound: {_format_rate_percent(rates['silent_rate_lower_bound'])} " - f"({rates['coverage_status']} / {rates['publication_status']})" - ), - ( - f"- silent among classified: {_format_rate_percent(rates['silent_rate_among_classified'])} " - f"({rates['classified_coverage_status']} / {rates['classified_publication_status']})" - ), - f"- acknowledged within next 3 assistant turns: {window3_totals['acknowledged']:,}", - f"- acknowledgments appearing only after the next turn: {rates['ack_later_within_3']:,}", - ( - f"- silent lower bound after next-3 sensitivity: " - f"{_format_rate_percent(rates['window3_silent_rate_lower_bound'])} " - f"({rates['coverage_status']} / {rates['publication_status']})" - ), - ( - f"- silent among classified after next-3 sensitivity: " - f"{_format_rate_percent(rates['window3_silent_rate_among_classified'])} " - f"({rates['window3_classified_coverage_status']} / " - f"{rates['window3_classified_publication_status']})" - ), - f"- configured limit: {report['limit']:,}", - f"- split-cell minimum n: {frame['n_min']:,} (below this, rates are not supported)", - f"- selection order: {frame['selection_order']}", - f"- selection strategy: {frame['selection_strategy']}", - "", - "### Handler-Class Split", - "", - "The headline should not mix ordinary read/search recovery with more consequential", - "shell/build/edit failures without saying so. Handler classes are explicit and", - "methodological: `benign_recovery` covers read/search/path-discovery tools,", - "`consequential` covers shell/build/edit/write-class tools, and `other` is not", - "folded into either claim.", - "", - *handler_class_rows, - "", - "### Inspected vs Total by Origin", - "", - *sampled_by_origin, - "", - "### Marker Calibration", - "", - "The classifier is an explicit marker detector, not an LLM judgment. This", - "calibration sample is deterministic and stratified across acknowledged,", - "silent-proceed, and ambiguous predicted classes. Human labels, when present,", - "live in `ack-marker-calibration.labels.csv` so regeneration does not overwrite", - "manual judgment.", - "", - f"- calibration sample size: {int(calibration['sample_size']):,}", - f"- calibration seed: {int(calibration['sample_seed'])}", - f"- labeled rows: {int(calibration_metrics['labeled_rows']):,}", - f"- labels in this current calibration sample: {int(calibration['frame_coverage']['labels_in_current_sample']):,}", - f"- labels outside this current calibration sample: {int(calibration['frame_coverage']['labels_outside_current_sample']):,}", - ( - f"- acknowledged-marker precision: {float(precision):.1%}" - if precision is not None - else "- acknowledged-marker precision: not enough labels" - ), - "", - "### Economy Lanes", - "", - "These lanes are restricted to sessions represented in this paired failure harness. Token and money", - "values come only from `session_model_usage` plus provider-usage event counts,", - "not profile text columns. Codex input/cache and output/reasoning semantics are kept disjoint by", - "the usage materializer. Provider-reported and catalog-derived money remain separate.", - *[ - ( - f"- {row['origin']}: calls {int(row['api_call_count']):,}; input {int(row['input_tokens']):,}; " - f"output {int(row['output_tokens']):,}; cache-read {int(row['cache_read_tokens']):,}; " - f"catalog ${float(row['catalog_cost_usd']):.2f}; provider-reported ${float(row['provider_reported_cost_usd']):.2f}; " - f"catalog $/silent {_format_dollars(row['catalog_usd_per_silent_proceed'])}" - ) - for row in report["economy"]["by_origin"] - ], - ( - f"- acknowledged-marker recall: {float(recall):.1%}" - if recall is not None - else "- acknowledged-marker recall: not enough labels" - ), - "", - "## Regenerate", - "", - "```bash", - "devtools workspace claim-vs-evidence \\", - " --limit 5000 \\", - " --out-dir .local/evidence/claim-vs-evidence \\", - " --json", - "```", - "", - "## Files", - "", - "- `claim-vs-evidence.report.json` — full machine-readable report.", - f"- `{_PUBLIC_SUMMARY_FILE}` — aggregate-only public-safe summary.", - f"- `{_PUBLIC_REPRODUCTION_FILE}` — seeded private-data-free reproduction instructions.", - f"- `{_COLD_READER_GATE_FILE}` — cold-reader prompt and passing-answer checklist.", - f"- `{_CALIBRATION_SAMPLE_FILE}` — deterministic sample for marker calibration.", - f"- `{_CALIBRATION_LABELS_FILE}` — optional human labels consumed on regeneration.", - "- `summary.json` — local claim/non-claim/proof/caveat summary.", - "- `README.md` — this human-readable packet.", - "", - ] - path.write_text("\n".join(lines), encoding="utf-8") - - -def main(argv: list[str] | None = None) -> int: - parsed = _parser().parse_args(argv) - config = _report_config(parsed) - evidence: object | None = None - try: - if parsed.materialize_evidence: - from devtools.claim_vs_evidence_evidence import materialize_claim_vs_evidence_evidence_under_lease - - file_set_root = active_archive_root(config) - with RebuildLease(file_set_root): - report = build_report(parsed, config=config) - evidence = materialize_claim_vs_evidence_evidence_under_lease( - report, - archive_root=file_set_root, - now_ms=int(datetime.now(UTC).timestamp() * 1000), - ) - else: - report = build_report(parsed, config=config) - except ValueError as exc: - print(f"claim-vs-evidence: {exc}", file=sys.stderr) - return 2 - if parsed.materialize_evidence: - assert evidence is not None - print(f"materialized evidence: {json.dumps(evidence, indent=2, sort_keys=True)}", file=sys.stderr) - if parsed.json: - sys.stdout.write(json.dumps(report, indent=2, sort_keys=True) + "\n") - elif parsed.out_dir is not None: - print(f"wrote claim-vs-evidence artifacts to {parsed.out_dir}") - else: - totals = report["totals"] - print( - f"failed={totals['failed_outcomes']} acknowledged={totals['acknowledged']} " - f"silent={totals['silent_proceed']} ambiguous={totals['ambiguous']}" - ) - return 0 - - -if __name__ == "__main__": # pragma: no cover - raise SystemExit(main()) diff --git a/devtools/claim_vs_evidence_evidence.py b/devtools/claim_vs_evidence_evidence.py deleted file mode 100644 index af03d64e44..0000000000 --- a/devtools/claim_vs_evidence_evidence.py +++ /dev/null @@ -1,348 +0,0 @@ -"""Represent a claim-vs-evidence report run as first-party archive evidence. - -The classification/economy logic stays in ``devtools/claim_vs_evidence.py`` -(the harness). This module only represents the harness's OUTPUT as durable -evidence: a content-addressed :class:`~polylogue.storage.sqlite.query_objects. -QueryObject` for the structured-failure selection (the AnalysisDefinition), a -:class:`~polylogue.storage.sqlite.query_objects.ResultSetManifest` for the -rows the run actually matched, an -:class:`~polylogue.storage.sqlite.query_objects.EvaluationReceipt` binding the -run to tier generations and the runtime build (the AnalysisRun), and -``AssertionKind.FINDING`` rows for the headline numbers (polylogue-rxdo.13). - -It writes through the same production primitives the daemon's own -standing-query convergence stage uses -(``polylogue/daemon/convergence_standing_queries.py``), but only after taking -the archive's exclusive offline-writer lease -- not through a second live -SQLite writer, not a new generic finding registry, not a -metric/pattern/cohort/experiment definition system, and not a scheduler. A -finding is written with ``public_claim=None`` (no ``PublicClaimDeclaration``) -unless the run's own construct-validity gates (``n_min``, non-zero classified -outcomes) are satisfied, so an unpublishable run still gets an honest private -evidence record without ever exposing a degenerate rate as a public claim. -""" - -from __future__ import annotations - -from pathlib import Path -from typing import Any, TypedDict - -from polylogue.archive.query.production_evaluator import ( - _index_epoch, # planner-internal seam, reused for the same tier-generation identity - _polylogue_runtime_build_ref, - _tier_generation, -) -from polylogue.core.hashing import hash_payload -from polylogue.core.json import JSONValue -from polylogue.core.query_identity import JsonValue -from polylogue.core.query_identity import query_ref as _query_object_ref -from polylogue.core.query_identity import result_set_ref as _result_set_object_ref -from polylogue.storage.archive_identity import archive_file_set_root -from polylogue.storage.index_generation import RebuildLease -from polylogue.storage.sqlite.archive_tiers.user_write import ( - ArchiveAssertionEnvelope, - FindingAssertion, - PublicClaimDeclaration, - upsert_findings_as_assertions, -) -from polylogue.storage.sqlite.connection_profile import open_daemon_connection -from polylogue.storage.sqlite.query_objects import ( - EvaluationReceipt, - QueryObject, - ResultSetManifest, - get_result_set, - membership_merkle_root, - put_evaluation_receipt, - put_query, - put_result_set, -) - -# Bump when the classifier's silent/acknowledged/ambiguous taxonomy or the -# handler-class split changes meaning, so the AnalysisDefinition identity -# (query_hash) changes with it instead of silently reusing a stale one. -CLASSIFIER_DEFINITION_VERSION = "2" -ANALYSIS_TARGET_REF = "analysis:claim-vs-evidence" -_QUERY_GRAIN = "structured-failure-followup" -_QUERY_LANE = "analysis" -_QUERY_RANK_POLICY = "origin,session_id,tool_id,tool_result_message_id,tool_result_block_id" - - -class MaterializedEvidence(TypedDict): - query_ref: str - result_set_ref: str - receipt_id: str - finding_assertion_ids: list[str] - public_claim_written: bool - - -def build_query_definition(report: dict[str, Any]) -> dict[str, JsonValue]: - """Return the content-addressed AnalysisDefinition payload for one report run. - - Deliberately not a DSL-executable plan: the paired next-assistant-turn - lookup with window-3 lookahead has no representation in the query - predicate grammar today. This is provenance identity, mirroring the - ``convergence_standing_queries`` doctrine that durable identity JSON is - "provenance, not source syntax to reverse-compile" -- the harness in - ``devtools/claim_vs_evidence.py`` remains the sole executor. - """ - frame = report["sample_frame"] - return { - "kind": "analysis-selection", - "analysis_id": "claim-vs-evidence", - "classifier_module": "polylogue.archive.actions.followup", - "classifier_function": "classify_failed_followup_evidence", - "classifier_definition_version": CLASSIFIER_DEFINITION_VERSION, - "failure_predicate": frame["failure_predicate"], - "classification_scope": frame["classification_scope"], - "sensitivity_scope": frame["sensitivity_scope"], - "selection_strategy": frame["selection_strategy"], - "n_min": frame["n_min"], - "limit": report["limit"], - "handler_class_definition": report["handler_class_definition"], - } - - -def build_result_set_members(report: dict[str, Any]) -> tuple[str, ...]: - """Return one sorted ``block:`` ref per failed outcome classified.""" - return tuple(report["evidence"]["member_refs"]) - - -def build_evaluation_receipt( - archive_root: Path, - index_db: Path, - *, - query_hash: str, - result_set_id: str, - created_at_ms: int, -) -> EvaluationReceipt: - """Bind one run to its source/user/index tier generations and runtime build. - - ``receipt_id`` is content-addressed (not a random UUID, unlike - ``ArchiveCanonicalPlanEvaluator``'s per-execution telemetry receipts) over - every field ``put_evaluation_receipt`` itself treats as significant, - including ``created_at_ms`` -- that function already rejects reusing a - receipt id with a changed ``created_at_ms``, so folding it into the hash - is what lets two calls at the same ``created_at_ms`` collapse into one - safe no-op while two calls at different times correctly get distinct - receipts instead of a spurious conflict. - """ - source_generation = _tier_generation(archive_root / "source.db", label="source") - user_generation = _tier_generation(archive_root / "user.db", label="user") - index_generation = _index_epoch(index_db) - runtime_build_ref = _polylogue_runtime_build_ref() - receipt_digest = hash_payload( - [ - query_hash, - result_set_id, - source_generation, - user_generation, - index_generation, - runtime_build_ref, - created_at_ms, - ] - ) - receipt_id = f"receipt-{receipt_digest}" - return EvaluationReceipt( - receipt_id=receipt_id, - source_generation=source_generation, - user_generation=user_generation, - index_generation=index_generation, - runtime_build_ref=runtime_build_ref, - ) - - -def build_findings( - report: dict[str, Any], - *, - query_reference: str, - result_set_reference: str, - receipt: EvaluationReceipt, -) -> list[FindingAssertion]: - """Return the headline-number FindingAssertions for one report run. - - ``public_claim`` stays ``None`` (no PublicClaimDeclaration) unless the - aggregate rate actually clears the run's own ``n_min``/classified-outcome - gates -- an unpublishable run still gets an honest private evidence - record, never a fabricated public rate. - """ - frame = report["sample_frame"] - totals = report["totals"] - rates = report["rates"] - run_ref = f"run:claim-vs-evidence-{report['captured_at']}" - aggregate_publishable = rates["publication_status"] == "supported" and rates["silent_rate_lower_bound"] is not None - statistic: dict[str, JSONValue] = { - "op": "lower_bound", - "value": rates["silent_rate_lower_bound"], - "unit": "ratio", - "numerator": totals["silent_proceed"], - "denominator": totals["failed_outcomes"], - "ambiguous": totals["ambiguous"], - "classified_outcomes": totals.get("classified_outcomes"), - } - body_text = ( - f"Structured-failure follow-up classification over {frame['inspected_structured_failures']} " - f"inspected failures: {totals['acknowledged']} acknowledged, {totals['silent_proceed']} silent, " - f"{totals['ambiguous']} ambiguous." - ) - public_claim: PublicClaimDeclaration | None = None - if aggregate_publishable: - body_text = ( - f"In one bounded private-archive sample, {totals['silent_proceed']} of " - f"{totals['failed_outcomes']} inspected structured failures were followed by silent " - f"continuation on the next assistant turn, a {rates['silent_rate_lower_bound']:.1%} lower bound." - ) - public_claim = PublicClaimDeclaration( - publication=body_text, - scope=( - f"One private archive; {frame['inspected_structured_failures']} inspected structured " - "failures from the run's bounded sample frame; next assistant turn only." - ), - caveat=( - "This is not a population estimate; ambiguous rows are excluded from the classified " - "denominator, and support must be recomputed when the evidence epoch, definition, or " - "frame changes." - ), - public_evidence_refs=("file:docs/findings/claim-vs-evidence.md",), - disclosure="public", - ) - return [ - FindingAssertion( - claim_key="finding.silent-proceed-lower-bound", - target_ref=ANALYSIS_TARGET_REF, - body_text=body_text, - finding_kind="claim-vs-evidence", - statistic=statistic, - n=totals["failed_outcomes"], - query_ref=query_reference, - result_set_ref=result_set_reference, - detector_ref=run_ref, - evidence_refs=("file:docs/findings/claim-vs-evidence.md",), - source_epoch=report["captured_at"], - evaluation_ref=f"receipt:{receipt.receipt_id}", - frame_ref=query_reference, - public_claim=public_claim, - ) - ] - - -def materialize_claim_vs_evidence_evidence_under_lease( - report: dict[str, Any], - *, - archive_root: Path, - now_ms: int, -) -> MaterializedEvidence: - """Register one report while the caller holds ``RebuildLease``. - - The AnalysisDefinition (query) and its matched-row ResultSetManifest are - content-addressed: identical selection logic and identical matched rows - always resolve to the same identity, at any ``now_ms``. The AnalysisRun - receipt and its FindingAssertion are scoped to ``now_ms``: calling this - twice with the same ``report`` and the same ``now_ms`` is a safe retry - no-op, but calling it again at a later ``now_ms`` records a new run and a - new finding row even if the numbers happen to match, because the archive - should carry that a re-verification happened under a later tier state -- - not silently collapse repeated regenerations into one row. - """ - archive_root = archive_root.resolve() - report_root = Path(report["archive_root"]).resolve() - index_db = Path(report["index_db"]).resolve() - if report_root != archive_root: - raise ValueError(f"report archive root {report_root} does not match materialization root {archive_root}") - index_file_set_root = archive_file_set_root(archive_root=archive_root, db_path=index_db).resolve() - if index_file_set_root != archive_root: - raise ValueError( - f"report index {index_db} belongs to {index_file_set_root}, not materialization root {archive_root}" - ) - query_definition = build_query_definition(report) - member_refs = build_result_set_members(report) - conn = open_daemon_connection(archive_root / "user.db", timeout=30.0) - try: - query: QueryObject = put_query( - conn, - query_definition, - grain=_QUERY_GRAIN, - lane=_QUERY_LANE, - rank_policy=_QUERY_RANK_POLICY, - created_at_ms=now_ms, - ) - query_reference = _query_object_ref(query.query_hash).format() - corpus_epoch = _index_epoch(index_db) - result_set_digest = hash_payload( - ( - query.query_hash, - _QUERY_GRAIN, - corpus_epoch, - membership_merkle_root(member_refs), - hash_payload(list(member_refs)), - "capped", - "finding", - ) - ) - result_set_id = f"finding-{result_set_digest}" - result_set: ResultSetManifest | None = get_result_set(conn, result_set_id) - if result_set is None: - result_set = put_result_set( - conn, - result_set_id=result_set_id, - query_hash=query.query_hash, - grain=_QUERY_GRAIN, - corpus_epoch=corpus_epoch, - member_refs=member_refs, - exactness="capped", - persistence_class="finding", - created_at_ms=now_ms, - ) - result_set_reference = _result_set_object_ref(result_set.result_set_id).format() - receipt = build_evaluation_receipt( - archive_root, - index_db, - query_hash=query.query_hash, - result_set_id=result_set.result_set_id, - created_at_ms=now_ms, - ) - put_evaluation_receipt( - conn, - query_hash=query.query_hash, - receipt=receipt, - result_set_id=result_set.result_set_id, - created_at_ms=now_ms, - ) - findings = build_findings( - report, - query_reference=query_reference, - result_set_reference=result_set_reference, - receipt=receipt, - ) - envelopes: list[ArchiveAssertionEnvelope] = upsert_findings_as_assertions(conn, findings, now_ms=now_ms) - conn.commit() - finally: - conn.close() - return { - "query_ref": query_reference, - "result_set_ref": result_set_reference, - "receipt_id": receipt.receipt_id, - "finding_assertion_ids": [envelope.assertion_id for envelope in envelopes], - "public_claim_written": any(finding.public_claim is not None for finding in findings), - } - - -def materialize_claim_vs_evidence_evidence( - report: dict[str, Any], - *, - archive_root: Path, - now_ms: int, -) -> MaterializedEvidence: - """Materialize an already-collected report under exclusive writer ownership. - - The CLI acquires this lease before report collection and calls the internal - under-lease function directly. This public helper remains safe for callers - that already hold a report: it excludes every live writer before opening - ``user.db`` and validates that the report's index and durable tiers belong - to the same archive file set. - """ - with RebuildLease(archive_root): - return materialize_claim_vs_evidence_evidence_under_lease( - report, - archive_root=archive_root, - now_ms=now_ms, - ) diff --git a/docs/findings/claim-vs-evidence.md b/docs/findings/claim-vs-evidence.md index fe1302fb2f..3567e976d7 100644 --- a/docs/findings/claim-vs-evidence.md +++ b/docs/findings/claim-vs-evidence.md @@ -107,14 +107,17 @@ The calibration is small. The method therefore keeps 3,375 cases ambiguous inste ## First-party evidence boundary -The command is read-only by default. With explicit `--materialize-evidence`, it -records a content-addressed analysis definition, result-set membership, -evaluation receipt, and finding through the archive's existing user-tier -writers. It emits a public-claim declaration only when the run's own -minimum-sample and classified-outcome gates pass. The surviving -`PublicClaimProjection` applies publication, privacy, freshness, frame, and -evidence-integrity state independently; report generation alone never upgrades -this historical page into a supported current claim. +The generating command was read-only by default. With explicit +`--materialize-evidence`, it recorded a content-addressed analysis definition, +result-set membership, evaluation receipt, and finding through the archive's +existing user-tier writers, and emitted a public-claim declaration only when +the run's own minimum-sample and classified-outcome gates passed. That +harness was retired 2026-08 (the closed claim-vs-evidence campaign's +private-report/calibration/publishing tooling — see #3950); the surviving +`PublicClaimProjection` (`polylogue/insights/measurement/public_claims.py`) +still applies publication, privacy, freshness, frame, and evidence-integrity +state independently on ordinary production routes. This page remains a frozen +historical record, not a currently regeneratable claim. ## Interpretation @@ -122,39 +125,17 @@ The finding establishes that Polylogue can ask and operationalize a question ord It does not establish why the assistant proceeded, whether the outcome was eventually repaired, whether silence was harmful in every case, or how frequently the behavior occurs outside the sampled archive. -## Reproduce the method without private data +## Reproducing this finding -```bash -export POLYLOGUE_ARCHIVE_ROOT=/tmp/polylogue-claim-vs-evidence-demo -polylogue demo seed --root "$POLYLOGUE_ARCHIVE_ROOT" --force --with-overlays --format json -polylogue demo verify --root "$POLYLOGUE_ARCHIVE_ROOT" --require-overlays --format json +The generating harness (`devtools workspace claim-vs-evidence` and its +private-report/calibration/publishing machinery) was retired 2026-08 once the +closed campaign it served (`polylogue-sru`) had its terminal artifacts. This +page is therefore frozen historical text: the numbers above are not +regeneratable through a current command. The reusable query semantics behind +them remain on ordinary production routes via `PublicClaimProjection` +(`polylogue/insights/measurement/public_claims.py`). -devtools workspace claim-vs-evidence \ - --archive-root "$POLYLOGUE_ARCHIVE_ROOT" \ - --limit 5000 \ - --out-dir /tmp/polylogue-claim-vs-evidence-repro \ - --json -``` - -The deterministic corpus reproduces the method and controls. It does not reproduce the private-archive prevalence result. - -## Regenerate the private packet locally - -Operators with the relevant archive can run: - -```bash -devtools workspace claim-vs-evidence \ - --limit 5000 \ - --out-dir .local/evidence/claim-vs-evidence \ - --json -``` - -## Evidence and caveats - -See: - -- `devtools/claim_vs_evidence.py`; -- `tests/unit/devtools/test_claim_vs_evidence.py`; -- the local `.local/evidence/claim-vs-evidence/` packet when generated. - -Publication requires the packet’s archive cursor, measure version, commit SHA, sample-frame predicate, and run date. If any is missing or stale, the finding page should refuse regeneration rather than silently retain an old number. +Publication of any future finding of this shape still requires the packet's +archive cursor, measure version, commit SHA, sample-frame predicate, and run +date; a finding page should refuse regeneration rather than silently retain +an old number when any of those is missing or stale. diff --git a/tests/unit/devtools/test_claim_vs_evidence.py b/tests/unit/devtools/test_claim_vs_evidence.py deleted file mode 100644 index b534e4d17e..0000000000 --- a/tests/unit/devtools/test_claim_vs_evidence.py +++ /dev/null @@ -1,850 +0,0 @@ -from __future__ import annotations - -import argparse -import json -import sqlite3 -from pathlib import Path - -import pytest - -from devtools import claim_vs_evidence -from devtools.claim_vs_evidence import _economy_rows, build_report -from polylogue.archive.actions.followup import classify_failed_followup_evidence -from polylogue.config import Config -from polylogue.demo import seed_demo_archive -from polylogue.storage.index_generation import ActiveWriterLease, RebuildLeaseUnavailableError -from polylogue.storage.sqlite.action_relation import action_relation_select_sql - - -def _report_args( - *, - archive_root: Path | None, - out_dir: Path | None, - limit: int, - sample_limit: int, - n_min: int = 1, - calibration_size: int = 3, - calibration_seed: int = 7, - calibration_labels: Path | None = None, -) -> argparse.Namespace: - return argparse.Namespace( - archive_root=archive_root, - out_dir=out_dir, - limit=limit, - sample_limit=sample_limit, - n_min=n_min, - calibration_size=calibration_size, - calibration_seed=calibration_seed, - calibration_labels=calibration_labels, - json=False, - ) - - -def _seed_archive(root: Path) -> None: - root.mkdir(parents=True) - conn = sqlite3.connect(root / "index.db") - conn.executescript( - f""" - PRAGMA user_version=22; - CREATE TABLE sessions ( - session_id TEXT PRIMARY KEY, - origin TEXT NOT NULL, - title TEXT, - created_at_ms INTEGER, - updated_at_ms INTEGER - ); - CREATE TABLE messages ( - session_id TEXT NOT NULL, - message_id TEXT PRIMARY KEY, - role TEXT NOT NULL, - position INTEGER NOT NULL, - variant_index INTEGER NOT NULL DEFAULT 0, - material_origin TEXT NOT NULL DEFAULT 'assistant_authored', - model_name TEXT - ); - CREATE TABLE blocks ( - block_id TEXT GENERATED ALWAYS AS (message_id || ':' || position) STORED UNIQUE, - message_id TEXT NOT NULL, - session_id TEXT NOT NULL, - position INTEGER NOT NULL, - block_type TEXT NOT NULL, - text TEXT, - tool_name TEXT, - tool_id TEXT, - tool_input TEXT, - semantic_type TEXT, - tool_result_is_error INTEGER, - tool_result_exit_code INTEGER, - tool_command TEXT GENERATED ALWAYS AS (json_extract(tool_input, '$.command')) VIRTUAL, - tool_path TEXT GENERATED ALWAYS AS ( - COALESCE(json_extract(tool_input, '$.file_path'), json_extract(tool_input, '$.path')) - ) VIRTUAL, - PRIMARY KEY(message_id, position) - ); - CREATE INDEX idx_blocks_type ON blocks(block_type); - CREATE INDEX idx_blocks_tool_result_outcome - ON blocks(block_type, tool_result_is_error, tool_result_exit_code, session_id, tool_id, message_id) - WHERE block_type = 'tool_result'; - CREATE INDEX idx_blocks_tool_id ON blocks(tool_id) WHERE tool_id IS NOT NULL; - CREATE INDEX idx_messages_session_position ON messages(session_id, position); - CREATE VIEW actions AS - {action_relation_select_sql()}; - CREATE TABLE session_model_usage ( - session_id TEXT NOT NULL, - model_name TEXT NOT NULL, - input_tokens INTEGER NOT NULL, - output_tokens INTEGER NOT NULL, - cache_read_tokens INTEGER NOT NULL, - cache_write_tokens INTEGER NOT NULL, - cost_usd REAL, - cost_provenance TEXT NOT NULL - ); - CREATE TABLE session_provider_usage_events ( - session_id TEXT NOT NULL, - model_name TEXT, - provider_event_type TEXT NOT NULL, - last_reasoning_output_tokens INTEGER - ); - """ - ) - conn.executemany( - "INSERT INTO sessions(session_id, origin, title, created_at_ms, updated_at_ms) VALUES (?, ?, ?, ?, ?)", - [ - ("s1", "claude-code-session", "fixture one", 1, 4), - ("s2", "codex-session", "fixture two", 1, 1), - ("s3", "claude-code-session", "unrelated fixture", 1, 1), - ], - ) - conn.executemany( - "INSERT INTO messages(session_id, message_id, role, position, model_name) VALUES (?, ?, ?, ?, ?)", - [ - ("s1", "tool-ack", "tool", 1, "claude-opus"), - ("s1", "next-ack", "assistant", 2, "claude-sonnet"), - ("s1", "tool-silent", "tool", 3, "claude-opus"), - ("s1", "next-silent", "assistant", 4, "claude-haiku"), - ("s1", "next-silent-ack", "assistant", 5, "claude-haiku"), - ("s2", "tool-missing-next", "tool", 1, "codex"), - ("s2", "next-prose", "assistant", 2, "codex"), - ("s2", "tool-wordless", "tool", 3, "codex"), - ("s2", "next-wordless", "assistant", 4, "codex"), - ], - ) - conn.executemany( - """ - INSERT INTO blocks( - message_id, session_id, position, block_type, text, tool_name, tool_id, - tool_input, tool_result_is_error, tool_result_exit_code - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - """, - [ - ("tool-ack", "s1", 0, "tool_use", None, "Bash", "t1", '{"command":"pytest"}', None, None), - ("tool-ack", "s1", 1, "tool_result", "failed", None, "t1", None, 1, None), - ( - "next-ack", - "s1", - 0, - "text", - "The command failed with exit code 2, so I will fix it.", - None, - None, - None, - None, - None, - ), - ("tool-silent", "s1", 0, "tool_use", None, "Bash", "t2", '{"command":"ls missing"}', None, None), - ("tool-silent", "s1", 1, "tool_result", "missing", None, "t2", None, 0, 2), - ( - "next-silent", - "s1", - 0, - "text", - "I will continue by inspecting the neighboring module now.", - None, - None, - None, - None, - None, - ), - ( - "next-silent-ack", - "s1", - 0, - "text", - "The ls command failed; I will switch to a different path.", - None, - None, - None, - None, - None, - ), - ("tool-missing-next", "s2", 0, "tool_use", None, "Read", "t3", '{"path":"x"}', None, None), - ("tool-missing-next", "s2", 1, "tool_result", "nope", None, "t3", None, 0, 1), - ( - "next-prose", - "s2", - 0, - "text", - "Ok.", - None, - None, - None, - None, - None, - ), - ("tool-wordless", "s2", 0, "tool_use", None, "Read", "t4", '{"path":"y"}', None, None), - ("tool-wordless", "s2", 1, "tool_result", "nope again", None, "t4", None, 0, 1), - ("next-wordless", "s2", 0, "tool_use", None, "Read", "t5", '{"path":"z"}', None, None), - ], - ) - # NOTE: this module builds its own minimal synthetic schema (not the real - # index.db DDL) with no priced_with column or CHECK constraint, so the - # polylogue-shnc invariant does not apply here. - conn.executemany( - """ - INSERT INTO session_model_usage( - session_id, model_name, input_tokens, output_tokens, cache_read_tokens, - cache_write_tokens, cost_usd, cost_provenance - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) - """, - [ - ("s1", "claude-sonnet", 11, 12, 13, 14, 1.25, "priced"), - ("s2", "codex", 21, 22, 23, 24, 3.5, "origin_reported"), - ("s3", "unrelated-model", 999, 999, 999, 999, 99.0, "priced"), - ], - ) - conn.executemany( - """ - INSERT INTO session_provider_usage_events( - session_id, model_name, provider_event_type, last_reasoning_output_tokens - ) VALUES (?, ?, ?, ?) - """, - [ - ("s1", "claude-sonnet", "message_usage", 5), - ("s2", "codex", "token_count", 89), - ], - ) - conn.commit() - conn.close() - - -def test_economy_rows_empty_session_ids_short_circuits(tmp_path: Path) -> None: - conn = sqlite3.connect(tmp_path / "empty.db") - conn.executescript( - """ - CREATE TABLE session_model_usage ( - session_id TEXT NOT NULL, model_name TEXT NOT NULL, - input_tokens INTEGER, output_tokens INTEGER, - cache_read_tokens INTEGER, cache_write_tokens INTEGER, - cost_usd REAL, cost_provenance TEXT - ); - """ - ) - economy = _economy_rows(conn, session_ids=(), silent_by_origin={}) - assert economy == {"by_model": [], "by_origin": []} - - -def test_economy_rows_missing_usage_table_returns_empty(tmp_path: Path) -> None: - conn = sqlite3.connect(tmp_path / "no-usage-table.db") - conn.execute("CREATE TABLE sessions (session_id TEXT PRIMARY KEY, origin TEXT)") - economy = _economy_rows(conn, session_ids=("s1",), silent_by_origin={}) - assert economy == {"by_model": [], "by_origin": []} - - -def test_protocol_only_followup_is_ambiguous_not_silent_proceed() -> None: - """Hidden reasoning contains no reader-visible acknowledgement evidence.""" - assert classify_failed_followup_evidence("inspect state privately") == { - "classification": "ambiguous", - "reason": "protocol_only_next_assistant_message", - "matched_marker": None, - } - - -def test_claim_vs_evidence_builds_bounded_artifacts(tmp_path: Path) -> None: - archive = tmp_path / "archive" - out_dir = tmp_path / "out" - _seed_archive(archive) - out_dir.mkdir() - (out_dir / "ack-marker-calibration.labels.csv").write_text( - "\n".join( - [ - "sample_id,human_label,classification,classification_reason,matched_marker,origin,model_name," - "tool_name,handler_class,session_ref,tool_result_message_ref,next_message_ref,next_text_preview," - "next3_classification,next3_matched_marker,next3_text_preview", - "cal-001,acknowledged,acknowledged,explicit_acknowledgment_marker,failed,claude-code-session," - "claude-sonnet,Bash,consequential,session:s1,message:tool-ack,message:next-ack," - "The command failed,acknowledged,failed,The command failed", - "cal-002,acknowledged,silent_proceed,no_acknowledgment_marker,,claude-code-session," - "claude-haiku,Bash,consequential,session:s1,message:tool-silent,message:next-silent," - "I will continue,acknowledged,failed,The ls command failed", - "", - ] - ), - encoding="utf-8", - ) - - report = build_report( - _report_args( - archive_root=archive, - out_dir=out_dir, - limit=4, - sample_limit=2, - ) - ) - - assert report["index_schema_version"] == 22 - assert report["sample_frame"] == { - "classification_scope": "immediately following assistant message only", - "complete_failure_frame": True, - "failure_predicate": "tool_result_is_error = 1 OR tool_result_exit_code != 0", - "inspected_structured_failures": 4, - "limit": 4, - "n_min": 1, - "time_window": "entire archive (no since/until filter)", - "sampled_by_origin": [ - { - "inspected_structured_failures": 2, - "origin": "claude-code-session", - "requested_limit": 2, - "total_structured_failures": 2, - }, - { - "inspected_structured_failures": 2, - "origin": "codex-session", - "requested_limit": 2, - "total_structured_failures": 2, - }, - ], - "selection_order": "origin, session_id, tool_id, tool_result_message_id, tool_result_block_id", - "selection_strategy": ( - "origin-stratified bounded sample; at least one row per origin when limit allows, " - "then proportional fill by origin failure count; each origin candidate frame is bounded " - "before pairing to tool-use rows" - ), - "sensitivity_scope": "next 3 assistant messages after the failed result, stopping before the next user message", - "thin_cell_policy": ( - "Split cells below n_min are retained for coverage accounting but publish no rates: " - "coverage_status=insufficient_n and publication_status=not_supported; " - "classified-denominator rates independently require classified_outcomes >= n_min." - ), - "total_by_origin": [ - {"failed_outcomes": 2, "origin": "claude-code-session"}, - {"failed_outcomes": 2, "origin": "codex-session"}, - ], - "total_structured_failures": 4, - "unpaired_structured_failures": 0, - } - assert report["totals"] == { - "failed_outcomes": 4, - "acknowledged": 1, - "silent_proceed": 1, - "ambiguous": 2, - "ambiguous_wordless_continuation": 1, - "ambiguous_prose_no_marker": 1, - "classified_outcomes": 2, - } - assert report["window3_totals"] == { - "failed_outcomes": 4, - "acknowledged": 2, - "silent_proceed": 0, - "ambiguous": 2, - "classified_outcomes": 2, - } - assert report["rates"]["silent_rate_lower_bound"] == 1 / 4 - assert report["rates"]["ack_later_within_3"] == 1 - assert report["rates"]["window3_silent_rate_lower_bound"] == 0 - assert report["economy"]["scope"] == "sessions represented in the paired structured-failure harness" - assert report["economy"]["sampled_session_count"] == 2 - economy_by_model = {str(row["model_name"]): row for row in report["economy"]["by_model"]} - assert set(economy_by_model) == {"claude-sonnet", "codex"} - assert economy_by_model["claude-sonnet"]["reasoning_tokens"] == 5 - assert economy_by_model["claude-sonnet"]["catalog_cost_usd"] == 1.25 - assert economy_by_model["codex"]["reasoning_tokens"] is None - assert economy_by_model["codex"]["provider_reported_cost_usd"] == 3.5 - assert economy_by_model["claude-sonnet"]["input_tokens"] == 11 - assert "unrelated-model" not in economy_by_model - assert report["calibration"]["sample_size"] == 3 - assert report["calibration"]["sample_seed"] == 7 - assert report["calibration"]["metrics"]["labeled_rows"] == 2 - assert report["calibration"]["metrics"]["ack_marker_precision"] == 1.0 - assert report["calibration"]["metrics"]["ack_marker_recall"] == 0.5 - assert report["by_handler_class"] == [ - { - "name": "benign_recovery", - "failed_outcomes": 2, - "acknowledged": 0, - "silent_proceed": 0, - "ambiguous": 2, - "ambiguous_wordless_continuation": 1, - "ambiguous_prose_no_marker": 1, - "classified_outcomes": 0, - "n_min": 1, - "coverage_status": "supported", - "publication_status": "supported", - "classified_coverage_status": "insufficient_n", - "classified_publication_status": "not_supported", - "silent_rate_lower_bound": 0.0, - "silent_rate_among_classified": None, - }, - { - "name": "consequential", - "failed_outcomes": 2, - "acknowledged": 1, - "silent_proceed": 1, - "ambiguous": 0, - "ambiguous_wordless_continuation": 0, - "ambiguous_prose_no_marker": 0, - "classified_outcomes": 2, - "n_min": 1, - "coverage_status": "supported", - "publication_status": "supported", - "classified_coverage_status": "supported", - "classified_publication_status": "supported", - "silent_rate_lower_bound": 0.5, - "silent_rate_among_classified": 0.5, - }, - ] - assert set(report["samples_by_origin_classification"]) == {"claude-code-session", "codex-session"} - assert report["samples_by_origin_classification"]["codex-session"]["ambiguous"][0]["origin"] == "codex-session" - codex_ambiguous = report["samples_by_origin_classification"]["codex-session"]["ambiguous"] - assert {sample["classification_reason"] for sample in codex_ambiguous} == { - "prose_no_marker", - "wordless_tool_continuation", - } - assert {sample["handler_class"] for sample in codex_ambiguous} == {"benign_recovery"} - assert any(sample["next_has_tool_use"] for sample in codex_ambiguous) - assert ( - report["samples_by_origin_classification"]["claude-code-session"]["acknowledged"][0]["next_text_preview"] - == "The command failed with exit code 2, so I will fix it." - ) - assert ( - report["samples_by_origin_classification"]["claude-code-session"]["acknowledged"][0]["classification_reason"] - == "explicit_acknowledgment_marker" - ) - silent_sample = report["samples_by_origin_classification"]["claude-code-session"]["silent_proceed"][0] - assert silent_sample["next3_classification"] == "acknowledged" - assert silent_sample["next3_matched_marker"] == "failed" - assert silent_sample["next3_message_refs"] == ["message:next-silent", "message:next-silent-ack"] - assert report["samples_by_origin_classification"]["claude-code-session"]["acknowledged"][0]["matched_marker"] == ( - "failed" - ) - summary = json.loads((out_dir / "summary.json").read_text()) - assert summary["claim"] - assert summary["non_claim"] - assert summary["proof_report"]["failed_outcomes"] == 4 - assert summary["proof_report"]["complete_failure_frame"] is True - assert summary["proof_report"]["ambiguous_wordless_continuation"] == 1 - assert summary["proof_report"]["ambiguous_prose_no_marker"] == 1 - assert summary["proof_report"]["acknowledged_within_3"] == 2 - assert summary["proof_report"]["silent_proceed_within_3"] == 0 - assert summary["proof_report"]["ack_later_within_3"] == 1 - assert summary["proof_report"]["window3_silent_rate_lower_bound"] == 0 - assert summary["proof_report"]["calibration"] == { - "sample_size": 3, - "sample_seed": 7, - "labeled_rows": 2, - "ack_marker_precision": 1.0, - "ack_marker_recall": 0.5, - } - assert summary["proof_report"]["by_handler_class"][0]["name"] == "benign_recovery" - assert summary["proof_report"]["by_handler_class"][1]["coverage_status"] == "supported" - assert summary["proof_report"]["by_handler_class"][1]["publication_status"] == "supported" - assert summary["proof_report"]["by_handler_class"][1]["silent_rate_lower_bound"] == 0.5 - assert summary["proof_report"]["time_window"] == "entire archive (no since/until filter)" - assert summary["proof_report"]["sampled_by_origin"] == [ - { - "inspected_structured_failures": 2, - "origin": "claude-code-session", - "requested_limit": 2, - "total_structured_failures": 2, - }, - { - "inspected_structured_failures": 2, - "origin": "codex-session", - "requested_limit": 2, - "total_structured_failures": 2, - }, - ] - assert (out_dir / "claim-vs-evidence.report.json").exists() - calibration_sample = (out_dir / "ack-marker-calibration.sample.csv").read_text() - assert "sample_id,human_label,classification" in calibration_sample - assert "acknowledged" in calibration_sample - public_summary = json.loads((out_dir / "public-summary.json").read_text()) - assert public_summary["claim"].startswith("Polylogue can ground") - assert "private archive" in public_summary["non_claim"] - assert public_summary["proofs"][0]["total_structured_failures"] == 4 - assert public_summary["proofs"][2]["ack_marker_precision"] == 1.0 - assert "samples_by_classification" not in public_summary - assert "calibration_sample" not in public_summary - assert "next_text_preview" not in json.dumps(public_summary) - public_reproduction = (out_dir / "PUBLIC_REPRODUCTION.md").read_text() - assert "polylogue demo seed" in public_reproduction - assert "actions where is_error:true | group by followup_class | count" in public_reproduction - assert "polylogue --plain --format json actions where is_error:true" in public_reproduction - assert "devtools workspace claim-vs-evidence" in public_reproduction - assert "reproduces the method and artifact shape" in public_reproduction - cold_reader_gate = (out_dir / "COLD_READER_GATE.md").read_text() - assert "Expected Passing Answer" in cold_reader_gate - assert "no private transcript previews" in cold_reader_gate - readme = (out_dir / "README.md").read_text() - assert "Claim-vs-Evidence" in readme - assert "- time window: entire archive (no since/until filter)" in readme - assert "### Handler-Class Split" in readme - assert "- consequential: failed 2; silent 1; ambiguous 0; silent lower bound 50.0%" in readme - assert "- acknowledgments appearing only after the next turn: 1" in readme - assert "- silent lower bound after next-3 sensitivity: 0.0%" in readme - assert "### Marker Calibration" in readme - assert "- calibration sample size: 3" in readme - assert "- acknowledged-marker precision: 100.0%" in readme - assert "- acknowledged-marker recall: 50.0%" in readme - assert "`public-summary.json`" in readme - assert "`PUBLIC_REPRODUCTION.md`" in readme - assert "`COLD_READER_GATE.md`" in readme - assert "- claude-code-session: inspected 2 / 2 structured failures (requested 2)" in readme - assert "- codex-session: inspected 2 / 2 structured failures (requested 2)" in readme - - -def test_claim_vs_evidence_bounded_sample_is_origin_stratified(tmp_path: Path) -> None: - archive = tmp_path / "archive" - _seed_archive(archive) - - report = build_report( - _report_args( - archive_root=archive, - out_dir=None, - limit=2, - sample_limit=2, - ) - ) - - assert report["sample_frame"]["complete_failure_frame"] is False - assert report["sample_frame"]["sampled_by_origin"] == [ - { - "inspected_structured_failures": 1, - "origin": "claude-code-session", - "requested_limit": 1, - "total_structured_failures": 2, - }, - { - "inspected_structured_failures": 1, - "origin": "codex-session", - "requested_limit": 1, - "total_structured_failures": 2, - }, - ] - assert {row["name"] for row in report["by_origin"]} == {"claude-code-session", "codex-session"} - - -def test_claim_vs_evidence_refuses_rates_for_cells_below_n_min(tmp_path: Path) -> None: - archive = tmp_path / "archive" - _seed_archive(archive) - - report = build_report( - _report_args( - archive_root=archive, - out_dir=None, - limit=4, - sample_limit=2, - n_min=3, - ) - ) - - assert report["sample_frame"]["n_min"] == 3 - assert "publication_status=not_supported" in report["sample_frame"]["thin_cell_policy"] - by_model = {str(row["name"]): row for row in report["by_model"]} - thin = by_model["claude-haiku"] - assert thin["failed_outcomes"] == 1 - assert thin["coverage_status"] == "insufficient_n" - assert thin["publication_status"] == "not_supported" - assert thin["silent_rate_lower_bound"] is None - assert thin["silent_rate_among_classified"] is None - assert report["rates"]["coverage_status"] == "supported" - assert report["rates"]["classified_coverage_status"] == "insufficient_n" - assert report["rates"]["silent_rate_lower_bound"] == 1 / 4 - assert report["rates"]["silent_rate_among_classified"] is None - - at_threshold = build_report( - _report_args( - archive_root=archive, - out_dir=None, - limit=4, - sample_limit=2, - n_min=2, - ) - ) - by_origin = {str(row["name"]): row for row in at_threshold["by_origin"]} - supported = by_origin["claude-code-session"] - assert supported["failed_outcomes"] == 2 - assert supported["coverage_status"] == "supported" - assert supported["publication_status"] == "supported" - assert supported["silent_rate_lower_bound"] == 0.5 - - benign = next(row for row in at_threshold["by_handler_class"] if row["name"] == "benign_recovery") - assert benign["coverage_status"] == "supported" - assert benign["classified_coverage_status"] == "insufficient_n" - assert benign["classified_publication_status"] == "not_supported" - assert benign["silent_rate_among_classified"] is None - - assert at_threshold["rates"]["coverage_status"] == "supported" - assert at_threshold["rates"]["silent_rate_lower_bound"] == 0.25 - - -def test_claim_vs_evidence_refuses_aggregate_rates_below_n_min(tmp_path: Path) -> None: - archive = tmp_path / "archive" - out_dir = tmp_path / "out" - _seed_archive(archive) - - report = build_report( - _report_args( - archive_root=archive, - out_dir=out_dir, - limit=4, - sample_limit=2, - n_min=5, - ) - ) - - assert report["rates"]["coverage_status"] == "insufficient_n" - assert report["rates"]["publication_status"] == "not_supported" - assert report["rates"]["silent_rate_lower_bound"] is None - assert report["rates"]["window3_silent_rate_lower_bound"] is None - public_summary = json.loads((out_dir / "public-summary.json").read_text()) - assert public_summary["proofs"][1]["silent_rate_lower_bound_coverage_status"] == "insufficient_n" - assert public_summary["proofs"][1]["silent_rate_lower_bound_publication_status"] == "not_supported" - summary = json.loads((out_dir / "summary.json").read_text()) - assert summary["proof_report"]["silent_rate_lower_bound_coverage_status"] == "insufficient_n" - assert summary["proof_report"]["window3_silent_rate_lower_bound_publication_status"] == "not_supported" - public_reproduction = (out_dir / "PUBLIC_REPRODUCTION.md").read_text() - assert "not enough labels (insufficient_n / not_supported)" in public_reproduction - - -def test_claim_vs_evidence_public_reproduction_handles_unlabeled_sample(tmp_path: Path) -> None: - archive = tmp_path / "archive" - out_dir = tmp_path / "out" - _seed_archive(archive) - - report = build_report( - _report_args( - archive_root=archive, - out_dir=out_dir, - limit=4, - sample_limit=2, - ) - ) - - assert report["calibration"]["metrics"]["labeled_rows"] == 0 - public_reproduction = (out_dir / "PUBLIC_REPRODUCTION.md").read_text() - assert "- acknowledged-marker precision: not enough labels" in public_reproduction - assert "- acknowledged-marker recall: not enough labels" in public_reproduction - - -@pytest.mark.asyncio -async def test_claim_vs_evidence_seeded_demo_reproduces_method(tmp_path: Path) -> None: - archive = tmp_path / "demo-archive" - out_dir = tmp_path / "demo-report" - - await seed_demo_archive(archive, force=True, with_overlays=True) - report = build_report( - _report_args( - archive_root=archive, - out_dir=out_dir, - limit=5000, - sample_limit=10, - calibration_size=10, - ) - ) - - assert report["sample_frame"]["total_structured_failures"] == 7 - assert report["sample_frame"]["inspected_structured_failures"] == 7 - assert report["totals"]["acknowledged"] == 3 - assert report["totals"]["silent_proceed"] == 4 - assert report["totals"]["ambiguous"] == 0 - public_summary = json.loads((out_dir / "public-summary.json").read_text()) - assert public_summary["proofs"][0]["total_structured_failures"] == 7 - public_reproduction = (out_dir / "PUBLIC_REPRODUCTION.md").read_text() - assert "Counts will differ because the deterministic demo archive is synthetic." in public_reproduction - - -def test_claim_vs_evidence_keeps_same_message_tool_result_identities(tmp_path: Path) -> None: - archive = tmp_path / "archive" - _seed_archive(archive) - conn = sqlite3.connect(archive / "index.db") - conn.executemany( - """ - INSERT INTO blocks( - message_id, session_id, position, block_type, text, tool_name, tool_id, - tool_input, tool_result_is_error, tool_result_exit_code - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - """, - [ - ("tool-missing-next", "s2", 2, "tool_use", None, "Read", "t6", '{"path":"y"}', None, None), - ("tool-missing-next", "s2", 3, "tool_result", "also nope", None, "t6", None, 0, 1), - ], - ) - conn.commit() - conn.close() - - report = build_report( - _report_args( - archive_root=archive, - out_dir=None, - limit=10, - sample_limit=10, - ) - ) - - assert report["sample_frame"]["total_structured_failures"] == 5 - assert {row["origin"]: row["failed_outcomes"] for row in report["sample_frame"]["total_by_origin"]} == { - "claude-code-session": 2, - "codex-session": 3, - } - codex_samples = report["samples_by_origin_classification"]["codex-session"]["ambiguous"] - assert len(codex_samples) == 3 - assert {sample["tool_result_tool_id"] for sample in codex_samples} == {"t3", "t4", "t6"} - assert {sample["tool_result_message_ref"] for sample in codex_samples} == { - "message:tool-missing-next", - "message:tool-wordless", - } - assert len(report["evidence"]["member_refs"]) == report["totals"]["failed_outcomes"] - assert all(ref.startswith("block:") for ref in report["evidence"]["member_refs"]) - - -def test_claim_vs_evidence_rank_pairs_reused_tool_ids_without_duplicate_members(tmp_path: Path) -> None: - archive = tmp_path / "archive" - _seed_archive(archive) - conn = sqlite3.connect(archive / "index.db") - conn.execute( - "INSERT INTO sessions(session_id, origin, title, created_at_ms, updated_at_ms) VALUES (?, ?, ?, ?, ?)", - ("s4", "codex-session", "reused tool id", 1, 4), - ) - conn.executemany( - "INSERT INTO messages(session_id, message_id, role, position, model_name) VALUES (?, ?, ?, ?, ?)", - [ - ("s4", "dup-tool-1", "tool", 1, "codex"), - ("s4", "dup-next-1", "assistant", 2, "codex"), - ("s4", "dup-tool-2", "tool", 3, "codex"), - ("s4", "dup-next-2", "assistant", 4, "codex"), - ], - ) - conn.executemany( - """ - INSERT INTO blocks( - message_id, session_id, position, block_type, text, tool_name, tool_id, - tool_input, tool_result_is_error, tool_result_exit_code - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - """, - [ - ("dup-tool-1", "s4", 0, "tool_use", None, "Read", "reused", '{"path":"a"}', None, None), - ("dup-tool-1", "s4", 1, "tool_result", "first failure", None, "reused", None, 1, None), - ("dup-next-1", "s4", 0, "text", "Continuing with another path.", None, None, None, None, None), - ("dup-tool-2", "s4", 0, "tool_use", None, "Read", "reused", '{"path":"b"}', None, None), - ("dup-tool-2", "s4", 1, "tool_result", "second failure", None, "reused", None, 1, None), - ("dup-next-2", "s4", 0, "text", "Continuing again.", None, None, None, None, None), - ], - ) - conn.commit() - conn.close() - - report = build_report(_report_args(archive_root=archive, out_dir=None, limit=20, sample_limit=20)) - - member_refs = report["evidence"]["member_refs"] - assert report["totals"]["failed_outcomes"] == 6 - assert len(member_refs) == len(set(member_refs)) == 6 - assert {"block:dup-tool-1:1", "block:dup-tool-2:1"} <= set(member_refs) - - -def test_followup_window_ignores_protocol_user_rows_and_stops_at_authored_human_turn(tmp_path: Path) -> None: - archive = tmp_path / "archive" - _seed_archive(archive) - conn = sqlite3.connect(archive / "index.db") - conn.execute( - "INSERT INTO sessions(session_id, origin, title, created_at_ms, updated_at_ms) VALUES (?, ?, ?, ?, ?)", - ("s4", "codex-session", "authored boundary", 1, 5), - ) - conn.executemany( - """ - INSERT INTO messages( - session_id, message_id, role, position, variant_index, material_origin, model_name - ) VALUES (?, ?, ?, ?, ?, ?, ?) - """, - [ - ("s4", "boundary-tool", "tool", 1, 0, "tool_result", "codex"), - ("s4", "protocol-user", "user", 2, 0, "runtime_protocol", None), - ("s4", "assistant-v0", "assistant", 3, 0, "assistant_authored", "codex"), - ("s4", "assistant-v1", "assistant", 3, 1, "assistant_authored", "codex"), - ("s4", "human-turn", "user", 4, 0, "human_authored", None), - ("s4", "after-human", "assistant", 5, 0, "assistant_authored", "codex"), - ], - ) - conn.executemany( - """ - INSERT INTO blocks( - message_id, session_id, position, block_type, text, tool_name, tool_id, - tool_input, tool_result_is_error, tool_result_exit_code - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - """, - [ - ("boundary-tool", "s4", 0, "tool_use", None, "Read", "boundary", '{"path":"x"}', None, None), - ("boundary-tool", "s4", 1, "tool_result", "failed", None, "boundary", None, 1, None), - ("assistant-v0", "s4", 0, "text", "I will inspect another path.", None, None, None, None, None), - ("assistant-v1", "s4", 0, "text", "Alternative continuation.", None, None, None, None, None), - ("after-human", "s4", 0, "text", "The earlier command failed.", None, None, None, None, None), - ], - ) - conn.commit() - conn.close() - - report = build_report(_report_args(archive_root=archive, out_dir=None, limit=20, sample_limit=20)) - samples = [sample for bucket in report["samples_by_classification"].values() for sample in bucket] - sample = next(item for item in samples if item["tool_result_tool_id"] == "boundary") - - assert sample["next_message_ref"] == "message:assistant-v0" - assert sample["next3_message_refs"] == ["message:assistant-v0", "message:assistant-v1"] - assert "message:after-human" not in sample["next3_message_refs"] - - -def test_split_index_root_is_the_report_file_set_authority(tmp_path: Path) -> None: - default_root = tmp_path / "default" - split_root = tmp_path / "split" - _seed_archive(split_root) - config = Config( - archive_root=default_root, - render_root=tmp_path / "render", - sources=[], - db_path=split_root / "index.db", - ) - - report = build_report( - _report_args(archive_root=None, out_dir=None, limit=10, sample_limit=10), - config=config, - ) - - assert report["archive_root"] == str(split_root) - assert report["index_db"] == str(split_root / "index.db") - - -def test_materializing_cli_holds_writer_exclusion_before_report_collection( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - archive = tmp_path / "archive" - archive.mkdir() - - def _build_report_under_lease(_args: argparse.Namespace, *, config: Config | None = None) -> dict[str, object]: - del config - competing_writer = ActiveWriterLease(archive) - with pytest.raises(RebuildLeaseUnavailableError): - competing_writer.acquire() - return {"archive_root": str(archive), "index_db": str(archive / "index.db"), "totals": {}} - - monkeypatch.setattr(claim_vs_evidence, "build_report", _build_report_under_lease) - monkeypatch.setattr( - "devtools.claim_vs_evidence_evidence.materialize_claim_vs_evidence_evidence_under_lease", - lambda *_args, **_kwargs: { - "query_ref": "query:test", - "result_set_ref": "result-set:test", - "receipt_id": "receipt-test", - "finding_assertion_ids": [], - "public_claim_written": False, - }, - ) - - assert claim_vs_evidence.main(["--archive-root", str(archive), "--materialize-evidence", "--json"]) == 0 diff --git a/tests/unit/devtools/test_claim_vs_evidence_evidence.py b/tests/unit/devtools/test_claim_vs_evidence_evidence.py deleted file mode 100644 index 8e3d02a3f1..0000000000 --- a/tests/unit/devtools/test_claim_vs_evidence_evidence.py +++ /dev/null @@ -1,305 +0,0 @@ -"""Tests for representing claim-vs-evidence report runs as archive evidence.""" - -from __future__ import annotations - -import sqlite3 -from pathlib import Path -from typing import Any - -import pytest - -from devtools.claim_vs_evidence_evidence import ( - build_findings, - build_query_definition, - build_result_set_members, - materialize_claim_vs_evidence_evidence, -) -from polylogue.core.enums import AssertionKind -from polylogue.core.query_identity import query_hash_for_plan -from polylogue.storage.index_generation import ActiveWriterLease, RebuildLeaseUnavailableError -from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_archive_database -from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier -from polylogue.storage.sqlite.archive_tiers.user_write import list_assertion_claims -from polylogue.storage.sqlite.finding_provenance import list_public_finding_inputs -from polylogue.storage.sqlite.query_objects import get_query, get_result_set - - -def _report( - *, - archive_root: Path, - silent: int, - acknowledged: int, - ambiguous: int, - n_min: int = 30, - member_refs: tuple[str, ...] = ("block:s1:tool-1-result:0", "block:s1:tool-2-result:0"), - captured_at: str = "2026-07-18T00:00:00+00:00", -) -> dict[str, Any]: - classified = silent + acknowledged - failed = classified + ambiguous - publishable = failed >= n_min and classified >= n_min - return { - "captured_at": captured_at, - "archive_root": str(archive_root), - "index_db": str(archive_root / "index.db"), - "limit": 5000, - "sample_frame": { - "inspected_structured_failures": failed, - "failure_predicate": "tool_result_is_error = 1 OR tool_result_exit_code != 0", - "classification_scope": "immediately following assistant message only", - "sensitivity_scope": "next 3 assistant messages", - "selection_strategy": "origin-stratified bounded sample", - "n_min": n_min, - }, - "totals": { - "failed_outcomes": failed, - "acknowledged": acknowledged, - "silent_proceed": silent, - "ambiguous": ambiguous, - "classified_outcomes": classified, - }, - "rates": { - "publication_status": "supported" if publishable else "not_supported", - "silent_rate_lower_bound": (silent / failed) if publishable else None, - }, - "handler_class_definition": { - "benign_recovery": ["glob", "grep"], - "consequential": ["bash", "edit"], - "other": "any other tool", - }, - "evidence": {"member_refs": sorted(member_refs)}, - } - - -def test_build_query_definition_is_content_addressed(tmp_path: Path) -> None: - report_a = _report(archive_root=tmp_path, silent=1, acknowledged=1, ambiguous=1, n_min=30) - report_b = _report(archive_root=tmp_path, silent=1, acknowledged=1, ambiguous=1, n_min=30) - report_c = _report(archive_root=tmp_path, silent=1, acknowledged=1, ambiguous=1, n_min=50) - - definition_a = build_query_definition(report_a) - definition_b = build_query_definition(report_b) - definition_c = build_query_definition(report_c) - - hash_a = query_hash_for_plan(definition_a, grain="g", lane="l", rank_policy="r") - hash_b = query_hash_for_plan(definition_b, grain="g", lane="l", rank_policy="r") - hash_c = query_hash_for_plan(definition_c, grain="g", lane="l", rank_policy="r") - - assert hash_a == hash_b - assert hash_a != hash_c - - -def test_build_result_set_members_returns_sorted_refs(tmp_path: Path) -> None: - report = _report( - archive_root=tmp_path, - silent=1, - acknowledged=1, - ambiguous=1, - member_refs=("block:s1:z:0", "block:s1:a:0"), - ) - assert build_result_set_members(report) == ("block:s1:a:0", "block:s1:z:0") - - -def test_build_findings_omits_public_claim_when_not_publishable(tmp_path: Path) -> None: - report = _report(archive_root=tmp_path, silent=2, acknowledged=2, ambiguous=16, n_min=30) - from polylogue.storage.sqlite.query_objects import EvaluationReceipt - - receipt = EvaluationReceipt( - receipt_id="receipt-test", - source_generation="source:absent", - user_generation="user:absent", - index_generation="index:absent", - runtime_build_ref="polylogue:test", - ) - findings = build_findings( - report, - query_reference="query:" + "0" * 64, - result_set_reference="result-set:test", - receipt=receipt, - ) - assert len(findings) == 1 - assert findings[0].public_claim is None - assert "acknowledged" in findings[0].body_text - - -def test_build_findings_includes_public_claim_when_publishable(tmp_path: Path) -> None: - report = _report(archive_root=tmp_path, silent=20, acknowledged=15, ambiguous=5, n_min=30) - from polylogue.storage.sqlite.query_objects import EvaluationReceipt - - receipt = EvaluationReceipt( - receipt_id="receipt-test", - source_generation="source:absent", - user_generation="user:absent", - index_generation="index:absent", - runtime_build_ref="polylogue:test", - ) - findings = build_findings( - report, - query_reference="query:" + "0" * 64, - result_set_reference="result-set:test", - receipt=receipt, - ) - assert len(findings) == 1 - assert findings[0].public_claim is not None - assert findings[0].public_claim.disclosure == "public" - assert "50.0%" in findings[0].public_claim.publication - - -def test_materialize_end_to_end_publishable_run_round_trips_through_public_claims(tmp_path: Path) -> None: - archive_root = tmp_path / "archive" - archive_root.mkdir() - initialize_archive_database(archive_root / "user.db", ArchiveTier.USER) - report = _report( - archive_root=archive_root, - silent=20, - acknowledged=15, - ambiguous=5, - n_min=30, - member_refs=("message:s1:tool-1-result", "message:s1:tool-2-result"), - ) - - result = materialize_claim_vs_evidence_evidence(report, archive_root=archive_root, now_ms=1_000) - - assert result["public_claim_written"] is True - assert len(result["finding_assertion_ids"]) == 1 - - conn = sqlite3.connect(archive_root / "user.db") - conn.row_factory = sqlite3.Row - try: - query_hash = result["query_ref"].removeprefix("query:") - query = get_query(conn, query_hash) - assert query is not None - assert query.grain == "structured-failure-followup" - - result_set_id = result["result_set_ref"].removeprefix("result-set:") - result_set = get_result_set(conn, result_set_id) - assert result_set is not None - assert result_set.member_count == 2 - assert result_set.exactness == "capped" - assert result_set.persistence_class == "finding" - - findings = list_assertion_claims(conn, kinds=(AssertionKind.FINDING,), statuses=None) - assert len(findings) == 1 - assert findings[0].assertion_id in result["finding_assertion_ids"] - - public_inputs = list_public_finding_inputs(conn) - assert len(public_inputs) == 1 - assert public_inputs[0].claim_key == "finding.silent-proceed-lower-bound" - assert public_inputs[0].disclosure == "public" - finally: - conn.close() - - -def test_materialize_unpublishable_run_writes_private_finding_without_public_claim(tmp_path: Path) -> None: - archive_root = tmp_path / "archive" - archive_root.mkdir() - initialize_archive_database(archive_root / "user.db", ArchiveTier.USER) - report = _report(archive_root=archive_root, silent=2, acknowledged=2, ambiguous=16, n_min=30) - - result = materialize_claim_vs_evidence_evidence(report, archive_root=archive_root, now_ms=1_000) - - assert result["public_claim_written"] is False - assert len(result["finding_assertion_ids"]) == 1 - - conn = sqlite3.connect(archive_root / "user.db") - conn.row_factory = sqlite3.Row - try: - findings = list_assertion_claims(conn, kinds=(AssertionKind.FINDING,), statuses=None) - assert len(findings) == 1 - public_inputs = list_public_finding_inputs(conn) - assert public_inputs == () - finally: - conn.close() - - -def test_materialize_is_idempotent_for_a_retried_identical_call(tmp_path: Path) -> None: - """A retried write (same report, same wall-clock) must not duplicate rows.""" - archive_root = tmp_path / "archive" - archive_root.mkdir() - initialize_archive_database(archive_root / "user.db", ArchiveTier.USER) - report = _report(archive_root=archive_root, silent=20, acknowledged=15, ambiguous=5, n_min=30) - - first = materialize_claim_vs_evidence_evidence(report, archive_root=archive_root, now_ms=1_000) - second = materialize_claim_vs_evidence_evidence(report, archive_root=archive_root, now_ms=1_000) - - assert first == second - - conn = sqlite3.connect(archive_root / "user.db") - conn.row_factory = sqlite3.Row - try: - findings = list_assertion_claims(conn, kinds=(AssertionKind.FINDING,), statuses=None) - assert len(findings) == 1 - finally: - conn.close() - - -def test_materialize_at_a_later_time_reuses_query_and_result_set_but_records_a_new_run(tmp_path: Path) -> None: - """A genuine regeneration keeps the stable AnalysisDefinition/result-set identity - - but records its own AnalysisRun receipt and finding row -- the archive should - carry that a re-verification happened at a later corpus/tier state, not silently - collapse it into the first run. - """ - archive_root = tmp_path / "archive" - archive_root.mkdir() - initialize_archive_database(archive_root / "user.db", ArchiveTier.USER) - report = _report(archive_root=archive_root, silent=20, acknowledged=15, ambiguous=5, n_min=30) - - first = materialize_claim_vs_evidence_evidence(report, archive_root=archive_root, now_ms=1_000) - second = materialize_claim_vs_evidence_evidence(report, archive_root=archive_root, now_ms=2_000) - - assert first["query_ref"] == second["query_ref"] - assert first["result_set_ref"] == second["result_set_ref"] - assert first["finding_assertion_ids"] != second["finding_assertion_ids"] - - conn = sqlite3.connect(archive_root / "user.db") - conn.row_factory = sqlite3.Row - try: - findings = list_assertion_claims(conn, kinds=(AssertionKind.FINDING,), statuses=None) - assert len(findings) == 2 - finally: - conn.close() - - -def test_materialize_distinguishes_query_identity_for_the_same_members(tmp_path: Path) -> None: - archive_root = tmp_path / "archive" - archive_root.mkdir() - initialize_archive_database(archive_root / "user.db", ArchiveTier.USER) - first_report = _report(archive_root=archive_root, silent=20, acknowledged=15, ambiguous=5, n_min=30) - second_report = _report(archive_root=archive_root, silent=20, acknowledged=15, ambiguous=5, n_min=50) - - first = materialize_claim_vs_evidence_evidence(first_report, archive_root=archive_root, now_ms=1_000) - second = materialize_claim_vs_evidence_evidence(second_report, archive_root=archive_root, now_ms=2_000) - - assert first["query_ref"] != second["query_ref"] - assert first["result_set_ref"] != second["result_set_ref"] - - -def test_materialize_refuses_a_second_writer_before_opening_user_tier( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - archive_root = tmp_path / "archive" - archive_root.mkdir() - initialize_archive_database(archive_root / "user.db", ArchiveTier.USER) - report = _report(archive_root=archive_root, silent=20, acknowledged=15, ambiguous=5, n_min=30) - monkeypatch.setattr( - "devtools.claim_vs_evidence_evidence.open_daemon_connection", - lambda *_args, **_kwargs: pytest.fail("user.db opened before writer exclusion"), - ) - writer = ActiveWriterLease(archive_root) - writer.acquire() - try: - with pytest.raises(RebuildLeaseUnavailableError): - materialize_claim_vs_evidence_evidence(report, archive_root=archive_root, now_ms=1_000) - finally: - writer.close() - - -def test_materialize_refuses_report_and_durable_tiers_from_different_file_sets(tmp_path: Path) -> None: - report_root = tmp_path / "report-archive" - wrong_root = tmp_path / "wrong-archive" - report_root.mkdir() - wrong_root.mkdir() - initialize_archive_database(wrong_root / "user.db", ArchiveTier.USER) - report = _report(archive_root=report_root, silent=20, acknowledged=15, ambiguous=5, n_min=30) - - with pytest.raises(ValueError, match="does not match materialization root"): - materialize_claim_vs_evidence_evidence(report, archive_root=wrong_root, now_ms=1_000) From 8c5a11ad4329e2d1cfb3b46340ad6547c33be5cf Mon Sep 17 00:00:00 2001 From: Sinity Date: Wed, 19 Aug 2026 02:16:57 +0200 Subject: [PATCH 03/10] chore(devtools): delete turso_probe (concluded storage-backend research) Turso compatibility research (#2257/#2281, June) concluded; direction settled on SQLite independence + Sinex mode. No hook/CI/doc references. Ref .agent/campaigns/2026-08-overhaul/ws-c-devtools-pruning.md --- devtools/turso_probe.py | 709 ------------------------ tests/unit/devtools/test_turso_probe.py | 160 ------ 2 files changed, 869 deletions(-) delete mode 100644 devtools/turso_probe.py delete mode 100644 tests/unit/devtools/test_turso_probe.py diff --git a/devtools/turso_probe.py b/devtools/turso_probe.py deleted file mode 100644 index 7f051bf1e7..0000000000 --- a/devtools/turso_probe.py +++ /dev/null @@ -1,709 +0,0 @@ -"""Probe Turso Database compatibility for Polylogue storage research.""" - -from __future__ import annotations - -import argparse -import importlib -import importlib.util -import json -import shutil -import sqlite3 -import subprocess -import sys -import tempfile -import time -from contextlib import suppress -from dataclasses import asdict, dataclass -from pathlib import Path -from typing import Any, Literal - -ProbeStatus = Literal["pass", "fail", "skip"] - - -@dataclass(frozen=True, slots=True) -class ProbeResult: - name: str - status: ProbeStatus - expected_status: ProbeStatus - summary: str - command: list[str] - stdout: str = "" - stderr: str = "" - - @property - def expected(self) -> bool: - return self.status == self.expected_status - - def to_dict(self) -> dict[str, object]: - payload = asdict(self) - payload["expected"] = self.expected - return payload - - -@dataclass(frozen=True, slots=True) -class BenchmarkResult: - name: str - status: ProbeStatus - summary: str - duration_ms: float | None - row_count: int - db_bytes: int | None - sidecar_bytes: int | None - command: list[str] - stdout: str = "" - stderr: str = "" - - def to_dict(self) -> dict[str, object]: - return asdict(self) - - -SQL_PROBES: tuple[tuple[str, tuple[str, ...], str, ProbeStatus, str], ...] = ( - ( - "strict_tables", - (), - "CREATE TABLE t(id INTEGER PRIMARY KEY, name TEXT) STRICT; " - "INSERT INTO t(name) VALUES ('polylogue'); SELECT count(*) FROM t;", - "pass", - "STRICT tables are required by every archive tier.", - ), - ( - "stored_generated_columns", - ("--experimental-generated-columns",), - "CREATE TABLE t(a INTEGER, b INTEGER GENERATED ALWAYS AS (a + 1) STORED) STRICT;", - "fail", - "Polylogue index.db uses stored generated IDs; this is a direct-swap blocker when unsupported.", - ), - ( - "virtual_generated_columns", - ("--experimental-generated-columns",), - "CREATE TABLE t(a INTEGER, b INTEGER GENERATED ALWAYS AS (a + 1) VIRTUAL) STRICT; " - "INSERT INTO t(a) VALUES (1); SELECT b FROM t;", - "pass", - "JSON-derived virtual columns are useful for tool/search projection fields.", - ), - ( - "fts5_virtual_table", - (), - "CREATE VIRTUAL TABLE messages_fts USING fts5(text);", - "fail", - "Polylogue's current FTS provider uses SQLite FTS5 virtual tables.", - ), - ( - "wal_journal_size_limit", - (), - "PRAGMA journal_mode=wal; PRAGMA journal_size_limit=1000;", - "fail", - "Polylogue's SQLite connection profile uses journal_size_limit to bound WAL sidecars.", - ), - ( - "mvcc_begin_concurrent", - (), - "PRAGMA journal_mode=mvcc; CREATE TABLE t(id INTEGER PRIMARY KEY, name TEXT) STRICT; " - "BEGIN CONCURRENT; INSERT INTO t(name) VALUES ('a'); COMMIT; SELECT count(*) FROM t;", - "pass", - "MVCC and BEGIN CONCURRENT are the concurrency feature worth measuring.", - ), - ( - "cdc_id_mode", - (), - "PRAGMA capture_data_changes_conn('id'); CREATE TABLE t(id INTEGER PRIMARY KEY, name TEXT) STRICT; " - "INSERT INTO t(name) VALUES ('a'); SELECT count(*) FROM turso_cdc;", - "pass", - "CDC is a plausible ops.db/archive-debt signal source if it remains stable.", - ), - ( - "vector_distance", - (), - "SELECT vector_distance_cos(vector32('[1,0]'), vector32('[0,1]')) AS distance;", - "pass", - "Built-in exact vector functions could support a separate vector-provider experiment.", - ), -) - -OPS_BENCHMARK_ROWS = 2500 - - -def _find_python_turso() -> object | None: - return importlib.util.find_spec("turso") - - -def _import_python_turso() -> Any: - return importlib.import_module("turso") - - -def _find_tursodb() -> str | None: - return shutil.which("tursodb") - - -def _run_command( - command: list[str], - *, - stdin: str | None = None, -) -> subprocess.CompletedProcess[str]: - return subprocess.run( - command, - input=stdin, - capture_output=True, - text=True, - check=False, - timeout=20, - ) - - -def _ops_sidecar_bytes(db_path: Path) -> int: - total = 0 - for suffix in ("-wal", "-shm", "-log"): - sidecar = Path(f"{db_path}{suffix}") - if sidecar.exists(): - total += sidecar.stat().st_size - return total - - -def _remove_db_with_sidecars(db_path: Path) -> None: - db_path.unlink(missing_ok=True) - for suffix in ("-wal", "-shm", "-log"): - Path(f"{db_path}{suffix}").unlink(missing_ok=True) - - -def _sqlite_ops_benchmark(*, scratch_dir: Path, row_count: int = OPS_BENCHMARK_ROWS) -> BenchmarkResult: - db_path = scratch_dir / "ops-workload-sqlite.db" - _remove_db_with_sidecars(db_path) - command = [ - sys.executable, - "-c", - "sqlite3 ops.db; insert ops-shaped daemon/convergence rows", - ] - started = time.perf_counter() - conn = sqlite3.connect(db_path) - try: - conn.execute("PRAGMA journal_mode=WAL") - conn.execute("PRAGMA synchronous=NORMAL") - conn.execute("PRAGMA busy_timeout=5000") - conn.executescript( - """ - CREATE TABLE ingest_attempts( - attempt_id INTEGER PRIMARY KEY, - source_path TEXT NOT NULL, - status TEXT NOT NULL, - duration_ms INTEGER NOT NULL - ) STRICT; - CREATE TABLE convergence_debt( - debt_id INTEGER PRIMARY KEY, - stage TEXT NOT NULL, - status TEXT NOT NULL, - updated_at_ms INTEGER NOT NULL - ) STRICT; - CREATE TABLE daemon_events( - event_id INTEGER PRIMARY KEY, - run_id TEXT NOT NULL, - event_type TEXT NOT NULL, - payload_json TEXT NOT NULL - ) STRICT; - """ - ) - conn.executemany( - "INSERT INTO ingest_attempts(source_path, status, duration_ms) VALUES (?, ?, ?)", - ((f"/source/{index % 17}.jsonl", "completed", index % 1000) for index in range(row_count)), - ) - conn.executemany( - "INSERT INTO convergence_debt(stage, status, updated_at_ms) VALUES (?, ?, ?)", - ( - (f"stage-{index % 5}", "open" if index % 7 else "resolved", 1_700_000_000_000 + index) - for index in range(row_count) - ), - ) - conn.executemany( - "INSERT INTO daemon_events(run_id, event_type, payload_json) VALUES (?, ?, ?)", - ( - (f"run-{index % 13}", "stage", json.dumps({"index": index, "stage": index % 5})) - for index in range(row_count) - ), - ) - conn.execute("UPDATE convergence_debt SET status = 'resolved' WHERE debt_id % 11 = 0") - conn.execute("SELECT stage, count(*) FROM convergence_debt WHERE status = 'open' GROUP BY stage").fetchall() - conn.execute("SELECT event_type, count(*) FROM daemon_events GROUP BY event_type").fetchall() - conn.commit() - finally: - conn.close() - duration_ms = (time.perf_counter() - started) * 1000 - return BenchmarkResult( - name="sqlite_ops_workload", - status="pass", - summary="SQLite WAL baseline for an ops.db-shaped write/read workload.", - duration_ms=round(duration_ms, 3), - row_count=row_count * 3, - db_bytes=db_path.stat().st_size if db_path.exists() else None, - sidecar_bytes=_ops_sidecar_bytes(db_path), - command=command, - ) - - -def _turso_ops_sql(row_count: int) -> str: - ingest_rows = ",\n".join( - f"('/source/{index % 17}.jsonl', 'completed', {index % 1000})" for index in range(row_count) - ) - debt_rows = ",\n".join( - (f"('stage-{index % 5}', '{'resolved' if index % 7 == 0 else 'open'}', {1_700_000_000_000 + index})") - for index in range(row_count) - ) - event_rows = ",\n".join( - (f"('run-{index % 13}', 'stage', '{{\"index\":{index},\"stage\":{index % 5}}}')") for index in range(row_count) - ) - return f""" - PRAGMA journal_mode=mvcc; - CREATE TABLE ingest_attempts( - attempt_id INTEGER PRIMARY KEY, - source_path TEXT NOT NULL, - status TEXT NOT NULL, - duration_ms INTEGER NOT NULL - ) STRICT; - CREATE TABLE convergence_debt( - debt_id INTEGER PRIMARY KEY, - stage TEXT NOT NULL, - status TEXT NOT NULL, - updated_at_ms INTEGER NOT NULL - ) STRICT; - CREATE TABLE daemon_events( - event_id INTEGER PRIMARY KEY, - run_id TEXT NOT NULL, - event_type TEXT NOT NULL, - payload_json TEXT NOT NULL - ) STRICT; - BEGIN CONCURRENT; - INSERT INTO ingest_attempts(source_path, status, duration_ms) VALUES - {ingest_rows}; - INSERT INTO convergence_debt(stage, status, updated_at_ms) VALUES - {debt_rows}; - INSERT INTO daemon_events(run_id, event_type, payload_json) VALUES - {event_rows}; - UPDATE convergence_debt SET status = 'resolved' WHERE debt_id % 11 = 0; - COMMIT; - SELECT stage, count(*) FROM convergence_debt WHERE status = 'open' GROUP BY stage; - SELECT event_type, count(*) FROM daemon_events GROUP BY event_type; - """ - - -def _turso_ops_benchmark( - *, - tursodb: str | None, - scratch_dir: Path, - row_count: int = OPS_BENCHMARK_ROWS, -) -> BenchmarkResult: - if tursodb is None: - return BenchmarkResult( - name="turso_ops_workload", - status="skip", - summary="Skipped because `tursodb` is not on PATH.", - duration_ms=None, - row_count=row_count * 3, - db_bytes=None, - sidecar_bytes=None, - command=["tursodb", "--quiet", "ops-workload-turso.db", ""], - ) - db_path = scratch_dir / "ops-workload-turso.db" - _remove_db_with_sidecars(db_path) - sql = _turso_ops_sql(row_count) - command = [tursodb, "--quiet", str(db_path)] - public_command = [tursodb, "--quiet", str(db_path), "< ops-workload.sql"] - started = time.perf_counter() - result = _run_command(command, stdin=sql) - duration_ms = (time.perf_counter() - started) * 1000 - return BenchmarkResult( - name="turso_ops_workload", - status="pass" if result.returncode == 0 else "fail", - summary="Turso CLI/MVCC run of the same ops.db-shaped write/read workload.", - duration_ms=round(duration_ms, 3), - row_count=row_count * 3, - db_bytes=db_path.stat().st_size if db_path.exists() else None, - sidecar_bytes=_ops_sidecar_bytes(db_path), - command=public_command, - stdout=result.stdout.strip(), - stderr=result.stderr.strip(), - ) - - -def _python_unavailable_result(name: str, summary: str) -> ProbeResult: - return ProbeResult( - name=name, - status="skip", - expected_status="skip", - summary=summary, - command=[sys.executable, "-c", "import turso"], - ) - - -def _python_binding_probe() -> ProbeResult: - spec = _find_python_turso() - if spec is None: - return _python_unavailable_result( - "python_binding", - "Python package `turso` is not importable in this environment.", - ) - try: - turso = _import_python_turso() - - conn = turso.connect(":memory:") - except Exception as exc: # pragma: no cover - depends on optional external package - return ProbeResult( - name="python_binding", - status="fail", - expected_status="pass", - summary=f"Python package `turso` imported but connect(':memory:') failed: {type(exc).__name__}: {exc}", - command=[sys.executable, "-c", "import turso; turso.connect(':memory:')"], - ) - with suppress(Exception): - conn.close() - return ProbeResult( - name="python_binding", - status="pass", - expected_status="pass", - summary="Python package `turso` imported and opened an in-memory connection.", - command=[sys.executable, "-c", "import turso; turso.connect(':memory:')"], - ) - - -def _python_runtime_api_probe() -> ProbeResult: - if _find_python_turso() is None: - return _python_unavailable_result( - "python_runtime_api", - "Skipped because Python package `turso` is not importable.", - ) - command = [ - sys.executable, - "-c", - "import turso; c=turso.connect(':memory:'); c.execute('select 1')", - ] - conn: Any | None = None - try: - turso = _import_python_turso() - conn = turso.connect(":memory:") - conn.execute("CREATE TABLE t(id INTEGER PRIMARY KEY, name TEXT) STRICT") - cursor = conn.execute("INSERT INTO t(name) VALUES (?)", ("polylogue",)) - rowcount = cursor.rowcount - lastrowid = cursor.lastrowid - conn.commit() - conn.row_factory = turso.Row - row = conn.execute("SELECT id, name FROM t WHERE name = ?", ("polylogue",)).fetchone() - if row is None: - raise RuntimeError("row_factory query returned no row") - if row["name"] != "polylogue": - raise RuntimeError("row_factory did not support name lookup") - if rowcount != 1: - raise RuntimeError(f"unexpected rowcount {rowcount!r}") - if lastrowid is None: - raise RuntimeError("lastrowid was not populated") - conn.executescript("CREATE TABLE aux(id INTEGER PRIMARY KEY) STRICT; INSERT INTO aux VALUES (1);") - aux_row = conn.execute("SELECT count(*) FROM aux").fetchone() - if aux_row is None or aux_row[0] != 1: - raise RuntimeError(f"executescript did not create aux row: {aux_row!r}") - conn.execute("INSERT INTO t(name) VALUES ('rollback-target')") - conn.rollback() - names = [record[0] for record in conn.execute("SELECT name FROM t ORDER BY id")] - if names != ["polylogue"]: - raise RuntimeError(f"rollback did not restore expected rows: {names!r}") - conn.execute("PRAGMA user_version=42") - version_row = conn.execute("PRAGMA user_version").fetchone() - if version_row is None or version_row[0] != 42: - raise RuntimeError(f"PRAGMA user_version roundtrip failed: {version_row!r}") - except Exception as exc: # pragma: no cover - depends on optional external package - return ProbeResult( - name="python_runtime_api", - status="fail", - expected_status="pass", - summary=f"Python runtime API smoke failed: {type(exc).__name__}: {exc}", - command=command, - ) - finally: - if conn is not None: - with suppress(Exception): - conn.close() - return ProbeResult( - name="python_runtime_api", - status="pass", - expected_status="pass", - summary="Parameter binding, row_factory, executescript, rollback, cursor metadata, and PRAGMA user_version work.", - command=command, - ) - - -def _python_readonly_uri_probe(*, scratch_dir: Path) -> ProbeResult: - if _find_python_turso() is None: - return _python_unavailable_result( - "python_readonly_uri", - "Skipped because Python package `turso` is not importable.", - ) - db_path = scratch_dir / "python-readonly-uri.db" - command = [ - sys.executable, - "-c", - "import turso; turso.connect('file:archive.db?mode=ro')", - ] - writer: Any | None = None - reader: Any | None = None - try: - turso = _import_python_turso() - db_path.unlink(missing_ok=True) - writer = turso.connect(str(db_path)) - writer.execute("CREATE TABLE t(id INTEGER PRIMARY KEY, name TEXT) STRICT") - writer.execute("INSERT INTO t(name) VALUES ('seed')") - writer.commit() - writer.close() - reader = turso.connect(f"file:{db_path}?mode=ro") - count_row = reader.execute("SELECT count(*) FROM t").fetchone() - if count_row is None or count_row[0] != 1: - raise RuntimeError(f"read-only URI did not read seeded row: {count_row!r}") - try: - reader.execute("INSERT INTO t(name) VALUES ('should-fail')") - reader.commit() - except Exception: - pass - else: - raise RuntimeError("read-only URI allowed a write") - except Exception as exc: # pragma: no cover - depends on optional external package - return ProbeResult( - name="python_readonly_uri", - status="fail", - expected_status="fail", - summary=f"sqlite3-style file:...?mode=ro URI is not usable as Polylogue's readonly connection pattern: {type(exc).__name__}: {exc}", - command=command, - ) - finally: - if writer is not None: - with suppress(Exception): - writer.close() - if reader is not None: - with suppress(Exception): - reader.close() - return ProbeResult( - name="python_readonly_uri", - status="pass", - expected_status="fail", - summary="sqlite3-style file:...?mode=ro URI opened and rejected writes.", - command=command, - ) - - -def _python_multiprocess_probe(*, scratch_dir: Path) -> ProbeResult: - if _find_python_turso() is None: - return _python_unavailable_result( - "python_multiprocess_wal", - "Skipped because Python package `turso` is not importable.", - ) - command = [ - sys.executable, - "-c", - "import turso; turso.connect('archive.db', experimental_features='multiprocess_wal')", - ] - conn: Any | None = None - try: - turso = _import_python_turso() - db_path = scratch_dir / "python-multiprocess-wal.db" - db_path.unlink(missing_ok=True) - conn = turso.connect(str(db_path), experimental_features="multiprocess_wal") - conn.execute("CREATE TABLE t(id INTEGER PRIMARY KEY) STRICT") - except Exception as exc: # pragma: no cover - depends on optional external package - return ProbeResult( - name="python_multiprocess_wal", - status="fail", - expected_status="pass", - summary=f"Python multiprocess_wal connection failed: {type(exc).__name__}: {exc}", - command=command, - ) - finally: - if conn is not None: - with suppress(Exception): - conn.close() - return ProbeResult( - name="python_multiprocess_wal", - status="pass", - expected_status="pass", - summary="Python binding accepts experimental_features='multiprocess_wal'.", - command=command, - ) - - -def _run_tursodb_sql( - *, - tursodb: str, - scratch_dir: Path, - name: str, - flags: tuple[str, ...], - sql: str, - expected_status: ProbeStatus, - summary: str, -) -> ProbeResult: - db_path = scratch_dir / f"{name}.db" - command = [tursodb, "--quiet", *flags, str(db_path), sql] - result = _run_command(command) - status: ProbeStatus = "pass" if result.returncode == 0 else "fail" - return ProbeResult( - name=name, - status=status, - expected_status=expected_status, - summary=summary, - command=command, - stdout=result.stdout.strip(), - stderr=result.stderr.strip(), - ) - - -def _attach_probe(*, tursodb: str, scratch_dir: Path) -> ProbeResult: - sibling = scratch_dir / "attach_sibling.db" - command_seed = [ - tursodb, - "--quiet", - str(sibling), - "CREATE TABLE b(id INTEGER PRIMARY KEY) STRICT; INSERT INTO b VALUES (1);", - ] - seed = _run_command(command_seed) - if seed.returncode != 0: - return ProbeResult( - name="attach_experimental", - status="fail", - expected_status="pass", - summary="Failed to create the sibling database used for ATTACH probing.", - command=command_seed, - stdout=seed.stdout.strip(), - stderr=seed.stderr.strip(), - ) - db_path = scratch_dir / "attach_parent.db" - sql = f"ATTACH DATABASE '{sibling}' AS sibling; SELECT count(*) FROM sibling.b;" - command = [tursodb, "--quiet", "--experimental-attach", str(db_path), sql] - result = _run_command(command) - return ProbeResult( - name="attach_experimental", - status="pass" if result.returncode == 0 else "fail", - expected_status="pass", - summary="Polylogue cross-tier reads currently depend on ATTACH-style access.", - command=command, - stdout=result.stdout.strip(), - stderr=result.stderr.strip(), - ) - - -def run_probe(*, tursodb: str | None = None, scratch_dir: Path | None = None) -> dict[str, object]: - resolved_tursodb = tursodb or _find_tursodb() - if scratch_dir is None: - tmp_context = tempfile.TemporaryDirectory() - scratch_root = Path(tmp_context.name) - else: - tmp_context = None - scratch_root = scratch_dir - try: - results = [ - _python_binding_probe(), - _python_runtime_api_probe(), - _python_readonly_uri_probe(scratch_dir=scratch_root), - _python_multiprocess_probe(scratch_dir=scratch_root), - ] - if resolved_tursodb is None: - results.append( - ProbeResult( - name="tursodb_binary", - status="skip", - expected_status="skip", - summary="`tursodb` is not on PATH; CLI feature probes were not run.", - command=["tursodb", "--version"], - ) - ) - else: - with tempfile.TemporaryDirectory(dir=scratch_root) as tmp: - root = Path(tmp) - for name, flags, sql, expected_status, summary in SQL_PROBES: - results.append( - _run_tursodb_sql( - tursodb=resolved_tursodb, - scratch_dir=root, - name=name, - flags=flags, - sql=sql, - expected_status=expected_status, - summary=summary, - ) - ) - results.append(_attach_probe(tursodb=resolved_tursodb, scratch_dir=root)) - benchmark_root = scratch_root / "benchmarks" - benchmark_root.mkdir(parents=True, exist_ok=True) - benchmarks = [ - _sqlite_ops_benchmark(scratch_dir=benchmark_root), - _turso_ops_benchmark(tursodb=resolved_tursodb, scratch_dir=benchmark_root), - ] - finally: - if tmp_context is not None: - tmp_context.cleanup() - blockers = [ - result.name - for result in results - if ( - (result.name == "python_binding" and result.status != "pass") - or (result.name == "python_readonly_uri" and result.status == "fail") - or (result.name in {"stored_generated_columns", "fts5_virtual_table"} and result.status != "pass") - ) - ] - unexpected = [result.name for result in results if not result.expected] - return { - "ok": not unexpected, - "tursodb": resolved_tursodb, - "results": [result.to_dict() for result in results], - "benchmarks": [benchmark.to_dict() for benchmark in benchmarks], - "compatibility_blockers": blockers, - "unexpected": unexpected, - "recommendation": _recommendation(blockers=blockers, unexpected=unexpected), - } - - -def _recommendation(*, blockers: list[str], unexpected: list[str]) -> str: - if unexpected: - return "Probe behavior changed; inspect unexpected results before drawing storage conclusions." - if blockers: - return ( - "Do not attempt a drop-in backend swap. Use this evidence to scope a tier-specific experiment, " - "with ops.db and a separate vector provider as the lowest-risk candidates." - ) - return "No immediate blocker found by the small probe; proceed to representative workload benchmarks." - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--json", action="store_true", help="Emit the probe payload as JSON.") - parser.add_argument("--tursodb", help="Path to the tursodb binary. Defaults to PATH lookup.") - parser.add_argument( - "--scratch-dir", - type=Path, - default=Path(".cache/turso-probe"), - help="Directory for temporary probe databases.", - ) - parser.add_argument( - "--check", action="store_true", help="Return non-zero when probe outcomes differ from expectations." - ) - args = parser.parse_args(argv) - args.scratch_dir.mkdir(parents=True, exist_ok=True) - payload = run_probe(tursodb=args.tursodb, scratch_dir=args.scratch_dir) - if args.json: - print(json.dumps(payload, indent=2, sort_keys=True)) - else: - print(f"Turso probe: {'ok' if payload['ok'] else 'unexpected'}") - print(f" tursodb: {payload['tursodb'] or 'missing'}") - compatibility_blockers = payload["compatibility_blockers"] - assert isinstance(compatibility_blockers, list) - print(f" blockers: {', '.join(str(blocker) for blocker in compatibility_blockers) or 'none'}") - results = payload["results"] - assert isinstance(results, list) - for result in results: - assert isinstance(result, dict) - marker = "expected" if result["expected"] else "UNEXPECTED" - print(f" - {result['name']}: {result['status']} ({marker})") - benchmarks = payload["benchmarks"] - assert isinstance(benchmarks, list) - print(" benchmarks:") - for benchmark in benchmarks: - assert isinstance(benchmark, dict) - duration = benchmark["duration_ms"] - duration_text = f"{duration} ms" if duration is not None else "n/a" - print(f" - {benchmark['name']}: {benchmark['status']} ({duration_text})") - print(f" recommendation: {payload['recommendation']}") - return 1 if args.check and not payload["ok"] else 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tests/unit/devtools/test_turso_probe.py b/tests/unit/devtools/test_turso_probe.py deleted file mode 100644 index 9998faf491..0000000000 --- a/tests/unit/devtools/test_turso_probe.py +++ /dev/null @@ -1,160 +0,0 @@ -from __future__ import annotations - -import subprocess -from pathlib import Path -from typing import Any, cast - -import pytest - -from devtools import turso_probe - - -def test_probe_reports_missing_python_binding_and_binary( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr(turso_probe, "_find_python_turso", lambda: None) - monkeypatch.setattr(turso_probe, "_find_tursodb", lambda: None) - - payload = turso_probe.run_probe(scratch_dir=tmp_path) - - assert payload["ok"] is True - assert payload["tursodb"] is None - assert payload["compatibility_blockers"] == ["python_binding"] - benchmarks = cast(list[dict[str, object]], payload["benchmarks"]) - benchmark_rows = {row["name"]: row for row in benchmarks} - assert benchmark_rows["sqlite_ops_workload"]["status"] == "pass" - assert benchmark_rows["turso_ops_workload"]["status"] == "skip" - rows = cast(list[dict[str, object]], payload["results"]) - results = {row["name"]: row for row in rows} - assert results["python_binding"]["status"] == "skip" - assert results["python_runtime_api"]["status"] == "skip" - assert results["python_readonly_uri"]["status"] == "skip" - assert results["python_multiprocess_wal"]["status"] == "skip" - assert results["tursodb_binary"]["status"] == "skip" - - -def test_probe_classifies_polylogue_sql_compatibility( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr(turso_probe, "_find_python_turso", lambda: None) - - def fake_run(command: list[str], *, stdin: str | None = None) -> subprocess.CompletedProcess[str]: - sql = stdin if stdin is not None else command[-1] - failing = "STORED" in sql or "fts5" in sql or "journal_size_limit" in sql - return subprocess.CompletedProcess( - command, - 1 if failing else 0, - stdout="" if failing else "ok", - stderr="unsupported" if failing else "", - ) - - monkeypatch.setattr(turso_probe, "_run_command", fake_run) - - payload = turso_probe.run_probe(tursodb="/fake/tursodb", scratch_dir=tmp_path) - - assert payload["ok"] is True - assert payload["unexpected"] == [] - assert payload["compatibility_blockers"] == [ - "python_binding", - "stored_generated_columns", - "fts5_virtual_table", - ] - benchmarks = cast(list[dict[str, object]], payload["benchmarks"]) - benchmark_rows = {row["name"]: row for row in benchmarks} - assert benchmark_rows["sqlite_ops_workload"]["status"] == "pass" - assert benchmark_rows["turso_ops_workload"]["status"] == "pass" - assert benchmark_rows["turso_ops_workload"]["row_count"] == turso_probe.OPS_BENCHMARK_ROWS * 3 - rows = cast(list[dict[str, object]], payload["results"]) - results = {row["name"]: row for row in rows} - assert results["strict_tables"]["status"] == "pass" - assert results["stored_generated_columns"]["status"] == "fail" - assert results["stored_generated_columns"]["expected"] is True - assert results["fts5_virtual_table"]["status"] == "fail" - assert results["wal_journal_size_limit"]["status"] == "fail" - assert results["attach_experimental"]["status"] == "pass" - - -def test_python_readonly_uri_failure_is_a_compatibility_blocker( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr( - turso_probe, - "_python_binding_probe", - lambda: turso_probe.ProbeResult( - name="python_binding", - status="pass", - expected_status="pass", - summary="available", - command=["python", "-c", "import turso"], - ), - ) - monkeypatch.setattr( - turso_probe, - "_python_runtime_api_probe", - lambda: turso_probe.ProbeResult( - name="python_runtime_api", - status="pass", - expected_status="pass", - summary="runtime ok", - command=["python", "-c", "import turso"], - ), - ) - monkeypatch.setattr( - turso_probe, - "_python_readonly_uri_probe", - lambda *, scratch_dir: turso_probe.ProbeResult( - name="python_readonly_uri", - status="fail", - expected_status="fail", - summary=f"readonly incompatible in {scratch_dir}", - command=["python", "-c", "import turso"], - ), - ) - monkeypatch.setattr( - turso_probe, - "_python_multiprocess_probe", - lambda *, scratch_dir: turso_probe.ProbeResult( - name="python_multiprocess_wal", - status="pass", - expected_status="pass", - summary=f"multiprocess ok in {scratch_dir}", - command=["python", "-c", "import turso"], - ), - ) - monkeypatch.setattr(turso_probe, "_find_tursodb", lambda: None) - - payload = turso_probe.run_probe(scratch_dir=tmp_path) - - assert payload["ok"] is True - assert payload["unexpected"] == [] - assert payload["compatibility_blockers"] == ["python_readonly_uri"] - - -def test_main_json_emits_probe_payload( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - def fake_probe(*, tursodb: str | None, scratch_dir: Path | None) -> dict[str, Any]: - _ = scratch_dir - return { - "ok": True, - "tursodb": tursodb, - "results": [], - "benchmarks": [], - "compatibility_blockers": [], - "unexpected": [], - "recommendation": "ok", - } - - monkeypatch.setattr( - turso_probe, - "run_probe", - fake_probe, - ) - - assert turso_probe.main(["--json", "--scratch-dir", str(tmp_path), "--tursodb", "/fake"]) == 0 - assert '"tursodb": "/fake"' in capsys.readouterr().out From ac4b667dd2829d0facaaa0533d91467cbae7d910 Mon Sep 17 00:00:00 2001 From: Sinity Date: Wed, 19 Aug 2026 02:17:00 +0200 Subject: [PATCH 04/10] chore(devtools): delete resume_ranking_eval (orphaned probe) No catalog entry, no importers beyond its own test, no hook/CI/doc references. Ref .agent/campaigns/2026-08-overhaul/ws-c-devtools-pruning.md --- devtools/resume_ranking_eval.py | 467 ------------------ .../unit/devtools/test_resume_ranking_eval.py | 105 ---- 2 files changed, 572 deletions(-) delete mode 100644 devtools/resume_ranking_eval.py delete mode 100644 tests/unit/devtools/test_resume_ranking_eval.py diff --git a/devtools/resume_ranking_eval.py b/devtools/resume_ranking_eval.py deleted file mode 100644 index 2ccc8b548c..0000000000 --- a/devtools/resume_ranking_eval.py +++ /dev/null @@ -1,467 +0,0 @@ -"""Offline before/after evaluation for resume-candidate ranking changes. - -The fixture format carries a current-work context, a candidate profile pool, -and lineage identifiers. Ground truth is derived from that lineage rather than -from an evaluator-only relevance label. Both ranking variants call the -production scorer in :mod:`polylogue.insights.resume`. -""" - -from __future__ import annotations - -import argparse -import json -import sys -import tempfile -from collections import defaultdict -from dataclasses import asdict, dataclass -from pathlib import Path -from typing import Literal - -from pydantic import BaseModel, ConfigDict, Field, ValidationError, model_validator - -from polylogue.insights.archive import ArchiveInsightProvenance, SessionProfileInsight -from polylogue.insights.archive_models import SessionEvidencePayload, SessionInferencePayload -from polylogue.insights.resume import ( - ResumeCandidate, - _PathResolutionContext, - _rank_resume_profiles, - _score_file_overlap, -) - -DEFAULT_FIXTURE = Path(__file__).resolve().parents[1] / "tests" / "data" / "resume-ranking-eval-v1.json" -FixtureSource = Literal["synthetic", "snapshot-derived"] - - -class _FixtureModel(BaseModel): - model_config = ConfigDict(extra="forbid", frozen=True) - - -class CurrentWorkFixture(_FixtureModel): - session_id: str - logical_session_id: str - parent_session_id: str | None = None - recent_files: tuple[str, ...] - cwd: str | None = None - - -class CandidateFixture(_FixtureModel): - session_id: str - logical_session_id: str - parent_session_id: str | None = None - title: str - last_message_at: str - file_paths_touched: tuple[str, ...] - repo_root_alias: str | None = None - cwd_paths: tuple[str, ...] = () - terminal_state: str = "unknown" - workflow_shape: str = "unknown" - - -class RankingScenarioFixture(_FixtureModel): - id: str - source: FixtureSource - present_paths: tuple[str, ...] - current: CurrentWorkFixture - candidates: tuple[CandidateFixture, ...] - - @model_validator(mode="after") - def _require_candidate_pool(self) -> RankingScenarioFixture: - if not self.candidates: - raise ValueError("ranking scenario must include at least one candidate") - return self - - -class EvidenceSampleFixture(_FixtureModel): - id: str - source: FixtureSource - resolvable_count: int = Field(ge=0) - recoverable_dead_count: int = Field(ge=0) - unrecoverable_dead_count: int = Field(ge=0) - - @model_validator(mode="after") - def _require_evidence(self) -> EvidenceSampleFixture: - if self.resolvable_count + self.recoverable_dead_count + self.unrecoverable_dead_count == 0: - raise ValueError("evidence sample must include at least one path") - return self - - -class RankingEvaluationFixture(_FixtureModel): - version: int - description: str - scenarios: tuple[RankingScenarioFixture, ...] - evidence_samples: tuple[EvidenceSampleFixture, ...] = () - - @model_validator(mode="after") - def _validate_version_and_scenarios(self) -> RankingEvaluationFixture: - if self.version != 1: - raise ValueError(f"unsupported resume ranking fixture version: {self.version}") - if not self.scenarios: - raise ValueError("fixture must include at least one ranking scenario") - scenario_ids = [scenario.id for scenario in self.scenarios] - if len(scenario_ids) != len(set(scenario_ids)): - raise ValueError("ranking scenario ids must be unique") - return self - - -@dataclass(frozen=True, slots=True) -class RankingMetrics: - scenarios: int - hit_at_1: float - hit_at_3: float - mrr: float - - -@dataclass(frozen=True, slots=True) -class BeforeAfterMetrics: - before: RankingMetrics - after: RankingMetrics - - -@dataclass(frozen=True, slots=True) -class ScenarioEvaluation: - scenario_id: str - source: FixtureSource - target_logical_session_ids: tuple[str, ...] - before_rank: int | None - after_rank: int | None - before_order: tuple[str, ...] - after_order: tuple[str, ...] - after_overlap_basis: dict[str, object] - - -@dataclass(frozen=True, slots=True) -class EvidenceRecoveryEvaluation: - sample_id: str - source: FixtureSource - path_count: int - resolvable_count: int - dead_count: int - directory_recovered_count: int - dead_excluded_count: int - usable_share_before: float - usable_share_after: float - dead_recovery_rate: float - - -@dataclass(frozen=True, slots=True) -class EvaluationVerdict: - non_regressing: bool - strict_overall_improvement: bool - all_fixed_targets_hit_at_1: bool - - -@dataclass(frozen=True, slots=True) -class EvaluationReport: - fixture_version: int - fixture_description: str - metrics: dict[str, BeforeAfterMetrics] - scenarios: tuple[ScenarioEvaluation, ...] - evidence_recovery: tuple[EvidenceRecoveryEvaluation, ...] - verdict: EvaluationVerdict - - def to_dict(self) -> dict[str, object]: - return asdict(self) - - -def load_fixture(path: Path) -> RankingEvaluationFixture: - """Load and strictly validate one ranking-evaluation fixture.""" - - return RankingEvaluationFixture.model_validate_json(path.read_text(encoding="utf-8")) - - -def _safe_fixture_path(repo_root: Path, relative_path: str) -> Path: - candidate = Path(relative_path) - if candidate.is_absolute(): - raise ValueError(f"present_paths entries must be repo-relative: {relative_path}") - resolved = (repo_root / candidate).resolve(strict=False) - try: - resolved.relative_to(repo_root) - except ValueError as exc: - raise ValueError(f"fixture path escapes repo root: {relative_path}") from exc - return resolved - - -def _materialize_present_paths(repo_root: Path, paths: tuple[str, ...]) -> None: - for relative_path in paths: - path = _safe_fixture_path(repo_root, relative_path) - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(f"# fixture: {relative_path}\n", encoding="utf-8") - - -def _profile_from_fixture(candidate: CandidateFixture, repo_root: Path) -> SessionProfileInsight: - profile_repo_root = candidate.repo_root_alias or str(repo_root) - return SessionProfileInsight( - session_id=candidate.session_id, - logical_session_id=candidate.logical_session_id, - origin="claude-code", - title=candidate.title, - provenance=ArchiveInsightProvenance( - materializer_version=1, - materialized_at=candidate.last_message_at, - source_updated_at=candidate.last_message_at, - ), - evidence=SessionEvidencePayload( - last_message_at=candidate.last_message_at, - repo_paths=(profile_repo_root,), - cwd_paths=candidate.cwd_paths, - file_paths_touched=candidate.file_paths_touched, - parent_id=candidate.parent_session_id, - logical_session_id=candidate.logical_session_id, - ), - inference=SessionInferencePayload( - terminal_state=candidate.terminal_state, - workflow_shape=candidate.workflow_shape, - ), - ) - - -def _lineage_targets(scenario: RankingScenarioFixture) -> tuple[str, ...]: - current = scenario.current - target_logical_ids: set[str] = set() - for candidate in scenario.candidates: - same_logical_family = candidate.logical_session_id == current.logical_session_id - true_parent = current.parent_session_id is not None and candidate.session_id == current.parent_session_id - true_sibling = ( - current.parent_session_id is not None - and candidate.parent_session_id is not None - and candidate.parent_session_id == current.parent_session_id - ) - if same_logical_family or true_parent or true_sibling: - target_logical_ids.add(candidate.logical_session_id) - if not target_logical_ids: - raise ValueError(f"scenario {scenario.id!r} has no lineage-derived resume target") - return tuple(sorted(target_logical_ids)) - - -def _first_relevant_rank(candidates: tuple[ResumeCandidate, ...], targets: tuple[str, ...]) -> int | None: - target_set = set(targets) - for rank, candidate in enumerate(candidates, start=1): - if candidate.logical_session_id in target_set: - return rank - return None - - -def _target_basis(candidates: tuple[ResumeCandidate, ...], targets: tuple[str, ...]) -> dict[str, object]: - target_set = set(targets) - for candidate in candidates: - if candidate.logical_session_id in target_set: - return candidate.overlap_basis.model_dump(mode="json") - return {} - - -def evaluate_scenario(scenario: RankingScenarioFixture) -> ScenarioEvaluation: - """Run one fixture through the production legacy and fixed rankers.""" - - with tempfile.TemporaryDirectory(prefix=f"polylogue-resume-eval-{scenario.id}-") as temporary: - repo_root = Path(temporary).resolve() - _materialize_present_paths(repo_root, scenario.present_paths) - profiles = [_profile_from_fixture(candidate, repo_root) for candidate in scenario.candidates] - target_ids = _lineage_targets(scenario) - logical_pool_size = len({candidate.logical_session_id for candidate in scenario.candidates}) - before = _rank_resume_profiles( - profiles, - repo_path=str(repo_root), - cwd=scenario.current.cwd, - recent_files=scenario.current.recent_files, - limit=logical_pool_size, - overlap_mode="legacy", - ) - after = _rank_resume_profiles( - profiles, - repo_path=str(repo_root), - cwd=scenario.current.cwd, - recent_files=scenario.current.recent_files, - limit=logical_pool_size, - overlap_mode="refactor-aware", - ) - - return ScenarioEvaluation( - scenario_id=scenario.id, - source=scenario.source, - target_logical_session_ids=target_ids, - before_rank=_first_relevant_rank(before, target_ids), - after_rank=_first_relevant_rank(after, target_ids), - before_order=tuple(candidate.logical_session_id for candidate in before), - after_order=tuple(candidate.logical_session_id for candidate in after), - after_overlap_basis=_target_basis(after, target_ids), - ) - - -def _ranking_metrics(rows: tuple[ScenarioEvaluation, ...], *, use_after: bool) -> RankingMetrics: - if not rows: - return RankingMetrics(scenarios=0, hit_at_1=0.0, hit_at_3=0.0, mrr=0.0) - ranks = [row.after_rank if use_after else row.before_rank for row in rows] - count = len(ranks) - return RankingMetrics( - scenarios=count, - hit_at_1=round(sum(rank is not None and rank <= 1 for rank in ranks) / count, 6), - hit_at_3=round(sum(rank is not None and rank <= 3 for rank in ranks) / count, 6), - mrr=round(sum(0.0 if rank is None else 1.0 / rank for rank in ranks) / count, 6), - ) - - -def _evaluate_evidence_sample(sample: EvidenceSampleFixture) -> EvidenceRecoveryEvaluation: - with tempfile.TemporaryDirectory(prefix=f"polylogue-resume-evidence-{sample.id}-") as temporary: - repo_root = Path(temporary).resolve() - resolvable = tuple(f"mass/live/file_{index:03d}.py" for index in range(sample.resolvable_count)) - recoverable_dead = tuple( - f"mass/refactor/retired_{index:03d}.py" for index in range(sample.recoverable_dead_count) - ) - replacement_files = tuple( - f"mass/refactor/current_{index:03d}.py" for index in range(sample.recoverable_dead_count) - ) - unrecoverable_dead = tuple( - f"retired/area_{index:03d}/ghost.py" for index in range(sample.unrecoverable_dead_count) - ) - _materialize_present_paths(repo_root, (*resolvable, *replacement_files)) - score = _score_file_overlap( - context=_PathResolutionContext.from_repo_path(str(repo_root)), - recent_files=set(replacement_files), - candidate_paths={*resolvable, *recoverable_dead, *unrecoverable_dead}, - ) - - path_count = sample.resolvable_count + sample.recoverable_dead_count + sample.unrecoverable_dead_count - resolvable_count = len(score.resolvable_paths) - dead_count = len(score.dead_paths) - recovered_count = len(score.basis.dir) - dead_excluded_count = len(score.basis.dead_excluded) - if not score.resolution_available: - raise RuntimeError(f"evidence sample {sample.id!r} could not resolve its temporary repo root") - expected_dead = sample.recoverable_dead_count + sample.unrecoverable_dead_count - if (resolvable_count, dead_count, recovered_count, dead_excluded_count) != ( - sample.resolvable_count, - expected_dead, - sample.recoverable_dead_count, - sample.unrecoverable_dead_count, - ): - raise RuntimeError( - f"evidence sample {sample.id!r} did not exercise the intended partition: " - f"resolved={resolvable_count}, dead={dead_count}, recovered={recovered_count}, " - f"excluded={dead_excluded_count}" - ) - return EvidenceRecoveryEvaluation( - sample_id=sample.id, - source=sample.source, - path_count=path_count, - resolvable_count=resolvable_count, - dead_count=dead_count, - directory_recovered_count=recovered_count, - dead_excluded_count=dead_excluded_count, - usable_share_before=round(resolvable_count / path_count, 6), - usable_share_after=round((resolvable_count + recovered_count) / path_count, 6), - dead_recovery_rate=round(recovered_count / dead_count, 6) if dead_count else 0.0, - ) - - -def evaluate_fixture(fixture: RankingEvaluationFixture) -> EvaluationReport: - """Evaluate every cohort and evidence-recovery sample in one fixture.""" - - scenario_rows = tuple(evaluate_scenario(scenario) for scenario in fixture.scenarios) - cohorts: dict[str, list[ScenarioEvaluation]] = defaultdict(list) - cohorts["overall"].extend(scenario_rows) - for row in scenario_rows: - cohorts[row.source].append(row) - metrics = { - cohort: BeforeAfterMetrics( - before=_ranking_metrics(tuple(rows), use_after=False), - after=_ranking_metrics(tuple(rows), use_after=True), - ) - for cohort, rows in sorted(cohorts.items()) - } - evidence_rows = tuple(_evaluate_evidence_sample(sample) for sample in fixture.evidence_samples) - non_regressing = all( - metric.after.hit_at_1 >= metric.before.hit_at_1 - and metric.after.hit_at_3 >= metric.before.hit_at_3 - and metric.after.mrr >= metric.before.mrr - for metric in metrics.values() - ) - overall = metrics["overall"] - strict_improvement = ( - overall.after.hit_at_1 > overall.before.hit_at_1 - or overall.after.hit_at_3 > overall.before.hit_at_3 - or overall.after.mrr > overall.before.mrr - ) - return EvaluationReport( - fixture_version=fixture.version, - fixture_description=fixture.description, - metrics=metrics, - scenarios=scenario_rows, - evidence_recovery=evidence_rows, - verdict=EvaluationVerdict( - non_regressing=non_regressing, - strict_overall_improvement=strict_improvement, - all_fixed_targets_hit_at_1=all(row.after_rank == 1 for row in scenario_rows), - ), - ) - - -def _pct(value: float) -> str: - return f"{value * 100:.1f}%" - - -def format_report(report: EvaluationReport) -> str: - lines = [ - "Resume ranking evaluation", - f"Fixture v{report.fixture_version}: {report.fixture_description}", - "", - "Ranking quality (before -> after)", - ] - for cohort, metrics in report.metrics.items(): - lines.append( - f" {cohort:16} n={metrics.before.scenarios:<2d} " - f"hit@1 {_pct(metrics.before.hit_at_1)} -> {_pct(metrics.after.hit_at_1)}; " - f"hit@3 {_pct(metrics.before.hit_at_3)} -> {_pct(metrics.after.hit_at_3)}; " - f"MRR {metrics.before.mrr:.3f} -> {metrics.after.mrr:.3f}" - ) - lines.extend(("", "Scenario ranks (before -> after)")) - for row in report.scenarios: - before_rank = str(row.before_rank) if row.before_rank is not None else "miss" - after_rank = str(row.after_rank) if row.after_rank is not None else "miss" - lines.append(f" {row.scenario_id:48} {before_rank} -> {after_rank}") - if report.evidence_recovery: - lines.extend(("", "Evidence usability")) - for evidence_row in report.evidence_recovery: - lines.append( - f" {evidence_row.sample_id}: " - f"{_pct(evidence_row.usable_share_before)} -> {_pct(evidence_row.usable_share_after)} usable; " - f"{_pct(evidence_row.dead_recovery_rate)} of dead paths directory-recovered " - f"({evidence_row.directory_recovered_count}/{evidence_row.dead_count})" - ) - lines.extend( - ( - "", - "Verdict: " - f"non-regressing={str(report.verdict.non_regressing).lower()}, " - f"strict-overall-improvement={str(report.verdict.strict_overall_improvement).lower()}, " - f"all-fixed-targets-hit@1={str(report.verdict.all_fixed_targets_hit_at_1).lower()}", - ) - ) - return "\n".join(lines) - - -def _parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser( - prog="python -m devtools.resume_ranking_eval", - description="Compare legacy and refactor-aware resume ranking over lineage-grounded offline fixtures.", - ) - parser.add_argument("--fixture", type=Path, default=DEFAULT_FIXTURE, help="Path to a v1 evaluation fixture.") - parser.add_argument("--json", action="store_true", help="Emit the complete machine-readable report.") - return parser - - -def main(argv: list[str] | None = None) -> int: - args = _parser().parse_args(argv) - try: - report = evaluate_fixture(load_fixture(args.fixture)) - except (OSError, ValueError, ValidationError, RuntimeError) as exc: - print(f"resume-ranking-eval: {type(exc).__name__}: {exc}", file=sys.stderr) - return 2 - if args.json: - print(json.dumps(report.to_dict(), indent=2, sort_keys=True)) - else: - print(format_report(report)) - return 0 if report.verdict.non_regressing else 1 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tests/unit/devtools/test_resume_ranking_eval.py b/tests/unit/devtools/test_resume_ranking_eval.py deleted file mode 100644 index df09ebf6e0..0000000000 --- a/tests/unit/devtools/test_resume_ranking_eval.py +++ /dev/null @@ -1,105 +0,0 @@ -"""Contracts for the offline resume-ranking quality evaluator.""" - -from __future__ import annotations - -import json - -import pytest - -from devtools import resume_ranking_eval - - -def test_resume_ranking_eval_improves_lineage_grounded_metrics() -> None: - fixture = resume_ranking_eval.load_fixture(resume_ranking_eval.DEFAULT_FIXTURE) - - report = resume_ranking_eval.evaluate_fixture(fixture) - - overall = report.metrics["overall"] - assert overall.before == resume_ranking_eval.RankingMetrics( - scenarios=5, - hit_at_1=0.2, - hit_at_3=0.2, - mrr=0.39, - ) - assert overall.after == resume_ranking_eval.RankingMetrics( - scenarios=5, - hit_at_1=1.0, - hit_at_3=1.0, - mrr=1.0, - ) - assert report.metrics["synthetic"].before.hit_at_1 == 0.333333 - assert report.metrics["snapshot-derived"].before.mrr == 0.25 - assert report.verdict.non_regressing is True - assert report.verdict.strict_overall_improvement is True - assert report.verdict.all_fixed_targets_hit_at_1 is True - - -def test_resume_ranking_eval_catches_legacy_dead_path_anti_selection() -> None: - """Replacing refactor-aware mode with legacy mode makes every asserted repair fail.""" - report = resume_ranking_eval.evaluate_fixture(resume_ranking_eval.load_fixture(resume_ranking_eval.DEFAULT_FIXTURE)) - by_id = {row.scenario_id: row for row in report.scenarios} - - assert by_id["synthetic-dead-shared-directory"].before_rank == 4 - assert by_id["synthetic-dead-shared-directory"].after_rank == 1 - assert by_id["synthetic-dead-union-deflation"].before_rank == 5 - assert by_id["synthetic-dead-union-deflation"].after_rank == 1 - assert by_id["snapshot-storage-repository-file-to-package"].before_rank == 4 - assert by_id["snapshot-storage-repository-file-to-package"].after_rank == 1 - assert by_id["snapshot-pipeline-runner-directory-recovery"].before_rank == 4 - assert by_id["snapshot-pipeline-runner-directory-recovery"].after_rank == 1 - assert by_id["synthetic-live-exact-control"].before_rank == 1 - assert by_id["synthetic-live-exact-control"].after_rank == 1 - - -def test_resume_ranking_eval_reproduces_seeded_evidence_recovery() -> None: - report = resume_ranking_eval.evaluate_fixture(resume_ranking_eval.load_fixture(resume_ranking_eval.DEFAULT_FIXTURE)) - - assert len(report.evidence_recovery) == 1 - evidence = report.evidence_recovery[0] - assert evidence.path_count == 1000 - assert evidence.resolvable_count == 570 - assert evidence.dead_count == 430 - assert evidence.directory_recovered_count == 254 - assert evidence.dead_excluded_count == 176 - assert evidence.usable_share_before == 0.57 - assert evidence.usable_share_after == 0.824 - assert evidence.dead_recovery_rate == pytest.approx(254 / 430, abs=1e-6) - - -def test_resume_ranking_fixture_has_no_manual_relevance_labels() -> None: - raw = json.loads(resume_ranking_eval.DEFAULT_FIXTURE.read_text(encoding="utf-8")) - - for scenario in raw["scenarios"]: - assert "relevant" not in scenario - assert "expected_target" not in scenario - assert scenario["current"]["logical_session_id"] - assert scenario["current"]["parent_session_id"] - - -def test_resume_ranking_metrics_assign_zero_reciprocal_rank_to_a_miss() -> None: - row = resume_ranking_eval.ScenarioEvaluation( - scenario_id="missing-target", - source="synthetic", - target_logical_session_ids=("target",), - before_rank=None, - after_rank=1, - before_order=("distractor",), - after_order=("target",), - after_overlap_basis={}, - ) - - before = resume_ranking_eval._ranking_metrics((row,), use_after=False) - - assert before.hit_at_1 == 0 - assert before.hit_at_3 == 0 - assert before.mrr == 0 - - -def test_resume_ranking_eval_main_emits_json(capsys: pytest.CaptureFixture[str]) -> None: - exit_code = resume_ranking_eval.main(["--fixture", str(resume_ranking_eval.DEFAULT_FIXTURE), "--json"]) - - assert exit_code == 0 - payload = json.loads(capsys.readouterr().out) - assert payload["metrics"]["overall"]["before"]["hit_at_1"] == 0.2 - assert payload["metrics"]["overall"]["after"]["hit_at_1"] == 1.0 - assert payload["evidence_recovery"][0]["usable_share_after"] == 0.824 From dc7333fb476a34710b9ddadf60ccab7cefa53e3f Mon Sep 17 00:00:00 2001 From: Sinity Date: Wed, 19 Aug 2026 02:17:04 +0200 Subject: [PATCH 05/10] chore(devtools): delete proof_world_real_slice (closed uplift campaign) The uplift campaign (jxe) it screened real-archive slices for is closed. No hook/CI/doc references. Ref .agent/campaigns/2026-08-overhaul/ws-c-devtools-pruning.md --- devtools/proof_world_real_slice.py | 440 ------------------ .../devtools/test_proof_world_real_slice.py | 325 ------------- 2 files changed, 765 deletions(-) delete mode 100644 devtools/proof_world_real_slice.py delete mode 100644 tests/unit/devtools/test_proof_world_real_slice.py diff --git a/devtools/proof_world_real_slice.py b/devtools/proof_world_real_slice.py deleted file mode 100644 index afcdb5332d..0000000000 --- a/devtools/proof_world_real_slice.py +++ /dev/null @@ -1,440 +0,0 @@ -"""Real-archive candidate-slice extraction and privacy screening. - -Support for polylogue-212.11 (shared deterministic proof world / Incident -14:32): the deterministic demo corpus should eventually be extended with a -*representative slice of real archive data*, not synthetic fixtures alone. -That extension is deliberately a two-step, human-gated process: - -1. This tool runs **read-only** queries against a real Polylogue archive, - flattens each candidate session to plain text, and screens that text for - secrets/credentials and personal-information patterns. It writes a report - plus rendered transcripts to an arbitrary output directory. It never - mutates the source archive (``Polylogue.get_session`` opens the archive - tiers with ``read_only=True``) and never writes into the product fixture - tree (``polylogue/scenarios/``) on its own. The per-session transcript - files under ``/transcripts/`` are full, unredacted flattened text — - they exist for a human to read the real session content. The - *report/manifest* (``SCREENING_REPORT.md``, ``manifest.json``) are a - different, narrower surface: any matched **secret** value is redacted - before it is written there (see ``scan_text``/``_snippet``), so a report - that later gets shared or accidentally committed doesn't itself become a - secret-leak vector. Matched **PII** text is kept verbatim in the report - since a reviewer needs the real value to judge placeholder vs. genuine - data. Point ``--out`` at a location outside version control (e.g. a - gitignored scratch directory) — this tool applies no guard against - writing into a tracked path. -2. An operator reviews the report and transcripts and decides, session by - session, whether the slice is safe to fold into the shared proof-world - corpus. Only after that explicit approval should the slice move into a - real fixture path. - -The screening pass is a best-effort heuristic layer, not a certification. -"Clean" means "no configured pattern fired" — an operator still has to read -the transcripts before promoting anything to a shared fixture. -""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import re -import sys -from dataclasses import dataclass, field -from pathlib import Path -from typing import TYPE_CHECKING, Any, Protocol - -if TYPE_CHECKING: - from collections.abc import Iterable - - from polylogue.api import Polylogue - - -class _MessageLike(Protocol): - """Structural shape ``_flatten_session_text`` reads from a message. - - A ``Protocol`` (not the concrete ``polylogue.archive.session.domain_models - .Session``/``Message`` classes) so the real session objects returned by - ``Polylogue.get_session`` and the lightweight duck-typed test doubles in - ``tests/unit/devtools/test_proof_world_real_slice.py`` both satisfy the - parameter type structurally, without the tests needing to construct or - subclass the full domain model. Declared as read-only ``@property`` - members rather than plain attributes: mypy checks plain-attribute - Protocol members *invariantly* (both read and write), which the real - ``Message.blocks: list[dict[str, object]]`` fails against a plain - ``blocks: object`` attribute even though every value it can hold is - assignable to ``object``. Properties are read-only, so the check is - covariant instead and both the real model and the test doubles conform. - """ - - @property - def text(self) -> str | None: ... - - @property - def blocks(self) -> object: ... - - -class _SessionLike(Protocol): - """Structural shape ``_flatten_session_text`` reads from a session. - - ``Iterable``, not ``Sequence`` — the real ``Session.messages`` is a - ``MessageCollection`` that supports iteration but is not a nominal - ``collections.abc.Sequence`` subclass, and ``Sequence`` is a concrete ABC - in typeshed (not a structural ``Protocol``) so mypy would reject it here - even though the object is sequence-*shaped*. Only iteration is needed. - """ - - @property - def messages(self) -> Iterable[_MessageLike]: ... - - -# Patterns that indicate a live secret/credential shape. Kept intentionally -# narrow (favor false negatives over drowning the report in noise) — this is -# a triage aid, not a DLP product. -_SECRET_PATTERNS: dict[str, re.Pattern[str]] = { - "aws_access_key_id": re.compile(r"AKIA[0-9A-Z]{16}"), - "generic_credential_assignment": re.compile( - r"(?i)\b(api[_-]?key|secret|password|passwd|access[_-]?token)\b\s*[:=]\s*" - r"['\"]?[A-Za-z0-9_\-/+=.]{12,}" - ), - "bearer_token": re.compile(r"(?i)\bBearer\s+[A-Za-z0-9_\-.=]{16,}"), - "private_key_block": re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----"), - "openai_style_key": re.compile(r"\bsk-[A-Za-z0-9]{20,}\b"), - "slack_token": re.compile(r"\bxox[baprs]-[A-Za-z0-9-]{10,}\b"), - "ssh_public_key": re.compile(r"\bssh-(?:rsa|ed25519) [A-Za-z0-9+/]{20,}"), - "jwt_like": re.compile(r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b"), -} - -# Patterns that may indicate personal information. These fire far more often -# on ordinary dev-work text (localhost IPs, placeholder emails), so callers -# should read the samples rather than treat any hit as disqualifying. -_PII_PATTERNS: dict[str, re.Pattern[str]] = { - "email": re.compile(r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}"), - "home_path": re.compile(r"/home/[a-zA-Z0-9_-]+"), - "ipv4": re.compile(r"\b(?:\d{1,3}\.){3}\d{1,3}\b"), -} - -# Values that are conventionally placeholders/loopback addresses, not real -# personal data. A hit is downgraded only when the *entire* matched string -# equals one of these exactly (never a substring check — "notuser@example.com" -# and "127.0.0.123" must NOT be treated as allowlisted merely because an -# allowlisted value happens to appear inside them). A downgraded hit is still -# reported (never silently dropped). -_ALLOWLIST_VALUES: frozenset[str] = frozenset( - ( - "test@example.com", - "user@example.com", - "127.0.0.1", - "0.0.0.0", - "255.255.255.255", - ) -) - -_SNIPPET_RADIUS = 40 -_MAX_SAMPLES_PER_PATTERN = 5 - - -@dataclass(slots=True) -class PatternHit: - pattern: str - kind: str # "secret" | "pii" - count: int - samples: list[str] = field(default_factory=list) - all_allowlisted: bool = False - - def to_dict(self) -> dict[str, Any]: - return { - "pattern": self.pattern, - "kind": self.kind, - "count": self.count, - "samples": self.samples, - "all_allowlisted": self.all_allowlisted, - } - - -@dataclass(slots=True) -class SessionScreeningResult: - session_id: str - origin: str - title: str | None - created_at: str | None - message_count: int - word_count: int - hits: list[PatternHit] - - @property - def verdict(self) -> str: - secret_hits = [h for h in self.hits if h.kind == "secret"] - if secret_hits: - return "flagged" - pii_hits = [h for h in self.hits if h.kind == "pii" and not h.all_allowlisted] - if pii_hits: - return "review" - return "clean" - - def to_dict(self) -> dict[str, Any]: - return { - "session_id": self.session_id, - "origin": self.origin, - "title": self.title, - "created_at": self.created_at, - "message_count": self.message_count, - "word_count": self.word_count, - "verdict": self.verdict, - "hits": [h.to_dict() for h in self.hits], - } - - -def _snippet(text: str, match: re.Match[str], *, redact: bool) -> str: - """Render a bounded context window around ``match``. - - When ``redact`` is true (secret-kind hits), the matched substring itself - is replaced with a placeholder — the *surrounding* context is still - useful for triage (which pattern fired, roughly where), but the actual - secret value never reaches the report/manifest on disk. PII hits are not - redacted: a human reviewer needs the real matched text (e.g. the actual - email/IP) to judge whether it is a placeholder or genuine personal data. - """ - - start = max(0, match.start() - _SNIPPET_RADIUS) - end = min(len(text), match.end() + _SNIPPET_RADIUS) - prefix = "…" if start > 0 else "" - suffix = "…" if end < len(text) else "" - window = text[start:end] - if redact: - rel_start = match.start() - start - rel_end = match.end() - start - window = f"{window[:rel_start]}{window[rel_end:]}" - return f"{prefix}{window!r}{suffix}" - - -def scan_text(text: str) -> list[PatternHit]: - """Run every configured secret/PII pattern over ``text``. - - Returns one :class:`PatternHit` per pattern that matched at least once, - each carrying up to ``_MAX_SAMPLES_PER_PATTERN`` samples for human - review. Secret-kind samples have the actual matched value redacted (see - :func:`_snippet`) — never the raw secret — to keep the report itself - from becoming a leak surface. PII-kind samples keep the real matched - text, which a reviewer needs to judge placeholder vs. genuine data. - """ - - hits: list[PatternHit] = [] - for kind, patterns in (("secret", _SECRET_PATTERNS), ("pii", _PII_PATTERNS)): - redact = kind == "secret" - for name, pattern in patterns.items(): - matches = list(pattern.finditer(text)) - if not matches: - continue - samples = [_snippet(text, m, redact=redact) for m in matches[:_MAX_SAMPLES_PER_PATTERN]] - all_allowlisted = all(m.group(0) in _ALLOWLIST_VALUES for m in matches) - hits.append( - PatternHit( - pattern=name, - kind=kind, - count=len(matches), - samples=samples, - all_allowlisted=all_allowlisted, - ) - ) - return hits - - -def _flatten_session_text(session: _SessionLike) -> str: - """Flatten every message's text and structured blocks to one string.""" - - parts: list[str] = [] - for message in session.messages: - if message.text: - parts.append(message.text) - if message.blocks: - parts.append(json.dumps(message.blocks, default=str)) - return "\n".join(parts) - - -async def _screen_session_with(poly: Polylogue, session_id: str) -> tuple[SessionScreeningResult, str]: - """Screen one session through an already-open ``Polylogue`` instance.""" - - session = await poly.get_session(session_id) - if session is None: - raise ValueError(f"session not found in archive: {session_id}") - text = _flatten_session_text(session) - word_count = len(text.split()) - result = SessionScreeningResult( - session_id=str(session.id), - origin=str(session.origin), - title=session.title, - created_at=str(session.created_at) if session.created_at else None, - message_count=len(session.messages), - word_count=word_count, - hits=scan_text(text), - ) - return result, text - - -async def screen_session(archive_root: Path, session_id: str) -> tuple[SessionScreeningResult, str]: - """Load one session read-only and screen it. Returns (result, transcript_text). - - Opens and closes its own scoped ``Polylogue`` instance — the convenient - single-session entry point used by tests and one-off callers. Batch - callers should use :func:`screen_sessions`, which opens the archive once - and reuses the same instance across every session id instead of paying - the open/close cost per id. - """ - - from polylogue.api import Polylogue - - async with Polylogue(archive_root=archive_root) as pl: - return await _screen_session_with(pl, session_id) - - -async def screen_sessions(archive_root: Path, session_ids: list[str]) -> list[tuple[SessionScreeningResult, str]]: - """Screen every id in ``session_ids`` through one shared archive open.""" - - from polylogue.api import Polylogue - - async with Polylogue(archive_root=archive_root) as pl: - return [await _screen_session_with(pl, session_id) for session_id in session_ids] - - -def render_report_markdown(results: list[SessionScreeningResult], *, archive_root: Path) -> str: - lines = [ - "# Real-archive candidate slice — privacy screening report", - "", - f"Archive root: `{archive_root}`", - f"Sessions screened: {len(results)}", - "", - "This is an automated triage pass (pattern matching only). It is not a", - "certification. An operator must read the transcripts before any of", - "this content is folded into the shared demo proof-world fixture.", - "", - "| session_id | origin | verdict | messages | words | flags |", - "| --- | --- | --- | ---: | ---: | --- |", - ] - for r in results: - flags = ", ".join(f"{h.pattern}×{h.count}" for h in r.hits) or "—" - lines.append( - f"| `{r.session_id}` | {r.origin} | **{r.verdict}** | {r.message_count} | {r.word_count} | {flags} |" - ) - lines.append("") - for r in results: - lines.append(f"## `{r.session_id}`") - lines.append("") - lines.append(f"- title: {r.title!r}") - lines.append(f"- created_at: {r.created_at}") - lines.append(f"- verdict: **{r.verdict}**") - if not r.hits: - lines.append("- no pattern hits") - for h in r.hits: - lines.append(f"- `{h.pattern}` ({h.kind}) × {h.count}{' (all allowlisted)' if h.all_allowlisted else ''}") - for sample in h.samples: - lines.append(f" - {sample}") - lines.append("") - return "\n".join(lines) - - -def _parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser( - prog="devtools demo real-slice-screen", - description=( - "Read-only extraction and privacy screening of a candidate real-archive " - "session slice, for later human-gated inclusion in the shared demo " - "proof-world corpus (polylogue-212.11)." - ), - ) - parser.add_argument("--archive-root", type=Path, required=True, help="Real archive root to read (read-only).") - parser.add_argument( - "--session", - dest="sessions", - action="append", - default=[], - help="Session id to screen (repeatable).", - ) - parser.add_argument( - "--refs-file", - type=Path, - default=None, - help="Optional file with one session id per line (blank lines and '#' comments ignored).", - ) - parser.add_argument("--out", type=Path, required=True, help="Output directory for the report + transcripts.") - return parser - - -def _safe_transcript_filename(session_id: str) -> str: - """Filesystem-safe, collision-free filename stem for a session id. - - Path-unsafe characters are replaced for readability, but readability - alone is not collision-safe: distinct session ids that differ only in - which punctuation character separates otherwise-identical characters - (e.g. ``origin:a:b`` vs. ``origin:a_b``) would sanitize to the same - stem. A short content hash of the *original, unsanitized* session id is - appended so two distinct session ids can never produce the same - filename, guaranteeing no transcript is silently overwritten. - """ - - sanitized = re.sub(r"[^A-Za-z0-9_.-]", "_", session_id) - digest = hashlib.sha256(session_id.encode("utf-8")).hexdigest()[:12] - return f"{sanitized}__{digest}" - - -def _read_refs_file(path: Path) -> list[str]: - refs: list[str] = [] - for line in path.read_text(encoding="utf-8").splitlines(): - stripped = line.strip() - if not stripped or stripped.startswith("#"): - continue - refs.append(stripped) - return refs - - -def main(argv: list[str] | None = None) -> int: - import asyncio - - args = _parser().parse_args(argv) - session_ids = list(args.sessions) - if args.refs_file is not None: - session_ids.extend(_read_refs_file(args.refs_file)) - session_ids = list(dict.fromkeys(session_ids)) # de-dupe, preserve order - if not session_ids: - print("no session ids given (use --session or --refs-file)", file=sys.stderr) - return 2 - - pairs = asyncio.run(screen_sessions(args.archive_root, session_ids)) - results = [r for r, _ in pairs] - - args.out.mkdir(parents=True, exist_ok=True) - transcripts_dir = args.out / "transcripts" - transcripts_dir.mkdir(parents=True, exist_ok=True) - seen_filenames: dict[str, str] = {} - for result, text in pairs: - safe_name = _safe_transcript_filename(result.session_id) - prior = seen_filenames.get(safe_name) - if prior is not None and prior != result.session_id: - # Should be unreachable (sha256 collision on distinct inputs), - # but fail loudly rather than silently overwrite a transcript. - raise RuntimeError( - f"transcript filename collision: {safe_name!r} claimed by both {prior!r} and {result.session_id!r}" - ) - seen_filenames[safe_name] = result.session_id - (transcripts_dir / f"{safe_name}.txt").write_text(text, encoding="utf-8") - - manifest = { - "archive_root": str(args.archive_root), - "sessions": [r.to_dict() for r in results], - } - (args.out / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") - report_md = render_report_markdown(results, archive_root=args.archive_root) - (args.out / "SCREENING_REPORT.md").write_text(report_md + "\n", encoding="utf-8") - - flagged = [r for r in results if r.verdict == "flagged"] - review = [r for r in results if r.verdict == "review"] - print( - f"screened {len(results)} sessions: {len(flagged)} flagged, {len(review)} need review, " - f"{len(results) - len(flagged) - len(review)} clean" - ) - print(f"report: {args.out / 'SCREENING_REPORT.md'}") - return 1 if flagged else 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tests/unit/devtools/test_proof_world_real_slice.py b/tests/unit/devtools/test_proof_world_real_slice.py deleted file mode 100644 index 455815930b..0000000000 --- a/tests/unit/devtools/test_proof_world_real_slice.py +++ /dev/null @@ -1,325 +0,0 @@ -"""Tests for the real-archive candidate-slice screening harness (polylogue-212.11).""" - -from __future__ import annotations - -from pathlib import Path - -import pytest - -from devtools import proof_world_real_slice as m -from polylogue.core.enums import Provider -from tests.infra.archive_scenarios import native_session_id_for -from tests.infra.storage_records import SessionBuilder, db_setup - -_CLEAN_ID = native_session_id_for("claude-ai", "clean-session") -_SECRET_ID = native_session_id_for("claude-ai", "secret-session") - - -class _FakeMessage: - def __init__(self, text: str | None, blocks: object = None) -> None: - self.text = text - self.blocks = blocks - - -class _FakeSession: - def __init__(self, messages: list[_FakeMessage]) -> None: - self.messages = messages - - -# --- pure scan_text() behavior --------------------------------------------- - - -def test_scan_text_finds_no_hits_on_clean_text() -> None: - hits = m.scan_text("just an ordinary sentence about pytest and git diffs") - assert hits == [] - - -def test_scan_text_flags_a_credential_shaped_string() -> None: - hits = m.scan_text("aws key: AKIAABCDEFGHIJKLMNOP") - names = {h.pattern for h in hits} - assert "aws_access_key_id" in names - secret_hit = next(h for h in hits if h.pattern == "aws_access_key_id") - assert secret_hit.kind == "secret" - assert secret_hit.count == 1 - assert not secret_hit.all_allowlisted - - -def test_scan_text_allowlists_placeholder_email_and_loopback() -> None: - hits = m.scan_text("git config user.email test@example.com; server on 127.0.0.1") - email_hit = next(h for h in hits if h.pattern == "email") - ipv4_hit = next(h for h in hits if h.pattern == "ipv4") - assert email_hit.all_allowlisted - assert ipv4_hit.all_allowlisted - - -def test_scan_text_does_not_allowlist_a_real_looking_email() -> None: - hits = m.scan_text("contact jane.doe@personalmail.example for details") - email_hit = next(h for h in hits if h.pattern == "email") - assert not email_hit.all_allowlisted - - -def test_scan_text_does_not_allowlist_by_substring_containment() -> None: - """A match that merely *contains* an allowlisted value must not be - downgraded — only an exact full-match equals check counts. Regression - for a bug where `notuser@example.com` and `127.0.0.123` were both - treated as fully allowlisted (and thus 'clean') purely because - `user@example.com`/`127.0.0.1` occur as substrings.""" - - hits = m.scan_text("contact notuser@example.com and reach server at 127.0.0.123 for details") - email_hit = next(h for h in hits if h.pattern == "email") - ipv4_hit = next(h for h in hits if h.pattern == "ipv4") - assert not email_hit.all_allowlisted - assert not ipv4_hit.all_allowlisted - - result = m.SessionScreeningResult( - session_id="x", - origin="claude-code-session", - title=None, - created_at=None, - message_count=1, - word_count=1, - hits=[email_hit, ipv4_hit], - ) - assert result.verdict == "review" - - -# --- verdict computation ----------------------------------------------------- - - -def test_verdict_clean_when_no_hits() -> None: - result = m.SessionScreeningResult( - session_id="x", - origin="claude-code-session", - title=None, - created_at=None, - message_count=1, - word_count=1, - hits=[], - ) - assert result.verdict == "clean" - - -def test_verdict_flagged_beats_review_when_a_secret_hits() -> None: - result = m.SessionScreeningResult( - session_id="x", - origin="claude-code-session", - title=None, - created_at=None, - message_count=1, - word_count=1, - hits=[ - m.PatternHit(pattern="email", kind="pii", count=1, samples=["s"], all_allowlisted=False), - m.PatternHit(pattern="openai_style_key", kind="secret", count=1, samples=["s"], all_allowlisted=False), - ], - ) - assert result.verdict == "flagged" - - -def test_verdict_review_when_only_non_allowlisted_pii_hits() -> None: - result = m.SessionScreeningResult( - session_id="x", - origin="claude-code-session", - title=None, - created_at=None, - message_count=1, - word_count=1, - hits=[m.PatternHit(pattern="home_path", kind="pii", count=1, samples=["s"], all_allowlisted=False)], - ) - assert result.verdict == "review" - - -def test_verdict_clean_when_pii_hits_are_fully_allowlisted() -> None: - result = m.SessionScreeningResult( - session_id="x", - origin="claude-code-session", - title=None, - created_at=None, - message_count=1, - word_count=1, - hits=[m.PatternHit(pattern="email", kind="pii", count=1, samples=["s"], all_allowlisted=True)], - ) - assert result.verdict == "clean" - - -# --- secret redaction in samples ---------------------------------------------- - - -def test_scan_text_redacts_the_actual_secret_value_in_samples() -> None: - """The report/manifest must never embed a raw secret value verbatim — - only PII context does that. Regression for a bug where the snippet - window always fully contained the matched secret text despite the - docstring's claim that samples 'never' leak the full match.""" - - text = "the real key is AKIAABCDEFGHIJKLMNOP and it must stay secret" - hits = m.scan_text(text) - secret_hit = next(h for h in hits if h.pattern == "aws_access_key_id") - joined = " ".join(secret_hit.samples) - assert "AKIAABCDEFGHIJKLMNOP" not in joined - assert "redacted" in joined - # surrounding context should still be present for triage - assert "real key" in joined - - -def test_scan_text_keeps_real_pii_text_in_samples_for_human_judgment() -> None: - hits = m.scan_text("contact jane.doe@personalmail.example for details") - email_hit = next(h for h in hits if h.pattern == "email") - joined = " ".join(email_hit.samples) - assert "jane.doe@personalmail.example" in joined - - -# --- transcript filename collision safety -------------------------------------- - - -def test_safe_transcript_filename_disambiguates_punctuation_variants() -> None: - """Two distinct session ids that differ only in which punctuation - character separates otherwise-identical characters must never collide - on the sanitized filename stem.""" - - a = m._safe_transcript_filename("origin:a:b") - b = m._safe_transcript_filename("origin:a_b") - assert a != b - - -def test_safe_transcript_filename_is_deterministic() -> None: - assert m._safe_transcript_filename("claude-code-session:abc") == m._safe_transcript_filename( - "claude-code-session:abc" - ) - - -# --- flatten helper ----------------------------------------------------------- - - -def test_flatten_session_text_includes_message_text_and_block_json() -> None: - session = _FakeSession( - [ - _FakeMessage(text="hello world"), - _FakeMessage(text=None, blocks=[{"kind": "tool_use", "input": {"cmd": "ls"}}]), - ] - ) - flat = m._flatten_session_text(session) - assert "hello world" in flat - assert "tool_use" in flat - assert '"cmd": "ls"' in flat - - -# --- report rendering ----------------------------------------------------- - - -def test_render_report_markdown_includes_verdict_and_samples() -> None: - result = m.SessionScreeningResult( - session_id="claude-code-session:abc", - origin="claude-code-session", - title="Some session", - created_at="2026-01-01T00:00:00Z", - message_count=3, - word_count=42, - hits=[m.PatternHit(pattern="home_path", kind="pii", count=2, samples=["…/home/x…"], all_allowlisted=False)], - ) - md = m.render_report_markdown([result], archive_root=Path("/fake/archive")) - assert "claude-code-session:abc" in md - assert "**review**" in md - assert "home_path" in md - assert "/home/x" in md - - -# --- end-to-end read path against a seeded archive -------------------------- - - -async def _seed(db_path: Path) -> None: - await ( - SessionBuilder(db_path, "clean-session") - .provider(Provider.CLAUDE_AI.value) - .title("Clean session") - .add_message(text="just discussing pytest fixtures, nothing sensitive") - .build() - ) - await ( - SessionBuilder(db_path, "secret-session") - .provider(Provider.CLAUDE_AI.value) - .title("Session with a planted secret") - .add_message(text="here is my key: AKIAABCDEFGHIJKLMNOP please rotate it") - .build() - ) - - -async def test_screen_session_reads_real_archive_and_flags_planted_secret( - workspace_env: dict[str, Path], -) -> None: - db_path = db_setup(workspace_env) - await _seed(db_path) - archive_root = db_path.parent - - clean_result, clean_text = await m.screen_session(archive_root, _CLEAN_ID) - assert clean_result.verdict == "clean" - assert "pytest fixtures" in clean_text - - secret_result, secret_text = await m.screen_session(archive_root, _SECRET_ID) - assert secret_result.verdict == "flagged" - assert any(h.pattern == "aws_access_key_id" for h in secret_result.hits) - # the raw transcript text is unredacted (it exists for full human review)... - assert "AKIAABCDEFGHIJKLMNOP" in secret_text - # ...but the report-facing samples must never carry the raw secret value - all_samples = [s for h in secret_result.hits for s in h.samples] - assert not any("AKIAABCDEFGHIJKLMNOP" in s for s in all_samples) - - -async def test_screen_session_raises_for_unknown_session(workspace_env: dict[str, Path]) -> None: - db_path = db_setup(workspace_env) - await _seed(db_path) - - with pytest.raises(ValueError, match="not found"): - await m.screen_session(db_path.parent, "claude-code-session:does-not-exist") - - -async def test_screen_sessions_opens_the_archive_once_for_the_whole_batch( - workspace_env: dict[str, Path], monkeypatch: pytest.MonkeyPatch -) -> None: - """Regression for a CodeRabbit finding on PR #2885: ``screen_sessions`` - must open one shared ``Polylogue`` instance and reuse it across every - session id in the batch, not reopen the archive tiers per id. Counts real - ``Polylogue.__aenter__`` calls (the actual archive-open chokepoint) while - driving the real screening path against a seeded archive — this fails if - ``screen_sessions`` regresses to calling ``screen_session`` (which opens - its own scoped instance) once per id.""" - - from polylogue.api import Polylogue - - db_path = db_setup(workspace_env) - await _seed(db_path) - archive_root = db_path.parent - - open_count = 0 - original_aenter = Polylogue.__aenter__ - - async def counting_aenter(self: Polylogue) -> Polylogue: - nonlocal open_count - open_count += 1 - return await original_aenter(self) - - monkeypatch.setattr(Polylogue, "__aenter__", counting_aenter) - - pairs = await m.screen_sessions(archive_root, [_CLEAN_ID, _SECRET_ID]) - - assert open_count == 1 - assert [r.session_id for r, _ in pairs] == [_CLEAN_ID, _SECRET_ID] - assert pairs[0][0].verdict == "clean" - assert pairs[1][0].verdict == "flagged" - - -# --- CLI argument handling --------------------------------------------------- - - -def test_main_exits_nonzero_with_no_session_ids(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: - code = m.main(["--archive-root", str(tmp_path), "--out", str(tmp_path / "out")]) - assert code == 2 - captured = capsys.readouterr() - assert "no session ids given" in captured.err - - -def test_read_refs_file_skips_blanks_and_comments(tmp_path: Path) -> None: - refs_path = tmp_path / "refs.txt" - refs_path.write_text("\n# a comment\nclaude-code-session:a:b\n\nclaude-code-session:c:d\n", encoding="utf-8") - assert m._read_refs_file(refs_path) == [ - "claude-code-session:a:b", - "claude-code-session:c:d", - ] From f82eef552ab53a3d411f5e757d7edb1b51c028a6 Mon Sep 17 00:00:00 2001 From: Sinity Date: Wed, 19 Aug 2026 02:17:07 +0200 Subject: [PATCH 06/10] chore(devtools): delete help_latency_probe (one-shot CLI-latency research) No dedicated test file, no hook/CI/doc references; the interactive-tier cold-CLI budget it checked is not enforced by any automated gate. Ref .agent/campaigns/2026-08-overhaul/ws-c-devtools-pruning.md --- devtools/help_latency_probe.py | 172 --------------------------------- 1 file changed, 172 deletions(-) delete mode 100644 devtools/help_latency_probe.py diff --git a/devtools/help_latency_probe.py b/devtools/help_latency_probe.py deleted file mode 100644 index b97c47d6a4..0000000000 --- a/devtools/help_latency_probe.py +++ /dev/null @@ -1,172 +0,0 @@ -"""Measure ``--help`` wall-clock latency against the interactive-tier cold-CLI budget. - -Use ``devtools bench help-latency`` to catch import-tax regressions on the -CLI ``--help`` path continuously (polylogue-20d.2). The interactive SLO tier -(polylogue-20d.14, ``docs/plans/slo-catalog.yaml``) states a <700ms budget for -a cold (no warm daemon) CLI invocation; ``--help`` is the cheapest possible -invocation of any command (no archive I/O, no query execution) so it is the -tightest floor on Python/import overhead. A regression here means every -daemonless invocation of that command pays the same tax. - -Each target is run as a fresh subprocess (``python -m polylogue.cli ``) -several times; the *minimum* wall time is compared against budget rather than -the mean, because process-launch jitter (scheduler contention, page-cache -misses) only ever adds latency on a shared dev host, never subtracts it. This -mirrors the host-variable framing of the other ``devtools bench`` probes: -wall-clock is diagnostic and campaign-comparable, but the budget comparison -here IS a CI gate for "required" targets (unlike the wall-clock-only probes), -because import cost is deterministic given the source tree, not host load. - -Targets marked ``gate="informational"`` are measured and reported but never -fail the check — they document a known-slow path with an open follow-up. -(No target currently uses this gate: the last one, ``ops maintenance -migrate-tier``, was promoted to ``required`` once polylogue-h1wt made the -``archive_tiers`` and parent ``polylogue.storage.sqlite`` package inits lazy; -see ``polylogue/cli/commands/maintenance/_migrate_tier.py``'s module -docstring.) -""" - -from __future__ import annotations - -import argparse -import json -import subprocess -import sys -from dataclasses import dataclass -from datetime import UTC, datetime -from pathlib import Path -from time import perf_counter -from typing import Literal, cast - -Gate = Literal["required", "informational"] - - -@dataclass(frozen=True, slots=True) -class HelpLatencyTarget: - label: str - args: tuple[str, ...] - budget_ms: int - gate: Gate - - -# Budget follows the 20d.14 interactive-tier "cold CLI (no daemon) <700ms" -# line in docs/plans/slo-catalog.yaml. -_DEFAULT_BUDGET_MS = 700 - -TARGETS: tuple[HelpLatencyTarget, ...] = ( - HelpLatencyTarget("root", ("--help",), _DEFAULT_BUDGET_MS, "required"), - HelpLatencyTarget("find", ("find", "--help"), _DEFAULT_BUDGET_MS, "required"), - HelpLatencyTarget("read", ("read", "--help"), _DEFAULT_BUDGET_MS, "required"), - HelpLatencyTarget("mark", ("mark", "--help"), _DEFAULT_BUDGET_MS, "required"), - HelpLatencyTarget("select", ("select", "--help"), _DEFAULT_BUDGET_MS, "required"), - HelpLatencyTarget("analyze", ("analyze", "--help"), _DEFAULT_BUDGET_MS, "required"), - HelpLatencyTarget("import", ("import", "--help"), _DEFAULT_BUDGET_MS, "required"), - HelpLatencyTarget("config", ("config", "--help"), _DEFAULT_BUDGET_MS, "required"), - HelpLatencyTarget("dashboard", ("dashboard", "--help"), _DEFAULT_BUDGET_MS, "required"), - HelpLatencyTarget("ops", ("ops", "--help"), _DEFAULT_BUDGET_MS, "required"), - HelpLatencyTarget("reset", ("reset", "--help"), _DEFAULT_BUDGET_MS, "required"), - # `ops maintenance` is now a package of one lazily-dispatched submodule - # per subcommand (polylogue-sod7): the group listing and every subcommand - # except migrate-tier import only click/paths/config at `--help` time, - # deferring ArchiveStore/blob_gc/blob_integrity/embeddings.reconcile/ - # migration_runner/the archive_tiers DDL stack into each command's own - # function body. - HelpLatencyTarget("ops-maintenance", ("ops", "maintenance", "--help"), _DEFAULT_BUDGET_MS, "required"), - HelpLatencyTarget( - "ops-maintenance-archive-read", - ("ops", "maintenance", "archive-read", "--help"), - _DEFAULT_BUDGET_MS, - "required", - ), - # migrate-tier's --help still needs DURABLE_MIGRATION_TIERS at - # Click-decoration time (to render the `tier` argument's valid choices), - # but polylogue-h1wt made both the archive_tiers package init and its - # parent polylogue.storage.sqlite package init lazy, so this now costs - # only ArchiveTier's own weight -- comfortably inside the same required - # budget as every sibling command. See _migrate_tier.py. - HelpLatencyTarget( - "ops-maintenance-migrate-tier", - ("ops", "maintenance", "migrate-tier", "--help"), - _DEFAULT_BUDGET_MS, - "required", - ), -) - - -def _time_invocation(args: tuple[str, ...], *, repeats: int) -> float: - """Return the minimum wall-clock ms across ``repeats`` fresh subprocess runs.""" - - best: float | None = None - for _ in range(repeats): - started = perf_counter() - subprocess.run( - [sys.executable, "-m", "polylogue.cli", *args], - check=False, - capture_output=True, - text=True, - ) - elapsed_ms = (perf_counter() - started) * 1_000 - if best is None or elapsed_ms < best: - best = elapsed_ms - assert best is not None - return best - - -def measure(*, repeats: int = 3, targets: tuple[HelpLatencyTarget, ...] = TARGETS) -> dict[str, object]: - results = [] - for target in targets: - elapsed_ms = _time_invocation(target.args, repeats=repeats) - within_budget = elapsed_ms <= target.budget_ms - results.append( - { - "label": target.label, - "args": list(target.args), - "gate": target.gate, - "budget_ms": target.budget_ms, - "elapsed_ms": round(elapsed_ms, 1), - "within_budget": within_budget, - } - ) - violations = [r["label"] for r in results if r["gate"] == "required" and not r["within_budget"]] - return { - "version": 1, - "generated_at": datetime.now(UTC).isoformat(), - "repeats": repeats, - "results": results, - "violations": violations, - "ok": not violations, - } - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--repeats", type=int, default=3, help="Subprocess runs per target; minimum wins.") - parser.add_argument("--json", action="store_true", help="Emit the full JSON report instead of a table.") - parser.add_argument("--out", type=Path, default=None, help="Also write the JSON report to this path.") - args = parser.parse_args(argv) - - report = measure(repeats=max(1, args.repeats)) - - if args.out is not None: - args.out.parent.mkdir(parents=True, exist_ok=True) - args.out.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") - - if args.json: - print(json.dumps(report, indent=2, sort_keys=True)) - else: - results = cast("list[dict[str, object]]", report["results"]) - for result in results: - marker = "OK" if result["within_budget"] else "OVER" - flag = "" if result["gate"] == "required" else " (informational)" - print(f"{marker:>4} {result['label']:<32} {result['elapsed_ms']:>7.1f}ms / {result['budget_ms']}ms{flag}") - violations = cast("list[str]", report["violations"]) - if violations: - print(f"\nBudget violations (required): {', '.join(violations)}") - else: - print("\nAll required targets within budget.") - - return 0 if report["ok"] else 1 - - -if __name__ == "__main__": - raise SystemExit(main()) From db29da00709f502857e0004c3b8e98d238cb6f9d Mon Sep 17 00:00:00 2001 From: Sinity Date: Wed, 19 Aug 2026 02:17:10 +0200 Subject: [PATCH 07/10] chore(devtools): delete temporal_read_profile + temporal_archive_aggregates One-shot read-profiling research (July 2). No hook/CI/doc references. Ref .agent/campaigns/2026-08-overhaul/ws-c-devtools-pruning.md --- devtools/temporal_archive_aggregates.py | 210 ------------------ devtools/temporal_read_profile.py | 114 ---------- .../test_temporal_archive_aggregates.py | 108 --------- .../devtools/test_temporal_read_profile.py | 84 ------- 4 files changed, 516 deletions(-) delete mode 100644 devtools/temporal_archive_aggregates.py delete mode 100644 devtools/temporal_read_profile.py delete mode 100644 tests/unit/devtools/test_temporal_archive_aggregates.py delete mode 100644 tests/unit/devtools/test_temporal_read_profile.py diff --git a/devtools/temporal_archive_aggregates.py b/devtools/temporal_archive_aggregates.py deleted file mode 100644 index fdb873c181..0000000000 --- a/devtools/temporal_archive_aggregates.py +++ /dev/null @@ -1,210 +0,0 @@ -"""Build aggregate run-projection artifacts from the active archive.""" - -from __future__ import annotations - -import argparse -import csv -import json -import sys -from collections.abc import Iterable -from datetime import UTC, datetime -from pathlib import Path -from sqlite3 import Connection -from typing import Any - -from polylogue.config import Config, get_config -from polylogue.storage.sqlite.connection_profile import open_readonly_connection -from polylogue.storage.sqlite.run_projection_relations import ( - context_snapshot_relation_sql, - observed_event_relation_sql, - run_relation_sql, -) - - -def _parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser( - prog="devtools workspace temporal-archive-aggregates", - description="Summarize run-projection aggregate tables from the active archive.", - ) - parser.add_argument("--archive-root", type=Path, default=None, help="Override the active archive root.") - parser.add_argument("--out-dir", type=Path, default=None, help="Write cardinality JSON and monthly CSV artifacts.") - parser.add_argument("--json", action="store_true", help="Emit JSON report to stdout. Accepted for devtools parity.") - return parser - - -def _config_with_archive_root(config: Config, archive_root: Path | None) -> Config: - if archive_root is None: - return config - resolved = archive_root.expanduser().resolve() - return Config( - archive_root=resolved, - render_root=config.render_root, - sources=config.sources, - db_path=resolved / "index.db", - drive_config=config.drive_config, - index_config=config.index_config, - ) - - -def _user_version(conn: Connection) -> int: - row = conn.execute("PRAGMA user_version").fetchone() - return int(row[0]) if row else 0 - - -def _count(conn: Connection, table: str) -> int: - row = conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone() - return int(row[0]) if row else 0 - - -def _relation_count(conn: Connection, relation_sql: str, relation: str) -> int: - row = conn.execute(f"{relation_sql} SELECT COUNT(*) FROM {relation}").fetchone() - return int(row[0]) if row else 0 - - -def _has_column(conn: Connection, table: str, column: str) -> bool: - return any(str(row[1]) == column for row in conn.execute(f"PRAGMA table_info({table})")) - - -def _has_table(conn: Connection, table: str) -> bool: - row = conn.execute( - "SELECT 1 FROM sqlite_master WHERE type IN ('table','view') AND name = ?", - (table,), - ).fetchone() - return row is not None - - -def _logical_root_session_count(conn: Connection, physical_sessions: int) -> int: - if not _has_column(conn, "sessions", "root_session_id"): - return physical_sessions - row = conn.execute( - """ - SELECT COUNT(DISTINCT COALESCE(root_session_id, session_id)) - FROM sessions - """ - ).fetchone() - return int(row[0]) if row else physical_sessions - - -def _session_profile_count(conn: Connection) -> int | None: - if not _has_table(conn, "session_profiles"): - return None - return _count(conn, "session_profiles") - - -def _rows(conn: Connection, sql: str) -> list[dict[str, object]]: - cursor = conn.execute(sql) - columns = [str(description[0]) for description in cursor.description or ()] - return [dict(zip(columns, row, strict=True)) for row in cursor.fetchall()] - - -def _write_json(path: Path, payload: object) -> None: - path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") - - -def _write_csv(path: Path, rows: Iterable[dict[str, object]]) -> None: - materialized = list(rows) - fieldnames = list(materialized[0].keys()) if materialized else [] - with path.open("w", encoding="utf-8", newline="") as handle: - writer = csv.DictWriter(handle, fieldnames=fieldnames) - writer.writeheader() - writer.writerows(materialized) - - -def build_report(args: argparse.Namespace) -> dict[str, Any]: - config = _config_with_archive_root(get_config(), args.archive_root) - index_db = config.db_path - conn = open_readonly_connection(index_db) - try: - physical_sessions = _count(conn, "sessions") - session_profiles = _session_profile_count(conn) - cardinality = { - "sessions": physical_sessions, - "physical_sessions": physical_sessions, - "logical_root_sessions": _logical_root_session_count(conn, physical_sessions), - "session_profiles": session_profiles, - "session_profile_coverage_exact": session_profiles is not None, - "runs": _relation_count(conn, run_relation_sql(), "runs"), - "observed_events": _relation_count(conn, observed_event_relation_sql(source_where="1"), "observed_events"), - "context_snapshots": _relation_count(conn, context_snapshot_relation_sql(), "context_snapshots"), - } - # polylogue-dab/itvd: session_runs/session_observed_events/ - # session_context_snapshots are source-derived CTE relations, not - # tables. Their `source_updated_at` column is a zero-padded 16-digit - # epoch-ms string (for lexicographic ORDER BY), not an ISO-8601 - # timestamp -- the old `substr(source_updated_at,1,7)` ISO-prefix - # trick would silently bucket everything into one bogus "month", so - # the epoch-ms string is cast back to a real date first. - monthly_runs = _rows( - conn, - f""" - {run_relation_sql()} - SELECT coalesce(strftime('%Y-%m', CAST(source_updated_at AS INTEGER) / 1000, 'unixepoch'), 'unknown') AS month, - harness, - role, - status, - count(*) AS runs - FROM runs - GROUP BY 1,2,3,4 - ORDER BY 1,2,3,4 - """, - ) - monthly_observed_events = _rows( - conn, - f""" - {observed_event_relation_sql(source_where="1")} - SELECT coalesce(strftime('%Y-%m', CAST(source_updated_at AS INTEGER) / 1000, 'unixepoch'), 'unknown') AS month, - kind, - delivery_state, - count(*) AS events - FROM observed_events - GROUP BY 1,2,3 - ORDER BY 1,2,3 - """, - ) - monthly_context_boundaries = _rows( - conn, - f""" - {context_snapshot_relation_sql()} - SELECT coalesce(strftime('%Y-%m', CAST(source_updated_at AS INTEGER) / 1000, 'unixepoch'), 'unknown') AS month, - boundary, - inheritance_mode, - count(*) AS snapshots - FROM context_snapshots - GROUP BY 1,2,3 - ORDER BY 1,2,3 - """, - ) - report: dict[str, Any] = { - "report_version": 1, - "captured_at": datetime.now(UTC).isoformat(), - "command": "devtools workspace temporal-archive-aggregates", - "archive_root": str(config.archive_root), - "index_db": str(index_db), - "index_schema_version": _user_version(conn), - "cardinality": cardinality, - "monthly_runs_by_harness_role_status": monthly_runs, - "monthly_observed_events_by_kind": monthly_observed_events, - "monthly_context_boundaries": monthly_context_boundaries, - } - finally: - conn.close() - if args.out_dir is not None: - out_dir = args.out_dir - out_dir.mkdir(parents=True, exist_ok=True) - _write_json(out_dir / "archive-cardinality.json", [cardinality]) - _write_csv(out_dir / "monthly-runs-by-harness-role-status.csv", monthly_runs) - _write_csv(out_dir / "monthly-observed-events-by-kind.csv", monthly_observed_events) - _write_csv(out_dir / "monthly-context-boundaries.csv", monthly_context_boundaries) - _write_json(out_dir / "temporal-archive-aggregates.report.json", report) - return report - - -def main(argv: list[str] | None = None) -> int: - args = _parser().parse_args(argv) - report = build_report(args) - sys.stdout.write(json.dumps(report, indent=2, sort_keys=True) + "\n") - return 0 - - -if __name__ == "__main__": # pragma: no cover - raise SystemExit(main()) diff --git a/devtools/temporal_read_profile.py b/devtools/temporal_read_profile.py deleted file mode 100644 index 406ea0554d..0000000000 --- a/devtools/temporal_read_profile.py +++ /dev/null @@ -1,114 +0,0 @@ -"""Profile the shared temporal read-view builder on the active archive.""" - -from __future__ import annotations - -import argparse -import json -import sys -import time -from collections.abc import Mapping -from datetime import UTC, datetime -from pathlib import Path -from typing import Any, cast - -from polylogue.cli.read_views.standard import build_read_temporal_window -from polylogue.cli.root_request import RootModeRequest -from polylogue.config import Config, get_config - - -def _parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser( - prog="devtools workspace temporal-read-profile", - description="Measure phase timings for read --view temporal using the shared read-view builder.", - ) - parser.add_argument("--query", default="repo:polylogue", help="Root query expression to select sessions.") - parser.add_argument("--limit", type=int, default=1, help="Session summary limit for the temporal read.") - parser.add_argument("--archive-root", type=Path, default=None, help="Override the active archive root.") - parser.add_argument("--out", type=Path, default=None, help="Write the JSON report to this path.") - parser.add_argument("--include-window", action="store_true", help="Include the full temporal_window payload.") - parser.add_argument("--json", action="store_true", help="Emit JSON to stdout. Accepted for devtools parity.") - return parser - - -def _config_with_archive_root(config: Config, archive_root: Path | None) -> Config: - if archive_root is None: - return config - resolved = archive_root.expanduser().resolve() - return Config( - archive_root=resolved, - render_root=config.render_root, - sources=config.sources, - db_path=resolved / "index.db", - drive_config=config.drive_config, - index_config=config.index_config, - ) - - -def _phase_summary(phases: list[dict[str, object]]) -> dict[str, object]: - def elapsed_ms(phase: dict[str, object]) -> float: - return cast(float, phase["elapsed_ms"]) - - by_phase = {str(phase["name"]): elapsed_ms(phase) for phase in phases} - slowest = max(phases, key=elapsed_ms, default=None) - return { - "phase_count": len(phases), - "elapsed_by_phase_ms": by_phase, - "slowest_phase": None if slowest is None else slowest["name"], - "slowest_phase_ms": None if slowest is None else slowest["elapsed_ms"], - } - - -def build_report(args: argparse.Namespace) -> dict[str, Any]: - config = _config_with_archive_root(get_config(), args.archive_root) - phases: list[dict[str, object]] = [] - - def record_phase(name: str, elapsed_ms: float, details: Mapping[str, object]) -> None: - phases.append( - { - "name": name, - "elapsed_ms": round(elapsed_ms, 3), - "details": details, - } - ) - - request = RootModeRequest.from_params({"query": (args.query,), "limit": args.limit}) - started = time.perf_counter() - window = build_read_temporal_window(config, request, phase_recorder=record_phase) - total_elapsed_ms = round((time.perf_counter() - started) * 1000, 3) - report: dict[str, Any] = { - "report_version": 1, - "captured_at": datetime.now(UTC).isoformat(), - "command": "devtools workspace temporal-read-profile", - "archive_root": str(config.archive_root), - "index_db": str(config.db_path), - "query": args.query, - "limit": args.limit, - "total_elapsed_ms": total_elapsed_ms, - "phases": phases, - "phase_summary": _phase_summary(phases), - "temporal_window_summary": { - "event_count": window.event_count, - "family_counts": dict(window.family_counts), - "kind_counts": dict(window.kind_counts), - "caveats": list(window.caveats), - }, - } - if args.include_window: - report["temporal_window"] = window.model_dump(mode="json") - return report - - -def main(argv: list[str] | None = None) -> int: - parser = _parser() - args = parser.parse_args(argv) - report = build_report(args) - rendered = json.dumps(report, indent=2, sort_keys=True) + "\n" - if args.out is not None: - args.out.parent.mkdir(parents=True, exist_ok=True) - args.out.write_text(rendered, encoding="utf-8") - sys.stdout.write(rendered) - return 0 - - -if __name__ == "__main__": # pragma: no cover - raise SystemExit(main()) diff --git a/tests/unit/devtools/test_temporal_archive_aggregates.py b/tests/unit/devtools/test_temporal_archive_aggregates.py deleted file mode 100644 index 4421956b2c..0000000000 --- a/tests/unit/devtools/test_temporal_archive_aggregates.py +++ /dev/null @@ -1,108 +0,0 @@ -from __future__ import annotations - -import argparse -import csv -import json -import sqlite3 -from pathlib import Path - -from devtools import temporal_archive_aggregates - - -def _make_index_db(root: Path) -> Path: - root.mkdir() - db = root / "index.db" - conn = sqlite3.connect(db) - try: - conn.executescript( - """ - PRAGMA user_version = 18; - CREATE TABLE sessions ( - session_id TEXT PRIMARY KEY, - parent_session_id TEXT, - root_session_id TEXT, - origin TEXT, - branch_type TEXT, - title TEXT, - git_branch TEXT, - native_id TEXT, - message_count INTEGER, - tool_use_count INTEGER, - created_at_ms INTEGER, - updated_at_ms INTEGER - ); - CREATE TABLE blocks ( - block_id TEXT PRIMARY KEY, - session_id TEXT, - block_type TEXT, - message_id TEXT, - position INTEGER, - semantic_type TEXT, - tool_command TEXT, - tool_id TEXT, - tool_name TEXT, - tool_result_exit_code INTEGER, - tool_result_is_error INTEGER, - search_text TEXT - ); - CREATE TABLE session_profiles (session_id TEXT PRIMARY KEY); - INSERT INTO sessions VALUES - ('s1', NULL, 's1', 'codex-session', NULL, 't1', NULL, 'n1', 1, 0, 1780308000000, 1780308000000), - ('s2', NULL, 's1', 'codex-session', NULL, 't2', NULL, 'n2', 1, 0, 1780394400000, 1780394400000), - ('s3', 's1', 's1', 'claude-code-session', 'subagent', 't3', NULL, 'n3', 1, 0, 1782900000000, 1782900000000); - INSERT INTO blocks VALUES - ('s1::m1::0', 's1', 'tool_use', 'm1', 0, NULL, NULL, 'tool-1', 'Bash', NULL, NULL, 'run tests'), - ('s1::m1::1', 's1', 'tool_result', 'm1', 1, NULL, NULL, 'tool-1', NULL, 0, 0, 'ok'); - INSERT INTO session_profiles VALUES ('s1'); - """ - ) - conn.commit() - finally: - conn.close() - return db - - -def test_temporal_archive_aggregates_report_and_files(tmp_path: Path) -> None: - """polylogue-dab/itvd: runs/observed_events/context_snapshots are now - source-derived CTE relations (run_projection_relations.py), computed - from `sessions`/`blocks`, not standalone materialized tables. Every - session unconditionally produces exactly one run row and one - context-snapshot row, plus one 'session_started' observed-event row - (more if it has tool_use/tool_result block pairs). - """ - archive_root = tmp_path / "archive" - _make_index_db(archive_root) - out_dir = tmp_path / "out" - args = argparse.Namespace(archive_root=archive_root, out_dir=out_dir, json=True) - - report = temporal_archive_aggregates.build_report(args) - - assert report["archive_root"] == str(archive_root.resolve()) - assert report["index_schema_version"] == 18 - assert report["cardinality"] == { - "sessions": 3, - "physical_sessions": 3, - "logical_root_sessions": 1, - "session_profiles": 1, - "session_profile_coverage_exact": True, - "runs": 3, - "observed_events": 4, - "context_snapshots": 3, - } - assert report["monthly_runs_by_harness_role_status"] == [ - {"month": "2026-06", "harness": "codex", "role": "main", "status": "completed", "runs": 2}, - {"month": "2026-07", "harness": "claude-code", "role": "subagent", "status": "completed", "runs": 1}, - ] - # The tool_finished event's source_updated_at is NULL (position-ordered, - # not time-ordered) and buckets to the 'unknown' month fallback. - assert {"month": "unknown", "kind": "tool_finished", "delivery_state": "observed", "events": 1} in report[ - "monthly_observed_events_by_kind" - ] - - cardinality = json.loads((out_dir / "archive-cardinality.json").read_text(encoding="utf-8")) - assert cardinality == [report["cardinality"]] - with (out_dir / "monthly-observed-events-by-kind.csv").open(encoding="utf-8", newline="") as handle: - rows = list(csv.DictReader(handle)) - assert {"month": "2026-06", "kind": "session_started", "delivery_state": "observed", "events": "2"} in rows - written_report = json.loads((out_dir / "temporal-archive-aggregates.report.json").read_text(encoding="utf-8")) - assert written_report["cardinality"] == report["cardinality"] diff --git a/tests/unit/devtools/test_temporal_read_profile.py b/tests/unit/devtools/test_temporal_read_profile.py deleted file mode 100644 index 22e201683e..0000000000 --- a/tests/unit/devtools/test_temporal_read_profile.py +++ /dev/null @@ -1,84 +0,0 @@ -from __future__ import annotations - -import argparse -import json -from collections.abc import Callable -from datetime import UTC, datetime -from pathlib import Path -from unittest.mock import patch - -import pytest - -from devtools import temporal_read_profile -from polylogue.cli.root_request import RootModeRequest -from polylogue.config import Config -from polylogue.surfaces.temporal_evidence import ( - TemporalEvidenceEvent, - TemporalEvidenceWindow, - build_temporal_evidence_window, -) - - -def test_temporal_read_profile_report_wraps_shared_builder(tmp_path: Path) -> None: - config = Config( - archive_root=tmp_path, - db_path=tmp_path / "index.db", - render_root=tmp_path / "render", - sources=[], - ) - window = build_temporal_evidence_window( - [ - TemporalEvidenceEvent( - event_id="session:abc:session", - occurred_at=datetime(2026, 6, 30, 8, 0, tzinfo=UTC), - family="archive-session", - kind="session", - label="Temporal profile", - source_ref="session:abc", - evidence_refs=("session:abc",), - ) - ] - ) - - def fake_builder( - _config: Config, - _request: RootModeRequest, - *, - phase_recorder: Callable[[str, float, dict[str, object]], None], - ) -> TemporalEvidenceWindow: - phase_recorder("prepare", 1.25, {"archive_root": str(tmp_path), "limit": 1}) - phase_recorder("select_sessions", 2.5, {"session_count": 1}) - return window - - args = argparse.Namespace( - query="repo:polylogue", - limit=1, - archive_root=None, - out=None, - include_window=False, - json=True, - ) - with ( - patch("devtools.temporal_read_profile.get_config", return_value=config), - patch("devtools.temporal_read_profile.build_read_temporal_window", side_effect=fake_builder), - ): - report = temporal_read_profile.build_report(args) - - assert report["archive_root"] == str(tmp_path) - assert report["query"] == "repo:polylogue" - assert report["phase_summary"]["slowest_phase"] == "select_sessions" - assert report["temporal_window_summary"]["family_counts"] == {"archive-session": 1} - assert "temporal_window" not in report - - -def test_temporal_read_profile_main_writes_report(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: - out = tmp_path / "profile.json" - with patch( - "devtools.temporal_read_profile.build_report", - return_value={"report_version": 1, "total_elapsed_ms": 3.0}, - ): - exit_code = temporal_read_profile.main(["--query", "repo:polylogue", "--out", str(out), "--json"]) - - assert exit_code == 0 - assert json.loads(out.read_text(encoding="utf-8"))["total_elapsed_ms"] == 3.0 - assert json.loads(capsys.readouterr().out)["report_version"] == 1 From 154a6ad9e9657eec41f548fbd50f46a6b7c341f6 Mon Sep 17 00:00:00 2001 From: Sinity Date: Wed, 19 Aug 2026 02:17:33 +0200 Subject: [PATCH 08/10] chore(devtools): regenerate devtools-reference after tranche-1 pruning Ref .agent/campaigns/2026-08-overhaul/ws-c-devtools-pruning.md --- docs/devtools.md | 7 ------- 1 file changed, 7 deletions(-) diff --git a/docs/devtools.md b/docs/devtools.md index 3173bf5ff9..5ea929b4ff 100644 --- a/docs/devtools.md +++ b/docs/devtools.md @@ -63,7 +63,6 @@ They are not a proof ledger or end-user archive workflow. | `devtools lab policy insight-honesty` | Enforce that polylogue.insights.registry.INSIGHT_REGISTRY and polylogue.insights.rigor's contract matrix/exemption list never drift apart (9e5.28) -- a registered product with neither a RigorContract nor a RIGOR_EXEMPT entry used to silently vanish from `polylogue ops insights audit` instead of showing as uncovered. | | `devtools lab probe cost-reconciliation` | Validate archive token accounting against optional local Codex state_5.sqlite and Claude stats-cache.json before publishing cost or usage-analysis claims. | | `devtools lab probe pipeline` | Run real pipeline stages and optionally capture emitted summaries as regression cases. | -| `devtools lab probe turso` | Collect executable evidence before changing production storage backends: Python binding availability, generated-column support, FTS compatibility, MVCC, CDC, vector functions, ATTACH, and WAL pragma behavior. | | `devtools lab run` | Run a scenario such as rebuild-safety through the direct lab command path. | | `devtools lab smoke` | Run direct archive and reader smoke sets outside the archive CLI. | | `devtools lab schema list` | Inspect committed provider schema package catalogs without presenting them as normal archive usage. | @@ -136,7 +135,6 @@ These are the commands worth remembering during normal repo work: | `devtools lab probe capture-regression` | Capture pipeline-probe summaries as durable local regression cases. | | `devtools lab probe cost-reconciliation` | Reconcile Polylogue token accounting against private provider stores. | | `devtools lab probe pipeline` | Run typed pipeline probes against synthetic, staged, or archive-subset inputs. | -| `devtools lab probe turso` | Probe Turso Database compatibility against Polylogue storage assumptions. | | `devtools lab provider completeness` | Report provider/importer package completeness by origin and capture mode. | | `devtools lab run` | Run a named archive verification scenario. | | `devtools lab schema audit` | Run committed provider schema package quality checks. | @@ -170,7 +168,6 @@ These are the commands worth remembering during normal repo work: | Command | Description | | --- | --- | -| `devtools bench help-latency` | Check `--help` wall-clock latency against the interactive-tier cold-CLI budget (polylogue-20d.2). | | `devtools bench ingest-amplification` | Measure deterministic per-tier ingest write amplification on a synthetic fixture (#1851). | | `devtools bench ingest-throughput` | Measure ingest wall-clock throughput on a synthetic fixture. | | `devtools bench memory` | Measure query-memory envelopes on generated fixtures. | @@ -183,7 +180,6 @@ These are the commands worth remembering during normal repo work: | Command | Description | | --- | --- | -| `devtools demo real-slice-screen` | Read-only extraction + privacy screening of a candidate real-archive session slice. | | `devtools workspace affordance-usage` | Analyze agent affordance/tool usage from archive tool-use rows. | | `devtools workspace agent-meta-sidecar-purge-apply` | Purge agent-*.meta.json subagent-sidecar phantom sessions from index.db. | | `devtools workspace agent-meta-sidecar-sweep` | Find agent-*.meta.json subagent-sidecar phantom sessions (message_count=0). | @@ -195,7 +191,6 @@ These are the commands worth remembering during normal repo work: | `devtools workspace bead-reimport-guard` | Monotonic, receipted guard/reconcile/export for bd's JSONL synchronization. | | `devtools workspace binary-artifact-reclassify-apply` | Persist raw_artifacts classification for binary-shaped raw rows. | | `devtools workspace binary-artifact-sweep` | Find raw_sessions rows whose bytes are a non-session binary format (SQLite, etc). | -| `devtools workspace claim-vs-evidence` | Analyze structured failures and the assistant behavior that followed. | | `devtools workspace continuity-evidence` | Replay continuity scenarios and verify their query routes are discoverable. | | `devtools workspace degraded-archive-proof` | Build a degraded archive self-healing proof artifact. | | `devtools workspace deployment-smoke` | Probe deployed Polylogue binaries, daemon/web routes, and browser-capture archive flow. | @@ -220,8 +215,6 @@ These are the commands worth remembering during normal repo work: | `devtools workspace raw-quarantine-group-dedup-apply` | Promote one representative raw per fully-quarantined byte-identical (source_path, blob_hash) group. | | `devtools workspace read-package` | Render a declarative package of Polylogue read artifacts. | | `devtools workspace scale-regression` | Run the seeded large-archive scale-regression probe. | -| `devtools workspace temporal-archive-aggregates` | Build run-projection aggregate artifacts from the active archive. | -| `devtools workspace temporal-read-profile` | Measure read --view temporal phase timings on the active archive. | | `devtools workspace tool-result-history-reclassify-apply` | Persist raw_artifacts classification for tool-result/file-history-shaped raw rows. | | `devtools workspace tool-result-history-sweep` | Find claude-code-session raw rows that should reclassify as tool-result/file-history sidecars. | | `devtools workspace unknown-export-reclassification` | Re-run the fixed browser-capture provider probe against stored unknown-export rows. | From 0c63100e0f02b05f4ee6b9228eb5e6cef8d45e8b Mon Sep 17 00:00:00 2001 From: Sinity Date: Wed, 19 Aug 2026 02:19:59 +0200 Subject: [PATCH 09/10] fix(docs): de-literal the deleted claim-vs-evidence command mention verify doc-commands flags any backtick-quoted 'devtools ...' span against the live Click tree; the retired command needed prose, not code, framing. Ref .agent/campaigns/2026-08-overhaul/ws-c-devtools-pruning.md --- docs/findings/claim-vs-evidence.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/findings/claim-vs-evidence.md b/docs/findings/claim-vs-evidence.md index 3567e976d7..c2e785da91 100644 --- a/docs/findings/claim-vs-evidence.md +++ b/docs/findings/claim-vs-evidence.md @@ -127,8 +127,9 @@ It does not establish why the assistant proceeded, whether the outcome was event ## Reproducing this finding -The generating harness (`devtools workspace claim-vs-evidence` and its -private-report/calibration/publishing machinery) was retired 2026-08 once the +The generating harness (formerly the devtools workspace claim-vs-evidence +command, now deleted, and its private-report/calibration/publishing +machinery) was retired 2026-08 once the closed campaign it served (`polylogue-sru`) had its terminal artifacts. This page is therefore frozen historical text: the numbers above are not regeneratable through a current command. The reusable query semantics behind From 4f78a0f9a5a4f062cd7bc06e3d6e6f1192a6fb90 Mon Sep 17 00:00:00 2001 From: Sinity Date: Wed, 19 Aug 2026 02:47:59 +0200 Subject: [PATCH 10/10] revert(devtools): restore proof_world_real_slice (verifier REFUTED) Cold adversarial review found the deletion rationale wrong: the ledger row cited the closed jxe uplift campaign, but the module and its test actually cite polylogue-212.11 -- OPEN, P3, whose notes name this tool as the harness half of the real-archive-data extension with a pending unpromoted deliverable. Same name-collision failure class as the dev_loop/mutmut_campaign exclusions caught before this PR was opened, just missed on this one row. Restores devtools/proof_world_real_slice.py, tests/unit/devtools/test_proof_world_real_slice.py, and the 'demo real-slice-screen' CommandSpec verbatim from before deletion (dc7333fb4^), then regenerates docs/devtools.md via devtools render devtools-reference. The other 5 deletions in this PR were independently verified safe and are untouched. Ref .agent/campaigns/2026-08-overhaul/ws-c-devtools-pruning.md --- devtools/command_catalog.py | 19 + devtools/proof_world_real_slice.py | 440 ++++++++++++++++++ docs/devtools.md | 1 + .../devtools/test_proof_world_real_slice.py | 325 +++++++++++++ 4 files changed, 785 insertions(+) create mode 100644 devtools/proof_world_real_slice.py create mode 100644 tests/unit/devtools/test_proof_world_real_slice.py diff --git a/devtools/command_catalog.py b/devtools/command_catalog.py index 7bdc343c97..d37e546d32 100644 --- a/devtools/command_catalog.py +++ b/devtools/command_catalog.py @@ -560,6 +560,25 @@ def to_dict(self) -> dict[str, object]: "devtools workspace bead-reimport-guard export /tmp/issues-snapshot.jsonl", ), ), + CommandSpec( + "demo real-slice-screen", + "workspace", + "Read-only extraction + privacy screening of a candidate real-archive session slice.", + "devtools.proof_world_real_slice", + use_when=( + "Assembling a candidate real-archive slice for the shared demo proof world " + "(polylogue-212.11): pulls sessions read-only via the Polylogue API, flattens them " + "to text, and screens for secret/credential and PII-adjacent patterns before any " + "operator decides to fold the slice into a shared fixture. Never mutates the source " + "archive and never writes into polylogue/scenarios/ on its own." + ), + examples=( + "devtools demo real-slice-screen --archive-root /realm/state/polylogue " + "--session claude-code-session:: --out .agent/scratch/real-slice", + "devtools demo real-slice-screen --archive-root /realm/state/polylogue " + "--refs-file refs.txt --out .agent/scratch/real-slice", + ), + ), CommandSpec( "workspace dev-loop", "workspace", diff --git a/devtools/proof_world_real_slice.py b/devtools/proof_world_real_slice.py new file mode 100644 index 0000000000..afcdb5332d --- /dev/null +++ b/devtools/proof_world_real_slice.py @@ -0,0 +1,440 @@ +"""Real-archive candidate-slice extraction and privacy screening. + +Support for polylogue-212.11 (shared deterministic proof world / Incident +14:32): the deterministic demo corpus should eventually be extended with a +*representative slice of real archive data*, not synthetic fixtures alone. +That extension is deliberately a two-step, human-gated process: + +1. This tool runs **read-only** queries against a real Polylogue archive, + flattens each candidate session to plain text, and screens that text for + secrets/credentials and personal-information patterns. It writes a report + plus rendered transcripts to an arbitrary output directory. It never + mutates the source archive (``Polylogue.get_session`` opens the archive + tiers with ``read_only=True``) and never writes into the product fixture + tree (``polylogue/scenarios/``) on its own. The per-session transcript + files under ``/transcripts/`` are full, unredacted flattened text — + they exist for a human to read the real session content. The + *report/manifest* (``SCREENING_REPORT.md``, ``manifest.json``) are a + different, narrower surface: any matched **secret** value is redacted + before it is written there (see ``scan_text``/``_snippet``), so a report + that later gets shared or accidentally committed doesn't itself become a + secret-leak vector. Matched **PII** text is kept verbatim in the report + since a reviewer needs the real value to judge placeholder vs. genuine + data. Point ``--out`` at a location outside version control (e.g. a + gitignored scratch directory) — this tool applies no guard against + writing into a tracked path. +2. An operator reviews the report and transcripts and decides, session by + session, whether the slice is safe to fold into the shared proof-world + corpus. Only after that explicit approval should the slice move into a + real fixture path. + +The screening pass is a best-effort heuristic layer, not a certification. +"Clean" means "no configured pattern fired" — an operator still has to read +the transcripts before promoting anything to a shared fixture. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING, Any, Protocol + +if TYPE_CHECKING: + from collections.abc import Iterable + + from polylogue.api import Polylogue + + +class _MessageLike(Protocol): + """Structural shape ``_flatten_session_text`` reads from a message. + + A ``Protocol`` (not the concrete ``polylogue.archive.session.domain_models + .Session``/``Message`` classes) so the real session objects returned by + ``Polylogue.get_session`` and the lightweight duck-typed test doubles in + ``tests/unit/devtools/test_proof_world_real_slice.py`` both satisfy the + parameter type structurally, without the tests needing to construct or + subclass the full domain model. Declared as read-only ``@property`` + members rather than plain attributes: mypy checks plain-attribute + Protocol members *invariantly* (both read and write), which the real + ``Message.blocks: list[dict[str, object]]`` fails against a plain + ``blocks: object`` attribute even though every value it can hold is + assignable to ``object``. Properties are read-only, so the check is + covariant instead and both the real model and the test doubles conform. + """ + + @property + def text(self) -> str | None: ... + + @property + def blocks(self) -> object: ... + + +class _SessionLike(Protocol): + """Structural shape ``_flatten_session_text`` reads from a session. + + ``Iterable``, not ``Sequence`` — the real ``Session.messages`` is a + ``MessageCollection`` that supports iteration but is not a nominal + ``collections.abc.Sequence`` subclass, and ``Sequence`` is a concrete ABC + in typeshed (not a structural ``Protocol``) so mypy would reject it here + even though the object is sequence-*shaped*. Only iteration is needed. + """ + + @property + def messages(self) -> Iterable[_MessageLike]: ... + + +# Patterns that indicate a live secret/credential shape. Kept intentionally +# narrow (favor false negatives over drowning the report in noise) — this is +# a triage aid, not a DLP product. +_SECRET_PATTERNS: dict[str, re.Pattern[str]] = { + "aws_access_key_id": re.compile(r"AKIA[0-9A-Z]{16}"), + "generic_credential_assignment": re.compile( + r"(?i)\b(api[_-]?key|secret|password|passwd|access[_-]?token)\b\s*[:=]\s*" + r"['\"]?[A-Za-z0-9_\-/+=.]{12,}" + ), + "bearer_token": re.compile(r"(?i)\bBearer\s+[A-Za-z0-9_\-.=]{16,}"), + "private_key_block": re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----"), + "openai_style_key": re.compile(r"\bsk-[A-Za-z0-9]{20,}\b"), + "slack_token": re.compile(r"\bxox[baprs]-[A-Za-z0-9-]{10,}\b"), + "ssh_public_key": re.compile(r"\bssh-(?:rsa|ed25519) [A-Za-z0-9+/]{20,}"), + "jwt_like": re.compile(r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b"), +} + +# Patterns that may indicate personal information. These fire far more often +# on ordinary dev-work text (localhost IPs, placeholder emails), so callers +# should read the samples rather than treat any hit as disqualifying. +_PII_PATTERNS: dict[str, re.Pattern[str]] = { + "email": re.compile(r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}"), + "home_path": re.compile(r"/home/[a-zA-Z0-9_-]+"), + "ipv4": re.compile(r"\b(?:\d{1,3}\.){3}\d{1,3}\b"), +} + +# Values that are conventionally placeholders/loopback addresses, not real +# personal data. A hit is downgraded only when the *entire* matched string +# equals one of these exactly (never a substring check — "notuser@example.com" +# and "127.0.0.123" must NOT be treated as allowlisted merely because an +# allowlisted value happens to appear inside them). A downgraded hit is still +# reported (never silently dropped). +_ALLOWLIST_VALUES: frozenset[str] = frozenset( + ( + "test@example.com", + "user@example.com", + "127.0.0.1", + "0.0.0.0", + "255.255.255.255", + ) +) + +_SNIPPET_RADIUS = 40 +_MAX_SAMPLES_PER_PATTERN = 5 + + +@dataclass(slots=True) +class PatternHit: + pattern: str + kind: str # "secret" | "pii" + count: int + samples: list[str] = field(default_factory=list) + all_allowlisted: bool = False + + def to_dict(self) -> dict[str, Any]: + return { + "pattern": self.pattern, + "kind": self.kind, + "count": self.count, + "samples": self.samples, + "all_allowlisted": self.all_allowlisted, + } + + +@dataclass(slots=True) +class SessionScreeningResult: + session_id: str + origin: str + title: str | None + created_at: str | None + message_count: int + word_count: int + hits: list[PatternHit] + + @property + def verdict(self) -> str: + secret_hits = [h for h in self.hits if h.kind == "secret"] + if secret_hits: + return "flagged" + pii_hits = [h for h in self.hits if h.kind == "pii" and not h.all_allowlisted] + if pii_hits: + return "review" + return "clean" + + def to_dict(self) -> dict[str, Any]: + return { + "session_id": self.session_id, + "origin": self.origin, + "title": self.title, + "created_at": self.created_at, + "message_count": self.message_count, + "word_count": self.word_count, + "verdict": self.verdict, + "hits": [h.to_dict() for h in self.hits], + } + + +def _snippet(text: str, match: re.Match[str], *, redact: bool) -> str: + """Render a bounded context window around ``match``. + + When ``redact`` is true (secret-kind hits), the matched substring itself + is replaced with a placeholder — the *surrounding* context is still + useful for triage (which pattern fired, roughly where), but the actual + secret value never reaches the report/manifest on disk. PII hits are not + redacted: a human reviewer needs the real matched text (e.g. the actual + email/IP) to judge whether it is a placeholder or genuine personal data. + """ + + start = max(0, match.start() - _SNIPPET_RADIUS) + end = min(len(text), match.end() + _SNIPPET_RADIUS) + prefix = "…" if start > 0 else "" + suffix = "…" if end < len(text) else "" + window = text[start:end] + if redact: + rel_start = match.start() - start + rel_end = match.end() - start + window = f"{window[:rel_start]}{window[rel_end:]}" + return f"{prefix}{window!r}{suffix}" + + +def scan_text(text: str) -> list[PatternHit]: + """Run every configured secret/PII pattern over ``text``. + + Returns one :class:`PatternHit` per pattern that matched at least once, + each carrying up to ``_MAX_SAMPLES_PER_PATTERN`` samples for human + review. Secret-kind samples have the actual matched value redacted (see + :func:`_snippet`) — never the raw secret — to keep the report itself + from becoming a leak surface. PII-kind samples keep the real matched + text, which a reviewer needs to judge placeholder vs. genuine data. + """ + + hits: list[PatternHit] = [] + for kind, patterns in (("secret", _SECRET_PATTERNS), ("pii", _PII_PATTERNS)): + redact = kind == "secret" + for name, pattern in patterns.items(): + matches = list(pattern.finditer(text)) + if not matches: + continue + samples = [_snippet(text, m, redact=redact) for m in matches[:_MAX_SAMPLES_PER_PATTERN]] + all_allowlisted = all(m.group(0) in _ALLOWLIST_VALUES for m in matches) + hits.append( + PatternHit( + pattern=name, + kind=kind, + count=len(matches), + samples=samples, + all_allowlisted=all_allowlisted, + ) + ) + return hits + + +def _flatten_session_text(session: _SessionLike) -> str: + """Flatten every message's text and structured blocks to one string.""" + + parts: list[str] = [] + for message in session.messages: + if message.text: + parts.append(message.text) + if message.blocks: + parts.append(json.dumps(message.blocks, default=str)) + return "\n".join(parts) + + +async def _screen_session_with(poly: Polylogue, session_id: str) -> tuple[SessionScreeningResult, str]: + """Screen one session through an already-open ``Polylogue`` instance.""" + + session = await poly.get_session(session_id) + if session is None: + raise ValueError(f"session not found in archive: {session_id}") + text = _flatten_session_text(session) + word_count = len(text.split()) + result = SessionScreeningResult( + session_id=str(session.id), + origin=str(session.origin), + title=session.title, + created_at=str(session.created_at) if session.created_at else None, + message_count=len(session.messages), + word_count=word_count, + hits=scan_text(text), + ) + return result, text + + +async def screen_session(archive_root: Path, session_id: str) -> tuple[SessionScreeningResult, str]: + """Load one session read-only and screen it. Returns (result, transcript_text). + + Opens and closes its own scoped ``Polylogue`` instance — the convenient + single-session entry point used by tests and one-off callers. Batch + callers should use :func:`screen_sessions`, which opens the archive once + and reuses the same instance across every session id instead of paying + the open/close cost per id. + """ + + from polylogue.api import Polylogue + + async with Polylogue(archive_root=archive_root) as pl: + return await _screen_session_with(pl, session_id) + + +async def screen_sessions(archive_root: Path, session_ids: list[str]) -> list[tuple[SessionScreeningResult, str]]: + """Screen every id in ``session_ids`` through one shared archive open.""" + + from polylogue.api import Polylogue + + async with Polylogue(archive_root=archive_root) as pl: + return [await _screen_session_with(pl, session_id) for session_id in session_ids] + + +def render_report_markdown(results: list[SessionScreeningResult], *, archive_root: Path) -> str: + lines = [ + "# Real-archive candidate slice — privacy screening report", + "", + f"Archive root: `{archive_root}`", + f"Sessions screened: {len(results)}", + "", + "This is an automated triage pass (pattern matching only). It is not a", + "certification. An operator must read the transcripts before any of", + "this content is folded into the shared demo proof-world fixture.", + "", + "| session_id | origin | verdict | messages | words | flags |", + "| --- | --- | --- | ---: | ---: | --- |", + ] + for r in results: + flags = ", ".join(f"{h.pattern}×{h.count}" for h in r.hits) or "—" + lines.append( + f"| `{r.session_id}` | {r.origin} | **{r.verdict}** | {r.message_count} | {r.word_count} | {flags} |" + ) + lines.append("") + for r in results: + lines.append(f"## `{r.session_id}`") + lines.append("") + lines.append(f"- title: {r.title!r}") + lines.append(f"- created_at: {r.created_at}") + lines.append(f"- verdict: **{r.verdict}**") + if not r.hits: + lines.append("- no pattern hits") + for h in r.hits: + lines.append(f"- `{h.pattern}` ({h.kind}) × {h.count}{' (all allowlisted)' if h.all_allowlisted else ''}") + for sample in h.samples: + lines.append(f" - {sample}") + lines.append("") + return "\n".join(lines) + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="devtools demo real-slice-screen", + description=( + "Read-only extraction and privacy screening of a candidate real-archive " + "session slice, for later human-gated inclusion in the shared demo " + "proof-world corpus (polylogue-212.11)." + ), + ) + parser.add_argument("--archive-root", type=Path, required=True, help="Real archive root to read (read-only).") + parser.add_argument( + "--session", + dest="sessions", + action="append", + default=[], + help="Session id to screen (repeatable).", + ) + parser.add_argument( + "--refs-file", + type=Path, + default=None, + help="Optional file with one session id per line (blank lines and '#' comments ignored).", + ) + parser.add_argument("--out", type=Path, required=True, help="Output directory for the report + transcripts.") + return parser + + +def _safe_transcript_filename(session_id: str) -> str: + """Filesystem-safe, collision-free filename stem for a session id. + + Path-unsafe characters are replaced for readability, but readability + alone is not collision-safe: distinct session ids that differ only in + which punctuation character separates otherwise-identical characters + (e.g. ``origin:a:b`` vs. ``origin:a_b``) would sanitize to the same + stem. A short content hash of the *original, unsanitized* session id is + appended so two distinct session ids can never produce the same + filename, guaranteeing no transcript is silently overwritten. + """ + + sanitized = re.sub(r"[^A-Za-z0-9_.-]", "_", session_id) + digest = hashlib.sha256(session_id.encode("utf-8")).hexdigest()[:12] + return f"{sanitized}__{digest}" + + +def _read_refs_file(path: Path) -> list[str]: + refs: list[str] = [] + for line in path.read_text(encoding="utf-8").splitlines(): + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + refs.append(stripped) + return refs + + +def main(argv: list[str] | None = None) -> int: + import asyncio + + args = _parser().parse_args(argv) + session_ids = list(args.sessions) + if args.refs_file is not None: + session_ids.extend(_read_refs_file(args.refs_file)) + session_ids = list(dict.fromkeys(session_ids)) # de-dupe, preserve order + if not session_ids: + print("no session ids given (use --session or --refs-file)", file=sys.stderr) + return 2 + + pairs = asyncio.run(screen_sessions(args.archive_root, session_ids)) + results = [r for r, _ in pairs] + + args.out.mkdir(parents=True, exist_ok=True) + transcripts_dir = args.out / "transcripts" + transcripts_dir.mkdir(parents=True, exist_ok=True) + seen_filenames: dict[str, str] = {} + for result, text in pairs: + safe_name = _safe_transcript_filename(result.session_id) + prior = seen_filenames.get(safe_name) + if prior is not None and prior != result.session_id: + # Should be unreachable (sha256 collision on distinct inputs), + # but fail loudly rather than silently overwrite a transcript. + raise RuntimeError( + f"transcript filename collision: {safe_name!r} claimed by both {prior!r} and {result.session_id!r}" + ) + seen_filenames[safe_name] = result.session_id + (transcripts_dir / f"{safe_name}.txt").write_text(text, encoding="utf-8") + + manifest = { + "archive_root": str(args.archive_root), + "sessions": [r.to_dict() for r in results], + } + (args.out / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + report_md = render_report_markdown(results, archive_root=args.archive_root) + (args.out / "SCREENING_REPORT.md").write_text(report_md + "\n", encoding="utf-8") + + flagged = [r for r in results if r.verdict == "flagged"] + review = [r for r in results if r.verdict == "review"] + print( + f"screened {len(results)} sessions: {len(flagged)} flagged, {len(review)} need review, " + f"{len(results) - len(flagged) - len(review)} clean" + ) + print(f"report: {args.out / 'SCREENING_REPORT.md'}") + return 1 if flagged else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/devtools.md b/docs/devtools.md index 5ea929b4ff..d8b88c85bd 100644 --- a/docs/devtools.md +++ b/docs/devtools.md @@ -180,6 +180,7 @@ These are the commands worth remembering during normal repo work: | Command | Description | | --- | --- | +| `devtools demo real-slice-screen` | Read-only extraction + privacy screening of a candidate real-archive session slice. | | `devtools workspace affordance-usage` | Analyze agent affordance/tool usage from archive tool-use rows. | | `devtools workspace agent-meta-sidecar-purge-apply` | Purge agent-*.meta.json subagent-sidecar phantom sessions from index.db. | | `devtools workspace agent-meta-sidecar-sweep` | Find agent-*.meta.json subagent-sidecar phantom sessions (message_count=0). | diff --git a/tests/unit/devtools/test_proof_world_real_slice.py b/tests/unit/devtools/test_proof_world_real_slice.py new file mode 100644 index 0000000000..455815930b --- /dev/null +++ b/tests/unit/devtools/test_proof_world_real_slice.py @@ -0,0 +1,325 @@ +"""Tests for the real-archive candidate-slice screening harness (polylogue-212.11).""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from devtools import proof_world_real_slice as m +from polylogue.core.enums import Provider +from tests.infra.archive_scenarios import native_session_id_for +from tests.infra.storage_records import SessionBuilder, db_setup + +_CLEAN_ID = native_session_id_for("claude-ai", "clean-session") +_SECRET_ID = native_session_id_for("claude-ai", "secret-session") + + +class _FakeMessage: + def __init__(self, text: str | None, blocks: object = None) -> None: + self.text = text + self.blocks = blocks + + +class _FakeSession: + def __init__(self, messages: list[_FakeMessage]) -> None: + self.messages = messages + + +# --- pure scan_text() behavior --------------------------------------------- + + +def test_scan_text_finds_no_hits_on_clean_text() -> None: + hits = m.scan_text("just an ordinary sentence about pytest and git diffs") + assert hits == [] + + +def test_scan_text_flags_a_credential_shaped_string() -> None: + hits = m.scan_text("aws key: AKIAABCDEFGHIJKLMNOP") + names = {h.pattern for h in hits} + assert "aws_access_key_id" in names + secret_hit = next(h for h in hits if h.pattern == "aws_access_key_id") + assert secret_hit.kind == "secret" + assert secret_hit.count == 1 + assert not secret_hit.all_allowlisted + + +def test_scan_text_allowlists_placeholder_email_and_loopback() -> None: + hits = m.scan_text("git config user.email test@example.com; server on 127.0.0.1") + email_hit = next(h for h in hits if h.pattern == "email") + ipv4_hit = next(h for h in hits if h.pattern == "ipv4") + assert email_hit.all_allowlisted + assert ipv4_hit.all_allowlisted + + +def test_scan_text_does_not_allowlist_a_real_looking_email() -> None: + hits = m.scan_text("contact jane.doe@personalmail.example for details") + email_hit = next(h for h in hits if h.pattern == "email") + assert not email_hit.all_allowlisted + + +def test_scan_text_does_not_allowlist_by_substring_containment() -> None: + """A match that merely *contains* an allowlisted value must not be + downgraded — only an exact full-match equals check counts. Regression + for a bug where `notuser@example.com` and `127.0.0.123` were both + treated as fully allowlisted (and thus 'clean') purely because + `user@example.com`/`127.0.0.1` occur as substrings.""" + + hits = m.scan_text("contact notuser@example.com and reach server at 127.0.0.123 for details") + email_hit = next(h for h in hits if h.pattern == "email") + ipv4_hit = next(h for h in hits if h.pattern == "ipv4") + assert not email_hit.all_allowlisted + assert not ipv4_hit.all_allowlisted + + result = m.SessionScreeningResult( + session_id="x", + origin="claude-code-session", + title=None, + created_at=None, + message_count=1, + word_count=1, + hits=[email_hit, ipv4_hit], + ) + assert result.verdict == "review" + + +# --- verdict computation ----------------------------------------------------- + + +def test_verdict_clean_when_no_hits() -> None: + result = m.SessionScreeningResult( + session_id="x", + origin="claude-code-session", + title=None, + created_at=None, + message_count=1, + word_count=1, + hits=[], + ) + assert result.verdict == "clean" + + +def test_verdict_flagged_beats_review_when_a_secret_hits() -> None: + result = m.SessionScreeningResult( + session_id="x", + origin="claude-code-session", + title=None, + created_at=None, + message_count=1, + word_count=1, + hits=[ + m.PatternHit(pattern="email", kind="pii", count=1, samples=["s"], all_allowlisted=False), + m.PatternHit(pattern="openai_style_key", kind="secret", count=1, samples=["s"], all_allowlisted=False), + ], + ) + assert result.verdict == "flagged" + + +def test_verdict_review_when_only_non_allowlisted_pii_hits() -> None: + result = m.SessionScreeningResult( + session_id="x", + origin="claude-code-session", + title=None, + created_at=None, + message_count=1, + word_count=1, + hits=[m.PatternHit(pattern="home_path", kind="pii", count=1, samples=["s"], all_allowlisted=False)], + ) + assert result.verdict == "review" + + +def test_verdict_clean_when_pii_hits_are_fully_allowlisted() -> None: + result = m.SessionScreeningResult( + session_id="x", + origin="claude-code-session", + title=None, + created_at=None, + message_count=1, + word_count=1, + hits=[m.PatternHit(pattern="email", kind="pii", count=1, samples=["s"], all_allowlisted=True)], + ) + assert result.verdict == "clean" + + +# --- secret redaction in samples ---------------------------------------------- + + +def test_scan_text_redacts_the_actual_secret_value_in_samples() -> None: + """The report/manifest must never embed a raw secret value verbatim — + only PII context does that. Regression for a bug where the snippet + window always fully contained the matched secret text despite the + docstring's claim that samples 'never' leak the full match.""" + + text = "the real key is AKIAABCDEFGHIJKLMNOP and it must stay secret" + hits = m.scan_text(text) + secret_hit = next(h for h in hits if h.pattern == "aws_access_key_id") + joined = " ".join(secret_hit.samples) + assert "AKIAABCDEFGHIJKLMNOP" not in joined + assert "redacted" in joined + # surrounding context should still be present for triage + assert "real key" in joined + + +def test_scan_text_keeps_real_pii_text_in_samples_for_human_judgment() -> None: + hits = m.scan_text("contact jane.doe@personalmail.example for details") + email_hit = next(h for h in hits if h.pattern == "email") + joined = " ".join(email_hit.samples) + assert "jane.doe@personalmail.example" in joined + + +# --- transcript filename collision safety -------------------------------------- + + +def test_safe_transcript_filename_disambiguates_punctuation_variants() -> None: + """Two distinct session ids that differ only in which punctuation + character separates otherwise-identical characters must never collide + on the sanitized filename stem.""" + + a = m._safe_transcript_filename("origin:a:b") + b = m._safe_transcript_filename("origin:a_b") + assert a != b + + +def test_safe_transcript_filename_is_deterministic() -> None: + assert m._safe_transcript_filename("claude-code-session:abc") == m._safe_transcript_filename( + "claude-code-session:abc" + ) + + +# --- flatten helper ----------------------------------------------------------- + + +def test_flatten_session_text_includes_message_text_and_block_json() -> None: + session = _FakeSession( + [ + _FakeMessage(text="hello world"), + _FakeMessage(text=None, blocks=[{"kind": "tool_use", "input": {"cmd": "ls"}}]), + ] + ) + flat = m._flatten_session_text(session) + assert "hello world" in flat + assert "tool_use" in flat + assert '"cmd": "ls"' in flat + + +# --- report rendering ----------------------------------------------------- + + +def test_render_report_markdown_includes_verdict_and_samples() -> None: + result = m.SessionScreeningResult( + session_id="claude-code-session:abc", + origin="claude-code-session", + title="Some session", + created_at="2026-01-01T00:00:00Z", + message_count=3, + word_count=42, + hits=[m.PatternHit(pattern="home_path", kind="pii", count=2, samples=["…/home/x…"], all_allowlisted=False)], + ) + md = m.render_report_markdown([result], archive_root=Path("/fake/archive")) + assert "claude-code-session:abc" in md + assert "**review**" in md + assert "home_path" in md + assert "/home/x" in md + + +# --- end-to-end read path against a seeded archive -------------------------- + + +async def _seed(db_path: Path) -> None: + await ( + SessionBuilder(db_path, "clean-session") + .provider(Provider.CLAUDE_AI.value) + .title("Clean session") + .add_message(text="just discussing pytest fixtures, nothing sensitive") + .build() + ) + await ( + SessionBuilder(db_path, "secret-session") + .provider(Provider.CLAUDE_AI.value) + .title("Session with a planted secret") + .add_message(text="here is my key: AKIAABCDEFGHIJKLMNOP please rotate it") + .build() + ) + + +async def test_screen_session_reads_real_archive_and_flags_planted_secret( + workspace_env: dict[str, Path], +) -> None: + db_path = db_setup(workspace_env) + await _seed(db_path) + archive_root = db_path.parent + + clean_result, clean_text = await m.screen_session(archive_root, _CLEAN_ID) + assert clean_result.verdict == "clean" + assert "pytest fixtures" in clean_text + + secret_result, secret_text = await m.screen_session(archive_root, _SECRET_ID) + assert secret_result.verdict == "flagged" + assert any(h.pattern == "aws_access_key_id" for h in secret_result.hits) + # the raw transcript text is unredacted (it exists for full human review)... + assert "AKIAABCDEFGHIJKLMNOP" in secret_text + # ...but the report-facing samples must never carry the raw secret value + all_samples = [s for h in secret_result.hits for s in h.samples] + assert not any("AKIAABCDEFGHIJKLMNOP" in s for s in all_samples) + + +async def test_screen_session_raises_for_unknown_session(workspace_env: dict[str, Path]) -> None: + db_path = db_setup(workspace_env) + await _seed(db_path) + + with pytest.raises(ValueError, match="not found"): + await m.screen_session(db_path.parent, "claude-code-session:does-not-exist") + + +async def test_screen_sessions_opens_the_archive_once_for_the_whole_batch( + workspace_env: dict[str, Path], monkeypatch: pytest.MonkeyPatch +) -> None: + """Regression for a CodeRabbit finding on PR #2885: ``screen_sessions`` + must open one shared ``Polylogue`` instance and reuse it across every + session id in the batch, not reopen the archive tiers per id. Counts real + ``Polylogue.__aenter__`` calls (the actual archive-open chokepoint) while + driving the real screening path against a seeded archive — this fails if + ``screen_sessions`` regresses to calling ``screen_session`` (which opens + its own scoped instance) once per id.""" + + from polylogue.api import Polylogue + + db_path = db_setup(workspace_env) + await _seed(db_path) + archive_root = db_path.parent + + open_count = 0 + original_aenter = Polylogue.__aenter__ + + async def counting_aenter(self: Polylogue) -> Polylogue: + nonlocal open_count + open_count += 1 + return await original_aenter(self) + + monkeypatch.setattr(Polylogue, "__aenter__", counting_aenter) + + pairs = await m.screen_sessions(archive_root, [_CLEAN_ID, _SECRET_ID]) + + assert open_count == 1 + assert [r.session_id for r, _ in pairs] == [_CLEAN_ID, _SECRET_ID] + assert pairs[0][0].verdict == "clean" + assert pairs[1][0].verdict == "flagged" + + +# --- CLI argument handling --------------------------------------------------- + + +def test_main_exits_nonzero_with_no_session_ids(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + code = m.main(["--archive-root", str(tmp_path), "--out", str(tmp_path / "out")]) + assert code == 2 + captured = capsys.readouterr() + assert "no session ids given" in captured.err + + +def test_read_refs_file_skips_blanks_and_comments(tmp_path: Path) -> None: + refs_path = tmp_path / "refs.txt" + refs_path.write_text("\n# a comment\nclaude-code-session:a:b\n\nclaude-code-session:c:d\n", encoding="utf-8") + assert m._read_refs_file(refs_path) == [ + "claude-code-session:a:b", + "claude-code-session:c:d", + ]