diff --git a/polylogue/sources/parsers/chatgpt.py b/polylogue/sources/parsers/chatgpt.py index 56df5be956..e3f602e6ad 100644 --- a/polylogue/sources/parsers/chatgpt.py +++ b/polylogue/sources/parsers/chatgpt.py @@ -11,7 +11,7 @@ from polylogue.archive.message.artifacts import classify_material_origin, classify_text_message_type from polylogue.archive.message.roles import Role from polylogue.archive.message.types import MessageType -from polylogue.core.enums import BlockType, Provider, SessionKind, WebConstructType +from polylogue.core.enums import BlockType, Provider, SessionKind, TitleSource, WebConstructType from polylogue.core.timestamps import parse_timestamp from polylogue.sources.providers.chatgpt_session_models import ChatGPTNode @@ -1282,7 +1282,18 @@ def parse(payload: Mapping[str, object], fallback_id: str) -> ParsedSession: ] session_events.extend(_block_metadata_evidence_events(messages)) duration_values = [message.duration_ms for message in messages if message.duration_ms is not None] - title = payload.get("title") or payload.get("name") or fallback_id + provider_title = payload.get("title") or payload.get("name") + title = provider_title or fallback_id + # polylogue-cijx.4 decision 3 / has_real_title (archive_tiers/archive.py): + # title_source is the sole gate distinguishing a genuine provider title + # from the bare native-id fallback this parser stores in `title` when + # ChatGPT's own export carries neither `title` nor `name` -- without it, + # every ChatGPT session (titled or not) silently degraded to the + # structural "N msgs" label once #3421 made that gate strict. Only a + # real payload title counts as ORIGIN evidence; the id fallback is + # exactly the "worse than the UUID it replaces" case that gate exists + # to catch. + title_source = TitleSource.ORIGIN if provider_title else None conv_id = payload.get("id") or payload.get("uuid") or payload.get("conversation_id") ingest_flags: list[str] = [] if not messages and payload.get("conversation_id") and payload.get("id") and "mapping" not in payload: @@ -1301,6 +1312,7 @@ def parse(payload: Mapping[str, object], fallback_id: str) -> ParsedSession: source_name=Provider.CHATGPT, provider_session_id=str(conv_id or fallback_id), title=str(title), + title_source=title_source, session_kind=session_kind, provider_project_ref=provider_project_ref, created_at=str(payload.get("create_time")) if payload.get("create_time") is not None else None, diff --git a/polylogue/storage/repair.py b/polylogue/storage/repair.py index 855f3d5c5b..f70dfad364 100644 --- a/polylogue/storage/repair.py +++ b/polylogue/storage/repair.py @@ -5681,24 +5681,40 @@ def _empty_session_debris_session_ids(conn: sqlite3.Connection) -> list[str]: Shared by ``count_empty_sessions_sync`` and ``repair_empty_sessions`` so the reported debt and the deleted rows can never diverge (polylogue-ne6k). + + Both ``_empty_session_candidate_ids`` and + ``_raw_artifact_positively_fails_classification`` access rows by column + name, so this sets ``row_factory = sqlite3.Row`` defensively on entry + (restoring the caller's original factory on exit) rather than assuming + every caller-supplied connection already has it -- ``count_empty_sessions_sync``'s + own docstring documents that it is "called with a caller-supplied, + possibly read-only, connection" (polylogue-9rdky: the maintenance + planner's preview/execute path opens a plain tuple-row connection via + ``open_readonly_connection``, which crashed both helpers with + ``TypeError: tuple indices must be integers or slices, not str``). """ - candidates = _empty_session_candidate_ids(conn) - if not candidates: - return [] - source_db = _sibling_source_db_path(conn) - if source_db is None or not source_db.exists(): - # No source tier reachable -> no way to obtain positive evidence for - # any candidate -> retain all of them. - return [] - conn.execute("ATTACH DATABASE ? AS source", (str(source_db),)) + original_row_factory = conn.row_factory + conn.row_factory = sqlite3.Row try: - return [ - session_id - for session_id, raw_id in candidates - if _raw_artifact_positively_fails_classification(conn, raw_id) - ] + candidates = _empty_session_candidate_ids(conn) + if not candidates: + return [] + source_db = _sibling_source_db_path(conn) + if source_db is None or not source_db.exists(): + # No source tier reachable -> no way to obtain positive evidence for + # any candidate -> retain all of them. + return [] + conn.execute("ATTACH DATABASE ? AS source", (str(source_db),)) + try: + return [ + session_id + for session_id, raw_id in candidates + if _raw_artifact_positively_fails_classification(conn, raw_id) + ] + finally: + conn.execute("DETACH DATABASE source") finally: - conn.execute("DETACH DATABASE source") + conn.row_factory = original_row_factory def repair_empty_sessions(config: Config, dry_run: bool = False) -> RepairResult: diff --git a/polylogue/storage/sqlite/queries/raw_writes.py b/polylogue/storage/sqlite/queries/raw_writes.py index cf391dc512..2a80510d80 100644 --- a/polylogue/storage/sqlite/queries/raw_writes.py +++ b/polylogue/storage/sqlite/queries/raw_writes.py @@ -45,8 +45,8 @@ async def save_raw_session( validated_at_ms, validation_status, validation_error, validation_drift_count, validation_mode, detection_warnings_json, logical_source_key, revision_kind, source_revision, predecessor_source_revision, predecessor_raw_id, baseline_raw_id, append_start_offset, - append_end_offset, acquisition_generation, revision_authority - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + append_end_offset, acquisition_generation, revision_authority, revision_authority_evidence + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( record.raw_id, @@ -77,6 +77,16 @@ async def save_raw_session( record.revision.append_end_offset if record.revision else None, record.revision.acquisition_generation if record.revision else None, record.revision.authority.value if record.revision else "quarantined", + # revision_authority_evidence (migration 017) is never computed at + # initial-write time -- it is only ever populated later by a + # dedicated, explicitly operator-invoked maintenance actuator + # (raw_live_source_reconciliation_apply.py / + # raw_append_chain_backfill_apply.py) re-verifying the raw + # against still-present live source bytes. This is `INSERT OR + # IGNORE`, so binding NULL here for a brand-new row is correct + # and a duplicate-key insert attempt never overwrites an + # already-recorded verification verdict. + None, ), ) inserted = bool(cursor.rowcount > 0) diff --git a/tests/infra/storage_records.py b/tests/infra/storage_records.py index e0480fd8fe..13b4c678b4 100644 --- a/tests/infra/storage_records.py +++ b/tests/infra/storage_records.py @@ -13,7 +13,15 @@ from polylogue.archive.message.roles import Role from polylogue.archive.session.branch_type import BranchType -from polylogue.core.enums import BlockType, Origin, Provider, SemanticBlockType, ValidationMode, ValidationStatus +from polylogue.core.enums import ( + BlockType, + Origin, + Provider, + SemanticBlockType, + TitleSource, + ValidationMode, + ValidationStatus, +) from polylogue.core.json import dumps, loads, require_json_document, require_json_value from polylogue.core.sources import origin_from_provider, provider_from_origin from polylogue.core.timestamps import _timestamp_sort_key @@ -937,6 +945,16 @@ def _blocks(message: MessageRecord) -> list[ParsedContentBlock]: source_name=provider_from_origin(session.origin), provider_session_id=session.native_id, title=session.title, + # A real parser that sets a title always sets title_source alongside + # it (assembly_codex.py, assembly_gemini.py, etc.) -- write.py's + # session upsert treats title_source as the sole gate for "is this a + # real title" (archive_tiers/archive.py's has_real_title check, + # polylogue-cijx.4 decision 3), so a builder-set title with no + # title_source silently degrades to the structural "N msgs" fallback + # at read time. Mirror real-parser provenance here rather than + # leaving every test-built session's explicit title invisible to + # that gate. + title_source=TitleSource.ORIGIN if session.title else None, created_at=session.created_at, updated_at=session.updated_at, messages=parsed_messages, @@ -1423,3 +1441,75 @@ def create_session( builder.save() return cid + + def mark_as_phantom_debris(self, native_id: str, *, provider: str = "test") -> str: + """Attach an ``agent-*.meta.json``-shaped phantom raw artifact to an + already-created session and link it via ``sessions.raw_id``. + + ``count_empty_sessions_sync``/``repair_empty_sessions`` + (``polylogue/storage/repair.py``) only ever count a message-less (or + all-zero-word) session as debris when its raw artifact *positively + fails* the current record-shape classifier + (``_raw_artifact_positively_fails_classification``) -- a session + created via :meth:`create_session` with no raw content at all has + ``raw_id IS NULL``, which the classifier treats as "no evidence + either way" and therefore always retains (never counted as debt). + This seeds the same phantom shape + ``tests/unit/storage/test_empty_session_repair_provenance.py``'s + ``_seed`` helper uses (an ``agent-*.meta.json`` sidecar path, a + genuinely-debris shape the classifier positively refuses), so a + caller that wants an "empty" session to actually register as + maintenance debt must call this after :meth:`create_session`. + + Resolves the blob store from ``self.db_path``'s own parent directory + (this factory's archive root), never the ambient + ``POLYLOGUE_ARCHIVE_ROOT``/``blob_store_root()`` config -- a caller + that seeds a ``DbFactory`` pointed at an archive root distinct from + the ambient one (e.g. verifying config-supplied paths win over + ambient defaults) must have the phantom blob land in the same + archive the classifier will actually read back from. + """ + from polylogue.storage.blob_store import BlobStore + + # SessionBuilder.__init__ always stores native_id as f"ext-{session_id}" + # (the "id" callers pass to create_session), so the lookup below must + # match that same transform, not the bare caller-supplied id. + stored_native_id = f"ext-{native_id}" + origin = _origin_value(provider) + archive_root = self.db_path.parent + source_db = archive_root / "source.db" + store = BlobStore(archive_root / "blob") + raw_id, blob_size = store.write_from_bytes( + f'{{"agentType":"general-purpose","for":"{stored_native_id}"}}'.encode() + ) + + with sqlite3.connect(source_db) as source_conn: + source_conn.execute( + """ + INSERT INTO raw_sessions ( + raw_id, origin, native_id, source_path, source_index, blob_hash, blob_size, acquired_at_ms + ) VALUES (?, ?, ?, ?, 0, ?, ?, 1) + """, + ( + raw_id, + origin.value, + stored_native_id, + f"agent-{stored_native_id}.meta.json", + bytes.fromhex(raw_id), + blob_size, + ), + ) + source_conn.commit() + + with sqlite3.connect(self.db_path) as index_conn: + cursor = index_conn.execute( + "UPDATE sessions SET raw_id = ? WHERE native_id = ? AND origin = ?", + (raw_id, stored_native_id, origin.value), + ) + if cursor.rowcount != 1: + raise AssertionError( + f"mark_as_phantom_debris: expected exactly one session row for " + f"native_id={stored_native_id!r} origin={origin.value!r}, updated {cursor.rowcount}" + ) + index_conn.commit() + return raw_id diff --git a/tests/unit/api/test_assertion_candidate_evidence_disclosure.py b/tests/unit/api/test_assertion_candidate_evidence_disclosure.py index e06ce398e3..b47a7b1ec0 100644 --- a/tests/unit/api/test_assertion_candidate_evidence_disclosure.py +++ b/tests/unit/api/test_assertion_candidate_evidence_disclosure.py @@ -15,7 +15,7 @@ from polylogue import Polylogue from polylogue.archive.message.roles import Role -from polylogue.core.enums import AssertionKind, BlockType, Provider +from polylogue.core.enums import AssertionKind, BlockType, Provider, TitleSource from polylogue.sources.parsers.base import ParsedContentBlock, ParsedMessage, ParsedSession from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore from polylogue.storage.sqlite.archive_tiers.user_write import upsert_assertion @@ -29,6 +29,7 @@ def _seed_candidate(root: Path) -> tuple[str, str]: source_name=Provider.CODEX, provider_session_id="evidence-review", title="Evidence review source", + title_source=TitleSource.ORIGIN, messages=[ ParsedMessage( provider_message_id="m1", diff --git a/tests/unit/api/test_facade_contracts.py b/tests/unit/api/test_facade_contracts.py index 1f29b9aff6..af3ac77e43 100644 --- a/tests/unit/api/test_facade_contracts.py +++ b/tests/unit/api/test_facade_contracts.py @@ -4444,7 +4444,10 @@ async def test_archive_tiers_api_delete_uses_index_tier_and_keeps_user_overlay(t await archive.close() -@pytest.mark.frozen_clock_modules("polylogue.storage.sqlite.archive_tiers.archive") +@pytest.mark.frozen_clock_modules( + "polylogue.storage.sqlite.archive_tiers.archive", + "polylogue.storage.sqlite.archive_tiers.revision_governance", +) async def test_archive_tiers_api_raw_artifacts_read_source_tier(tmp_path: Path, frozen_clock: FrozenClock) -> None: """Raw artifact facade reads ``source.db`` rows and their parse-lifecycle timestamp. diff --git a/tests/unit/cli/__snapshots__/test_help_snapshots.ambr b/tests/unit/cli/__snapshots__/test_help_snapshots.ambr index ffba61829f..6e5e703c0b 100644 --- a/tests/unit/cli/__snapshots__/test_help_snapshots.ambr +++ b/tests/unit/cli/__snapshots__/test_help_snapshots.ambr @@ -118,6 +118,11 @@ Sort by field --reverse Reverse sort order --sample INTEGER Random sample of N sessions + --root / --no-root Only top-level sessions (--root, the implicit + default when unset) or only subagent/branch + children (--no-root). Session counts count + top-level sessions unless --no-root or + root:false is used (polylogue-j8u2). -o, --output TEXT Output destinations: browser, clipboard, stdout (comma-separated) --json Shortcut for --format json. Disables color and @@ -192,6 +197,7 @@ Other commands: agent Install executable agent guidance. annotations Import typed annotation batches. + compare Blind pairwise comparative judgment and calibration. ''' # --- diff --git a/tests/unit/cli/__snapshots__/test_plain_cli_snapshots.ambr b/tests/unit/cli/__snapshots__/test_plain_cli_snapshots.ambr index 1a93fef6e2..222500b5b6 100644 --- a/tests/unit/cli/__snapshots__/test_plain_cli_snapshots.ambr +++ b/tests/unit/cli/__snapshots__/test_plain_cli_snapshots.ambr @@ -50,8 +50,8 @@ "total_cost_usd": 0.0, "cost_is_estimated": true, "tokens": { - "input_tokens": 191, - "output_tokens": 0, + "input_tokens": 100, + "output_tokens": 91, "cache_read_tokens": 0, "cache_write_tokens": 0 }, @@ -162,8 +162,8 @@ "total_cost_usd": 0.0, "cost_is_estimated": true, "tokens": { - "input_tokens": 191, - "output_tokens": 0, + "input_tokens": 100, + "output_tokens": 91, "cache_read_tokens": 0, "cache_write_tokens": 0 }, @@ -524,7 +524,8 @@ "next_offset": null, "offset": 0, "origin": null, - "total": 2 + "total": 2, + "total_unit": "top-level sessions" } ''' @@ -546,6 +547,7 @@ "sessions": 2, "messages": 12, "raw_records": 2, + "unidentified_artifacts": 0, "next_action": "polylogue init", "component_readiness": { "raw_materialization": { @@ -1015,8 +1017,8 @@ "path": "", "exists": true, "size_bytes": , - "expected_user_version": 15, - "user_version": 15, + "expected_user_version": 20, + "user_version": 20, "version_status": "ok", "table_counts": { "raw_sessions": 2, @@ -1039,8 +1041,8 @@ "path": "", "exists": true, "size_bytes": , - "expected_user_version": 46, - "user_version": 46, + "expected_user_version": 57, + "user_version": 57, "version_status": "ok", "table_counts": { "sessions": 2, @@ -1150,7 +1152,7 @@ "path": "", "exists": true, "wal_bytes": 0, - "sqlite_stat1_rows": 10, + "sqlite_stat1_rows": 12, "planner_stats_present": true }, "index": { @@ -1350,7 +1352,7 @@ - **Sessions:** 4 analyzed / 4 matched - **Origins:** chatgpt-export (2), claude-code-session (2) - **Total cost:** $0.00 _(estimated)_ - - **Tokens:** in 191, out 0, cache-read 0, cache-write 0 + - **Tokens:** in 100, out 91, cache-read 0, cache-write 0 ## Distributions @@ -1383,7 +1385,7 @@ | session_count | 4 | | wallclock_span | span_ms=75663126945, summed_wall_ms= | | estimated_cost_usd | $0.000000 (estimated) | - | token_lanes | input=191, output=0, cache_read=0, cache_write=0 | + | token_lanes | input=100, output=91, cache_read=0, cache_write=0 | | top_expensive_session | no_signal: no session in scope carries a positive cost figure | | repos_touched | (none) | | subagent_branch_count | 0 | @@ -1417,7 +1419,7 @@ sessions: analyzed=4 matched=4 origins: chatgpt-export=2, claude-code-session=2 total_cost: $0.00 (estimated) - tokens: in=191 out=0 cache_read=0 cache_write=0 + tokens: in=100 out=91 cache_read=0 cache_write=0 cost_per_session_usd: no signal wall_per_session_s: n=4 total=1.45443e+ min=240 p50=360 p90=7.38324e+07 max=7.38324e+07 mean=3.63606e+07 wallclock_span_ms: 75663126945 @@ -1433,7 +1435,7 @@ sessions: 4 wallclock: span_ms=75663126945 summed_wall_ms= ( -> ) cost: $0.000000 (estimated) - tokens: input=191 output=0 cache_read=0 cache_write=0 + tokens: input=100 output=91 cache_read=0 cache_write=0 top_expensive_session: no_signal (no session in scope carries a positive cost figure) repos_touched: (none) subagent_branch_count: 0 diff --git a/tests/unit/cli/test_archive_maintenance_cli.py b/tests/unit/cli/test_archive_maintenance_cli.py index ccf77aa58a..d324db671e 100644 --- a/tests/unit/cli/test_archive_maintenance_cli.py +++ b/tests/unit/cli/test_archive_maintenance_cli.py @@ -1926,9 +1926,9 @@ def test_rebuild_index_full_source_resumes_one_candidate_until_terminal_promotio archive.write_raw_payload( provider=Provider.CODEX, payload=( - f'{{"type":"session_meta","payload":{{"id":"{native_id}"}}}}\\n' + f'{{"type":"session_meta","payload":{{"id":"{native_id}"}}}}\n' f'{{"type":"response_item","payload":{{"type":"message","role":"user",' - f'"content":[{{"type":"input_text","text":"{native_id}"}}]}}}}\\n' + f'"content":[{{"type":"input_text","text":"{native_id}"}}]}}}}\n' ).encode(), source_path=f"{native_id}.jsonl", acquired_at_ms=acquired_at_ms, @@ -2004,9 +2004,9 @@ def test_rebuild_index_persists_durable_pass_receipt_alongside_transaction( archive.write_raw_payload( provider=Provider.CODEX, payload=( - f'{{"type":"session_meta","payload":{{"id":"{native_id}"}}}}\\n' + f'{{"type":"session_meta","payload":{{"id":"{native_id}"}}}}\n' f'{{"type":"response_item","payload":{{"type":"message","role":"user",' - f'"content":[{{"type":"input_text","text":"{native_id}"}}]}}}}\\n' + f'"content":[{{"type":"input_text","text":"{native_id}"}}]}}}}\n' ).encode(), source_path=f"{native_id}.jsonl", acquired_at_ms=acquired_at_ms, @@ -2075,9 +2075,9 @@ def test_rebuild_index_byte_budget_defers_then_reaches_terminal_ready_candidate( archive.write_raw_payload( provider=Provider.CODEX, payload=( - f'{{"type":"session_meta","payload":{{"id":"{native_id}"}}}}\\n' + f'{{"type":"session_meta","payload":{{"id":"{native_id}"}}}}\n' f'{{"type":"response_item","payload":{{"type":"message","role":"user",' - f'"content":[{{"type":"input_text","text":"{padding}"}}]}}}}\\n' + f'"content":[{{"type":"input_text","text":"{padding}"}}]}}}}\n' ).encode(), source_path=f"{native_id}.jsonl", acquired_at_ms=acquired_at_ms, diff --git a/tests/unit/cli/test_command_aux_runtime.py b/tests/unit/cli/test_command_aux_runtime.py index 9bd19d94da..c64e0913ea 100644 --- a/tests/unit/cli/test_command_aux_runtime.py +++ b/tests/unit/cli/test_command_aux_runtime.py @@ -45,7 +45,11 @@ def test_completion_functions_cover_origin_session_tag_tool_and_open_targets() - ctx, param = _ctx_param() origin_items = shell_completion_values.complete_origin_values(ctx, param, "chatgpt,cla") - assert [item.value for item in origin_items] == ["chatgpt,claude-ai-export", "chatgpt,claude-code-session"] + assert [item.value for item in origin_items] == [ + "chatgpt,claude-ai-export", + "chatgpt,claude-code-session", + "chatgpt,claude-design-session", + ] action_items = shell_completion_values.complete_action_values(ctx, param, "file") action_sequence_items = shell_completion_values.complete_action_sequence_values(ctx, param, "shell,file") material_origin_items = shell_completion_values.complete_material_origin_values(ctx, param, "runtime") @@ -260,7 +264,11 @@ def test_query_expression_value_completion_uses_field_completion_source() -> Non ctx, param = _ctx_param() origin_items = shell_completion_values.complete_query_expression_fields(ctx, param, "origin:cla") - assert [item.value for item in origin_items] == ["origin:claude-ai-export", "origin:claude-code-session"] + assert [item.value for item in origin_items] == [ + "origin:claude-ai-export", + "origin:claude-code-session", + "origin:claude-design-session", + ] with ( patch("polylogue.cli.shell_completion_values._db_exists", return_value=True), diff --git a/tests/unit/core/test_enums.py b/tests/unit/core/test_enums.py index 6fec463e2b..1f668592ca 100644 --- a/tests/unit/core/test_enums.py +++ b/tests/unit/core/test_enums.py @@ -38,6 +38,7 @@ def test_origin_values_match_archive_issue_contract() -> None: "grok-export", "chatgpt-export", "claude-ai-export", + "claude-design-session", "aistudio-drive", "unknown-export", ) diff --git a/tests/unit/core/test_paths.py b/tests/unit/core/test_paths.py index 664a61b1c2..b63f7f5024 100644 --- a/tests/unit/core/test_paths.py +++ b/tests/unit/core/test_paths.py @@ -257,6 +257,7 @@ def test_paths_root_exports_only_directory_layout_symbols(self) -> None: assert set(paths.__all__) == { "GEMINI_DRIVE_FOLDER", "antigravity_path", + "api_auth_token_path", "archive_root", "blob_store_root", "browser_capture_pairing_state_path", @@ -266,6 +267,7 @@ def test_paths_root_exports_only_directory_layout_symbols(self) -> None: "cache_home", "cache_root", "claude_code_path", + "claude_code_todos_path", "codex_path", "config_home", "config_root", diff --git a/tests/unit/core/test_query_fields.py b/tests/unit/core/test_query_fields.py index cdaac0806f..e08d2773fc 100644 --- a/tests/unit/core/test_query_fields.py +++ b/tests/unit/core/test_query_fields.py @@ -142,11 +142,16 @@ def test_query_field_catalog_drives_plan_presence_descriptions_and_pushdown() -> def test_query_field_catalog_covers_public_spec_fields() -> None: descriptor_spec_attrs = {descriptor.spec_attr for descriptor in QUERY_FIELD_DESCRIPTORS if descriptor.spec_attr} spec_fields = {field.name for field in fields(SessionQuerySpec)} - internal_spec_fields = {"boolean_predicate", "with_unit_fields"} + internal_spec_fields = {"boolean_predicate", "with_unit_fields", "with_unit_windows"} # All public spec fields have descriptors (cursor was promoted from # internal-only to exposed). ``boolean_predicate`` is the compiled AST - # carrier, not a user-authored field token. + # carrier, not a user-authored field token. ``with_unit_fields``/ + # ``with_unit_windows`` are projection sub-details parsed out of the + # ``with [bracket]`` clause by ``_split_with_projection_clause`` + # -- ``with_units`` itself is the one user-authored field token and has + # its own descriptor; there is no separate DSL token for the field/window + # sub-selection, so neither carries its own descriptor. assert spec_fields - descriptor_spec_attrs - internal_spec_fields == set() diff --git a/tests/unit/daemon/test_daemon_http_security.py b/tests/unit/daemon/test_daemon_http_security.py index 293e793d08..67d2b9b7a7 100644 --- a/tests/unit/daemon/test_daemon_http_security.py +++ b/tests/unit/daemon/test_daemon_http_security.py @@ -1036,6 +1036,11 @@ def test_no_token_in_log_or_print_calls(self) -> None: } intentional_secret_outputs = { (Path("polylogue/daemon/browser_capture.py"), "token_show", "echo"), + # `polylogue ops api token show` (#3488/#3549): the deliberate CLI + # surface for printing the daemon HTTP API's auto-minted bearer + # token to its own operator, same intentional pattern as the + # browser-capture pairing token_show above. + (Path("polylogue/daemon/api_auth.py"), "token_show", "echo"), # `_handle_query_units` reads a client-supplied pagination # `continuation` param and decodes it into `continuation_token`/ # `continuation_request` (an opaque query-unit cursor, not a diff --git a/tests/unit/devtools/test_lineage_validation.py b/tests/unit/devtools/test_lineage_validation.py index 464fddee38..7880023694 100644 --- a/tests/unit/devtools/test_lineage_validation.py +++ b/tests/unit/devtools/test_lineage_validation.py @@ -37,7 +37,8 @@ def _make_index_db(root: Path, *, with_gap: bool = False) -> Path: git_branch TEXT, git_repository_url TEXT, provider_project_ref TEXT, - message_count INTEGER DEFAULT 0 + message_count INTEGER DEFAULT 0, + reported_cost_usd REAL ); CREATE TABLE session_profiles ( session_id TEXT PRIMARY KEY, @@ -90,7 +91,8 @@ def _make_index_db(root: Path, *, with_gap: bool = False) -> Path: occurred_at_ms INTEGER, paste_boundary TEXT, duration_ms INTEGER, - parent_message_id TEXT + parent_message_id TEXT, + stop_reason TEXT ); CREATE TABLE blocks ( block_id TEXT, @@ -104,6 +106,7 @@ def _make_index_db(root: Path, *, with_gap: bool = False) -> Path: language TEXT, tool_result_is_error INTEGER, tool_result_exit_code INTEGER, + tool_result_outcome_unknown_reason TEXT, position INTEGER ); INSERT INTO sessions(session_id, native_id, origin, title, root_session_id, branch_type, message_count) diff --git a/tests/unit/devtools/test_render_quality_reference.py b/tests/unit/devtools/test_render_quality_reference.py index c1ba25f665..7dad7758f9 100644 --- a/tests/unit/devtools/test_render_quality_reference.py +++ b/tests/unit/devtools/test_render_quality_reference.py @@ -170,7 +170,7 @@ def test_build_document_includes_live_registry_sections() -> None: assert "| `validation-lane` | `runtime-substrate-hardening` |" in rendered assert "| `inferred-corpus-scenario` | `chatgpt:v1` |" in rendered assert "`session-insight-repair-loop`" in rendered - assert "new coverage information should land in the closure matrix first" in rendered + assert "When adding a new domain, parser, or surface, add or update its row in the matrix" in rendered assert "legacy free-text catalog" not in rendered diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 06f04821b4..ff43dc9c5f 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -98,6 +98,9 @@ def test_quick_verify_omits_pytest() -> None: "lab policy schema-versioning", "lab policy classifier-fingerprints", "lab policy demo-tour-freshness", + "lab policy raw-payload-hash-purity", + "lab policy position-derived-identity", + "lab policy raw-authority-frontier-executability", "schema promotion audit", ] diff --git a/tests/unit/maintenance/test_planner_contract.py b/tests/unit/maintenance/test_planner_contract.py index fab86584df..863d466a43 100644 --- a/tests/unit/maintenance/test_planner_contract.py +++ b/tests/unit/maintenance/test_planner_contract.py @@ -347,7 +347,13 @@ def test_preview_reads_the_callers_seeded_archive(self, workspace_env: dict[str, """Planner debt comes from the supplied archive, not ambient paths.""" caller_workspace = _caller_archive_workspace(workspace_env) index_db = db_setup(caller_workspace) - DbFactory(index_db).create_session(id="planner-config") + factory = DbFactory(index_db) + factory.create_session(id="planner-config") + # A message-less session with no raw artifact is "no evidence either + # way" to the empty_sessions debt classifier and never counts as + # debt (polylogue-9rdky) -- give it a phantom raw the classifier + # positively refuses so this fixture produces real debt. + factory.mark_as_phantom_debris("planner-config") config = Config( archive_root=caller_workspace["archive_root"], render_root=workspace_env["data_root"] / "render", @@ -364,7 +370,10 @@ def test_preview_reads_the_callers_seeded_archive(self, workspace_env: dict[str, def test_execute_dry_run_reads_the_callers_seeded_archive(self, workspace_env: dict[str, Path]) -> None: caller_workspace = _caller_archive_workspace(workspace_env) index_db = db_setup(caller_workspace) - DbFactory(index_db).create_session(id="planner-execute") + factory = DbFactory(index_db) + factory.create_session(id="planner-execute") + # See test_preview_reads_the_callers_seeded_archive above. + factory.mark_as_phantom_debris("planner-execute") config = Config( archive_root=caller_workspace["archive_root"], render_root=workspace_env["data_root"] / "render", diff --git a/tests/unit/maintenance/test_planner_filter_narrowing.py b/tests/unit/maintenance/test_planner_filter_narrowing.py index 2596eb508a..11a7994299 100644 --- a/tests/unit/maintenance/test_planner_filter_narrowing.py +++ b/tests/unit/maintenance/test_planner_filter_narrowing.py @@ -33,7 +33,15 @@ def _seeded_config(workspace_env: dict[str, Path], *, sessions: int = 3) -> Conf index_db = db_setup(workspace_env) factory = DbFactory(index_db) for index in range(sessions): - factory.create_session(id=f"empty-{index}") + native_id = f"empty-{index}" + factory.create_session(id=native_id) + # A message-less session with no raw artifact at all (raw_id IS + # NULL) is "no evidence either way" to the empty_sessions debt + # classifier and is never counted as debt -- give each one a + # phantom raw artifact the classifier positively refuses, so this + # fixture actually produces real archive debt for the tests below + # to narrow (polylogue-9rdky). + factory.mark_as_phantom_debris(native_id) return Config( archive_root=workspace_env["archive_root"], render_root=workspace_env["data_root"] / "render", diff --git a/tests/unit/pipeline/test_branching.py b/tests/unit/pipeline/test_branching.py index 37824405ea..e3e2b08b66 100644 --- a/tests/unit/pipeline/test_branching.py +++ b/tests/unit/pipeline/test_branching.py @@ -58,9 +58,31 @@ def _archive_session_id(provider: Provider, native_id: str) -> str: def _codex_continuation_payload(*, child_id: str, parent_id: str) -> list[dict[str, object]]: + # #3484 requires structural evidence for the legacy (no `forked_from_id`) + # continuation fallback: the second distinct session_meta only counts as + # the replayed parent header when its timestamp does not postdate the + # first's AND the two share a matching cwd or git repository_url -- a + # bare second session_meta with an unrelated id proves nothing on its + # own (polylogue.sources.parsers.codex._has_continuation_evidence). return [ - {"type": "session_meta", "payload": {"id": child_id, "timestamp": "2025-01-02T10:00:00Z"}}, - {"type": "session_meta", "payload": {"id": parent_id, "timestamp": "2025-01-01T10:00:00Z"}}, + { + "type": "session_meta", + "payload": { + "id": child_id, + "timestamp": "2025-01-02T10:00:00Z", + "cwd": "/realm/project/continuation-fixture", + "git": {"repository_url": "git@github.com:example/continuation-fixture.git"}, + }, + }, + { + "type": "session_meta", + "payload": { + "id": parent_id, + "timestamp": "2025-01-01T10:00:00Z", + "cwd": "/realm/project/continuation-fixture", + "git": {"repository_url": "git@github.com:example/continuation-fixture.git"}, + }, + }, { "type": "response_item", "payload": { @@ -397,8 +419,15 @@ async def test_tree_and_filter_contract(self, workspace_env: WorkspaceEnv) -> No tree = await archive.get_session_tree(ids["grandchild"]) children = [conv for conv in tree if conv.parent_id is not None and str(conv.parent_id) == ids["root"]] root = next(conv for conv in tree if conv.is_root) - continuations = await archive.filter().is_continuation().list() - sidechains = await archive.filter().is_sidechain().list() + # #3495 defaults every session query's SQL-level `root` filter to + # True (top-level only) unless the caller explicitly overrides + # it. A continuation/sidechain session is a child by + # definition (it always has a parent), so it can never be + # "root" -- without this override the implicit root-only + # default silently empties both result sets before + # is_continuation()/is_sidechain() even run. + continuations = await archive.filter().is_continuation().is_root(False).list() + sidechains = await archive.filter().is_sidechain().is_root(False).list() roots = await archive.filter().is_root().list() with_branches = await archive.filter().has_branches().list() diff --git a/tests/unit/sources/test_codex_event_stream_contract.py b/tests/unit/sources/test_codex_event_stream_contract.py index 8e5f0c5d7c..fd750d0d7a 100644 --- a/tests/unit/sources/test_codex_event_stream_contract.py +++ b/tests/unit/sources/test_codex_event_stream_contract.py @@ -136,7 +136,14 @@ def test_unknown_inner_payload_type_still_recorded_as_session_event(self) -> Non This is the canary for OpenAI introducing a new event class — the parser must still surface it as a session event so downstream - consumers can see something is happening. + consumers can see something is happening. #3567 added an explicit + allowlist-vs-passthrough classification at the generic session_event + dispatch: a type outside the known set (``_CODEX_KNOWN_RESPONSE_ITEM_TYPES`` + in ``sources/parsers/codex.py``) now routes to a distinct + ``codex_unclassified_response_item`` bucket instead of silently + adopting its own wire name, matching the ``claude_attachment_unclassified`` + precedent -- the event is still surfaced, just under the shared + "something unrecognized happened" name rather than the raw token. """ records: list[object] = [ { @@ -145,7 +152,7 @@ def test_unknown_inner_payload_type_still_recorded_as_session_event(self) -> Non } ] session = _parse(records) - assert [event.event_type for event in session.session_events] == ["future_event_kind"] + assert [event.event_type for event in session.session_events] == ["codex_unclassified_response_item"] # --------------------------------------------------------------------------- diff --git a/tests/unit/sources/test_dispatch_payloads.py b/tests/unit/sources/test_dispatch_payloads.py index 14306e2efd..c609fecad0 100644 --- a/tests/unit/sources/test_dispatch_payloads.py +++ b/tests/unit/sources/test_dispatch_payloads.py @@ -717,8 +717,19 @@ def test_parse_stream_payload_codex_long_rollout_with_repeated_session_meta_yiel assert len(sessions) == 1 total_messages = len(sessions[0].messages) assert total_messages > 0, "long multi-session_meta Codex rollout must not parse to zero messages" - # 20 turns * (user + assistant + function_call use/output + custom_tool_call use/output) - assert total_messages == 20 * 6 + # 20 turns * (user + assistant + function_call use/output + custom_tool_call + # use/output + reasoning). #3447 (polylogue-vf9x) started materializing + # every Codex `reasoning` response_item as its own THINKING-block message + # (previously read only by the generic session_event compactor, which + # never surfaced it as content) -- even one with an empty `summary` and + # no `content` still produces a message (block text=None) so the fact + # that the model reasoned here survives. + assert total_messages == 20 * 7 + thinking_blocks = [ + block for message in sessions[0].messages for block in message.blocks if block.type is BlockType.THINKING + ] + assert len(thinking_blocks) == 20 + assert all(block.text is None for block in thinking_blocks) def test_require_positive_conversational_evidence_refuses_claude_code_stream_with_no_conversational_records( diff --git a/tests/unit/sources/test_live_catchup_planning.py b/tests/unit/sources/test_live_catchup_planning.py index f135c9ba46..68c37b3891 100644 --- a/tests/unit/sources/test_live_catchup_planning.py +++ b/tests/unit/sources/test_live_catchup_planning.py @@ -294,8 +294,14 @@ def test_codex_append_plan_recovers_identity_from_session_meta_when_source_row_m assert isinstance(plan, _AppendPlan) assert plan.start_offset == old_offset - assert b'"type":"session_meta","payload":{"id":"conv-hot"}' in plan.payload - assert b'"content":"new"' in plan.payload + # #3539 (polylogue-u19l) retired splicing a synthetic session_meta header + # into a Codex append payload before hashing/storing it -- the stored + # blob must stay a literal byte-slice of the live file so live-source + # byte-identity re-verification stays possible. Recovered identity now + # flows as a sidecar hint (native_id_hint) instead, applied as the + # parser's fallback_id at replay time. + assert plan.native_id_hint == "conv-hot" + assert plan.payload == source.read_bytes()[old_offset:] def test_catch_up_ingests_needed_files_in_bounded_chunks( diff --git a/tests/unit/sources/test_live_watcher.py b/tests/unit/sources/test_live_watcher.py index 2def6046a6..9b114f6a3e 100644 --- a/tests/unit/sources/test_live_watcher.py +++ b/tests/unit/sources/test_live_watcher.py @@ -1221,7 +1221,12 @@ async def test_live_batch_processor_records_durable_attempt(tmp_path: Path) -> N root = tmp_path / "sessions" root.mkdir() source_path = root / "session.jsonl" - source_path.write_text('{"type":"session_meta","payload":{"id":"s"}}\n', encoding="utf-8") + source_path.write_text( + '{"type":"session_meta","payload":{"id":"s"}}\n' + '{"type":"response_item","payload":{"type":"message","role":"user",' + '"content":[{"type":"input_text","text":"hello"}]}}\n', + encoding="utf-8", + ) db_path = tmp_path / "live.sqlite" cursor = CursorStore(db_path) polylogue = SimpleNamespace(archive_root=tmp_path, backend=None) diff --git a/tests/unit/storage/test_blob_gc.py b/tests/unit/storage/test_blob_gc.py index 1d77217e47..00fdbf5640 100644 --- a/tests/unit/storage/test_blob_gc.py +++ b/tests/unit/storage/test_blob_gc.py @@ -263,6 +263,9 @@ def test_run_blob_gc_preserves_referenced_blobs(tmp_path: Path) -> None: assert blob_store.exists(h) +@pytest.mark.uses_real_clock( + "backdates a real blob mtime via os.utime; blob_gc.py's age gate compares it against a real time.time() call in production code, so frozen_clock cannot intercept either side" +) def test_run_blob_gc_preserves_archive_source_referenced_blobs(tmp_path: Path) -> None: """GC run from ``index.db`` must preserve blobs referenced by sibling ``source.db``.""" index_db_path = tmp_path / "index.db" @@ -313,6 +316,9 @@ def test_run_blob_gc_max_batch_bound(tmp_path: Path) -> None: assert 0 <= deleted <= 2 +@pytest.mark.uses_real_clock( + "backdates a real blob mtime via os.utime; blob_gc.py's age gate compares it against a real time.time() call in production code, so frozen_clock cannot intercept either side" +) def test_run_blob_gc_bounds_final_lock_rechecks_with_many_references( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -378,6 +384,9 @@ def _backdate(blob_store: BlobStore, blob_hash: str, *, seconds: float = 3600) - os.utime(path, (past, past)) +@pytest.mark.uses_real_clock( + "backdates a real blob mtime via os.utime; blob_gc.py's age gate compares it against a real time.time() call in production code, so frozen_clock cannot intercept either side" +) def test_run_blob_gc_unlinks_sharded_path_and_increments_counter(tmp_path: Path) -> None: """#1190 regression: an orphan blob present at the sharded path ``{root}/{prefix}/{remainder}`` must actually be removed and the @@ -408,6 +417,9 @@ def test_run_blob_gc_unlinks_sharded_path_and_increments_counter(tmp_path: Path) assert not sharded.exists(), "sharded blob must actually be removed from disk" +@pytest.mark.uses_real_clock( + "backdates a real blob mtime via os.utime; blob_gc.py's age gate compares it against a real time.time() call in production code, so frozen_clock cannot intercept either side" +) def test_run_blob_gc_does_not_increment_when_file_already_missing(tmp_path: Path) -> None: """#1190 regression: when the candidate file has vanished between discovery and unlink (concurrent reclaimer, stale candidate, manual @@ -461,6 +473,9 @@ def patched(root: Path, *, older_than: float) -> list[tuple[str, float]]: assert history[0].reclaimed_bytes == 0 +@pytest.mark.uses_real_clock( + "backdates a real blob mtime via os.utime; blob_gc.py's age gate compares it against a real time.time() call in production code, so frozen_clock cannot intercept either side" +) def test_run_blob_gc_dry_run_does_not_delete_or_record_generation(tmp_path: Path) -> None: """#1190 ambitious-expansion: --dry-run previews without committing. @@ -490,6 +505,9 @@ def test_run_blob_gc_dry_run_does_not_delete_or_record_generation(tmp_path: Path assert row[0] == 0, "dry-run must not consume a generation slot" +@pytest.mark.uses_real_clock( + "backdates a real blob mtime via os.utime; blob_gc.py's age gate compares it against a real time.time() call in production code, so frozen_clock cannot intercept either side" +) def test_run_blob_gc_records_reclaim_counters(tmp_path: Path) -> None: """#1743: each committed pass writes a typed ``gc_generations`` row capturing the reclaimed blob count and freed bytes — the durable @@ -527,6 +545,9 @@ def test_run_blob_gc_records_reclaim_counters(tmp_path: Path) -> None: assert row.started_at_ms <= row.completed_at_ms +@pytest.mark.uses_real_clock( + "backdates a real blob mtime via os.utime; blob_gc.py's age gate compares it against a real time.time() call in production code, so frozen_clock cannot intercept either side" +) def test_read_gc_history_returns_recent_passes(tmp_path: Path) -> None: """#1743: ``read_gc_history`` surfaces one typed row per committed pass, so a ``gc-history`` operator surface can show recent reclamation without diff --git a/tests/unit/storage/test_bulk_delete_guarded.py b/tests/unit/storage/test_bulk_delete_guarded.py index f02b424cca..8da8237f71 100644 --- a/tests/unit/storage/test_bulk_delete_guarded.py +++ b/tests/unit/storage/test_bulk_delete_guarded.py @@ -224,7 +224,7 @@ def test_delete_sessions_bulk_leaves_fts_trigram_and_action_pairs_coherent(tmp_p """ INSERT INTO delegation_facts ( delegation_id, parent_session_id, mapping_state, result_status, parent_origin - ) VALUES (?, ?, 'mapped', 'ok', 'codex-session') + ) VALUES (?, ?, 'resolved', 'ok', 'codex-session') """, (f"deleg-{session_ids[0]}", session_ids[0]), ) diff --git a/tests/unit/storage/test_hermes_artifact_inspection.py b/tests/unit/storage/test_hermes_artifact_inspection.py index 829a2db610..240759cd8c 100644 --- a/tests/unit/storage/test_hermes_artifact_inspection.py +++ b/tests/unit/storage/test_hermes_artifact_inspection.py @@ -166,7 +166,12 @@ def test_generic_sqlite_lookalike_is_not_claimed_as_hermes(blob_store: BlobStore observation = inspect_raw_artifact(_record(blob_store, snapshot, raw_id="hermes:profile-a:lookalike")) - assert observation.support_status is ArtifactSupportStatus.DECODE_FAILED - assert observation.artifact_kind == "unknown" + # #3576 (polylogue-hbtj2) added a shared magic-byte chokepoint that + # positively refuses a recognized-but-unclaimed binary payload before any + # JSON/JSONL decode is attempted -- a generic SQLite lookalike is now + # classified as a recognized (if unparsed) binary database, not merely an + # incidental decode failure the old code fell through to. + assert observation.support_status is ArtifactSupportStatus.RECOGNIZED_UNPARSED + assert observation.artifact_kind == "binary_database" assert observation.resolved_package_version is None assert observation.classification_reason != "Hermes state.db SQLite archive marker"