diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4084db3..71ed905 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -153,7 +153,8 @@ jobs: browser-xss: name: Browser XSS (Playwright) needs: [lockfile] - runs-on: ubuntu-latest + # ubuntu-latest is 26.04. Playwright 1.50.0 refuses Chromium there. + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: diff --git a/CHANGELOG.md b/CHANGELOG.md index 64b0799..4c62726 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed +- **Cursor 3 conversation index**: export, search, and workspace tabs read + `composerConvHeaders` when `fullConversationHeadersOnly` is empty, and keep + bubble ids that contain colons (`{uuid}/s:0`). + ## [0.2.1] - 2026-07-30 ### Added diff --git a/pyproject.toml b/pyproject.toml index 70563ba..5559016 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,9 @@ requires-python = ">=3.10" dependencies = [ "flask>=3.0,<4", "fpdf2>=2.7,<3", + # fonttools 4.66+ requires Python >=3.11. The single lock must still + # install on the 3.10 matrix jobs. + "fonttools>=4.34,<4.66", # Security floor: fpdf2 allows Pillow>=8.3.2 (no upper cap); pin 12.x to avoid # known high-severity CVEs in Pillow 10.x (e.g. CVE-2024-28219 and later advisories). "pillow>=12.2.0,<13", diff --git a/requirements-lock.txt b/requirements-lock.txt index 74f5c5a..86f7bb1 100644 --- a/requirements-lock.txt +++ b/requirements-lock.txt @@ -4,13 +4,13 @@ # Run periodically (e.g. via the "Update dependency lock file" CI workflow) to pick up # upstream patch / security releases within the bounded ranges in requirements.txt. blinker==1.9.0 # via flask -click==8.4.2 # via flask +click==8.5.0 # via flask defusedxml==0.7.1 # via fpdf2 flask==3.1.3 # via -r requirements.txt -fonttools==4.63.0 # via fpdf2 -fpdf2==2.8.8 # via -r requirements.txt +fonttools==4.65.0 # via -r requirements.txt, fpdf2 +fpdf2==2.8.9 # via -r requirements.txt itsdangerous==2.2.0 # via flask jinja2==3.1.6 # via flask -markupsafe==3.0.3 # via flask, jinja2, werkzeug +markupsafe==3.0.4 # via flask, jinja2, werkzeug pillow==12.3.0 # via -r requirements.txt, fpdf2 -werkzeug==3.1.8 # via flask +werkzeug==3.1.9 # via flask diff --git a/requirements.txt b/requirements.txt index f08a70a..e56d9d2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,5 +6,8 @@ # pip install -e ".[desktop]" (+ pywebview for the GUI launcher) flask>=3.0,<4 fpdf2>=2.7,<3 +# fonttools 4.66+ requires Python >=3.11. The single lock must still +# install on the 3.10 matrix jobs. +fonttools>=4.34,<4.66 pillow>=12.2.0,<13 # pywebview is desktop-only — install with: pip install -e ".[desktop]" diff --git a/services/export_engine.py b/services/export_engine.py index b563518..988a475 100644 --- a/services/export_engine.py +++ b/services/export_engine.py @@ -6,7 +6,7 @@ import logging import os import sqlite3 -from dataclasses import dataclass +from dataclasses import dataclass, field from datetime import datetime from typing import Any, Literal @@ -35,6 +35,7 @@ messages_to_bubbles, traverse_blobs, ) +from utils.conversation_index import composer_data_with_headers, load_conv_header_map from utils.cursor_md_exporter import ( cursor_cli_session_to_markdown, cursor_ide_chat_to_markdown, @@ -97,6 +98,7 @@ class GlobalDbExportData: code_block_diff_map: dict[str, list[Any]] ide_composer_rows: list[sqlite3.Row] invalid_workspace_aliases: dict[str, str] + conv_header_map: dict[str, list[dict[str, Any]]] = field(default_factory=dict) def json_dump_safe(value: object) -> str: @@ -186,6 +188,7 @@ def load_global_db_export_data( project_layouts_map = enriched.project_layouts_map bubble_map = enriched.bubble_map code_block_diff_map = load_code_block_diff_map(global_db) + conv_header_map = load_conv_header_map(global_db) ide_composer_rows = safe_fetchall(global_db, COMPOSER_ROWS_WITH_HEADERS_SQL) invalid_workspace_aliases = resolve_invalid_workspace_aliases_cached( @@ -203,6 +206,7 @@ def load_global_db_export_data( code_block_diff_map=code_block_diff_map, ide_composer_rows=ide_composer_rows, invalid_workspace_aliases=invalid_workspace_aliases, + conv_header_map=conv_header_map, ) @@ -245,6 +249,7 @@ def _collect_ide_export_entries( ) continue + cd = composer_data_with_headers(cd, composer_id, db_data.conv_header_map) headers = cd.get("fullConversationHeadersOnly") or [] if not isinstance(headers, list) or not headers: continue diff --git a/services/search.py b/services/search.py index c0471f6..3e18506 100644 --- a/services/search.py +++ b/services/search.py @@ -51,6 +51,7 @@ ) from services.workspace_resolver import infer_invalid_workspace_aliases from utils.cli_chat_reader import list_cli_projects, messages_to_bubbles, traverse_blobs +from utils.conversation_index import composer_data_with_headers, load_conv_header_map from utils.exclusion_rules import RuleTokens, build_searchable_text, is_excluded_by_rules from utils.text_extract import extract_text_from_bubble from utils.path_helpers import ( @@ -506,6 +507,11 @@ def _search_global_storage_via_index( workspace_path, workspace_entries, rules, ) + with open_global_db(workspace_path) as (header_db, _): + conv_header_map = ( + load_conv_header_map(header_db) if header_db is not None else {} + ) + composers_in_window = query_composer_rows_in_window(since_ms) if since_ms is not None and not composers_in_window: return results @@ -551,8 +557,9 @@ def _search_global_storage_via_index( continue try: + cd = composer_data_with_headers(cd, composer_id, conv_header_map) headers = cd.get("fullConversationHeadersOnly") or [] - if not headers: + if not isinstance(headers, list) or not headers: continue title = composer_row["title"] or cd.get("name") or "" @@ -677,6 +684,7 @@ def _search_global_storage_live_scan( return results composer_rows = conn.execute(COMPOSER_ROWS_WITH_HEADERS_SQL).fetchall() + conv_header_map = load_conv_header_map(conn) window_composer_ids: set[str] | None = None if since_ms is not None: @@ -726,8 +734,9 @@ def _search_global_storage_live_scan( continue try: + cd = composer_data_with_headers(cd, composer_id, conv_header_map) headers = cd.get("fullConversationHeadersOnly") or [] - if not headers: + if not isinstance(headers, list) or not headers: continue title = cd.get("name") or "" diff --git a/services/search_index.py b/services/search_index.py index bcbfee2..c404430 100644 --- a/services/search_index.py +++ b/services/search_index.py @@ -57,7 +57,10 @@ _logger = logging.getLogger(__name__) -INDEX_VERSION = 1 +# Bumped when the indexed composer set changes. Version 2 includes composers +# whose inline fullConversationHeadersOnly list is empty (Cursor 3.x stores +# the index in composerConvHeaders instead). +INDEX_VERSION = 2 SEARCH_INDEX_POINTER_FILE = CACHE_DIR / "search_index.active" # Legacy single-file path (pre-pointer builds); still honored when no pointer exists. SEARCH_INDEX_FILE = CACHE_DIR / "search_index.sqlite" @@ -224,7 +227,10 @@ def build_search_index( *, force: bool = False, ) -> bool: - """Rebuild search index when fingerprint differs. Returns True if rebuilt.""" + """Rebuild search index when the fingerprint or stored version differs. + + Returns True if rebuilt. + """ if not index_search_enabled(): return False @@ -237,10 +243,8 @@ def build_search_index( active_path = _resolve_active_index_db_path() if not force and active_path is not None: with _index_db_conn(readonly=True) as existing: - if existing is not None: - stored = _read_stored_fingerprint(existing) - if stored is not None and _fingerprints_match(stored, fingerprint): - return False + if existing is not None and not _index_needs_rebuild(existing, fingerprint): + return False CACHE_DIR.mkdir(parents=True, exist_ok=True) new_path = CACHE_DIR / f"search_index.{uuid.uuid4().hex[:12]}.sqlite" @@ -362,8 +366,7 @@ def ensure_search_index(workspace_path: str, rules: list[RuleTokens]) -> None: if conn is None: build_search_index(workspace_path, rules) return - stored = _read_stored_fingerprint(conn) - if stored is None or not _fingerprints_match(stored, fingerprint): + if _index_needs_rebuild(conn, fingerprint): build_search_index(workspace_path, rules) @@ -541,13 +544,35 @@ def query_composer_rows_in_window( return {row["composer_id"]: row for row in rows} +def _index_needs_rebuild( + conn: sqlite3.Connection, + fingerprint: dict[str, Any], +) -> bool: + """True when the stored fingerprint or index version is not current.""" + stored = _read_stored_fingerprint(conn) + if stored is None or not _fingerprints_match(stored, fingerprint): + return True + return _stored_index_version(conn) != str(INDEX_VERSION) + + +def _stored_index_version(conn: sqlite3.Connection) -> str | None: + try: + row = conn.execute( + "SELECT value FROM index_meta WHERE key = 'version'" + ).fetchone() + except sqlite3.Error: + return None + if not row or row[0] is None: + return None + return str(row[0]) + + def index_is_usable(workspace_path: str, rules: list[RuleTokens]) -> bool: - """True when the on-disk index matches the current Cursor storage fingerprint.""" + """True when the on-disk index matches the storage fingerprint and INDEX_VERSION.""" if not index_search_enabled() or _resolve_active_index_db_path() is None: return False fingerprint = _storage_fingerprint(workspace_path, rules) with _index_db_conn(readonly=True) as conn: if conn is None: return False - stored = _read_stored_fingerprint(conn) - return stored is not None and _fingerprints_match(stored, fingerprint) + return not _index_needs_rebuild(conn, fingerprint) diff --git a/services/workspace_composer_scan.py b/services/workspace_composer_scan.py index 74da715..ff8d437 100644 --- a/services/workspace_composer_scan.py +++ b/services/workspace_composer_scan.py @@ -12,6 +12,7 @@ from typing import Any from models import Bubble, Composer, ParseWarningCollector, SchemaError +from utils.conversation_index import composer_data_with_headers from utils.exclusion_rules import RuleTokens, build_searchable_text, is_excluded_by_rules from services.workspace_resolver import determine_project_for_conversation @@ -23,11 +24,15 @@ def parse_composer_data_row( raw_value: object | None, *, parse_warnings: ParseWarningCollector, + conv_header_map: Mapping[str, list[dict[str, Any]]] | None = None, ) -> Composer | None: """Parse a ``composerData:*`` KV row into a :class:`Composer`. Returns ``None`` for null/placeholder payloads (e.g. ``empty-state-draft``) without emitting decode warnings. Logs and records skips for malformed JSON. + When ``fullConversationHeadersOnly`` is empty, headers are taken from + *conv_header_map* (``composerConvHeaders`` chunks). A composer with no + resolved headers is omitted. """ if not row_key.startswith("composerData:"): return None @@ -53,6 +58,7 @@ def parse_composer_data_row( if not isinstance(cd, dict): parse_warnings.record_composer_skipped() return None + cd = composer_data_with_headers(cd, composer_id, conv_header_map or {}) try: composer = Composer.from_dict(cd, composer_id=composer_id) except SchemaError as e: diff --git a/services/workspace_db.py b/services/workspace_db.py index 1ccd386..bc00c82 100644 --- a/services/workspace_db.py +++ b/services/workspace_db.py @@ -15,6 +15,7 @@ _logger = logging.getLogger(__name__) from models import Bubble, ParseWarningCollector, SchemaError +from utils.conversation_index import bubble_id_from_kv_key from utils.path_helpers import get_workspace_folder_paths from utils.workspace_descriptor import read_json_file @@ -44,11 +45,14 @@ def _parse_bubble_kv_row( *, parse_warnings: ParseWarningCollector | None = None, ) -> tuple[str, Bubble] | None: - """Parse one ``bubbleId:…`` row; return ``(bubble_id, Bubble)`` or skip.""" - parts = row_key.split(":") - if len(parts) < 3: + """Parse one ``bubbleId:…`` row; return ``(bubble_id, Bubble)`` or skip. + + Bubble ids may contain colons (``{uuid}/s:0``). The id is the key tail + after ``bubbleId:{composerId}:``. + """ + bid = bubble_id_from_kv_key(row_key) + if not bid: return None - bid = parts[2] try: parsed = json.loads(row_value) bubble = Bubble.from_dict(parsed, bubble_id=bid) @@ -344,13 +348,14 @@ def collect_invalid_workspace_ids(workspace_entries: list[dict[str, Any]]) -> se return invalid -# Composers that have at least one header entry (list/summary paths). +# Composers that store a conversation index field. Cursor 3.x leaves +# fullConversationHeadersOnly as [] and writes the ordered headers to +# composerConvHeaders:{id}:{chunk}. Callers resolve that fallback and skip +# composers whose resolved header list is still empty (drafts). COMPOSER_ROWS_WITH_HEADERS_SQL = ( "SELECT key, value FROM cursorDiskKV WHERE key LIKE 'composerData:%'" " AND LENGTH(value) > 10" " AND value LIKE '%fullConversationHeadersOnly%'" - " AND value NOT LIKE '%fullConversationHeadersOnly\":[]%'" - " AND value NOT LIKE '%fullConversationHeadersOnly\": []%'" ) diff --git a/services/workspace_listing.py b/services/workspace_listing.py index b88b6d1..f58ffd8 100644 --- a/services/workspace_listing.py +++ b/services/workspace_listing.py @@ -7,6 +7,7 @@ _logger = logging.getLogger(__name__) +from utils.conversation_index import load_conv_header_map from utils.cli_chat_reader import list_cli_projects from utils.exclusion_rules import RuleTokens, build_searchable_text, is_excluded_by_rules from utils.path_helpers import ( @@ -120,6 +121,7 @@ def _build_workspace_projects_uncached( if global_db: try: composer_rows = safe_fetchall(global_db, COMPOSER_ROWS_WITH_HEADERS_SQL) + conv_header_map = load_conv_header_map(global_db) project_layouts_map = load_project_layouts_map(global_db) bubble_map: dict[str, Bubble] = {} @@ -134,7 +136,10 @@ def _build_workspace_projects_uncached( for row in composer_rows: composer = parse_composer_data_row( - row["key"], row["value"], parse_warnings=parse_warnings, + row["key"], + row["value"], + parse_warnings=parse_warnings, + conv_header_map=conv_header_map, ) if composer is None: continue diff --git a/services/workspace_tabs.py b/services/workspace_tabs.py index 3b2db88..8927949 100644 --- a/services/workspace_tabs.py +++ b/services/workspace_tabs.py @@ -10,6 +10,7 @@ _logger = logging.getLogger(__name__) +from utils.conversation_index import load_conv_header_map, load_conv_headers_for_composer from utils.path_helpers import ( get_workspace_folder_paths, normalize_file_path, @@ -380,11 +381,15 @@ def _iter_assigned_workspace_composers( invalid_workspace_aliases: dict[str, str], matching_ws_ids: set[str], bubble_map: Mapping[str, Bubble], + conv_header_map: Mapping[str, list[dict[str, Any]]] | None = None, ) -> Iterator[tuple[Composer, str]]: """Yield composers assigned to the requested workspace, skipping parse failures.""" for row in composer_rows: composer = parse_composer_data_row( - row["key"], row["value"], parse_warnings=parse_warnings, + row["key"], + row["value"], + parse_warnings=parse_warnings, + conv_header_map=conv_header_map, ) if composer is None: continue @@ -575,6 +580,7 @@ def _build_workspace_tab_summaries_uncached( project_layouts_map = load_project_layouts_map(global_db) composer_rows = safe_fetchall(global_db, COMPOSER_ROWS_WITH_HEADERS_SQL) + conv_header_map = load_conv_header_map(global_db) ctx = with_invalid_workspace_aliases( ctx, @@ -598,6 +604,7 @@ def _build_workspace_tab_summaries_uncached( invalid_workspace_aliases=invalid_workspace_aliases, matching_ws_ids=matching_ws_ids, bubble_map={}, + conv_header_map=conv_header_map, ): if is_composer_excluded( rules, @@ -683,8 +690,12 @@ def assemble_single_tab( return {"error": ERR_CONVERSATION_NOT_FOUND, "code": CODE_CONVERSATION_NOT_FOUND}, 404 row = rows[0] + conv_header_map = load_conv_headers_for_composer(global_db, composer_id) composer = parse_composer_data_row( - row["key"], row["value"], parse_warnings=parse_warnings, + row["key"], + row["value"], + parse_warnings=parse_warnings, + conv_header_map=conv_header_map, ) if composer is None: return {"error": ERR_CONVERSATION_NOT_FOUND, "code": CODE_CONVERSATION_NOT_FOUND}, 404 @@ -822,6 +833,7 @@ def assemble_workspace_tabs( project_layouts_map[chat_id].append(layout["rootPath"]) composer_rows = safe_fetchall(global_db, COMPOSER_ROWS_WITH_HEADERS_SQL) + conv_header_map = load_conv_header_map(global_db) ctx = with_invalid_workspace_aliases( ctx, @@ -845,6 +857,7 @@ def assemble_workspace_tabs( invalid_workspace_aliases=invalid_workspace_aliases, matching_ws_ids=matching_ws_ids, bubble_map={}, + conv_header_map=conv_header_map, ): tab = _safe_assemble_tab_from_composer_data( composer_id=composer_id, diff --git a/tests/test_conversation_index.py b/tests/test_conversation_index.py new file mode 100644 index 0000000..25feda0 --- /dev/null +++ b/tests/test_conversation_index.py @@ -0,0 +1,150 @@ +"""Tests for Cursor 3.x composerConvHeaders and colon-bearing bubble ids.""" + +import json +import sqlite3 +import unittest + +from services.workspace_db import load_bubbles_for_composer +from utils.conversation_index import ( + bubble_id_from_kv_key, + composer_data_with_headers, + load_conv_header_map, + load_conv_headers_for_composer, +) + + +class TestBubbleIdFromKvKey(unittest.TestCase): + def test_plain_uuid(self) -> None: + key = "bubbleId:composer-1:11111111-1111-1111-1111-111111111111" + self.assertEqual( + bubble_id_from_kv_key(key), + "11111111-1111-1111-1111-111111111111", + ) + + def test_sub_bubble_keeps_colon(self) -> None: + bubble_id = "11111111-1111-1111-1111-111111111111/s:0" + key = f"bubbleId:composer-1:{bubble_id}" + self.assertEqual(bubble_id_from_kv_key(key), bubble_id) + + def test_tool_bubble_keeps_newline(self) -> None: + bubble_id = "11111111-1111-1111-1111-111111111111/t:call-1\nfc_abc" + key = f"bubbleId:composer-1:{bubble_id}" + self.assertEqual(bubble_id_from_kv_key(key), bubble_id) + + def test_missing_bubble_id(self) -> None: + self.assertIsNone(bubble_id_from_kv_key("bubbleId:composer-1")) + self.assertIsNone(bubble_id_from_kv_key("composerData:composer-1")) + + +class TestConvHeaderMap(unittest.TestCase): + def test_chunks_merge_in_suffix_order(self) -> None: + conn = sqlite3.connect(":memory:") + conn.row_factory = sqlite3.Row + conn.execute("CREATE TABLE cursorDiskKV ([key] TEXT PRIMARY KEY, value TEXT)") + composer_id = "cmp-new" + second = [{"bubbleId": "parent/s:0", "type": 2}] + first = [{"bubbleId": "parent", "type": 1}] + conn.execute( + "INSERT INTO cursorDiskKV ([key], value) VALUES (?, ?)", + (f"composerConvHeaders:{composer_id}:000001", json.dumps(second)), + ) + conn.execute( + "INSERT INTO cursorDiskKV ([key], value) VALUES (?, ?)", + (f"composerConvHeaders:{composer_id}:000000", json.dumps(first)), + ) + conn.commit() + + merged = load_conv_header_map(conn) + self.assertEqual( + [item["bubbleId"] for item in merged[composer_id]], + ["parent", "parent/s:0"], + ) + + def test_non_string_bubble_id_is_omitted(self) -> None: + conn = sqlite3.connect(":memory:") + conn.row_factory = sqlite3.Row + conn.execute("CREATE TABLE cursorDiskKV ([key] TEXT PRIMARY KEY, value TEXT)") + composer_id = "cmp-bad-id" + payload = [ + {"bubbleId": "parent", "type": 1}, + {"bubbleId": [], "type": 2}, + {"type": 2}, + ] + conn.execute( + "INSERT INTO cursorDiskKV ([key], value) VALUES (?, ?)", + (f"composerConvHeaders:{composer_id}:000000", json.dumps(payload)), + ) + conn.commit() + + merged = load_conv_header_map(conn) + self.assertEqual( + [item["bubbleId"] for item in merged[composer_id]], + ["parent"], + ) + + def test_scoped_loader_returns_one_composer_in_suffix_order(self) -> None: + conn = sqlite3.connect(":memory:") + conn.row_factory = sqlite3.Row + conn.execute("CREATE TABLE cursorDiskKV ([key] TEXT PRIMARY KEY, value TEXT)") + wanted = "cmp-wanted" + other = "cmp-other" + rows = [ + (f"composerConvHeaders:{wanted}:000001", [{"bubbleId": "second", "type": 2}]), + (f"composerConvHeaders:{wanted}:000000", [{"bubbleId": "first", "type": 1}]), + (f"composerConvHeaders:{other}:000000", [{"bubbleId": "elsewhere", "type": 1}]), + ( + "composerConvHeaders:cmp_wanted:000000", + [{"bubbleId": "underscore-neighbor", "type": 1}], + ), + ] + for key, payload in rows: + conn.execute( + "INSERT INTO cursorDiskKV ([key], value) VALUES (?, ?)", + (key, json.dumps(payload)), + ) + conn.commit() + + scoped = load_conv_headers_for_composer(conn, wanted) + self.assertEqual(list(scoped), [wanted]) + self.assertEqual( + [item["bubbleId"] for item in scoped[wanted]], + ["first", "second"], + ) + + def test_inline_headers_win(self) -> None: + inline = [{"bubbleId": "old", "type": 1}] + resolved = composer_data_with_headers( + {"fullConversationHeadersOnly": inline}, + "cmp-1", + {"cmp-1": [{"bubbleId": "chunk", "type": 2}]}, + ) + self.assertEqual(resolved["fullConversationHeadersOnly"], inline) + + def test_empty_inline_uses_chunks(self) -> None: + chunks = [{"bubbleId": "parent/s:0", "type": 2}] + original = {"fullConversationHeadersOnly": [], "name": "Chat"} + resolved = composer_data_with_headers(original, "cmp-1", {"cmp-1": chunks}) + self.assertEqual(resolved["fullConversationHeadersOnly"], chunks) + self.assertEqual(original["fullConversationHeadersOnly"], []) + + def test_load_bubbles_keeps_sub_bubble_id(self) -> None: + conn = sqlite3.connect(":memory:") + conn.row_factory = sqlite3.Row + conn.execute("CREATE TABLE cursorDiskKV ([key] TEXT PRIMARY KEY, value TEXT)") + composer_id = "cmp-new" + bubble_id = "11111111-1111-1111-1111-111111111111/s:0" + conn.execute( + "INSERT INTO cursorDiskKV ([key], value) VALUES (?, ?)", + ( + f"bubbleId:{composer_id}:{bubble_id}", + json.dumps({"text": "assistant turn"}), + ), + ) + conn.commit() + bubbles = load_bubbles_for_composer(conn, composer_id) + self.assertIn(bubble_id, bubbles) + self.assertEqual(bubbles[bubble_id].text, "assistant turn") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_export_exclusion_filtering.py b/tests/test_export_exclusion_filtering.py index 57950bc..9db796f 100644 --- a/tests/test_export_exclusion_filtering.py +++ b/tests/test_export_exclusion_filtering.py @@ -145,6 +145,58 @@ def test_filters_when_term_appears_after_long_prefix(self): self.assertIn("roadmap", content) self.assertNotIn("kwd", content) + def test_exports_when_headers_live_in_conv_chunks(self): + # Cursor 3.x leaves fullConversationHeadersOnly empty and stores the + # ordered index in composerConvHeaders, with bubble ids that contain + # colons. A draft with neither source must stay out of the export. + parent_id = "11111111-1111-1111-1111-111111111111" + sub_id = f"{parent_id}/s:0" + self._insert_bubble( + "cmp-new", parent_id, {"type": "user", "text": "user turn alpha"}, + ) + self._insert_bubble( + "cmp-new", sub_id, {"type": "assistant", "text": "assistant turn beta"}, + ) + self._insert_composer("cmp-draft", "Empty draft should stay hidden", "gpt-4.1", []) + conn = sqlite3.connect(self.global_db_path) + conn.execute( + "INSERT INTO cursorDiskKV ([key], value) VALUES (?, ?)", + ( + "composerData:cmp-new", + json.dumps({ + "name": "Chunked chat", + "modelConfig": {"modelName": "gpt-4.1"}, + "fullConversationHeadersOnly": [], + "lastUpdatedAt": 1739300000000, + "createdAt": 1739200000000, + }), + ), + ) + conn.execute( + "INSERT INTO cursorDiskKV ([key], value) VALUES (?, ?)", + ( + "composerConvHeaders:cmp-new:000001", + json.dumps([{"bubbleId": sub_id, "type": 2}]), + ), + ) + conn.execute( + "INSERT INTO cursorDiskKV ([key], value) VALUES (?, ?)", + ( + "composerConvHeaders:cmp-new:000000", + json.dumps([{"bubbleId": parent_id, "type": 1}]), + ), + ) + conn.commit() + conn.close() + + out_dir = self._run_export("kwd\n") + md_files = self._collect_exported_markdown(out_dir) + self.assertEqual(len(md_files), 1) + content = md_files[0].read_text(encoding="utf-8") + self.assertIn("user turn alpha", content) + self.assertIn("assistant turn beta", content) + self.assertNotIn("Empty draft should stay hidden", content) + def test_writes_manifest_to_global_state_dir(self): self._insert_bubble("cmp-safe", "b-safe-4", {"type": "assistant", "text": "General roadmap update."}) self._insert_composer("cmp-safe", "Roadmap", "gpt-4.1-mini", ["b-safe-4"]) diff --git a/tests/test_search_index.py b/tests/test_search_index.py index 2fb345b..0b66a6d 100644 --- a/tests/test_search_index.py +++ b/tests/test_search_index.py @@ -120,6 +120,20 @@ def test_index_is_usable_after_build(self, indexed_storage): with patches[0], patches[1], patches[2]: assert index_is_usable(indexed_storage["ws_root"], []) is True + def test_stale_index_version_rebuilds(self, indexed_storage): + patches = _index_patches(indexed_storage["cache_dir"]) + with patches[0], patches[1], patches[2]: + with sqlite3.connect(indexed_storage["index_path"]) as conn: + conn.execute( + "UPDATE index_meta SET value = ? WHERE key = 'version'", + ("1",), + ) + conn.commit() + assert index_is_usable(indexed_storage["ws_root"], []) is False + rebuilt = build_search_index(indexed_storage["ws_root"], [], force=False) + assert rebuilt is True + assert index_is_usable(indexed_storage["ws_root"], []) is True + def test_bubble_fts_finds_term(self, indexed_storage): patches = _index_patches(indexed_storage["cache_dir"]) with patches[0], patches[1], patches[2]: diff --git a/utils/conversation_index.py b/utils/conversation_index.py new file mode 100644 index 0000000..73c2337 --- /dev/null +++ b/utils/conversation_index.py @@ -0,0 +1,187 @@ +"""Conversation index helpers for Cursor 3.x global storage. + +Cursor still writes ``composerData.fullConversationHeadersOnly``, but recent +sessions leave that list empty and store the ordered header entries in +``composerConvHeaders:{composerId}:{chunk}``. Bubble ids in those headers +may contain colons (``{uuid}/s:0``, ``{uuid}/t:call-…``). +""" + +from __future__ import annotations + +import json +import logging +import sqlite3 +from collections.abc import Mapping +from typing import Any + +_logger = logging.getLogger(__name__) + +_BUBBLE_KEY_PREFIX = "bubbleId:" +_CONV_HEADER_KEY_PREFIX = "composerConvHeaders:" + + +def bubble_id_from_kv_key(key: str) -> str | None: + """Return the bubble id from a ``bubbleId:{composerId}:{bubbleId}`` key. + + The bubble id is everything after the composer id, including colons and + newlines. ``str.split(":")`` drops that tail. + """ + if not key.startswith(_BUBBLE_KEY_PREFIX): + return None + _composer_id, sep, bubble_id = key[len(_BUBBLE_KEY_PREFIX):].partition(":") + if not sep or not bubble_id: + return None + return bubble_id + + +def _conv_header_key_parts(key: str) -> tuple[str, str] | None: + """Return ``(composer_id, chunk_suffix)`` for a conv-header key.""" + if not key.startswith(_CONV_HEADER_KEY_PREFIX): + return None + composer_id, sep, chunk = key[len(_CONV_HEADER_KEY_PREFIX):].partition(":") + if not sep or not composer_id or not chunk: + return None + return composer_id, chunk + + +def _json_object(value: object) -> object | None: + if isinstance(value, bytes): + try: + value = value.decode("utf-8") + except UnicodeDecodeError: + return None + if not isinstance(value, str): + return None + try: + parsed: object = json.loads(value) + except (json.JSONDecodeError, TypeError, ValueError): + return None + return parsed + + +def _string_bubble_headers(parsed: object) -> list[dict[str, Any]] | None: + """Return header dicts whose ``bubbleId`` is a string. + + A list or dict id is unhashable and would raise ``TypeError`` in + ``bubble_map.get``. Non-list rows are rejected so the caller can log them. + """ + if not isinstance(parsed, list): + return None + return [ + item + for item in parsed + if isinstance(item, dict) and isinstance(item.get("bubbleId"), str) + ] + + +def _merge_header_rows(rows: list[Any]) -> dict[str, list[dict[str, Any]]]: + """Concatenate header chunks in suffix order.""" + grouped: dict[str, list[tuple[str, list[dict[str, Any]]]]] = {} + for row in rows: + key = row["key"] + parts = _conv_header_key_parts(key if isinstance(key, str) else "") + if parts is None: + continue + composer_id, chunk = parts + parsed = _json_object(row["value"]) + headers = _string_bubble_headers(parsed) + if headers is None: + _logger.debug("Skipping non-list composerConvHeaders row %s", key) + continue + grouped.setdefault(composer_id, []).append((chunk, headers)) + + merged: dict[str, list[dict[str, Any]]] = {} + for composer_id, chunks in grouped.items(): + chunks.sort(key=lambda pair: pair[0]) + combined: list[dict[str, Any]] = [] + for _chunk, headers in chunks: + combined.extend(headers) + if combined: + merged[composer_id] = combined + return merged + + +def _conv_header_like_pattern(composer_id: str) -> str: + """Return a ``LIKE`` pattern for one composer's header chunks. + + ``%`` and ``_`` in *composer_id* are escaped for ``ESCAPE '\\'``. + """ + escaped = ( + composer_id + .replace("\\", "\\\\") + .replace("%", "\\%") + .replace("_", "\\_") + ) + return f"{_CONV_HEADER_KEY_PREFIX}{escaped}:%" + + +def load_conv_header_map( + conn: sqlite3.Connection, +) -> dict[str, list[dict[str, Any]]]: + """Load ``composerConvHeaders:*`` chunks into ``{composer_id: headers}``. + + Chunks for one composer are concatenated in suffix order (``000000``, + then ``000001``, …). Malformed rows and entries whose ``bubbleId`` is + not a string are skipped. + """ + try: + rows = conn.execute( + "SELECT key, value FROM cursorDiskKV" + " WHERE key LIKE 'composerConvHeaders:%'" + ).fetchall() + except sqlite3.Error as exc: + _logger.debug("Could not read composerConvHeaders: %s", exc) + return {} + return _merge_header_rows(rows) + + +def load_conv_headers_for_composer( + conn: sqlite3.Connection, + composer_id: str, +) -> dict[str, list[dict[str, Any]]]: + """Load header chunks for one composer. + + Returns ``{composer_id: headers}`` when that composer has string + ``bubbleId`` entries, otherwise ``{}``. Chunks are concatenated in + suffix order. + """ + if not composer_id: + return {} + try: + rows = conn.execute( + "SELECT key, value FROM cursorDiskKV" + " WHERE key LIKE ? ESCAPE '\\'", + (_conv_header_like_pattern(composer_id),), + ).fetchall() + except sqlite3.Error as exc: + _logger.debug( + "Could not read composerConvHeaders for %s: %s", composer_id, exc, + ) + return {} + merged = _merge_header_rows(rows) + headers = merged.get(composer_id) + if not headers: + return {} + return {composer_id: headers} + + +def composer_data_with_headers( + composer_data: dict[str, Any], + composer_id: str, + conv_header_map: Mapping[str, list[dict[str, Any]]], +) -> dict[str, Any]: + """Return composer JSON whose header list is non-empty when chunks exist. + + A non-empty ``fullConversationHeadersOnly`` is left unchanged. An empty + or missing list is replaced with the chunk map for *composer_id* when + that map has entries. The input dict is not mutated. + """ + inline = composer_data.get("fullConversationHeadersOnly") + if isinstance(inline, list) and inline: + return composer_data + resolved = conv_header_map.get(composer_id) or [] + if not resolved: + return composer_data + copied = dict(composer_data) + copied["fullConversationHeadersOnly"] = list(resolved) + return copied