From 15fdf2e16f57e648b49850d133892ea61171fab8 Mon Sep 17 00:00:00 2001 From: wpak-ai Date: Wed, 28 Oct 2026 10:26:36 -0400 Subject: [PATCH 1/3] Enhance conversation index handling in Cursor 3.x --- CHANGELOG.md | 5 + services/export_engine.py | 7 +- services/search.py | 13 ++- services/search_index.py | 21 +++- services/workspace_composer_scan.py | 6 ++ services/workspace_db.py | 19 ++-- services/workspace_listing.py | 7 +- services/workspace_tabs.py | 17 +++- tests/test_conversation_index.py | 98 ++++++++++++++++++ tests/test_export_exclusion_filtering.py | 52 ++++++++++ utils/conversation_index.py | 123 +++++++++++++++++++++++ 11 files changed, 353 insertions(+), 15 deletions(-) create mode 100644 tests/test_conversation_index.py create mode 100644 utils/conversation_index.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 64b0799..4c62726 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed +- **Cursor 3 conversation index**: export, search, and workspace tabs read + `composerConvHeaders` when `fullConversationHeadersOnly` is empty, and keep + bubble ids that contain colons (`{uuid}/s:0`). + ## [0.2.1] - 2026-07-30 ### Added diff --git a/services/export_engine.py b/services/export_engine.py index b563518..988a475 100644 --- a/services/export_engine.py +++ b/services/export_engine.py @@ -6,7 +6,7 @@ import logging import os import sqlite3 -from dataclasses import dataclass +from dataclasses import dataclass, field from datetime import datetime from typing import Any, Literal @@ -35,6 +35,7 @@ messages_to_bubbles, traverse_blobs, ) +from utils.conversation_index import composer_data_with_headers, load_conv_header_map from utils.cursor_md_exporter import ( cursor_cli_session_to_markdown, cursor_ide_chat_to_markdown, @@ -97,6 +98,7 @@ class GlobalDbExportData: code_block_diff_map: dict[str, list[Any]] ide_composer_rows: list[sqlite3.Row] invalid_workspace_aliases: dict[str, str] + conv_header_map: dict[str, list[dict[str, Any]]] = field(default_factory=dict) def json_dump_safe(value: object) -> str: @@ -186,6 +188,7 @@ def load_global_db_export_data( project_layouts_map = enriched.project_layouts_map bubble_map = enriched.bubble_map code_block_diff_map = load_code_block_diff_map(global_db) + conv_header_map = load_conv_header_map(global_db) ide_composer_rows = safe_fetchall(global_db, COMPOSER_ROWS_WITH_HEADERS_SQL) invalid_workspace_aliases = resolve_invalid_workspace_aliases_cached( @@ -203,6 +206,7 @@ def load_global_db_export_data( code_block_diff_map=code_block_diff_map, ide_composer_rows=ide_composer_rows, invalid_workspace_aliases=invalid_workspace_aliases, + conv_header_map=conv_header_map, ) @@ -245,6 +249,7 @@ def _collect_ide_export_entries( ) continue + cd = composer_data_with_headers(cd, composer_id, db_data.conv_header_map) headers = cd.get("fullConversationHeadersOnly") or [] if not isinstance(headers, list) or not headers: continue diff --git a/services/search.py b/services/search.py index c0471f6..3e18506 100644 --- a/services/search.py +++ b/services/search.py @@ -51,6 +51,7 @@ ) from services.workspace_resolver import infer_invalid_workspace_aliases from utils.cli_chat_reader import list_cli_projects, messages_to_bubbles, traverse_blobs +from utils.conversation_index import composer_data_with_headers, load_conv_header_map from utils.exclusion_rules import RuleTokens, build_searchable_text, is_excluded_by_rules from utils.text_extract import extract_text_from_bubble from utils.path_helpers import ( @@ -506,6 +507,11 @@ def _search_global_storage_via_index( workspace_path, workspace_entries, rules, ) + with open_global_db(workspace_path) as (header_db, _): + conv_header_map = ( + load_conv_header_map(header_db) if header_db is not None else {} + ) + composers_in_window = query_composer_rows_in_window(since_ms) if since_ms is not None and not composers_in_window: return results @@ -551,8 +557,9 @@ def _search_global_storage_via_index( continue try: + cd = composer_data_with_headers(cd, composer_id, conv_header_map) headers = cd.get("fullConversationHeadersOnly") or [] - if not headers: + if not isinstance(headers, list) or not headers: continue title = composer_row["title"] or cd.get("name") or "" @@ -677,6 +684,7 @@ def _search_global_storage_live_scan( return results composer_rows = conn.execute(COMPOSER_ROWS_WITH_HEADERS_SQL).fetchall() + conv_header_map = load_conv_header_map(conn) window_composer_ids: set[str] | None = None if since_ms is not None: @@ -726,8 +734,9 @@ def _search_global_storage_live_scan( continue try: + cd = composer_data_with_headers(cd, composer_id, conv_header_map) headers = cd.get("fullConversationHeadersOnly") or [] - if not headers: + if not isinstance(headers, list) or not headers: continue title = cd.get("name") or "" diff --git a/services/search_index.py b/services/search_index.py index bcbfee2..177e03b 100644 --- a/services/search_index.py +++ b/services/search_index.py @@ -57,7 +57,10 @@ _logger = logging.getLogger(__name__) -INDEX_VERSION = 1 +# Bumped when the indexed composer set changes. Version 2 includes composers +# whose inline fullConversationHeadersOnly list is empty (Cursor 3.x stores +# the index in composerConvHeaders instead). +INDEX_VERSION = 2 SEARCH_INDEX_POINTER_FILE = CACHE_DIR / "search_index.active" # Legacy single-file path (pre-pointer builds); still honored when no pointer exists. SEARCH_INDEX_FILE = CACHE_DIR / "search_index.sqlite" @@ -541,6 +544,18 @@ def query_composer_rows_in_window( return {row["composer_id"]: row for row in rows} +def _stored_index_version(conn: sqlite3.Connection) -> str | None: + try: + row = conn.execute( + "SELECT value FROM index_meta WHERE key = 'version'" + ).fetchone() + except sqlite3.Error: + return None + if not row or row[0] is None: + return None + return str(row[0]) + + def index_is_usable(workspace_path: str, rules: list[RuleTokens]) -> bool: """True when the on-disk index matches the current Cursor storage fingerprint.""" if not index_search_enabled() or _resolve_active_index_db_path() is None: @@ -550,4 +565,6 @@ def index_is_usable(workspace_path: str, rules: list[RuleTokens]) -> bool: if conn is None: return False stored = _read_stored_fingerprint(conn) - return stored is not None and _fingerprints_match(stored, fingerprint) + if stored is None or not _fingerprints_match(stored, fingerprint): + return False + return _stored_index_version(conn) == str(INDEX_VERSION) diff --git a/services/workspace_composer_scan.py b/services/workspace_composer_scan.py index 74da715..ff8d437 100644 --- a/services/workspace_composer_scan.py +++ b/services/workspace_composer_scan.py @@ -12,6 +12,7 @@ from typing import Any from models import Bubble, Composer, ParseWarningCollector, SchemaError +from utils.conversation_index import composer_data_with_headers from utils.exclusion_rules import RuleTokens, build_searchable_text, is_excluded_by_rules from services.workspace_resolver import determine_project_for_conversation @@ -23,11 +24,15 @@ def parse_composer_data_row( raw_value: object | None, *, parse_warnings: ParseWarningCollector, + conv_header_map: Mapping[str, list[dict[str, Any]]] | None = None, ) -> Composer | None: """Parse a ``composerData:*`` KV row into a :class:`Composer`. Returns ``None`` for null/placeholder payloads (e.g. ``empty-state-draft``) without emitting decode warnings. Logs and records skips for malformed JSON. + When ``fullConversationHeadersOnly`` is empty, headers are taken from + *conv_header_map* (``composerConvHeaders`` chunks). A composer with no + resolved headers is omitted. """ if not row_key.startswith("composerData:"): return None @@ -53,6 +58,7 @@ def parse_composer_data_row( if not isinstance(cd, dict): parse_warnings.record_composer_skipped() return None + cd = composer_data_with_headers(cd, composer_id, conv_header_map or {}) try: composer = Composer.from_dict(cd, composer_id=composer_id) except SchemaError as e: diff --git a/services/workspace_db.py b/services/workspace_db.py index 1ccd386..bc00c82 100644 --- a/services/workspace_db.py +++ b/services/workspace_db.py @@ -15,6 +15,7 @@ _logger = logging.getLogger(__name__) from models import Bubble, ParseWarningCollector, SchemaError +from utils.conversation_index import bubble_id_from_kv_key from utils.path_helpers import get_workspace_folder_paths from utils.workspace_descriptor import read_json_file @@ -44,11 +45,14 @@ def _parse_bubble_kv_row( *, parse_warnings: ParseWarningCollector | None = None, ) -> tuple[str, Bubble] | None: - """Parse one ``bubbleId:…`` row; return ``(bubble_id, Bubble)`` or skip.""" - parts = row_key.split(":") - if len(parts) < 3: + """Parse one ``bubbleId:…`` row; return ``(bubble_id, Bubble)`` or skip. + + Bubble ids may contain colons (``{uuid}/s:0``). The id is the key tail + after ``bubbleId:{composerId}:``. + """ + bid = bubble_id_from_kv_key(row_key) + if not bid: return None - bid = parts[2] try: parsed = json.loads(row_value) bubble = Bubble.from_dict(parsed, bubble_id=bid) @@ -344,13 +348,14 @@ def collect_invalid_workspace_ids(workspace_entries: list[dict[str, Any]]) -> se return invalid -# Composers that have at least one header entry (list/summary paths). +# Composers that store a conversation index field. Cursor 3.x leaves +# fullConversationHeadersOnly as [] and writes the ordered headers to +# composerConvHeaders:{id}:{chunk}. Callers resolve that fallback and skip +# composers whose resolved header list is still empty (drafts). COMPOSER_ROWS_WITH_HEADERS_SQL = ( "SELECT key, value FROM cursorDiskKV WHERE key LIKE 'composerData:%'" " AND LENGTH(value) > 10" " AND value LIKE '%fullConversationHeadersOnly%'" - " AND value NOT LIKE '%fullConversationHeadersOnly\":[]%'" - " AND value NOT LIKE '%fullConversationHeadersOnly\": []%'" ) diff --git a/services/workspace_listing.py b/services/workspace_listing.py index b88b6d1..f58ffd8 100644 --- a/services/workspace_listing.py +++ b/services/workspace_listing.py @@ -7,6 +7,7 @@ _logger = logging.getLogger(__name__) +from utils.conversation_index import load_conv_header_map from utils.cli_chat_reader import list_cli_projects from utils.exclusion_rules import RuleTokens, build_searchable_text, is_excluded_by_rules from utils.path_helpers import ( @@ -120,6 +121,7 @@ def _build_workspace_projects_uncached( if global_db: try: composer_rows = safe_fetchall(global_db, COMPOSER_ROWS_WITH_HEADERS_SQL) + conv_header_map = load_conv_header_map(global_db) project_layouts_map = load_project_layouts_map(global_db) bubble_map: dict[str, Bubble] = {} @@ -134,7 +136,10 @@ def _build_workspace_projects_uncached( for row in composer_rows: composer = parse_composer_data_row( - row["key"], row["value"], parse_warnings=parse_warnings, + row["key"], + row["value"], + parse_warnings=parse_warnings, + conv_header_map=conv_header_map, ) if composer is None: continue diff --git a/services/workspace_tabs.py b/services/workspace_tabs.py index 3b2db88..5394902 100644 --- a/services/workspace_tabs.py +++ b/services/workspace_tabs.py @@ -10,6 +10,7 @@ _logger = logging.getLogger(__name__) +from utils.conversation_index import load_conv_header_map from utils.path_helpers import ( get_workspace_folder_paths, normalize_file_path, @@ -380,11 +381,15 @@ def _iter_assigned_workspace_composers( invalid_workspace_aliases: dict[str, str], matching_ws_ids: set[str], bubble_map: Mapping[str, Bubble], + conv_header_map: Mapping[str, list[dict[str, Any]]] | None = None, ) -> Iterator[tuple[Composer, str]]: """Yield composers assigned to the requested workspace, skipping parse failures.""" for row in composer_rows: composer = parse_composer_data_row( - row["key"], row["value"], parse_warnings=parse_warnings, + row["key"], + row["value"], + parse_warnings=parse_warnings, + conv_header_map=conv_header_map, ) if composer is None: continue @@ -575,6 +580,7 @@ def _build_workspace_tab_summaries_uncached( project_layouts_map = load_project_layouts_map(global_db) composer_rows = safe_fetchall(global_db, COMPOSER_ROWS_WITH_HEADERS_SQL) + conv_header_map = load_conv_header_map(global_db) ctx = with_invalid_workspace_aliases( ctx, @@ -598,6 +604,7 @@ def _build_workspace_tab_summaries_uncached( invalid_workspace_aliases=invalid_workspace_aliases, matching_ws_ids=matching_ws_ids, bubble_map={}, + conv_header_map=conv_header_map, ): if is_composer_excluded( rules, @@ -683,8 +690,12 @@ def assemble_single_tab( return {"error": ERR_CONVERSATION_NOT_FOUND, "code": CODE_CONVERSATION_NOT_FOUND}, 404 row = rows[0] + conv_header_map = load_conv_header_map(global_db) composer = parse_composer_data_row( - row["key"], row["value"], parse_warnings=parse_warnings, + row["key"], + row["value"], + parse_warnings=parse_warnings, + conv_header_map=conv_header_map, ) if composer is None: return {"error": ERR_CONVERSATION_NOT_FOUND, "code": CODE_CONVERSATION_NOT_FOUND}, 404 @@ -822,6 +833,7 @@ def assemble_workspace_tabs( project_layouts_map[chat_id].append(layout["rootPath"]) composer_rows = safe_fetchall(global_db, COMPOSER_ROWS_WITH_HEADERS_SQL) + conv_header_map = load_conv_header_map(global_db) ctx = with_invalid_workspace_aliases( ctx, @@ -845,6 +857,7 @@ def assemble_workspace_tabs( invalid_workspace_aliases=invalid_workspace_aliases, matching_ws_ids=matching_ws_ids, bubble_map={}, + conv_header_map=conv_header_map, ): tab = _safe_assemble_tab_from_composer_data( composer_id=composer_id, diff --git a/tests/test_conversation_index.py b/tests/test_conversation_index.py new file mode 100644 index 0000000..ab449bc --- /dev/null +++ b/tests/test_conversation_index.py @@ -0,0 +1,98 @@ +"""Tests for Cursor 3.x composerConvHeaders and colon-bearing bubble ids.""" + +import json +import sqlite3 +import unittest + +from services.workspace_db import load_bubbles_for_composer +from utils.conversation_index import ( + bubble_id_from_kv_key, + composer_data_with_headers, + load_conv_header_map, +) + + +class TestBubbleIdFromKvKey(unittest.TestCase): + def test_plain_uuid(self) -> None: + key = "bubbleId:composer-1:11111111-1111-1111-1111-111111111111" + self.assertEqual( + bubble_id_from_kv_key(key), + "11111111-1111-1111-1111-111111111111", + ) + + def test_sub_bubble_keeps_colon(self) -> None: + bubble_id = "11111111-1111-1111-1111-111111111111/s:0" + key = f"bubbleId:composer-1:{bubble_id}" + self.assertEqual(bubble_id_from_kv_key(key), bubble_id) + + def test_tool_bubble_keeps_newline(self) -> None: + bubble_id = "11111111-1111-1111-1111-111111111111/t:call-1\nfc_abc" + key = f"bubbleId:composer-1:{bubble_id}" + self.assertEqual(bubble_id_from_kv_key(key), bubble_id) + + def test_missing_bubble_id(self) -> None: + self.assertIsNone(bubble_id_from_kv_key("bubbleId:composer-1")) + self.assertIsNone(bubble_id_from_kv_key("composerData:composer-1")) + + +class TestConvHeaderMap(unittest.TestCase): + def test_chunks_merge_in_suffix_order(self) -> None: + conn = sqlite3.connect(":memory:") + conn.row_factory = sqlite3.Row + conn.execute("CREATE TABLE cursorDiskKV ([key] TEXT PRIMARY KEY, value TEXT)") + composer_id = "cmp-new" + second = [{"bubbleId": "parent/s:0", "type": 2}] + first = [{"bubbleId": "parent", "type": 1}] + conn.execute( + "INSERT INTO cursorDiskKV ([key], value) VALUES (?, ?)", + (f"composerConvHeaders:{composer_id}:000001", json.dumps(second)), + ) + conn.execute( + "INSERT INTO cursorDiskKV ([key], value) VALUES (?, ?)", + (f"composerConvHeaders:{composer_id}:000000", json.dumps(first)), + ) + conn.commit() + + merged = load_conv_header_map(conn) + self.assertEqual( + [item["bubbleId"] for item in merged[composer_id]], + ["parent", "parent/s:0"], + ) + + def test_inline_headers_win(self) -> None: + inline = [{"bubbleId": "old", "type": 1}] + resolved = composer_data_with_headers( + {"fullConversationHeadersOnly": inline}, + "cmp-1", + {"cmp-1": [{"bubbleId": "chunk", "type": 2}]}, + ) + self.assertEqual(resolved["fullConversationHeadersOnly"], inline) + + def test_empty_inline_uses_chunks(self) -> None: + chunks = [{"bubbleId": "parent/s:0", "type": 2}] + original = {"fullConversationHeadersOnly": [], "name": "Chat"} + resolved = composer_data_with_headers(original, "cmp-1", {"cmp-1": chunks}) + self.assertEqual(resolved["fullConversationHeadersOnly"], chunks) + self.assertEqual(original["fullConversationHeadersOnly"], []) + + def test_load_bubbles_keeps_sub_bubble_id(self) -> None: + conn = sqlite3.connect(":memory:") + conn.row_factory = sqlite3.Row + conn.execute("CREATE TABLE cursorDiskKV ([key] TEXT PRIMARY KEY, value TEXT)") + composer_id = "cmp-new" + bubble_id = "11111111-1111-1111-1111-111111111111/s:0" + conn.execute( + "INSERT INTO cursorDiskKV ([key], value) VALUES (?, ?)", + ( + f"bubbleId:{composer_id}:{bubble_id}", + json.dumps({"text": "assistant turn"}), + ), + ) + conn.commit() + bubbles = load_bubbles_for_composer(conn, composer_id) + self.assertIn(bubble_id, bubbles) + self.assertEqual(bubbles[bubble_id].text, "assistant turn") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_export_exclusion_filtering.py b/tests/test_export_exclusion_filtering.py index 57950bc..9db796f 100644 --- a/tests/test_export_exclusion_filtering.py +++ b/tests/test_export_exclusion_filtering.py @@ -145,6 +145,58 @@ def test_filters_when_term_appears_after_long_prefix(self): self.assertIn("roadmap", content) self.assertNotIn("kwd", content) + def test_exports_when_headers_live_in_conv_chunks(self): + # Cursor 3.x leaves fullConversationHeadersOnly empty and stores the + # ordered index in composerConvHeaders, with bubble ids that contain + # colons. A draft with neither source must stay out of the export. + parent_id = "11111111-1111-1111-1111-111111111111" + sub_id = f"{parent_id}/s:0" + self._insert_bubble( + "cmp-new", parent_id, {"type": "user", "text": "user turn alpha"}, + ) + self._insert_bubble( + "cmp-new", sub_id, {"type": "assistant", "text": "assistant turn beta"}, + ) + self._insert_composer("cmp-draft", "Empty draft should stay hidden", "gpt-4.1", []) + conn = sqlite3.connect(self.global_db_path) + conn.execute( + "INSERT INTO cursorDiskKV ([key], value) VALUES (?, ?)", + ( + "composerData:cmp-new", + json.dumps({ + "name": "Chunked chat", + "modelConfig": {"modelName": "gpt-4.1"}, + "fullConversationHeadersOnly": [], + "lastUpdatedAt": 1739300000000, + "createdAt": 1739200000000, + }), + ), + ) + conn.execute( + "INSERT INTO cursorDiskKV ([key], value) VALUES (?, ?)", + ( + "composerConvHeaders:cmp-new:000001", + json.dumps([{"bubbleId": sub_id, "type": 2}]), + ), + ) + conn.execute( + "INSERT INTO cursorDiskKV ([key], value) VALUES (?, ?)", + ( + "composerConvHeaders:cmp-new:000000", + json.dumps([{"bubbleId": parent_id, "type": 1}]), + ), + ) + conn.commit() + conn.close() + + out_dir = self._run_export("kwd\n") + md_files = self._collect_exported_markdown(out_dir) + self.assertEqual(len(md_files), 1) + content = md_files[0].read_text(encoding="utf-8") + self.assertIn("user turn alpha", content) + self.assertIn("assistant turn beta", content) + self.assertNotIn("Empty draft should stay hidden", content) + def test_writes_manifest_to_global_state_dir(self): self._insert_bubble("cmp-safe", "b-safe-4", {"type": "assistant", "text": "General roadmap update."}) self._insert_composer("cmp-safe", "Roadmap", "gpt-4.1-mini", ["b-safe-4"]) diff --git a/utils/conversation_index.py b/utils/conversation_index.py new file mode 100644 index 0000000..7de0aee --- /dev/null +++ b/utils/conversation_index.py @@ -0,0 +1,123 @@ +"""Conversation index helpers for Cursor 3.x global storage. + +Cursor still writes ``composerData.fullConversationHeadersOnly``, but recent +sessions leave that list empty and store the ordered header entries in +``composerConvHeaders:{composerId}:{chunk}``. Bubble ids in those headers +may contain colons (``{uuid}/s:0``, ``{uuid}/t:call-…``). +""" + +from __future__ import annotations + +import json +import logging +import sqlite3 +from collections.abc import Mapping +from typing import Any + +_logger = logging.getLogger(__name__) + +_BUBBLE_KEY_PREFIX = "bubbleId:" +_CONV_HEADER_KEY_PREFIX = "composerConvHeaders:" + + +def bubble_id_from_kv_key(key: str) -> str | None: + """Return the bubble id from a ``bubbleId:{composerId}:{bubbleId}`` key. + + The bubble id is everything after the composer id, including colons and + newlines. ``str.split(":")`` drops that tail. + """ + if not key.startswith(_BUBBLE_KEY_PREFIX): + return None + _composer_id, sep, bubble_id = key[len(_BUBBLE_KEY_PREFIX):].partition(":") + if not sep or not bubble_id: + return None + return bubble_id + + +def _conv_header_key_parts(key: str) -> tuple[str, str] | None: + """Return ``(composer_id, chunk_suffix)`` for a conv-header key.""" + if not key.startswith(_CONV_HEADER_KEY_PREFIX): + return None + composer_id, sep, chunk = key[len(_CONV_HEADER_KEY_PREFIX):].partition(":") + if not sep or not composer_id or not chunk: + return None + return composer_id, chunk + + +def _json_object(value: object) -> object | None: + if isinstance(value, bytes): + try: + value = value.decode("utf-8") + except UnicodeDecodeError: + return None + if not isinstance(value, str): + return None + try: + parsed: object = json.loads(value) + except (json.JSONDecodeError, TypeError, ValueError): + return None + return parsed + + +def load_conv_header_map( + conn: sqlite3.Connection, +) -> dict[str, list[dict[str, Any]]]: + """Load ``composerConvHeaders:*`` chunks into ``{composer_id: headers}``. + + Chunks for one composer are concatenated in suffix order (``000000``, + then ``000001``, …). Malformed rows are skipped. + """ + grouped: dict[str, list[tuple[str, list[dict[str, Any]]]]] = {} + try: + rows = conn.execute( + "SELECT key, value FROM cursorDiskKV" + " WHERE key LIKE 'composerConvHeaders:%'" + ).fetchall() + except sqlite3.Error as exc: + _logger.debug("Could not read composerConvHeaders: %s", exc) + return {} + + for row in rows: + key = row["key"] + parts = _conv_header_key_parts(key if isinstance(key, str) else "") + if parts is None: + continue + composer_id, chunk = parts + parsed = _json_object(row["value"]) + if not isinstance(parsed, list): + _logger.debug("Skipping non-list composerConvHeaders row %s", key) + continue + headers = [item for item in parsed if isinstance(item, dict)] + grouped.setdefault(composer_id, []).append((chunk, headers)) + + merged: dict[str, list[dict[str, Any]]] = {} + for composer_id, chunks in grouped.items(): + chunks.sort(key=lambda pair: pair[0]) + combined: list[dict[str, Any]] = [] + for _chunk, headers in chunks: + combined.extend(headers) + if combined: + merged[composer_id] = combined + return merged + + +def composer_data_with_headers( + composer_data: dict[str, Any], + composer_id: str, + conv_header_map: Mapping[str, list[dict[str, Any]]], +) -> dict[str, Any]: + """Return composer JSON whose header list is non-empty when chunks exist. + + A non-empty ``fullConversationHeadersOnly`` is left unchanged. An empty + or missing list is replaced with the chunk map for *composer_id* when + that map has entries. The input dict is not mutated. + """ + inline = composer_data.get("fullConversationHeadersOnly") + if isinstance(inline, list) and inline: + return composer_data + resolved = conv_header_map.get(composer_id) or [] + if not resolved: + return composer_data + copied = dict(composer_data) + copied["fullConversationHeadersOnly"] = list(resolved) + return copied From 8b38cf970f942107d9c911ef90779bf7a5db68ce Mon Sep 17 00:00:00 2001 From: wpak-ai Date: Wed, 28 Oct 2026 11:45:57 -0400 Subject: [PATCH 2/3] Rebuild stale search indexes and scope single-tab header reads. A version-1 index with a matching fingerprint never rebuilt, so search stayed on the live scan. Opening one tab now reads only that composer's header rows, and non-string bubble ids are dropped before export. Refresh the lock file so the freshness check matches current pins. --- requirements-lock.txt | 10 ++-- services/search_index.py | 32 +++++++---- services/workspace_tabs.py | 4 +- tests/test_conversation_index.py | 52 +++++++++++++++++ tests/test_search_index.py | 14 +++++ utils/conversation_index.py | 98 ++++++++++++++++++++++++++------ 6 files changed, 174 insertions(+), 36 deletions(-) diff --git a/requirements-lock.txt b/requirements-lock.txt index 74f5c5a..9f7c822 100644 --- a/requirements-lock.txt +++ b/requirements-lock.txt @@ -4,13 +4,13 @@ # Run periodically (e.g. via the "Update dependency lock file" CI workflow) to pick up # upstream patch / security releases within the bounded ranges in requirements.txt. blinker==1.9.0 # via flask -click==8.4.2 # via flask +click==8.5.0 # via flask defusedxml==0.7.1 # via fpdf2 flask==3.1.3 # via -r requirements.txt -fonttools==4.63.0 # via fpdf2 -fpdf2==2.8.8 # via -r requirements.txt +fonttools==4.68.0 # via fpdf2 +fpdf2==2.8.9 # via -r requirements.txt itsdangerous==2.2.0 # via flask jinja2==3.1.6 # via flask -markupsafe==3.0.3 # via flask, jinja2, werkzeug +markupsafe==3.0.4 # via flask, jinja2, werkzeug pillow==12.3.0 # via -r requirements.txt, fpdf2 -werkzeug==3.1.8 # via flask +werkzeug==3.1.9 # via flask diff --git a/services/search_index.py b/services/search_index.py index 177e03b..c404430 100644 --- a/services/search_index.py +++ b/services/search_index.py @@ -227,7 +227,10 @@ def build_search_index( *, force: bool = False, ) -> bool: - """Rebuild search index when fingerprint differs. Returns True if rebuilt.""" + """Rebuild search index when the fingerprint or stored version differs. + + Returns True if rebuilt. + """ if not index_search_enabled(): return False @@ -240,10 +243,8 @@ def build_search_index( active_path = _resolve_active_index_db_path() if not force and active_path is not None: with _index_db_conn(readonly=True) as existing: - if existing is not None: - stored = _read_stored_fingerprint(existing) - if stored is not None and _fingerprints_match(stored, fingerprint): - return False + if existing is not None and not _index_needs_rebuild(existing, fingerprint): + return False CACHE_DIR.mkdir(parents=True, exist_ok=True) new_path = CACHE_DIR / f"search_index.{uuid.uuid4().hex[:12]}.sqlite" @@ -365,8 +366,7 @@ def ensure_search_index(workspace_path: str, rules: list[RuleTokens]) -> None: if conn is None: build_search_index(workspace_path, rules) return - stored = _read_stored_fingerprint(conn) - if stored is None or not _fingerprints_match(stored, fingerprint): + if _index_needs_rebuild(conn, fingerprint): build_search_index(workspace_path, rules) @@ -544,6 +544,17 @@ def query_composer_rows_in_window( return {row["composer_id"]: row for row in rows} +def _index_needs_rebuild( + conn: sqlite3.Connection, + fingerprint: dict[str, Any], +) -> bool: + """True when the stored fingerprint or index version is not current.""" + stored = _read_stored_fingerprint(conn) + if stored is None or not _fingerprints_match(stored, fingerprint): + return True + return _stored_index_version(conn) != str(INDEX_VERSION) + + def _stored_index_version(conn: sqlite3.Connection) -> str | None: try: row = conn.execute( @@ -557,14 +568,11 @@ def _stored_index_version(conn: sqlite3.Connection) -> str | None: def index_is_usable(workspace_path: str, rules: list[RuleTokens]) -> bool: - """True when the on-disk index matches the current Cursor storage fingerprint.""" + """True when the on-disk index matches the storage fingerprint and INDEX_VERSION.""" if not index_search_enabled() or _resolve_active_index_db_path() is None: return False fingerprint = _storage_fingerprint(workspace_path, rules) with _index_db_conn(readonly=True) as conn: if conn is None: return False - stored = _read_stored_fingerprint(conn) - if stored is None or not _fingerprints_match(stored, fingerprint): - return False - return _stored_index_version(conn) == str(INDEX_VERSION) + return not _index_needs_rebuild(conn, fingerprint) diff --git a/services/workspace_tabs.py b/services/workspace_tabs.py index 5394902..8927949 100644 --- a/services/workspace_tabs.py +++ b/services/workspace_tabs.py @@ -10,7 +10,7 @@ _logger = logging.getLogger(__name__) -from utils.conversation_index import load_conv_header_map +from utils.conversation_index import load_conv_header_map, load_conv_headers_for_composer from utils.path_helpers import ( get_workspace_folder_paths, normalize_file_path, @@ -690,7 +690,7 @@ def assemble_single_tab( return {"error": ERR_CONVERSATION_NOT_FOUND, "code": CODE_CONVERSATION_NOT_FOUND}, 404 row = rows[0] - conv_header_map = load_conv_header_map(global_db) + conv_header_map = load_conv_headers_for_composer(global_db, composer_id) composer = parse_composer_data_row( row["key"], row["value"], diff --git a/tests/test_conversation_index.py b/tests/test_conversation_index.py index ab449bc..25feda0 100644 --- a/tests/test_conversation_index.py +++ b/tests/test_conversation_index.py @@ -9,6 +9,7 @@ bubble_id_from_kv_key, composer_data_with_headers, load_conv_header_map, + load_conv_headers_for_composer, ) @@ -59,6 +60,57 @@ def test_chunks_merge_in_suffix_order(self) -> None: ["parent", "parent/s:0"], ) + def test_non_string_bubble_id_is_omitted(self) -> None: + conn = sqlite3.connect(":memory:") + conn.row_factory = sqlite3.Row + conn.execute("CREATE TABLE cursorDiskKV ([key] TEXT PRIMARY KEY, value TEXT)") + composer_id = "cmp-bad-id" + payload = [ + {"bubbleId": "parent", "type": 1}, + {"bubbleId": [], "type": 2}, + {"type": 2}, + ] + conn.execute( + "INSERT INTO cursorDiskKV ([key], value) VALUES (?, ?)", + (f"composerConvHeaders:{composer_id}:000000", json.dumps(payload)), + ) + conn.commit() + + merged = load_conv_header_map(conn) + self.assertEqual( + [item["bubbleId"] for item in merged[composer_id]], + ["parent"], + ) + + def test_scoped_loader_returns_one_composer_in_suffix_order(self) -> None: + conn = sqlite3.connect(":memory:") + conn.row_factory = sqlite3.Row + conn.execute("CREATE TABLE cursorDiskKV ([key] TEXT PRIMARY KEY, value TEXT)") + wanted = "cmp-wanted" + other = "cmp-other" + rows = [ + (f"composerConvHeaders:{wanted}:000001", [{"bubbleId": "second", "type": 2}]), + (f"composerConvHeaders:{wanted}:000000", [{"bubbleId": "first", "type": 1}]), + (f"composerConvHeaders:{other}:000000", [{"bubbleId": "elsewhere", "type": 1}]), + ( + "composerConvHeaders:cmp_wanted:000000", + [{"bubbleId": "underscore-neighbor", "type": 1}], + ), + ] + for key, payload in rows: + conn.execute( + "INSERT INTO cursorDiskKV ([key], value) VALUES (?, ?)", + (key, json.dumps(payload)), + ) + conn.commit() + + scoped = load_conv_headers_for_composer(conn, wanted) + self.assertEqual(list(scoped), [wanted]) + self.assertEqual( + [item["bubbleId"] for item in scoped[wanted]], + ["first", "second"], + ) + def test_inline_headers_win(self) -> None: inline = [{"bubbleId": "old", "type": 1}] resolved = composer_data_with_headers( diff --git a/tests/test_search_index.py b/tests/test_search_index.py index 2fb345b..0b66a6d 100644 --- a/tests/test_search_index.py +++ b/tests/test_search_index.py @@ -120,6 +120,20 @@ def test_index_is_usable_after_build(self, indexed_storage): with patches[0], patches[1], patches[2]: assert index_is_usable(indexed_storage["ws_root"], []) is True + def test_stale_index_version_rebuilds(self, indexed_storage): + patches = _index_patches(indexed_storage["cache_dir"]) + with patches[0], patches[1], patches[2]: + with sqlite3.connect(indexed_storage["index_path"]) as conn: + conn.execute( + "UPDATE index_meta SET value = ? WHERE key = 'version'", + ("1",), + ) + conn.commit() + assert index_is_usable(indexed_storage["ws_root"], []) is False + rebuilt = build_search_index(indexed_storage["ws_root"], [], force=False) + assert rebuilt is True + assert index_is_usable(indexed_storage["ws_root"], []) is True + def test_bubble_fts_finds_term(self, indexed_storage): patches = _index_patches(indexed_storage["cache_dir"]) with patches[0], patches[1], patches[2]: diff --git a/utils/conversation_index.py b/utils/conversation_index.py index 7de0aee..73c2337 100644 --- a/utils/conversation_index.py +++ b/utils/conversation_index.py @@ -59,24 +59,24 @@ def _json_object(value: object) -> object | None: return parsed -def load_conv_header_map( - conn: sqlite3.Connection, -) -> dict[str, list[dict[str, Any]]]: - """Load ``composerConvHeaders:*`` chunks into ``{composer_id: headers}``. +def _string_bubble_headers(parsed: object) -> list[dict[str, Any]] | None: + """Return header dicts whose ``bubbleId`` is a string. - Chunks for one composer are concatenated in suffix order (``000000``, - then ``000001``, …). Malformed rows are skipped. + A list or dict id is unhashable and would raise ``TypeError`` in + ``bubble_map.get``. Non-list rows are rejected so the caller can log them. """ - grouped: dict[str, list[tuple[str, list[dict[str, Any]]]]] = {} - try: - rows = conn.execute( - "SELECT key, value FROM cursorDiskKV" - " WHERE key LIKE 'composerConvHeaders:%'" - ).fetchall() - except sqlite3.Error as exc: - _logger.debug("Could not read composerConvHeaders: %s", exc) - return {} + if not isinstance(parsed, list): + return None + return [ + item + for item in parsed + if isinstance(item, dict) and isinstance(item.get("bubbleId"), str) + ] + +def _merge_header_rows(rows: list[Any]) -> dict[str, list[dict[str, Any]]]: + """Concatenate header chunks in suffix order.""" + grouped: dict[str, list[tuple[str, list[dict[str, Any]]]]] = {} for row in rows: key = row["key"] parts = _conv_header_key_parts(key if isinstance(key, str) else "") @@ -84,10 +84,10 @@ def load_conv_header_map( continue composer_id, chunk = parts parsed = _json_object(row["value"]) - if not isinstance(parsed, list): + headers = _string_bubble_headers(parsed) + if headers is None: _logger.debug("Skipping non-list composerConvHeaders row %s", key) continue - headers = [item for item in parsed if isinstance(item, dict)] grouped.setdefault(composer_id, []).append((chunk, headers)) merged: dict[str, list[dict[str, Any]]] = {} @@ -101,6 +101,70 @@ def load_conv_header_map( return merged +def _conv_header_like_pattern(composer_id: str) -> str: + """Return a ``LIKE`` pattern for one composer's header chunks. + + ``%`` and ``_`` in *composer_id* are escaped for ``ESCAPE '\\'``. + """ + escaped = ( + composer_id + .replace("\\", "\\\\") + .replace("%", "\\%") + .replace("_", "\\_") + ) + return f"{_CONV_HEADER_KEY_PREFIX}{escaped}:%" + + +def load_conv_header_map( + conn: sqlite3.Connection, +) -> dict[str, list[dict[str, Any]]]: + """Load ``composerConvHeaders:*`` chunks into ``{composer_id: headers}``. + + Chunks for one composer are concatenated in suffix order (``000000``, + then ``000001``, …). Malformed rows and entries whose ``bubbleId`` is + not a string are skipped. + """ + try: + rows = conn.execute( + "SELECT key, value FROM cursorDiskKV" + " WHERE key LIKE 'composerConvHeaders:%'" + ).fetchall() + except sqlite3.Error as exc: + _logger.debug("Could not read composerConvHeaders: %s", exc) + return {} + return _merge_header_rows(rows) + + +def load_conv_headers_for_composer( + conn: sqlite3.Connection, + composer_id: str, +) -> dict[str, list[dict[str, Any]]]: + """Load header chunks for one composer. + + Returns ``{composer_id: headers}`` when that composer has string + ``bubbleId`` entries, otherwise ``{}``. Chunks are concatenated in + suffix order. + """ + if not composer_id: + return {} + try: + rows = conn.execute( + "SELECT key, value FROM cursorDiskKV" + " WHERE key LIKE ? ESCAPE '\\'", + (_conv_header_like_pattern(composer_id),), + ).fetchall() + except sqlite3.Error as exc: + _logger.debug( + "Could not read composerConvHeaders for %s: %s", composer_id, exc, + ) + return {} + merged = _merge_header_rows(rows) + headers = merged.get(composer_id) + if not headers: + return {} + return {composer_id: headers} + + def composer_data_with_headers( composer_data: dict[str, Any], composer_id: str, From 2a628e43f257ccc978725bad9a9e763548091a78 Mon Sep 17 00:00:00 2001 From: wpak-ai Date: Wed, 28 Oct 2026 12:07:53 -0400 Subject: [PATCH 3/3] Keep the lock installable on Python 3.10. fonttools 4.66 and later require Python 3.11, so the 3.10 matrix could not install the lock. Cap fonttools below 4.66 and run the Playwright job on Ubuntu 24.04, where Chromium 1.50 still installs. --- .github/workflows/tests.yml | 3 ++- pyproject.toml | 3 +++ requirements-lock.txt | 2 +- requirements.txt | 3 +++ 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4084db3..71ed905 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -153,7 +153,8 @@ jobs: browser-xss: name: Browser XSS (Playwright) needs: [lockfile] - runs-on: ubuntu-latest + # ubuntu-latest is 26.04. Playwright 1.50.0 refuses Chromium there. + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: diff --git a/pyproject.toml b/pyproject.toml index 70563ba..5559016 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,9 @@ requires-python = ">=3.10" dependencies = [ "flask>=3.0,<4", "fpdf2>=2.7,<3", + # fonttools 4.66+ requires Python >=3.11. The single lock must still + # install on the 3.10 matrix jobs. + "fonttools>=4.34,<4.66", # Security floor: fpdf2 allows Pillow>=8.3.2 (no upper cap); pin 12.x to avoid # known high-severity CVEs in Pillow 10.x (e.g. CVE-2024-28219 and later advisories). "pillow>=12.2.0,<13", diff --git a/requirements-lock.txt b/requirements-lock.txt index 9f7c822..86f7bb1 100644 --- a/requirements-lock.txt +++ b/requirements-lock.txt @@ -7,7 +7,7 @@ blinker==1.9.0 # via flask click==8.5.0 # via flask defusedxml==0.7.1 # via fpdf2 flask==3.1.3 # via -r requirements.txt -fonttools==4.68.0 # via fpdf2 +fonttools==4.65.0 # via -r requirements.txt, fpdf2 fpdf2==2.8.9 # via -r requirements.txt itsdangerous==2.2.0 # via flask jinja2==3.1.6 # via flask diff --git a/requirements.txt b/requirements.txt index f08a70a..e56d9d2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,5 +6,8 @@ # pip install -e ".[desktop]" (+ pywebview for the GUI launcher) flask>=3.0,<4 fpdf2>=2.7,<3 +# fonttools 4.66+ requires Python >=3.11. The single lock must still +# install on the 3.10 matrix jobs. +fonttools>=4.34,<4.66 pillow>=12.2.0,<13 # pywebview is desktop-only — install with: pip install -e ".[desktop]"