fix(retrieval): match locator keyword queries to the trigram index

packages/infinity_context_adapters/infinity_context_adapters/postgres/locator_retrieval.py

@@ -13,7 +13,7 @@
     LocatorProviderResult,
     LocatorRetrievalRequest,
 )
-from sqlalchemy import case, cast, func, not_, or_, select, text
+from sqlalchemy import case, cast, not_, or_, select, text
 from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
 
@@ -103,7 +103,7 @@ async def hydrate_final_locator_read(
 def _candidate_statement(request: LocatorRetrievalRequest, query: str):
     terms = tuple(dict.fromkeys(term.casefold() for term in query.split() if term))
     matches = tuple(
-        func.lower(MemoryChunkRow.normalized_text).contains(term, autoescape=True) for term in terms
+        MemoryChunkRow.normalized_text.contains(term, autoescape=True) for term in terms
     )
     relevance = sum((case((match, 1), else_=0) for match in matches), start=0)
     conditions = list(_hard_sql_conditions(request))

tests/adapters/test_locator_parent_lifecycle.py

@@ -23,6 +23,10 @@ def test_parent_lifecycle_and_binding_are_canonical_for_every_locator_read() ->
     asyncio.run(_assert_parent_authority())
 
 
+def test_locator_keyword_matching_preserves_case_unicode_and_literal_wildcards() -> None:
+    asyncio.run(_assert_keyword_matching_semantics())
+
+
 async def _assert_parent_authority() -> None:
     engine = create_async_engine("sqlite+aiosqlite:///:memory:")
     async with engine.begin() as connection:
@@ -63,13 +67,49 @@ async def _assert_parent_authority() -> None:
     await engine.dispose()
 
 
-def _request() -> core.LocatorRetrievalRequest:
+async def _assert_keyword_matching_semantics() -> None:
+    engine = create_async_engine("sqlite+aiosqlite:///:memory:")
+    async with engine.begin() as connection:
+        await connection.run_sync(Base.metadata.create_all)
+    sessions = async_sessionmaker(engine, expire_on_commit=False)
+    cases = (
+        ("mixed-unicode", "Café Δέλτα", "café δέλτα"),
+        ("literal-percent", "Budget 100% complete", "budget 100% complete"),
+        ("literal-underscore", "release_candidate ready", "release_candidate ready"),
+        ("percent-decoy", "Budget 1000 complete", "budget 1000 complete"),
+        ("underscore-decoy", "releaseXcandidate ready", "releasexcandidate ready"),
+    )
+    async with sessions.begin() as session:
+        for ordinal, (name, text, normalized_text) in enumerate(cases):
+            session.add(_document(name))
+            session.add(
+                _chunk(
+                    name,
+                    ordinal,
+                    text=text,
+                    normalized_text=normalized_text,
+                )
+            )
+
+    provider = PostgresLocatorCandidateProvider(sessions)
+    expected = {
+        "CAFÉ ΔΈΛΤΑ": ["chunk-mixed-unicode"],
+        "100%": ["chunk-literal-percent"],
+        "release_candidate": ["chunk-literal-underscore"],
+    }
+    for query, identities in expected.items():
+        result = await provider.retrieve_locator_candidates(_request(query))
+        assert [hit.canonical_identity for hit in result.hits] == identities
+    await engine.dispose()
+
+
+def _request(query: str = "evidence") -> core.LocatorRetrievalRequest:
     return core.LocatorRetrievalRequest(
         "context-retrieval.v2",
         "a" * 64,
         "profile",
         core.LocatorRetrievalScope("space", "scope", "thread"),
-        (core.LocatorQueryVariant("q1", "evidence"),),
+        (core.LocatorQueryVariant("q1", query),),
         core.LocatorHardFilters(
             source_generations=(core.LocatorSourceGeneration("source", "generation"),)
         ),

tests/adapters/test_locator_retrieval_adapters.py

@@ -108,6 +108,24 @@ def test_postgres_array_filters_compile_to_jsonb_containment() -> None:
     assert "memory_chunks.retrieval_actor_keys_json LIKE" not in statement
 
 
+def test_postgres_keyword_match_targets_the_indexed_normalized_column() -> None:
+    compiled = _candidate_statement(
+        _request(), "CAFÉ 100% release_candidate"
+    ).compile(
+        dialect=postgresql.dialect()
+    )
+    statement = str(compiled)
+
+    assert "lower(" not in statement
+    assert statement.count("memory_chunks.normalized_text LIKE") == 9
+    assert [compiled.params[f"normalized_text_{ordinal}"] for ordinal in range(1, 4)] == [
+        "café",
+        "100/%",
+        "release/_candidate",
+    ]
+    assert "ESCAPE '/'" in statement
+
+
 def test_qdrant_provider_preserves_raw_score_rank_and_version() -> None:
     search = _Search()
     result = asyncio.run(

tests/adapters/test_postgres_canonical_keyword_trigram.py

@@ -10,6 +10,7 @@
 from hashlib import sha256
 from pathlib import Path
 
+import infinity_context_core.features.context_building.public as core
 import pytest
 from infinity_context_adapters.postgres import create_schema
 from infinity_context_adapters.postgres.canonical_keyword_trigram import (
@@ -21,6 +22,7 @@
     _keyword_batch_statement,
     _keyword_fragments,
 )
+from infinity_context_adapters.postgres.locator_retrieval import _candidate_statement
 from infinity_context_adapters.postgres.models import MemoryChunkRow, MemoryDocumentRow
 from infinity_context_adapters.postgres.repositories import (
     PostgresChunkRepository,
@@ -40,6 +42,14 @@
 )
 MIGRATION = _MIGRATIONS / "0022_canonical_keyword_trigram.sql"
 _LOGGER = logging.getLogger(__name__)
+_LOCATOR_SEMANTIC_IDS = (
+    "locator-unicode-all",
+    "locator-literal-percent",
+    "locator-literal-underscore",
+    "locator-percent-decoy",
+    "locator-underscore-decoy",
+    "locator-no-match-decoy",
+)
 
 
 def test_migration_and_runtime_installer_share_the_partial_trigram_contract() -> None:
@@ -121,6 +131,53 @@ async def _assert_real_postgres_access_path(database_url: str) -> None:
             await session.commit()
         await create_schema(engine)
         async with AsyncSession(engine, expire_on_commit=False) as session:
+            # Build a locator-shaped corpus without emitting profile events. This is a
+            # disposable planner fixture, not a production write-path substitute.
+            await session.execute(text("ALTER TABLE memory_documents DISABLE TRIGGER USER"))
+            await session.execute(text("ALTER TABLE memory_chunks DISABLE TRIGGER USER"))
+            await session.execute(
+                text(
+                    "UPDATE memory_documents SET retrieval_projected = TRUE "
+                    "WHERE id = 'filler-document' OR id LIKE 'document-locator-%'"
+                )
+            )
+            await session.execute(
+                text(
+                    """
+                    UPDATE memory_chunks
+                    SET source_external_id = 'filler-source',
+                        normalized_text = CASE WHEN id = 'filler-1'
+                            THEN 'unique locator needle evidence'
+                            ELSE normalized_text END,
+                        retrieval_locator = id,
+                        retrieval_source_key = 'source',
+                        retrieval_projection_generation = 'generation',
+                        retrieval_sequence_ordinal = sequence,
+                        retrieval_kind = 'document',
+                        retrieval_category = 'document'
+                    WHERE document_id = 'filler-document'
+                    """
+                )
+            )
+            await session.execute(
+                text(
+                    """
+                    UPDATE memory_chunks
+                    SET retrieval_locator = id,
+                        retrieval_source_key = 'semantic-source',
+                        retrieval_projection_generation = 'generation',
+                        retrieval_sequence_ordinal = sequence,
+                        retrieval_kind = 'document',
+                        retrieval_category = 'document'
+                    WHERE id = ANY(:fixture_ids)
+                    """
+                ),
+                {"fixture_ids": list(_LOCATOR_SEMANTIC_IDS)},
+            )
+            await session.execute(text("ALTER TABLE memory_chunks ENABLE TRIGGER USER"))
+            await session.execute(text("ALTER TABLE memory_documents ENABLE TRIGGER USER"))
+            await session.commit()
+            await session.execute(text("ANALYZE memory_chunks"))
             corpus_size = int(
                 (await session.execute(text("SELECT count(*) FROM memory_chunks"))).scalar_one()
             )
@@ -155,6 +212,8 @@ async def _assert_real_postgres_access_path(database_url: str) -> None:
                 "turn-thread-a",
             ]
 
+            await _assert_locator_semantic_parity(session)
+
             index_definition = (
                 await session.execute(
                     text(
@@ -232,6 +291,46 @@ async def _assert_real_postgres_access_path(database_url: str) -> None:
             assert CANONICAL_KEYWORD_TRIGRAM_INDEX in _plan_index_names(after_batch)
             assert "Bitmap Index Scan" in _plan_node_types(after_scalar)
             assert "Bitmap Index Scan" in _plan_node_types(after_batch)
+
+            locator_request = core.LocatorRetrievalRequest(
+                "context-retrieval.v2",
+                "a" * 64,
+                "profile",
+                core.LocatorRetrievalScope("space-a", "scope-a", None, "any"),
+                (core.LocatorQueryVariant("q1", "NEEDLE"),),
+                core.LocatorHardFilters(
+                    source_generations=(
+                        core.LocatorSourceGeneration("source", "generation"),
+                    )
+                ),
+                core.LocatorSoftPreferences(),
+                core.LocatorRetrievalBounds(candidate_limit=10, result_limit=5),
+            )
+            locator_sql = _literal_postgres_sql(
+                _candidate_statement(locator_request, "NEEDLE").limit(10)
+            )
+            assert "lower(" not in locator_sql
+            await session.execute(text("SET LOCAL enable_seqscan = off"))
+            locator_plan = await _explain(session, locator_sql)
+            locator_index_conditions = _named_bitmap_index_conditions(
+                locator_plan,
+                CANONICAL_KEYWORD_TRIGRAM_INDEX,
+            )
+            assert len(locator_index_conditions) == 1
+            assert "normalized_text" in locator_index_conditions[0]
+            assert "~~" in locator_index_conditions[0]
+            assert "needle" in locator_index_conditions[0]
+
+            old_locator_sql = locator_sql.replace(
+                "memory_chunks.normalized_text LIKE",
+                "lower(memory_chunks.normalized_text) LIKE",
+            )
+            assert "lower(memory_chunks.normalized_text) LIKE" in old_locator_sql
+            old_locator_plan = await _explain(session, old_locator_sql)
+            assert not _named_bitmap_index_conditions(
+                old_locator_plan,
+                CANONICAL_KEYWORD_TRIGRAM_INDEX,
+            )
             _LOGGER.info(
                 "canonical keyword plan evidence: %s",
                 {
@@ -384,7 +483,7 @@ def _semantic_rows(now: datetime) -> list[MemoryChunkRow]:
         ("restricted", "scope-a", None, "active", "restricted", 0, "manual", "turn-private"),
         ("deleted", "scope-a", None, "deleted", "internal", 0, "manual", "turn-deleted"),
     )
-    return [
+    ordinary_rows = [
         MemoryChunkRow(
             id=item_id,
             space_id="space-a",
@@ -419,6 +518,97 @@ def _semantic_rows(now: datetime) -> list[MemoryChunkRow]:
             source_external_id,
         ) in values
     ]
+    locator_values = (
+        ("locator-unicode-all", "café 100% release_candidate", 70_001),
+        ("locator-literal-percent", "café 100% candidate", 70_002),
+        ("locator-literal-underscore", "café release_candidate candidate", 70_003),
+        ("locator-percent-decoy", "café 1000 candidate", 70_004),
+        ("locator-underscore-decoy", "café releasexcandidate", 70_005),
+        ("locator-no-match-decoy", "cafe 1000 releasexcandidate", 70_006),
+    )
+    return ordinary_rows + [
+        MemoryChunkRow(
+            id=item_id,
+            space_id="space-a",
+            memory_scope_id="scope-a",
+            thread_id=None,
+            document_id=f"document-{item_id}",
+            episode_id=None,
+            source_type="manual",
+            source_external_id=f"source-{item_id}",
+            source_hash=f"hash-{item_id}",
+            kind="document_section",
+            text=normalized_text,
+            normalized_text=normalized_text,
+            status="active",
+            sequence=sequence,
+            char_start=0,
+            char_end=len(normalized_text),
+            token_estimate=4,
+            classification="internal",
+            created_at=now + timedelta(seconds=sequence),
+            updated_at=now + timedelta(seconds=sequence),
+            metadata_json={"source_identity": f"source-{item_id}"},
+        )
+        for item_id, normalized_text, sequence in locator_values
+    ]
+
+
+async def _assert_locator_semantic_parity(session: AsyncSession) -> None:
+    request = core.LocatorRetrievalRequest(
+        "context-retrieval.v2",
+        "b" * 64,
+        "profile",
+        core.LocatorRetrievalScope("space-a", "scope-a", None, "any"),
+        (core.LocatorQueryVariant("semantic-query", "CAFÉ 100% RELEASE_CANDIDATE"),),
+        core.LocatorHardFilters(
+            source_generations=(
+                core.LocatorSourceGeneration("semantic-source", "generation"),
+            )
+        ),
+        core.LocatorSoftPreferences(),
+        core.LocatorRetrievalBounds(candidate_limit=10, result_limit=10),
+    )
+    statement = _candidate_statement(request, request.queries[0].query).limit(10)
+    raw_rows = [
+        (str(row.id), int(row.relevance))
+        for row in (await session.execute(statement)).all()
+    ]
+    expected = [
+        ("locator-unicode-all", 3),
+        ("locator-literal-percent", 2),
+        ("locator-literal-underscore", 2),
+        ("locator-percent-decoy", 1),
+        ("locator-underscore-decoy", 1),
+    ]
+    assert raw_rows == expected
+    assert [
+        (str(row.id), int(row.relevance))
+        for row in (await session.execute(statement)).all()
+    ] == expected
+
+    lowercase_mismatches = (
+        await session.execute(
+            text(
+                "SELECT id FROM memory_chunks "
+                "WHERE id = ANY(:fixture_ids) AND lower(normalized_text) <> normalized_text"
+            ),
+            {"fixture_ids": list(_LOCATOR_SEMANTIC_IDS)},
+        )
+    ).all()
+    assert lowercase_mismatches == []
+
+    raw_sql = _literal_postgres_sql(statement)
+    old_lower_sql = raw_sql.replace(
+        "memory_chunks.normalized_text LIKE",
+        "lower(memory_chunks.normalized_text) LIKE",
+    )
+    assert "lower(memory_chunks.normalized_text) LIKE" in old_lower_sql
+    old_lower_rows = [
+        (str(row.id), int(row.relevance))
+        for row in (await session.execute(text(old_lower_sql))).all()
+    ]
+    assert old_lower_rows == raw_rows
 
 
 def _document_rows(
@@ -468,7 +658,7 @@ def _normalize_sql(value: str) -> str:
 def _literal_postgres_sql(statement) -> str:
     compiled = str(
         statement.compile(
-            dialect=postgresql.dialect(),
+            dialect=postgresql.dialect(paramstyle="named"),
             compile_kwargs={"literal_binds": True},
         )
     )
@@ -483,6 +673,10 @@ async def _explain_analyze(session: AsyncSession, sql: str):
     ).scalar_one()
 
 
+async def _explain(session: AsyncSession, sql: str):
+    return (await session.execute(text(f"EXPLAIN (FORMAT JSON) {sql}"))).scalar_one()
+
+
 def _plan_index_names(plan) -> set[str]:
     return {str(node["Index Name"]) for node in _plan_nodes(plan) if "Index Name" in node}
 
@@ -491,6 +685,16 @@ def _plan_node_types(plan) -> set[str]:
     return {str(node["Node Type"]) for node in _plan_nodes(plan) if "Node Type" in node}
 
 
+def _named_bitmap_index_conditions(plan, index_name: str) -> tuple[str, ...]:
+    return tuple(
+        str(node["Index Cond"])
+        for node in _plan_nodes(plan)
+        if node.get("Node Type") == "Bitmap Index Scan"
+        and node.get("Index Name") == index_name
+        and "Index Cond" in node
+    )
+
+
 def _plan_summary(plan) -> dict[str, object]:
     root = plan[0]["Plan"]
     return {