From 121d09059be0e43a174763c80e5427e582f75379 Mon Sep 17 00:00:00 2001 From: iliya Date: Sat, 12 Sep 2026 17:31:50 +0000 Subject: [PATCH 1/3] fix(retrieval): checkpoint keyword index patch for test correction --- .../postgres/locator_retrieval.py | 4 +- .../adapters/test_locator_parent_lifecycle.py | 44 ++++++++++++- .../test_locator_retrieval_adapters.py | 15 +++++ ...test_postgres_canonical_keyword_trigram.py | 61 +++++++++++++++++++ .../test_projected_document_rendering.py | 4 +- 5 files changed, 122 insertions(+), 6 deletions(-) diff --git a/packages/infinity_context_adapters/infinity_context_adapters/postgres/locator_retrieval.py b/packages/infinity_context_adapters/infinity_context_adapters/postgres/locator_retrieval.py index e07b7478..80ce2b6f 100644 --- a/packages/infinity_context_adapters/infinity_context_adapters/postgres/locator_retrieval.py +++ b/packages/infinity_context_adapters/infinity_context_adapters/postgres/locator_retrieval.py @@ -13,7 +13,7 @@ LocatorProviderResult, LocatorRetrievalRequest, ) -from sqlalchemy import case, cast, func, not_, or_, select, text +from sqlalchemy import case, cast, not_, or_, select, text from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker @@ -103,7 +103,7 @@ async def hydrate_final_locator_read( def _candidate_statement(request: LocatorRetrievalRequest, query: str): terms = tuple(dict.fromkeys(term.casefold() for term in query.split() if term)) matches = tuple( - func.lower(MemoryChunkRow.normalized_text).contains(term, autoescape=True) for term in terms + MemoryChunkRow.normalized_text.contains(term, autoescape=True) for term in terms ) relevance = sum((case((match, 1), else_=0) for match in matches), start=0) conditions = list(_hard_sql_conditions(request)) diff --git a/tests/adapters/test_locator_parent_lifecycle.py b/tests/adapters/test_locator_parent_lifecycle.py index 09c7c01c..6dd0cb36 100644 --- a/tests/adapters/test_locator_parent_lifecycle.py +++ b/tests/adapters/test_locator_parent_lifecycle.py @@ -23,6 +23,10 @@ def test_parent_lifecycle_and_binding_are_canonical_for_every_locator_read() -> asyncio.run(_assert_parent_authority()) +def test_locator_keyword_matching_preserves_case_unicode_and_literal_wildcards() -> None: + asyncio.run(_assert_keyword_matching_semantics()) + + async def _assert_parent_authority() -> None: engine = create_async_engine("sqlite+aiosqlite:///:memory:") async with engine.begin() as connection: @@ -63,13 +67,49 @@ async def _assert_parent_authority() -> None: await engine.dispose() -def _request() -> core.LocatorRetrievalRequest: +async def _assert_keyword_matching_semantics() -> None: + engine = create_async_engine("sqlite+aiosqlite:///:memory:") + async with engine.begin() as connection: + await connection.run_sync(Base.metadata.create_all) + sessions = async_sessionmaker(engine, expire_on_commit=False) + cases = ( + ("mixed-unicode", "Café Δέλτα", "café δέλτα"), + ("literal-percent", "Budget 100% complete", "budget 100% complete"), + ("literal-underscore", "release_candidate ready", "release_candidate ready"), + ("percent-decoy", "Budget 1000 complete", "budget 1000 complete"), + ("underscore-decoy", "releaseXcandidate ready", "releasexcandidate ready"), + ) + async with sessions.begin() as session: + for ordinal, (name, text, normalized_text) in enumerate(cases): + session.add(_document(name)) + session.add( + _chunk( + name, + ordinal, + text=text, + normalized_text=normalized_text, + ) + ) + + provider = PostgresLocatorCandidateProvider(sessions) + expected = { + "CAFÉ ΔΈΛΤΑ": ["chunk-mixed-unicode"], + "100%": ["chunk-literal-percent"], + "release_candidate": ["chunk-literal-underscore"], + } + for query, identities in expected.items(): + result = await provider.retrieve_locator_candidates(_request(query)) + assert [hit.canonical_identity for hit in result.hits] == identities + await engine.dispose() + + +def _request(query: str = "evidence") -> core.LocatorRetrievalRequest: return core.LocatorRetrievalRequest( "context-retrieval.v2", "a" * 64, "profile", core.LocatorRetrievalScope("space", "scope", "thread"), - (core.LocatorQueryVariant("q1", "evidence"),), + (core.LocatorQueryVariant("q1", query),), core.LocatorHardFilters( source_generations=(core.LocatorSourceGeneration("source", "generation"),) ), diff --git a/tests/adapters/test_locator_retrieval_adapters.py b/tests/adapters/test_locator_retrieval_adapters.py index 813dc56d..657fac60 100644 --- a/tests/adapters/test_locator_retrieval_adapters.py +++ b/tests/adapters/test_locator_retrieval_adapters.py @@ -108,6 +108,21 @@ def test_postgres_array_filters_compile_to_jsonb_containment() -> None: assert "memory_chunks.retrieval_actor_keys_json LIKE" not in statement +def test_postgres_keyword_match_targets_the_indexed_normalized_column() -> None: + statement = str( + _candidate_statement(_request(), "CAFÉ 100% release_candidate").compile( + dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True} + ) + ) + + assert "lower(" not in statement + assert statement.count("memory_chunks.normalized_text LIKE") == 6 + assert "café" in statement + assert "100/%" in statement + assert "release/_candidate" in statement + assert "ESCAPE '/'" in statement + + def test_qdrant_provider_preserves_raw_score_rank_and_version() -> None: search = _Search() result = asyncio.run( diff --git a/tests/adapters/test_postgres_canonical_keyword_trigram.py b/tests/adapters/test_postgres_canonical_keyword_trigram.py index 6fabc3e5..bb8fa6d3 100644 --- a/tests/adapters/test_postgres_canonical_keyword_trigram.py +++ b/tests/adapters/test_postgres_canonical_keyword_trigram.py @@ -10,6 +10,7 @@ from hashlib import sha256 from pathlib import Path +import infinity_context_core.features.context_building.public as core import pytest from infinity_context_adapters.postgres import create_schema from infinity_context_adapters.postgres.canonical_keyword_trigram import ( @@ -21,6 +22,7 @@ _keyword_batch_statement, _keyword_fragments, ) +from infinity_context_adapters.postgres.locator_retrieval import _candidate_statement from infinity_context_adapters.postgres.models import MemoryChunkRow, MemoryDocumentRow from infinity_context_adapters.postgres.repositories import ( PostgresChunkRepository, @@ -121,6 +123,38 @@ async def _assert_real_postgres_access_path(database_url: str) -> None: await session.commit() await create_schema(engine) async with AsyncSession(engine, expire_on_commit=False) as session: + # Build a locator-shaped corpus without emitting profile events. This is a + # disposable planner fixture, not a production write-path substitute. + await session.execute(text("ALTER TABLE memory_documents DISABLE TRIGGER USER")) + await session.execute(text("ALTER TABLE memory_chunks DISABLE TRIGGER USER")) + await session.execute( + text( + "UPDATE memory_documents SET retrieval_projected = TRUE " + "WHERE id = 'filler-document'" + ) + ) + await session.execute( + text( + """ + UPDATE memory_chunks + SET source_external_id = 'filler-source', + normalized_text = CASE WHEN id = 'filler-1' + THEN 'unique locator needle evidence' + ELSE normalized_text END, + retrieval_locator = id, + retrieval_source_key = 'source', + retrieval_projection_generation = 'generation', + retrieval_sequence_ordinal = sequence, + retrieval_kind = 'document', + retrieval_category = 'document' + WHERE document_id = 'filler-document' + """ + ) + ) + await session.execute(text("ALTER TABLE memory_chunks ENABLE TRIGGER USER")) + await session.execute(text("ALTER TABLE memory_documents ENABLE TRIGGER USER")) + await session.commit() + await session.execute(text("ANALYZE memory_chunks")) corpus_size = int( (await session.execute(text("SELECT count(*) FROM memory_chunks"))).scalar_one() ) @@ -232,6 +266,29 @@ async def _assert_real_postgres_access_path(database_url: str) -> None: assert CANONICAL_KEYWORD_TRIGRAM_INDEX in _plan_index_names(after_batch) assert "Bitmap Index Scan" in _plan_node_types(after_scalar) assert "Bitmap Index Scan" in _plan_node_types(after_batch) + + locator_request = core.LocatorRetrievalRequest( + "context-retrieval.v2", + "a" * 64, + "profile", + core.LocatorRetrievalScope("space-a", "scope-a", None, "any"), + (core.LocatorQueryVariant("q1", "NEEDLE"),), + core.LocatorHardFilters( + source_generations=( + core.LocatorSourceGeneration("source", "generation"), + ) + ), + core.LocatorSoftPreferences(), + core.LocatorRetrievalBounds(candidate_limit=10, result_limit=5), + ) + locator_sql = _literal_postgres_sql( + _candidate_statement(locator_request, "NEEDLE").limit(10) + ) + assert "lower(" not in locator_sql + await session.execute(text("SET LOCAL enable_seqscan = off")) + locator_plan = await _explain(session, locator_sql) + assert CANONICAL_KEYWORD_TRIGRAM_INDEX in _plan_index_names(locator_plan) + assert "Bitmap Index Scan" in _plan_node_types(locator_plan) _LOGGER.info( "canonical keyword plan evidence: %s", { @@ -483,6 +540,10 @@ async def _explain_analyze(session: AsyncSession, sql: str): ).scalar_one() +async def _explain(session: AsyncSession, sql: str): + return (await session.execute(text(f"EXPLAIN (FORMAT JSON) {sql}"))).scalar_one() + + def _plan_index_names(plan) -> set[str]: return {str(node["Index Name"]) for node in _plan_nodes(plan) if "Index Name" in node} diff --git a/tests/adapters/test_projected_document_rendering.py b/tests/adapters/test_projected_document_rendering.py index 43eace50..20a1742d 100644 --- a/tests/adapters/test_projected_document_rendering.py +++ b/tests/adapters/test_projected_document_rendering.py @@ -38,7 +38,7 @@ def test_projected_ingestion_keeps_source_refs_out_of_canonical_embedding_text() title = "mkevidence1." + base64.urlsafe_b64encode( hashlib.sha256(b"synthetic-render-audit").digest() ).decode().rstrip("=") - body = "hello " * 33 + body = "CAFÉ ΔΈΛΤΑ 100% release_candidate " * 8 source_refs = [ { "source_type": "transcript", @@ -87,7 +87,7 @@ def test_projected_ingestion_keeps_source_refs_out_of_canonical_embedding_text() ) ) - expected = title.lower() + " " + body.strip() + expected = (title + " " + body.strip()).lower() chunk = result.chunks[0] row = next(row for row in session.rows if isinstance(row, MemoryChunkRow)) # Database defaults are assigned explicitly in this provider-free row fixture. From 7ca92896563d57dd3dec7ec7514d78a85ebf8677 Mon Sep 17 00:00:00 2001 From: iliya Date: Sat, 12 Sep 2026 17:41:32 +0000 Subject: [PATCH 2/3] test(retrieval): assert escaped keyword binds without inlining JSONB --- .../test_locator_retrieval_adapters.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/adapters/test_locator_retrieval_adapters.py b/tests/adapters/test_locator_retrieval_adapters.py index 657fac60..24f5744e 100644 --- a/tests/adapters/test_locator_retrieval_adapters.py +++ b/tests/adapters/test_locator_retrieval_adapters.py @@ -109,17 +109,20 @@ def test_postgres_array_filters_compile_to_jsonb_containment() -> None: def test_postgres_keyword_match_targets_the_indexed_normalized_column() -> None: - statement = str( - _candidate_statement(_request(), "CAFÉ 100% release_candidate").compile( - dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True} - ) + compiled = _candidate_statement( + _request(), "CAFÉ 100% release_candidate" + ).compile( + dialect=postgresql.dialect() ) + statement = str(compiled) assert "lower(" not in statement - assert statement.count("memory_chunks.normalized_text LIKE") == 6 - assert "café" in statement - assert "100/%" in statement - assert "release/_candidate" in statement + assert statement.count("memory_chunks.normalized_text LIKE") == 9 + assert [compiled.params[f"normalized_text_{ordinal}"] for ordinal in range(1, 4)] == [ + "café", + "100/%", + "release/_candidate", + ] assert "ESCAPE '/'" in statement From f95d5cca4470430f76ccafa5c927e572cb206045 Mon Sep 17 00:00:00 2001 From: iliya Date: Sat, 12 Sep 2026 18:25:13 +0000 Subject: [PATCH 3/3] test(retrieval): prove PostgreSQL locator parity and index conditions --- ...test_postgres_canonical_keyword_trigram.py | 153 +++++++++++++++++- 1 file changed, 148 insertions(+), 5 deletions(-) diff --git a/tests/adapters/test_postgres_canonical_keyword_trigram.py b/tests/adapters/test_postgres_canonical_keyword_trigram.py index bb8fa6d3..56e8f3ae 100644 --- a/tests/adapters/test_postgres_canonical_keyword_trigram.py +++ b/tests/adapters/test_postgres_canonical_keyword_trigram.py @@ -42,6 +42,14 @@ ) MIGRATION = _MIGRATIONS / "0022_canonical_keyword_trigram.sql" _LOGGER = logging.getLogger(__name__) +_LOCATOR_SEMANTIC_IDS = ( + "locator-unicode-all", + "locator-literal-percent", + "locator-literal-underscore", + "locator-percent-decoy", + "locator-underscore-decoy", + "locator-no-match-decoy", +) def test_migration_and_runtime_installer_share_the_partial_trigram_contract() -> None: @@ -130,7 +138,7 @@ async def _assert_real_postgres_access_path(database_url: str) -> None: await session.execute( text( "UPDATE memory_documents SET retrieval_projected = TRUE " - "WHERE id = 'filler-document'" + "WHERE id = 'filler-document' OR id LIKE 'document-locator-%'" ) ) await session.execute( @@ -151,6 +159,21 @@ async def _assert_real_postgres_access_path(database_url: str) -> None: """ ) ) + await session.execute( + text( + """ + UPDATE memory_chunks + SET retrieval_locator = id, + retrieval_source_key = 'semantic-source', + retrieval_projection_generation = 'generation', + retrieval_sequence_ordinal = sequence, + retrieval_kind = 'document', + retrieval_category = 'document' + WHERE id = ANY(:fixture_ids) + """ + ), + {"fixture_ids": list(_LOCATOR_SEMANTIC_IDS)}, + ) await session.execute(text("ALTER TABLE memory_chunks ENABLE TRIGGER USER")) await session.execute(text("ALTER TABLE memory_documents ENABLE TRIGGER USER")) await session.commit() @@ -189,6 +212,8 @@ async def _assert_real_postgres_access_path(database_url: str) -> None: "turn-thread-a", ] + await _assert_locator_semantic_parity(session) + index_definition = ( await session.execute( text( @@ -287,8 +312,25 @@ async def _assert_real_postgres_access_path(database_url: str) -> None: assert "lower(" not in locator_sql await session.execute(text("SET LOCAL enable_seqscan = off")) locator_plan = await _explain(session, locator_sql) - assert CANONICAL_KEYWORD_TRIGRAM_INDEX in _plan_index_names(locator_plan) - assert "Bitmap Index Scan" in _plan_node_types(locator_plan) + locator_index_conditions = _named_bitmap_index_conditions( + locator_plan, + CANONICAL_KEYWORD_TRIGRAM_INDEX, + ) + assert len(locator_index_conditions) == 1 + assert "normalized_text" in locator_index_conditions[0] + assert "~~" in locator_index_conditions[0] + assert "needle" in locator_index_conditions[0] + + old_locator_sql = locator_sql.replace( + "memory_chunks.normalized_text LIKE", + "lower(memory_chunks.normalized_text) LIKE", + ) + assert "lower(memory_chunks.normalized_text) LIKE" in old_locator_sql + old_locator_plan = await _explain(session, old_locator_sql) + assert not _named_bitmap_index_conditions( + old_locator_plan, + CANONICAL_KEYWORD_TRIGRAM_INDEX, + ) _LOGGER.info( "canonical keyword plan evidence: %s", { @@ -441,7 +483,7 @@ def _semantic_rows(now: datetime) -> list[MemoryChunkRow]: ("restricted", "scope-a", None, "active", "restricted", 0, "manual", "turn-private"), ("deleted", "scope-a", None, "deleted", "internal", 0, "manual", "turn-deleted"), ) - return [ + ordinary_rows = [ MemoryChunkRow( id=item_id, space_id="space-a", @@ -476,6 +518,97 @@ def _semantic_rows(now: datetime) -> list[MemoryChunkRow]: source_external_id, ) in values ] + locator_values = ( + ("locator-unicode-all", "café 100% release_candidate", 70_001), + ("locator-literal-percent", "café 100% candidate", 70_002), + ("locator-literal-underscore", "café release_candidate candidate", 70_003), + ("locator-percent-decoy", "café 1000 candidate", 70_004), + ("locator-underscore-decoy", "café releasexcandidate", 70_005), + ("locator-no-match-decoy", "cafe 1000 releasexcandidate", 70_006), + ) + return ordinary_rows + [ + MemoryChunkRow( + id=item_id, + space_id="space-a", + memory_scope_id="scope-a", + thread_id=None, + document_id=f"document-{item_id}", + episode_id=None, + source_type="manual", + source_external_id=f"source-{item_id}", + source_hash=f"hash-{item_id}", + kind="document_section", + text=normalized_text, + normalized_text=normalized_text, + status="active", + sequence=sequence, + char_start=0, + char_end=len(normalized_text), + token_estimate=4, + classification="internal", + created_at=now + timedelta(seconds=sequence), + updated_at=now + timedelta(seconds=sequence), + metadata_json={"source_identity": f"source-{item_id}"}, + ) + for item_id, normalized_text, sequence in locator_values + ] + + +async def _assert_locator_semantic_parity(session: AsyncSession) -> None: + request = core.LocatorRetrievalRequest( + "context-retrieval.v2", + "b" * 64, + "profile", + core.LocatorRetrievalScope("space-a", "scope-a", None, "any"), + (core.LocatorQueryVariant("semantic-query", "CAFÉ 100% RELEASE_CANDIDATE"),), + core.LocatorHardFilters( + source_generations=( + core.LocatorSourceGeneration("semantic-source", "generation"), + ) + ), + core.LocatorSoftPreferences(), + core.LocatorRetrievalBounds(candidate_limit=10, result_limit=10), + ) + statement = _candidate_statement(request, request.queries[0].query).limit(10) + raw_rows = [ + (str(row.id), int(row.relevance)) + for row in (await session.execute(statement)).all() + ] + expected = [ + ("locator-unicode-all", 3), + ("locator-literal-percent", 2), + ("locator-literal-underscore", 2), + ("locator-percent-decoy", 1), + ("locator-underscore-decoy", 1), + ] + assert raw_rows == expected + assert [ + (str(row.id), int(row.relevance)) + for row in (await session.execute(statement)).all() + ] == expected + + lowercase_mismatches = ( + await session.execute( + text( + "SELECT id FROM memory_chunks " + "WHERE id = ANY(:fixture_ids) AND lower(normalized_text) <> normalized_text" + ), + {"fixture_ids": list(_LOCATOR_SEMANTIC_IDS)}, + ) + ).all() + assert lowercase_mismatches == [] + + raw_sql = _literal_postgres_sql(statement) + old_lower_sql = raw_sql.replace( + "memory_chunks.normalized_text LIKE", + "lower(memory_chunks.normalized_text) LIKE", + ) + assert "lower(memory_chunks.normalized_text) LIKE" in old_lower_sql + old_lower_rows = [ + (str(row.id), int(row.relevance)) + for row in (await session.execute(text(old_lower_sql))).all() + ] + assert old_lower_rows == raw_rows def _document_rows( @@ -525,7 +658,7 @@ def _normalize_sql(value: str) -> str: def _literal_postgres_sql(statement) -> str: compiled = str( statement.compile( - dialect=postgresql.dialect(), + dialect=postgresql.dialect(paramstyle="named"), compile_kwargs={"literal_binds": True}, ) ) @@ -552,6 +685,16 @@ def _plan_node_types(plan) -> set[str]: return {str(node["Node Type"]) for node in _plan_nodes(plan) if "Node Type" in node} +def _named_bitmap_index_conditions(plan, index_name: str) -> tuple[str, ...]: + return tuple( + str(node["Index Cond"]) + for node in _plan_nodes(plan) + if node.get("Node Type") == "Bitmap Index Scan" + and node.get("Index Name") == index_name + and "Index Cond" in node + ) + + def _plan_summary(plan) -> dict[str, object]: root = plan[0]["Plan"] return {