From 0e021419320b8d5a536d5bdbeac835c026b50211 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Sat, 21 Nov 2026 02:31:43 +0000 Subject: [PATCH] Skip candidate indexes for stored entity mappings Signed-off-by: zlareb1 --- server/memory/app/entity_resolution.py | 7 ++- tests/unit/test_entity_resolution.py | 86 ++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 3 deletions(-) diff --git a/server/memory/app/entity_resolution.py b/server/memory/app/entity_resolution.py index 7dde1d7df7..03919c8ee6 100644 --- a/server/memory/app/entity_resolution.py +++ b/server/memory/app/entity_resolution.py @@ -418,9 +418,8 @@ async def resolve_entities_for_write( self_evidence[m_text] = max( self_evidence.get(m_text, 0), int(row.get("evidence_count") or 0) ) - # The rows are normalized once; ranking still receives the same sorted, - # distinct names and uses the same length and character-overlap indexes. - prepared_candidates = _prepare_canonical_candidates(candidates) + # Existing mappings need no candidate indexes. Prepare once for the first unseen mention. + prepared_candidates: PreparedCandidates | None = None decisions: list[EntityResolutionDecision] = [] redirects: dict[str, str] = {} # flipped canonical -> new canonical @@ -447,6 +446,8 @@ async def resolve_entities_for_write( ) ) continue + if prepared_candidates is None: + prepared_candidates = _prepare_canonical_candidates(candidates) decision = choose_canonical_candidate(mention, prepared_candidates) if decision.strategy == "fuzzy" and self_evidence.get(decision.canonical_text, 0) < 1: # The matched canonical was never directly observed as a diff --git a/tests/unit/test_entity_resolution.py b/tests/unit/test_entity_resolution.py index b3001841fb..debaeb4157 100644 --- a/tests/unit/test_entity_resolution.py +++ b/tests/unit/test_entity_resolution.py @@ -317,6 +317,7 @@ async def test_normalized_map_preserves_duplicate_rows_and_attestation( ) -> None: from copy import deepcopy from datetime import datetime + from unittest.mock import Mock from server.core.ch_writes import committed_insert_settings from server.memory.app import entity_resolution as owner @@ -366,6 +367,8 @@ async def test_normalized_map_preserves_duplicate_rows_and_attestation( before = deepcopy(rows) ch = StubCH() ch.query_all_returns.append(rows) + prepare = Mock(wraps=owner._prepare_canonical_candidates) + monkeypatch.setattr(owner, "_prepare_canonical_candidates", prepare) monkeypatch.setattr(owner, "naive_utc_now", lambda: now) resolved = await owner.resolve_entities_for_write( ch, org="clickhouse", namespace="private-agent", entity_texts=["Naomi Alderman"] @@ -404,6 +407,7 @@ async def test_normalized_map_preserves_duplicate_rows_and_attestation( for born in (created, duplicate_created) ) assert ch.insert_calls == [("loom.entity_canonical_map", expected, committed_insert_settings())] + assert prepare.call_count == 0 assert rows == before assert ch.query_all_calls[0][1] == {"org": "clickhouse", "ns": "private-agent", "lim": 5000} fuzzy = owner.choose_canonical_candidate("Northwind AI", ["northwindai", "unrelated product"]) @@ -636,6 +640,7 @@ def counted(value): async def test_prepared_scoring_preserves_complete_resolver_rows(monkeypatch, self_evidence): from copy import deepcopy from datetime import datetime + from unittest.mock import Mock from server.core.ch_writes import committed_insert_settings from server.memory.app import entity_resolution as owner @@ -661,9 +666,12 @@ async def test_prepared_scoring_preserves_complete_resolver_rows(monkeypatch, se actual, reference = StubCH(), StubCH() actual.query_all_returns.append(rows) reference.query_all_returns.append(deepcopy(rows)) + prepare = Mock(wraps=owner._prepare_canonical_candidates) + monkeypatch.setattr(owner, "_prepare_canonical_candidates", prepare) monkeypatch.setattr(owner, "naive_utc_now", lambda: now) args = dict(org="clickhouse", namespace="private-agent", entity_texts=["Northwind AI"]) resolved = await owner.resolve_entities_for_write(actual, **args) + assert prepare.call_count == 1 with monkeypatch.context() as control: control.setattr(owner, "choose_canonical_candidate", _frozen_raw_entity_decision) control.setattr(owner, "entity_similarity", _frozen_raw_entity_similarity) @@ -802,3 +810,81 @@ def observed_score(left, right, compact_left, compact_right): ) if gap != "above": assert pairs == [(mention, candidate) for candidate in raw] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("known", [None, "aster known", "zulu known"]) +@pytest.mark.parametrize("attested", [False, True], ids=["self", "fuzzy"]) +async def test_candidate_index_is_reused_across_missing_and_existing_mentions( + monkeypatch, known, attested +): + from copy import deepcopy + from datetime import datetime + from unittest.mock import Mock + + from server.core.ch_writes import committed_insert_settings + from server.memory.app import entity_resolution as owner + + created, now = datetime(2024, 1, 2), datetime(2025, 1, 2) + rows = [] + if attested: + rows.append( + { + "mention_text": "northwindai", + "canonical_text": "northwindai", + "confidence": 1.0, + "evidence_count": 2, + "created_at": created, + } + ) + if known is not None: + rows.append( + { + "mention_text": known, + "canonical_text": known, + "confidence": 0.8, + "evidence_count": 3, + "created_at": created, + } + ) + before = deepcopy(rows) + ch = StubCH() + ch.query_all_returns.append(rows) + prepare = Mock(wraps=owner._prepare_canonical_candidates) + monkeypatch.setattr(owner, "_prepare_canonical_candidates", prepare) + monkeypatch.setattr(owner, "naive_utc_now", lambda: now) + mentions = ["Unrelated Galaxy", "Northwind AI", "Northwind AI"] + if known is not None: + mentions.insert(1, known.upper()) + resolved = await owner.resolve_entities_for_write( + ch, org="clickhouse", namespace="private-agent", entity_texts=mentions + ) + + canonical = "northwindai" if attested else "northwind ai" + expected = { + "northwind ai": (canonical, 1.0, "fuzzy" if attested else "self", 1, now), + "unrelated galaxy": ("unrelated galaxy", 1.0, "self", 1, now), + } + if known is not None: + expected[known] = (known, 0.8, "existing", 4, created) + assert resolved == {mention: fields[0] for mention, fields in expected.items()} + written = [ + { + "org": "clickhouse", + "namespace": "private-agent", + "mention_text": mention, + "canonical_text": fields[0], + "confidence": fields[1], + "strategy": fields[2], + "evidence_count": fields[3], + "created_at": fields[4], + "updated_at": now, + "tombstoned_at": None, + } + for mention, fields in sorted(expected.items()) + ] + assert ch.insert_calls == [("loom.entity_canonical_map", written, committed_insert_settings())] + assert prepare.call_count == 1 + assert len(ch.query_all_calls) == 1 + assert ch.query_all_calls[0][1] == {"org": "clickhouse", "ns": "private-agent", "lim": 5000} + assert rows == before