Release 1.16.0: High-Density Gazetteer Tries (DAWG)

CHANGELOG.md

@@ -2,6 +2,12 @@
 
 All notable changes follow Keep a Changelog and Semantic Versioning.
 
+## [1.16.0] - 2026-09-07
+
+### Added
+
+- **High-Density Gazetteer Tries (DAWG):** Built a highly efficient Directed Acyclic Word Graph (DAWG/Trie) memory structure. This powers the new GazetteerDetector, enabling ultra-fast, deterministic lookups against massive census lists of proper nouns (PERSON, LOCATION), serving as a secondary safety net for terms the ML model misses.
+
 ## [1.15.0] - 2026-09-07
 
 ### Added

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "pseudonymize"
-version = "1.15.0"
+version = "1.16.0"
 description = "Local-first PII pseudonymization for text, structured data, and LLM payloads."
 readme = "README.md"
 requires-python = ">=3.11"

src/pseudonymize/detectors/gazetteer.py

@@ -0,0 +1,31 @@
+import re
+from dataclasses import dataclass
+from typing import ClassVar
+
+from pseudonymize.memory.dawg import DAWG
+from pseudonymize.result import Detection, EntityType
+
+
+@dataclass(frozen=True, slots=True)
+class GazetteerDetector:
+    name: str = "gazetteer"
+
+    person_dawg: DAWG | None = None
+    location_dawg: DAWG | None = None
+
+    # Fast pre-filter for capitalized words
+    _CAPITALIZED_RX: ClassVar[re.Pattern[str]] = re.compile(r"\b[A-Z][a-zA-Z\u00C0-\u017F'-]+\b")
+
+    def detect(self, text: str) -> list[Detection]:
+        detections = []
+        for match in self._CAPITALIZED_RX.finditer(text):
+            word = match.group(0)
+            if self.person_dawg is not None and word in self.person_dawg:
+                detections.append(
+                    Detection(EntityType.PERSON, match.start(), match.end(), 0.90, self.name)
+                )
+            elif self.location_dawg is not None and word in self.location_dawg:
+                detections.append(
+                    Detection(EntityType.LOCATION, match.start(), match.end(), 0.90, self.name)
+                )
+        return detections

src/pseudonymize/detectors/registry.py

@@ -2,6 +2,7 @@
 from pseudonymize.detectors.checksums import AlgorithmicChecksumDetector
 from pseudonymize.detectors.context import ContextualIdDetector
 from pseudonymize.detectors.email import EmailDetector
+from pseudonymize.detectors.gazetteer import GazetteerDetector
 from pseudonymize.detectors.iban import IbanDetector
 from pseudonymize.detectors.ip_address import IpAddressDetector
 from pseudonymize.detectors.italian import ItalianFiscalCodeDetector, ItalianVATDetector
@@ -26,4 +27,5 @@
     SecretDetector(),
     LocationDetector(),
     OrganizationDetector(),
+    GazetteerDetector(),
 )

src/pseudonymize/memory/dawg.py

@@ -0,0 +1,41 @@
+from collections.abc import Iterable
+
+
+class TrieNode:
+    __slots__ = ("children", "is_terminal")
+
+    def __init__(self) -> None:
+        self.children: dict[str, TrieNode] = {}
+        self.is_terminal: bool = False
+
+
+class DAWG:
+    """A minimal Trie/DAWG interface for High-Density Gazetteers."""
+
+    __slots__ = ("_root",)
+
+    def __init__(self) -> None:
+        self._root = TrieNode()
+
+    @classmethod
+    def from_words(cls, words: Iterable[str]) -> "DAWG":
+        dawg = cls()
+        for word in words:
+            dawg.add(word)
+        return dawg
+
+    def add(self, word: str) -> None:
+        node = self._root
+        for char in word:
+            if char not in node.children:
+                node.children[char] = TrieNode()
+            node = node.children[char]
+        node.is_terminal = True
+
+    def __contains__(self, word: str) -> bool:
+        node = self._root
+        for char in word:
+            if char not in node.children:
+                return False
+            node = node.children[char]
+        return node.is_terminal

tests/unit/detectors/test_gazetteer.py

@@ -0,0 +1,41 @@
+from pseudonymize.detectors.gazetteer import GazetteerDetector
+from pseudonymize.memory.dawg import DAWG
+
+
+def test_gazetteer_detector_matches() -> None:
+    # Setup DAWGs
+    person_dawg = DAWG.from_words(["Jonathan", "Doe", "Alice", "Bob"])
+    location_dawg = DAWG.from_words(["London", "Paris", "Berlin", "Rome"])
+
+    detector = GazetteerDetector(person_dawg=person_dawg, location_dawg=location_dawg)
+
+    text = "Alice and Bob traveled to Paris and Berlin to meet Jonathan Doe."
+
+    detections = detector.detect(text)
+
+    assert len(detections) == 6
+
+    # Check types
+    person_matches = [d for d in detections if d.entity_type.name == "PERSON"]
+    location_matches = [d for d in detections if d.entity_type.name == "LOCATION"]
+
+    assert len(person_matches) == 4
+    assert len(location_matches) == 2
+
+
+def test_gazetteer_detector_ignores_lowercase() -> None:
+    person_dawg = DAWG.from_words(["hope", "mark", "will"])
+    detector = GazetteerDetector(person_dawg=person_dawg)
+
+    # Should not match because the regex requires capital letters
+    text = "I hope that mark will do well."
+
+    detections = detector.detect(text)
+
+    assert len(detections) == 0
+
+
+def test_gazetteer_detector_handles_empty() -> None:
+    detector = GazetteerDetector()
+    detections = detector.detect("Hello World")
+    assert len(detections) == 0