diff --git a/CHANGELOG.md b/CHANGELOG.md index 88c0781..fa067de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ All notable changes follow Keep a Changelog and Semantic Versioning. +## [1.16.0] - 2026-09-07 + +### Added + +- **High-Density Gazetteer Tries (DAWG):** Built a highly efficient Directed Acyclic Word Graph (DAWG/Trie) memory structure. This powers the new GazetteerDetector, enabling ultra-fast, deterministic lookups against massive census lists of proper nouns (PERSON, LOCATION), serving as a secondary safety net for terms the ML model misses. + ## [1.15.0] - 2026-09-07 ### Added diff --git a/pyproject.toml b/pyproject.toml index bfa0370..fbae3a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "pseudonymize" -version = "1.15.0" +version = "1.16.0" description = "Local-first PII pseudonymization for text, structured data, and LLM payloads." readme = "README.md" requires-python = ">=3.11" diff --git a/src/pseudonymize/detectors/gazetteer.py b/src/pseudonymize/detectors/gazetteer.py new file mode 100644 index 0000000..b8a5fb4 --- /dev/null +++ b/src/pseudonymize/detectors/gazetteer.py @@ -0,0 +1,31 @@ +import re +from dataclasses import dataclass +from typing import ClassVar + +from pseudonymize.memory.dawg import DAWG +from pseudonymize.result import Detection, EntityType + + +@dataclass(frozen=True, slots=True) +class GazetteerDetector: + name: str = "gazetteer" + + person_dawg: DAWG | None = None + location_dawg: DAWG | None = None + + # Fast pre-filter for capitalized words + _CAPITALIZED_RX: ClassVar[re.Pattern[str]] = re.compile(r"\b[A-Z][a-zA-Z\u00C0-\u017F'-]+\b") + + def detect(self, text: str) -> list[Detection]: + detections = [] + for match in self._CAPITALIZED_RX.finditer(text): + word = match.group(0) + if self.person_dawg is not None and word in self.person_dawg: + detections.append( + Detection(EntityType.PERSON, match.start(), match.end(), 0.90, self.name) + ) + elif self.location_dawg is not None and word in self.location_dawg: + detections.append( + Detection(EntityType.LOCATION, match.start(), match.end(), 0.90, self.name) + ) + return detections diff --git a/src/pseudonymize/detectors/registry.py b/src/pseudonymize/detectors/registry.py index 431ccb0..449ac8a 100644 --- a/src/pseudonymize/detectors/registry.py +++ b/src/pseudonymize/detectors/registry.py @@ -2,6 +2,7 @@ from pseudonymize.detectors.checksums import AlgorithmicChecksumDetector from pseudonymize.detectors.context import ContextualIdDetector from pseudonymize.detectors.email import EmailDetector +from pseudonymize.detectors.gazetteer import GazetteerDetector from pseudonymize.detectors.iban import IbanDetector from pseudonymize.detectors.ip_address import IpAddressDetector from pseudonymize.detectors.italian import ItalianFiscalCodeDetector, ItalianVATDetector @@ -26,4 +27,5 @@ SecretDetector(), LocationDetector(), OrganizationDetector(), + GazetteerDetector(), ) diff --git a/src/pseudonymize/memory/dawg.py b/src/pseudonymize/memory/dawg.py new file mode 100644 index 0000000..d49d7fd --- /dev/null +++ b/src/pseudonymize/memory/dawg.py @@ -0,0 +1,41 @@ +from collections.abc import Iterable + + +class TrieNode: + __slots__ = ("children", "is_terminal") + + def __init__(self) -> None: + self.children: dict[str, TrieNode] = {} + self.is_terminal: bool = False + + +class DAWG: + """A minimal Trie/DAWG interface for High-Density Gazetteers.""" + + __slots__ = ("_root",) + + def __init__(self) -> None: + self._root = TrieNode() + + @classmethod + def from_words(cls, words: Iterable[str]) -> "DAWG": + dawg = cls() + for word in words: + dawg.add(word) + return dawg + + def add(self, word: str) -> None: + node = self._root + for char in word: + if char not in node.children: + node.children[char] = TrieNode() + node = node.children[char] + node.is_terminal = True + + def __contains__(self, word: str) -> bool: + node = self._root + for char in word: + if char not in node.children: + return False + node = node.children[char] + return node.is_terminal diff --git a/tests/unit/detectors/test_gazetteer.py b/tests/unit/detectors/test_gazetteer.py new file mode 100644 index 0000000..9d33522 --- /dev/null +++ b/tests/unit/detectors/test_gazetteer.py @@ -0,0 +1,41 @@ +from pseudonymize.detectors.gazetteer import GazetteerDetector +from pseudonymize.memory.dawg import DAWG + + +def test_gazetteer_detector_matches() -> None: + # Setup DAWGs + person_dawg = DAWG.from_words(["Jonathan", "Doe", "Alice", "Bob"]) + location_dawg = DAWG.from_words(["London", "Paris", "Berlin", "Rome"]) + + detector = GazetteerDetector(person_dawg=person_dawg, location_dawg=location_dawg) + + text = "Alice and Bob traveled to Paris and Berlin to meet Jonathan Doe." + + detections = detector.detect(text) + + assert len(detections) == 6 + + # Check types + person_matches = [d for d in detections if d.entity_type.name == "PERSON"] + location_matches = [d for d in detections if d.entity_type.name == "LOCATION"] + + assert len(person_matches) == 4 + assert len(location_matches) == 2 + + +def test_gazetteer_detector_ignores_lowercase() -> None: + person_dawg = DAWG.from_words(["hope", "mark", "will"]) + detector = GazetteerDetector(person_dawg=person_dawg) + + # Should not match because the regex requires capital letters + text = "I hope that mark will do well." + + detections = detector.detect(text) + + assert len(detections) == 0 + + +def test_gazetteer_detector_handles_empty() -> None: + detector = GazetteerDetector() + detections = detector.detect("Hello World") + assert len(detections) == 0 diff --git a/uv.lock b/uv.lock index 576daa7..d166b85 100644 --- a/uv.lock +++ b/uv.lock @@ -2567,7 +2567,7 @@ wheels = [ [[package]] name = "pseudonymize" -version = "1.14.0" +version = "1.16.0" source = { editable = "." } [package.optional-dependencies]