Release 1.18.0: Tabular & Delimited Structure Inference

CHANGELOG.md

@@ -2,6 +2,12 @@
 
 All notable changes follow Keep a Changelog and Semantic Versioning.
 
+## [1.18.0] - 2026-09-08
+
+### Added
+
+- **Tabular & Delimited Structure Inference:** Implemented a pre-parsing layout pass for dense tabular structures (like CSVs). If a column header exactly matches a known PII semantic class (e.g., `phone_number`, `ssn`), the engine dynamically injects high-confidence detections for all cells within that column, completely bypassing ML context requirements and dramatically improving recall on raw data dumps.
+
 ## [1.17.0] - 2026-09-08
 
 ### Added

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "pseudonymize"
-version = "1.17.0"
+version = "1.18.0"
 description = "Local-first PII pseudonymization for text, structured data, and LLM payloads."
 readme = "README.md"
 requires-python = ">=3.11"

src/pseudonymize/engine.py

@@ -48,6 +48,7 @@
     FileFormat,
     select_file_format,
 )
+from pseudonymize.inference import TabularInferenceLayout
 from pseudonymize.memory.bloom import BloomFilter
 from pseudonymize.policy import Policy
 from pseudonymize.processing import (
@@ -183,6 +184,7 @@ def _detect_block(
         statistics: "_OperationStatistics",
         remote: bool = False,
         coreferences: CoreferenceGraph | None = None,
+        tabular_layout: TabularInferenceLayout | None = None,
     ) -> tuple[Detection, ...]:
         if not remote:
             statistics.blocks_processed += 1
@@ -191,6 +193,12 @@ def _detect_block(
         stripped_block = block if stripped_to_orig is None else replace(block, text=stripped_text)
 
         candidates: list[Detection] = []
+
+        if not remote and tabular_layout is not None:
+            # We don't map tabular detections through stripped text because they
+            # generally just span the whole original text content of the cell.
+            candidates.extend(tabular_layout.extract_csv_detections(block))
+
         for backend in self.backends:
             capabilities = backend_capabilities(backend)
             if capabilities.remote != remote:
@@ -441,11 +449,18 @@ def process_document(self, document: Document) -> ProcessingResult[Document]:
         reports: list[DetectionReport] = []
         context = AliasContext()
         coreferences = CoreferenceGraph()
+        tabular_layout = TabularInferenceLayout(document)
         blocks: list[ContentBlock] = []
         for block in document.blocks:
             if self._allows_block(block):
                 result = self._process_block(
-                    block, context, False, statistics, reports, coreferences=coreferences
+                    block,
+                    context,
+                    False,
+                    statistics,
+                    reports,
+                    coreferences=coreferences,
+                    tabular_layout=tabular_layout,
                 )
                 blocks.append(replace(block, text=result.text))
             else:
@@ -457,9 +472,12 @@ def process_document(self, document: Document) -> ProcessingResult[Document]:
     def inspect_document(self, document: Document) -> ProcessingResult[None]:
         statistics = _OperationStatistics()
         reports: list[DetectionReport] = []
+        tabular_layout = TabularInferenceLayout(document)
         for block in document.blocks:
             if self._allows_block(block):
-                detections = self._detect_block(block, statistics, remote=False)
+                detections = self._detect_block(
+                    block, statistics, remote=False, tabular_layout=tabular_layout
+                )
                 reports.extend(_detection_reports(block, detections))
             else:
                 statistics.blocks_processed += 1
@@ -572,6 +590,7 @@ def _process_block(
         statistics: "_OperationStatistics",
         reports: list[DetectionReport],
         coreferences: CoreferenceGraph | None = None,
+        tabular_layout: TabularInferenceLayout | None = None,
     ) -> Result:
         if include_mapping and self.mode not in {
             TransformationMode.NUMBERED,
@@ -582,7 +601,11 @@ def _process_block(
         # 1. Local detection
         text = block.text
         local_detections = self._detect_block(
-            block, statistics, remote=False, coreferences=coreferences
+            block,
+            statistics,
+            remote=False,
+            coreferences=coreferences,
+            tabular_layout=tabular_layout,
         )
         local_entities = self.resolver.resolve(text, local_detections)
         local_aliases = tuple(self.assigner.assign(entity, context) for entity in local_entities)
@@ -625,7 +648,11 @@ def _process_block(
         # 3. Remote detection on sanitized text
         sanitized_block = replace(block, text=sanitized_text)
         remote_detections_raw = self._detect_block(
-            sanitized_block, statistics, remote=True, coreferences=coreferences
+            sanitized_block,
+            statistics,
+            remote=True,
+            coreferences=coreferences,
+            tabular_layout=tabular_layout,
         )
 
         remote_detections_mapped = []

src/pseudonymize/inference.py

@@ -0,0 +1,79 @@
+import re
+
+from pseudonymize.document import ContentBlock, CSVCellLocation, Document
+from pseudonymize.result import Detection, EntityType
+
+COLUMN_SEMANTICS: dict[str, EntityType] = {
+    "phone": EntityType.PHONE,
+    "phone_number": EntityType.PHONE,
+    "mobile": EntityType.PHONE,
+    "mobile_number": EntityType.PHONE,
+    "cell": EntityType.PHONE,
+    "ssn": EntityType.NATIONAL_ID,
+    "social_security": EntityType.NATIONAL_ID,
+    "national_id": EntityType.NATIONAL_ID,
+    "nino": EntityType.NATIONAL_ID,
+    "cpf": EntityType.NATIONAL_ID,
+    "email": EntityType.EMAIL,
+    "email_address": EntityType.EMAIL,
+    "card": EntityType.PAYMENT_CARD,
+    "credit_card": EntityType.PAYMENT_CARD,
+    "pan": EntityType.PAYMENT_CARD,
+    "card_number": EntityType.PAYMENT_CARD,
+    "iban": EntityType.IBAN,
+    "account": EntityType.IBAN,
+    "account_number": EntityType.IBAN,
+    "ip": EntityType.IP_ADDRESS,
+    "ip_address": EntityType.IP_ADDRESS,
+    "password": EntityType.SECRET,
+    "secret": EntityType.SECRET,
+    "token": EntityType.SECRET,
+    "tax_id": EntityType.TAX_ID,
+    "vat": EntityType.TAX_ID,
+    "tin": EntityType.TAX_ID,
+    "fiscal_code": EntityType.TAX_ID,
+    "name": EntityType.PERSON,
+    "first_name": EntityType.PERSON,
+    "last_name": EntityType.PERSON,
+    "full_name": EntityType.PERSON,
+    "person": EntityType.PERSON,
+    "employee": EntityType.PERSON,
+    "customer": EntityType.PERSON,
+    "city": EntityType.LOCATION,
+    "address": EntityType.LOCATION,
+    "zip": EntityType.LOCATION,
+    "zipcode": EntityType.LOCATION,
+    "location": EntityType.LOCATION,
+    "company": EntityType.ORGANIZATION,
+    "organization": EntityType.ORGANIZATION,
+    "org": EntityType.ORGANIZATION,
+}
+
+
+def _normalize_header(header: str) -> str:
+    return re.sub(r"[^a-z0-9]", "_", header.strip().lower()).strip("_")
+
+
+class TabularInferenceLayout:
+    def __init__(self, document: Document) -> None:
+        self.csv_semantics: dict[int, EntityType] = {}
+
+        # Determine CSV layout
+        for block in document.blocks:
+            loc = block.location
+            if isinstance(loc, CSVCellLocation) and loc.row == 0:
+                norm = _normalize_header(block.text)
+                if norm in COLUMN_SEMANTICS:
+                    self.csv_semantics[loc.column] = COLUMN_SEMANTICS[norm]
+            # Since blocks are ordered, we can stop early if we pass row 0, but no harm continuing
+
+    def extract_csv_detections(self, block: ContentBlock) -> list[Detection]:
+        loc = block.location
+        if isinstance(loc, CSVCellLocation) and loc.row > 0 and loc.column in self.csv_semantics:
+            entity_type = self.csv_semantics[loc.column]
+            text = block.text.strip()
+            if text:
+                start = block.text.find(text)
+                end = start + len(text)
+                return [Detection(entity_type, start, end, 1.0, "tabular", "layout_pass")]
+        return []

src/pseudonymize/spans.py

@@ -4,6 +4,9 @@
 from pseudonymize.result import Detection
 
 _DETECTOR_WEIGHT = {
+    # Tabular Layout / Column Headers (Absolute Highest)
+    "tabular": 1.0,
+
     # Checksums / Deterministic structures - Highest priority (1.0)
     "payment_card": 1.0,
     "iban": 1.0,

tests/integration/test_builtin_files.py

@@ -104,7 +104,7 @@ def test_jsonl_uses_one_alias_scope_and_record_locations(tmp_path: Path) -> None
 def test_csv_preserves_matrix_and_reports_cell_locations(tmp_path: Path) -> None:
     source = tmp_path / "payload.csv"
     source.write_text(
-        "email,note,formula\n"
+        "contact,note,formula\n"
         'maria@example.com,"line one\nline two","=A2"\n'
         "192.0.2.10,,tail,extra\n",
         encoding="utf-8",
@@ -120,7 +120,7 @@ def test_csv_preserves_matrix_and_reports_cell_locations(tmp_path: Path) -> None
     )
 
     assert rows == (
-        ["email", "note", "formula"],
+        ["contact", "note", "formula"],
         ["<EMAIL_1>", "line one\nline two", "=A2"],
         ["<IP_ADDRESS_1>", "", "tail", "extra"],
     )

tests/unit/test_inference.py

@@ -0,0 +1,43 @@
+from pseudonymize.document import ContentBlock, CSVCellLocation, Document
+from pseudonymize.engine import Pseudonymizer
+from pseudonymize.inference import TabularInferenceLayout
+from pseudonymize.result import EntityType
+
+
+def test_csv_tabular_inference() -> None:
+    blocks = (
+        ContentBlock("r0c0", "ssn", CSVCellLocation(0, 0)),
+        ContentBlock("r0c1", "name", CSVCellLocation(0, 1)),
+        ContentBlock("r1c0", "999-99-9999", CSVCellLocation(1, 0)),
+        ContentBlock("r1c1", "John Doe", CSVCellLocation(1, 1)),
+    )
+    document = Document("test", blocks, {})
+    layout = TabularInferenceLayout(document)
+
+    assert layout.csv_semantics[0] == EntityType.NATIONAL_ID
+    assert layout.csv_semantics[1] == EntityType.PERSON
+
+    d0 = layout.extract_csv_detections(blocks[2])
+    assert len(d0) == 1
+    assert d0[0].entity_type == EntityType.NATIONAL_ID
+    assert d0[0].start == 0
+    assert d0[0].end == 11
+
+    d1 = layout.extract_csv_detections(blocks[3])
+    assert len(d1) == 1
+    assert d1[0].entity_type == EntityType.PERSON
+
+
+def test_engine_csv_tabular_inference() -> None:
+    engine = Pseudonymizer()
+    blocks = (
+        ContentBlock("r0c0", "phone_number", CSVCellLocation(0, 0)),
+        ContentBlock("r1c0", "555-0199", CSVCellLocation(1, 0)),
+    )
+    document = Document("test", blocks, {})
+    result = engine.process_document(document)
+
+    # Check detections
+    phone_detections = [d for d in result.detections if d.entity_type == EntityType.PHONE]
+    assert len(phone_detections) >= 1
+    assert any(d.detector == "tabular" for d in phone_detections)