Release 1.17.0: Detector-Aware Conflict Matrix

CHANGELOG.md

@@ -2,6 +2,12 @@
 
 All notable changes follow Keep a Changelog and Semantic Versioning.
 
+## [1.17.0] - 2026-09-08
+
+### Added
+
+- **Detector-Aware Conflict Matrix:** Deprecated the static, entity-based priority list in favor of a dynamic confidence-scaling matrix. The engine now assigns weights based on the *originating detector*, allowing deterministic algorithmic heuristics (like PAN checksums) to intelligently override lower-confidence ML predictions, resolving conflicts dynamically based on actual certainty.
+
 ## [1.16.0] - 2026-09-07
 
 ### Added

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "pseudonymize"
-version = "1.16.0"
+version = "1.17.0"
 description = "Local-first PII pseudonymization for text, structured data, and LLM payloads."
 readme = "README.md"
 requires-python = ">=3.11"

src/pseudonymize/spans.py

@@ -1,24 +1,27 @@
 import bisect
 from collections.abc import Iterable
 
-from pseudonymize.result import Detection, EntityType
+from pseudonymize.result import Detection
 
-_ENTITY_PRIORITY = {
-    EntityType.PAYMENT_CARD: 70,
-    EntityType.IBAN: 70,
-    EntityType.NATIONAL_ID: 70,
-    EntityType.TAX_ID: 70,
-    # URL credentials outrank emails: a password such as "s3cret" followed by
-    # "@host" also matches the email pattern, and letting the email span win
-    # would leave the "user:" part of the userinfo unmasked.
-    EntityType.URL_CREDENTIAL: 65,
-    EntityType.EMAIL: 60,
-    EntityType.IP_ADDRESS: 60,
-    EntityType.SECRET: 50,
-    EntityType.PHONE: 40,
-    EntityType.PERSON: 30,
-    EntityType.ORGANIZATION: 30,
-    EntityType.LOCATION: 30,
+_DETECTOR_WEIGHT = {
+    # Checksums / Deterministic structures - Highest priority (1.0)
+    "payment_card": 1.0,
+    "iban": 1.0,
+    "italian_fiscal_code": 1.0,
+    "italian_vat": 1.0,
+    "checksum": 1.0,
+    # URL credentials outrank emails (password vs email)
+    "url": 0.95,
+    "email": 0.90,
+    "ip_address": 0.90,
+    "secret": 0.80,
+    "phone": 0.70,
+    # Local heuristics & gazetteers
+    "context_id": 0.60,
+    "gazetteer": 0.55,
+    "location": 0.50,
+    "organization": 0.50,
+    "ensemble": 0.40,
 }
 
 
@@ -28,21 +31,20 @@ def resolve_overlaps(
     configured = {
         name: len(detector_priority) - index for index, name in enumerate(detector_priority)
     }
-    # Priority:
-    # 3 if ML >= 0.95
-    # 2 if local_rules
-    # 1 otherwise
+
+    def resolution_score(detection: Detection) -> float:
+        base_weight = _DETECTOR_WEIGHT.get(detection.detector, 0.3)
+        if detection.backend == "local_onnx_pii" and detection.confidence >= 0.95:
+            # Overwhelmingly confident ML overrides generic heuristics,
+            # but stays below valid deterministic checksums.
+            return max(base_weight * detection.confidence, 0.85)
+        return base_weight * detection.confidence
 
     ranked = sorted(
         detections,
         key=lambda detection: (
-            -_ENTITY_PRIORITY[detection.entity_type],
+            -resolution_score(detection),
             -configured.get(detection.detector, 0),
-            -3
-            if (detection.backend == "local_onnx_pii" and detection.confidence >= 0.95)
-            else -2
-            if detection.backend == "local_rules"
-            else -1,
             -(detection.end - detection.start),
             -detection.confidence,
             detection.start,

tests/unit/test_spans.py

@@ -6,7 +6,7 @@
 
 def test_overlap_prefers_validated_entity_then_stable_order() -> None:
     phone = Detection(EntityType.PHONE, 0, 19, 0.99, "phone")
-    card = Detection(EntityType.PAYMENT_CARD, 0, 19, 1.0, "card")
+    card = Detection(EntityType.PAYMENT_CARD, 0, 19, 1.0, "payment_card")
     email = Detection(EntityType.EMAIL, 30, 40, 0.9, "email")
     assert resolve_overlaps([phone, email, card]) == (card, email)
 
@@ -41,12 +41,12 @@ def test_dense_overlaps_yield_disjoint_and_maximal_selection() -> None:
 
 def test_rules_outrank_ml_unless_ml_highly_confident() -> None:
     # Rule match and ML match with normal confidence -> Rule wins
-    rule_normal = Detection(EntityType.PERSON, 0, 10, 1.0, "context", "local_rules")
+    rule_normal = Detection(EntityType.PERSON, 0, 10, 1.0, "context_id", "local_rules")
     ml_normal = Detection(EntityType.PERSON, 0, 10, 0.85, "onnx", "local_onnx_pii")
     assert resolve_overlaps([ml_normal, rule_normal]) == (rule_normal,)
 
     # Rule match and ML match with > 0.95 confidence -> ML wins
-    rule_overridden = Detection(EntityType.PERSON, 0, 10, 1.0, "context", "local_rules")
+    rule_overridden = Detection(EntityType.PERSON, 0, 10, 1.0, "context_id", "local_rules")
     ml_high = Detection(EntityType.PERSON, 0, 10, 0.96, "onnx", "local_onnx_pii")
     assert resolve_overlaps([ml_high, rule_overridden]) == (ml_high,)