From bd8a8da5c71bff9f0d3478ed31a023add9c74dbb Mon Sep 17 00:00:00 2001 From: Paolo Mazza Date: Tue, 8 Sep 2026 13:04:31 +0200 Subject: [PATCH] Release 1.17.0: Detector-Aware Conflict Matrix --- CHANGELOG.md | 6 +++++ pyproject.toml | 2 +- src/pseudonymize/spans.py | 56 ++++++++++++++++++++------------------- tests/unit/test_spans.py | 6 ++--- uv.lock | 2 +- 5 files changed, 40 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fa067de..413c8f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ All notable changes follow Keep a Changelog and Semantic Versioning. +## [1.17.0] - 2026-09-08 + +### Added + +- **Detector-Aware Conflict Matrix:** Deprecated the static, entity-based priority list in favor of a dynamic confidence-scaling matrix. The engine now assigns weights based on the *originating detector*, allowing deterministic algorithmic heuristics (like PAN checksums) to intelligently override lower-confidence ML predictions, resolving conflicts dynamically based on actual certainty. + ## [1.16.0] - 2026-09-07 ### Added diff --git a/pyproject.toml b/pyproject.toml index fbae3a0..e6a3113 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "pseudonymize" -version = "1.16.0" +version = "1.17.0" description = "Local-first PII pseudonymization for text, structured data, and LLM payloads." readme = "README.md" requires-python = ">=3.11" diff --git a/src/pseudonymize/spans.py b/src/pseudonymize/spans.py index e7cfe48..0434639 100644 --- a/src/pseudonymize/spans.py +++ b/src/pseudonymize/spans.py @@ -1,24 +1,27 @@ import bisect from collections.abc import Iterable -from pseudonymize.result import Detection, EntityType +from pseudonymize.result import Detection -_ENTITY_PRIORITY = { - EntityType.PAYMENT_CARD: 70, - EntityType.IBAN: 70, - EntityType.NATIONAL_ID: 70, - EntityType.TAX_ID: 70, - # URL credentials outrank emails: a password such as "s3cret" followed by - # "@host" also matches the email pattern, and letting the email span win - # would leave the "user:" part of the userinfo unmasked. - EntityType.URL_CREDENTIAL: 65, - EntityType.EMAIL: 60, - EntityType.IP_ADDRESS: 60, - EntityType.SECRET: 50, - EntityType.PHONE: 40, - EntityType.PERSON: 30, - EntityType.ORGANIZATION: 30, - EntityType.LOCATION: 30, +_DETECTOR_WEIGHT = { + # Checksums / Deterministic structures - Highest priority (1.0) + "payment_card": 1.0, + "iban": 1.0, + "italian_fiscal_code": 1.0, + "italian_vat": 1.0, + "checksum": 1.0, + # URL credentials outrank emails (password vs email) + "url": 0.95, + "email": 0.90, + "ip_address": 0.90, + "secret": 0.80, + "phone": 0.70, + # Local heuristics & gazetteers + "context_id": 0.60, + "gazetteer": 0.55, + "location": 0.50, + "organization": 0.50, + "ensemble": 0.40, } @@ -28,21 +31,20 @@ def resolve_overlaps( configured = { name: len(detector_priority) - index for index, name in enumerate(detector_priority) } - # Priority: - # 3 if ML >= 0.95 - # 2 if local_rules - # 1 otherwise + + def resolution_score(detection: Detection) -> float: + base_weight = _DETECTOR_WEIGHT.get(detection.detector, 0.3) + if detection.backend == "local_onnx_pii" and detection.confidence >= 0.95: + # Overwhelmingly confident ML overrides generic heuristics, + # but stays below valid deterministic checksums. + return max(base_weight * detection.confidence, 0.85) + return base_weight * detection.confidence ranked = sorted( detections, key=lambda detection: ( - -_ENTITY_PRIORITY[detection.entity_type], + -resolution_score(detection), -configured.get(detection.detector, 0), - -3 - if (detection.backend == "local_onnx_pii" and detection.confidence >= 0.95) - else -2 - if detection.backend == "local_rules" - else -1, -(detection.end - detection.start), -detection.confidence, detection.start, diff --git a/tests/unit/test_spans.py b/tests/unit/test_spans.py index a5e7547..3d26467 100644 --- a/tests/unit/test_spans.py +++ b/tests/unit/test_spans.py @@ -6,7 +6,7 @@ def test_overlap_prefers_validated_entity_then_stable_order() -> None: phone = Detection(EntityType.PHONE, 0, 19, 0.99, "phone") - card = Detection(EntityType.PAYMENT_CARD, 0, 19, 1.0, "card") + card = Detection(EntityType.PAYMENT_CARD, 0, 19, 1.0, "payment_card") email = Detection(EntityType.EMAIL, 30, 40, 0.9, "email") assert resolve_overlaps([phone, email, card]) == (card, email) @@ -41,12 +41,12 @@ def test_dense_overlaps_yield_disjoint_and_maximal_selection() -> None: def test_rules_outrank_ml_unless_ml_highly_confident() -> None: # Rule match and ML match with normal confidence -> Rule wins - rule_normal = Detection(EntityType.PERSON, 0, 10, 1.0, "context", "local_rules") + rule_normal = Detection(EntityType.PERSON, 0, 10, 1.0, "context_id", "local_rules") ml_normal = Detection(EntityType.PERSON, 0, 10, 0.85, "onnx", "local_onnx_pii") assert resolve_overlaps([ml_normal, rule_normal]) == (rule_normal,) # Rule match and ML match with > 0.95 confidence -> ML wins - rule_overridden = Detection(EntityType.PERSON, 0, 10, 1.0, "context", "local_rules") + rule_overridden = Detection(EntityType.PERSON, 0, 10, 1.0, "context_id", "local_rules") ml_high = Detection(EntityType.PERSON, 0, 10, 0.96, "onnx", "local_onnx_pii") assert resolve_overlaps([ml_high, rule_overridden]) == (ml_high,) diff --git a/uv.lock b/uv.lock index d166b85..380097e 100644 --- a/uv.lock +++ b/uv.lock @@ -2567,7 +2567,7 @@ wheels = [ [[package]] name = "pseudonymize" -version = "1.16.0" +version = "1.17.0" source = { editable = "." } [package.optional-dependencies]