diff --git a/CHANGELOG.md b/CHANGELOG.md index e9ebd1bb44..f0645c82fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. ## [unreleased] +### Added + +- Added Taiwan National ID (`TW_NATIONAL_ID`) predefined recognizer in `presidio-analyzer`. +- Added Taiwan phone number (`TW_PHONE_NUMBER`) predefined recognizer in `presidio-analyzer`. + ### Anonymizer ### General #### Fixed diff --git a/docs/supported_entities.md b/docs/supported_entities.md index 4273461a41..69d2dbf853 100644 --- a/docs/supported_entities.md +++ b/docs/supported_entities.md @@ -146,6 +146,8 @@ For more information, refer to the [adding new recognizers documentation](analyz | FieldType | Description | Detection Method | |------------|---------------------------------------------------------------------------------------------------------|------------------------------------------| | TH_TNIN | The Thai National ID Number (TNIN) is a unique 13-digit number issued to all Thai residents. | Pattern match, context and custom logic. | +| TW_NATIONAL_ID | Taiwan National Identification Number (國民身分證統一編號 / 身分證字號): 1 leading letter followed by 9 digits, with the second digit indicating holder category and a public checksum rule. | Pattern match, context and checksum | +| TW_PHONE_NUMBER | Taiwan phone number (電話號碼 / 手機號碼): validated by `python-phonenumbers` with the Taiwan (`TW`) region for mobile and landline formats. | Pattern match, context and region-aware phone validation | ### Turkey diff --git a/presidio-analyzer/presidio_analyzer/conf/default_recognizers.yaml b/presidio-analyzer/presidio_analyzer/conf/default_recognizers.yaml index 0ad50b1603..b65f5dd37b 100644 --- a/presidio-analyzer/presidio_analyzer/conf/default_recognizers.yaml +++ b/presidio-analyzer/presidio_analyzer/conf/default_recognizers.yaml @@ -329,6 +329,20 @@ recognizers: enabled: false country_code: th + - name: TwNationalIdRecognizer + supported_languages: + - zh + type: predefined + enabled: false + country_code: tw + + - name: TwPhoneNumberRecognizer + supported_languages: + - zh + type: predefined + enabled: false + country_code: tw + - name: TrNationalIdRecognizer supported_languages: - tr diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/__init__.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/__init__.py index 2823ff035d..0a4ad0a221 100644 --- a/presidio-analyzer/presidio_analyzer/predefined_recognizers/__init__.py +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/__init__.py @@ -101,6 +101,10 @@ ) from .country_specific.sweden.se_personnummer_recognizer import SePersonnummerRecognizer +# Taiwan recognizers +from .country_specific.taiwan.tw_national_id_recognizer import TwNationalIdRecognizer +from .country_specific.taiwan.tw_phone_number_recognizer import TwPhoneNumberRecognizer + # Thai recognizers from .country_specific.thai.th_tnin_recognizer import ThTninRecognizer @@ -245,6 +249,8 @@ "KrFrnRecognizer", "SeOrganisationsnummerRecognizer", "ThTninRecognizer", + "TwNationalIdRecognizer", + "TwPhoneNumberRecognizer", "TrLicensePlateRecognizer", "TrNationalIdRecognizer", "SePersonnummerRecognizer", diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/taiwan/tw_national_id_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/taiwan/tw_national_id_recognizer.py new file mode 100644 index 0000000000..df10c44799 --- /dev/null +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/taiwan/tw_national_id_recognizer.py @@ -0,0 +1,131 @@ +from typing import List, Optional, Tuple, Union + +from presidio_analyzer import EntityRecognizer, Pattern, PatternRecognizer + + +class TwNationalIdRecognizer(PatternRecognizer): + """ + Recognize Taiwan National Identification Numbers. + + Taiwan national IDs use one leading letter followed by 9 digits. + The second digit is typically 1 or 2, and the full value follows + a public checksum rule defined by Taiwan's National Identification Card + numbering scheme. + + Public reference for format and checksum background: + https://www.ris.gov.tw/app/portal/3053 + + :param patterns: List of patterns to be used by this recognizer + :param context: List of context words to increase confidence in detection + :param supported_language: Language this recognizer supports + :param supported_entity: The entity this recognizer can detect + :param replacement_pairs: List of tuples with potential replacement values + for different strings to be used during pattern matching. + """ + + COUNTRY_CODE = "tw" + + PATTERNS = [ + Pattern( + "TW_NATIONAL_ID", + r"\b[A-Z][12]\d{8}\b", + 0.3, + ), + ] + + CONTEXT = [ + "身分證", + "身分證字號", + "身份證", + "身份證字號", + "國民身分證", + "national id", + "id number", + ] + + LETTER_MAPPING = { + "A": 10, + "B": 11, + "C": 12, + "D": 13, + "E": 14, + "F": 15, + "G": 16, + "H": 17, + "I": 34, + "J": 18, + "K": 19, + "L": 20, + "M": 21, + "N": 22, + "O": 35, + "P": 23, + "Q": 24, + "R": 25, + "S": 26, + "T": 27, + "U": 28, + "V": 29, + "W": 32, + "X": 30, + "Y": 31, + "Z": 33, + } + + WEIGHTS = [1, 9, 8, 7, 6, 5, 4, 3, 2, 1, 1] + + def __init__( + self, + patterns: Optional[List[Pattern]] = None, + context: Optional[List[str]] = None, + supported_language: str = "zh", + supported_entity: str = "TW_NATIONAL_ID", + replacement_pairs: Optional[List[Tuple[str, str]]] = None, + name: Optional[str] = None, + ): + self.replacement_pairs = replacement_pairs if replacement_pairs else [] + + patterns = patterns if patterns else self.PATTERNS + context = context if context else self.CONTEXT + super().__init__( + supported_entity=supported_entity, + patterns=patterns, + context=context, + supported_language=supported_language, + name=name, + ) + + def validate_result(self, pattern_text: str) -> Union[bool, None]: + """ + Validate the pattern logic by running Taiwan ID checksum verification. + + :param pattern_text: the text to validated. + Only the part in text that was detected by the regex engine + :return: A bool or None, indicating whether the validation was successful. + """ + sanitized_value = EntityRecognizer.sanitize_value( + pattern_text, self.replacement_pairs + ) + + if len(sanitized_value) != 10: + return False + + if not sanitized_value[0].isalpha() or not sanitized_value[1:].isdigit(): + return False + + sanitized_value = sanitized_value.upper() + + if sanitized_value[0] not in self.LETTER_MAPPING: + return False + + if sanitized_value[1] not in {"1", "2"}: + return False + + return self._validate_checksum(sanitized_value) + + def _validate_checksum(self, national_id: str) -> bool: + """Validate Taiwan National ID using the public checksum algorithm.""" + mapped_prefix = str(self.LETTER_MAPPING[national_id[0]]) + digits = [int(digit) for digit in mapped_prefix + national_id[1:]] + checksum = sum(digit * weight for digit, weight in zip(digits, self.WEIGHTS)) + return checksum % 10 == 0 diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/taiwan/tw_phone_number_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/taiwan/tw_phone_number_recognizer.py new file mode 100644 index 0000000000..91c16f4b9f --- /dev/null +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/taiwan/tw_phone_number_recognizer.py @@ -0,0 +1,56 @@ +from typing import List, Optional + +from presidio_analyzer.predefined_recognizers.generic.phone_recognizer import ( + PhoneRecognizer, +) + + +class TwPhoneNumberRecognizer(PhoneRecognizer): + """ + Recognize Taiwan phone numbers using python-phonenumbers with TW region hints. + + This recognizer intentionally delegates parsing and validation to the + generic PhoneRecognizer with the supported region restricted to Taiwan. + + Public reference for numbering background: + https://www.itu.int/itudoc/itu-t/ob-lists/icc/e164_886.html + + :param context: Base context words for enhancing the assurance scores. + :param supported_language: Language this recognizer supports + :param supported_entity: The entity this recognizer can detect + :param name: Optional recognizer name override + """ + + COUNTRY_CODE = "tw" + + CONTEXT = [ + "電話", + "電話號碼", + "手機", + "手機號碼", + "行動電話", + "聯絡電話", + "市話", + "office phone", + "phone", + "phone number", + "mobile", + "cell", + "call", + "contact", + ] + + def __init__( + self, + context: Optional[List[str]] = None, + supported_language: str = "zh", + supported_entity: str = "TW_PHONE_NUMBER", + name: Optional[str] = None, + ): + super().__init__( + context=context if context else self.CONTEXT, + supported_language=supported_language, + supported_entity=supported_entity, + supported_regions=["TW"], + name=name, + ) diff --git a/presidio-analyzer/tests/test_recognizer_registry.py b/presidio-analyzer/tests/test_recognizer_registry.py index e0c6b8db53..c681c6ec35 100644 --- a/presidio-analyzer/tests/test_recognizer_registry.py +++ b/presidio-analyzer/tests/test_recognizer_registry.py @@ -1,6 +1,7 @@ from pathlib import Path import pytest +import yaml import regex as re from presidio_analyzer import ( AnalyzerEngine, @@ -10,6 +11,7 @@ RecognizerRegistry, ) from presidio_analyzer.predefined_recognizers import SpacyRecognizer, UsSsnRecognizer +from presidio_analyzer.recognizer_registry.recognizers_loader_utils import RecognizerListLoader def create_mock_pattern_recognizer(lang, entity, name): @@ -617,3 +619,24 @@ def test_load_predefined_recognizers_validates_countries_input(): # --------------------------------------------------------------------------- # YAML ``country_code`` cross-validation # --------------------------------------------------------------------------- + + +def test_default_yaml_defines_tw_recognizers_and_loader_can_resolve_classes(): + """Default YAML should define Taiwan recognizers with matching loader classes.""" + config_path = Path(__file__).resolve().parents[1] / "presidio_analyzer" / "conf" / "default_recognizers.yaml" + config = yaml.safe_load(config_path.read_text()) + + recognizer_entries = { + entry["name"]: entry for entry in config["recognizers"] if isinstance(entry, dict) + } + + tw_national_id = recognizer_entries["TwNationalIdRecognizer"] + tw_phone = recognizer_entries["TwPhoneNumberRecognizer"] + + assert tw_national_id["supported_languages"] == ["zh"] + assert tw_national_id["country_code"] == "tw" + assert tw_phone["supported_languages"] == ["zh"] + assert tw_phone["country_code"] == "tw" + + assert RecognizerListLoader.get_existing_recognizer_cls("TwNationalIdRecognizer").__name__ == "TwNationalIdRecognizer" + assert RecognizerListLoader.get_existing_recognizer_cls("TwPhoneNumberRecognizer").__name__ == "TwPhoneNumberRecognizer" diff --git a/presidio-analyzer/tests/test_tw_national_id_recognizer.py b/presidio-analyzer/tests/test_tw_national_id_recognizer.py new file mode 100644 index 0000000000..c60e3a34a3 --- /dev/null +++ b/presidio-analyzer/tests/test_tw_national_id_recognizer.py @@ -0,0 +1,123 @@ +"""Tests for Taiwan National ID recognizer.""" + +import pytest +from presidio_analyzer.predefined_recognizers import TwNationalIdRecognizer + +from tests import assert_result_within_score_range + + +@pytest.fixture(scope="module") +def recognizer(): + """Create a Taiwan National ID recognizer instance for testing.""" + return TwNationalIdRecognizer() + + +@pytest.fixture(scope="module") +def entities(): + """Return the Taiwan National ID entity type for testing.""" + return ["TW_NATIONAL_ID"] + + +@pytest.mark.parametrize( + "text, expected_len, expected_positions, expected_score_ranges", + [ + ("A100000001", 1, ((0, 10),), ((0.5, 1.0),),), + ("B120863514", 1, ((0, 10),), ((0.5, 1.0),),), + ("(A100000001)", 1, ((1, 11),), ((0.5, 1.0),),), + ( + "身分證字號：A100000001", + 1, + ((6, 16),), + ((0.5, 1.0),), + ), + ( + "National ID A100000001 belongs to the applicant.", + 1, + ((12, 22),), + ((0.5, 1.0),), + ), + ( + "Primary ID A100000001, secondary ID B120863514", + 2, + ((11, 21), (36, 46),), + ((0.5, 1.0), (0.5, 1.0),), + ), + ("A100000002", 0, (), (),), + ("Z100000000", 0, (), (),), + ("A300000001", 0, (), (),), + ("AA00000001", 0, (), (),), + ("A10000001", 0, (), (),), + ("A1000000011", 0, (), (),), + ("身分證 A100000002", 0, (), (),), + ("a100000001", 1, ((0, 10),), ((0.5, 1.0),),), + ], +) +def test_when_tw_national_id_in_text_then_all_matches_are_found( + text, + expected_len, + expected_positions, + expected_score_ranges, + recognizer, + entities, + max_score, +): + """Test that Taiwan National ID recognizer correctly identifies IDs.""" + results = sorted( + recognizer.analyze(text, entities), + key=lambda result: result.start, + ) + assert len(results) == expected_len + + for res, (st_pos, fn_pos), (st_score, fn_score) in zip( + results, expected_positions, expected_score_ranges + ): + if st_score == "max": + st_score = max_score + if fn_score == "max": + fn_score = max_score + assert_result_within_score_range( + res, entities[0], st_pos, fn_pos, st_score, fn_score + ) + + +@pytest.mark.parametrize( + "value, expected", + [ + ("A100000001", True), + ("B120863514", True), + ("A100000002", False), + ("Z100000000", False), + ("A300000001", False), + ("AA00000001", False), + ("A10000001", False), + ("A1000000011", False), + ("a100000001", True), + ], +) +def test_validate_result(value, expected, recognizer): + """Test validate_result with valid and invalid Taiwan IDs.""" + assert recognizer.validate_result(value) is expected + + +def test_supported_entity(recognizer): + """Test that supported entity is correctly set.""" + assert recognizer.supported_entities == ["TW_NATIONAL_ID"] + + +def test_supported_language(recognizer): + """Test that supported language is correctly set.""" + assert recognizer.supported_language == "zh" + + +def test_context_words(recognizer): + """Test that context words are properly set.""" + expected_context = [ + "身分證", + "身分證字號", + "身份證", + "身份證字號", + "國民身分證", + "national id", + "id number", + ] + assert recognizer.context == expected_context diff --git a/presidio-analyzer/tests/test_tw_phone_number_recognizer.py b/presidio-analyzer/tests/test_tw_phone_number_recognizer.py new file mode 100644 index 0000000000..b191234fab --- /dev/null +++ b/presidio-analyzer/tests/test_tw_phone_number_recognizer.py @@ -0,0 +1,106 @@ +"""Tests for Taiwan phone number (TW_PHONE_NUMBER) recognizer.""" + +import pytest +from presidio_analyzer.predefined_recognizers import TwPhoneNumberRecognizer + +from tests import assert_result_within_score_range + +TAIWAN_CONTEXT = [ + "電話", + "電話號碼", + "手機", + "手機號碼", + "行動電話", + "聯絡電話", + "市話", + "office phone", + "phone", + "phone number", + "mobile", + "cell", + "call", + "contact", +] + + +@pytest.fixture(scope="module") +def recognizer(): + """Create a TW-configured PhoneRecognizer instance for testing.""" + return TwPhoneNumberRecognizer() + + +@pytest.fixture(scope="module") +def entities(): + """Return the TW_PHONE_NUMBER entity type for testing.""" + return ["TW_PHONE_NUMBER"] + + +@pytest.mark.parametrize( + "text, expected_len, expected_positions, expected_score_ranges", + [ + ("+886912345678", 1, ((0, 13),), ((0.4, 1.0),)), + ("+886 912 345 678", 1, ((0, 16),), ((0.4, 1.0),)), + ("0912345678", 1, ((0, 10),), ((0.4, 1.0),)), + ("02 2345 6789", 1, ((0, 12),), ((0.4, 1.0),)), + ("(02) 2345-6789", 1, ((0, 14),), ((0.4, 1.0),)), + ( + "電話：02 2345 6789", + 1, + ((3, 15),), + ((0.4, 1.0),), + ), + ( + "手機號碼 0912345678 可於上班時間聯絡。", + 1, + ((5, 15),), + ((0.4, 1.0),), + ), + ( + "Office phone +886 2 2345 6789, mobile 0912345678", + 2, + ((13, 29), (38, 48),), + ((0.4, 1.0), (0.4, 1.0),), + ), + ("1234567890", 0, (), ()), + ("10000000146", 0, (), ()), + ("091234567", 0, (), ()), + ("09123456789", 0, (), ()), + ("hello world", 0, (), ()), + ], +) +def test_when_tw_phone_number_in_text_then_all_matches_are_found( + text, + expected_len, + expected_positions, + expected_score_ranges, + recognizer, + entities, +): + """Test that Taiwan phone number recognizer correctly identifies numbers.""" + results = sorted( + recognizer.analyze(text, entities), + key=lambda result: result.start, + ) + assert len(results) == expected_len + + for res, (st_pos, fn_pos), (st_score, fn_score) in zip( + results, expected_positions, expected_score_ranges + ): + assert_result_within_score_range( + res, entities[0], st_pos, fn_pos, st_score, fn_score + ) + + +def test_supported_entity(recognizer): + """Test that supported entity is correctly set.""" + assert recognizer.supported_entities == ["TW_PHONE_NUMBER"] + + +def test_supported_language(recognizer): + """Test that supported language is correctly set.""" + assert recognizer.supported_language == "zh" + + +def test_context_words(recognizer): + """Test that context words are properly set.""" + assert recognizer.context == TAIWAN_CONTEXT