From 14da1a6db3822baa0b47300064e605e0ab11164a Mon Sep 17 00:00:00 2001 From: Arjun Pakhan Date: Thu, 18 Jun 2026 05:53:00 +0000 Subject: [PATCH] feat: add Taiwan (TW) country-specific national ID recognizer --- bn_test/config.cfg | 104 ++++ bn_test/meta.json | 31 + bn_test/tokenizer | 4 + bn_test/vocab/key2row | 1 + bn_test/vocab/lookups.bin | 1 + bn_test/vocab/strings.json | 400 ++++++++++++ bn_test/vocab/vectors | Bin 0 -> 128 bytes bn_test/vocab/vectors.cfg | 3 + he_test/config.cfg | 104 ++++ he_test/meta.json | 31 + he_test/tokenizer | 3 + he_test/vocab/key2row | 1 + he_test/vocab/lookups.bin | 1 + he_test/vocab/strings.json | 348 +++++++++++ he_test/vocab/vectors | Bin 0 -> 128 bytes he_test/vocab/vectors.cfg | 3 + presidio-analyzer/app.py | 7 +- .../presidio_analyzer/analyzer_engine.py | 1 + .../presidio_analyzer/chunkers/__init__.py | 1 - .../chunkers/base_chunker.py | 1 + .../chunkers/character_based_text_chunker.py | 1 + .../chunkers/text_chunker_provider.py | 1 - .../conf/default_analyzer_full.yaml | 589 +++++++++++++----- .../conf/default_recognizers.yaml | 102 ++- .../presidio_analyzer/conf/slim.yaml | 158 ++--- .../yaml_recognizer_models.py | 4 +- .../presidio_analyzer/llm_utils/__init__.py | 1 - .../llm_utils/azure_auth_helper.py | 8 +- .../llm_utils/config_loader.py | 7 +- .../llm_utils/entity_mapper.py | 36 +- .../llm_utils/examples_loader.py | 5 +- .../llm_utils/langextract_helper.py | 28 +- .../llm_utils/prompt_loader.py | 1 + .../presidio_analyzer/lm_recognizer.py | 26 +- .../nlp_engine/stanza_nlp_engine.py | 6 +- .../predefined_recognizers/__init__.py | 2 + .../germany/de_bsnr_recognizer.py | 44 +- .../germany/de_social_security_recognizer.py | 10 +- .../country_specific/tw/__init__.py | 5 + .../tw/tw_national_id_recognizer.py | 113 ++++ .../generic/iban_recognizer.py | 1 + .../generic/mac_recognizer.py | 6 +- .../ner/gliner_recognizer.py | 4 +- .../third_party/ahds_recognizer.py | 9 +- .../azure_openai_langextract_recognizer.py | 1 + .../third_party/azure_openai_provider.py | 32 +- .../basic_langextract_recognizer.py | 28 +- .../third_party/langextract_recognizer.py | 33 +- presidio-analyzer/tests/__init__.py | 6 +- presidio-analyzer/tests/assertions.py | 11 +- presidio-analyzer/tests/conftest.py | 5 +- .../tests/test_ahds_recognizer.py | 12 +- ...st_ahds_recognizer_credential_selection.py | 171 +++-- .../tests/test_analyzer_engine.py | 60 +- .../tests/test_analyzer_engine_provider.py | 53 +- .../tests/test_analyzer_request.py | 51 +- .../tests/test_au_abn_recognizer.py | 62 +- .../tests/test_au_acn_recognizer.py | 67 +- .../tests/test_au_medicare_recognizer.py | 53 +- .../tests/test_au_tfn_recognizer.py | 62 +- .../test_azure_ai_language_recognizer.py | 23 +- .../tests/test_azure_auth_helper.py | 34 +- ...est_azure_openai_langextract_recognizer.py | 292 +++++---- presidio-analyzer/tests/test_base_chunker.py | 5 +- .../test_basic_langextract_recognizer.py | 354 +++++++---- .../tests/test_batch_analyzer_engine.py | 31 +- .../test_certificate_number_recognizer.py | 4 +- .../test_character_based_text_chunker.py | 8 +- presidio-analyzer/tests/test_config_loader.py | 67 +- .../tests/test_configuration_validator.py | 141 ++--- .../tests/test_context_support.py | 5 +- .../tests/test_date_recognizer.py | 2 +- .../tests/test_de_bsnr_recognizer.py | 29 +- .../tests/test_de_fuehrerschein_recognizer.py | 1 + .../test_de_handelsregister_recognizer.py | 1 + .../test_de_health_insurance_recognizer.py | 3 +- .../tests/test_de_id_card_recognizer.py | 11 +- .../tests/test_de_kfz_recognizer.py | 1 + .../tests/test_de_lanr_recognizer.py | 3 +- .../tests/test_de_passport_recognizer.py | 1 + .../tests/test_de_plz_recognizer.py | 1 + .../test_de_social_security_recognizer.py | 3 +- .../tests/test_de_tax_id_recognizer.py | 7 +- .../tests/test_de_tax_number_recognizer.py | 1 + .../tests/test_de_vat_id_recognizer.py | 4 +- .../tests/test_device_detector.py | 65 +- presidio-analyzer/tests/test_entity_mapper.py | 77 +-- .../tests/test_entity_recognizer.py | 2 + .../tests/test_es_nie_recognizer.py | 43 +- .../tests/test_es_passport_recognizer.py | 94 ++- .../tests/test_examples_loader.py | 102 +-- .../tests/test_gliner_recognizer.py | 52 +- .../tests/test_huggingface_ner_recognizer.py | 6 +- .../tests/test_iban_recognizer.py | 4 +- .../tests/test_in_aadhaar_recognizer.py | 6 +- .../tests/test_in_gstin_recognizer.py | 56 +- .../tests/test_in_passport_recognizer.py | 17 +- .../tests/test_kr_brn_recognizer.py | 7 +- .../tests/test_kr_frn_recognizer.py | 137 +++- .../tests/test_kr_passport_recognizer.py | 55 +- .../tests/test_kr_rrn_recognizer.py | 133 +++- .../tests/test_langextract_helper.py | 54 +- .../tests/test_language_validation.py | 1 + .../test_lemma_context_aware_enhancer.py | 98 ++- presidio-analyzer/tests/test_lm_recognizer.py | 79 +-- .../tests/test_medical_ner_recognizer.py | 3 +- .../tests/test_ner_model_configuration.py | 8 +- .../tests/test_nlp_engine_provider.py | 49 +- presidio-analyzer/tests/test_pattern.py | 7 +- .../tests/test_pattern_recognizer.py | 27 +- .../tests/test_phone_recognizer.py | 11 +- presidio-analyzer/tests/test_prompt_loader.py | 14 +- .../tests/test_recognizer_registry.py | 7 +- .../test_recognizer_registry_provider.py | 88 ++- .../tests/test_recognizers_loader_utils.py | 3 + .../test_se_organisationsnummer_recognizer.py | 5 +- .../tests/test_se_personnummer_recognizer.py | 3 +- .../tests/test_sg_fin_recognizer.py | 3 +- .../tests/test_slim_spacy_nlp_engine.py | 22 +- .../tests/test_spacy_nlp_engine.py | 70 ++- .../tests/test_spacy_recognizer.py | 19 +- .../tests/test_stanza_batch_processing.py | 151 ++--- .../tests/test_stanza_nlp_engine.py | 2 + .../tests/test_text_chunker_provider.py | 29 +- .../tests/test_th_tnin_recognizer.py | 261 ++++++-- .../tests/test_tr_national_id_recognizer.py | 143 ++++- .../tests/test_tr_phone_number_recognizer.py | 1 - .../tests/test_transformers_recognizer.py | 1 - .../tests/test_tw_national_id_recognizer.py | 60 ++ .../tests/test_uk_nino_recognizer.py | 2 +- .../tests/test_url_recognizer.py | 2 +- .../tests/test_us_mbi_recognizer.py | 8 +- .../tests/test_yaml_recognizer_models.py | 194 ++---- 133 files changed, 4295 insertions(+), 1937 deletions(-) create mode 100644 bn_test/config.cfg create mode 100644 bn_test/meta.json create mode 100644 bn_test/tokenizer create mode 100644 bn_test/vocab/key2row create mode 100644 bn_test/vocab/lookups.bin create mode 100644 bn_test/vocab/strings.json create mode 100644 bn_test/vocab/vectors create mode 100644 bn_test/vocab/vectors.cfg create mode 100644 he_test/config.cfg create mode 100644 he_test/meta.json create mode 100644 he_test/tokenizer create mode 100644 he_test/vocab/key2row create mode 100644 he_test/vocab/lookups.bin create mode 100644 he_test/vocab/strings.json create mode 100644 he_test/vocab/vectors create mode 100644 he_test/vocab/vectors.cfg create mode 100644 presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/tw/__init__.py create mode 100644 presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/tw/tw_national_id_recognizer.py create mode 100644 presidio-analyzer/tests/test_tw_national_id_recognizer.py diff --git a/bn_test/config.cfg b/bn_test/config.cfg new file mode 100644 index 0000000000..5e8a7ff8e0 --- /dev/null +++ b/bn_test/config.cfg @@ -0,0 +1,104 @@ +[paths] +train = null +dev = null +vectors = null +init_tok2vec = null + +[system] +seed = 0 +gpu_allocator = null + +[nlp] +lang = "bn" +pipeline = [] +disabled = [] +before_creation = null +after_creation = null +after_pipeline_creation = null +batch_size = 1000 + +[corpora] + +[training] +seed = ${system.seed} +gpu_allocator = ${system.gpu_allocator} +dropout = 0.1 +accumulate_gradient = 1 +patience = 1600 +max_epochs = 0 +max_steps = 20000 +eval_frequency = 200 +frozen_components = [] +annotating_components = [] +dev_corpus = "corpora.dev" +train_corpus = "corpora.train" +before_to_disk = null +before_update = null + +[initialize] +vectors = ${paths.vectors} +init_tok2vec = ${paths.init_tok2vec} +vocab_data = null +lookups = null +before_init = null +after_init = null + +[components] + +[pretraining] + +[nlp.tokenizer] +@tokenizers = "spacy.Tokenizer.v1" + +[nlp.vectors] +@vectors = "spacy.Vectors.v1" + +[corpora.train] +@readers = "spacy.Corpus.v1" +path = ${paths.train} +gold_preproc = false +max_length = 0 +limit = 0 +augmenter = null + +[corpora.dev] +@readers = "spacy.Corpus.v1" +path = ${paths.dev} +gold_preproc = false +max_length = 0 +limit = 0 +augmenter = null + +[training.logger] +@loggers = "spacy.ConsoleLogger.v1" +progress_bar = false + +[training.batcher] +@batchers = "spacy.batch_by_words.v1" +discard_oversize = false +tolerance = 0.2 +get_length = null + +[training.optimizer] +@optimizers = "Adam.v1" +beta1 = 0.9 +beta2 = 0.999 +L2_is_weight_decay = true +L2 = 0.01 +grad_clip = 1.0 +use_averages = false +eps = 1e-08 +learn_rate = 0.001 + +[training.score_weights] + +[initialize.tokenizer] + +[initialize.components] + +[training.batcher.size] +@schedules = "compounding.v1" +start = 100 +stop = 1000 +compound = 1.001 +t = 0.0 \ No newline at end of file diff --git a/bn_test/meta.json b/bn_test/meta.json new file mode 100644 index 0000000000..34ea2b5a95 --- /dev/null +++ b/bn_test/meta.json @@ -0,0 +1,31 @@ +{ + "lang":"bn", + "name":"pipeline", + "version":"0.0.0", + "spacy_version":">=3.8.14,<3.9.0", + "description":"", + "author":"", + "email":"", + "url":"", + "license":"", + "spacy_git_version":"0069cf9", + "vectors":{ + "width":0, + "vectors":0, + "keys":0, + "name":null, + "mode":"default" + }, + "labels":{ + + }, + "pipeline":[ + + ], + "components":[ + + ], + "disabled":[ + + ] +} \ No newline at end of file diff --git a/bn_test/tokenizer b/bn_test/tokenizer new file mode 100644 index 0000000000..d35811ea93 --- /dev/null +++ b/bn_test/tokenizer @@ -0,0 +1,4 @@ +prefix_search +^\+|^…|^……|^,|^:|^;|^\!|^\?|^¿|^؟|^¡|^\(|^\)|^\[|^\]|^\{|^\}|^<|^>|^_|^#|^\*|^&|^。|^？|^！|^，|^、|^；|^：|^～|^·|^।|^،|^۔|^؛|^٪|^।|^॥|^\.\.+|^…|^\'|^"|^”|^“|^`|^‘|^´|^’|^‚|^,|^„|^»|^«|^「|^」|^『|^』|^（|^）|^〔|^〕|^【|^】|^《|^》|^〈|^〉|^〈|^〉|^⟦|^⟧|^[\u00A6\u00A9\u00AE\u00B0\u0482\u058D\u058E\u060E\u060F\u06DE\u06E9\u06FD\u06FE\u07F6\u09FA\u0B70\u0BF3-\u0BF8\u0BFA\u0C7F\u0D4F\u0D79\u0F01-\u0F03\u0F13\u0F15-\u0F17\u0F1A-\u0F1F\u0F34\u0F36\u0F38\u0FBE-\u0FC5\u0FC7-\u0FCC\u0FCE\u0FCF\u0FD5-\u0FD8\u109E\u109F\u1390-\u1399\u1940\u19DE-\u19FF\u1B61-\u1B6A\u1B74-\u1B7C\u2100\u2101\u2103-\u2106\u2108\u2109\u2114\u2116\u2117\u211E-\u2123\u2125\u2127\u2129\u212E\u213A\u213B\u214A\u214C\u214D\u214F\u218A\u218B\u2195-\u2199\u219C-\u219F\u21A1\u21A2\u21A4\u21A5\u21A7-\u21AD\u21AF-\u21CD\u21D0\u21D1\u21D3\u21D5-\u21F3\u2300-\u2307\u230C-\u231F\u2322-\u2328\u232B-\u237B\u237D-\u239A\u23B4-\u23DB\u23E2-\u2426\u2440-\u244A\u249C-\u24E9\u2500-\u25B6\u25B8-\u25C0\u25C2-\u25F7\u2600-\u266E\u2670-\u2767\u2794-\u27BF\u2800-\u28FF\u2B00-\u2B2F\u2B45\u2B46\u2B4D-\u2B73\u2B76-\u2B95\u2B98-\u2BC8\u2BCA-\u2BFE\u2CE5-\u2CEA\u2E80-\u2E99\u2E9B-\u2EF3\u2F00-\u2FD5\u2FF0-\u2FFB\u3004\u3012\u3013\u3020\u3036\u3037\u303E\u303F\u3190\u3191\u3196-\u319F\u31C0-\u31E3\u3200-\u321E\u322A-\u3247\u3250\u3260-\u327F\u328A-\u32B0\u32C0-\u32FE\u3300-\u33FF\u4DC0-\u4DFF\uA490-\uA4C6\uA828-\uA82B\uA836\uA837\uA839\uAA77-\uAA79\uFDFD\uFFE4\uFFE8\uFFED\uFFEE\uFFFC\uFFFD\U00010137-\U0001013F\U00010179-\U00010189\U0001018C-\U0001018E\U00010190-\U0001019B\U000101A0\U000101D0-\U000101FC\U00010877\U00010878\U00010AC8\U0001173F\U00016B3C-\U00016B3F\U00016B45\U0001BC9C\U0001D000-\U0001D0F5\U0001D100-\U0001D126\U0001D129-\U0001D164\U0001D16A-\U0001D16C\U0001D183\U0001D184\U0001D18C-\U0001D1A9\U0001D1AE-\U0001D1E8\U0001D200-\U0001D241\U0001D245\U0001D300-\U0001D356\U0001D800-\U0001D9FF\U0001DA37-\U0001DA3A\U0001DA6D-\U0001DA74\U0001DA76-\U0001DA83\U0001DA85\U0001DA86\U0001ECAC\U0001F000-\U0001F02B\U0001F030-\U0001F093\U0001F0A0-\U0001F0AE\U0001F0B1-\U0001F0BF\U0001F0C1-\U0001F0CF\U0001F0D1-\U0001F0F5\U0001F110-\U0001F16B\U0001F170-\U0001F1AC\U0001F1E6-\U0001F202\U0001F210-\U0001F23B\U0001F240-\U0001F248\U0001F250\U0001F251\U0001F260-\U0001F265\U0001F300-\U0001F3FA\U0001F400-\U0001F6D4\U0001F6E0-\U0001F6EC\U0001F6F0-\U0001F6F9\U0001F700-\U0001F773\U0001F780-\U0001F7D8\U0001F800-\U0001F80B\U0001F810-\U0001F847\U0001F850-\U0001F859\U0001F860-\U0001F887\U0001F890-\U0001F8AD\U0001F900-\U0001F90B\U0001F910-\U0001F93E\U0001F940-\U0001F970\U0001F973-\U0001F976\U0001F97A\U0001F97C-\U0001F9A2\U0001F9B0-\U0001F9B9\U0001F9C0-\U0001F9C2\U0001F9D0-\U0001F9FF\U0001FA60-\U0001FA6D]suffix_search…$|……$|,$|:$|;$|\!$|\?$|¿$|؟$|¡$|\($|\)$|\[$|\]$|\{$|\}$|<$|>$|_$|#$|\*$|&$|。$|？$|！$|，$|、$|；$|：$|～$|·$|।$|،$|۔$|؛$|٪$|।$|॥$|\.\.+$|…$|\'$|"$|”$|“$|`$|‘$|´$|’$|‚$|,$|„$|»$|«$|「$|」$|『$|』$|（$|）$|〔$|〕$|【$|】$|《$|》$|〈$|〉$|〈$|〉$|⟦$|⟧$|[\u00A6\u00A9\u00AE\u00B0\u0482\u058D\u058E\u060E\u060F\u06DE\u06E9\u06FD\u06FE\u07F6\u09FA\u0B70\u0BF3-\u0BF8\u0BFA\u0C7F\u0D4F\u0D79\u0F01-\u0F03\u0F13\u0F15-\u0F17\u0F1A-\u0F1F\u0F34\u0F36\u0F38\u0FBE-\u0FC5\u0FC7-\u0FCC\u0FCE\u0FCF\u0FD5-\u0FD8\u109E\u109F\u1390-\u1399\u1940\u19DE-\u19FF\u1B61-\u1B6A\u1B74-\u1B7C\u2100\u2101\u2103-\u2106\u2108\u2109\u2114\u2116\u2117\u211E-\u2123\u2125\u2127\u2129\u212E\u213A\u213B\u214A\u214C\u214D\u214F\u218A\u218B\u2195-\u2199\u219C-\u219F\u21A1\u21A2\u21A4\u21A5\u21A7-\u21AD\u21AF-\u21CD\u21D0\u21D1\u21D3\u21D5-\u21F3\u2300-\u2307\u230C-\u231F\u2322-\u2328\u232B-\u237B\u237D-\u239A\u23B4-\u23DB\u23E2-\u2426\u2440-\u244A\u249C-\u24E9\u2500-\u25B6\u25B8-\u25C0\u25C2-\u25F7\u2600-\u266E\u2670-\u2767\u2794-\u27BF\u2800-\u28FF\u2B00-\u2B2F\u2B45\u2B46\u2B4D-\u2B73\u2B76-\u2B95\u2B98-\u2BC8\u2BCA-\u2BFE\u2CE5-\u2CEA\u2E80-\u2E99\u2E9B-\u2EF3\u2F00-\u2FD5\u2FF0-\u2FFB\u3004\u3012\u3013\u3020\u3036\u3037\u303E\u303F\u3190\u3191\u3196-\u319F\u31C0-\u31E3\u3200-\u321E\u322A-\u3247\u3250\u3260-\u327F\u328A-\u32B0\u32C0-\u32FE\u3300-\u33FF\u4DC0-\u4DFF\uA490-\uA4C6\uA828-\uA82B\uA836\uA837\uA839\uAA77-\uAA79\uFDFD\uFFE4\uFFE8\uFFED\uFFEE\uFFFC\uFFFD\U00010137-\U0001013F\U00010179-\U00010189\U0001018C-\U0001018E\U00010190-\U0001019B\U000101A0\U000101D0-\U000101FC\U00010877\U00010878\U00010AC8\U0001173F\U00016B3C-\U00016B3F\U00016B45\U0001BC9C\U0001D000-\U0001D0F5\U0001D100-\U0001D126\U0001D129-\U0001D164\U0001D16A-\U0001D16C\U0001D183\U0001D184\U0001D18C-\U0001D1A9\U0001D1AE-\U0001D1E8\U0001D200-\U0001D241\U0001D245\U0001D300-\U0001D356\U0001D800-\U0001D9FF\U0001DA37-\U0001DA3A\U0001DA6D-\U0001DA74\U0001DA76-\U0001DA83\U0001DA85\U0001DA86\U0001ECAC\U0001F000-\U0001F02B\U0001F030-\U0001F093\U0001F0A0-\U0001F0AE\U0001F0B1-\U0001F0BF\U0001F0C1-\U0001F0CF\U0001F0D1-\U0001F0F5\U0001F110-\U0001F16B\U0001F170-\U0001F1AC\U0001F1E6-\U0001F202\U0001F210-\U0001F23B\U0001F240-\U0001F248\U0001F250\U0001F251\U0001F260-\U0001F265\U0001F300-\U0001F3FA\U0001F400-\U0001F6D4\U0001F6E0-\U0001F6EC\U0001F6F0-\U0001F6F9\U0001F700-\U0001F773\U0001F780-\U0001F7D8\U0001F800-\U0001F80B\U0001F810-\U0001F847\U0001F850-\U0001F859\U0001F860-\U0001F887\U0001F890-\U0001F8AD\U0001F900-\U0001F90B\U0001F910-\U0001F93E\U0001F940-\U0001F970\U0001F973-\U0001F976\U0001F97A\U0001F97C-\U0001F9A2\U0001F9B0-\U0001F9B9\U0001F9C0-\U0001F9C2\U0001F9D0-\U0001F9FF\U0001FA60-\U0001FA6D]$|(?<=[0-9])\+$|(?<=°[FfCcKk])\.$|(?<=[0-9])(?:[\$¢£€¥฿৳])$|(?<=[0-9])(?:km|km²|km³|m|m²|m³|dm|dm²|dm³|cm|cm²|cm³|mm|mm²|mm³|ha|µm|nm|yd|in|ft|kg|g|mg|µg|t|lb|oz|m/s|km/h|kmh|mph|hPa|Pa|mbar|mb|MB|kb|KB|gb|GB|tb|TB|T|G|M|K|%|км|км²|км³|м|м²|м³|дм|дм²|дм³|см|см²|см³|мм|мм²|мм³|нм|кг|г|мг|м/с|км/ч|кПа|Па|мбар|Кб|КБ|кб|Мб|МБ|мб|Гб|ГБ|гб|Тб|ТБ|тбكم|كم²|كم³|م|م²|م³|سم|سم²|سم³|مم|مم²|مم³|كم|غرام|جرام|جم|كغ|ملغ|كوب|اكواب)$|(?<=[a-z\uFF41-\uFF5A\u00DF-\u00F6\u00F8-\u00FF\u0101\u0103\u0105\u0107\u0109\u010B\u010D\u010F\u0111\u0113\u0115\u0117\u0119\u011B\u011D\u011F\u0121\u0123\u0125\u0127\u0129\u012B\u012D\u012F\u0131\u0133\u0135\u0137\u0138\u013A\u013C\u013E\u0140\u0142\u0144\u0146\u0148\u0149\u014B\u014D\u014F\u0151\u0153\u0155\u0157\u0159\u015B\u015D\u015F\u0161\u0163\u0165\u0167\u0169\u016B\u016D\u016F\u0171\u0173\u0175\u0177\u017A\u017C\u017E\u017F\u0180\u0183\u0185\u0188\u018C\u018D\u0192\u0195\u0199-\u019B\u019E\u01A1\u01A3\u01A5\u01A8\u01AA\u01AB\u01AD\u01B0\u01B4\u01B6\u01B9\u01BA\u01BD-\u01BF\u01C6\u01C9\u01CC\u01CE\u01D0\u01D2\u01D4\u01D6\u01D8\u01DA\u01DC\u01DD\u01DF\u01E1\u01E3\u01E5\u01E7\u01E9\u01EB\u01ED\u01EF\u01F0\u01F3\u01F5\u01F9\u01FB\u01FD\u01FF\u0201\u0203\u0205\u0207\u0209\u020B\u020D\u020F\u0211\u0213\u0215\u0217\u0219\u021B\u021D\u021F\u0221\u0223\u0225\u0227\u0229\u022B\u022D\u022F\u0231\u0233-\u0239\u023C\u023F\u0240\u0242\u0247\u0249\u024B\u024D\u024F\u2C61\u2C65\u2C66\u2C68\u2C6A\u2C6C\u2C71\u2C73\u2C74\u2C76-\u2C7B\uA723\uA725\uA727\uA729\uA72B\uA72D\uA72F-\uA731\uA733\uA735\uA737\uA739\uA73B\uA73D\uA73F\uA741\uA743\uA745\uA747\uA749\uA74B\uA74D\uA74F\uA751\uA753\uA755\uA757\uA759\uA75B\uA75D\uA75F\uA761\uA763\uA765\uA767\uA769\uA76B\uA76D\uA76F\uA771-\uA778\uA77A\uA77C\uA77F\uA781\uA783\uA785\uA787\uA78C\uA78E\uA791\uA793-\uA795\uA797\uA799\uA79B\uA79D\uA79F\uA7A1\uA7A3\uA7A5\uA7A7\uA7A9\uA7AF\uA7B5\uA7B7\uA7B9\uA7FA\uAB30-\uAB5A\uAB60-\uAB64\u0250-\u02AF\u1D00-\u1D25\u1D6B-\u1D77\u1D79-\u1D9A\u1E01\u1E03\u1E05\u1E07\u1E09\u1E0B\u1E0D\u1E0F\u1E11\u1E13\u1E15\u1E17\u1E19\u1E1B\u1E1D\u1E1F\u1E21\u1E23\u1E25\u1E27\u1E29\u1E2B\u1E2D\u1E2F\u1E31\u1E33\u1E35\u1E37\u1E39\u1E3B\u1E3D\u1E3F\u1E41\u1E43\u1E45\u1E47\u1E49\u1E4B\u1E4D\u1E4F\u1E51\u1E53\u1E55\u1E57\u1E59\u1E5B\u1E5D\u1E5F\u1E61\u1E63\u1E65\u1E67\u1E69\u1E6B\u1E6D\u1E6F\u1E71\u1E73\u1E75\u1E77\u1E79\u1E7B\u1E7D\u1E7F\u1E81\u1E83\u1E85\u1E87\u1E89\u1E8B\u1E8D\u1E8F\u1E91\u1E93\u1E95-\u1E9D\u1E9F\u1EA1\u1EA3\u1EA5\u1EA7\u1EA9\u1EAB\u1EAD\u1EAF\u1EB1\u1EB3\u1EB5\u1EB7\u1EB9\u1EBB\u1EBD\u1EBF\u1EC1\u1EC3\u1EC5\u1EC7\u1EC9\u1ECB\u1ECD\u1ECF\u1ED1\u1ED3\u1ED5\u1ED7\u1ED9\u1EDB\u1EDD\u1EDF\u1EE1\u1EE3\u1EE5\u1EE7\u1EE9\u1EEB\u1EED\u1EEF\u1EF1\u1EF3\u1EF5\u1EF7\u1EF9\u1EFB\u1EFD\u1EFFёа-яәөүҗңһα-ωάέίόώήύа-щюяіїєґѓѕјљњќѐѝ\u1200-\u137F\u0980-\u09FF\u0591-\u05F4\uFB1D-\uFB4F\u0620-\u064A\u066E-\u06D5\u06E5-\u06FF\u0750-\u077F\u08A0-\u08BD\uFB50-\uFBB1\uFBD3-\uFD3D\uFD50-\uFDC7\uFDF0-\uFDFB\uFE70-\uFEFC\U0001EE00-\U0001EEBB\u0D80-\u0DFF\u0900-\u097F\u0C80-\u0CFF\u0B80-\u0BFF\u0C00-\u0C7F\uAC00-\uD7AF\u1100-\u11FF\u3040-\u309F\u30A0-\u30FFー\u4E00-\u62FF\u6300-\u77FF\u7800-\u8CFF\u8D00-\u9FFF\u3400-\u4DBF\U00020000-\U000215FF\U00021600-\U000230FF\U00023100-\U000245FF\U00024600-\U000260FF\U00026100-\U000275FF\U00027600-\U000290FF\U00029100-\U0002A6DF\U0002A700-\U0002B73F\U0002B740-\U0002B81F\U0002B820-\U0002CEAF\U0002CEB0-\U0002EBEF\u2E80-\u2EFF\u2F00-\u2FDF\u2FF0-\u2FFF\u3000-\u303F\u31C0-\u31EF\u3200-\u32FF\u3300-\u33FF\uF900-\uFAFF\uFE30-\uFE4F\U0001F200-\U0001F2FF\U0002F800-\U0002FA1F%²\-\+\'"”“`‘´’‚,„»«「」『』（）〔〕【】《》〈〉〈〉⟦⟧(?:\$¢£€¥฿৳)])\.$infix_finditer+\.\.+|…|[\u00A6\u00A9\u00AE\u00B0\u0482\u058D\u058E\u060E\u060F\u06DE\u06E9\u06FD\u06FE\u07F6\u09FA\u0B70\u0BF3-\u0BF8\u0BFA\u0C7F\u0D4F\u0D79\u0F01-\u0F03\u0F13\u0F15-\u0F17\u0F1A-\u0F1F\u0F34\u0F36\u0F38\u0FBE-\u0FC5\u0FC7-\u0FCC\u0FCE\u0FCF\u0FD5-\u0FD8\u109E\u109F\u1390-\u1399\u1940\u19DE-\u19FF\u1B61-\u1B6A\u1B74-\u1B7C\u2100\u2101\u2103-\u2106\u2108\u2109\u2114\u2116\u2117\u211E-\u2123\u2125\u2127\u2129\u212E\u213A\u213B\u214A\u214C\u214D\u214F\u218A\u218B\u2195-\u2199\u219C-\u219F\u21A1\u21A2\u21A4\u21A5\u21A7-\u21AD\u21AF-\u21CD\u21D0\u21D1\u21D3\u21D5-\u21F3\u2300-\u2307\u230C-\u231F\u2322-\u2328\u232B-\u237B\u237D-\u239A\u23B4-\u23DB\u23E2-\u2426\u2440-\u244A\u249C-\u24E9\u2500-\u25B6\u25B8-\u25C0\u25C2-\u25F7\u2600-\u266E\u2670-\u2767\u2794-\u27BF\u2800-\u28FF\u2B00-\u2B2F\u2B45\u2B46\u2B4D-\u2B73\u2B76-\u2B95\u2B98-\u2BC8\u2BCA-\u2BFE\u2CE5-\u2CEA\u2E80-\u2E99\u2E9B-\u2EF3\u2F00-\u2FD5\u2FF0-\u2FFB\u3004\u3012\u3013\u3020\u3036\u3037\u303E\u303F\u3190\u3191\u3196-\u319F\u31C0-\u31E3\u3200-\u321E\u322A-\u3247\u3250\u3260-\u327F\u328A-\u32B0\u32C0-\u32FE\u3300-\u33FF\u4DC0-\u4DFF\uA490-\uA4C6\uA828-\uA82B\uA836\uA837\uA839\uAA77-\uAA79\uFDFD\uFFE4\uFFE8\uFFED\uFFEE\uFFFC\uFFFD\U00010137-\U0001013F\U00010179-\U00010189\U0001018C-\U0001018E\U00010190-\U0001019B\U000101A0\U000101D0-\U000101FC\U00010877\U00010878\U00010AC8\U0001173F\U00016B3C-\U00016B3F\U00016B45\U0001BC9C\U0001D000-\U0001D0F5\U0001D100-\U0001D126\U0001D129-\U0001D164\U0001D16A-\U0001D16C\U0001D183\U0001D184\U0001D18C-\U0001D1A9\U0001D1AE-\U0001D1E8\U0001D200-\U0001D241\U0001D245\U0001D300-\U0001D356\U0001D800-\U0001D9FF\U0001DA37-\U0001DA3A\U0001DA6D-\U0001DA74\U0001DA76-\U0001DA83\U0001DA85\U0001DA86\U0001ECAC\U0001F000-\U0001F02B\U0001F030-\U0001F093\U0001F0A0-\U0001F0AE\U0001F0B1-\U0001F0BF\U0001F0C1-\U0001F0CF\U0001F0D1-\U0001F0F5\U0001F110-\U0001F16B\U0001F170-\U0001F1AC\U0001F1E6-\U0001F202\U0001F210-\U0001F23B\U0001F240-\U0001F248\U0001F250\U0001F251\U0001F260-\U0001F265\U0001F300-\U0001F3FA\U0001F400-\U0001F6D4\U0001F6E0-\U0001F6EC\U0001F6F0-\U0001F6F9\U0001F700-\U0001F773\U0001F780-\U0001F7D8\U0001F800-\U0001F80B\U0001F810-\U0001F847\U0001F850-\U0001F859\U0001F860-\U0001F887\U0001F890-\U0001F8AD\U0001F900-\U0001F90B\U0001F910-\U0001F93E\U0001F940-\U0001F970\U0001F973-\U0001F976\U0001F97A\U0001F97C-\U0001F9A2\U0001F9B0-\U0001F9B9\U0001F9C0-\U0001F9C2\U0001F9D0-\U0001F9FF\U0001FA60-\U0001FA6D]|(?<=[0-9০-৯])[+\-\*^=](?=[0-9০-৯-])|(?<=[A-Za-z\uFF21-\uFF3A\uFF41-\uFF5A\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u00FF\u0100-\u017F\u0180-\u01BF\u01C4-\u024F\u2C60-\u2C7B\u2C7E\u2C7F\uA722-\uA76F\uA771-\uA787\uA78B-\uA78E\uA790-\uA7B9\uA7FA\uAB30-\uAB5A\uAB60-\uAB64\u0250-\u02AF\u1D00-\u1D25\u1D6B-\u1D77\u1D79-\u1D9A\u1E00-\u1EFFёа-яЁА-ЯәөүҗңһӘӨҮҖҢҺα-ωάέίόώήύΑ-ΩΆΈΊΌΏΉΎа-щюяіїєґА-ЩЮЯІЇЄҐѓѕјљњќѐѝЃЅЈЉЊЌЀЍ\u1200-\u137F\u0980-\u09FF\u0591-\u05F4\uFB1D-\uFB4F\u0620-\u064A\u066E-\u06D5\u06E5-\u06FF\u0750-\u077F\u08A0-\u08BD\uFB50-\uFBB1\uFBD3-\uFD3D\uFD50-\uFDC7\uFDF0-\uFDFB\uFE70-\uFEFC\U0001EE00-\U0001EEBB\u0D80-\u0DFF\u0900-\u097F\u0C80-\u0CFF\u0B80-\u0BFF\u0C00-\u0C7F\uAC00-\uD7AF\u1100-\u11FF\u3040-\u309F\u30A0-\u30FFー\u4E00-\u62FF\u6300-\u77FF\u7800-\u8CFF\u8D00-\u9FFF\u3400-\u4DBF\U00020000-\U000215FF\U00021600-\U000230FF\U00023100-\U000245FF\U00024600-\U000260FF\U00026100-\U000275FF\U00027600-\U000290FF\U00029100-\U0002A6DF\U0002A700-\U0002B73F\U0002B740-\U0002B81F\U0002B820-\U0002CEAF\U0002CEB0-\U0002EBEF\u2E80-\u2EFF\u2F00-\u2FDF\u2FF0-\u2FFF\u3000-\u303F\u31C0-\u31EF\u3200-\u32FF\u3300-\u33FF\uF900-\uFAFF\uFE30-\uFE4F\U0001F200-\U0001F2FF\U0002F800-\U0002FA1F]),(?=[A-Za-z\uFF21-\uFF3A\uFF41-\uFF5A\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u00FF\u0100-\u017F\u0180-\u01BF\u01C4-\u024F\u2C60-\u2C7B\u2C7E\u2C7F\uA722-\uA76F\uA771-\uA787\uA78B-\uA78E\uA790-\uA7B9\uA7FA\uAB30-\uAB5A\uAB60-\uAB64\u0250-\u02AF\u1D00-\u1D25\u1D6B-\u1D77\u1D79-\u1D9A\u1E00-\u1EFFёа-яЁА-ЯәөүҗңһӘӨҮҖҢҺα-ωάέίόώήύΑ-ΩΆΈΊΌΏΉΎа-щюяіїєґА-ЩЮЯІЇЄҐѓѕјљњќѐѝЃЅЈЉЊЌЀЍ\u1200-\u137F\u0980-\u09FF\u0591-\u05F4\uFB1D-\uFB4F\u0620-\u064A\u066E-\u06D5\u06E5-\u06FF\u0750-\u077F\u08A0-\u08BD\uFB50-\uFBB1\uFBD3-\uFD3D\uFD50-\uFDC7\uFDF0-\uFDFB\uFE70-\uFEFC\U0001EE00-\U0001EEBB\u0D80-\u0DFF\u0900-\u097F\u0C80-\u0CFF\u0B80-\u0BFF\u0C00-\u0C7F\uAC00-\uD7AF\u1100-\u11FF\u3040-\u309F\u30A0-\u30FFー\u4E00-\u62FF\u6300-\u77FF\u7800-\u8CFF\u8D00-\u9FFF\u3400-\u4DBF\U00020000-\U000215FF\U00021600-\U000230FF\U00023100-\U000245FF\U00024600-\U000260FF\U00026100-\U000275FF\U00027600-\U000290FF\U00029100-\U0002A6DF\U0002A700-\U0002B73F\U0002B740-\U0002B81F\U0002B820-\U0002CEAF\U0002CEB0-\U0002EBEF\u2E80-\u2EFF\u2F00-\u2FDF\u2FF0-\u2FFF\u3000-\u303F\u31C0-\u31EF\u3200-\u32FF\u3300-\u33FF\uF900-\uFAFF\uFE30-\uFE4F\U0001F200-\U0001F2FF\U0002F800-\U0002FA1F])|(?<=[A-Za-z\uFF21-\uFF3A\uFF41-\uFF5A\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u00FF\u0100-\u017F\u0180-\u01BF\u01C4-\u024F\u2C60-\u2C7B\u2C7E\u2C7F\uA722-\uA76F\uA771-\uA787\uA78B-\uA78E\uA790-\uA7B9\uA7FA\uAB30-\uAB5A\uAB60-\uAB64\u0250-\u02AF\u1D00-\u1D25\u1D6B-\u1D77\u1D79-\u1D9A\u1E00-\u1EFFёа-яЁА-ЯәөүҗңһӘӨҮҖҢҺα-ωάέίόώήύΑ-ΩΆΈΊΌΏΉΎа-щюяіїєґА-ЩЮЯІЇЄҐѓѕјљњќѐѝЃЅЈЉЊЌЀЍ\u1200-\u137F\u0980-\u09FF\u0591-\u05F4\uFB1D-\uFB4F\u0620-\u064A\u066E-\u06D5\u06E5-\u06FF\u0750-\u077F\u08A0-\u08BD\uFB50-\uFBB1\uFBD3-\uFD3D\uFD50-\uFDC7\uFDF0-\uFDFB\uFE70-\uFEFC\U0001EE00-\U0001EEBB\u0D80-\u0DFF\u0900-\u097F\u0C80-\u0CFF\u0B80-\u0BFF\u0C00-\u0C7F\uAC00-\uD7AF\u1100-\u11FF\u3040-\u309F\u30A0-\u30FFー\u4E00-\u62FF\u6300-\u77FF\u7800-\u8CFF\u8D00-\u9FFF\u3400-\u4DBF\U00020000-\U000215FF\U00021600-\U000230FF\U00023100-\U000245FF\U00024600-\U000260FF\U00026100-\U000275FF\U00027600-\U000290FF\U00029100-\U0002A6DF\U0002A700-\U0002B73F\U0002B740-\U0002B81F\U0002B820-\U0002CEAF\U0002CEB0-\U0002EBEF\u2E80-\u2EFF\u2F00-\u2FDF\u2FF0-\u2FFF\u3000-\u303F\u31C0-\u31EF\u3200-\u32FF\u3300-\u33FF\uF900-\uFAFF\uFE30-\uFE4F\U0001F200-\U0001F2FF\U0002F800-\U0002FA1F])(-|–|—|--|---|——|~)(?=[এ])|(?<=[A-Za-z\uFF21-\uFF3A\uFF41-\uFF5A\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u00FF\u0100-\u017F\u0180-\u01BF\u01C4-\u024F\u2C60-\u2C7B\u2C7E\u2C7F\uA722-\uA76F\uA771-\uA787\uA78B-\uA78E\uA790-\uA7B9\uA7FA\uAB30-\uAB5A\uAB60-\uAB64\u0250-\u02AF\u1D00-\u1D25\u1D6B-\u1D77\u1D79-\u1D9A\u1E00-\u1EFFёа-яЁА-ЯәөүҗңһӘӨҮҖҢҺα-ωάέίόώήύΑ-ΩΆΈΊΌΏΉΎа-щюяіїєґА-ЩЮЯІЇЄҐѓѕјљњќѐѝЃЅЈЉЊЌЀЍ\u1200-\u137F\u0980-\u09FF\u0591-\u05F4\uFB1D-\uFB4F\u0620-\u064A\u066E-\u06D5\u06E5-\u06FF\u0750-\u077F\u08A0-\u08BD\uFB50-\uFBB1\uFBD3-\uFD3D\uFD50-\uFDC7\uFDF0-\uFDFB\uFE70-\uFEFC\U0001EE00-\U0001EEBB\u0D80-\u0DFF\u0900-\u097F\u0C80-\u0CFF\u0B80-\u0BFF\u0C00-\u0C7F\uAC00-\uD7AF\u1100-\u11FF\u3040-\u309F\u30A0-\u30FFー\u4E00-\u62FF\u6300-\u77FF\u7800-\u8CFF\u8D00-\u9FFF\u3400-\u4DBF\U00020000-\U000215FF\U00021600-\U000230FF\U00023100-\U000245FF\U00024600-\U000260FF\U00026100-\U000275FF\U00027600-\U000290FF\U00029100-\U0002A6DF\U0002A700-\U0002B73F\U0002B740-\U0002B81F\U0002B820-\U0002CEAF\U0002CEB0-\U0002EBEF\u2E80-\u2EFF\u2F00-\u2FDF\u2FF0-\u2FFF\u3000-\u303F\u31C0-\u31EF\u3200-\u32FF\u3300-\u33FF\uF900-\uFAFF\uFE30-\uFE4F\U0001F200-\U0001F2FF\U0002F800-\U0002FA1F])(?:-|–|—|--|---|——|~)(?=[A-Za-z\uFF21-\uFF3A\uFF41-\uFF5A\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u00FF\u0100-\u017F\u0180-\u01BF\u01C4-\u024F\u2C60-\u2C7B\u2C7E\u2C7F\uA722-\uA76F\uA771-\uA787\uA78B-\uA78E\uA790-\uA7B9\uA7FA\uAB30-\uAB5A\uAB60-\uAB64\u0250-\u02AF\u1D00-\u1D25\u1D6B-\u1D77\u1D79-\u1D9A\u1E00-\u1EFFёа-яЁА-ЯәөүҗңһӘӨҮҖҢҺα-ωάέίόώήύΑ-ΩΆΈΊΌΏΉΎа-щюяіїєґА-ЩЮЯІЇЄҐѓѕјљњќѐѝЃЅЈЉЊЌЀЍ\u1200-\u137F\u0980-\u09FF\u0591-\u05F4\uFB1D-\uFB4F\u0620-\u064A\u066E-\u06D5\u06E5-\u06FF\u0750-\u077F\u08A0-\u08BD\uFB50-\uFBB1\uFBD3-\uFD3D\uFD50-\uFDC7\uFDF0-\uFDFB\uFE70-\uFEFC\U0001EE00-\U0001EEBB\u0D80-\u0DFF\u0900-\u097F\u0C80-\u0CFF\u0B80-\u0BFF\u0C00-\u0C7F\uAC00-\uD7AF\u1100-\u11FF\u3040-\u309F\u30A0-\u30FFー\u4E00-\u62FF\u6300-\u77FF\u7800-\u8CFF\u8D00-\u9FFF\u3400-\u4DBF\U00020000-\U000215FF\U00021600-\U000230FF\U00023100-\U000245FF\U00024600-\U000260FF\U00026100-\U000275FF\U00027600-\U000290FF\U00029100-\U0002A6DF\U0002A700-\U0002B73F\U0002B740-\U0002B81F\U0002B820-\U0002CEAF\U0002CEB0-\U0002EBEF\u2E80-\u2EFF\u2F00-\u2FDF\u2FF0-\u2FFF\u3000-\u303F\u31C0-\u31EF\u3200-\u32FF\u3300-\u33FF\uF900-\uFAFF\uFE30-\uFE4F\U0001F200-\U0001F2FF\U0002F800-\U0002FA1F])|(?<=[A-Za-z\uFF21-\uFF3A\uFF41-\uFF5A\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u00FF\u0100-\u017F\u0180-\u01BF\u01C4-\u024F\u2C60-\u2C7B\u2C7E\u2C7F\uA722-\uA76F\uA771-\uA787\uA78B-\uA78E\uA790-\uA7B9\uA7FA\uAB30-\uAB5A\uAB60-\uAB64\u0250-\u02AF\u1D00-\u1D25\u1D6B-\u1D77\u1D79-\u1D9A\u1E00-\u1EFFёа-яЁА-ЯәөүҗңһӘӨҮҖҢҺα-ωάέίόώήύΑ-ΩΆΈΊΌΏΉΎа-щюяіїєґА-ЩЮЯІЇЄҐѓѕјљњќѐѝЃЅЈЉЊЌЀЍ\u1200-\u137F\u0980-\u09FF\u0591-\u05F4\uFB1D-\uFB4F\u0620-\u064A\u066E-\u06D5\u06E5-\u06FF\u0750-\u077F\u08A0-\u08BD\uFB50-\uFBB1\uFBD3-\uFD3D\uFD50-\uFDC7\uFDF0-\uFDFB\uFE70-\uFEFC\U0001EE00-\U0001EEBB\u0D80-\u0DFF\u0900-\u097F\u0C80-\u0CFF\u0B80-\u0BFF\u0C00-\u0C7F\uAC00-\uD7AF\u1100-\u11FF\u3040-\u309F\u30A0-\u30FFー\u4E00-\u62FF\u6300-\u77FF\u7800-\u8CFF\u8D00-\u9FFF\u3400-\u4DBF\U00020000-\U000215FF\U00021600-\U000230FF\U00023100-\U000245FF\U00024600-\U000260FF\U00026100-\U000275FF\U00027600-\U000290FF\U00029100-\U0002A6DF\U0002A700-\U0002B73F\U0002B740-\U0002B81F\U0002B820-\U0002CEAF\U0002CEB0-\U0002EBEF\u2E80-\u2EFF\u2F00-\u2FDF\u2FF0-\u2FFF\u3000-\u303F\u31C0-\u31EF\u3200-\u32FF\u3300-\u33FF\uF900-\uFAFF\uFE30-\uFE4F\U0001F200-\U0001F2FF\U0002F800-\U0002FA1F])[:<>=/](?=[A-Za-z\uFF21-\uFF3A\uFF41-\uFF5A\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u00FF\u0100-\u017F\u0180-\u01BF\u01C4-\u024F\u2C60-\u2C7B\u2C7E\u2C7F\uA722-\uA76F\uA771-\uA787\uA78B-\uA78E\uA790-\uA7B9\uA7FA\uAB30-\uAB5A\uAB60-\uAB64\u0250-\u02AF\u1D00-\u1D25\u1D6B-\u1D77\u1D79-\u1D9A\u1E00-\u1EFFёа-яЁА-ЯәөүҗңһӘӨҮҖҢҺα-ωάέίόώήύΑ-ΩΆΈΊΌΏΉΎа-щюяіїєґА-ЩЮЯІЇЄҐѓѕјљњќѐѝЃЅЈЉЊЌЀЍ\u1200-\u137F\u0980-\u09FF\u0591-\u05F4\uFB1D-\uFB4F\u0620-\u064A\u066E-\u06D5\u06E5-\u06FF\u0750-\u077F\u08A0-\u08BD\uFB50-\uFBB1\uFBD3-\uFD3D\uFD50-\uFDC7\uFDF0-\uFDFB\uFE70-\uFEFC\U0001EE00-\U0001EEBB\u0D80-\u0DFF\u0900-\u097F\u0C80-\u0CFF\u0B80-\u0BFF\u0C00-\u0C7F\uAC00-\uD7AF\u1100-\u11FF\u3040-\u309F\u30A0-\u30FFー\u4E00-\u62FF\u6300-\u77FF\u7800-\u8CFF\u8D00-\u9FFF\u3400-\u4DBF\U00020000-\U000215FF\U00021600-\U000230FF\U00023100-\U000245FF\U00024600-\U000260FF\U00026100-\U000275FF\U00027600-\U000290FF\U00029100-\U0002A6DF\U0002A700-\U0002B73F\U0002B740-\U0002B81F\U0002B820-\U0002CEAF\U0002CEB0-\U0002EBEF\u2E80-\u2EFF\u2F00-\u2FDF\u2FF0-\u2FFF\u3000-\u303F\u31C0-\u31EF\u3200-\u32FF\u3300-\u33FF\uF900-\uFAFF\uFE30-\uFE4F\U0001F200-\U0001F2FF\U0002F800-\U0002FA1F])token_matchurl_match (?u)^(?:(?:[\w\+\-\.]{2,})://)?(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[A-Za-z0-9\u00a1-\uffff][A-Za-z0-9\u00a1-\uffff_-]{0,62})?[A-Za-z0-9\u00a1-\uffff]\.)+(?:[a-z\uFF41-\uFF5A\u00DF-\u00F6\u00F8-\u00FF\u0101\u0103\u0105\u0107\u0109\u010B\u010D\u010F\u0111\u0113\u0115\u0117\u0119\u011B\u011D\u011F\u0121\u0123\u0125\u0127\u0129\u012B\u012D\u012F\u0131\u0133\u0135\u0137\u0138\u013A\u013C\u013E\u0140\u0142\u0144\u0146\u0148\u0149\u014B\u014D\u014F\u0151\u0153\u0155\u0157\u0159\u015B\u015D\u015F\u0161\u0163\u0165\u0167\u0169\u016B\u016D\u016F\u0171\u0173\u0175\u0177\u017A\u017C\u017E\u017F\u0180\u0183\u0185\u0188\u018C\u018D\u0192\u0195\u0199-\u019B\u019E\u01A1\u01A3\u01A5\u01A8\u01AA\u01AB\u01AD\u01B0\u01B4\u01B6\u01B9\u01BA\u01BD-\u01BF\u01C6\u01C9\u01CC\u01CE\u01D0\u01D2\u01D4\u01D6\u01D8\u01DA\u01DC\u01DD\u01DF\u01E1\u01E3\u01E5\u01E7\u01E9\u01EB\u01ED\u01EF\u01F0\u01F3\u01F5\u01F9\u01FB\u01FD\u01FF\u0201\u0203\u0205\u0207\u0209\u020B\u020D\u020F\u0211\u0213\u0215\u0217\u0219\u021B\u021D\u021F\u0221\u0223\u0225\u0227\u0229\u022B\u022D\u022F\u0231\u0233-\u0239\u023C\u023F\u0240\u0242\u0247\u0249\u024B\u024D\u024F\u2C61\u2C65\u2C66\u2C68\u2C6A\u2C6C\u2C71\u2C73\u2C74\u2C76-\u2C7B\uA723\uA725\uA727\uA729\uA72B\uA72D\uA72F-\uA731\uA733\uA735\uA737\uA739\uA73B\uA73D\uA73F\uA741\uA743\uA745\uA747\uA749\uA74B\uA74D\uA74F\uA751\uA753\uA755\uA757\uA759\uA75B\uA75D\uA75F\uA761\uA763\uA765\uA767\uA769\uA76B\uA76D\uA76F\uA771-\uA778\uA77A\uA77C\uA77F\uA781\uA783\uA785\uA787\uA78C\uA78E\uA791\uA793-\uA795\uA797\uA799\uA79B\uA79D\uA79F\uA7A1\uA7A3\uA7A5\uA7A7\uA7A9\uA7AF\uA7B5\uA7B7\uA7B9\uA7FA\uAB30-\uAB5A\uAB60-\uAB64\u0250-\u02AF\u1D00-\u1D25\u1D6B-\u1D77\u1D79-\u1D9A\u1E01\u1E03\u1E05\u1E07\u1E09\u1E0B\u1E0D\u1E0F\u1E11\u1E13\u1E15\u1E17\u1E19\u1E1B\u1E1D\u1E1F\u1E21\u1E23\u1E25\u1E27\u1E29\u1E2B\u1E2D\u1E2F\u1E31\u1E33\u1E35\u1E37\u1E39\u1E3B\u1E3D\u1E3F\u1E41\u1E43\u1E45\u1E47\u1E49\u1E4B\u1E4D\u1E4F\u1E51\u1E53\u1E55\u1E57\u1E59\u1E5B\u1E5D\u1E5F\u1E61\u1E63\u1E65\u1E67\u1E69\u1E6B\u1E6D\u1E6F\u1E71\u1E73\u1E75\u1E77\u1E79\u1E7B\u1E7D\u1E7F\u1E81\u1E83\u1E85\u1E87\u1E89\u1E8B\u1E8D\u1E8F\u1E91\u1E93\u1E95-\u1E9D\u1E9F\u1EA1\u1EA3\u1EA5\u1EA7\u1EA9\u1EAB\u1EAD\u1EAF\u1EB1\u1EB3\u1EB5\u1EB7\u1EB9\u1EBB\u1EBD\u1EBF\u1EC1\u1EC3\u1EC5\u1EC7\u1EC9\u1ECB\u1ECD\u1ECF\u1ED1\u1ED3\u1ED5\u1ED7\u1ED9\u1EDB\u1EDD\u1EDF\u1EE1\u1EE3\u1EE5\u1EE7\u1EE9\u1EEB\u1EED\u1EEF\u1EF1\u1EF3\u1EF5\u1EF7\u1EF9\u1EFB\u1EFD\u1EFFёа-яәөүҗңһα-ωάέίόώήύа-щюяіїєґѓѕјљњќѐѝ\u1200-\u137F\u0980-\u09FF\u0591-\u05F4\uFB1D-\uFB4F\u0620-\u064A\u066E-\u06D5\u06E5-\u06FF\u0750-\u077F\u08A0-\u08BD\uFB50-\uFBB1\uFBD3-\uFD3D\uFD50-\uFDC7\uFDF0-\uFDFB\uFE70-\uFEFC\U0001EE00-\U0001EEBB\u0D80-\u0DFF\u0900-\u097F\u0C80-\u0CFF\u0B80-\u0BFF\u0C00-\u0C7F\uAC00-\uD7AF\u1100-\u11FF\u3040-\u309F\u30A0-\u30FFー\u4E00-\u62FF\u6300-\u77FF\u7800-\u8CFF\u8D00-\u9FFF\u3400-\u4DBF\U00020000-\U000215FF\U00021600-\U000230FF\U00023100-\U000245FF\U00024600-\U000260FF\U00026100-\U000275FF\U00027600-\U000290FF\U00029100-\U0002A6DF\U0002A700-\U0002B73F\U0002B740-\U0002B81F\U0002B820-\U0002CEAF\U0002CEB0-\U0002EBEF\u2E80-\u2EFF\u2F00-\u2FDF\u2FF0-\u2FFF\u3000-\u303F\u31C0-\u31EF\u3200-\u32FF\u3300-\u33FF\uF900-\uFAFF\uFE30-\uFE4F\U0001F200-\U0001F2FF\U0002F800-\U0002FA1F]{2,63}))(?::\d{2,5})?(?:[/?#]\S*)?$exceptions¡ A +A + A 'A'''A''(*_*)A(*_*)(-8A(-8(-:A(-:(-;A(-;(-_-)A(-_-)(._.)A(._.)(:A(:(;A(;(=A(=(>_<)A(>_<)(^_^)A(^_^)(o:A(o:(¬_¬)A(¬_¬)(ಠ_ಠ)A(ಠ_ಠ)(╯°□°）╯︵┻━┻A(╯°□°）╯︵┻━┻)-:A)-:):A):-_-A-_--__-A-__-._.A._.0.0A0.00.oA0.o0_0A0_00_oA0_o8)A8)8-)A8-)8-DA8-D8DA8D:'(A:'(:')A:'):'-(A:'-(:'-)A:'-):(A:(:((A:((:(((A:(((:()A:():)A:):))A:)):)))A:))):*A:*:-(A:-(:-((A:-((:-(((A:-(((:-)A:-):-))A:-)):-)))A:-))):-*A:-*:-/A:-/:-0A:-0:-3A:-3:->A:->:-DA:-D:-OA:-O:-PA:-P:-XA:-X:-]A:-]:-oA:-o:-pA:-p:-xA:-x:-|A:-|:-}A:-}:/A:/:0A:0:1A:1:3A:3:>A:>:DA:D:OA:O:PA:P:XA:X:]A:]:oA:o:o)A:o):pA:p:xA:x:|A:|:}A:}:’(A:’(:’)A:’):’-(A:’-(:’-)A:’-);)A;);-)A;-);-DA;-D;DA;D;_;A;_;<.A=(A=(=)A=)=/A=/=3A=3=DA=D=[A=[=]A=]=|A=|>..<>.>A>.>>:(A>:(>:oA>:o><(((*>A><(((*>@_@A@_@C++AC++O.OAO.OO.oAO.oO_OAO_OO_oAO_oV.VAV.VV_VAV_VXDAXDXDDAXDD[-:A[-:[:A[:[=A[=\")A\")\nA\n\tA\t]=A]=^_^A^_^^__^A^__^^___^A^___^a.Aa.b.Ab.c.Ac.d.Ad.e.Ae.f.Af.g.Ag.h.Ah.i.Ai.j.Aj.k.Ak.l.Al.m.Am.n.An.o.Ao.o.0Ao.0o.OAo.Oo.oAo.oo_0Ao_0o_OAo_Oo_oAo_op.Ap.q.Aq.r.Ar.s.As.t.At.u.Au.v.Av.v.vAv.vv_vAv_vw.Aw.x.Ax.xDAxDxDDAxDDy.Ay.z.Az. A C ¯\(ツ)/¯A¯\(ツ)/¯°C.A°ACA.°F.A°AFA.°K.A°AKA.°c.A°AcA.°f.A°AfA.°k.A°AkA.ä.Aä.ö.Aö.ü.Aü.কি.মিAকি.মিCকিলোমিটারকি.মি.Aকি.মি.Cকিলোমিটারড.Aড.Cডক্টরডঃAডঃCডক্টরডা.Aডা.Cডাক্তারডাঃAডাঃCডাক্তারমি.লি.Aমি.লি.Cমিলিলিটারমো.Aমো.Cমোহাম্মদমোঃAমোঃCমোহাম্মদসে.Aসে.Cসেলসিয়াসসে.মিAসে.মিC!সেন্টিমিটারসে.মি.Aসে.মি.C!সেন্টিমিটারಠ_ಠAಠ_ಠಠ︵ಠAಠ︵ಠ—A—’A’’’A’’faster_heuristics \ No newline at end of file diff --git a/bn_test/vocab/key2row b/bn_test/vocab/key2row new file mode 100644 index 0000000000..5416677bc7 --- /dev/null +++ b/bn_test/vocab/key2row @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/bn_test/vocab/lookups.bin b/bn_test/vocab/lookups.bin new file mode 100644 index 0000000000..5416677bc7 --- /dev/null +++ b/bn_test/vocab/lookups.bin @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/bn_test/vocab/strings.json b/bn_test/vocab/strings.json new file mode 100644 index 0000000000..f4b951a9b8 --- /dev/null +++ b/bn_test/vocab/strings.json @@ -0,0 +1,400 @@ +[ + "\t", + "\n", + " ", + " ", + "\"", + "'", + "''", + "'-(", + "'-)", + "(", + "(((", + "(*>", + "(*_*)", + "(-8", + "(-:", + "(-;", + "(-_-)", + "(-d", + "(._.)", + "(:", + "(;", + "(=", + "(>_<)", + "(^_^)", + "(o:", + "(x:", + "(x_x)", + "(\u00ac_\u00ac)", + "(\u0ca0_\u0ca0)", + "(\u256f\u00b0\u25a1\u00b0\uff09\u256f\ufe35\u253b\u2501\u253b", + ")", + ")))", + ")-:", + ")/\u00af", + "):", + "*", + "-", + "-((", + "-))", + "-/", + "-0", + "-3", + "-8", + "-D", + "-O", + "-P", + "-X", + "-_-", + "-__-", + "-d", + "-o", + "-p", + "-x", + "-|", + ".", + "._.", + ".\u09ae\u09bf", + ".\u09b2\u09bf", + "/", + "/3", + "/d", + "0", + "0.0", + "0.o", + "0_0", + "0_o", + "1", + "3", + "33", + "333", + "8", + "8)", + "8-", + "8-)", + "8-D", + "8-d", + "8D", + "8d", + ":", + ":'(", + ":')", + ":'-(", + ":'-)", + ":(", + ":((", + ":(((", + ":()", + ":)", + ":))", + ":)))", + ":*", + ":-(", + ":-((", + ":-(((", + ":-)", + ":-))", + ":-)))", + ":-*", + ":-/", + ":-0", + ":-3", + ":->", + ":-D", + ":-O", + ":-P", + ":-X", + ":-]", + ":-d", + ":-o", + ":-p", + ":-x", + ":-|", + ":-}", + ":/", + ":0", + ":1", + ":3", + ":>", + ":D", + ":O", + ":P", + ":X", + ":]", + ":d", + ":o", + ":o)", + ":p", + ":x", + ":x)", + ":|", + ":}", + ":\u2019(", + ":\u2019)", + ":\u2019-(", + ":\u2019-)", + ";", + ";)", + ";-)", + ";-D", + ";-X", + ";-d", + ";D", + ";X", + ";_;", + ";d", + "<", + "<.<", + "", + "", + "=", + "=(", + "=)", + "=/", + "=3", + "=D", + "=X", + "=[", + "=]", + "=d", + "=|", + ">", + ">.<", + ">.>", + ">:(", + ">:o", + ">:x", + "><(((*>", + "@", + "@_@", + "C", + "C++", + "C.", + "D", + "F", + "F.", + "K", + "K.", + "O", + "O.O", + "O.o", + "O_O", + "O_o", + "P", + "V", + "V.V", + "V_V", + "X++", + "X.", + "X.X", + "X.x", + "XD", + "XDD", + "XX", + "XXX", + "X_X", + "X_x", + "[", + "[-:", + "[:", + "[=", + "\\", + "\\\")", + "\\n", + "\\t", + "\\x", + "]", + "]=", + "^", + "^_^", + "^__^", + "^___^", + "_*)", + "_-)", + "_.)", + "_<)", + "_^)", + "__-", + "__^", + "_\u00ac)", + "_\u0ca0)", + "a", + "a.", + "ace", + "b", + "b.", + "bn", + "c", + "c++", + "c.", + "ce>", + "d", + "d)", + "d-", + "d-)", + "d-X", + "d.", + "d.d", + "d.x", + "dX", + "d_d", + "d_x", + "dd", + "ddd", + "e", + "e.", + "f", + "f.", + "g", + "g.", + "h", + "h.", + "i", + "i.", + "j", + "j.", + "k", + "k.", + "l", + "l.", + "m", + "m.", + "n", + "n.", + "o", + "o.", + "o.0", + "o.O", + "o.o", + "o_0", + "o_O", + "o_o", + "p", + "p.", + "q", + "q.", + "r", + "r.", + "s", + "s.", + "space", + "t", + "t.", + "u", + "u.", + "v", + "v.", + "v.v", + "v_v", + "w", + "w.", + "x", + "x.", + "x.X", + "x.d", + "x.x", + "xD", + "xDD", + "xX", + "xXX", + "x_X", + "x_d", + "x_x", + "xd", + "xdd", + "xxxx", + "x\u0983", + "x\u09be", + "x\u09be.", + "x\u09be\u0983", + "x\u09bf.x\u09bf", + "x\u09bf.x\u09bf.", + "x\u09c7", + "x\u09c7.", + "x\u09c7.x\u09bf", + "x\u09c7.x\u09bf.", + "x\u09cb", + "x\u09cb.", + "x\u09cb\u0983", + "x\ufe35x", + "y", + "y.", + "z", + "z.", + "|", + "}", + "\u00a0", + "\u00ac", + "\u00ac_\u00ac", + "\u00af", + "\u00af\\(x)/\u00af", + "\u00af\\(\u30c4)/\u00af", + "\u00b0", + "\u00b0C.", + "\u00b0F.", + "\u00b0K.", + "\u00b0X.", + "\u00b0c.", + "\u00b0f.", + "\u00b0k.", + "\u00b0x.", + "\u00e4", + "\u00e4.", + "\u00f6", + "\u00f6.", + "\u00fc", + "\u00fc.", + "\u0995", + "\u0995\u09bf.\u09ae\u09bf", + "\u0995\u09bf.\u09ae\u09bf.", + "\u0995\u09bf\u09b2\u09cb\u09ae\u09bf\u099f\u09be\u09b0", + "\u09a1", + "\u09a1.", + "\u09a1\u0983", + "\u09a1\u0995\u09cd\u099f\u09b0", + "\u09a1\u09be", + "\u09a1\u09be.", + "\u09a1\u09be\u0983", + "\u09a1\u09be\u0995\u09cd\u09a4\u09be\u09b0", + "\u09ae", + "\u09ae\u09bf.", + "\u09ae\u09bf.\u09b2\u09bf", + "\u09ae\u09bf.\u09b2\u09bf.", + "\u09ae\u09bf\u09b2\u09bf\u09b2\u09bf\u099f\u09be\u09b0", + "\u09ae\u09cb", + "\u09ae\u09cb.", + "\u09ae\u09cb\u0983", + "\u09ae\u09cb\u09b9\u09be\u09ae\u09cd\u09ae\u09a6", + "\u09b2\u09bf.", + "\u09b8", + "\u09b8\u09c7", + "\u09b8\u09c7.", + "\u09b8\u09c7.\u09ae\u09bf", + "\u09b8\u09c7.\u09ae\u09bf.", + "\u09b8\u09c7\u09a8\u09cd\u099f\u09bf\u09ae\u09bf\u099f\u09be\u09b0", + "\u09b8\u09c7\u09b2\u09b8\u09bf\u09af\u09bc\u09be\u09b8", + "\u0ca0", + "\u0ca0_\u0ca0", + "\u0ca0\ufe35\u0ca0", + "\u2014", + "\u2019", + "\u2019-(", + "\u2019-)", + "\u2019\u2019", + "\u2501", + "\u253b", + "\u253b\u2501\u253b", + "\u256f", + "\u25a1", + "\ufe35", + "\uff09" +] \ No newline at end of file diff --git a/bn_test/vocab/vectors b/bn_test/vocab/vectors new file mode 100644 index 0000000000000000000000000000000000000000..ebadaa58b83a76a8ce0dbf1b5a75d8fb4aa559ad GIT binary patch literal 128 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1ZlV+l>qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= XXCxM+0{I#SItm7wItsN4WCJb$RVy23 literal 0 HcmV?d00001 diff --git a/bn_test/vocab/vectors.cfg b/bn_test/vocab/vectors.cfg new file mode 100644 index 0000000000..32c800af49 --- /dev/null +++ b/bn_test/vocab/vectors.cfg @@ -0,0 +1,3 @@ +{ + "mode":"default" +} \ No newline at end of file diff --git a/he_test/config.cfg b/he_test/config.cfg new file mode 100644 index 0000000000..42f94e1914 --- /dev/null +++ b/he_test/config.cfg @@ -0,0 +1,104 @@ +[paths] +train = null +dev = null +vectors = null +init_tok2vec = null + +[system] +seed = 0 +gpu_allocator = null + +[nlp] +lang = "he" +pipeline = [] +disabled = [] +before_creation = null +after_creation = null +after_pipeline_creation = null +batch_size = 1000 + +[corpora] + +[training] +seed = ${system.seed} +gpu_allocator = ${system.gpu_allocator} +dropout = 0.1 +accumulate_gradient = 1 +patience = 1600 +max_epochs = 0 +max_steps = 20000 +eval_frequency = 200 +frozen_components = [] +annotating_components = [] +dev_corpus = "corpora.dev" +train_corpus = "corpora.train" +before_to_disk = null +before_update = null + +[initialize] +vectors = ${paths.vectors} +init_tok2vec = ${paths.init_tok2vec} +vocab_data = null +lookups = null +before_init = null +after_init = null + +[components] + +[pretraining] + +[nlp.tokenizer] +@tokenizers = "spacy.Tokenizer.v1" + +[nlp.vectors] +@vectors = "spacy.Vectors.v1" + +[corpora.train] +@readers = "spacy.Corpus.v1" +path = ${paths.train} +gold_preproc = false +max_length = 0 +limit = 0 +augmenter = null + +[corpora.dev] +@readers = "spacy.Corpus.v1" +path = ${paths.dev} +gold_preproc = false +max_length = 0 +limit = 0 +augmenter = null + +[training.logger] +@loggers = "spacy.ConsoleLogger.v1" +progress_bar = false + +[training.batcher] +@batchers = "spacy.batch_by_words.v1" +discard_oversize = false +tolerance = 0.2 +get_length = null + +[training.optimizer] +@optimizers = "Adam.v1" +beta1 = 0.9 +beta2 = 0.999 +L2_is_weight_decay = true +L2 = 0.01 +grad_clip = 1.0 +use_averages = false +eps = 1e-08 +learn_rate = 0.001 + +[training.score_weights] + +[initialize.tokenizer] + +[initialize.components] + +[training.batcher.size] +@schedules = "compounding.v1" +start = 100 +stop = 1000 +compound = 1.001 +t = 0.0 \ No newline at end of file diff --git a/he_test/meta.json b/he_test/meta.json new file mode 100644 index 0000000000..e39defd815 --- /dev/null +++ b/he_test/meta.json @@ -0,0 +1,31 @@ +{ + "lang":"he", + "name":"pipeline", + "version":"0.0.0", + "spacy_version":">=3.8.14,<3.9.0", + "description":"", + "author":"", + "email":"", + "url":"", + "license":"", + "spacy_git_version":"0069cf9", + "vectors":{ + "width":0, + "vectors":0, + "keys":0, + "name":null, + "mode":"default" + }, + "labels":{ + + }, + "pipeline":[ + + ], + "components":[ + + ], + "disabled":[ + + ] +} \ No newline at end of file diff --git a/he_test/tokenizer b/he_test/tokenizer new file mode 100644 index 0000000000..cf1ad87920 --- /dev/null +++ b/he_test/tokenizer @@ -0,0 +1,3 @@ +prefix_search ^§|^%|^=|^—|^–|^\+(?![0-9])|^…|^……|^,|^:|^;|^\!|^\?|^¿|^؟|^¡|^\(|^\)|^\[|^\]|^\{|^\}|^<|^>|^_|^#|^\*|^&|^。|^？|^！|^，|^、|^；|^：|^～|^·|^।|^،|^۔|^؛|^٪|^\.\.+|^…|^\'|^"|^”|^“|^`|^‘|^´|^’|^‚|^,|^„|^»|^«|^「|^」|^『|^』|^（|^）|^〔|^〕|^【|^】|^《|^》|^〈|^〉|^〈|^〉|^⟦|^⟧|^\$|^£|^€|^¥|^฿|^US\$|^C\$|^A\$|^₽|^﷼|^₴|^₠|^₡|^₢|^₣|^₤|^₥|^₦|^₧|^₨|^₩|^₪|^₫|^€|^₭|^₮|^₯|^₰|^₱|^₲|^₳|^₴|^₵|^₶|^₷|^₸|^₹|^₺|^₻|^₼|^₽|^₾|^₿|^[\u00A6\u00A9\u00AE\u00B0\u0482\u058D\u058E\u060E\u060F\u06DE\u06E9\u06FD\u06FE\u07F6\u09FA\u0B70\u0BF3-\u0BF8\u0BFA\u0C7F\u0D4F\u0D79\u0F01-\u0F03\u0F13\u0F15-\u0F17\u0F1A-\u0F1F\u0F34\u0F36\u0F38\u0FBE-\u0FC5\u0FC7-\u0FCC\u0FCE\u0FCF\u0FD5-\u0FD8\u109E\u109F\u1390-\u1399\u1940\u19DE-\u19FF\u1B61-\u1B6A\u1B74-\u1B7C\u2100\u2101\u2103-\u2106\u2108\u2109\u2114\u2116\u2117\u211E-\u2123\u2125\u2127\u2129\u212E\u213A\u213B\u214A\u214C\u214D\u214F\u218A\u218B\u2195-\u2199\u219C-\u219F\u21A1\u21A2\u21A4\u21A5\u21A7-\u21AD\u21AF-\u21CD\u21D0\u21D1\u21D3\u21D5-\u21F3\u2300-\u2307\u230C-\u231F\u2322-\u2328\u232B-\u237B\u237D-\u239A\u23B4-\u23DB\u23E2-\u2426\u2440-\u244A\u249C-\u24E9\u2500-\u25B6\u25B8-\u25C0\u25C2-\u25F7\u2600-\u266E\u2670-\u2767\u2794-\u27BF\u2800-\u28FF\u2B00-\u2B2F\u2B45\u2B46\u2B4D-\u2B73\u2B76-\u2B95\u2B98-\u2BC8\u2BCA-\u2BFE\u2CE5-\u2CEA\u2E80-\u2E99\u2E9B-\u2EF3\u2F00-\u2FD5\u2FF0-\u2FFB\u3004\u3012\u3013\u3020\u3036\u3037\u303E\u303F\u3190\u3191\u3196-\u319F\u31C0-\u31E3\u3200-\u321E\u322A-\u3247\u3250\u3260-\u327F\u328A-\u32B0\u32C0-\u32FE\u3300-\u33FF\u4DC0-\u4DFF\uA490-\uA4C6\uA828-\uA82B\uA836\uA837\uA839\uAA77-\uAA79\uFDFD\uFFE4\uFFE8\uFFED\uFFEE\uFFFC\uFFFD\U00010137-\U0001013F\U00010179-\U00010189\U0001018C-\U0001018E\U00010190-\U0001019B\U000101A0\U000101D0-\U000101FC\U00010877\U00010878\U00010AC8\U0001173F\U00016B3C-\U00016B3F\U00016B45\U0001BC9C\U0001D000-\U0001D0F5\U0001D100-\U0001D126\U0001D129-\U0001D164\U0001D16A-\U0001D16C\U0001D183\U0001D184\U0001D18C-\U0001D1A9\U0001D1AE-\U0001D1E8\U0001D200-\U0001D241\U0001D245\U0001D300-\U0001D356\U0001D800-\U0001D9FF\U0001DA37-\U0001DA3A\U0001DA6D-\U0001DA74\U0001DA76-\U0001DA83\U0001DA85\U0001DA86\U0001ECAC\U0001F000-\U0001F02B\U0001F030-\U0001F093\U0001F0A0-\U0001F0AE\U0001F0B1-\U0001F0BF\U0001F0C1-\U0001F0CF\U0001F0D1-\U0001F0F5\U0001F110-\U0001F16B\U0001F170-\U0001F1AC\U0001F1E6-\U0001F202\U0001F210-\U0001F23B\U0001F240-\U0001F248\U0001F250\U0001F251\U0001F260-\U0001F265\U0001F300-\U0001F3FA\U0001F400-\U0001F6D4\U0001F6E0-\U0001F6EC\U0001F6F0-\U0001F6F9\U0001F700-\U0001F773\U0001F780-\U0001F7D8\U0001F800-\U0001F80B\U0001F810-\U0001F847\U0001F850-\U0001F859\U0001F860-\U0001F887\U0001F890-\U0001F8AD\U0001F900-\U0001F90B\U0001F910-\U0001F93E\U0001F940-\U0001F970\U0001F973-\U0001F976\U0001F97A\U0001F97C-\U0001F9A2\U0001F9B0-\U0001F9B9\U0001F9C0-\U0001F9C2\U0001F9D0-\U0001F9FF\U0001FA60-\U0001FA6D]suffix_search2…$|……$|,$|:$|;$|\!$|\?$|¿$|؟$|¡$|\($|\)$|\[$|\]$|\{$|\}$|<$|>$|_$|#$|\*$|&$|。$|？$|！$|，$|、$|；$|：$|～$|·$|।$|،$|۔$|؛$|٪$|\.\.+$|…$|\'$|"$|”$|“$|`$|‘$|´$|’$|‚$|,$|„$|»$|«$|「$|」$|『$|』$|（$|）$|〔$|〕$|【$|】$|《$|》$|〈$|〉$|〈$|〉$|⟦$|⟧$|[\u00A6\u00A9\u00AE\u00B0\u0482\u058D\u058E\u060E\u060F\u06DE\u06E9\u06FD\u06FE\u07F6\u09FA\u0B70\u0BF3-\u0BF8\u0BFA\u0C7F\u0D4F\u0D79\u0F01-\u0F03\u0F13\u0F15-\u0F17\u0F1A-\u0F1F\u0F34\u0F36\u0F38\u0FBE-\u0FC5\u0FC7-\u0FCC\u0FCE\u0FCF\u0FD5-\u0FD8\u109E\u109F\u1390-\u1399\u1940\u19DE-\u19FF\u1B61-\u1B6A\u1B74-\u1B7C\u2100\u2101\u2103-\u2106\u2108\u2109\u2114\u2116\u2117\u211E-\u2123\u2125\u2127\u2129\u212E\u213A\u213B\u214A\u214C\u214D\u214F\u218A\u218B\u2195-\u2199\u219C-\u219F\u21A1\u21A2\u21A4\u21A5\u21A7-\u21AD\u21AF-\u21CD\u21D0\u21D1\u21D3\u21D5-\u21F3\u2300-\u2307\u230C-\u231F\u2322-\u2328\u232B-\u237B\u237D-\u239A\u23B4-\u23DB\u23E2-\u2426\u2440-\u244A\u249C-\u24E9\u2500-\u25B6\u25B8-\u25C0\u25C2-\u25F7\u2600-\u266E\u2670-\u2767\u2794-\u27BF\u2800-\u28FF\u2B00-\u2B2F\u2B45\u2B46\u2B4D-\u2B73\u2B76-\u2B95\u2B98-\u2BC8\u2BCA-\u2BFE\u2CE5-\u2CEA\u2E80-\u2E99\u2E9B-\u2EF3\u2F00-\u2FD5\u2FF0-\u2FFB\u3004\u3012\u3013\u3020\u3036\u3037\u303E\u303F\u3190\u3191\u3196-\u319F\u31C0-\u31E3\u3200-\u321E\u322A-\u3247\u3250\u3260-\u327F\u328A-\u32B0\u32C0-\u32FE\u3300-\u33FF\u4DC0-\u4DFF\uA490-\uA4C6\uA828-\uA82B\uA836\uA837\uA839\uAA77-\uAA79\uFDFD\uFFE4\uFFE8\uFFED\uFFEE\uFFFC\uFFFD\U00010137-\U0001013F\U00010179-\U00010189\U0001018C-\U0001018E\U00010190-\U0001019B\U000101A0\U000101D0-\U000101FC\U00010877\U00010878\U00010AC8\U0001173F\U00016B3C-\U00016B3F\U00016B45\U0001BC9C\U0001D000-\U0001D0F5\U0001D100-\U0001D126\U0001D129-\U0001D164\U0001D16A-\U0001D16C\U0001D183\U0001D184\U0001D18C-\U0001D1A9\U0001D1AE-\U0001D1E8\U0001D200-\U0001D241\U0001D245\U0001D300-\U0001D356\U0001D800-\U0001D9FF\U0001DA37-\U0001DA3A\U0001DA6D-\U0001DA74\U0001DA76-\U0001DA83\U0001DA85\U0001DA86\U0001ECAC\U0001F000-\U0001F02B\U0001F030-\U0001F093\U0001F0A0-\U0001F0AE\U0001F0B1-\U0001F0BF\U0001F0C1-\U0001F0CF\U0001F0D1-\U0001F0F5\U0001F110-\U0001F16B\U0001F170-\U0001F1AC\U0001F1E6-\U0001F202\U0001F210-\U0001F23B\U0001F240-\U0001F248\U0001F250\U0001F251\U0001F260-\U0001F265\U0001F300-\U0001F3FA\U0001F400-\U0001F6D4\U0001F6E0-\U0001F6EC\U0001F6F0-\U0001F6F9\U0001F700-\U0001F773\U0001F780-\U0001F7D8\U0001F800-\U0001F80B\U0001F810-\U0001F847\U0001F850-\U0001F859\U0001F860-\U0001F887\U0001F890-\U0001F8AD\U0001F900-\U0001F90B\U0001F910-\U0001F93E\U0001F940-\U0001F970\U0001F973-\U0001F976\U0001F97A\U0001F97C-\U0001F9A2\U0001F9B0-\U0001F9B9\U0001F9C0-\U0001F9C2\U0001F9D0-\U0001F9FF\U0001FA60-\U0001FA6D]$|'s$|'S$|’s$|’S$|—$|–$|(?<=[0-9])\+$|(?<=°[FfCcKk])\.$|(?<=[0-9])(?:\$|£|€|¥|฿|US\$|C\$|A\$|₽|﷼|₴|₠|₡|₢|₣|₤|₥|₦|₧|₨|₩|₪|₫|€|₭|₮|₯|₰|₱|₲|₳|₴|₵|₶|₷|₸|₹|₺|₻|₼|₽|₾|₿)$|(?<=[0-9])(?:km|km²|km³|m|m²|m³|dm|dm²|dm³|cm|cm²|cm³|mm|mm²|mm³|ha|µm|nm|yd|in|ft|kg|g|mg|µg|t|lb|oz|m/s|km/h|kmh|mph|hPa|Pa|mbar|mb|MB|kb|KB|gb|GB|tb|TB|T|G|M|K|%|км|км²|км³|м|м²|м³|дм|дм²|дм³|см|см²|см³|мм|мм²|мм³|нм|кг|г|мг|м/с|км/ч|кПа|Па|мбар|Кб|КБ|кб|Мб|МБ|мб|Гб|ГБ|гб|Тб|ТБ|тбكم|كم²|كم³|م|م²|م³|سم|سم²|سم³|مم|مم²|مم³|كم|غرام|جرام|جم|كغ|ملغ|كوب|اكواب)$|(?<=[0-9a-z\uFF41-\uFF5A\u00DF-\u00F6\u00F8-\u00FF\u0101\u0103\u0105\u0107\u0109\u010B\u010D\u010F\u0111\u0113\u0115\u0117\u0119\u011B\u011D\u011F\u0121\u0123\u0125\u0127\u0129\u012B\u012D\u012F\u0131\u0133\u0135\u0137\u0138\u013A\u013C\u013E\u0140\u0142\u0144\u0146\u0148\u0149\u014B\u014D\u014F\u0151\u0153\u0155\u0157\u0159\u015B\u015D\u015F\u0161\u0163\u0165\u0167\u0169\u016B\u016D\u016F\u0171\u0173\u0175\u0177\u017A\u017C\u017E\u017F\u0180\u0183\u0185\u0188\u018C\u018D\u0192\u0195\u0199-\u019B\u019E\u01A1\u01A3\u01A5\u01A8\u01AA\u01AB\u01AD\u01B0\u01B4\u01B6\u01B9\u01BA\u01BD-\u01BF\u01C6\u01C9\u01CC\u01CE\u01D0\u01D2\u01D4\u01D6\u01D8\u01DA\u01DC\u01DD\u01DF\u01E1\u01E3\u01E5\u01E7\u01E9\u01EB\u01ED\u01EF\u01F0\u01F3\u01F5\u01F9\u01FB\u01FD\u01FF\u0201\u0203\u0205\u0207\u0209\u020B\u020D\u020F\u0211\u0213\u0215\u0217\u0219\u021B\u021D\u021F\u0221\u0223\u0225\u0227\u0229\u022B\u022D\u022F\u0231\u0233-\u0239\u023C\u023F\u0240\u0242\u0247\u0249\u024B\u024D\u024F\u2C61\u2C65\u2C66\u2C68\u2C6A\u2C6C\u2C71\u2C73\u2C74\u2C76-\u2C7B\uA723\uA725\uA727\uA729\uA72B\uA72D\uA72F-\uA731\uA733\uA735\uA737\uA739\uA73B\uA73D\uA73F\uA741\uA743\uA745\uA747\uA749\uA74B\uA74D\uA74F\uA751\uA753\uA755\uA757\uA759\uA75B\uA75D\uA75F\uA761\uA763\uA765\uA767\uA769\uA76B\uA76D\uA76F\uA771-\uA778\uA77A\uA77C\uA77F\uA781\uA783\uA785\uA787\uA78C\uA78E\uA791\uA793-\uA795\uA797\uA799\uA79B\uA79D\uA79F\uA7A1\uA7A3\uA7A5\uA7A7\uA7A9\uA7AF\uA7B5\uA7B7\uA7B9\uA7FA\uAB30-\uAB5A\uAB60-\uAB64\u0250-\u02AF\u1D00-\u1D25\u1D6B-\u1D77\u1D79-\u1D9A\u1E01\u1E03\u1E05\u1E07\u1E09\u1E0B\u1E0D\u1E0F\u1E11\u1E13\u1E15\u1E17\u1E19\u1E1B\u1E1D\u1E1F\u1E21\u1E23\u1E25\u1E27\u1E29\u1E2B\u1E2D\u1E2F\u1E31\u1E33\u1E35\u1E37\u1E39\u1E3B\u1E3D\u1E3F\u1E41\u1E43\u1E45\u1E47\u1E49\u1E4B\u1E4D\u1E4F\u1E51\u1E53\u1E55\u1E57\u1E59\u1E5B\u1E5D\u1E5F\u1E61\u1E63\u1E65\u1E67\u1E69\u1E6B\u1E6D\u1E6F\u1E71\u1E73\u1E75\u1E77\u1E79\u1E7B\u1E7D\u1E7F\u1E81\u1E83\u1E85\u1E87\u1E89\u1E8B\u1E8D\u1E8F\u1E91\u1E93\u1E95-\u1E9D\u1E9F\u1EA1\u1EA3\u1EA5\u1EA7\u1EA9\u1EAB\u1EAD\u1EAF\u1EB1\u1EB3\u1EB5\u1EB7\u1EB9\u1EBB\u1EBD\u1EBF\u1EC1\u1EC3\u1EC5\u1EC7\u1EC9\u1ECB\u1ECD\u1ECF\u1ED1\u1ED3\u1ED5\u1ED7\u1ED9\u1EDB\u1EDD\u1EDF\u1EE1\u1EE3\u1EE5\u1EE7\u1EE9\u1EEB\u1EED\u1EEF\u1EF1\u1EF3\u1EF5\u1EF7\u1EF9\u1EFB\u1EFD\u1EFFёа-яәөүҗңһα-ωάέίόώήύа-щюяіїєґѓѕјљњќѐѝ\u1200-\u137F\u0980-\u09FF\u0591-\u05F4\uFB1D-\uFB4F\u0620-\u064A\u066E-\u06D5\u06E5-\u06FF\u0750-\u077F\u08A0-\u08BD\uFB50-\uFBB1\uFBD3-\uFD3D\uFD50-\uFDC7\uFDF0-\uFDFB\uFE70-\uFEFC\U0001EE00-\U0001EEBB\u0D80-\u0DFF\u0900-\u097F\u0C80-\u0CFF\u0B80-\u0BFF\u0C00-\u0C7F\uAC00-\uD7AF\u1100-\u11FF\u3040-\u309F\u30A0-\u30FFー\u4E00-\u62FF\u6300-\u77FF\u7800-\u8CFF\u8D00-\u9FFF\u3400-\u4DBF\U00020000-\U000215FF\U00021600-\U000230FF\U00023100-\U000245FF\U00024600-\U000260FF\U00026100-\U000275FF\U00027600-\U000290FF\U00029100-\U0002A6DF\U0002A700-\U0002B73F\U0002B740-\U0002B81F\U0002B820-\U0002CEAF\U0002CEB0-\U0002EBEF\u2E80-\u2EFF\u2F00-\u2FDF\u2FF0-\u2FFF\u3000-\u303F\u31C0-\u31EF\u3200-\u32FF\u3300-\u33FF\uF900-\uFAFF\uFE30-\uFE4F\U0001F200-\U0001F2FF\U0002F800-\U0002FA1F%²\-\+…|……|,|:|;|\!|\?|¿|؟|¡|\(|\)|\[|\]|\{|\}|<|>|_|#|\*|&|。|？|！|，|、|；|：|～|·|।|،|۔|؛|٪(?:\'"”“`‘´’‚,„»«「」『』（）〔〕【】《》〈〉〈〉⟦⟧)])\.$|(?<=[A-Z\uFF21-\uFF3A\u00C0-\u00D6\u00D8-\u00DE\u0100\u0102\u0104\u0106\u0108\u010A\u010C\u010E\u0110\u0112\u0114\u0116\u0118\u011A\u011C\u011E\u0120\u0122\u0124\u0126\u0128\u012A\u012C\u012E\u0130\u0132\u0134\u0136\u0139\u013B\u013D\u013F\u0141\u0143\u0145\u0147\u014A\u014C\u014E\u0150\u0152\u0154\u0156\u0158\u015A\u015C\u015E\u0160\u0162\u0164\u0166\u0168\u016A\u016C\u016E\u0170\u0172\u0174\u0176\u0178\u0179\u017B\u017D\u0181\u0182\u0184\u0186\u0187\u0189-\u018B\u018E-\u0191\u0193\u0194\u0196-\u0198\u019C\u019D\u019F\u01A0\u01A2\u01A4\u01A6\u01A7\u01A9\u01AC\u01AE\u01AF\u01B1-\u01B3\u01B5\u01B7\u01B8\u01BC\u01C4\u01C7\u01CA\u01CD\u01CF\u01D1\u01D3\u01D5\u01D7\u01D9\u01DB\u01DE\u01E0\u01E2\u01E4\u01E6\u01E8\u01EA\u01EC\u01EE\u01F1\u01F4\u01F6-\u01F8\u01FA\u01FC\u01FE\u0200\u0202\u0204\u0206\u0208\u020A\u020C\u020E\u0210\u0212\u0214\u0216\u0218\u021A\u021C\u021E\u0220\u0222\u0224\u0226\u0228\u022A\u022C\u022E\u0230\u0232\u023A\u023B\u023D\u023E\u0241\u0243-\u0246\u0248\u024A\u024C\u024E\u2C60\u2C62-\u2C64\u2C67\u2C69\u2C6B\u2C6D-\u2C70\u2C72\u2C75\u2C7E\u2C7F\uA722\uA724\uA726\uA728\uA72A\uA72C\uA72E\uA732\uA734\uA736\uA738\uA73A\uA73C\uA73E\uA740\uA742\uA744\uA746\uA748\uA74A\uA74C\uA74E\uA750\uA752\uA754\uA756\uA758\uA75A\uA75C\uA75E\uA760\uA762\uA764\uA766\uA768\uA76A\uA76C\uA76E\uA779\uA77B\uA77D\uA77E\uA780\uA782\uA784\uA786\uA78B\uA78D\uA790\uA792\uA796\uA798\uA79A\uA79C\uA79E\uA7A0\uA7A2\uA7A4\uA7A6\uA7A8\uA7AA-\uA7AE\uA7B0-\uA7B4\uA7B6\uA7B8\u1E00\u1E02\u1E04\u1E06\u1E08\u1E0A\u1E0C\u1E0E\u1E10\u1E12\u1E14\u1E16\u1E18\u1E1A\u1E1C\u1E1E\u1E20\u1E22\u1E24\u1E26\u1E28\u1E2A\u1E2C\u1E2E\u1E30\u1E32\u1E34\u1E36\u1E38\u1E3A\u1E3C\u1E3E\u1E40\u1E42\u1E44\u1E46\u1E48\u1E4A\u1E4C\u1E4E\u1E50\u1E52\u1E54\u1E56\u1E58\u1E5A\u1E5C\u1E5E\u1E60\u1E62\u1E64\u1E66\u1E68\u1E6A\u1E6C\u1E6E\u1E70\u1E72\u1E74\u1E76\u1E78\u1E7A\u1E7C\u1E7E\u1E80\u1E82\u1E84\u1E86\u1E88\u1E8A\u1E8C\u1E8E\u1E90\u1E92\u1E94\u1E9E\u1EA0\u1EA2\u1EA4\u1EA6\u1EA8\u1EAA\u1EAC\u1EAE\u1EB0\u1EB2\u1EB4\u1EB6\u1EB8\u1EBA\u1EBC\u1EBE\u1EC0\u1EC2\u1EC4\u1EC6\u1EC8\u1ECA\u1ECC\u1ECE\u1ED0\u1ED2\u1ED4\u1ED6\u1ED8\u1EDA\u1EDC\u1EDE\u1EE0\u1EE2\u1EE4\u1EE6\u1EE8\u1EEA\u1EEC\u1EEE\u1EF0\u1EF2\u1EF4\u1EF6\u1EF8\u1EFA\u1EFC\u1EFEЁА-ЯӘӨҮҖҢҺΑ-ΩΆΈΊΌΏΉΎА-ЩЮЯІЇЄҐЃЅЈЉЊЌЀЍ\u1200-\u137F\u0980-\u09FF\u0591-\u05F4\uFB1D-\uFB4F\u0620-\u064A\u066E-\u06D5\u06E5-\u06FF\u0750-\u077F\u08A0-\u08BD\uFB50-\uFBB1\uFBD3-\uFD3D\uFD50-\uFDC7\uFDF0-\uFDFB\uFE70-\uFEFC\U0001EE00-\U0001EEBB\u0D80-\u0DFF\u0900-\u097F\u0C80-\u0CFF\u0B80-\u0BFF\u0C00-\u0C7F\uAC00-\uD7AF\u1100-\u11FF\u3040-\u309F\u30A0-\u30FFー\u4E00-\u62FF\u6300-\u77FF\u7800-\u8CFF\u8D00-\u9FFF\u3400-\u4DBF\U00020000-\U000215FF\U00021600-\U000230FF\U00023100-\U000245FF\U00024600-\U000260FF\U00026100-\U000275FF\U00027600-\U000290FF\U00029100-\U0002A6DF\U0002A700-\U0002B73F\U0002B740-\U0002B81F\U0002B820-\U0002CEAF\U0002CEB0-\U0002EBEF\u2E80-\u2EFF\u2F00-\u2FDF\u2FF0-\u2FFF\u3000-\u303F\u31C0-\u31EF\u3200-\u32FF\u3300-\u33FF\uF900-\uFAFF\uFE30-\uFE4F\U0001F200-\U0001F2FF\U0002F800-\U0002FA1F][A-Z\uFF21-\uFF3A\u00C0-\u00D6\u00D8-\u00DE\u0100\u0102\u0104\u0106\u0108\u010A\u010C\u010E\u0110\u0112\u0114\u0116\u0118\u011A\u011C\u011E\u0120\u0122\u0124\u0126\u0128\u012A\u012C\u012E\u0130\u0132\u0134\u0136\u0139\u013B\u013D\u013F\u0141\u0143\u0145\u0147\u014A\u014C\u014E\u0150\u0152\u0154\u0156\u0158\u015A\u015C\u015E\u0160\u0162\u0164\u0166\u0168\u016A\u016C\u016E\u0170\u0172\u0174\u0176\u0178\u0179\u017B\u017D\u0181\u0182\u0184\u0186\u0187\u0189-\u018B\u018E-\u0191\u0193\u0194\u0196-\u0198\u019C\u019D\u019F\u01A0\u01A2\u01A4\u01A6\u01A7\u01A9\u01AC\u01AE\u01AF\u01B1-\u01B3\u01B5\u01B7\u01B8\u01BC\u01C4\u01C7\u01CA\u01CD\u01CF\u01D1\u01D3\u01D5\u01D7\u01D9\u01DB\u01DE\u01E0\u01E2\u01E4\u01E6\u01E8\u01EA\u01EC\u01EE\u01F1\u01F4\u01F6-\u01F8\u01FA\u01FC\u01FE\u0200\u0202\u0204\u0206\u0208\u020A\u020C\u020E\u0210\u0212\u0214\u0216\u0218\u021A\u021C\u021E\u0220\u0222\u0224\u0226\u0228\u022A\u022C\u022E\u0230\u0232\u023A\u023B\u023D\u023E\u0241\u0243-\u0246\u0248\u024A\u024C\u024E\u2C60\u2C62-\u2C64\u2C67\u2C69\u2C6B\u2C6D-\u2C70\u2C72\u2C75\u2C7E\u2C7F\uA722\uA724\uA726\uA728\uA72A\uA72C\uA72E\uA732\uA734\uA736\uA738\uA73A\uA73C\uA73E\uA740\uA742\uA744\uA746\uA748\uA74A\uA74C\uA74E\uA750\uA752\uA754\uA756\uA758\uA75A\uA75C\uA75E\uA760\uA762\uA764\uA766\uA768\uA76A\uA76C\uA76E\uA779\uA77B\uA77D\uA77E\uA780\uA782\uA784\uA786\uA78B\uA78D\uA790\uA792\uA796\uA798\uA79A\uA79C\uA79E\uA7A0\uA7A2\uA7A4\uA7A6\uA7A8\uA7AA-\uA7AE\uA7B0-\uA7B4\uA7B6\uA7B8\u1E00\u1E02\u1E04\u1E06\u1E08\u1E0A\u1E0C\u1E0E\u1E10\u1E12\u1E14\u1E16\u1E18\u1E1A\u1E1C\u1E1E\u1E20\u1E22\u1E24\u1E26\u1E28\u1E2A\u1E2C\u1E2E\u1E30\u1E32\u1E34\u1E36\u1E38\u1E3A\u1E3C\u1E3E\u1E40\u1E42\u1E44\u1E46\u1E48\u1E4A\u1E4C\u1E4E\u1E50\u1E52\u1E54\u1E56\u1E58\u1E5A\u1E5C\u1E5E\u1E60\u1E62\u1E64\u1E66\u1E68\u1E6A\u1E6C\u1E6E\u1E70\u1E72\u1E74\u1E76\u1E78\u1E7A\u1E7C\u1E7E\u1E80\u1E82\u1E84\u1E86\u1E88\u1E8A\u1E8C\u1E8E\u1E90\u1E92\u1E94\u1E9E\u1EA0\u1EA2\u1EA4\u1EA6\u1EA8\u1EAA\u1EAC\u1EAE\u1EB0\u1EB2\u1EB4\u1EB6\u1EB8\u1EBA\u1EBC\u1EBE\u1EC0\u1EC2\u1EC4\u1EC6\u1EC8\u1ECA\u1ECC\u1ECE\u1ED0\u1ED2\u1ED4\u1ED6\u1ED8\u1EDA\u1EDC\u1EDE\u1EE0\u1EE2\u1EE4\u1EE6\u1EE8\u1EEA\u1EEC\u1EEE\u1EF0\u1EF2\u1EF4\u1EF6\u1EF8\u1EFA\u1EFC\u1EFEЁА-ЯӘӨҮҖҢҺΑ-ΩΆΈΊΌΏΉΎА-ЩЮЯІЇЄҐЃЅЈЉЊЌЀЍ\u1200-\u137F\u0980-\u09FF\u0591-\u05F4\uFB1D-\uFB4F\u0620-\u064A\u066E-\u06D5\u06E5-\u06FF\u0750-\u077F\u08A0-\u08BD\uFB50-\uFBB1\uFBD3-\uFD3D\uFD50-\uFDC7\uFDF0-\uFDFB\uFE70-\uFEFC\U0001EE00-\U0001EEBB\u0D80-\u0DFF\u0900-\u097F\u0C80-\u0CFF\u0B80-\u0BFF\u0C00-\u0C7F\uAC00-\uD7AF\u1100-\u11FF\u3040-\u309F\u30A0-\u30FFー\u4E00-\u62FF\u6300-\u77FF\u7800-\u8CFF\u8D00-\u9FFF\u3400-\u4DBF\U00020000-\U000215FF\U00021600-\U000230FF\U00023100-\U000245FF\U00024600-\U000260FF\U00026100-\U000275FF\U00027600-\U000290FF\U00029100-\U0002A6DF\U0002A700-\U0002B73F\U0002B740-\U0002B81F\U0002B820-\U0002CEAF\U0002CEB0-\U0002EBEF\u2E80-\u2EFF\u2F00-\u2FDF\u2FF0-\u2FFF\u3000-\u303F\u31C0-\u31EF\u3200-\u32FF\u3300-\u33FF\uF900-\uFAFF\uFE30-\uFE4F\U0001F200-\U0001F2FF\U0002F800-\U0002FA1F])\.$infix_finditer>\.\.+|…|[\u00A6\u00A9\u00AE\u00B0\u0482\u058D\u058E\u060E\u060F\u06DE\u06E9\u06FD\u06FE\u07F6\u09FA\u0B70\u0BF3-\u0BF8\u0BFA\u0C7F\u0D4F\u0D79\u0F01-\u0F03\u0F13\u0F15-\u0F17\u0F1A-\u0F1F\u0F34\u0F36\u0F38\u0FBE-\u0FC5\u0FC7-\u0FCC\u0FCE\u0FCF\u0FD5-\u0FD8\u109E\u109F\u1390-\u1399\u1940\u19DE-\u19FF\u1B61-\u1B6A\u1B74-\u1B7C\u2100\u2101\u2103-\u2106\u2108\u2109\u2114\u2116\u2117\u211E-\u2123\u2125\u2127\u2129\u212E\u213A\u213B\u214A\u214C\u214D\u214F\u218A\u218B\u2195-\u2199\u219C-\u219F\u21A1\u21A2\u21A4\u21A5\u21A7-\u21AD\u21AF-\u21CD\u21D0\u21D1\u21D3\u21D5-\u21F3\u2300-\u2307\u230C-\u231F\u2322-\u2328\u232B-\u237B\u237D-\u239A\u23B4-\u23DB\u23E2-\u2426\u2440-\u244A\u249C-\u24E9\u2500-\u25B6\u25B8-\u25C0\u25C2-\u25F7\u2600-\u266E\u2670-\u2767\u2794-\u27BF\u2800-\u28FF\u2B00-\u2B2F\u2B45\u2B46\u2B4D-\u2B73\u2B76-\u2B95\u2B98-\u2BC8\u2BCA-\u2BFE\u2CE5-\u2CEA\u2E80-\u2E99\u2E9B-\u2EF3\u2F00-\u2FD5\u2FF0-\u2FFB\u3004\u3012\u3013\u3020\u3036\u3037\u303E\u303F\u3190\u3191\u3196-\u319F\u31C0-\u31E3\u3200-\u321E\u322A-\u3247\u3250\u3260-\u327F\u328A-\u32B0\u32C0-\u32FE\u3300-\u33FF\u4DC0-\u4DFF\uA490-\uA4C6\uA828-\uA82B\uA836\uA837\uA839\uAA77-\uAA79\uFDFD\uFFE4\uFFE8\uFFED\uFFEE\uFFFC\uFFFD\U00010137-\U0001013F\U00010179-\U00010189\U0001018C-\U0001018E\U00010190-\U0001019B\U000101A0\U000101D0-\U000101FC\U00010877\U00010878\U00010AC8\U0001173F\U00016B3C-\U00016B3F\U00016B45\U0001BC9C\U0001D000-\U0001D0F5\U0001D100-\U0001D126\U0001D129-\U0001D164\U0001D16A-\U0001D16C\U0001D183\U0001D184\U0001D18C-\U0001D1A9\U0001D1AE-\U0001D1E8\U0001D200-\U0001D241\U0001D245\U0001D300-\U0001D356\U0001D800-\U0001D9FF\U0001DA37-\U0001DA3A\U0001DA6D-\U0001DA74\U0001DA76-\U0001DA83\U0001DA85\U0001DA86\U0001ECAC\U0001F000-\U0001F02B\U0001F030-\U0001F093\U0001F0A0-\U0001F0AE\U0001F0B1-\U0001F0BF\U0001F0C1-\U0001F0CF\U0001F0D1-\U0001F0F5\U0001F110-\U0001F16B\U0001F170-\U0001F1AC\U0001F1E6-\U0001F202\U0001F210-\U0001F23B\U0001F240-\U0001F248\U0001F250\U0001F251\U0001F260-\U0001F265\U0001F300-\U0001F3FA\U0001F400-\U0001F6D4\U0001F6E0-\U0001F6EC\U0001F6F0-\U0001F6F9\U0001F700-\U0001F773\U0001F780-\U0001F7D8\U0001F800-\U0001F80B\U0001F810-\U0001F847\U0001F850-\U0001F859\U0001F860-\U0001F887\U0001F890-\U0001F8AD\U0001F900-\U0001F90B\U0001F910-\U0001F93E\U0001F940-\U0001F970\U0001F973-\U0001F976\U0001F97A\U0001F97C-\U0001F9A2\U0001F9B0-\U0001F9B9\U0001F9C0-\U0001F9C2\U0001F9D0-\U0001F9FF\U0001FA60-\U0001FA6D]|(?<=[0-9])[+\-\*^](?=[0-9-])|(?<=[a-z\uFF41-\uFF5A\u00DF-\u00F6\u00F8-\u00FF\u0101\u0103\u0105\u0107\u0109\u010B\u010D\u010F\u0111\u0113\u0115\u0117\u0119\u011B\u011D\u011F\u0121\u0123\u0125\u0127\u0129\u012B\u012D\u012F\u0131\u0133\u0135\u0137\u0138\u013A\u013C\u013E\u0140\u0142\u0144\u0146\u0148\u0149\u014B\u014D\u014F\u0151\u0153\u0155\u0157\u0159\u015B\u015D\u015F\u0161\u0163\u0165\u0167\u0169\u016B\u016D\u016F\u0171\u0173\u0175\u0177\u017A\u017C\u017E\u017F\u0180\u0183\u0185\u0188\u018C\u018D\u0192\u0195\u0199-\u019B\u019E\u01A1\u01A3\u01A5\u01A8\u01AA\u01AB\u01AD\u01B0\u01B4\u01B6\u01B9\u01BA\u01BD-\u01BF\u01C6\u01C9\u01CC\u01CE\u01D0\u01D2\u01D4\u01D6\u01D8\u01DA\u01DC\u01DD\u01DF\u01E1\u01E3\u01E5\u01E7\u01E9\u01EB\u01ED\u01EF\u01F0\u01F3\u01F5\u01F9\u01FB\u01FD\u01FF\u0201\u0203\u0205\u0207\u0209\u020B\u020D\u020F\u0211\u0213\u0215\u0217\u0219\u021B\u021D\u021F\u0221\u0223\u0225\u0227\u0229\u022B\u022D\u022F\u0231\u0233-\u0239\u023C\u023F\u0240\u0242\u0247\u0249\u024B\u024D\u024F\u2C61\u2C65\u2C66\u2C68\u2C6A\u2C6C\u2C71\u2C73\u2C74\u2C76-\u2C7B\uA723\uA725\uA727\uA729\uA72B\uA72D\uA72F-\uA731\uA733\uA735\uA737\uA739\uA73B\uA73D\uA73F\uA741\uA743\uA745\uA747\uA749\uA74B\uA74D\uA74F\uA751\uA753\uA755\uA757\uA759\uA75B\uA75D\uA75F\uA761\uA763\uA765\uA767\uA769\uA76B\uA76D\uA76F\uA771-\uA778\uA77A\uA77C\uA77F\uA781\uA783\uA785\uA787\uA78C\uA78E\uA791\uA793-\uA795\uA797\uA799\uA79B\uA79D\uA79F\uA7A1\uA7A3\uA7A5\uA7A7\uA7A9\uA7AF\uA7B5\uA7B7\uA7B9\uA7FA\uAB30-\uAB5A\uAB60-\uAB64\u0250-\u02AF\u1D00-\u1D25\u1D6B-\u1D77\u1D79-\u1D9A\u1E01\u1E03\u1E05\u1E07\u1E09\u1E0B\u1E0D\u1E0F\u1E11\u1E13\u1E15\u1E17\u1E19\u1E1B\u1E1D\u1E1F\u1E21\u1E23\u1E25\u1E27\u1E29\u1E2B\u1E2D\u1E2F\u1E31\u1E33\u1E35\u1E37\u1E39\u1E3B\u1E3D\u1E3F\u1E41\u1E43\u1E45\u1E47\u1E49\u1E4B\u1E4D\u1E4F\u1E51\u1E53\u1E55\u1E57\u1E59\u1E5B\u1E5D\u1E5F\u1E61\u1E63\u1E65\u1E67\u1E69\u1E6B\u1E6D\u1E6F\u1E71\u1E73\u1E75\u1E77\u1E79\u1E7B\u1E7D\u1E7F\u1E81\u1E83\u1E85\u1E87\u1E89\u1E8B\u1E8D\u1E8F\u1E91\u1E93\u1E95-\u1E9D\u1E9F\u1EA1\u1EA3\u1EA5\u1EA7\u1EA9\u1EAB\u1EAD\u1EAF\u1EB1\u1EB3\u1EB5\u1EB7\u1EB9\u1EBB\u1EBD\u1EBF\u1EC1\u1EC3\u1EC5\u1EC7\u1EC9\u1ECB\u1ECD\u1ECF\u1ED1\u1ED3\u1ED5\u1ED7\u1ED9\u1EDB\u1EDD\u1EDF\u1EE1\u1EE3\u1EE5\u1EE7\u1EE9\u1EEB\u1EED\u1EEF\u1EF1\u1EF3\u1EF5\u1EF7\u1EF9\u1EFB\u1EFD\u1EFFёа-яәөүҗңһα-ωάέίόώήύа-щюяіїєґѓѕјљњќѐѝ\u1200-\u137F\u0980-\u09FF\u0591-\u05F4\uFB1D-\uFB4F\u0620-\u064A\u066E-\u06D5\u06E5-\u06FF\u0750-\u077F\u08A0-\u08BD\uFB50-\uFBB1\uFBD3-\uFD3D\uFD50-\uFDC7\uFDF0-\uFDFB\uFE70-\uFEFC\U0001EE00-\U0001EEBB\u0D80-\u0DFF\u0900-\u097F\u0C80-\u0CFF\u0B80-\u0BFF\u0C00-\u0C7F\uAC00-\uD7AF\u1100-\u11FF\u3040-\u309F\u30A0-\u30FFー\u4E00-\u62FF\u6300-\u77FF\u7800-\u8CFF\u8D00-\u9FFF\u3400-\u4DBF\U00020000-\U000215FF\U00021600-\U000230FF\U00023100-\U000245FF\U00024600-\U000260FF\U00026100-\U000275FF\U00027600-\U000290FF\U00029100-\U0002A6DF\U0002A700-\U0002B73F\U0002B740-\U0002B81F\U0002B820-\U0002CEAF\U0002CEB0-\U0002EBEF\u2E80-\u2EFF\u2F00-\u2FDF\u2FF0-\u2FFF\u3000-\u303F\u31C0-\u31EF\u3200-\u32FF\u3300-\u33FF\uF900-\uFAFF\uFE30-\uFE4F\U0001F200-\U0001F2FF\U0002F800-\U0002FA1F\'"”“`‘´’‚,„»«「」『』（）〔〕【】《》〈〉〈〉⟦⟧])\.(?=[A-Z\uFF21-\uFF3A\u00C0-\u00D6\u00D8-\u00DE\u0100\u0102\u0104\u0106\u0108\u010A\u010C\u010E\u0110\u0112\u0114\u0116\u0118\u011A\u011C\u011E\u0120\u0122\u0124\u0126\u0128\u012A\u012C\u012E\u0130\u0132\u0134\u0136\u0139\u013B\u013D\u013F\u0141\u0143\u0145\u0147\u014A\u014C\u014E\u0150\u0152\u0154\u0156\u0158\u015A\u015C\u015E\u0160\u0162\u0164\u0166\u0168\u016A\u016C\u016E\u0170\u0172\u0174\u0176\u0178\u0179\u017B\u017D\u0181\u0182\u0184\u0186\u0187\u0189-\u018B\u018E-\u0191\u0193\u0194\u0196-\u0198\u019C\u019D\u019F\u01A0\u01A2\u01A4\u01A6\u01A7\u01A9\u01AC\u01AE\u01AF\u01B1-\u01B3\u01B5\u01B7\u01B8\u01BC\u01C4\u01C7\u01CA\u01CD\u01CF\u01D1\u01D3\u01D5\u01D7\u01D9\u01DB\u01DE\u01E0\u01E2\u01E4\u01E6\u01E8\u01EA\u01EC\u01EE\u01F1\u01F4\u01F6-\u01F8\u01FA\u01FC\u01FE\u0200\u0202\u0204\u0206\u0208\u020A\u020C\u020E\u0210\u0212\u0214\u0216\u0218\u021A\u021C\u021E\u0220\u0222\u0224\u0226\u0228\u022A\u022C\u022E\u0230\u0232\u023A\u023B\u023D\u023E\u0241\u0243-\u0246\u0248\u024A\u024C\u024E\u2C60\u2C62-\u2C64\u2C67\u2C69\u2C6B\u2C6D-\u2C70\u2C72\u2C75\u2C7E\u2C7F\uA722\uA724\uA726\uA728\uA72A\uA72C\uA72E\uA732\uA734\uA736\uA738\uA73A\uA73C\uA73E\uA740\uA742\uA744\uA746\uA748\uA74A\uA74C\uA74E\uA750\uA752\uA754\uA756\uA758\uA75A\uA75C\uA75E\uA760\uA762\uA764\uA766\uA768\uA76A\uA76C\uA76E\uA779\uA77B\uA77D\uA77E\uA780\uA782\uA784\uA786\uA78B\uA78D\uA790\uA792\uA796\uA798\uA79A\uA79C\uA79E\uA7A0\uA7A2\uA7A4\uA7A6\uA7A8\uA7AA-\uA7AE\uA7B0-\uA7B4\uA7B6\uA7B8\u1E00\u1E02\u1E04\u1E06\u1E08\u1E0A\u1E0C\u1E0E\u1E10\u1E12\u1E14\u1E16\u1E18\u1E1A\u1E1C\u1E1E\u1E20\u1E22\u1E24\u1E26\u1E28\u1E2A\u1E2C\u1E2E\u1E30\u1E32\u1E34\u1E36\u1E38\u1E3A\u1E3C\u1E3E\u1E40\u1E42\u1E44\u1E46\u1E48\u1E4A\u1E4C\u1E4E\u1E50\u1E52\u1E54\u1E56\u1E58\u1E5A\u1E5C\u1E5E\u1E60\u1E62\u1E64\u1E66\u1E68\u1E6A\u1E6C\u1E6E\u1E70\u1E72\u1E74\u1E76\u1E78\u1E7A\u1E7C\u1E7E\u1E80\u1E82\u1E84\u1E86\u1E88\u1E8A\u1E8C\u1E8E\u1E90\u1E92\u1E94\u1E9E\u1EA0\u1EA2\u1EA4\u1EA6\u1EA8\u1EAA\u1EAC\u1EAE\u1EB0\u1EB2\u1EB4\u1EB6\u1EB8\u1EBA\u1EBC\u1EBE\u1EC0\u1EC2\u1EC4\u1EC6\u1EC8\u1ECA\u1ECC\u1ECE\u1ED0\u1ED2\u1ED4\u1ED6\u1ED8\u1EDA\u1EDC\u1EDE\u1EE0\u1EE2\u1EE4\u1EE6\u1EE8\u1EEA\u1EEC\u1EEE\u1EF0\u1EF2\u1EF4\u1EF6\u1EF8\u1EFA\u1EFC\u1EFEЁА-ЯӘӨҮҖҢҺΑ-ΩΆΈΊΌΏΉΎА-ЩЮЯІЇЄҐЃЅЈЉЊЌЀЍ\u1200-\u137F\u0980-\u09FF\u0591-\u05F4\uFB1D-\uFB4F\u0620-\u064A\u066E-\u06D5\u06E5-\u06FF\u0750-\u077F\u08A0-\u08BD\uFB50-\uFBB1\uFBD3-\uFD3D\uFD50-\uFDC7\uFDF0-\uFDFB\uFE70-\uFEFC\U0001EE00-\U0001EEBB\u0D80-\u0DFF\u0900-\u097F\u0C80-\u0CFF\u0B80-\u0BFF\u0C00-\u0C7F\uAC00-\uD7AF\u1100-\u11FF\u3040-\u309F\u30A0-\u30FFー\u4E00-\u62FF\u6300-\u77FF\u7800-\u8CFF\u8D00-\u9FFF\u3400-\u4DBF\U00020000-\U000215FF\U00021600-\U000230FF\U00023100-\U000245FF\U00024600-\U000260FF\U00026100-\U000275FF\U00027600-\U000290FF\U00029100-\U0002A6DF\U0002A700-\U0002B73F\U0002B740-\U0002B81F\U0002B820-\U0002CEAF\U0002CEB0-\U0002EBEF\u2E80-\u2EFF\u2F00-\u2FDF\u2FF0-\u2FFF\u3000-\u303F\u31C0-\u31EF\u3200-\u32FF\u3300-\u33FF\uF900-\uFAFF\uFE30-\uFE4F\U0001F200-\U0001F2FF\U0002F800-\U0002FA1F\'"”“`‘´’‚,„»«「」『』（）〔〕【】《》〈〉〈〉⟦⟧])|(?<=[A-Za-z\uFF21-\uFF3A\uFF41-\uFF5A\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u00FF\u0100-\u017F\u0180-\u01BF\u01C4-\u024F\u2C60-\u2C7B\u2C7E\u2C7F\uA722-\uA76F\uA771-\uA787\uA78B-\uA78E\uA790-\uA7B9\uA7FA\uAB30-\uAB5A\uAB60-\uAB64\u0250-\u02AF\u1D00-\u1D25\u1D6B-\u1D77\u1D79-\u1D9A\u1E00-\u1EFFёа-яЁА-ЯәөүҗңһӘӨҮҖҢҺα-ωάέίόώήύΑ-ΩΆΈΊΌΏΉΎа-щюяіїєґА-ЩЮЯІЇЄҐѓѕјљњќѐѝЃЅЈЉЊЌЀЍ\u1200-\u137F\u0980-\u09FF\u0591-\u05F4\uFB1D-\uFB4F\u0620-\u064A\u066E-\u06D5\u06E5-\u06FF\u0750-\u077F\u08A0-\u08BD\uFB50-\uFBB1\uFBD3-\uFD3D\uFD50-\uFDC7\uFDF0-\uFDFB\uFE70-\uFEFC\U0001EE00-\U0001EEBB\u0D80-\u0DFF\u0900-\u097F\u0C80-\u0CFF\u0B80-\u0BFF\u0C00-\u0C7F\uAC00-\uD7AF\u1100-\u11FF\u3040-\u309F\u30A0-\u30FFー\u4E00-\u62FF\u6300-\u77FF\u7800-\u8CFF\u8D00-\u9FFF\u3400-\u4DBF\U00020000-\U000215FF\U00021600-\U000230FF\U00023100-\U000245FF\U00024600-\U000260FF\U00026100-\U000275FF\U00027600-\U000290FF\U00029100-\U0002A6DF\U0002A700-\U0002B73F\U0002B740-\U0002B81F\U0002B820-\U0002CEAF\U0002CEB0-\U0002EBEF\u2E80-\u2EFF\u2F00-\u2FDF\u2FF0-\u2FFF\u3000-\u303F\u31C0-\u31EF\u3200-\u32FF\u3300-\u33FF\uF900-\uFAFF\uFE30-\uFE4F\U0001F200-\U0001F2FF\U0002F800-\U0002FA1F]),(?=[A-Za-z\uFF21-\uFF3A\uFF41-\uFF5A\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u00FF\u0100-\u017F\u0180-\u01BF\u01C4-\u024F\u2C60-\u2C7B\u2C7E\u2C7F\uA722-\uA76F\uA771-\uA787\uA78B-\uA78E\uA790-\uA7B9\uA7FA\uAB30-\uAB5A\uAB60-\uAB64\u0250-\u02AF\u1D00-\u1D25\u1D6B-\u1D77\u1D79-\u1D9A\u1E00-\u1EFFёа-яЁА-ЯәөүҗңһӘӨҮҖҢҺα-ωάέίόώήύΑ-ΩΆΈΊΌΏΉΎа-щюяіїєґА-ЩЮЯІЇЄҐѓѕјљњќѐѝЃЅЈЉЊЌЀЍ\u1200-\u137F\u0980-\u09FF\u0591-\u05F4\uFB1D-\uFB4F\u0620-\u064A\u066E-\u06D5\u06E5-\u06FF\u0750-\u077F\u08A0-\u08BD\uFB50-\uFBB1\uFBD3-\uFD3D\uFD50-\uFDC7\uFDF0-\uFDFB\uFE70-\uFEFC\U0001EE00-\U0001EEBB\u0D80-\u0DFF\u0900-\u097F\u0C80-\u0CFF\u0B80-\u0BFF\u0C00-\u0C7F\uAC00-\uD7AF\u1100-\u11FF\u3040-\u309F\u30A0-\u30FFー\u4E00-\u62FF\u6300-\u77FF\u7800-\u8CFF\u8D00-\u9FFF\u3400-\u4DBF\U00020000-\U000215FF\U00021600-\U000230FF\U00023100-\U000245FF\U00024600-\U000260FF\U00026100-\U000275FF\U00027600-\U000290FF\U00029100-\U0002A6DF\U0002A700-\U0002B73F\U0002B740-\U0002B81F\U0002B820-\U0002CEAF\U0002CEB0-\U0002EBEF\u2E80-\u2EFF\u2F00-\u2FDF\u2FF0-\u2FFF\u3000-\u303F\u31C0-\u31EF\u3200-\u32FF\u3300-\u33FF\uF900-\uFAFF\uFE30-\uFE4F\U0001F200-\U0001F2FF\U0002F800-\U0002FA1F])|(?<=[A-Za-z\uFF21-\uFF3A\uFF41-\uFF5A\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u00FF\u0100-\u017F\u0180-\u01BF\u01C4-\u024F\u2C60-\u2C7B\u2C7E\u2C7F\uA722-\uA76F\uA771-\uA787\uA78B-\uA78E\uA790-\uA7B9\uA7FA\uAB30-\uAB5A\uAB60-\uAB64\u0250-\u02AF\u1D00-\u1D25\u1D6B-\u1D77\u1D79-\u1D9A\u1E00-\u1EFFёа-яЁА-ЯәөүҗңһӘӨҮҖҢҺα-ωάέίόώήύΑ-ΩΆΈΊΌΏΉΎа-щюяіїєґА-ЩЮЯІЇЄҐѓѕјљњќѐѝЃЅЈЉЊЌЀЍ\u1200-\u137F\u0980-\u09FF\u0591-\u05F4\uFB1D-\uFB4F\u0620-\u064A\u066E-\u06D5\u06E5-\u06FF\u0750-\u077F\u08A0-\u08BD\uFB50-\uFBB1\uFBD3-\uFD3D\uFD50-\uFDC7\uFDF0-\uFDFB\uFE70-\uFEFC\U0001EE00-\U0001EEBB\u0D80-\u0DFF\u0900-\u097F\u0C80-\u0CFF\u0B80-\u0BFF\u0C00-\u0C7F\uAC00-\uD7AF\u1100-\u11FF\u3040-\u309F\u30A0-\u30FFー\u4E00-\u62FF\u6300-\u77FF\u7800-\u8CFF\u8D00-\u9FFF\u3400-\u4DBF\U00020000-\U000215FF\U00021600-\U000230FF\U00023100-\U000245FF\U00024600-\U000260FF\U00026100-\U000275FF\U00027600-\U000290FF\U00029100-\U0002A6DF\U0002A700-\U0002B73F\U0002B740-\U0002B81F\U0002B820-\U0002CEAF\U0002CEB0-\U0002EBEF\u2E80-\u2EFF\u2F00-\u2FDF\u2FF0-\u2FFF\u3000-\u303F\u31C0-\u31EF\u3200-\u32FF\u3300-\u33FF\uF900-\uFAFF\uFE30-\uFE4F\U0001F200-\U0001F2FF\U0002F800-\U0002FA1F])(?:-|–|—|--|---|——|~)(?=[A-Za-z\uFF21-\uFF3A\uFF41-\uFF5A\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u00FF\u0100-\u017F\u0180-\u01BF\u01C4-\u024F\u2C60-\u2C7B\u2C7E\u2C7F\uA722-\uA76F\uA771-\uA787\uA78B-\uA78E\uA790-\uA7B9\uA7FA\uAB30-\uAB5A\uAB60-\uAB64\u0250-\u02AF\u1D00-\u1D25\u1D6B-\u1D77\u1D79-\u1D9A\u1E00-\u1EFFёа-яЁА-ЯәөүҗңһӘӨҮҖҢҺα-ωάέίόώήύΑ-ΩΆΈΊΌΏΉΎа-щюяіїєґА-ЩЮЯІЇЄҐѓѕјљњќѐѝЃЅЈЉЊЌЀЍ\u1200-\u137F\u0980-\u09FF\u0591-\u05F4\uFB1D-\uFB4F\u0620-\u064A\u066E-\u06D5\u06E5-\u06FF\u0750-\u077F\u08A0-\u08BD\uFB50-\uFBB1\uFBD3-\uFD3D\uFD50-\uFDC7\uFDF0-\uFDFB\uFE70-\uFEFC\U0001EE00-\U0001EEBB\u0D80-\u0DFF\u0900-\u097F\u0C80-\u0CFF\u0B80-\u0BFF\u0C00-\u0C7F\uAC00-\uD7AF\u1100-\u11FF\u3040-\u309F\u30A0-\u30FFー\u4E00-\u62FF\u6300-\u77FF\u7800-\u8CFF\u8D00-\u9FFF\u3400-\u4DBF\U00020000-\U000215FF\U00021600-\U000230FF\U00023100-\U000245FF\U00024600-\U000260FF\U00026100-\U000275FF\U00027600-\U000290FF\U00029100-\U0002A6DF\U0002A700-\U0002B73F\U0002B740-\U0002B81F\U0002B820-\U0002CEAF\U0002CEB0-\U0002EBEF\u2E80-\u2EFF\u2F00-\u2FDF\u2FF0-\u2FFF\u3000-\u303F\u31C0-\u31EF\u3200-\u32FF\u3300-\u33FF\uF900-\uFAFF\uFE30-\uFE4F\U0001F200-\U0001F2FF\U0002F800-\U0002FA1F])|(?<=[A-Za-z\uFF21-\uFF3A\uFF41-\uFF5A\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u00FF\u0100-\u017F\u0180-\u01BF\u01C4-\u024F\u2C60-\u2C7B\u2C7E\u2C7F\uA722-\uA76F\uA771-\uA787\uA78B-\uA78E\uA790-\uA7B9\uA7FA\uAB30-\uAB5A\uAB60-\uAB64\u0250-\u02AF\u1D00-\u1D25\u1D6B-\u1D77\u1D79-\u1D9A\u1E00-\u1EFFёа-яЁА-ЯәөүҗңһӘӨҮҖҢҺα-ωάέίόώήύΑ-ΩΆΈΊΌΏΉΎа-щюяіїєґА-ЩЮЯІЇЄҐѓѕјљњќѐѝЃЅЈЉЊЌЀЍ\u1200-\u137F\u0980-\u09FF\u0591-\u05F4\uFB1D-\uFB4F\u0620-\u064A\u066E-\u06D5\u06E5-\u06FF\u0750-\u077F\u08A0-\u08BD\uFB50-\uFBB1\uFBD3-\uFD3D\uFD50-\uFDC7\uFDF0-\uFDFB\uFE70-\uFEFC\U0001EE00-\U0001EEBB\u0D80-\u0DFF\u0900-\u097F\u0C80-\u0CFF\u0B80-\u0BFF\u0C00-\u0C7F\uAC00-\uD7AF\u1100-\u11FF\u3040-\u309F\u30A0-\u30FFー\u4E00-\u62FF\u6300-\u77FF\u7800-\u8CFF\u8D00-\u9FFF\u3400-\u4DBF\U00020000-\U000215FF\U00021600-\U000230FF\U00023100-\U000245FF\U00024600-\U000260FF\U00026100-\U000275FF\U00027600-\U000290FF\U00029100-\U0002A6DF\U0002A700-\U0002B73F\U0002B740-\U0002B81F\U0002B820-\U0002CEAF\U0002CEB0-\U0002EBEF\u2E80-\u2EFF\u2F00-\u2FDF\u2FF0-\u2FFF\u3000-\u303F\u31C0-\u31EF\u3200-\u32FF\u3300-\u33FF\uF900-\uFAFF\uFE30-\uFE4F\U0001F200-\U0001F2FF\U0002F800-\U0002FA1F0-9])[:<>=/](?=[A-Za-z\uFF21-\uFF3A\uFF41-\uFF5A\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u00FF\u0100-\u017F\u0180-\u01BF\u01C4-\u024F\u2C60-\u2C7B\u2C7E\u2C7F\uA722-\uA76F\uA771-\uA787\uA78B-\uA78E\uA790-\uA7B9\uA7FA\uAB30-\uAB5A\uAB60-\uAB64\u0250-\u02AF\u1D00-\u1D25\u1D6B-\u1D77\u1D79-\u1D9A\u1E00-\u1EFFёа-яЁА-ЯәөүҗңһӘӨҮҖҢҺα-ωάέίόώήύΑ-ΩΆΈΊΌΏΉΎа-щюяіїєґА-ЩЮЯІЇЄҐѓѕјљњќѐѝЃЅЈЉЊЌЀЍ\u1200-\u137F\u0980-\u09FF\u0591-\u05F4\uFB1D-\uFB4F\u0620-\u064A\u066E-\u06D5\u06E5-\u06FF\u0750-\u077F\u08A0-\u08BD\uFB50-\uFBB1\uFBD3-\uFD3D\uFD50-\uFDC7\uFDF0-\uFDFB\uFE70-\uFEFC\U0001EE00-\U0001EEBB\u0D80-\u0DFF\u0900-\u097F\u0C80-\u0CFF\u0B80-\u0BFF\u0C00-\u0C7F\uAC00-\uD7AF\u1100-\u11FF\u3040-\u309F\u30A0-\u30FFー\u4E00-\u62FF\u6300-\u77FF\u7800-\u8CFF\u8D00-\u9FFF\u3400-\u4DBF\U00020000-\U000215FF\U00021600-\U000230FF\U00023100-\U000245FF\U00024600-\U000260FF\U00026100-\U000275FF\U00027600-\U000290FF\U00029100-\U0002A6DF\U0002A700-\U0002B73F\U0002B740-\U0002B81F\U0002B820-\U0002CEAF\U0002CEB0-\U0002EBEF\u2E80-\u2EFF\u2F00-\u2FDF\u2FF0-\u2FFF\u3000-\u303F\u31C0-\u31EF\u3200-\u32FF\u3300-\u33FF\uF900-\uFAFF\uFE30-\uFE4F\U0001F200-\U0001F2FF\U0002F800-\U0002FA1F])token_matchurl_match (?u)^(?:(?:[\w\+\-\.]{2,})://)?(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[A-Za-z0-9\u00a1-\uffff][A-Za-z0-9\u00a1-\uffff_-]{0,62})?[A-Za-z0-9\u00a1-\uffff]\.)+(?:[a-z\uFF41-\uFF5A\u00DF-\u00F6\u00F8-\u00FF\u0101\u0103\u0105\u0107\u0109\u010B\u010D\u010F\u0111\u0113\u0115\u0117\u0119\u011B\u011D\u011F\u0121\u0123\u0125\u0127\u0129\u012B\u012D\u012F\u0131\u0133\u0135\u0137\u0138\u013A\u013C\u013E\u0140\u0142\u0144\u0146\u0148\u0149\u014B\u014D\u014F\u0151\u0153\u0155\u0157\u0159\u015B\u015D\u015F\u0161\u0163\u0165\u0167\u0169\u016B\u016D\u016F\u0171\u0173\u0175\u0177\u017A\u017C\u017E\u017F\u0180\u0183\u0185\u0188\u018C\u018D\u0192\u0195\u0199-\u019B\u019E\u01A1\u01A3\u01A5\u01A8\u01AA\u01AB\u01AD\u01B0\u01B4\u01B6\u01B9\u01BA\u01BD-\u01BF\u01C6\u01C9\u01CC\u01CE\u01D0\u01D2\u01D4\u01D6\u01D8\u01DA\u01DC\u01DD\u01DF\u01E1\u01E3\u01E5\u01E7\u01E9\u01EB\u01ED\u01EF\u01F0\u01F3\u01F5\u01F9\u01FB\u01FD\u01FF\u0201\u0203\u0205\u0207\u0209\u020B\u020D\u020F\u0211\u0213\u0215\u0217\u0219\u021B\u021D\u021F\u0221\u0223\u0225\u0227\u0229\u022B\u022D\u022F\u0231\u0233-\u0239\u023C\u023F\u0240\u0242\u0247\u0249\u024B\u024D\u024F\u2C61\u2C65\u2C66\u2C68\u2C6A\u2C6C\u2C71\u2C73\u2C74\u2C76-\u2C7B\uA723\uA725\uA727\uA729\uA72B\uA72D\uA72F-\uA731\uA733\uA735\uA737\uA739\uA73B\uA73D\uA73F\uA741\uA743\uA745\uA747\uA749\uA74B\uA74D\uA74F\uA751\uA753\uA755\uA757\uA759\uA75B\uA75D\uA75F\uA761\uA763\uA765\uA767\uA769\uA76B\uA76D\uA76F\uA771-\uA778\uA77A\uA77C\uA77F\uA781\uA783\uA785\uA787\uA78C\uA78E\uA791\uA793-\uA795\uA797\uA799\uA79B\uA79D\uA79F\uA7A1\uA7A3\uA7A5\uA7A7\uA7A9\uA7AF\uA7B5\uA7B7\uA7B9\uA7FA\uAB30-\uAB5A\uAB60-\uAB64\u0250-\u02AF\u1D00-\u1D25\u1D6B-\u1D77\u1D79-\u1D9A\u1E01\u1E03\u1E05\u1E07\u1E09\u1E0B\u1E0D\u1E0F\u1E11\u1E13\u1E15\u1E17\u1E19\u1E1B\u1E1D\u1E1F\u1E21\u1E23\u1E25\u1E27\u1E29\u1E2B\u1E2D\u1E2F\u1E31\u1E33\u1E35\u1E37\u1E39\u1E3B\u1E3D\u1E3F\u1E41\u1E43\u1E45\u1E47\u1E49\u1E4B\u1E4D\u1E4F\u1E51\u1E53\u1E55\u1E57\u1E59\u1E5B\u1E5D\u1E5F\u1E61\u1E63\u1E65\u1E67\u1E69\u1E6B\u1E6D\u1E6F\u1E71\u1E73\u1E75\u1E77\u1E79\u1E7B\u1E7D\u1E7F\u1E81\u1E83\u1E85\u1E87\u1E89\u1E8B\u1E8D\u1E8F\u1E91\u1E93\u1E95-\u1E9D\u1E9F\u1EA1\u1EA3\u1EA5\u1EA7\u1EA9\u1EAB\u1EAD\u1EAF\u1EB1\u1EB3\u1EB5\u1EB7\u1EB9\u1EBB\u1EBD\u1EBF\u1EC1\u1EC3\u1EC5\u1EC7\u1EC9\u1ECB\u1ECD\u1ECF\u1ED1\u1ED3\u1ED5\u1ED7\u1ED9\u1EDB\u1EDD\u1EDF\u1EE1\u1EE3\u1EE5\u1EE7\u1EE9\u1EEB\u1EED\u1EEF\u1EF1\u1EF3\u1EF5\u1EF7\u1EF9\u1EFB\u1EFD\u1EFFёа-яәөүҗңһα-ωάέίόώήύа-щюяіїєґѓѕјљњќѐѝ\u1200-\u137F\u0980-\u09FF\u0591-\u05F4\uFB1D-\uFB4F\u0620-\u064A\u066E-\u06D5\u06E5-\u06FF\u0750-\u077F\u08A0-\u08BD\uFB50-\uFBB1\uFBD3-\uFD3D\uFD50-\uFDC7\uFDF0-\uFDFB\uFE70-\uFEFC\U0001EE00-\U0001EEBB\u0D80-\u0DFF\u0900-\u097F\u0C80-\u0CFF\u0B80-\u0BFF\u0C00-\u0C7F\uAC00-\uD7AF\u1100-\u11FF\u3040-\u309F\u30A0-\u30FFー\u4E00-\u62FF\u6300-\u77FF\u7800-\u8CFF\u8D00-\u9FFF\u3400-\u4DBF\U00020000-\U000215FF\U00021600-\U000230FF\U00023100-\U000245FF\U00024600-\U000260FF\U00026100-\U000275FF\U00027600-\U000290FF\U00029100-\U0002A6DF\U0002A700-\U0002B73F\U0002B740-\U0002B81F\U0002B820-\U0002CEAF\U0002CEB0-\U0002EBEF\u2E80-\u2EFF\u2F00-\u2FDF\u2FF0-\u2FFF\u3000-\u303F\u31C0-\u31EF\u3200-\u32FF\u3300-\u33FF\uF900-\uFAFF\uFE30-\uFE4F\U0001F200-\U0001F2FF\U0002F800-\U0002FA1F]{2,63}))(?::\d{2,5})?(?:[/?#]\S*)?$exceptions A +A + A 'A'''A''(*_*)A(*_*)(-8A(-8(-:A(-:(-;A(-;(-_-)A(-_-)(._.)A(._.)(:A(:(;A(;(=A(=(>_<)A(>_<)(^_^)A(^_^)(o:A(o:(¬_¬)A(¬_¬)(ಠ_ಠ)A(ಠ_ಠ)(╯°□°）╯︵┻━┻A(╯°□°）╯︵┻━┻)-:A)-:):A):-_-A-_--__-A-__-._.A._.0.0A0.00.oA0.o0_0A0_00_oA0_o8)A8)8-)A8-)8-DA8-D8DA8D:'(A:'(:')A:'):'-(A:'-(:'-)A:'-):(A:(:((A:((:(((A:(((:()A:():)A:):))A:)):)))A:))):*A:*:-(A:-(:-((A:-((:-(((A:-(((:-)A:-):-))A:-)):-)))A:-))):-*A:-*:-/A:-/:-0A:-0:-3A:-3:->A:->:-DA:-D:-OA:-O:-PA:-P:-XA:-X:-]A:-]:-oA:-o:-pA:-p:-xA:-x:-|A:-|:-}A:-}:/A:/:0A:0:1A:1:3A:3:>A:>:DA:D:OA:O:PA:P:XA:X:]A:]:oA:o:o)A:o):pA:p:xA:x:|A:|:}A:};)A;);-)A;-);-DA;-D;DA;D;_;A;_;<.A=(A=(=)A=)=/A=/=3A=3=DA=D=[A=[=]A=]=|A=|>..<>.>A>.>>:(A>:(>:oA>:o><(((*>A><(((*>@_@A@_@C++AC++O.OAO.OO.oAO.oO_OAO_OO_oAO_oV.VAV.VV_VAV_VXDAXDXDDAXDD[-:A[-:[:A[:[=A[=\")A\")\nA\n\tA\t]=A]=^_^A^_^^__^A^__^^___^A^___^a.Aa.b.Ab.c.Ac.d.Ad.e.Ae.f.Af.g.Ag.h.Ah.i.Ai.j.Aj.k.Ak.l.Al.m.Am.n.An.o.Ao.o.0Ao.0o.OAo.Oo.oAo.oo_0Ao_0o_OAo_Oo_oAo_op.Ap.q.Aq.r.Ar.s.As.t.At.u.Au.v.Av.v.vAv.vv_vAv_vw.Aw.x.Ax.xDAxDxDDAxDDy.Ay.z.Az. A C ¯\(ツ)/¯A¯\(ツ)/¯°C.A°ACA.°F.A°AFA.°K.A°AKA.°c.A°AcA.°f.A°AfA.°k.A°AkA.ä.Aä.ö.Aö.ü.Aü.ಠ_ಠAಠ_ಠಠ︵ಠAಠ︵ಠ—A—faster_heuristics \ No newline at end of file diff --git a/he_test/vocab/key2row b/he_test/vocab/key2row new file mode 100644 index 0000000000..5416677bc7 --- /dev/null +++ b/he_test/vocab/key2row @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/he_test/vocab/lookups.bin b/he_test/vocab/lookups.bin new file mode 100644 index 0000000000..5416677bc7 --- /dev/null +++ b/he_test/vocab/lookups.bin @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/he_test/vocab/strings.json b/he_test/vocab/strings.json new file mode 100644 index 0000000000..e116405cda --- /dev/null +++ b/he_test/vocab/strings.json @@ -0,0 +1,348 @@ +[ + "\t", + "\n", + " ", + " ", + "\"", + "'", + "''", + "'-(", + "'-)", + "(", + "(((", + "(*>", + "(*_*)", + "(-8", + "(-:", + "(-;", + "(-_-)", + "(-d", + "(._.)", + "(:", + "(;", + "(=", + "(>_<)", + "(^_^)", + "(o:", + "(x:", + "(x_x)", + "(\u00ac_\u00ac)", + "(\u0ca0_\u0ca0)", + "(\u256f\u00b0\u25a1\u00b0\uff09\u256f\ufe35\u253b\u2501\u253b", + ")", + ")))", + ")-:", + ")/\u00af", + "):", + "*", + "-", + "-((", + "-))", + "-/", + "-0", + "-3", + "-8", + "-D", + "-O", + "-P", + "-X", + "-_-", + "-__-", + "-d", + "-o", + "-p", + "-x", + "-|", + ".", + "._.", + "/", + "/3", + "/d", + "0", + "0.0", + "0.o", + "0_0", + "0_o", + "1", + "3", + "33", + "333", + "8", + "8)", + "8-", + "8-)", + "8-D", + "8-d", + "8D", + "8d", + ":", + ":'(", + ":')", + ":'-(", + ":'-)", + ":(", + ":((", + ":(((", + ":()", + ":)", + ":))", + ":)))", + ":*", + ":-(", + ":-((", + ":-(((", + ":-)", + ":-))", + ":-)))", + ":-*", + ":-/", + ":-0", + ":-3", + ":->", + ":-D", + ":-O", + ":-P", + ":-X", + ":-]", + ":-d", + ":-o", + ":-p", + ":-x", + ":-|", + ":-}", + ":/", + ":0", + ":1", + ":3", + ":>", + ":D", + ":O", + ":P", + ":X", + ":]", + ":d", + ":o", + ":o)", + ":p", + ":x", + ":x)", + ":|", + ":}", + ";", + ";)", + ";-)", + ";-D", + ";-X", + ";-d", + ";D", + ";X", + ";_;", + ";d", + "<", + "<.<", + "", + "", + "=", + "=(", + "=)", + "=/", + "=3", + "=D", + "=X", + "=[", + "=]", + "=d", + "=|", + ">", + ">.<", + ">.>", + ">:(", + ">:o", + ">:x", + "><(((*>", + "@", + "@_@", + "C", + "C++", + "C.", + "D", + "F", + "F.", + "K", + "K.", + "O", + "O.O", + "O.o", + "O_O", + "O_o", + "P", + "V", + "V.V", + "V_V", + "X++", + "X.", + "X.X", + "X.x", + "XD", + "XDD", + "XX", + "XXX", + "X_X", + "X_x", + "[", + "[-:", + "[:", + "[=", + "\\", + "\\\")", + "\\n", + "\\t", + "\\x", + "]", + "]=", + "^", + "^_^", + "^__^", + "^___^", + "_*)", + "_-)", + "_.)", + "_<)", + "_^)", + "__-", + "__^", + "_\u00ac)", + "_\u0ca0)", + "a", + "a.", + "ace", + "b", + "b.", + "c", + "c++", + "c.", + "ce>", + "d", + "d)", + "d-", + "d-)", + "d-X", + "d.", + "d.d", + "d.x", + "dX", + "d_d", + "d_x", + "dd", + "ddd", + "e", + "e.", + "f", + "f.", + "g", + "g.", + "h", + "h.", + "he", + "i", + "i.", + "j", + "j.", + "k", + "k.", + "l", + "l.", + "m", + "m.", + "n", + "n.", + "o", + "o.", + "o.0", + "o.O", + "o.o", + "o_0", + "o_O", + "o_o", + "p", + "p.", + "q", + "q.", + "r", + "r.", + "s", + "s.", + "space", + "t", + "t.", + "u", + "u.", + "v", + "v.", + "v.v", + "v_v", + "w", + "w.", + "x", + "x.", + "x.X", + "x.d", + "x.x", + "xD", + "xDD", + "xX", + "xXX", + "x_X", + "x_d", + "x_x", + "xd", + "xdd", + "xxxx", + "x\ufe35x", + "y", + "y.", + "z", + "z.", + "|", + "}", + "\u00a0", + "\u00ac", + "\u00ac_\u00ac", + "\u00af", + "\u00af\\(x)/\u00af", + "\u00af\\(\u30c4)/\u00af", + "\u00b0", + "\u00b0C.", + "\u00b0F.", + "\u00b0K.", + "\u00b0X.", + "\u00b0c.", + "\u00b0f.", + "\u00b0k.", + "\u00b0x.", + "\u00e4", + "\u00e4.", + "\u00f6", + "\u00f6.", + "\u00fc", + "\u00fc.", + "\u0ca0", + "\u0ca0_\u0ca0", + "\u0ca0\ufe35\u0ca0", + "\u2014", + "\u2501", + "\u253b", + "\u253b\u2501\u253b", + "\u256f", + "\u25a1", + "\ufe35", + "\uff09" +] \ No newline at end of file diff --git a/he_test/vocab/vectors b/he_test/vocab/vectors new file mode 100644 index 0000000000000000000000000000000000000000..ebadaa58b83a76a8ce0dbf1b5a75d8fb4aa559ad GIT binary patch literal 128 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1ZlV+l>qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= XXCxM+0{I#SItm7wItsN4WCJb$RVy23 literal 0 HcmV?d00001 diff --git a/he_test/vocab/vectors.cfg b/he_test/vocab/vectors.cfg new file mode 100644 index 0000000000..32c800af49 --- /dev/null +++ b/he_test/vocab/vectors.cfg @@ -0,0 +1,3 @@ +{ + "mode":"default" +} \ No newline at end of file diff --git a/presidio-analyzer/app.py b/presidio-analyzer/app.py index 567a796cd0..9a9b5b3ec6 100644 --- a/presidio-analyzer/app.py +++ b/presidio-analyzer/app.py @@ -86,7 +86,7 @@ def analyze() -> Tuple[str, int]: texts=batch, batch_size=min( len(batch), - int(os.environ.get("BATCH_SIZE", DEFAULT_BATCH_SIZE)) + int(os.environ.get("BATCH_SIZE", DEFAULT_BATCH_SIZE)), ), language=req_data.language, correlation_id=req_data.correlation_id, @@ -99,9 +99,8 @@ def analyze() -> Tuple[str, int]: allow_list_match=req_data.allow_list_match, regex_flags=req_data.regex_flags, n_process=min( - len(batch), - int(os.environ.get("N_PROCESS", DEFAULT_N_PROCESS)) - ) + len(batch), int(os.environ.get("N_PROCESS", DEFAULT_N_PROCESS)) + ), ) results = [] for recognizer_result_list in iterator: diff --git a/presidio-analyzer/presidio_analyzer/analyzer_engine.py b/presidio-analyzer/presidio_analyzer/analyzer_engine.py index b68d24d6ab..9780d3a0f5 100644 --- a/presidio-analyzer/presidio_analyzer/analyzer_engine.py +++ b/presidio-analyzer/presidio_analyzer/analyzer_engine.py @@ -25,6 +25,7 @@ REGEX_TIMEOUT_SECONDS = int(os.environ.get("REGEX_TIMEOUT_SECONDS", 60)) + class AnalyzerEngine: """ Entry point for Presidio Analyzer. diff --git a/presidio-analyzer/presidio_analyzer/chunkers/__init__.py b/presidio-analyzer/presidio_analyzer/chunkers/__init__.py index fd9c825ce9..c21d006247 100644 --- a/presidio-analyzer/presidio_analyzer/chunkers/__init__.py +++ b/presidio-analyzer/presidio_analyzer/chunkers/__init__.py @@ -12,4 +12,3 @@ "CharacterBasedTextChunker", "TextChunkerProvider", ] - diff --git a/presidio-analyzer/presidio_analyzer/chunkers/base_chunker.py b/presidio-analyzer/presidio_analyzer/chunkers/base_chunker.py index 5547b87881..ab90bacb3a 100644 --- a/presidio-analyzer/presidio_analyzer/chunkers/base_chunker.py +++ b/presidio-analyzer/presidio_analyzer/chunkers/base_chunker.py @@ -1,4 +1,5 @@ """Abstract base class for text chunking strategies.""" + from abc import ABC, abstractmethod from dataclasses import dataclass from typing import TYPE_CHECKING, Callable, List diff --git a/presidio-analyzer/presidio_analyzer/chunkers/character_based_text_chunker.py b/presidio-analyzer/presidio_analyzer/chunkers/character_based_text_chunker.py index 52a739089b..7b6e5a190d 100644 --- a/presidio-analyzer/presidio_analyzer/chunkers/character_based_text_chunker.py +++ b/presidio-analyzer/presidio_analyzer/chunkers/character_based_text_chunker.py @@ -3,6 +3,7 @@ Based on gliner-spacy implementation: https://github.com/theirstory/gliner-spacy/blob/main/gliner_spacy/pipeline.py#L60-L96 """ + import logging from typing import Iterable, List, Tuple diff --git a/presidio-analyzer/presidio_analyzer/chunkers/text_chunker_provider.py b/presidio-analyzer/presidio_analyzer/chunkers/text_chunker_provider.py index cb668265be..4ef016673e 100644 --- a/presidio-analyzer/presidio_analyzer/chunkers/text_chunker_provider.py +++ b/presidio-analyzer/presidio_analyzer/chunkers/text_chunker_provider.py @@ -57,4 +57,3 @@ def create_chunker(self) -> BaseTextChunker: raise ValueError( f"Invalid configuration for chunker_type '{chunker_type}': {config}" ) from exc - diff --git a/presidio-analyzer/presidio_analyzer/conf/default_analyzer_full.yaml b/presidio-analyzer/presidio_analyzer/conf/default_analyzer_full.yaml index a3603e3947..836d290e84 100644 --- a/presidio-analyzer/presidio_analyzer/conf/default_analyzer_full.yaml +++ b/presidio-analyzer/presidio_analyzer/conf/default_analyzer_full.yaml @@ -1,154 +1,463 @@ -supported_languages: +supported_languages: - en -default_score_threshold: 0 -nlp_configuration: - nlp_engine_name: spacy - models: - - lang_code: en - model_name: en_core_web_lg - - ner_model_configuration: - model_to_presidio_entity_mapping: - PER: PERSON - PERSON: PERSON - NORP: NRP - FAC: LOCATION - LOC: LOCATION - LOCATION: LOCATION - GPE: LOCATION - ORG: ORGANIZATION - ORGANIZATION: ORGANIZATION - DATE: DATE_TIME - TIME: DATE_TIME - - low_confidence_score_multiplier: 0.4 - low_score_entity_names: - - - labels_to_ignore: - - ORG - - ORGANIZATION # has many false positives - - CARDINAL - - EVENT - - LANGUAGE - - LAW - - MONEY - - ORDINAL - - PERCENT - - PRODUCT - - QUANTITY - - WORK_OF_ART - - -recognizer_registry: - # global_regex_flags: 26 - recognizers: - # Recognizers listed here can either be loaded from the recognizers defined in code (type: predefined), - # or created based on the provided configuration (type: custom). - # For predefined: - # - If only a recognizer name is provided, a predefined recognizer with this name and default parameters will be loaded. - # - If a parameter isn't provided, the default one would be loaded. - # For custom: - # - See an example configuration here: https://github.com/microsoft/presidio/blob/main/presidio-analyzer/presidio_analyzer/conf/example_recognizers.yaml - # - Custom pattern recognizers with this configuration can be added to this file, with type: custom - # For recognizers supporting more than one language, an instance of the recognizer for each language will be created. - # For example, see the CreditCardRecognizer definition below: - - name: CreditCardRecognizer - supported_languages: - - language: en - context: [credit, card, visa, mastercard, cc, amex, discover, jcb, diners, maestro, instapayment] - type: predefined - - - name: UsBankRecognizer - type: predefined - - - name: UsLicenseRecognizer - type: predefined - - - name: UsItinRecognizer - type: predefined - - - name: UsPassportRecognizer - type: predefined - - - name: UsSsnRecognizer - type: predefined - - - name: NhsRecognizer - type: predefined - - - name: UkNinoRecognizer - type: predefined - enabled: false - - - name: SgFinRecognizer - type: predefined - enabled: false - - - name: AuAbnRecognizer - type: predefined - enabled: false - - - name: AuAcnRecognizer - type: predefined - enabled: false - - - name: AuTfnRecognizer - type: predefined - enabled: false - - - name: AuMedicareRecognizer - type: predefined - enabled: false - - - name: InPanRecognizer - type: predefined - enabled: false - - - name: InAadhaarRecognizer - supported_languages: - - en - type: predefined - enabled: false +global_regex_flags: 26 + +recognizers: + - name: CreditCardRecognizer + supported_languages: + - language: en + context: [credit, card, visa, mastercard, cc, amex, discover, jcb, diners, maestro, instapayment] + - language: es + context: [tarjeta, credito, visa, mastercard, cc, amex, discover, jcb, diners, maestro, instapayment] + - language: it + - language: pl + type: predefined + + - name: UsBankRecognizer + supported_languages: + - en + type: predefined + country_code: us + + - name: UsLicenseRecognizer + supported_languages: + - en + type: predefined + country_code: us + + - name: UsItinRecognizer + supported_languages: + - en + type: predefined + country_code: us + + - name: UsPassportRecognizer + supported_languages: + - en + type: predefined + country_code: us + + - name: UsSsnRecognizer + supported_languages: + - en + type: predefined + country_code: us + + - name: TwNationalIdRecognizer + supported_languages: + - en + type: predefined + country_code: tw + + - name: UsMbiRecognizer + supported_languages: + - en + type: predefined + enabled: false + country_code: us + + - name: UsNpiRecognizer + supported_languages: + - en + type: predefined + enabled: false + country_code: us + + - name: NhsRecognizer + supported_languages: + - en + type: predefined + country_code: uk + + - name: UkDrivingLicenceRecognizer + supported_languages: + - en + type: predefined + enabled: false + country_code: uk + + - name: UkNinoRecognizer + supported_languages: + - en + type: predefined + enabled: false + country_code: uk + + - name: UkPassportRecognizer + supported_languages: + - en + type: predefined + enabled: false + country_code: uk + + - name: UkPostcodeRecognizer + supported_languages: + - en + type: predefined + enabled: false + country_code: uk + + - name: UkVehicleRegistrationRecognizer + supported_languages: + - en + type: predefined + enabled: false + country_code: uk + + - name: SgFinRecognizer + supported_languages: + - en + type: predefined + enabled: false + country_code: sg + + - name: AuAbnRecognizer + supported_languages: + - en + type: predefined + enabled: false + country_code: au + + - name: AuAcnRecognizer + supported_languages: + - en + type: predefined + enabled: false + country_code: au + + - name: AuTfnRecognizer + supported_languages: + - en + type: predefined + enabled: false + country_code: au + + - name: AuMedicareRecognizer + supported_languages: + - en + type: predefined + enabled: false + country_code: au + + - name: NgNinRecognizer + supported_languages: + - en + type: predefined + enabled: false + country_code: ng + + - name: NgVehicleRegistrationRecognizer + supported_languages: + - en + type: predefined + enabled: false + country_code: ng + + - name: InPanRecognizer + supported_languages: + - en + type: predefined + enabled: false + country_code: in + + - name: InAadhaarRecognizer + supported_languages: + - en + type: predefined + enabled: false + country_code: in + + - name: InVehicleRegistrationRecognizer + supported_languages: + - en + type: predefined + enabled: false + country_code: in + + - name: InPassportRecognizer + supported_languages: + - en + type: predefined + enabled: false + country_code: in + + - name: EsNifRecognizer + supported_languages: + - es + type: predefined + country_code: es + + - name: EsNieRecognizer + supported_languages: + - es + type: predefined + country_code: es + + - name: EsPassportRecognizer + supported_languages: + - es + type: predefined + enabled: false + country_code: es + + - name: ItDriverLicenseRecognizer + supported_languages: + - it + type: predefined + country_code: it - - name: InVehicleRegistrationRecognizer - type: predefined - enabled: false + - name: ItFiscalCodeRecognizer + supported_languages: + - it + type: predefined + country_code: it - - name: InPassportRecognizer - type: predefined - enabled: false + - name: ItVatCodeRecognizer + supported_languages: + - it + type: predefined + country_code: it - - name: CryptoRecognizer - type: predefined + - name: ItIdentityCardRecognizer + supported_languages: + - it + type: predefined + country_code: it - - name: DateRecognizer - type: predefined + - name: ItPassportRecognizer + supported_languages: + - it + type: predefined + country_code: it - - name: EmailRecognizer - type: predefined + - name: PlPeselRecognizer + supported_languages: + - pl + type: predefined + country_code: pl - - name: IbanRecognizer - type: predefined + - name: KrBrnRecognizer + supported_languages: + - ko + - kr + type: predefined + enabled: false + country_code: kr - - name: IpRecognizer - type: predefined + - name: KrRrnRecognizer + supported_languages: + - ko + - kr + type: predefined + enabled: false + country_code: kr - - name: MedicalLicenseRecognizer - type: predefined + - name: KrDriverLicenseRecognizer + supported_languages: + - ko + - kr + type: predefined + enabled: false + country_code: kr - - name: PhoneRecognizer - type: predefined + - name: KrFrnRecognizer + supported_languages: + - ko + - kr + type: predefined + enabled: false + country_code: kr - - name: UrlRecognizer - type: predefined + - name: SeOrganisationsnummerRecognizer + supported_languages: + - sv + type: predefined + enabled: false + country_code: se - - name: InVoterRecognizer - type: predefined - enabled: false + - name: SePersonnummerRecognizer + supported_languages: + - sv + type: predefined + enabled: false + country_code: se - - name: InGstinRecognizer - type: predefined - enabled: false + - name: ZaIdNumberRecognizer + supported_languages: + - en + type: predefined + enabled: false + country_code: za + + - name: ThTninRecognizer + supported_languages: + - th + type: predefined + enabled: false + country_code: th + + - name: TrNationalIdRecognizer + supported_languages: + - tr + type: predefined + enabled: false + country_code: tr + + - name: TrLicensePlateRecognizer + supported_languages: + - tr + type: predefined + enabled: false + country_code: tr + + - name: HuggingFaceNerRecognizer + supported_languages: + - en + type: predefined + enabled: false + + - name: CryptoRecognizer + type: predefined + + - name: DateRecognizer + type: predefined + + - name: EmailRecognizer + type: predefined + + - name: IbanRecognizer + type: predefined + + - name: IpRecognizer + type: predefined + + - name: MedicalLicenseRecognizer + type: predefined + country_code: us + + - name: MacAddressRecognizer + type: predefined + + - name: PhoneRecognizer + type: predefined + + - name: UrlRecognizer + type: predefined + + - name: InVoterRecognizer + type: predefined + enabled: false + country_code: in + + - name: InGstinRecognizer + supported_languages: + - en + type: predefined + enabled: false + country_code: in + + - name: DeTaxIdRecognizer + supported_languages: + - de + type: predefined + enabled: false + country_code: de + + - name: DeTaxNumberRecognizer + supported_languages: + - de + type: predefined + enabled: false + country_code: de + + - name: DePassportRecognizer + supported_languages: + - de + type: predefined + enabled: false + country_code: de + + - name: DeIdCardRecognizer + supported_languages: + - de + type: predefined + enabled: false + country_code: de + + - name: DeSocialSecurityRecognizer + supported_languages: + - de + type: predefined + enabled: false + country_code: de + + - name: DeHealthInsuranceRecognizer + supported_languages: + - de + type: predefined + enabled: false + country_code: de + + - name: DeKfzRecognizer + supported_languages: + - de + type: predefined + enabled: false + country_code: de + + - name: DeHandelsregisterRecognizer + supported_languages: + - de + type: predefined + enabled: false + country_code: de + + - name: DePlzRecognizer + supported_languages: + - de + type: predefined + enabled: false + country_code: de + + - name: DeLanrRecognizer + supported_languages: + - de + type: predefined + enabled: false + country_code: de + + - name: DeBsnrRecognizer + supported_languages: + - de + type: predefined + enabled: false + country_code: de + + - name: DeVatIdRecognizer + supported_languages: + - de + type: predefined + enabled: false + country_code: de + + - name: DeFuehrerscheinRecognizer + supported_languages: + - de + type: predefined + enabled: false + country_code: de + + - name: BasicLangExtractRecognizer + supported_languages: + - en + type: predefined + enabled: false + config_path: presidio_analyzer/conf/langextract_config_basic.yaml - - name: SpacyRecognizer - type: predefined + - name: CaSinRecognizer + supported_languages: + - en + - fr + type: predefined + enabled: false + country_code: ca diff --git a/presidio-analyzer/presidio_analyzer/conf/default_recognizers.yaml b/presidio-analyzer/presidio_analyzer/conf/default_recognizers.yaml index 17f09fd2e5..836d290e84 100644 --- a/presidio-analyzer/presidio_analyzer/conf/default_recognizers.yaml +++ b/presidio-analyzer/presidio_analyzer/conf/default_recognizers.yaml @@ -2,31 +2,7 @@ supported_languages: - en global_regex_flags: 26 -# To restrict the loaded predefined recognizers to a subset of countries -# (mirrors ``supported_languages``), uncomment ``supported_countries`` and -# list ISO-3166-1 alpha-2 codes (case-insensitive). Locale-agnostic -# recognizers (those without a ``country_code`` below) are always loaded -# regardless of this filter. -# -# supported_countries: -# - us -# - uk - recognizers: - # Recognizers listed here can either be loaded from the recognizers defined in code (type: predefined), - # or created based on the provided configuration (type: custom). - # For predefined: - # - If only a recognizer name is provided, a predefined recognizer with this name and default parameters will be loaded. - # - If a parameter isn't provided, the default one would be loaded. - # - Use 'class_name' to specify the Python class when using a custom 'name' for display/metadata - # - The optional ``country_code`` field (ISO-3166-1 alpha-2) makes the recognizer's country tag - # visible to no-code YAML readers. It must match the recognizer class's ``COUNTRY_CODE`` - # attribute; mismatches raise at load time. Locale-agnostic recognizers omit ``country_code``. - # For custom: - # - See an example configuration here: https://github.com/microsoft/presidio/blob/main/presidio-analyzer/presidio_analyzer/conf/example_recognizers.yaml - # - Custom pattern recognizers with this configuration can be added to this file, with type: custom - # For recognizers supporting more than one language, an instance of the recognizer for each language will be created. - # For example, see the CreditCardRecognizer definition below: - name: CreditCardRecognizer supported_languages: - language: en @@ -38,35 +14,41 @@ recognizers: type: predefined - name: UsBankRecognizer - supported_languages: + supported_languages: - en type: predefined country_code: us - name: UsLicenseRecognizer - supported_languages: + supported_languages: - en type: predefined country_code: us - name: UsItinRecognizer - supported_languages: + supported_languages: - en type: predefined country_code: us - + - name: UsPassportRecognizer - supported_languages: + supported_languages: - en type: predefined country_code: us - + - name: UsSsnRecognizer supported_languages: - en type: predefined country_code: us + - name: TwNationalIdRecognizer + supported_languages: + - en + type: predefined + country_code: tw + - name: UsMbiRecognizer supported_languages: - en @@ -82,13 +64,13 @@ recognizers: country_code: us - name: NhsRecognizer - supported_languages: + supported_languages: - en type: predefined country_code: uk - + - name: UkDrivingLicenceRecognizer - supported_languages: + supported_languages: - en type: predefined enabled: false @@ -108,13 +90,6 @@ recognizers: enabled: false country_code: uk - - name: UkPostcodeRecognizer - supported_languages: - - en - type: predefined - enabled: false - country_code: uk - - name: UkPostcodeRecognizer supported_languages: - en @@ -130,35 +105,35 @@ recognizers: country_code: uk - name: SgFinRecognizer - supported_languages: + supported_languages: - en type: predefined enabled: false country_code: sg - name: AuAbnRecognizer - supported_languages: + supported_languages: - en type: predefined enabled: false country_code: au - name: AuAcnRecognizer - supported_languages: + supported_languages: - en type: predefined enabled: false country_code: au - name: AuTfnRecognizer - supported_languages: + supported_languages: - en type: predefined enabled: false country_code: au - name: AuMedicareRecognizer - supported_languages: + supported_languages: - en type: predefined enabled: false @@ -179,41 +154,41 @@ recognizers: country_code: ng - name: InPanRecognizer - supported_languages: + supported_languages: - en type: predefined enabled: false country_code: in - name: InAadhaarRecognizer - supported_languages: + supported_languages: - en type: predefined enabled: false country_code: in - name: InVehicleRegistrationRecognizer - supported_languages: + supported_languages: - en type: predefined enabled: false country_code: in - name: InPassportRecognizer - supported_languages: + supported_languages: - en type: predefined enabled: false country_code: in - name: EsNifRecognizer - supported_languages: + supported_languages: - es type: predefined country_code: es - name: EsNieRecognizer - supported_languages: + supported_languages: - es type: predefined country_code: es @@ -224,45 +199,45 @@ recognizers: type: predefined enabled: false country_code: es - + - name: ItDriverLicenseRecognizer - supported_languages: + supported_languages: - it type: predefined country_code: it - name: ItFiscalCodeRecognizer - supported_languages: + supported_languages: - it type: predefined country_code: it - name: ItVatCodeRecognizer - supported_languages: + supported_languages: - it type: predefined country_code: it - name: ItIdentityCardRecognizer - supported_languages: + supported_languages: - it type: predefined country_code: it - name: ItPassportRecognizer - supported_languages: + supported_languages: - it type: predefined country_code: it - name: PlPeselRecognizer - supported_languages: + supported_languages: - pl type: predefined country_code: pl - name: KrBrnRecognizer - supported_languages: + supported_languages: - ko - kr type: predefined @@ -270,7 +245,7 @@ recognizers: country_code: kr - name: KrRrnRecognizer - supported_languages: + supported_languages: - ko - kr type: predefined @@ -278,7 +253,7 @@ recognizers: country_code: kr - name: KrDriverLicenseRecognizer - supported_languages: + supported_languages: - ko - kr type: predefined @@ -299,7 +274,7 @@ recognizers: type: predefined enabled: false country_code: se - + - name: SePersonnummerRecognizer supported_languages: - sv @@ -375,13 +350,12 @@ recognizers: country_code: in - name: InGstinRecognizer - supported_languages: + supported_languages: - en type: predefined enabled: false country_code: in - # Germany recognizers (disabled by default, enable for German-language pipelines) - name: DeTaxIdRecognizer supported_languages: - de diff --git a/presidio-analyzer/presidio_analyzer/conf/slim.yaml b/presidio-analyzer/presidio_analyzer/conf/slim.yaml index 1cb55be437..f139ed062b 100644 --- a/presidio-analyzer/presidio_analyzer/conf/slim.yaml +++ b/presidio-analyzer/presidio_analyzer/conf/slim.yaml @@ -1,120 +1,42 @@ +# Keep standard base configurations intact supported_languages: - en -default_score_threshold: 0 -nlp_configuration: - nlp_engine_name: slim - models: - - lang_code: en - model_name: en_core_web_sm - -recognizer_registry: - # global_regex_flags: 26 - recognizers: - # GLiNERRecognizer replaces SpacyRecognizer for NER-based entity detection. - # The slim NLP engine provides only tokenization and lemmatization; - # entity extraction is handled by self-contained recognizers like GLiNER. - - name: GLiNERRecognizer - type: predefined - enabled: true - - - name: CreditCardRecognizer - supported_languages: - - language: en - context: [credit, card, visa, mastercard, cc, amex, discover, jcb, diners, maestro, instapayment] - type: predefined - - - name: UsBankRecognizer - type: predefined - - - name: UsLicenseRecognizer - type: predefined - - - name: UsItinRecognizer - type: predefined - - - name: UsPassportRecognizer - type: predefined - - - name: UsSsnRecognizer - type: predefined - - - name: NhsRecognizer - type: predefined - - - name: UkNinoRecognizer - type: predefined - enabled: false - - - name: UkPassportRecognizer - type: predefined - enabled: false - - - name: SgFinRecognizer - type: predefined - enabled: false - - - name: AuAbnRecognizer - type: predefined - enabled: false - - - name: AuAcnRecognizer - type: predefined - enabled: false - - - name: AuTfnRecognizer - type: predefined - enabled: false - - - name: AuMedicareRecognizer - type: predefined - enabled: false - - - name: InPanRecognizer - type: predefined - enabled: false - - - name: InAadhaarRecognizer - supported_languages: - - en - type: predefined - enabled: false - - - name: InVehicleRegistrationRecognizer - type: predefined - enabled: false - - - name: InPassportRecognizer - type: predefined - enabled: false - - - name: CryptoRecognizer - type: predefined - - - name: DateRecognizer - type: predefined - - - name: EmailRecognizer - type: predefined - - - name: IbanRecognizer - type: predefined - - - name: IpRecognizer - type: predefined - - - name: MedicalLicenseRecognizer - type: predefined - - - name: PhoneRecognizer - type: predefined - - - name: UrlRecognizer - type: predefined - - - name: InVoterRecognizer - type: predefined - enabled: false - - - name: InGstinRecognizer - type: predefined - enabled: false +global_regex_flags: 26 + +recognizers: + - name: CreditCardRecognizer + type: predefined + - name: CryptoRecognizer + type: predefined + - name: DateRecognizer + type: predefined + - name: EmailRecognizer + type: predefined + - name: IbanRecognizer + type: predefined + - name: IpRecognizer + type: predefined + - name: MacAddressRecognizer + type: predefined + - name: PhoneRecognizer + type: predefined + - name: UrlRecognizer + type: predefined + - name: UsBankRecognizer + type: predefined + country_code: us + - name: UsLicenseRecognizer + type: predefined + country_code: us + - name: UsItinRecognizer + type: predefined + country_code: us + - name: UsPassportRecognizer + type: predefined + country_code: us + - name: UsSsnRecognizer + type: predefined + country_code: us + - name: TwNationalIdRecognizer + type: predefined + country_code: tw diff --git a/presidio-analyzer/presidio_analyzer/input_validation/yaml_recognizer_models.py b/presidio-analyzer/presidio_analyzer/input_validation/yaml_recognizer_models.py index 3b58101d92..c42261517b 100644 --- a/presidio-analyzer/presidio_analyzer/input_validation/yaml_recognizer_models.py +++ b/presidio-analyzer/presidio_analyzer/input_validation/yaml_recognizer_models.py @@ -233,9 +233,7 @@ class CustomRecognizerConfig(BaseRecognizerConfig): ) country_code: Optional[str] = Field( default=None, - description=( - "Optional ISO 3166-1 alpha-2 country tag for country filtering" - ), + description=("Optional ISO 3166-1 alpha-2 country tag for country filtering"), ) patterns: Optional[List[Dict[str, Any]]] = Field( default=None, description="List of patterns" diff --git a/presidio-analyzer/presidio_analyzer/llm_utils/__init__.py b/presidio-analyzer/presidio_analyzer/llm_utils/__init__.py index 6f148f2625..90ca0ade64 100644 --- a/presidio-analyzer/presidio_analyzer/llm_utils/__init__.py +++ b/presidio-analyzer/presidio_analyzer/llm_utils/__init__.py @@ -58,4 +58,3 @@ "load_prompt_file", "render_jinja_template", ] - diff --git a/presidio-analyzer/presidio_analyzer/llm_utils/azure_auth_helper.py b/presidio-analyzer/presidio_analyzer/llm_utils/azure_auth_helper.py index e226854e94..01bc914f84 100644 --- a/presidio-analyzer/presidio_analyzer/llm_utils/azure_auth_helper.py +++ b/presidio-analyzer/presidio_analyzer/llm_utils/azure_auth_helper.py @@ -15,6 +15,7 @@ ManagedIdentityCredential, WorkloadIdentityCredential, ) + AZURE_IDENTITY_AVAILABLE = True except ImportError: # pragma: no cover AZURE_IDENTITY_AVAILABLE = False @@ -53,7 +54,7 @@ def get_azure_credential() -> "TokenCredential": "Install it with: pip install azure-identity" ) - if os.getenv('ENV') == 'development': + if os.getenv("ENV") == "development": credential = DefaultAzureCredential() # CodeQL [SM05139] OK for dev logger.debug("Using DefaultAzureCredential (development mode)") return credential @@ -61,15 +62,14 @@ def get_azure_credential() -> "TokenCredential": credential = ChainedTokenCredential( EnvironmentCredential(), WorkloadIdentityCredential(), - ManagedIdentityCredential() + ManagedIdentityCredential(), ) logger.debug("Using ChainedTokenCredential (production mode)") return credential def get_bearer_token_provider_for_scope( - scope: str, - credential: Optional["TokenCredential"] = None + scope: str, credential: Optional["TokenCredential"] = None ): """ Get a bearer token provider function for a specific Azure scope. diff --git a/presidio-analyzer/presidio_analyzer/llm_utils/config_loader.py b/presidio-analyzer/presidio_analyzer/llm_utils/config_loader.py index 5bd46f08fa..a36e776444 100644 --- a/presidio-analyzer/presidio_analyzer/llm_utils/config_loader.py +++ b/presidio-analyzer/presidio_analyzer/llm_utils/config_loader.py @@ -1,4 +1,5 @@ """Configuration loading utilities for LLM recognizers.""" + import logging from pathlib import Path from typing import Dict, List, Union @@ -93,7 +94,7 @@ def get_model_config(config: Dict, provider_key: str) -> Dict: (provider_key,), (provider_key, "model"), (provider_key, "model", "model_id"), - ] + ], ) return config[provider_key]["model"] @@ -102,7 +103,7 @@ def get_model_config(config: Dict, provider_key: str) -> Dict: def validate_config_fields( config: Dict, required_fields: List[Union[str, tuple]], - config_name: str = "Configuration" + config_name: str = "Configuration", ) -> None: """Validate that required fields exist in configuration. @@ -119,7 +120,7 @@ def validate_config_fields( current = config for i, key in enumerate(field): if not isinstance(current, dict) or key not in current: - path = ".".join(field[:i+1]) + path = ".".join(field[: i + 1]) raise ValueError(f"{config_name} must contain '{path}'") current = current[key] if i == len(field) - 1 and not current: diff --git a/presidio-analyzer/presidio_analyzer/llm_utils/entity_mapper.py b/presidio-analyzer/presidio_analyzer/llm_utils/entity_mapper.py index b599073e81..c8a14478c1 100644 --- a/presidio-analyzer/presidio_analyzer/llm_utils/entity_mapper.py +++ b/presidio-analyzer/presidio_analyzer/llm_utils/entity_mapper.py @@ -1,4 +1,5 @@ """Entity mapping and filtering utilities for LLM recognizers.""" + import logging from typing import List, Set @@ -21,8 +22,7 @@ def filter_results_by_labels( - results: List[RecognizerResult], - labels_to_ignore: List[str] + results: List[RecognizerResult], labels_to_ignore: List[str] ) -> List[RecognizerResult]: """Filter out results with ignored entity labels. @@ -41,7 +41,9 @@ def filter_results_by_labels( if result.entity_type.lower() in labels_to_ignore_lower: logger.debug( "Entity %s at [%d:%d] is in labels_to_ignore, skipping", - result.entity_type, result.start, result.end + result.entity_type, + result.start, + result.end, ) continue @@ -51,8 +53,7 @@ def filter_results_by_labels( def filter_results_by_score( - results: List[RecognizerResult], - min_score: float + results: List[RecognizerResult], min_score: float ) -> List[RecognizerResult]: """Filter out results below minimum confidence score. @@ -66,8 +67,11 @@ def filter_results_by_score( if result.score < min_score: logger.debug( "Entity %s at [%d:%d] below min_score (%.2f < %.2f), skipping", - result.entity_type, result.start, result.end, - result.score, min_score + result.entity_type, + result.start, + result.end, + result.score, + min_score, ) continue @@ -77,8 +81,7 @@ def filter_results_by_score( def filter_results_by_entities( - results: List[RecognizerResult], - requested_entities: List[str] + results: List[RecognizerResult], requested_entities: List[str] ) -> List[RecognizerResult]: """Filter results to only include requested entity types. @@ -95,7 +98,10 @@ def filter_results_by_entities( if result.entity_type not in requested_entities: logger.debug( "Entity %s at [%d:%d] not in requested entities %s, skipping", - result.entity_type, result.start, result.end, requested_entities + result.entity_type, + result.start, + result.end, + requested_entities, ) continue @@ -105,7 +111,7 @@ def filter_results_by_entities( def validate_result_positions( - results: List[RecognizerResult] + results: List[RecognizerResult], ) -> List[RecognizerResult]: """Filter out results with invalid or missing start/end positions. @@ -129,7 +135,7 @@ def validate_result_positions( def consolidate_generic_entities( results: List[RecognizerResult], supported_entities: List[str], - generic_entities_logged: Set[str] + generic_entities_logged: Set[str], ) -> List[RecognizerResult]: """Consolidate unmapped entity types to GENERIC_PII_ENTITY. @@ -167,8 +173,7 @@ def consolidate_generic_entities( def skip_unmapped_entities( - results: List[RecognizerResult], - supported_entities: List[str] + results: List[RecognizerResult], supported_entities: List[str] ) -> List[RecognizerResult]: """Skip unmapped entities instead of consolidating them. @@ -195,8 +200,7 @@ def skip_unmapped_entities( def ensure_generic_entity_support( - supported_entities: List[str], - enable_generic_consolidation: bool + supported_entities: List[str], enable_generic_consolidation: bool ) -> List[str]: """Ensure GENERIC_PII_ENTITY is in supported entities list if consolidation enabled. diff --git a/presidio-analyzer/presidio_analyzer/llm_utils/examples_loader.py b/presidio-analyzer/presidio_analyzer/llm_utils/examples_loader.py index 72aa318c31..9411b9a049 100644 --- a/presidio-analyzer/presidio_analyzer/llm_utils/examples_loader.py +++ b/presidio-analyzer/presidio_analyzer/llm_utils/examples_loader.py @@ -1,4 +1,5 @@ """Examples loading utilities for LLM recognizers.""" + import logging from typing import Dict, List @@ -13,9 +14,7 @@ ] -def load_yaml_examples( - examples_file: str, conf_subdir: str = "conf" -) -> List[Dict]: +def load_yaml_examples(examples_file: str, conf_subdir: str = "conf") -> List[Dict]: """Load and validate examples from YAML configuration file. :param examples_file: Path to YAML file with examples (repo-root-relative). diff --git a/presidio-analyzer/presidio_analyzer/llm_utils/langextract_helper.py b/presidio-analyzer/presidio_analyzer/llm_utils/langextract_helper.py index 25a517f1eb..ef9448429c 100644 --- a/presidio-analyzer/presidio_analyzer/llm_utils/langextract_helper.py +++ b/presidio-analyzer/presidio_analyzer/llm_utils/langextract_helper.py @@ -1,4 +1,5 @@ """LangExtract helper utilities.""" + import logging from typing import Dict, List, Optional @@ -69,8 +70,7 @@ def extract_lm_config(config: Dict) -> Dict: def get_supported_entities( - lm_config: Dict, - langextract_config: Dict + lm_config: Dict, langextract_config: Dict ) -> Optional[List[str]]: """Get supported entities list, checking LM config first then LangExtract config. @@ -78,9 +78,8 @@ def get_supported_entities( :param langextract_config: LangExtract configuration dictionary. :return: List of supported entity types, or None if not specified. """ - return ( - lm_config.get("supported_entities") - or langextract_config.get("supported_entities") + return lm_config.get("supported_entities") or langextract_config.get( + "supported_entities" ) @@ -94,8 +93,7 @@ def create_reverse_entity_mapping(entity_mappings: Dict) -> Dict: def calculate_extraction_confidence( - extraction, - alignment_scores: Optional[Dict[str, float]] = None + extraction, alignment_scores: Optional[Dict[str, float]] = None ) -> float: """Calculate confidence score based on extraction alignment status. @@ -108,9 +106,7 @@ def calculate_extraction_confidence( if alignment_scores is None: alignment_scores = DEFAULT_ALIGNMENT_SCORES - if not hasattr(extraction, "alignment_status") or not ( - extraction.alignment_status - ): + if not hasattr(extraction, "alignment_status") or not (extraction.alignment_status): return default_score status = str(extraction.alignment_status).upper() @@ -127,7 +123,7 @@ def convert_langextract_to_presidio_results( supported_entities: List[str], enable_generic_consolidation: bool, recognizer_name: str, - alignment_scores: Optional[Dict[str, float]] = None + alignment_scores: Optional[Dict[str, float]] = None, ) -> List[RecognizerResult]: """Convert LangExtract extraction results to Presidio RecognizerResult objects. @@ -177,10 +173,10 @@ def convert_langextract_to_presidio_results( confidence = calculate_extraction_confidence(extraction, alignment_scores) metadata = {} - if hasattr(extraction, 'attributes') and extraction.attributes: - metadata['attributes'] = extraction.attributes - if hasattr(extraction, 'alignment_status') and extraction.alignment_status: - metadata['alignment'] = str(extraction.alignment_status) + if hasattr(extraction, "attributes") and extraction.attributes: + metadata["attributes"] = extraction.attributes + if hasattr(extraction, "alignment_status") and extraction.alignment_status: + metadata["alignment"] = str(extraction.alignment_status) explanation = AnalysisExplanation( recognizer=recognizer_name, @@ -200,7 +196,7 @@ def convert_langextract_to_presidio_results( end=extraction.char_interval.end_pos, score=confidence, analysis_explanation=explanation, - recognition_metadata=metadata if metadata else None + recognition_metadata=metadata if metadata else None, ) results.append(result) diff --git a/presidio-analyzer/presidio_analyzer/llm_utils/prompt_loader.py b/presidio-analyzer/presidio_analyzer/llm_utils/prompt_loader.py index 83fb3e0741..22d4b89de6 100644 --- a/presidio-analyzer/presidio_analyzer/llm_utils/prompt_loader.py +++ b/presidio-analyzer/presidio_analyzer/llm_utils/prompt_loader.py @@ -1,4 +1,5 @@ """Prompt loading and rendering utilities for LLM recognizers.""" + import logging from .config_loader import resolve_config_path diff --git a/presidio-analyzer/presidio_analyzer/lm_recognizer.py b/presidio-analyzer/presidio_analyzer/lm_recognizer.py index b4a4a08680..8e840374b1 100644 --- a/presidio-analyzer/presidio_analyzer/lm_recognizer.py +++ b/presidio-analyzer/presidio_analyzer/lm_recognizer.py @@ -72,10 +72,7 @@ def __init__( @abstractmethod def _call_llm( - self, - text: str, - entities: List[str], - **kwargs + self, text: str, entities: List[str], **kwargs ) -> List[RecognizerResult]: """ Call LLM service and return RecognizerResult objects. @@ -92,7 +89,7 @@ def analyze( self, text: str, entities: Optional[List[str]] = None, - nlp_artifacts: Optional["NlpArtifacts"] = None + nlp_artifacts: Optional["NlpArtifacts"] = None, ) -> List[RecognizerResult]: """Analyze text for PII/PHI using LLM.""" if not text or not text.strip(): @@ -107,15 +104,14 @@ def analyze( if not requested_entities: logger.debug( "No requested entities (%s) match supported entities (%s)", - entities, self.supported_entities + entities, + self.supported_entities, ) return [] results = self._call_llm(text, requested_entities) - filtered_results = self._filter_and_process_results( - results, requested_entities - ) + filtered_results = self._filter_and_process_results(results, requested_entities) if filtered_results: logger.debug( @@ -128,27 +124,23 @@ def analyze( def _filter_and_process_results( self, results: List[RecognizerResult], - requested_entities: Optional[List[str]] = None + requested_entities: Optional[List[str]] = None, ) -> List[RecognizerResult]: """Filter and process results.""" filtered_results = filter_results_by_labels(results, self.labels_to_ignore) if self.enable_generic_consolidation: filtered_results = consolidate_generic_entities( - filtered_results, - self.supported_entities, - self._generic_entities_logged + filtered_results, self.supported_entities, self._generic_entities_logged ) else: filtered_results = skip_unmapped_entities( - filtered_results, - self.supported_entities + filtered_results, self.supported_entities ) if requested_entities: filtered_results = filter_results_by_entities( - filtered_results, - requested_entities + filtered_results, requested_entities ) filtered_results = validate_result_positions(filtered_results) diff --git a/presidio-analyzer/presidio_analyzer/nlp_engine/stanza_nlp_engine.py b/presidio-analyzer/presidio_analyzer/nlp_engine/stanza_nlp_engine.py index 7a4f306c35..88b8ae3663 100644 --- a/presidio-analyzer/presidio_analyzer/nlp_engine/stanza_nlp_engine.py +++ b/presidio-analyzer/presidio_analyzer/nlp_engine/stanza_nlp_engine.py @@ -69,9 +69,9 @@ def load(self) -> None: self.nlp[model["lang_code"]] = load_pipeline( model["model_name"], processors="tokenize,pos,lemma,ner", - download_method="DOWNLOAD_RESOURCES" - if self.download_if_missing - else None, + download_method=( + "DOWNLOAD_RESOURCES" if self.download_if_missing else None + ), device=self.device, ) diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/__init__.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/__init__.py index a6dc79eb26..32a8b05eac 100644 --- a/presidio-analyzer/presidio_analyzer/predefined_recognizers/__init__.py +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/__init__.py @@ -130,6 +130,7 @@ from .country_specific.us.us_mbi_recognizer import UsMbiRecognizer from .country_specific.us.us_npi_recognizer import UsNpiRecognizer from .country_specific.us.us_passport_recognizer import UsPassportRecognizer +from .country_specific.tw.tw_national_id_recognizer import TwNationalIdRecognizer from .country_specific.us.us_ssn_recognizer import UsSsnRecognizer # Generic recognizers @@ -200,6 +201,7 @@ "UsMbiRecognizer", "UsNpiRecognizer", "UsPassportRecognizer", + "TwNationalIdRecognizer", "UsSsnRecognizer", "EsNifRecognizer", "SpacyRecognizer", diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/germany/de_bsnr_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/germany/de_bsnr_recognizer.py index 42b5358e38..e78cfbdf61 100644 --- a/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/germany/de_bsnr_recognizer.py +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/germany/de_bsnr_recognizer.py @@ -53,27 +53,29 @@ class DeBsnrRecognizer(PatternRecognizer): # Valid KV regional codes per KBV Arztnummern-Richtlinie Anlage 1. # Includes the standard 17 KV regions, the KBV itself, and Anlage-8 BMV-Ä # special codes for Krankenhäuser. - VALID_KV_CODES = frozenset({ - "01", # Schleswig-Holstein - "02", # Hamburg - "03", # Bremen - "17", # Niedersachsen - "20", # Westfalen-Lippe - "35", # Krankenhäuser (Anlage 8 BMV-Ä) - "38", # Nordrhein - "46", # Hessen - "51", # Rheinland-Pfalz - "52", # Baden-Württemberg - "71", # Bayern - "72", # Berlin - "73", # Saarland - "74", # KBV (Kassenärztliche Bundesvereinigung) - "78", # Mecklenburg-Vorpommern - "83", # Brandenburg - "88", # Sachsen-Anhalt - "93", # Thüringen - "98", # Sachsen - }) + VALID_KV_CODES = frozenset( + { + "01", # Schleswig-Holstein + "02", # Hamburg + "03", # Bremen + "17", # Niedersachsen + "20", # Westfalen-Lippe + "35", # Krankenhäuser (Anlage 8 BMV-Ä) + "38", # Nordrhein + "46", # Hessen + "51", # Rheinland-Pfalz + "52", # Baden-Württemberg + "71", # Bayern + "72", # Berlin + "73", # Saarland + "74", # KBV (Kassenärztliche Bundesvereinigung) + "78", # Mecklenburg-Vorpommern + "83", # Brandenburg + "88", # Sachsen-Anhalt + "93", # Thüringen + "98", # Sachsen + } + ) PATTERNS = [ Pattern( diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/germany/de_social_security_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/germany/de_social_security_recognizer.py index 0e01dc8e43..d66624ccc6 100644 --- a/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/germany/de_social_security_recognizer.py +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/germany/de_social_security_recognizer.py @@ -63,11 +63,11 @@ class DeSocialSecurityRecognizer(PatternRecognizer): "Rentenversicherungsnummer (Strict, with birth date structure)", r"\b\d{2}" r"(0[1-9]|[12]\d|3[01]|5[1-9]|[67]\d|8[01])" # day: 01-31 or 51-81 - r"(0[1-9]|1[0-2])" # month 01-12 - r"\d{2}" # year - r"[A-Z]" # surname initial - r"\d{2}" # serial - r"[0-9]\b", # check digit + r"(0[1-9]|1[0-2])" # month 01-12 + r"\d{2}" # year + r"[A-Z]" # surname initial + r"\d{2}" # serial + r"[0-9]\b", # check digit 0.5, ), Pattern( diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/tw/__init__.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/tw/__init__.py new file mode 100644 index 0000000000..d38bad3350 --- /dev/null +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/tw/__init__.py @@ -0,0 +1,5 @@ +from .tw_national_id_recognizer import TwNationalIdRecognizer + +__all__ = [ + "TwNationalIdRecognizer", +] diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/tw/tw_national_id_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/tw/tw_national_id_recognizer.py new file mode 100644 index 0000000000..3401c020ef --- /dev/null +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/tw/tw_national_id_recognizer.py @@ -0,0 +1,113 @@ +from typing import List, Optional +from presidio_analyzer.pattern import Pattern +from presidio_analyzer.pattern_recognizer import PatternRecognizer + + +class TwNationalIdRecognizer(PatternRecognizer): + """ + Recognizes Taiwan National Identification Number (National ID / Resident Certificate). + """ + + PATTERNS = [ + Pattern( + "National ID (weak)", + r"(? bool: + """ + Validates Taiwan National ID / Resident UI No. using the Modulus-10 checksum algorithm. + """ + if not pattern_text or len(pattern_text) != 10: + return True + + # Presidio ignores case by default; strictly reject lowercase starting letters + if not pattern_text[0].isupper(): + return True + + # Letter to numerical weight conversion mapping + letter_mapping = { + "A": 10, + "B": 11, + "C": 12, + "D": 13, + "E": 14, + "F": 15, + "G": 16, + "H": 17, + "J": 18, + "K": 19, + "L": 20, + "M": 21, + "N": 22, + "P": 23, + "Q": 24, + "R": 25, + "S": 26, + "T": 27, + "U": 28, + "V": 29, + "X": 30, + "Y": 31, + "W": 32, + "Z": 33, + "I": 34, + "O": 35, + } + + first_char = pattern_text[0] + if first_char not in letter_mapping: + return True + + letter_code = letter_mapping[first_char] + + # Calculate base weight from the alphabet letter code conversion split + checksum = (letter_code // 10) + (letter_code % 10) * 9 + + # Multiply the remaining digits by weights [8, 7, 6, 5, 4, 3, 2, 1] + weights = [8, 7, 6, 5, 4, 3, 2, 1] + for i in range(8): + checksum += int(pattern_text[i + 1]) * weights[i] + + # Add the final check digit (index 9) + checksum += int(pattern_text[9]) + + # A valid structural national ID must be perfectly divisible by 10 + return checksum % 10 != 0 diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/iban_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/iban_recognizer.py index ec831114b3..f0b4254681 100644 --- a/presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/iban_recognizer.py +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/iban_recognizer.py @@ -22,6 +22,7 @@ REGEX_TIMEOUT_SECONDS = int(os.environ.get("REGEX_TIMEOUT_SECONDS", 60)) + class IbanRecognizer(PatternRecognizer): """ Recognize IBAN code using regex and checksum. diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/mac_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/mac_recognizer.py index de8e422740..334525719e 100644 --- a/presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/mac_recognizer.py +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/mac_recognizer.py @@ -48,7 +48,7 @@ def __init__( patterns=patterns, context=context, supported_language=supported_language, - name=name + name=name, ) def invalidate_result(self, pattern_text: str) -> bool: @@ -59,7 +59,7 @@ def invalidate_result(self, pattern_text: str) -> bool: :return: True if invalidated (invalid MAC address) """ # Remove separators and validate hex characters and length - cleaned = re.sub(r'[:\-.]', '', pattern_text) + cleaned = re.sub(r"[:\-.]", "", pattern_text) # All characters must be valid hex if re.fullmatch(r"[0-9A-Fa-f]{12}", cleaned) is None: @@ -67,7 +67,7 @@ def invalidate_result(self, pattern_text: str) -> bool: # Optionally reject broadcast/multicast addresses if needed # Broadcast: FF:FF:FF:FF:FF:FF - if cleaned.upper() == 'FFFFFFFFFFFF' or cleaned.upper() == '000000000000': + if cleaned.upper() == "FFFFFFFFFFFF" or cleaned.upper() == "000000000000": return True return False diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/ner/gliner_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/ner/gliner_recognizer.py index 4e12f7e634..009444d1b8 100644 --- a/presidio-analyzer/presidio_analyzer/predefined_recognizers/ner/gliner_recognizer.py +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/ner/gliner_recognizer.py @@ -106,9 +106,7 @@ def __init__( self.model_name = model_name self.map_location = ( - map_location - if map_location is not None - else device_detector.get_device() + map_location if map_location is not None else device_detector.get_device() ) self.flat_ner = flat_ner diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/third_party/ahds_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/third_party/ahds_recognizer.py index 88477610aa..0e30266464 100644 --- a/presidio-analyzer/presidio_analyzer/predefined_recognizers/third_party/ahds_recognizer.py +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/third_party/ahds_recognizer.py @@ -18,6 +18,7 @@ from presidio_analyzer.llm_utils.azure_auth_helper import ( get_azure_credential, ) + AZURE_AUTH_AVAILABLE = True except ImportError: # pragma: no cover get_azure_credential = None @@ -51,7 +52,6 @@ def __init__( version="1.0.0", ) - endpoint = os.getenv("AHDS_ENDPOINT", None) if client is None: @@ -111,8 +111,7 @@ def analyze( entities = self.supported_entities body = DeidentificationContent( - input_text=text, - operation_type=DeidentificationOperationType.TAG + input_text=text, operation_type=DeidentificationOperationType.TAG ) result = self.deid_client.deidentify_text(body) @@ -142,8 +141,8 @@ def _build_explanation(entity_type: str) -> AnalysisExplanation: recognizer=AzureHealthDeidRecognizer.__class__.__name__, original_score=1.0, textual_explanation=( - f"Identified as {entity_type} by Azure Health Data Services " - "Deidentification" + f"Identified as {entity_type} by Azure Health Data Services " + "Deidentification" ), ) return explanation diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/third_party/azure_openai_langextract_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/third_party/azure_openai_langextract_recognizer.py index 223b8172da..1e0de80835 100644 --- a/presidio-analyzer/presidio_analyzer/predefined_recognizers/third_party/azure_openai_langextract_recognizer.py +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/third_party/azure_openai_langextract_recognizer.py @@ -7,6 +7,7 @@ try: import langextract as lx + LANGEXTRACT_AVAILABLE = True except ImportError: # pragma: no cover LANGEXTRACT_AVAILABLE = False diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/third_party/azure_openai_provider.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/third_party/azure_openai_provider.py index 9f9bd30bc8..de0f3e342f 100644 --- a/presidio-analyzer/presidio_analyzer/predefined_recognizers/third_party/azure_openai_provider.py +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/third_party/azure_openai_provider.py @@ -8,6 +8,7 @@ import langextract as lx import openai from langextract.providers.openai import OpenAILanguageModel + LANGEXTRACT_OPENAI_AVAILABLE = True except ImportError: # pragma: no cover LANGEXTRACT_OPENAI_AVAILABLE = False @@ -20,6 +21,7 @@ get_azure_credential, get_bearer_token_provider_for_scope, ) + AZURE_IDENTITY_AVAILABLE = True except ImportError: # pragma: no cover AZURE_IDENTITY_AVAILABLE = False @@ -30,6 +32,7 @@ if LANGEXTRACT_OPENAI_AVAILABLE: + class AzureOpenAILanguageModel(OpenAILanguageModel): """ Custom LangExtract provider for Azure OpenAI. @@ -50,7 +53,7 @@ def __init__( api_version: Optional[str] = None, azure_deployment: Optional[str] = None, azure_ad_token_provider: Optional[any] = None, - **kwargs + **kwargs, ): """ Initialize Azure OpenAI provider. @@ -77,9 +80,9 @@ def __init__( self.model_id = clean_model_id self.api_key = api_key or os.environ.get("AZURE_OPENAI_API_KEY") self.organization = None - self.format_type = kwargs.get('format_type', lx.data.FormatType.JSON) - self.temperature = kwargs.get('temperature', 0.0) - self.max_workers = kwargs.get('max_workers', 10) + self.format_type = kwargs.get("format_type", lx.data.FormatType.JSON) + self.temperature = kwargs.get("temperature", 0.0) + self.max_workers = kwargs.get("max_workers", 10) self._extra_kwargs = kwargs # Azure-specific configuration @@ -92,13 +95,10 @@ def __init__( self.azure_deployment = azure_deployment or clean_model_id # Validate and initialize Azure OpenAI client - self._client = self._create_azure_openai_client( - azure_ad_token_provider - ) + self._client = self._create_azure_openai_client(azure_ad_token_provider) def _create_azure_openai_client( - self, - azure_ad_token_provider: Optional[any] = None + self, azure_ad_token_provider: Optional[any] = None ): """ Create and configure Azure OpenAI client with appropriate authentication. @@ -130,14 +130,14 @@ def _create_azure_openai_client( ) credential_type = ( "DefaultAzureCredential (development)" - if os.getenv('ENV') == 'development' + if os.getenv("ENV") == "development" else "ChainedTokenCredential" ) client = openai.AzureOpenAI( azure_ad_token_provider=token_provider, azure_endpoint=self.azure_endpoint, - api_version=self.api_version + api_version=self.api_version, ) logger.debug( @@ -148,7 +148,7 @@ def _create_azure_openai_client( client = openai.AzureOpenAI( api_key=self.api_key, azure_endpoint=self.azure_endpoint, - api_version=self.api_version + api_version=self.api_version, ) logger.debug( @@ -165,7 +165,9 @@ def _get_client_model_id(self) -> str: :return: Azure deployment name. """ return self.azure_deployment + else: # pragma: no cover + class AzureOpenAILanguageModel: """Placeholder when langextract is not installed.""" @@ -180,10 +182,8 @@ def __init__(self, *args, **kwargs): # Register the provider with LangExtract if LANGEXTRACT_OPENAI_AVAILABLE: try: - @lx.providers.registry.register( - r'^azure:', - priority=20 - ) + + @lx.providers.registry.register(r"^azure:", priority=20) class RegisteredAzureOpenAILanguageModel(AzureOpenAILanguageModel): """ Registered version of Azure OpenAI provider for LangExtract. diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/third_party/basic_langextract_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/third_party/basic_langextract_recognizer.py index 756214be8e..a2ad13f00c 100644 --- a/presidio-analyzer/presidio_analyzer/predefined_recognizers/third_party/basic_langextract_recognizer.py +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/third_party/basic_langextract_recognizer.py @@ -4,8 +4,9 @@ from typing import Any, Dict, Optional from presidio_analyzer.llm_utils import lx_factory -from presidio_analyzer.predefined_recognizers.third_party.\ - langextract_recognizer import LangExtractRecognizer +from presidio_analyzer.predefined_recognizers.third_party.langextract_recognizer import ( + LangExtractRecognizer, +) logger = logging.getLogger("presidio-analyzer") @@ -15,10 +16,8 @@ "fence_output": False, } -DEFAULT_LANGUAGE_MODEL_PARAMS = { - "timeout": 240, - "num_ctx": 8192 -} +DEFAULT_LANGUAGE_MODEL_PARAMS = {"timeout": 240, "num_ctx": 8192} + class BasicLangExtractRecognizer(LangExtractRecognizer): """Basic LangExtract recognizer using configurable backend.""" @@ -53,8 +52,8 @@ def __init__( supported_language=supported_language, extract_params={ "extract": DEFAULT_EXTRACT_PARAMS, - "language_model": DEFAULT_LANGUAGE_MODEL_PARAMS - } + "language_model": DEFAULT_LANGUAGE_MODEL_PARAMS, + }, ) model_config: Dict[str, Any] = self.config.get("model", {}) @@ -71,11 +70,14 @@ def __init__( ) if not self.provider: - raise ValueError("Configuration must contain " - "'langextract.model.provider.name'") - - if ("api_key" not in self.provider_kwargs - and "LANGEXTRACT_API_KEY" in os.environ): + raise ValueError( + "Configuration must contain " "'langextract.model.provider.name'" + ) + + if ( + "api_key" not in self.provider_kwargs + and "LANGEXTRACT_API_KEY" in os.environ + ): self.provider_kwargs["api_key"] = os.environ["LANGEXTRACT_API_KEY"] self.lx_model_config = lx_factory.ModelConfig( diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/third_party/langextract_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/third_party/langextract_recognizer.py index cfaa08c1f4..220abd2e40 100644 --- a/presidio-analyzer/presidio_analyzer/predefined_recognizers/third_party/langextract_recognizer.py +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/third_party/langextract_recognizer.py @@ -67,13 +67,11 @@ def __init__( ("langextract", "entity_mappings"), ("langextract", "prompt_file"), ("langextract", "examples_file"), - ] + ], ) self.config = langextract_config - model_config = get_model_config( - full_config, provider_key="langextract" - ) + model_config = get_model_config(full_config, provider_key="langextract") super().__init__( supported_entities=supported_entities, @@ -84,19 +82,13 @@ def __init__( temperature=model_config.get("temperature"), min_score=lm_config.get("min_score"), labels_to_ignore=lm_config.get("labels_to_ignore"), - enable_generic_consolidation=lm_config.get( - "enable_generic_consolidation" - ), + enable_generic_consolidation=lm_config.get("enable_generic_consolidation"), ) - examples_data = load_yaml_examples( - langextract_config["examples_file"] - ) + examples_data = load_yaml_examples(langextract_config["examples_file"]) self.examples = convert_to_langextract_format(examples_data) - prompt_template = load_prompt_file( - langextract_config["prompt_file"] - ) + prompt_template = load_prompt_file(langextract_config["prompt_file"]) self.prompt_description = render_jinja_template( prompt_template, supported_entities=self.supported_entities, @@ -120,11 +112,11 @@ def __init__( ) if "language_model" in extract_params: - for param_name, default_value in ( - extract_params["language_model"].items() - ): - self._language_model_params[param_name] = ( - self._model_config.get(param_name, default_value) + for param_name, default_value in extract_params[ + "language_model" + ].items(): + self._language_model_params[param_name] = self._model_config.get( + param_name, default_value ) def _call_llm(self, text: str, entities: List[str], **kwargs): @@ -151,7 +143,7 @@ def _call_llm(self, text: str, entities: List[str], **kwargs): entity_mappings=self.entity_mappings, supported_entities=self.supported_entities, enable_generic_consolidation=self.enable_generic_consolidation, - recognizer_name=self.__class__.__name__ + recognizer_name=self.__class__.__name__, ) def _call_langextract(self, **kwargs): @@ -172,8 +164,7 @@ def _call_langextract(self, **kwargs): return lx.extract(**extract_params) except Exception: logger.exception( - "LangExtract extraction failed (model '%s')", - self.model_id + "LangExtract extraction failed (model '%s')", self.model_id ) raise diff --git a/presidio-analyzer/tests/__init__.py b/presidio-analyzer/tests/__init__.py index 26fe605e40..aeec75f55d 100644 --- a/presidio-analyzer/tests/__init__.py +++ b/presidio-analyzer/tests/__init__.py @@ -1 +1,5 @@ -from .assertions import assert_result, assert_result_within_score_range, assert_result_with_textual_explanation # noqa: E501 +from .assertions import ( + assert_result, + assert_result_within_score_range, + assert_result_with_textual_explanation, +) # noqa: E501 diff --git a/presidio-analyzer/tests/assertions.py b/presidio-analyzer/tests/assertions.py index 7194a6cdce..11ad72c4ae 100644 --- a/presidio-analyzer/tests/assertions.py +++ b/presidio-analyzer/tests/assertions.py @@ -35,9 +35,16 @@ def assert_result_within_score_range( def assert_result_with_textual_explanation( - result, expected_entity_type, expected_start, expected_end, expected_score, expected_textual_explanation + result, + expected_entity_type, + expected_start, + expected_end, + expected_score, + expected_textual_explanation, ): assert_result( result, expected_entity_type, expected_start, expected_end, expected_score ) - assert result.analysis_explanation.textual_explanation == expected_textual_explanation \ No newline at end of file + assert ( + result.analysis_explanation.textual_explanation == expected_textual_explanation + ) diff --git a/presidio-analyzer/tests/conftest.py b/presidio-analyzer/tests/conftest.py index 890d603417..26ce9a3c11 100644 --- a/presidio-analyzer/tests/conftest.py +++ b/presidio-analyzer/tests/conftest.py @@ -12,7 +12,10 @@ ) from presidio_analyzer import RecognizerRegistry from presidio_analyzer.nlp_engine import NlpEngineProvider, NlpEngine -from presidio_analyzer.predefined_recognizers import NLP_RECOGNIZERS, PREDEFINED_RECOGNIZERS +from presidio_analyzer.predefined_recognizers import ( + NLP_RECOGNIZERS, + PREDEFINED_RECOGNIZERS, +) from tests.mocks import RecognizerRegistryMock, NlpEngineMock diff --git a/presidio-analyzer/tests/test_ahds_recognizer.py b/presidio-analyzer/tests/test_ahds_recognizer.py index 231d380680..1442961e74 100644 --- a/presidio-analyzer/tests/test_ahds_recognizer.py +++ b/presidio-analyzer/tests/test_ahds_recognizer.py @@ -21,16 +21,14 @@ def import_modules(): from azure.identity import DefaultAzureCredential - - def requires_env_vars(): dotenv.load_dotenv() endpoint = os.getenv("AHDS_ENDPOINT", "") return pytest.mark.skipif( - endpoint == "", - reason="AHDS_ENDPOINT environment variable not set" + endpoint == "", reason="AHDS_ENDPOINT environment variable not set" ) + @requires_env_vars() def test_get_supported_entities(import_modules): recognizer = AzureHealthDeidRecognizer() @@ -80,7 +78,9 @@ def test_mocked_entities_match_recognizer_results(): try: importlib.import_module("azure.health.deidentification") except ImportError: - pytest.skip("Skipping test because 'azure.health.deidentification' is not installed") + pytest.skip( + "Skipping test because 'azure.health.deidentification' is not installed" + ) # Mocking the DeidentificationClient and its analyze method from presidio_analyzer.predefined_recognizers import AzureHealthDeidRecognizer @@ -94,7 +94,7 @@ def __init__(self, entity_type, start, end, score): mock_entities = [ MockResult(entity_type="PATIENT", start=14, end=17, score=1.0), - MockResult(entity_type="EMAIL", start=29, end=41, score=1.0) + MockResult(entity_type="EMAIL", start=29, end=41, score=1.0), ] mock_client = MagicMock() diff --git a/presidio-analyzer/tests/test_ahds_recognizer_credential_selection.py b/presidio-analyzer/tests/test_ahds_recognizer_credential_selection.py index 9d120f45ae..0d6d100b91 100644 --- a/presidio-analyzer/tests/test_ahds_recognizer_credential_selection.py +++ b/presidio-analyzer/tests/test_ahds_recognizer_credential_selection.py @@ -4,120 +4,169 @@ from unittest.mock import Mock, patch, MagicMock import pytest -from presidio_analyzer.predefined_recognizers.third_party.ahds_recognizer import AzureHealthDeidRecognizer +from presidio_analyzer.predefined_recognizers.third_party.ahds_recognizer import ( + AzureHealthDeidRecognizer, +) @pytest.fixture def mock_azure_modules(): """Mock Azure modules to avoid import dependencies.""" - with patch.dict('sys.modules', { - 'azure.health.deidentification': MagicMock(), - 'azure.identity': MagicMock(), - }): + with patch.dict( + "sys.modules", + { + "azure.health.deidentification": MagicMock(), + "azure.identity": MagicMock(), + }, + ): # Mock the classes and enums we need mock_deid_client = MagicMock() mock_get_credential = MagicMock() - + # Mock PhiCategory enum for _get_supported_entities mock_phi_category = MagicMock() - mock_phi_category.__iter__ = lambda self: iter(['PATIENT', 'DOCTOR', 'DATE']) - - with patch('presidio_analyzer.predefined_recognizers.third_party.ahds_recognizer.DeidentificationClient', mock_deid_client), \ - patch('presidio_analyzer.predefined_recognizers.third_party.ahds_recognizer.get_azure_credential', mock_get_credential), \ - patch('presidio_analyzer.predefined_recognizers.third_party.ahds_recognizer.PhiCategory', mock_phi_category): + mock_phi_category.__iter__ = lambda self: iter(["PATIENT", "DOCTOR", "DATE"]) + + with patch( + "presidio_analyzer.predefined_recognizers.third_party.ahds_recognizer.DeidentificationClient", + mock_deid_client, + ), patch( + "presidio_analyzer.predefined_recognizers.third_party.ahds_recognizer.get_azure_credential", + mock_get_credential, + ), patch( + "presidio_analyzer.predefined_recognizers.third_party.ahds_recognizer.PhiCategory", + mock_phi_category, + ): yield { - 'DeidentificationClient': mock_deid_client, - 'get_azure_credential': mock_get_credential, - 'PhiCategory': mock_phi_category, + "DeidentificationClient": mock_deid_client, + "get_azure_credential": mock_get_credential, + "PhiCategory": mock_phi_category, } class TestAHDSRecognizerCredentialSelection: """Test credential selection based on environment variables.""" - def test_uses_default_credential_in_development_environment(self, mock_azure_modules): + def test_uses_default_credential_in_development_environment( + self, mock_azure_modules + ): """Test that get_azure_credential is used when ENV=development.""" - with patch.dict(os.environ, {'ENV': 'development', 'AHDS_ENDPOINT': 'https://test.endpoint.com'}): + with patch.dict( + os.environ, + {"ENV": "development", "AHDS_ENDPOINT": "https://test.endpoint.com"}, + ): mock_client_instance = MagicMock() - mock_azure_modules['DeidentificationClient'].return_value = mock_client_instance + mock_azure_modules["DeidentificationClient"].return_value = ( + mock_client_instance + ) mock_credential = MagicMock() - mock_azure_modules['get_azure_credential'].return_value = mock_credential - + mock_azure_modules["get_azure_credential"].return_value = mock_credential + recognizer = AzureHealthDeidRecognizer() - + # Verify get_azure_credential was called - mock_azure_modules['get_azure_credential'].assert_called_once() - + mock_azure_modules["get_azure_credential"].assert_called_once() + # Verify DeidentificationClient was initialized with credential from get_azure_credential - mock_azure_modules['DeidentificationClient'].assert_called_once() - call_args = mock_azure_modules['DeidentificationClient'].call_args + mock_azure_modules["DeidentificationClient"].assert_called_once() + call_args = mock_azure_modules["DeidentificationClient"].call_args assert call_args[0][1] == mock_credential - def test_uses_chained_credential_in_production_environment(self, mock_azure_modules): + def test_uses_chained_credential_in_production_environment( + self, mock_azure_modules + ): """Test that get_azure_credential is used when ENV=production.""" - with patch.dict(os.environ, {'ENV': 'production', 'AHDS_ENDPOINT': 'https://test.endpoint.com'}): + with patch.dict( + os.environ, + {"ENV": "production", "AHDS_ENDPOINT": "https://test.endpoint.com"}, + ): mock_client_instance = MagicMock() - mock_azure_modules['DeidentificationClient'].return_value = mock_client_instance + mock_azure_modules["DeidentificationClient"].return_value = ( + mock_client_instance + ) mock_credential = MagicMock() - mock_azure_modules['get_azure_credential'].return_value = mock_credential - + mock_azure_modules["get_azure_credential"].return_value = mock_credential + recognizer = AzureHealthDeidRecognizer() - + # Verify get_azure_credential was called - mock_azure_modules['get_azure_credential'].assert_called_once() - + mock_azure_modules["get_azure_credential"].assert_called_once() + # Verify DeidentificationClient was initialized with credential from get_azure_credential - mock_azure_modules['DeidentificationClient'].assert_called_once() - call_args = mock_azure_modules['DeidentificationClient'].call_args + mock_azure_modules["DeidentificationClient"].assert_called_once() + call_args = mock_azure_modules["DeidentificationClient"].call_args assert call_args[0][1] == mock_credential def test_uses_chained_credential_when_env_var_not_set(self, mock_azure_modules): """Test that get_azure_credential is used when ENV is not set (default).""" # Ensure ENV is not set - env_without_presidio = {k: v for k, v in os.environ.items() if k != 'ENV'} - env_without_presidio['AHDS_ENDPOINT'] = 'https://test.endpoint.com' - + env_without_presidio = {k: v for k, v in os.environ.items() if k != "ENV"} + env_without_presidio["AHDS_ENDPOINT"] = "https://test.endpoint.com" + with patch.dict(os.environ, env_without_presidio, clear=True): mock_client_instance = MagicMock() - mock_azure_modules['DeidentificationClient'].return_value = mock_client_instance + mock_azure_modules["DeidentificationClient"].return_value = ( + mock_client_instance + ) mock_credential = MagicMock() - mock_azure_modules['get_azure_credential'].return_value = mock_credential - + mock_azure_modules["get_azure_credential"].return_value = mock_credential + recognizer = AzureHealthDeidRecognizer() - + # Verify get_azure_credential was called (secure by default) - mock_azure_modules['get_azure_credential'].assert_called_once() + mock_azure_modules["get_azure_credential"].assert_called_once() - def test_uses_chained_credential_for_non_development_environment_values(self, mock_azure_modules): + def test_uses_chained_credential_for_non_development_environment_values( + self, mock_azure_modules + ): """Test that get_azure_credential is used for any ENV value other than 'development'.""" - test_environments = ['prod', 'production', 'staging', 'test', 'local', 'DEVELOPMENT'] - + test_environments = [ + "prod", + "production", + "staging", + "test", + "local", + "DEVELOPMENT", + ] + for env_value in test_environments: - with patch.dict(os.environ, {'ENV': env_value, 'AHDS_ENDPOINT': 'https://test.endpoint.com'}): + with patch.dict( + os.environ, + {"ENV": env_value, "AHDS_ENDPOINT": "https://test.endpoint.com"}, + ): mock_client_instance = MagicMock() - mock_azure_modules['DeidentificationClient'].return_value = mock_client_instance + mock_azure_modules["DeidentificationClient"].return_value = ( + mock_client_instance + ) mock_credential = MagicMock() - mock_azure_modules['get_azure_credential'].return_value = mock_credential - + mock_azure_modules["get_azure_credential"].return_value = ( + mock_credential + ) + # Reset mocks - mock_azure_modules['get_azure_credential'].reset_mock() - mock_azure_modules['DeidentificationClient'].reset_mock() - + mock_azure_modules["get_azure_credential"].reset_mock() + mock_azure_modules["DeidentificationClient"].reset_mock() + recognizer = AzureHealthDeidRecognizer() - + # Verify get_azure_credential was called for this environment - mock_azure_modules['get_azure_credential'].assert_called_once(), f"Failed for environment: {env_value}" + mock_azure_modules[ + "get_azure_credential" + ].assert_called_once(), f"Failed for environment: {env_value}" def test_respects_provided_client_parameter(self, mock_azure_modules): """Test that when a client is provided, no credential creation occurs.""" mock_client_instance = MagicMock() - - with patch.dict(os.environ, {'ENV': 'development', 'AHDS_ENDPOINT': 'https://test.endpoint.com'}): + + with patch.dict( + os.environ, + {"ENV": "development", "AHDS_ENDPOINT": "https://test.endpoint.com"}, + ): recognizer = AzureHealthDeidRecognizer(client=mock_client_instance) - + # Verify no credential helper was called - mock_azure_modules['get_azure_credential'].assert_not_called() - mock_azure_modules['DeidentificationClient'].assert_not_called() - + mock_azure_modules["get_azure_credential"].assert_not_called() + mock_azure_modules["DeidentificationClient"].assert_not_called() + # Verify the provided client is used - assert recognizer.deid_client == mock_client_instance \ No newline at end of file + assert recognizer.deid_client == mock_client_instance diff --git a/presidio-analyzer/tests/test_analyzer_engine.py b/presidio-analyzer/tests/test_analyzer_engine.py index 9e21e93c43..938c6ba3ef 100644 --- a/presidio-analyzer/tests/test_analyzer_engine.py +++ b/presidio-analyzer/tests/test_analyzer_engine.py @@ -85,6 +85,7 @@ def test_when_analyze_with_predefined_recognizers_then_return_results( assert len(results) == 1 assert_result(results[0], "CREDIT_CARD", 14, 33, max_score) + @pytest.mark.parametrize( "registry_config,analyzer_lang,expectation", [ @@ -93,22 +94,27 @@ def test_when_analyze_with_predefined_recognizers_then_return_results( ({"supported_languages": ["es", "de"]}, None, pytest.raises(ValueError)), ({"supported_languages": ["es", "de"]}, ["de", "es"], nullcontext()), (None, None, nullcontext()), - ] + ], ) -def test_when_analyze_with_unsupported_language_must_match(registry_config, analyzer_lang, expectation): +def test_when_analyze_with_unsupported_language_must_match( + registry_config, analyzer_lang, expectation +): with expectation: - registry = RecognizerRegistryProvider(registry_configuration=registry_config).create_recognizer_registry() + registry = RecognizerRegistryProvider( + registry_configuration=registry_config + ).create_recognizer_registry() AnalyzerEngine( registry=registry, supported_languages=analyzer_lang, nlp_engine=NlpEngineMock(), ) -def test_when_analyze_with_defaults_success( -): + +def test_when_analyze_with_defaults_success(): registry = RecognizerRegistryProvider().create_recognizer_registry() AnalyzerEngine(registry=registry) + def test_when_analyze_with_multiple_predefined_recognizers_then_succeed( loaded_registry, unit_test_guid, spacy_nlp_engine, max_score ): @@ -271,7 +277,7 @@ def test_when_regex_allow_list_specified(loaded_analyzer_engine): assert_result(results[0], "URL", 0, 8, 0.5) results = loaded_analyzer_engine.analyze( - text=text, language="en", allow_list=["bing"], allow_list_match = "regex" + text=text, language="en", allow_list=["bing"], allow_list_match="regex" ) assert len(results) == 2 assert text[results[0].start : results[0].end] == "microsoft.com" @@ -289,13 +295,15 @@ def test_when_regex_allow_list_specified_but_none_in_file(loaded_analyzer_engine assert_result(results[0], "URL", 0, 8, 0.5) results = loaded_analyzer_engine.analyze( - text=text, language="en", allow_list=["microsoft"], allow_list_match = "regex" + text=text, language="en", allow_list=["microsoft"], allow_list_match="regex" ) assert len(results) == 1 assert_result(results[0], "URL", 0, 8, 0.5) -def test_when_regex_allow_list_specified_multiple_items_with_missing_flags(loaded_analyzer_engine): +def test_when_regex_allow_list_specified_multiple_items_with_missing_flags( + loaded_analyzer_engine, +): text = "bing.com is his favorite website, microsoft.com is his second favorite, azure.com is his third favorite" results = loaded_analyzer_engine.analyze( text=text, @@ -305,7 +313,10 @@ def test_when_regex_allow_list_specified_multiple_items_with_missing_flags(loade assert_result(results[0], "URL", 0, 8, 0.5) results = loaded_analyzer_engine.analyze( - text=text, language="en", allow_list=["bing", "microsoft"], allow_list_match = "regex", + text=text, + language="en", + allow_list=["bing", "microsoft"], + allow_list_match="regex", ) assert len(results) == 1 assert text[results[0].start : results[0].end] == "azure.com" @@ -321,12 +332,20 @@ def test_when_regex_allow_list_specified_with_regex_flags(loaded_analyzer_engine assert_result(results[0], "URL", 0, 8, 0.5) results = loaded_analyzer_engine.analyze( - text=text, language="en", allow_list=["BING", "MICROSOFT", "AZURE"], allow_list_match = "regex", regex_flags=0 + text=text, + language="en", + allow_list=["BING", "MICROSOFT", "AZURE"], + allow_list_match="regex", + regex_flags=0, ) assert len(results) == 3 results = loaded_analyzer_engine.analyze( - text=text, language="en", allow_list=["BING", "MICROSOFT", "AZURE"], allow_list_match = "regex", regex_flags=re.IGNORECASE + text=text, + language="en", + allow_list=["BING", "MICROSOFT", "AZURE"], + allow_list_match="regex", + regex_flags=re.IGNORECASE, ) assert len(results) == 0 @@ -419,7 +438,7 @@ def test_when_entities_is_none_then_return_all_fields(loaded_registry): def test_when_entities_is_none_all_recognizers_loaded_then_return_all_fields( - spacy_nlp_engine, + spacy_nlp_engine, ): analyze_engine = AnalyzerEngine( registry=RecognizerRegistry(), nlp_engine=spacy_nlp_engine @@ -753,7 +772,9 @@ def test_entities_filter_for_ad_hoc_removes_recognizer(loaded_analyzer_engine): assert "MR" not in [resp.entity_type for resp in responses2] -def test_ad_hoc_with_context_support_higher_confidence(spacy_nlp_engine, zip_code_recognizer): +def test_ad_hoc_with_context_support_higher_confidence( + spacy_nlp_engine, zip_code_recognizer +): text = "Mr. John Smith's zip code is 10023" analyzer_engine = AnalyzerEngine(nlp_engine=spacy_nlp_engine) @@ -836,7 +857,9 @@ def analyze(self, text: str, entities: List[str], nlp_artifacts: NlpArtifacts): ) -def test_when_recognizer_overrides_enhance_score_then_it_get_boosted_once(spacy_nlp_engine): +def test_when_recognizer_overrides_enhance_score_then_it_get_boosted_once( + spacy_nlp_engine, +): class MockRecognizer(EntityRecognizer, ABC): def analyze(self, text: str, entities: List[str], nlp_artifacts: NlpArtifacts): return [ @@ -939,12 +962,8 @@ def test_when_regex_allow_list_times_out_then_result_is_kept(loaded_analyzer_eng """Test that a timed-out allow list regex keeps the result (conservative behavior).""" text = "bing.com is his favorite website" - with patch( - "presidio_analyzer.analyzer_engine.REGEX_TIMEOUT_SECONDS", 0.001 - ): - with patch( - "presidio_analyzer.analyzer_engine.re.compile" - ) as mock_compile: + with patch("presidio_analyzer.analyzer_engine.REGEX_TIMEOUT_SECONDS", 0.001): + with patch("presidio_analyzer.analyzer_engine.re.compile") as mock_compile: mock_compiled = mock_compile.return_value mock_compiled.search.side_effect = TimeoutError("regex timed out") @@ -993,4 +1012,3 @@ def test_when_regex_allow_list_is_all_empty_entries_then_results_are_kept(): ) assert filtered == results - diff --git a/presidio-analyzer/tests/test_analyzer_engine_provider.py b/presidio-analyzer/tests/test_analyzer_engine_provider.py index ed58e4cfaf..f544f85e4e 100644 --- a/presidio-analyzer/tests/test_analyzer_engine_provider.py +++ b/presidio-analyzer/tests/test_analyzer_engine_provider.py @@ -3,7 +3,11 @@ from typing import List from unittest.mock import patch -from presidio_analyzer import AnalyzerEngineProvider, RecognizerResult, PatternRecognizer +from presidio_analyzer import ( + AnalyzerEngineProvider, + RecognizerResult, + PatternRecognizer, +) from presidio_analyzer.nlp_engine import SpacyNlpEngine, NlpArtifacts @@ -176,7 +180,10 @@ def analyze( assert len(analyzer_engine.analyze("This is a test", language="en")) > 0 -@pytest.mark.skipif(pytest.importorskip("azure"), reason="Optional dependency not installed") # noqa: E501 + +@pytest.mark.skipif( + pytest.importorskip("azure"), reason="Optional dependency not installed" +) # noqa: E501 def test_analyzer_engine_provider_with_ahds(): analyzer_yaml, _, _ = get_full_paths( "conf/test_ahds_reco.yaml", @@ -205,7 +212,6 @@ def analyze( assert len(ahds_recognizers) == 1 assert len(analyzer_engine.analyze("This is a test", language="en")) > 0 - def test_analyzer_engine_provider_no_nlp_recognizer(): @@ -221,7 +227,14 @@ def test_analyzer_engine_provider_no_nlp_recognizer(): recognizer = analyzer_engine.get_recognizers()[0] assert isinstance(recognizer, CreditCardRecognizer) - assert len(analyzer_engine.analyze("My Credit card number is 4917300800000000", language="en")) > 0 + assert ( + len( + analyzer_engine.analyze( + "My Credit card number is 4917300800000000", language="en" + ) + ) + > 0 + ) def test_analyzer_engine_provider_no_nlp_recognizer_is_added(): @@ -249,7 +262,9 @@ def test_analyzer_engine_provider_no_nlp_recognizer_is_added_per_language(): analyzer_engine = provider.create_engine() - assert len(analyzer_engine.get_recognizers()) == 4 # Two CreditCardRecognizers and two SpacyRecognizers + assert ( + len(analyzer_engine.get_recognizers()) == 4 + ) # Two CreditCardRecognizers and two SpacyRecognizers nlp_recognizers = [ rec for rec in analyzer_engine.get_recognizers() @@ -277,7 +292,8 @@ def test_analyzer_engine_provider_multiple_nlp_recognizers_raises_exception(): with pytest.raises( ValueError, match=f"Multiple NLP recognizers for language en found in the configuration. " - f"Please remove the duplicates."): + f"Please remove the duplicates.", + ): provider = AnalyzerEngineProvider(analyzer_engine_conf_file=analyzer_yaml) analyzer_engine = provider.create_engine() @@ -290,7 +306,9 @@ def test_analyzer_engine_provider_no_nlp_engine_or_provider_results_in_default_n analyzer_engine = provider.create_engine() - assert len(analyzer_engine.get_recognizers()) == 2 # SpacyRecognizer, CreditCardRecognizer + assert ( + len(analyzer_engine.get_recognizers()) == 2 + ) # SpacyRecognizer, CreditCardRecognizer nlp_recognizer = [ rec for rec in analyzer_engine.get_recognizers() @@ -323,6 +341,7 @@ def test_analyzer_engine_stanza_without_recognizer_creates_recognizer(): } assert supported_languages == {"en", "es"} + def test_analyzer_engine_provider_one_custom_recognizer(): analyzer_yaml, _, _ = get_full_paths( "conf/custom_recognizer_yaml.yaml", @@ -331,7 +350,9 @@ def test_analyzer_engine_provider_one_custom_recognizer(): analyzer_engine = provider.create_engine() assert len(analyzer_engine.get_recognizers()) == 1 - assert analyzer_engine.analyze("My zip code is 12345", language="en")[0].score == pytest.approx(0.4) + assert analyzer_engine.analyze("My zip code is 12345", language="en")[ + 0 + ].score == pytest.approx(0.4) def test_analyzer_engine_provider_invalid_analyzer_conf_file(): @@ -349,7 +370,9 @@ def test_analyzer_engine_provider_invalid_nlp_conf_file(): def test_analyzer_engine_provider_invalid_registry_conf_file(): """Test that invalid recognizer registry configuration file path raises error.""" with pytest.raises(ValueError): - AnalyzerEngineProvider(recognizer_registry_conf_file="/nonexistent/path/file.yaml") + AnalyzerEngineProvider( + recognizer_registry_conf_file="/nonexistent/path/file.yaml" + ) def test_analyzer_engine_provider_get_configuration_with_nonexistent_file(): @@ -370,7 +393,7 @@ def test_analyzer_engine_provider_get_configuration_with_invalid_yaml(): import os # Create a temporary file with invalid YAML - with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as f: + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: f.write("invalid: yaml: content: [[[") temp_file = f.name @@ -599,8 +622,6 @@ def test_analyzer_engine_provider_configuration_logging(caplog): assert len(caplog.records) > 0 - - # --- install_models / _install_models_from_nlp_config tests --- _NLP_CONF_CONTENT = ( @@ -640,7 +661,9 @@ def test_install_models_analyzer_conf_without_nlp_falls_back_to_nlp_conf(tmp_pat nlp_yaml.write_text(_NLP_CONF_CONTENT) with patch("install_nlp_models._download_model") as mock_dl: - install_models(nlp_conf_file=str(nlp_yaml), analyzer_conf_file=str(analyzer_yaml)) + install_models( + nlp_conf_file=str(nlp_yaml), analyzer_conf_file=str(analyzer_yaml) + ) mock_dl.assert_called_once_with("spacy", "en_core_web_sm") @@ -664,7 +687,9 @@ def test_install_models_analyzer_conf_takes_priority_over_nlp_conf(tmp_path): nlp_yaml.write_text(_NLP_CONF_CONTENT) with patch("install_nlp_models._download_model") as mock_dl: - install_models(nlp_conf_file=str(nlp_yaml), analyzer_conf_file=str(analyzer_yaml)) + install_models( + nlp_conf_file=str(nlp_yaml), analyzer_conf_file=str(analyzer_yaml) + ) mock_dl.assert_called_once_with("spacy", "en_core_web_lg") diff --git a/presidio-analyzer/tests/test_analyzer_request.py b/presidio-analyzer/tests/test_analyzer_request.py index c704ac9d57..4b26b83cea 100644 --- a/presidio-analyzer/tests/test_analyzer_request.py +++ b/presidio-analyzer/tests/test_analyzer_request.py @@ -30,7 +30,7 @@ def test_analyzer_request_with_context(self): req_data = { "text": "Test text", "language": "en", - "context": ["previous message", "current message"] + "context": ["previous message", "current message"], } request = AnalyzerRequest(req_data) @@ -42,7 +42,7 @@ def test_analyzer_request_with_allow_list(self): req_data = { "text": "Test text", "language": "en", - "allow_list": ["John", "Microsoft", "Seattle"] + "allow_list": ["John", "Microsoft", "Seattle"], } request = AnalyzerRequest(req_data) @@ -66,7 +66,7 @@ def test_analyzer_request_with_allow_list_match_custom(self): req_data = { "text": "Test text", "language": "en", - "allow_list_match": "partial" + "allow_list_match": "partial", } request = AnalyzerRequest(req_data) @@ -89,11 +89,7 @@ def test_analyzer_request_with_regex_flags_default(self): def test_analyzer_request_with_regex_flags_custom(self): """Test regex_flags field with custom value (line 40).""" custom_flags = re.IGNORECASE | re.UNICODE - req_data = { - "text": "Test text", - "language": "en", - "regex_flags": custom_flags - } + req_data = {"text": "Test text", "language": "en", "regex_flags": custom_flags} request = AnalyzerRequest(req_data) @@ -138,15 +134,15 @@ def test_analyzer_request_all_fields(self): { "name": "custom_pattern", "regex": r"\d{3}-\d{3}", - "score": 0.5 + "score": 0.5, } - ] + ], } ], "context": ["user profile", "chat history"], "allow_list": ["John", "Microsoft"], "allow_list_match": "fuzzy", - "regex_flags": re.IGNORECASE + "regex_flags": re.IGNORECASE, } request = AnalyzerRequest(req_data) @@ -174,14 +170,10 @@ def test_analyzer_request_with_ad_hoc_recognizers(self): "supported_entity": "CUSTOM_ID", "supported_language": "en", "patterns": [ - { - "name": "id_pattern", - "regex": r"ID-\d{5}", - "score": 0.8 - } - ] + {"name": "id_pattern", "regex": r"ID-\d{5}", "score": 0.8} + ], } - ] + ], } request = AnalyzerRequest(req_data) @@ -227,8 +219,8 @@ def test_analyzer_request_with_complex_context(self): "context": { "user_id": "12345", "session": "abc", - "metadata": {"key": "value"} - } + "metadata": {"key": "value"}, + }, } request = AnalyzerRequest(req_data) @@ -236,17 +228,13 @@ def test_analyzer_request_with_complex_context(self): assert request.context == { "user_id": "12345", "session": "abc", - "metadata": {"key": "value"} + "metadata": {"key": "value"}, } def test_analyzer_request_with_multiple_regex_flags(self): """Test regex_flags with multiple combined flags.""" custom_flags = re.IGNORECASE | re.MULTILINE | re.DOTALL | re.VERBOSE - req_data = { - "text": "Test text", - "language": "en", - "regex_flags": custom_flags - } + req_data = {"text": "Test text", "language": "en", "regex_flags": custom_flags} request = AnalyzerRequest(req_data) @@ -259,21 +247,14 @@ def test_analyzer_request_with_multiple_regex_flags(self): def test_analyzer_request_allow_list_match_variations(self): """Test various allow_list_match values.""" - test_cases = [ - "exact", - "partial", - "fuzzy", - "regex", - "custom_match_type" - ] + test_cases = ["exact", "partial", "fuzzy", "regex", "custom_match_type"] for match_type in test_cases: req_data = { "text": "Test text", "language": "en", - "allow_list_match": match_type + "allow_list_match": match_type, } request = AnalyzerRequest(req_data) assert request.allow_list_match == match_type - diff --git a/presidio-analyzer/tests/test_au_abn_recognizer.py b/presidio-analyzer/tests/test_au_abn_recognizer.py index 1033f257aa..d0625304d6 100644 --- a/presidio-analyzer/tests/test_au_abn_recognizer.py +++ b/presidio-analyzer/tests/test_au_abn_recognizer.py @@ -17,17 +17,57 @@ def entities(): @pytest.mark.parametrize( "text, expected_len, expected_positions, expected_score_ranges", [ - # Valid formatting and valid ABNs - ("51 824 753 556", 1, ((0, 14),), ((1.0, 1.0),), ), - ("51824753556", 1, ((0, 11),), ((1.0, 1.0),), ), - # Valid formatting but invalid ABNs - ("52 824 753 556", 0, (), (),), - ("52824753556", 0, (), (),), - # Invalid formatting and ABNs. - ("5282475355632", 0, (), (),), - ("52824753556AF", 0, (), (),), - ("51 824 753 5564", 0, (), (),), - ("123 456\n789", 0, (), (),), + # Valid formatting and valid ABNs + ( + "51 824 753 556", + 1, + ((0, 14),), + ((1.0, 1.0),), + ), + ( + "51824753556", + 1, + ((0, 11),), + ((1.0, 1.0),), + ), + # Valid formatting but invalid ABNs + ( + "52 824 753 556", + 0, + (), + (), + ), + ( + "52824753556", + 0, + (), + (), + ), + # Invalid formatting and ABNs. + ( + "5282475355632", + 0, + (), + (), + ), + ( + "52824753556AF", + 0, + (), + (), + ), + ( + "51 824 753 5564", + 0, + (), + (), + ), + ( + "123 456\n789", + 0, + (), + (), + ), ], ) def test_when_all_abns_then_succeed( diff --git a/presidio-analyzer/tests/test_au_acn_recognizer.py b/presidio-analyzer/tests/test_au_acn_recognizer.py index ed928e6cba..007571055a 100644 --- a/presidio-analyzer/tests/test_au_acn_recognizer.py +++ b/presidio-analyzer/tests/test_au_acn_recognizer.py @@ -17,18 +17,63 @@ def entities(): @pytest.mark.parametrize( "text, expected_len, expected_positions, expected_score_ranges", [ - # Valid formatting and valid ACNs - ("000 000 019", 1, ((0, 11),), ((1.0, 1.0),), ), - ("005 499 981", 1, ((0, 11),), ((1.0, 1.0),), ), - ("006249976", 1, ((0, 9),), ((1.0, 1.0),), ), - # Valid formatting but invalid ACNs - ("824 753 557", 0, (), (),), - ("824753557", 0, (), (),), + # Valid formatting and valid ACNs + ( + "000 000 019", + 1, + ((0, 11),), + ((1.0, 1.0),), + ), + ( + "005 499 981", + 1, + ((0, 11),), + ((1.0, 1.0),), + ), + ( + "006249976", + 1, + ((0, 9),), + ((1.0, 1.0),), + ), + # Valid formatting but invalid ACNs + ( + "824 753 557", + 0, + (), + (), + ), + ( + "824753557", + 0, + (), + (), + ), # Invalid formatting and ACNs. - ("5282475355632", 0, (), (),), - ("52824753556AF", 0, (), (),), - ("51 824 753 5564", 0, (), (),), - ("123 456\n789", 0, (), (),), + ( + "5282475355632", + 0, + (), + (), + ), + ( + "52824753556AF", + 0, + (), + (), + ), + ( + "51 824 753 5564", + 0, + (), + (), + ), + ( + "123 456\n789", + 0, + (), + (), + ), ], ) def test_when_all_acns_then_succeed( diff --git a/presidio-analyzer/tests/test_au_medicare_recognizer.py b/presidio-analyzer/tests/test_au_medicare_recognizer.py index 0f9a28f087..7a105d1257 100644 --- a/presidio-analyzer/tests/test_au_medicare_recognizer.py +++ b/presidio-analyzer/tests/test_au_medicare_recognizer.py @@ -17,16 +17,51 @@ def entities(): @pytest.mark.parametrize( "text, expected_len, expected_positions, expected_score_ranges", [ - # Valid formatting and valid Medicare number. - ("2123 45670 1", 1, ((0, 12),), ((1.0, 1.0),), ), - ("2123456701", 1, ((0, 10),), ((1.0, 1.0),), ), + # Valid formatting and valid Medicare number. + ( + "2123 45670 1", + 1, + ((0, 12),), + ((1.0, 1.0),), + ), + ( + "2123456701", + 1, + ((0, 10),), + ((1.0, 1.0),), + ), # Valid formatting but invalid Medicare number. - ("2123 25870 1", 0, (), (),), - ("2123258701", 0, (), (),), - # Invalid formatting and Medicare number. - ("212345670221", 0, (), (),), - ("2123456702AF", 0, (), (),), - ("123 456\n789", 0, (), (),), + ( + "2123 25870 1", + 0, + (), + (), + ), + ( + "2123258701", + 0, + (), + (), + ), + # Invalid formatting and Medicare number. + ( + "212345670221", + 0, + (), + (), + ), + ( + "2123456702AF", + 0, + (), + (), + ), + ( + "123 456\n789", + 0, + (), + (), + ), ], ) def test_when_all_medicares_then_succeed( diff --git a/presidio-analyzer/tests/test_au_tfn_recognizer.py b/presidio-analyzer/tests/test_au_tfn_recognizer.py index aa2613e911..f6d7162dcd 100644 --- a/presidio-analyzer/tests/test_au_tfn_recognizer.py +++ b/presidio-analyzer/tests/test_au_tfn_recognizer.py @@ -17,17 +17,57 @@ def entities(): @pytest.mark.parametrize( "text, expected_len, expected_positions, expected_score_ranges", [ - # Valid formatting and valid TFNs - ("876 543 210", 1, ((0, 11),), ((1.0, 1.0),), ), - ("876543210", 1, ((0, 9),), ((1.0, 1.0),), ), - # Valid formatting but invalid TFNs - ("824 753 557", 0, (), (),), - ("824753557", 0, (), (),), - # Invalid formatting and TFNs. - ("5282475355632", 0, (), (),), - ("52824753556AF", 0, (), (),), - ("51 824 753 5564", 0, (), (),), - ("123 456\n789", 0, (), (),), + # Valid formatting and valid TFNs + ( + "876 543 210", + 1, + ((0, 11),), + ((1.0, 1.0),), + ), + ( + "876543210", + 1, + ((0, 9),), + ((1.0, 1.0),), + ), + # Valid formatting but invalid TFNs + ( + "824 753 557", + 0, + (), + (), + ), + ( + "824753557", + 0, + (), + (), + ), + # Invalid formatting and TFNs. + ( + "5282475355632", + 0, + (), + (), + ), + ( + "52824753556AF", + 0, + (), + (), + ), + ( + "51 824 753 5564", + 0, + (), + (), + ), + ( + "123 456\n789", + 0, + (), + (), + ), ], ) def test_when_all_tfns_then_succeed( diff --git a/presidio-analyzer/tests/test_azure_ai_language_recognizer.py b/presidio-analyzer/tests/test_azure_ai_language_recognizer.py index 381b3f3413..37282c1311 100644 --- a/presidio-analyzer/tests/test_azure_ai_language_recognizer.py +++ b/presidio-analyzer/tests/test_azure_ai_language_recognizer.py @@ -16,7 +16,7 @@ def requires_env_vars(): return pytest.mark.skipif( azure_ai_key == "" or azure_ai_endpoint == "", - reason=f"Environment variables are not set" + reason=f"Environment variables are not set", ) @@ -63,14 +63,23 @@ def test_mocked_entities_match_recognizer_results(): except ImportError: pytest.skip("Skipping test because 'azure.ai.textanalytics' is not installed") - from azure.ai.textanalytics import PiiEntity, TextAnalyticsClient, \ - RecognizePiiEntitiesResult + from azure.ai.textanalytics import ( + PiiEntity, + TextAnalyticsClient, + RecognizePiiEntitiesResult, + ) from azure.core.credentials import AzureKeyCredential - ent1 = PiiEntity(text="Raj", category="Person", - length=3, offset=0, confidence_score=0.8) - ent2 = PiiEntity(text="My email is ab@email.com", - category="EMAIL", length=12, offset=12, confidence_score=0.9) + ent1 = PiiEntity( + text="Raj", category="Person", length=3, offset=0, confidence_score=0.8 + ) + ent2 = PiiEntity( + text="My email is ab@email.com", + category="EMAIL", + length=12, + offset=12, + confidence_score=0.9, + ) mock_entities = [ent1, ent2] ta_client = TextAnalyticsClient(endpoint="", credential=AzureKeyCredential(key="")) diff --git a/presidio-analyzer/tests/test_azure_auth_helper.py b/presidio-analyzer/tests/test_azure_auth_helper.py index 4a4d8e2cbf..220153cef3 100644 --- a/presidio-analyzer/tests/test_azure_auth_helper.py +++ b/presidio-analyzer/tests/test_azure_auth_helper.py @@ -7,7 +7,7 @@ class TestGetAzureCredential: """Test get_azure_credential function.""" - @patch('presidio_analyzer.llm_utils.azure_auth_helper.DefaultAzureCredential') + @patch("presidio_analyzer.llm_utils.azure_auth_helper.DefaultAzureCredential") def test_when_development_env_then_uses_default_credential(self, mock_default_cred): """Test that development mode uses DefaultAzureCredential.""" from presidio_analyzer.llm_utils.azure_auth_helper import get_azure_credential @@ -15,17 +15,17 @@ def test_when_development_env_then_uses_default_credential(self, mock_default_cr mock_cred_instance = MagicMock() mock_default_cred.return_value = mock_cred_instance - with patch.dict(os.environ, {'ENV': 'development'}): + with patch.dict(os.environ, {"ENV": "development"}): credential = get_azure_credential() # Verify DefaultAzureCredential was called mock_default_cred.assert_called_once() assert credential == mock_cred_instance - @patch('presidio_analyzer.llm_utils.azure_auth_helper.ChainedTokenCredential') - @patch('presidio_analyzer.llm_utils.azure_auth_helper.EnvironmentCredential') - @patch('presidio_analyzer.llm_utils.azure_auth_helper.WorkloadIdentityCredential') - @patch('presidio_analyzer.llm_utils.azure_auth_helper.ManagedIdentityCredential') + @patch("presidio_analyzer.llm_utils.azure_auth_helper.ChainedTokenCredential") + @patch("presidio_analyzer.llm_utils.azure_auth_helper.EnvironmentCredential") + @patch("presidio_analyzer.llm_utils.azure_auth_helper.WorkloadIdentityCredential") + @patch("presidio_analyzer.llm_utils.azure_auth_helper.ManagedIdentityCredential") def test_when_production_env_then_uses_chained_credential( self, mock_managed, mock_workload, mock_env, mock_chained ): @@ -35,17 +35,17 @@ def test_when_production_env_then_uses_chained_credential( mock_cred_instance = MagicMock() mock_chained.return_value = mock_cred_instance - with patch.dict(os.environ, {'ENV': 'production'}): + with patch.dict(os.environ, {"ENV": "production"}): credential = get_azure_credential() # Verify ChainedTokenCredential was created with correct credentials mock_chained.assert_called_once() assert credential == mock_cred_instance - @patch('presidio_analyzer.llm_utils.azure_auth_helper.ChainedTokenCredential') - @patch('presidio_analyzer.llm_utils.azure_auth_helper.EnvironmentCredential') - @patch('presidio_analyzer.llm_utils.azure_auth_helper.WorkloadIdentityCredential') - @patch('presidio_analyzer.llm_utils.azure_auth_helper.ManagedIdentityCredential') + @patch("presidio_analyzer.llm_utils.azure_auth_helper.ChainedTokenCredential") + @patch("presidio_analyzer.llm_utils.azure_auth_helper.EnvironmentCredential") + @patch("presidio_analyzer.llm_utils.azure_auth_helper.WorkloadIdentityCredential") + @patch("presidio_analyzer.llm_utils.azure_auth_helper.ManagedIdentityCredential") def test_when_no_env_set_then_uses_chained_credential( self, mock_managed, mock_workload, mock_env, mock_chained ): @@ -57,8 +57,8 @@ def test_when_no_env_set_then_uses_chained_credential( # Remove ENV from environment env = os.environ.copy() - if 'ENV' in env: - del env['ENV'] + if "ENV" in env: + del env["ENV"] with patch.dict(os.environ, env, clear=True): credential = get_azure_credential() @@ -71,8 +71,8 @@ def test_when_no_env_set_then_uses_chained_credential( class TestGetBearerTokenProvider: """Test get_bearer_token_provider_for_scope function.""" - @patch('azure.identity.get_bearer_token_provider') - @patch('presidio_analyzer.llm_utils.azure_auth_helper.DefaultAzureCredential') + @patch("azure.identity.get_bearer_token_provider") + @patch("presidio_analyzer.llm_utils.azure_auth_helper.DefaultAzureCredential") def test_when_scope_provided_then_returns_provider( self, mock_default_cred, mock_get_provider ): @@ -86,14 +86,14 @@ def test_when_scope_provided_then_returns_provider( scope = "https://cognitiveservices.azure.com/.default" - with patch.dict(os.environ, {'ENV': 'development'}): + with patch.dict(os.environ, {"ENV": "development"}): provider = get_bearer_token_provider_for_scope(scope) # Verify get_bearer_token_provider was called with scope mock_get_provider.assert_called_once() assert provider == mock_provider - @patch('azure.identity.get_bearer_token_provider') + @patch("azure.identity.get_bearer_token_provider") def test_when_custom_credential_provided_then_uses_it(self, mock_get_provider): """Test that custom credential is used when provided.""" from presidio_analyzer.llm_utils.azure_auth_helper import ( diff --git a/presidio-analyzer/tests/test_azure_openai_langextract_recognizer.py b/presidio-analyzer/tests/test_azure_openai_langextract_recognizer.py index 04a3d6fbe4..09d89cead7 100644 --- a/presidio-analyzer/tests/test_azure_openai_langextract_recognizer.py +++ b/presidio-analyzer/tests/test_azure_openai_langextract_recognizer.py @@ -6,22 +6,31 @@ try: importlib.import_module("langextract") except ImportError: - pytest.skip("Skipping test because 'langextract' is not installed", allow_module_level=True) + pytest.skip( + "Skipping test because 'langextract' is not installed", allow_module_level=True + ) import openai from presidio_analyzer.predefined_recognizers import AzureOpenAILangExtractRecognizer -from presidio_analyzer.predefined_recognizers.third_party.azure_openai_provider import AzureOpenAILanguageModel +from presidio_analyzer.predefined_recognizers.third_party.azure_openai_provider import ( + AzureOpenAILanguageModel, +) @pytest.fixture def mock_langextract(): """Mock langextract to avoid actual API calls.""" - with patch('presidio_analyzer.llm_utils.langextract_helper.lx') as mock_lx: + with patch("presidio_analyzer.llm_utils.langextract_helper.lx") as mock_lx: # Mock successful extraction mock_result = MagicMock() mock_result.to_dict.return_value = { "entities": [ - {"text": "john.doe@example.com", "label": "email", "start": 12, "end": 32} + { + "text": "john.doe@example.com", + "label": "email", + "start": 12, + "end": 32, + } ] } mock_lx.extract.return_value = mock_result @@ -33,25 +42,19 @@ class TestAzureOpenAILangExtractRecognizerInitialization: def test_import_error_when_langextract_not_installed(self): """Test that ImportError is raised when langextract is not installed.""" - with patch( - 'presidio_analyzer.llm_utils.langextract_helper.lx', - None - ): + with patch("presidio_analyzer.llm_utils.langextract_helper.lx", None): with pytest.raises(ImportError, match="LangExtract is not installed"): AzureOpenAILangExtractRecognizer( - model_id="gpt-4o", - azure_endpoint="https://test.openai.azure.com/" + model_id="gpt-4o", azure_endpoint="https://test.openai.azure.com/" ) def test_missing_endpoint_raises_error(self, tmp_path): """Test that initialization fails when Azure endpoint is missing.""" import yaml import os - + config = { - "lm_recognizer": { - "type": "AzureOpenAILangExtractRecognizer" - }, + "lm_recognizer": {"type": "AzureOpenAILangExtractRecognizer"}, "langextract": { "supported_entities": ["PERSON"], "entity_mappings": {"person_name": "PERSON"}, @@ -61,99 +64,106 @@ def test_missing_endpoint_raises_error(self, tmp_path): "model": { "model_id": "gpt-4", "temperature": 0.0, - } + }, }, "azure_openai": { "api_key": "PLACEHOLDER_NOT_A_REAL_KEY", "api_version": "2024-02-01", - } + }, } config_file = tmp_path / "test_config.yaml" - with open(config_file, 'w') as f: + with open(config_file, "w") as f: yaml.dump(config, f) # Clear any environment variables that might provide endpoint - env_without_azure = {k: v for k, v in os.environ.items() - if not k.startswith('AZURE_OPENAI')} - - with patch.dict('os.environ', env_without_azure, clear=True): + env_without_azure = { + k: v for k, v in os.environ.items() if not k.startswith("AZURE_OPENAI") + } + + with patch.dict("os.environ", env_without_azure, clear=True): with pytest.raises(ValueError, match="azure_endpoint"): AzureOpenAILangExtractRecognizer(config_path=str(config_file)) class TestAzureOpenAILangExtractRecognizerUsage: """Test the simplified API that doesn't require config file download.""" - + def test_simple_usage_with_parameters(self, mock_langextract): """Best practice: Pass model_id and credentials as parameters.""" # Simple usage - no config file needed! recognizer = AzureOpenAILangExtractRecognizer( model_id="gpt-4o", # Your Azure deployment name azure_endpoint="https://test-resource.openai.azure.com/", - api_key="PLACEHOLDER_NOT_A_REAL_KEY" + api_key="PLACEHOLDER_NOT_A_REAL_KEY", ) - + assert recognizer.model_id == "gpt-4o" assert recognizer.azure_endpoint == "https://test-resource.openai.azure.com/" assert recognizer.api_key == "PLACEHOLDER_NOT_A_REAL_KEY" - + def test_environment_variables(self, mock_langextract): """Alternative: Use environment variables for credentials.""" env_vars = { "AZURE_OPENAI_ENDPOINT": "https://test-resource.openai.azure.com/", - "AZURE_OPENAI_API_KEY": "PLACEHOLDER_NOT_A_REAL_KEY" + "AZURE_OPENAI_API_KEY": "PLACEHOLDER_NOT_A_REAL_KEY", } - + with patch.dict(os.environ, env_vars): recognizer = AzureOpenAILangExtractRecognizer( model_id="gpt-4o" # Just pass deployment name ) - + assert recognizer.model_id == "gpt-4o" - assert recognizer.azure_endpoint == "https://test-resource.openai.azure.com/" + assert ( + recognizer.azure_endpoint == "https://test-resource.openai.azure.com/" + ) assert recognizer.api_key == "PLACEHOLDER_NOT_A_REAL_KEY" - + def test_managed_identity_no_api_key(self, mock_langextract): """Production: Use managed identity (no API key).""" recognizer = AzureOpenAILangExtractRecognizer( model_id="gpt-4o", - azure_endpoint="https://test-resource.openai.azure.com/" + azure_endpoint="https://test-resource.openai.azure.com/", # No api_key = uses managed identity ) - + assert recognizer.model_id == "gpt-4o" assert recognizer.api_key is None # Will use managed identity - + def test_model_id_overrides_config(self, mock_langextract, tmp_path): """Parameter model_id overrides config file model_id.""" # Mock file loading to avoid needing actual prompt/example files - with patch('presidio_analyzer.llm_utils.config_loader.load_yaml_file') as mock_load: + with patch( + "presidio_analyzer.llm_utils.config_loader.load_yaml_file" + ) as mock_load: mock_load.return_value = {} # Empty prompt/examples - + # Parameter overrides config - using default config recognizer = AzureOpenAILangExtractRecognizer( model_id="gpt-4o", # Parameter overrides default config azure_endpoint="https://test-resource.openai.azure.com/", - api_key="test-api-key" + api_key="test-api-key", ) - + # Should use parameter value assert recognizer.model_id == "gpt-4o" - + def test_config_file_provides_custom_entities(self, mock_langextract): """Advanced: Default config provides custom entities/mappings.""" # Mock file loading to avoid needing actual files - with patch('presidio_analyzer.llm_utils.config_loader.load_yaml_file') as mock_load: + with patch( + "presidio_analyzer.llm_utils.config_loader.load_yaml_file" + ) as mock_load: mock_load.return_value = {} # Empty prompt/examples - + # Use default config but override model_id recognizer = AzureOpenAILangExtractRecognizer( model_id="gpt-4o", # Override deployment name azure_endpoint="https://test-resource.openai.azure.com/", - api_key="test-api-key" + api_key="test-api-key", ) - + # Default config provides many entities assert recognizer.model_id == "gpt-4o" # From parameter assert "PERSON" in recognizer.supported_entities # From default config @@ -163,19 +173,23 @@ def test_config_file_provides_custom_entities(self, mock_langextract): class TestAzureOpenAILangExtractRecognizerCallMethod: """Test the _call_langextract method to achieve >90% coverage.""" - def test_call_langextract_builds_correct_params_with_api_key(self, mock_langextract): + def test_call_langextract_builds_correct_params_with_api_key( + self, mock_langextract + ): """Test that _call_langextract builds correct parameters including API key.""" recognizer = AzureOpenAILangExtractRecognizer( model_id="gpt-4o", azure_endpoint="https://test.openai.azure.com/", - api_key="PLACEHOLDER_NOT_A_REAL_KEY" + api_key="PLACEHOLDER_NOT_A_REAL_KEY", ) - + # Mock to capture the call mock_langextract.extract.return_value = MagicMock() - + # Call through analyze which calls _call_langextract - with patch('presidio_analyzer.llm_utils.config_loader.load_yaml_file') as mock_load: + with patch( + "presidio_analyzer.llm_utils.config_loader.load_yaml_file" + ) as mock_load: mock_load.return_value = {} # Trigger the call try: @@ -183,35 +197,45 @@ def test_call_langextract_builds_correct_params_with_api_key(self, mock_langextr except Exception: # Intentionally ignore exceptions - we only need to verify mock call arguments pass - + # Verify extract was called with correct params if mock_langextract.extract.called: call_kwargs = mock_langextract.extract.call_args[1] assert call_kwargs["model_id"] == "azure:gpt-4o" assert "language_model_params" in call_kwargs - assert call_kwargs["language_model_params"]["api_key"] == "PLACEHOLDER_NOT_A_REAL_KEY" - assert call_kwargs["language_model_params"]["azure_endpoint"] == "https://test.openai.azure.com/" + assert ( + call_kwargs["language_model_params"]["api_key"] + == "PLACEHOLDER_NOT_A_REAL_KEY" + ) + assert ( + call_kwargs["language_model_params"]["azure_endpoint"] + == "https://test.openai.azure.com/" + ) assert call_kwargs["fence_output"] is True assert call_kwargs["use_schema_constraints"] is False - def test_call_langextract_without_api_key_for_managed_identity(self, mock_langextract): + def test_call_langextract_without_api_key_for_managed_identity( + self, mock_langextract + ): """Test that _call_langextract omits API key when using managed identity.""" recognizer = AzureOpenAILangExtractRecognizer( model_id="gpt-4o", - azure_endpoint="https://test.openai.azure.com/" + azure_endpoint="https://test.openai.azure.com/", # No API key - uses managed identity ) - + mock_langextract.extract.return_value = MagicMock() - - with patch('presidio_analyzer.llm_utils.config_loader.load_yaml_file') as mock_load: + + with patch( + "presidio_analyzer.llm_utils.config_loader.load_yaml_file" + ) as mock_load: mock_load.return_value = {} try: recognizer.analyze("test", []) except Exception: # Intentionally ignore exceptions - we only need to verify mock call arguments pass - + if mock_langextract.extract.called: call_kwargs = mock_langextract.extract.call_args[1] # API key should NOT be in params @@ -222,11 +246,11 @@ def test_call_langextract_passes_through_kwargs(self, mock_langextract): recognizer = AzureOpenAILangExtractRecognizer( model_id="gpt-4o", azure_endpoint="https://test.openai.azure.com/", - api_key="key" + api_key="key", ) - + mock_langextract.extract.return_value = MagicMock() - + # Call _call_langextract directly with extra kwargs try: recognizer._call_langextract( @@ -234,12 +258,12 @@ def test_call_langextract_passes_through_kwargs(self, mock_langextract): prompt="prompt", examples=[], temperature=0.7, - max_workers=5 + max_workers=5, ) except Exception: # Intentionally ignore exceptions - we only need to verify mock call arguments pass - + if mock_langextract.extract.called: call_kwargs = mock_langextract.extract.call_args[1] assert call_kwargs.get("temperature") == 0.7 @@ -250,17 +274,21 @@ def test_call_langextract_handles_exceptions(self, mock_langextract): recognizer = AzureOpenAILangExtractRecognizer( model_id="gpt-4o", azure_endpoint="https://test.openai.azure.com/", - api_key="key" + api_key="key", ) - + # Patch langextract.extract at the module level where it's actually called - with patch('langextract.extract', side_effect=RuntimeError("Connection failed")): + with patch( + "langextract.extract", side_effect=RuntimeError("Connection failed") + ): # Should re-raise the exception with pytest.raises(RuntimeError, match="Connection failed"): recognizer._call_langextract( text="test", - prompt="prompt", - examples=[MagicMock()] # Provide mock example to avoid validation error + prompt="prompt", + examples=[ + MagicMock() + ], # Provide mock example to avoid validation error ) @@ -269,13 +297,13 @@ class TestAzureOpenAIProvider: def test_provider_initialization_with_api_key(self): """Test provider initialization with API key.""" - with patch.object(openai, 'AzureOpenAI') as mock_client: + with patch.object(openai, "AzureOpenAI") as mock_client: provider = AzureOpenAILanguageModel( model_id="gpt-4o", api_key="PLACEHOLDER_NOT_A_REAL_KEY", - azure_endpoint="https://test.openai.azure.com/" + azure_endpoint="https://test.openai.azure.com/", ) - + assert provider.model_id == "gpt-4o" assert provider.api_key == "PLACEHOLDER_NOT_A_REAL_KEY" assert provider.azure_endpoint == "https://test.openai.azure.com/" @@ -284,32 +312,39 @@ def test_provider_initialization_with_api_key(self): def test_provider_strips_azure_prefix(self): """Test that azure: prefix is stripped from model_id.""" - with patch.object(openai, 'AzureOpenAI'): + with patch.object(openai, "AzureOpenAI"): provider = AzureOpenAILanguageModel( model_id="azure:gpt-4o", api_key="PLACEHOLDER_KEY", - azure_endpoint="https://test.openai.azure.com/" + azure_endpoint="https://test.openai.azure.com/", ) - + assert provider.model_id == "gpt-4o" assert provider.azure_deployment == "gpt-4o" def test_provider_requires_endpoint(self): """Test that provider raises error if endpoint is missing.""" # Clear environment variable to ensure test fails without endpoint - env_without_endpoint = {k: v for k, v in os.environ.items() if k != 'AZURE_OPENAI_ENDPOINT'} + env_without_endpoint = { + k: v for k, v in os.environ.items() if k != "AZURE_OPENAI_ENDPOINT" + } with patch.dict(os.environ, env_without_endpoint, clear=True): with pytest.raises(ValueError, match="Azure OpenAI endpoint is required"): - AzureOpenAILanguageModel( - model_id="gpt-4o", - api_key="PLACEHOLDER_KEY" - ) + AzureOpenAILanguageModel(model_id="gpt-4o", api_key="PLACEHOLDER_KEY") def test_provider_managed_identity_development_env(self): """Test that provider uses get_bearer_token_provider_for_scope in development.""" - with patch.dict(os.environ, {'ENV': 'development', 'AZURE_OPENAI_ENDPOINT': 'https://test.openai.azure.com/'}): - with patch.object(openai, 'AzureOpenAI'): - with patch('presidio_analyzer.predefined_recognizers.third_party.azure_openai_provider.get_bearer_token_provider_for_scope') as mock_token_provider: + with patch.dict( + os.environ, + { + "ENV": "development", + "AZURE_OPENAI_ENDPOINT": "https://test.openai.azure.com/", + }, + ): + with patch.object(openai, "AzureOpenAI"): + with patch( + "presidio_analyzer.predefined_recognizers.third_party.azure_openai_provider.get_bearer_token_provider_for_scope" + ) as mock_token_provider: mock_token_provider.return_value = MagicMock() AzureOpenAILanguageModel(model_id="gpt-4o") # Verify token provider was called with correct scope @@ -319,13 +354,15 @@ def test_provider_managed_identity_development_env(self): def test_provider_managed_identity_production_env(self): """Test that provider uses get_bearer_token_provider_for_scope in production.""" - env_vars = {'AZURE_OPENAI_ENDPOINT': 'https://test.openai.azure.com/'} - env_without_dev = {k: v for k, v in os.environ.items() if k != 'ENV'} + env_vars = {"AZURE_OPENAI_ENDPOINT": "https://test.openai.azure.com/"} + env_without_dev = {k: v for k, v in os.environ.items() if k != "ENV"} env_without_dev.update(env_vars) - + with patch.dict(os.environ, env_without_dev, clear=True): - with patch.object(openai, 'AzureOpenAI'): - with patch('presidio_analyzer.predefined_recognizers.third_party.azure_openai_provider.get_bearer_token_provider_for_scope') as mock_token_provider: + with patch.object(openai, "AzureOpenAI"): + with patch( + "presidio_analyzer.predefined_recognizers.third_party.azure_openai_provider.get_bearer_token_provider_for_scope" + ) as mock_token_provider: mock_token_provider.return_value = MagicMock() AzureOpenAILanguageModel(model_id="gpt-4o") # Verify token provider was called with correct scope @@ -336,38 +373,41 @@ def test_provider_managed_identity_production_env(self): def test_provider_custom_token_provider(self): """Test that provider can use custom Azure AD token provider.""" mock_token_provider = MagicMock() - - with patch.object(openai, 'AzureOpenAI') as mock_client: + + with patch.object(openai, "AzureOpenAI") as mock_client: AzureOpenAILanguageModel( model_id="gpt-4o", azure_endpoint="https://test.openai.azure.com/", - azure_ad_token_provider=mock_token_provider + azure_ad_token_provider=mock_token_provider, ) - + mock_client.assert_called_once() call_kwargs = mock_client.call_args[1] - assert call_kwargs['azure_ad_token_provider'] == mock_token_provider + assert call_kwargs["azure_ad_token_provider"] == mock_token_provider def test_provider_get_client_model_id(self): """Test that _get_client_model_id returns deployment name.""" - with patch.object(openai, 'AzureOpenAI'): + with patch.object(openai, "AzureOpenAI"): provider = AzureOpenAILanguageModel( model_id="gpt-4o", api_key="PLACEHOLDER_KEY", azure_endpoint="https://test.openai.azure.com/", - azure_deployment="my-deployment" + azure_deployment="my-deployment", ) - + assert provider._get_client_model_id() == "my-deployment" def test_provider_import_error_without_azure_identity(self): """Test that provider raises error when azure-identity is missing for managed identity.""" - with patch('presidio_analyzer.predefined_recognizers.third_party.azure_openai_provider.AZURE_IDENTITY_AVAILABLE', False): - with patch.object(openai, 'AzureOpenAI') as mock_client: + with patch( + "presidio_analyzer.predefined_recognizers.third_party.azure_openai_provider.AZURE_IDENTITY_AVAILABLE", + False, + ): + with patch.object(openai, "AzureOpenAI") as mock_client: with pytest.raises(ImportError, match="azure-identity is required"): AzureOpenAILanguageModel( model_id="gpt-4o", - azure_endpoint="https://test.openai.azure.com/" + azure_endpoint="https://test.openai.azure.com/", # No API key, so should try managed identity ) @@ -378,7 +418,7 @@ class TestAzureOpenAILangExtractRecognizerParameterConfiguration: def test_when_no_config_params_then_uses_defaults(self, mock_langextract, tmp_path): """Test that default extract params are used when not in config.""" import yaml - + config = { "lm_recognizer": { "supported_entities": ["PERSON"], @@ -389,28 +429,30 @@ def test_when_no_config_params_then_uses_defaults(self, mock_langextract, tmp_pa "entity_mappings": {"person": "PERSON"}, "model": { "model_id": "gpt-4o", - } - } + }, + }, } - + config_file = tmp_path / "test_config.yaml" - with open(config_file, 'w') as f: + with open(config_file, "w") as f: yaml.dump(config, f) recognizer = AzureOpenAILangExtractRecognizer( config_path=str(config_file), azure_endpoint="https://test.openai.azure.com/", - api_key="test-key" + api_key="test-key", ) - + # Verify Azure defaults are set (different from Ollama) assert recognizer._extract_params["fence_output"] is True assert recognizer._extract_params["use_schema_constraints"] is False - def test_when_config_has_params_then_overrides_defaults(self, mock_langextract, tmp_path): + def test_when_config_has_params_then_overrides_defaults( + self, mock_langextract, tmp_path + ): """Test that config values override defaults.""" import yaml - + config = { "lm_recognizer": { "supported_entities": ["PERSON"], @@ -423,20 +465,20 @@ def test_when_config_has_params_then_overrides_defaults(self, mock_langextract, "model_id": "gpt-4o", "fence_output": False, # Override default "use_schema_constraints": True, # Override default - } - } + }, + }, } - + config_file = tmp_path / "test_config.yaml" - with open(config_file, 'w') as f: + with open(config_file, "w") as f: yaml.dump(config, f) recognizer = AzureOpenAILangExtractRecognizer( config_path=str(config_file), azure_endpoint="https://test.openai.azure.com/", - api_key="test-key" + api_key="test-key", ) - + # Verify config values override defaults assert recognizer._extract_params["fence_output"] is False assert recognizer._extract_params["use_schema_constraints"] is True @@ -444,7 +486,7 @@ def test_when_config_has_params_then_overrides_defaults(self, mock_langextract, def test_when_analyze_called_then_params_passed_to_langextract(self, tmp_path): """Test that configured params are passed to langextract.extract().""" import yaml - + config = { "lm_recognizer": { "supported_entities": ["PERSON"], @@ -456,22 +498,22 @@ def test_when_analyze_called_then_params_passed_to_langextract(self, tmp_path): "model": { "model_id": "gpt-4o", "fence_output": False, - } - } + }, + }, } - + config_file = tmp_path / "test_config.yaml" - with open(config_file, 'w') as f: + with open(config_file, "w") as f: yaml.dump(config, f) recognizer = AzureOpenAILangExtractRecognizer( config_path=str(config_file), azure_endpoint="https://test.openai.azure.com/", - api_key="test-key" + api_key="test-key", ) text = "My name is John Doe" - + mock_extraction = MagicMock() mock_extraction.extraction_class = "person" mock_extraction.extraction_text = "John Doe" @@ -482,21 +524,23 @@ def test_when_analyze_called_then_params_passed_to_langextract(self, tmp_path): mock_result = MagicMock() mock_result.extractions = [mock_extraction] - with patch('langextract.extract', return_value=mock_result) as mock_extract: + with patch("langextract.extract", return_value=mock_result) as mock_extract: recognizer.analyze(text) - + # Verify extract was called assert mock_extract.called call_kwargs = mock_extract.call_args[1] - + # Verify extract params were passed assert call_kwargs["fence_output"] is False assert call_kwargs["use_schema_constraints"] is False - + # Verify Azure-specific provider params assert call_kwargs["model_id"] == "azure:gpt-4o" assert "language_model_params" in call_kwargs - assert call_kwargs["language_model_params"]["azure_endpoint"] == "https://test.openai.azure.com/" + assert ( + call_kwargs["language_model_params"]["azure_endpoint"] + == "https://test.openai.azure.com/" + ) assert call_kwargs["language_model_params"]["azure_deployment"] == "gpt-4o" assert call_kwargs["language_model_params"]["api_key"] == "test-key" - diff --git a/presidio-analyzer/tests/test_base_chunker.py b/presidio-analyzer/tests/test_base_chunker.py index 189fff1134..f2bab533ca 100644 --- a/presidio-analyzer/tests/test_base_chunker.py +++ b/presidio-analyzer/tests/test_base_chunker.py @@ -1,4 +1,5 @@ """Tests for BaseTextChunker methods.""" + import pytest from presidio_analyzer import RecognizerResult @@ -29,7 +30,9 @@ def predict_func(chunk): if "Jane" in chunk: idx = chunk.index("Jane") return [ - RecognizerResult(entity_type="PERSON", start=idx, end=idx + 4, score=0.9) + RecognizerResult( + entity_type="PERSON", start=idx, end=idx + 4, score=0.9 + ) ] return [] diff --git a/presidio-analyzer/tests/test_basic_langextract_recognizer.py b/presidio-analyzer/tests/test_basic_langextract_recognizer.py index 1de766ea06..bb3244c068 100644 --- a/presidio-analyzer/tests/test_basic_langextract_recognizer.py +++ b/presidio-analyzer/tests/test_basic_langextract_recognizer.py @@ -1,4 +1,5 @@ """Tests for LangExtract recognizer hierarchy using mocks.""" + import pytest import urllib.error from unittest.mock import Mock, patch, MagicMock @@ -12,7 +13,7 @@ def create_test_config( temperature=0.0, min_score=0.5, labels_to_ignore=None, - enable_generic_consolidation=True + enable_generic_consolidation=True, ): """Create test config.""" if supported_entities is None: @@ -21,7 +22,7 @@ def create_test_config( entity_mappings = {"person": "PERSON", "email": "EMAIL_ADDRESS"} if labels_to_ignore is None: labels_to_ignore = [] - + return { "lm_recognizer": { "supported_entities": supported_entities, @@ -39,15 +40,13 @@ def create_test_config( "kwargs": { "model_url": model_url, }, - "extract_params": { - }, - "language_model_params": { - } + "extract_params": {}, + "language_model_params": {}, }, "temperature": temperature, }, "entity_mappings": entity_mappings, - } + }, } @@ -56,64 +55,77 @@ class TestBasicLangExtractRecognizerInitialization: def test_when_langextract_not_installed_then_raises_import_error(self): """Test that ImportError is raised when langextract is not installed.""" - with patch( - 'presidio_analyzer.llm_utils.langextract_helper.lx', - None - ): - from presidio_analyzer.predefined_recognizers.third_party.basic_langextract_recognizer import BasicLangExtractRecognizer + with patch("presidio_analyzer.llm_utils.langextract_helper.lx", None): + from presidio_analyzer.predefined_recognizers.third_party.basic_langextract_recognizer import ( + BasicLangExtractRecognizer, + ) + with pytest.raises(ImportError, match="LangExtract is not installed"): BasicLangExtractRecognizer() def test_when_initialized_with_mocked_ollama_then_succeeds(self, tmp_path): """Test BasicLangExtractRecognizer initialization.""" import yaml - + config = create_test_config( supported_entities=["PERSON", "EMAIL_ADDRESS"], entity_mappings={"person": "PERSON", "email": "EMAIL_ADDRESS"}, model_id="qwen2.5:1.5b", model_url="http://localhost:11434", temperature=0.0, - min_score=0.5 + min_score=0.5, ) config_file = tmp_path / "test_config.yaml" - with open(config_file, 'w') as f: + with open(config_file, "w") as f: yaml.dump(config, f) - with patch('presidio_analyzer.llm_utils.langextract_helper.lx', - return_value=Mock()): - - from presidio_analyzer.predefined_recognizers.third_party.basic_langextract_recognizer import BasicLangExtractRecognizer + with patch( + "presidio_analyzer.llm_utils.langextract_helper.lx", return_value=Mock() + ): + + from presidio_analyzer.predefined_recognizers.third_party.basic_langextract_recognizer import ( + BasicLangExtractRecognizer, + ) + recognizer = BasicLangExtractRecognizer(config_path=str(config_file)) - + # Verify initialization assert recognizer.name == "BasicLangExtractRecognizer" assert recognizer.model_id == "qwen2.5:1.5b" - assert len(recognizer.supported_entities) == 3 # PERSON, EMAIL_ADDRESS, GENERIC_PII_ENTITY + assert ( + len(recognizer.supported_entities) == 3 + ) # PERSON, EMAIL_ADDRESS, GENERIC_PII_ENTITY assert "PERSON" in recognizer.supported_entities assert "EMAIL_ADDRESS" in recognizer.supported_entities assert "GENERIC_PII_ENTITY" in recognizer.supported_entities - + # Verify inheritance hierarchy from presidio_analyzer.lm_recognizer import LMRecognizer - from presidio_analyzer.predefined_recognizers.third_party.langextract_recognizer import LangExtractRecognizer + from presidio_analyzer.predefined_recognizers.third_party.langextract_recognizer import ( + LangExtractRecognizer, + ) + assert isinstance(recognizer, BasicLangExtractRecognizer) assert isinstance(recognizer, LangExtractRecognizer) assert isinstance(recognizer, LMRecognizer) def test_when_config_file_missing_then_raises_file_not_found_error(self): """Test FileNotFoundError when config file doesn't exist.""" - with patch('presidio_analyzer.llm_utils.langextract_helper.lx', - return_value=Mock()): - from presidio_analyzer.predefined_recognizers.third_party.basic_langextract_recognizer import BasicLangExtractRecognizer + with patch( + "presidio_analyzer.llm_utils.langextract_helper.lx", return_value=Mock() + ): + from presidio_analyzer.predefined_recognizers.third_party.basic_langextract_recognizer import ( + BasicLangExtractRecognizer, + ) + with pytest.raises(FileNotFoundError, match="File not found"): BasicLangExtractRecognizer(config_path="/nonexistent/path.yaml") def test_when_model_section_missing_then_raises_value_error(self, tmp_path): """Test ValueError when config missing 'langextract.model' section.""" import yaml - + config = { "lm_recognizer": { "supported_entities": ["PERSON"], @@ -126,23 +138,29 @@ def test_when_model_section_missing_then_raises_value_error(self, tmp_path): "examples_file": "presidio-analyzer/presidio_analyzer/conf/langextract_prompts/default_pii_phi_examples.yaml", "entity_mappings": {"person": "PERSON"}, # Missing 'model' section - } + }, } - + config_file = tmp_path / "bad_config.yaml" - with open(config_file, 'w') as f: + with open(config_file, "w") as f: yaml.dump(config, f) - - with patch('presidio_analyzer.llm_utils.langextract_helper.lx', - return_value=Mock()): - from presidio_analyzer.predefined_recognizers.third_party.basic_langextract_recognizer import BasicLangExtractRecognizer - with pytest.raises(ValueError, match="Configuration must contain 'langextract.model'"): + + with patch( + "presidio_analyzer.llm_utils.langextract_helper.lx", return_value=Mock() + ): + from presidio_analyzer.predefined_recognizers.third_party.basic_langextract_recognizer import ( + BasicLangExtractRecognizer, + ) + + with pytest.raises( + ValueError, match="Configuration must contain 'langextract.model'" + ): BasicLangExtractRecognizer(config_path=str(config_file)) def test_when_model_id_missing_then_raises_value_error(self, tmp_path): """Test ValueError when model_id is missing.""" import yaml - + config = { "lm_recognizer": { "supported_entities": ["PERSON"], @@ -160,23 +178,30 @@ def test_when_model_id_missing_then_raises_value_error(self, tmp_path): "name": "ollama", }, }, - } + }, } - + config_file = tmp_path / "bad_config.yaml" - with open(config_file, 'w') as f: + with open(config_file, "w") as f: yaml.dump(config, f) - - with patch('presidio_analyzer.llm_utils.langextract_helper.lx', - return_value=Mock()): - from presidio_analyzer.predefined_recognizers.third_party.basic_langextract_recognizer import BasicLangExtractRecognizer - with pytest.raises(ValueError, match="Configuration must contain 'langextract.model.model_id'"): + + with patch( + "presidio_analyzer.llm_utils.langextract_helper.lx", return_value=Mock() + ): + from presidio_analyzer.predefined_recognizers.third_party.basic_langextract_recognizer import ( + BasicLangExtractRecognizer, + ) + + with pytest.raises( + ValueError, + match="Configuration must contain 'langextract.model.model_id'", + ): BasicLangExtractRecognizer(config_path=str(config_file)) def test_when_provider_missing_then_raises_value_error(self, tmp_path): """Test ValueError when model_url is missing.""" import yaml - + config = { "lm_recognizer": { "supported_entities": ["PERSON"], @@ -191,18 +216,25 @@ def test_when_provider_missing_then_raises_value_error(self, tmp_path): "model": { "model_id": "qwen2.5:1.5b" # Missing provider - } - } + }, + }, } - + config_file = tmp_path / "bad_config.yaml" - with open(config_file, 'w') as f: + with open(config_file, "w") as f: yaml.dump(config, f) - - with patch('presidio_analyzer.llm_utils.langextract_helper.lx', - return_value=Mock()): - from presidio_analyzer.predefined_recognizers.third_party.basic_langextract_recognizer import BasicLangExtractRecognizer - with pytest.raises(ValueError, match="Configuration must contain 'langextract.model.provider.name'"): + + with patch( + "presidio_analyzer.llm_utils.langextract_helper.lx", return_value=Mock() + ): + from presidio_analyzer.predefined_recognizers.third_party.basic_langextract_recognizer import ( + BasicLangExtractRecognizer, + ) + + with pytest.raises( + ValueError, + match="Configuration must contain 'langextract.model.provider.name'", + ): BasicLangExtractRecognizer(config_path=str(config_file)) @@ -213,27 +245,31 @@ class TestBasicLangExtractRecognizerAnalyze: def mock_recognizer(self, tmp_path): """Fixture to create a mocked BasicLangExtractRecognizer.""" import yaml - + config = create_test_config( supported_entities=["PERSON", "EMAIL_ADDRESS", "PHONE_NUMBER"], entity_mappings={ "person": "PERSON", "email": "EMAIL_ADDRESS", - "phone": "PHONE_NUMBER" + "phone": "PHONE_NUMBER", }, model_id="qwen2.5:1.5b", model_url="http://localhost:11434", temperature=0.0, - min_score=0.5 + min_score=0.5, ) config_file = tmp_path / "test_config.yaml" - with open(config_file, 'w') as f: + with open(config_file, "w") as f: yaml.dump(config, f) - with patch('presidio_analyzer.llm_utils.langextract_helper.lx', - return_value=Mock()): - from presidio_analyzer.predefined_recognizers.third_party.basic_langextract_recognizer import BasicLangExtractRecognizer + with patch( + "presidio_analyzer.llm_utils.langextract_helper.lx", return_value=Mock() + ): + from presidio_analyzer.predefined_recognizers.third_party.basic_langextract_recognizer import ( + BasicLangExtractRecognizer, + ) + return BasicLangExtractRecognizer(config_path=str(config_file)) def test_when_text_contains_person_then_detects_entity(self, mock_recognizer): @@ -253,7 +289,7 @@ def test_when_text_contains_person_then_detects_entity(self, mock_recognizer): mock_result = Mock() mock_result.extractions = [mock_extraction] - with patch('langextract.extract', return_value=mock_result): + with patch("langextract.extract", return_value=mock_result): results = mock_recognizer.analyze(text, entities=["PERSON"]) assert len(results) == 1 @@ -262,7 +298,9 @@ def test_when_text_contains_person_then_detects_entity(self, mock_recognizer): assert results[0].end == 19 assert results[0].score == 0.95 # MATCH_EXACT score - def test_when_text_contains_multiple_entities_then_detects_all(self, mock_recognizer): + def test_when_text_contains_multiple_entities_then_detects_all( + self, mock_recognizer + ): """Test analysis detecting multiple entity types.""" text = "Contact John Doe at john@example.com or 555-1234" @@ -289,9 +327,13 @@ def test_when_text_contains_multiple_entities_then_detects_all(self, mock_recogn phone_extraction.attributes = {} mock_result = Mock() - mock_result.extractions = [person_extraction, email_extraction, phone_extraction] + mock_result.extractions = [ + person_extraction, + email_extraction, + phone_extraction, + ] - with patch('langextract.extract', return_value=mock_result): + with patch("langextract.extract", return_value=mock_result): results = mock_recognizer.analyze(text) assert len(results) == 3 @@ -311,7 +353,7 @@ def test_when_text_is_empty_then_returns_no_results(self, mock_recognizer): def test_when_no_entities_match_then_returns_empty_list(self, mock_recognizer): """Test analysis when requested entities don't match supported entities.""" text = "Some text here" - + # Request unsupported entity type results = mock_recognizer.analyze(text, entities=["CREDIT_CARD"]) assert len(results) == 0 @@ -319,25 +361,21 @@ def test_when_no_entities_match_then_returns_empty_list(self, mock_recognizer): def test_when_entities_requested_then_filters_results(self, mock_recognizer): """Test that analyze filters results based on requested entities.""" from presidio_analyzer import RecognizerResult - + text = "Contact John Doe at john@example.com" # Create RecognizerResult objects (what _call_llm returns) person_result = RecognizerResult( - entity_type="PERSON", - start=8, - end=16, - score=0.95 + entity_type="PERSON", start=8, end=16, score=0.95 ) - + email_result = RecognizerResult( - entity_type="EMAIL_ADDRESS", - start=20, - end=36, - score=0.95 + entity_type="EMAIL_ADDRESS", start=20, end=36, score=0.95 ) - with patch.object(mock_recognizer, '_call_llm', return_value=[person_result, email_result]): + with patch.object( + mock_recognizer, "_call_llm", return_value=[person_result, email_result] + ): # Request only PERSON entities - EMAIL_ADDRESS should be filtered out by analyze() results = mock_recognizer.analyze(text, entities=["PERSON"]) @@ -347,7 +385,9 @@ def test_when_entities_requested_then_filters_results(self, mock_recognizer): assert results[0].start == 8 assert results[0].end == 16 - def test_when_min_score_set_then_filters_low_confidence_results(self, mock_recognizer): + def test_when_min_score_set_then_filters_low_confidence_results( + self, mock_recognizer + ): """Test that results below min_score are filtered out.""" # Set min_score to 0.5 (default in config) text = "Some text" @@ -362,17 +402,19 @@ def test_when_min_score_set_then_filters_low_confidence_results(self, mock_recog mock_result = Mock() mock_result.extractions = [mock_extraction] - with patch('langextract.extract', return_value=mock_result): + with patch("langextract.extract", return_value=mock_result): results = mock_recognizer.analyze(text) # NOT_ALIGNED has score 0.60, which is above min_score 0.5 assert len(results) == 1 - def test_when_langextract_raises_exception_then_exception_propagates(self, mock_recognizer): + def test_when_langextract_raises_exception_then_exception_propagates( + self, mock_recognizer + ): """Test that exceptions from LangExtract propagate to caller.""" text = "Some text" - with patch('langextract.extract', side_effect=Exception("LangExtract error")): + with patch("langextract.extract", side_effect=Exception("LangExtract error")): with pytest.raises(Exception, match="LangExtract error"): mock_recognizer.analyze(text) @@ -392,7 +434,7 @@ def test_when_entity_has_no_mapping_and_consolidation_enabled_then_creates_gener mock_result = Mock() mock_result.extractions = [mock_extraction] - with patch('langextract.extract', return_value=mock_result): + with patch("langextract.extract", return_value=mock_result): results = mock_recognizer.analyze(text) # Unknown entity type should be consolidated to GENERIC_PII_ENTITY @@ -402,25 +444,31 @@ def test_when_entity_has_no_mapping_and_consolidation_enabled_then_creates_gener assert results[0].start == 0 assert results[0].end == 9 - def test_when_entity_has_no_mapping_and_consolidation_disabled_then_skips(self, tmp_path): + def test_when_entity_has_no_mapping_and_consolidation_disabled_then_skips( + self, tmp_path + ): """Test that unknown entities are skipped when consolidation is disabled.""" import yaml - + # Create config with consolidation disabled config = create_test_config( supported_entities=["PERSON", "EMAIL_ADDRESS"], entity_mappings={"person": "PERSON", "email": "EMAIL_ADDRESS"}, - enable_generic_consolidation=False + enable_generic_consolidation=False, ) config_file = tmp_path / "test_config.yaml" - with open(config_file, 'w') as f: + with open(config_file, "w") as f: yaml.dump(config, f) - with patch('presidio_analyzer.llm_utils.langextract_helper.lx', - return_value=Mock()): - - from presidio_analyzer.predefined_recognizers.third_party.basic_langextract_recognizer import BasicLangExtractRecognizer + with patch( + "presidio_analyzer.llm_utils.langextract_helper.lx", return_value=Mock() + ): + + from presidio_analyzer.predefined_recognizers.third_party.basic_langextract_recognizer import ( + BasicLangExtractRecognizer, + ) + recognizer = BasicLangExtractRecognizer(config_path=str(config_file)) text = "Some text" @@ -435,7 +483,7 @@ def test_when_entity_has_no_mapping_and_consolidation_disabled_then_skips(self, mock_result = Mock() mock_result.extractions = [mock_extraction] - with patch('langextract.extract', return_value=mock_result): + with patch("langextract.extract", return_value=mock_result): results = recognizer.analyze(text) # Unknown entity type should be skipped when consolidation is disabled @@ -448,19 +496,23 @@ class TestBasicLangExtractRecognizerParameterConfiguration: def test_when_no_config_params_then_uses_defaults(self, tmp_path): """Test that default extract params are used when not in config.""" import yaml - + config = create_test_config() # No extract params in config - should use defaults - + config_file = tmp_path / "test_config.yaml" - with open(config_file, 'w') as f: + with open(config_file, "w") as f: yaml.dump(config, f) - with patch('presidio_analyzer.llm_utils.langextract_helper.lx', - return_value=Mock()): - from presidio_analyzer.predefined_recognizers.third_party.basic_langextract_recognizer import BasicLangExtractRecognizer + with patch( + "presidio_analyzer.llm_utils.langextract_helper.lx", return_value=Mock() + ): + from presidio_analyzer.predefined_recognizers.third_party.basic_langextract_recognizer import ( + BasicLangExtractRecognizer, + ) + recognizer = BasicLangExtractRecognizer(config_path=str(config_file)) - + # Verify defaults are set assert recognizer._extract_params["max_char_buffer"] == 400 assert recognizer._extract_params["use_schema_constraints"] is False @@ -471,24 +523,38 @@ def test_when_no_config_params_then_uses_defaults(self, tmp_path): def test_when_config_has_params_then_overrides_defaults(self, tmp_path): """Test that config values override defaults.""" import yaml - + config = create_test_config() # Add custom values to override defaults - config["langextract"]["model"]["provider"]["extract_params"]["max_char_buffer"] = 1000 - config["langextract"]["model"]["provider"]["extract_params"]["use_schema_constraints"] = True - config["langextract"]["model"]["provider"]["extract_params"]["fence_output"] = True - config["langextract"]["model"]["provider"]["language_model_params"]["timeout"] = 120 - config["langextract"]["model"]["provider"]["language_model_params"]["num_ctx"] = 4096 - + config["langextract"]["model"]["provider"]["extract_params"][ + "max_char_buffer" + ] = 1000 + config["langextract"]["model"]["provider"]["extract_params"][ + "use_schema_constraints" + ] = True + config["langextract"]["model"]["provider"]["extract_params"][ + "fence_output" + ] = True + config["langextract"]["model"]["provider"]["language_model_params"][ + "timeout" + ] = 120 + config["langextract"]["model"]["provider"]["language_model_params"][ + "num_ctx" + ] = 4096 + config_file = tmp_path / "test_config.yaml" - with open(config_file, 'w') as f: + with open(config_file, "w") as f: yaml.dump(config, f) - with patch('presidio_analyzer.llm_utils.langextract_helper.lx', - return_value=Mock()): - from presidio_analyzer.predefined_recognizers.third_party.basic_langextract_recognizer import BasicLangExtractRecognizer + with patch( + "presidio_analyzer.llm_utils.langextract_helper.lx", return_value=Mock() + ): + from presidio_analyzer.predefined_recognizers.third_party.basic_langextract_recognizer import ( + BasicLangExtractRecognizer, + ) + recognizer = BasicLangExtractRecognizer(config_path=str(config_file)) - + # Verify config values override defaults assert recognizer._extract_params["max_char_buffer"] == 1000 assert recognizer._extract_params["use_schema_constraints"] is True @@ -499,25 +565,33 @@ def test_when_config_has_params_then_overrides_defaults(self, tmp_path): def test_when_partial_config_params_then_uses_defaults_for_missing(self, tmp_path): """Test that only some params can be overridden.""" import yaml - + config = create_test_config() # Override only some params - config["langextract"]["model"]["provider"]["extract_params"]["max_char_buffer"] = 500 - config["langextract"]["model"]["provider"]["language_model_params"]["timeout"] = 60 - + config["langextract"]["model"]["provider"]["extract_params"][ + "max_char_buffer" + ] = 500 + config["langextract"]["model"]["provider"]["language_model_params"][ + "timeout" + ] = 60 + config_file = tmp_path / "test_config.yaml" - with open(config_file, 'w') as f: + with open(config_file, "w") as f: yaml.dump(config, f) - with patch('presidio_analyzer.llm_utils.langextract_helper.lx', - return_value=Mock()): - from presidio_analyzer.predefined_recognizers.third_party.basic_langextract_recognizer import BasicLangExtractRecognizer + with patch( + "presidio_analyzer.llm_utils.langextract_helper.lx", return_value=Mock() + ): + from presidio_analyzer.predefined_recognizers.third_party.basic_langextract_recognizer import ( + BasicLangExtractRecognizer, + ) + recognizer = BasicLangExtractRecognizer(config_path=str(config_file)) - + # Verify overridden values assert recognizer._extract_params["max_char_buffer"] == 500 assert recognizer._language_model_params["timeout"] == 60 - + # Verify defaults for non-overridden params assert recognizer._extract_params["use_schema_constraints"] is False assert recognizer._extract_params["fence_output"] is False @@ -526,22 +600,30 @@ def test_when_partial_config_params_then_uses_defaults_for_missing(self, tmp_pat def test_when_analyze_called_then_params_passed_to_langextract(self, tmp_path): """Test that configured params are passed to langextract.extract().""" import yaml - + config = create_test_config() - config["langextract"]["model"]["provider"]["extract_params"]["max_char_buffer"] = 1500 - config["langextract"]["model"]["provider"]["language_model_params"]["timeout"] = 180 - + config["langextract"]["model"]["provider"]["extract_params"][ + "max_char_buffer" + ] = 1500 + config["langextract"]["model"]["provider"]["language_model_params"][ + "timeout" + ] = 180 + config_file = tmp_path / "test_config.yaml" - with open(config_file, 'w') as f: + with open(config_file, "w") as f: yaml.dump(config, f) - with patch('presidio_analyzer.llm_utils.langextract_helper.lx', - return_value=Mock()): - from presidio_analyzer.predefined_recognizers.third_party.basic_langextract_recognizer import BasicLangExtractRecognizer + with patch( + "presidio_analyzer.llm_utils.langextract_helper.lx", return_value=Mock() + ): + from presidio_analyzer.predefined_recognizers.third_party.basic_langextract_recognizer import ( + BasicLangExtractRecognizer, + ) + recognizer = BasicLangExtractRecognizer(config_path=str(config_file)) text = "My name is John Doe" - + mock_extraction = Mock() mock_extraction.extraction_class = "person" mock_extraction.extraction_text = "John Doe" @@ -552,9 +634,9 @@ def test_when_analyze_called_then_params_passed_to_langextract(self, tmp_path): mock_result = Mock() mock_result.extractions = [mock_extraction] - with patch('langextract.extract', return_value=mock_result) as mock_extract: + with patch("langextract.extract", return_value=mock_result) as mock_extract: recognizer.analyze(text) - + # Verify extract was called assert mock_extract.called call_kwargs = mock_extract.call_args[1] @@ -563,14 +645,16 @@ def test_when_analyze_called_then_params_passed_to_langextract(self, tmp_path): assert call_kwargs["max_char_buffer"] == 1500 assert call_kwargs["use_schema_constraints"] is False assert call_kwargs["fence_output"] is False - + # Verify language model params were passed assert "language_model_params" in call_kwargs assert call_kwargs["language_model_params"]["timeout"] == 180 assert call_kwargs["language_model_params"]["num_ctx"] == 8192 - + # Verify provider params assert call_kwargs["config"].model_id == "qwen2.5:1.5b" assert call_kwargs["config"].provider == "ollama" - assert call_kwargs["config"].provider_kwargs["model_url"] == "http://localhost:11434" - + assert ( + call_kwargs["config"].provider_kwargs["model_url"] + == "http://localhost:11434" + ) diff --git a/presidio-analyzer/tests/test_batch_analyzer_engine.py b/presidio-analyzer/tests/test_batch_analyzer_engine.py index 07acdaf943..d59cbf7c09 100644 --- a/presidio-analyzer/tests/test_batch_analyzer_engine.py +++ b/presidio-analyzer/tests/test_batch_analyzer_engine.py @@ -41,10 +41,11 @@ def test_analyze_iterator_returns_list_of_recognizer_results( assert result == expected_result # fmt: on -@pytest.mark.parametrize("n_process, batch_size", [(1,1), (4,2), (2,1)]) -def test_analyze_dict_one_value_per_key(batch_analyzer_engine_simple, - n_process, - batch_size): + +@pytest.mark.parametrize("n_process, batch_size", [(1, 1), (4, 2), (2, 1)]) +def test_analyze_dict_one_value_per_key( + batch_analyzer_engine_simple, n_process, batch_size +): d = { "url": "https://microsoft.com", @@ -53,10 +54,9 @@ def test_analyze_dict_one_value_per_key(batch_analyzer_engine_simple, "misc": "microsoft.com or (202)-555-1234", } - results = batch_analyzer_engine_simple.analyze_dict(input_dict=d, - language="en", - n_process=n_process, - batch_size=batch_size) + results = batch_analyzer_engine_simple.analyze_dict( + input_dict=d, language="en", n_process=n_process, batch_size=batch_size + ) results = list(results) # url @@ -273,17 +273,20 @@ def test_analyze_dict_with_nones_returns_empty_result(batch_analyzer_engine_simp assert not r -@pytest.mark.parametrize("n_process, batch_size", [(1,1), (4,2), (2,1)]) +@pytest.mark.parametrize("n_process, batch_size", [(1, 1), (4, 2), (2, 1)]) def test_batch_analyze_iterator_returns_list_of_recognizer_results( batch_analyzer_engine_simple, n_process, batch_size ): texts = ["My name is David", "Call me at 2352351232", "I was born at 1/5/1922"] - expected_output = [[], [RecognizerResult(entity_type="PHONE_NUMBER", start=11, end=21, score= 0.4)], []] + expected_output = [ + [], + [RecognizerResult(entity_type="PHONE_NUMBER", start=11, end=21, score=0.4)], + [], + ] - results = batch_analyzer_engine_simple.analyze_iterator(texts=texts, - language="en", - batch_size=batch_size, - n_process=n_process) + results = batch_analyzer_engine_simple.analyze_iterator( + texts=texts, language="en", batch_size=batch_size, n_process=n_process + ) assert len(results) == len(expected_output) for result, expected_result in zip(results, expected_output): diff --git a/presidio-analyzer/tests/test_certificate_number_recognizer.py b/presidio-analyzer/tests/test_certificate_number_recognizer.py index 59186936ec..b1aff3c366 100644 --- a/presidio-analyzer/tests/test_certificate_number_recognizer.py +++ b/presidio-analyzer/tests/test_certificate_number_recognizer.py @@ -1,7 +1,9 @@ import pytest from tests import assert_result -from presidio_analyzer.predefined_recognizers.country_specific.us import MedicalLicenseRecognizer +from presidio_analyzer.predefined_recognizers.country_specific.us import ( + MedicalLicenseRecognizer, +) @pytest.fixture(scope="module") diff --git a/presidio-analyzer/tests/test_character_based_text_chunker.py b/presidio-analyzer/tests/test_character_based_text_chunker.py index 5b0012eb55..98ec846454 100644 --- a/presidio-analyzer/tests/test_character_based_text_chunker.py +++ b/presidio-analyzer/tests/test_character_based_text_chunker.py @@ -64,7 +64,7 @@ def test_chunks_extend_to_word_boundary(self): # Verify chunks don't cut words in the middle for chunk in chunks: # Each chunk text should not start/end mid-word (except first/last) - assert text[chunk.start:chunk.end] == chunk.text + assert text[chunk.start : chunk.end] == chunk.text def test_offset_calculation_is_correct(self): """Test that chunk offsets map correctly to original text.""" @@ -74,7 +74,7 @@ def test_offset_calculation_is_correct(self): # Critical: offsets must point to correct positions for chunk in chunks: - assert text[chunk.start:chunk.end] == chunk.text + assert text[chunk.start : chunk.end] == chunk.text class TestCharacterBasedTextChunkerEdgeCases: @@ -99,7 +99,7 @@ def test_newline_boundary(self): # The chunk stops AT the boundary (space), not including it assert chunks[0].text == "Hello\nworld" assert chunks[0].end == 11 # Position of space - assert text[chunks[0].start:chunks[0].end] == chunks[0].text + assert text[chunks[0].start : chunks[0].end] == chunks[0].text def test_text_without_spaces_cjk(self): """Test chunking CJK text without spaces extends to end.""" @@ -135,7 +135,7 @@ def test_long_text_produces_multiple_chunks(self): assert len(chunks) > 1 # Verify all offsets are correct for chunk in chunks: - assert text[chunk.start:chunk.end] == chunk.text + assert text[chunk.start : chunk.end] == chunk.text def test_overlap_captures_entity_at_boundary(self): """Test that overlap prevents missing entities at chunk boundaries.""" diff --git a/presidio-analyzer/tests/test_config_loader.py b/presidio-analyzer/tests/test_config_loader.py index f9a30a3ad8..194e755f5b 100644 --- a/presidio-analyzer/tests/test_config_loader.py +++ b/presidio-analyzer/tests/test_config_loader.py @@ -1,4 +1,5 @@ """Tests for llm_utils.config_loader module.""" + import pytest import tempfile from pathlib import Path @@ -15,7 +16,7 @@ class TestLoadYamlFile: def test_when_config_file_exists_then_loads_yaml(self): """Test loading a valid YAML configuration file.""" - with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as f: + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: f.write(""" lm_recognizer: supported_entities: ["PERSON", "EMAIL"] @@ -40,7 +41,7 @@ def test_when_config_file_missing_then_raises_file_not_found_error(self): def test_when_config_has_invalid_yaml_then_raises_value_error(self): """Test that invalid YAML raises ValueError.""" - with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as f: + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: f.write("invalid: yaml: content:\n - [unclosed") config_path = f.name @@ -52,7 +53,7 @@ def test_when_config_has_invalid_yaml_then_raises_value_error(self): def test_when_config_is_empty_then_returns_none(self): """Test loading an empty YAML file.""" - with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as f: + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: f.write("") config_path = f.name @@ -65,7 +66,7 @@ def test_when_config_is_empty_then_returns_none(self): def test_when_config_has_multiple_sections_then_loads_all(self): """Test loading config with multiple sections.""" - with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as f: + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: f.write(""" lm_recognizer: supported_entities: ["PERSON"] @@ -93,11 +94,7 @@ class TestExtractLmConfig: def test_when_lm_recognizer_section_exists_then_extracts_with_defaults(self): """Test extracting lm_recognizer config applies defaults for missing fields.""" - full_config = { - "lm_recognizer": { - "supported_entities": ["PERSON", "EMAIL"] - } - } + full_config = {"lm_recognizer": {"supported_entities": ["PERSON", "EMAIL"]}} lm_config = extract_lm_config(full_config) @@ -115,7 +112,7 @@ def test_when_all_fields_present_then_uses_provided_values(self): "supported_entities": ["PERSON"], "min_score": 0.8, "labels_to_ignore": ["system", "metadata"], - "enable_generic_consolidation": False + "enable_generic_consolidation": False, } } @@ -143,7 +140,7 @@ def test_when_partial_config_then_merges_with_defaults(self): full_config = { "lm_recognizer": { "supported_entities": ["PHONE"], - "min_score": 0.6 + "min_score": 0.6, # labels_to_ignore and enable_generic_consolidation missing } } @@ -166,7 +163,7 @@ def test_when_provider_key_exists_then_extracts_model_config(self): "model": { "model_id": "qwen2.5:1.5b", "temperature": 0.0, - "model_url": "http://localhost:11434" + "model_url": "http://localhost:11434", } } } @@ -180,37 +177,29 @@ def test_when_provider_key_exists_then_extracts_model_config(self): def test_when_provider_key_missing_then_raises_value_error(self): """Test that missing provider key raises ValueError.""" - config = { - "other_provider": { - "model": {"model_id": "some-model"} - } - } + config = {"other_provider": {"model": {"model_id": "some-model"}}} - with pytest.raises(ValueError, match="Configuration must contain 'langextract'"): + with pytest.raises( + ValueError, match="Configuration must contain 'langextract'" + ): get_model_config(config, "langextract") def test_when_model_section_missing_then_raises_value_error(self): """Test that missing model section raises ValueError.""" - config = { - "langextract": { - "other_section": {} - } - } + config = {"langextract": {"other_section": {}}} - with pytest.raises(ValueError, match="Configuration must contain 'langextract.model'"): + with pytest.raises( + ValueError, match="Configuration must contain 'langextract.model'" + ): get_model_config(config, "langextract") def test_when_model_id_missing_then_raises_value_error(self): """Test that missing model_id raises ValueError.""" - config = { - "langextract": { - "model": { - "temperature": 0.0 - } - } - } + config = {"langextract": {"model": {"temperature": 0.0}}} - with pytest.raises(ValueError, match="Configuration must contain 'langextract.model.model_id'"): + with pytest.raises( + ValueError, match="Configuration must contain 'langextract.model.model_id'" + ): get_model_config(config, "langextract") def test_when_model_config_has_extra_params_then_includes_all(self): @@ -221,7 +210,7 @@ def test_when_model_config_has_extra_params_then_includes_all(self): "model_id": "qwen2.5:1.5b", "temperature": 0.1, "model_url": "http://localhost:11434", - "custom_param": "custom_value" + "custom_param": "custom_value", } } } @@ -239,7 +228,7 @@ class TestIntegration: def test_when_loading_full_config_workflow_then_extracts_correctly(self): """Test complete workflow: load YAML → extract lm config → get model config.""" - with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as f: + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: f.write(""" lm_recognizer: supported_entities: ["PERSON", "EMAIL", "PHONE"] @@ -258,19 +247,19 @@ def test_when_loading_full_config_workflow_then_extracts_correctly(self): try: # Step 1: Load YAML full_config = load_yaml_file(config_path) - + # Step 2: Extract lm_recognizer config lm_config = extract_lm_config(full_config) assert lm_config["supported_entities"] == ["PERSON", "EMAIL", "PHONE"] assert lm_config["min_score"] == 0.75 assert lm_config["labels_to_ignore"] == ["system"] assert lm_config["enable_generic_consolidation"] is False - + # Step 3: Get model config model_config = get_model_config(full_config, "langextract") assert model_config["model_id"] == "qwen2.5:1.5b" assert model_config["temperature"] == 0.0 - + finally: Path(config_path).unlink() @@ -280,9 +269,7 @@ class TestResolveConfigPath: def test_when_conf_relative_path_then_resolves_to_package_conf(self): """Test that paths relative to conf/ resolve via the package directory.""" - resolved = resolve_config_path( - "langextract_prompts/default_pii_phi_prompt.j2" - ) + resolved = resolve_config_path("langextract_prompts/default_pii_phi_prompt.j2") assert resolved.exists() assert "conf" in str(resolved) assert resolved.name == "default_pii_phi_prompt.j2" diff --git a/presidio-analyzer/tests/test_configuration_validator.py b/presidio-analyzer/tests/test_configuration_validator.py index ad5ed5687a..7ecb957f25 100644 --- a/presidio-analyzer/tests/test_configuration_validator.py +++ b/presidio-analyzer/tests/test_configuration_validator.py @@ -1,11 +1,12 @@ """Tests for the Pydantic-based validation system using existing adapted classes.""" + import pytest from presidio_analyzer.input_validation import ConfigurationValidator - # ========== Language Code Validation Tests ========== + def test_validate_language_codes_valid(): """Test valid language codes.""" valid_languages = ["en", "es", "fr", "de"] @@ -38,6 +39,7 @@ def test_validate_language_codes_mixed_invalid(): # ========== File Path Validation Tests ========== + def test_file_path_validation_success(tmp_path): """Test file path validation with existing file.""" test_file = tmp_path / "test.yaml" @@ -77,6 +79,7 @@ def test_file_path_validation_directory(tmp_path): # ========== Score Threshold Validation Tests ========== + def test_validate_score_threshold_valid(): """Test valid score thresholds.""" valid_thresholds = [0.0, 0.5, 1.0, 0.25, 0.75] @@ -108,13 +111,12 @@ def test_validate_score_threshold_way_above(): # ========== NLP Configuration Validation Tests ========== + def test_configuration_validator_nlp_config_valid(): """Test ConfigurationValidator accepts valid NLP validation.""" valid_config = { "nlp_engine_name": "spacy", - "models": [ - {"lang_code": "en", "model_name": "en_core_web_lg"} - ] + "models": [{"lang_code": "en", "model_name": "en_core_web_lg"}], } validated = ConfigurationValidator.validate_nlp_configuration(valid_config) @@ -127,8 +129,8 @@ def test_nlp_config_multiple_models(): "nlp_engine_name": "spacy", "models": [ {"lang_code": "en", "model_name": "en_core_web_lg"}, - {"lang_code": "es", "model_name": "es_core_news_lg"} - ] + {"lang_code": "es", "model_name": "es_core_news_lg"}, + ], } validated = ConfigurationValidator.validate_nlp_configuration(valid_config) @@ -150,9 +152,7 @@ def test_configuration_validator_nlp_config_missing_fields(): def test_nlp_config_missing_nlp_engine_name(): """Test NLP config missing nlp_engine_name.""" - invalid_config = { - "models": [{"lang_code": "en", "model_name": "en_core_web_lg"}] - } + invalid_config = {"models": [{"lang_code": "en", "model_name": "en_core_web_lg"}]} with pytest.raises(ValueError) as exc_info: ConfigurationValidator.validate_nlp_configuration(invalid_config) @@ -170,7 +170,7 @@ def test_nlp_config_models_not_list(): """Test NLP configuration with models not as list.""" invalid_config = { "nlp_engine_name": "spacy", - "models": {"lang_code": "en", "model_name": "en_core_web_lg"} + "models": {"lang_code": "en", "model_name": "en_core_web_lg"}, } with pytest.raises(ValueError) as exc_info: @@ -180,10 +180,7 @@ def test_nlp_config_models_not_list(): def test_nlp_config_models_empty_list(): """Test NLP configuration with empty models list.""" - invalid_config = { - "nlp_engine_name": "spacy", - "models": [] - } + invalid_config = {"nlp_engine_name": "spacy", "models": []} with pytest.raises(ValueError) as exc_info: ConfigurationValidator.validate_nlp_configuration(invalid_config) @@ -192,10 +189,7 @@ def test_nlp_config_models_empty_list(): def test_nlp_config_model_not_dict(): """Test NLP configuration with model that is not a dict.""" - invalid_config = { - "nlp_engine_name": "spacy", - "models": ["en_core_web_lg"] - } + invalid_config = {"nlp_engine_name": "spacy", "models": ["en_core_web_lg"]} with pytest.raises(ValueError) as exc_info: ConfigurationValidator.validate_nlp_configuration(invalid_config) @@ -206,7 +200,7 @@ def test_nlp_config_model_missing_lang_code(): """Test NLP configuration with model missing lang_code.""" invalid_config = { "nlp_engine_name": "spacy", - "models": [{"model_name": "en_core_web_lg"}] + "models": [{"model_name": "en_core_web_lg"}], } with pytest.raises(ValueError) as exc_info: @@ -216,10 +210,7 @@ def test_nlp_config_model_missing_lang_code(): def test_nlp_config_model_missing_model_name(): """Test NLP configuration with model missing model_name.""" - invalid_config = { - "nlp_engine_name": "spacy", - "models": [{"lang_code": "en"}] - } + invalid_config = {"nlp_engine_name": "spacy", "models": [{"lang_code": "en"}]} with pytest.raises(ValueError) as exc_info: ConfigurationValidator.validate_nlp_configuration(invalid_config) @@ -228,6 +219,7 @@ def test_nlp_config_model_missing_model_name(): # ========== Recognizer Registry Configuration Tests ========== + def test_recognizer_registry_valid_custom_recognizer(): """Test valid recognizer registry configuration with custom recognizer.""" valid_config = { @@ -237,18 +229,14 @@ def test_recognizer_registry_valid_custom_recognizer(): "name": "CustomRecognizer", "type": "custom", "supported_entity": "CUSTOM_ENTITY", - "patterns": [ - { - "name": "pattern1", - "regex": "test", - "score": 0.5 - } - ] + "patterns": [{"name": "pattern1", "regex": "test", "score": 0.5}], } - ] + ], } - result = ConfigurationValidator.validate_recognizer_registry_configuration(valid_config) + result = ConfigurationValidator.validate_recognizer_registry_configuration( + valid_config + ) assert result is not None assert "recognizers" in result @@ -257,27 +245,23 @@ def test_recognizer_registry_valid_predefined_recognizer(): """Test valid recognizer registry configuration with predefined recognizer.""" valid_config = { "supported_languages": ["en"], - "recognizers": [ - { - "name": "CreditCardRecognizer", - "type": "predefined" - } - ] + "recognizers": [{"name": "CreditCardRecognizer", "type": "predefined"}], } - result = ConfigurationValidator.validate_recognizer_registry_configuration(valid_config) + result = ConfigurationValidator.validate_recognizer_registry_configuration( + valid_config + ) assert result is not None def test_recognizer_registry_empty_recognizers_list(): """Test recognizer registry with empty recognizers list.""" - invalid_config = { - "supported_languages": ["en"], - "recognizers": [] - } + invalid_config = {"supported_languages": ["en"], "recognizers": []} with pytest.raises(ValueError) as exc_info: - ConfigurationValidator.validate_recognizer_registry_configuration(invalid_config) + ConfigurationValidator.validate_recognizer_registry_configuration( + invalid_config + ) assert "Invalid recognizer registry configuration" in str(exc_info.value) @@ -286,23 +270,21 @@ def test_configuration_validator_recognizer_registry_unknown_keys(): invalid_config = { "supported_languages": ["en"], "global_regex_flags": 26, - "recognizers": [ - { - "name": "CreditCardRecognizer", - "type": "predefined" - } - ], + "recognizers": [{"name": "CreditCardRecognizer", "type": "predefined"}], "invalid_field": "value", - "typo_key": 456 + "typo_key": 456, } with pytest.raises(ValueError) as exc_info: - ConfigurationValidator.validate_recognizer_registry_configuration(invalid_config) + ConfigurationValidator.validate_recognizer_registry_configuration( + invalid_config + ) assert "Invalid recognizer registry configuration" in str(exc_info.value) # ========== Analyzer Configuration Tests ========== + def test_configuration_validator_analyzer_config_valid(): """Test ConfigurationValidator accepts valid analyzer validation.""" valid_config = { @@ -310,8 +292,8 @@ def test_configuration_validator_analyzer_config_valid(): "default_score_threshold": 0.5, "nlp_configuration": { "nlp_engine_name": "spacy", - "models": [{"lang_code": "en", "model_name": "en_core_web_lg"}] - } + "models": [{"lang_code": "en", "model_name": "en_core_web_lg"}], + }, } validated = ConfigurationValidator.validate_analyzer_configuration(valid_config) @@ -320,9 +302,7 @@ def test_configuration_validator_analyzer_config_valid(): def test_analyzer_config_minimal(): """Test minimal valid analyzer configuration.""" - valid_config = { - "supported_languages": ["en"] - } + valid_config = {"supported_languages": ["en"]} validated = ConfigurationValidator.validate_analyzer_configuration(valid_config) assert validated == valid_config @@ -334,13 +314,8 @@ def test_analyzer_config_with_recognizer_registry(): "supported_languages": ["en"], "recognizer_registry": { "supported_languages": ["en"], - "recognizers": [ - { - "name": "CreditCardRecognizer", - "type": "predefined" - } - ] - } + "recognizers": [{"name": "CreditCardRecognizer", "type": "predefined"}], + }, } validated = ConfigurationValidator.validate_analyzer_configuration(valid_config) @@ -351,7 +326,7 @@ def test_configuration_validator_analyzer_config_invalid_threshold(): """Test ConfigurationValidator rejects invalid score threshold.""" invalid_config = { "supported_languages": ["en"], - "default_score_threshold": 1.5 # Invalid: > 1.0 + "default_score_threshold": 1.5, # Invalid: > 1.0 } with pytest.raises(ValueError) as exc_info: @@ -373,7 +348,7 @@ def test_configuration_validator_analyzer_config_unknown_keys(): "supported_languages": ["en"], "default_score_threshold": 0.5, "unknown_key": "some_value", - "another_typo": 123 + "another_typo": 123, } with pytest.raises(ValueError) as exc_info: @@ -383,9 +358,7 @@ def test_configuration_validator_analyzer_config_unknown_keys(): def test_analyzer_config_invalid_languages(): """Test analyzer configuration with invalid language codes.""" - invalid_config = { - "supported_languages": ["invalid_lang"] - } + invalid_config = {"supported_languages": ["invalid_lang"]} with pytest.raises(ValueError) as exc_info: ConfigurationValidator.validate_analyzer_configuration(invalid_config) @@ -399,7 +372,7 @@ def test_analyzer_config_invalid_nlp_nested(): "nlp_configuration": { "nlp_engine_name": "spacy" # Missing models - } + }, } with pytest.raises(ValueError) as exc_info: @@ -411,9 +384,7 @@ def test_analyzer_config_invalid_recognizer_registry_nested(): """Test analyzer configuration with invalid nested recognizer registry.""" invalid_config = { "supported_languages": ["en"], - "recognizer_registry": { - "recognizers": [] # Empty list not allowed - } + "recognizer_registry": {"recognizers": []}, # Empty list not allowed } with pytest.raises(ValueError) as exc_info: @@ -424,18 +395,12 @@ def test_analyzer_config_invalid_recognizer_registry_nested(): def test_analyzer_config_threshold_at_boundaries(): """Test analyzer configuration with threshold at boundaries.""" # Test 0.0 - config_zero = { - "supported_languages": ["en"], - "default_score_threshold": 0.0 - } + config_zero = {"supported_languages": ["en"], "default_score_threshold": 0.0} validated = ConfigurationValidator.validate_analyzer_configuration(config_zero) assert validated["default_score_threshold"] == 0.0 # Test 1.0 - config_one = { - "supported_languages": ["en"], - "default_score_threshold": 1.0 - } + config_one = {"supported_languages": ["en"], "default_score_threshold": 1.0} validated = ConfigurationValidator.validate_analyzer_configuration(config_one) assert validated["default_score_threshold"] == 1.0 @@ -449,21 +414,15 @@ def test_analyzer_config_all_fields(): "nlp_engine_name": "spacy", "models": [ {"lang_code": "en", "model_name": "en_core_web_lg"}, - {"lang_code": "es", "model_name": "es_core_news_lg"} - ] + {"lang_code": "es", "model_name": "es_core_news_lg"}, + ], }, "recognizer_registry": { "supported_languages": ["en"], "global_regex_flags": 26, - "recognizers": [ - { - "name": "CreditCardRecognizer", - "type": "predefined" - } - ] - } + "recognizers": [{"name": "CreditCardRecognizer", "type": "predefined"}], + }, } validated = ConfigurationValidator.validate_analyzer_configuration(valid_config) assert validated == valid_config - diff --git a/presidio-analyzer/tests/test_context_support.py b/presidio-analyzer/tests/test_context_support.py index 7fdd7874a4..c23bfe1607 100644 --- a/presidio-analyzer/tests/test_context_support.py +++ b/presidio-analyzer/tests/test_context_support.py @@ -17,7 +17,6 @@ SgFinRecognizer, InPanRecognizer, InPassportRecognizer, - ) from presidio_analyzer.nlp_engine import NlpArtifacts from presidio_analyzer.context_aware_enhancers import LemmaContextAwareEnhancer @@ -192,7 +191,9 @@ def test_when_text_with_context_then_improves_score( assert res_wo.score <= res_w.score -def test_when_context_custom_recognizer_then_succeed(spacy_nlp_engine, mock_nlp_artifacts): +def test_when_context_custom_recognizer_then_succeed( + spacy_nlp_engine, mock_nlp_artifacts +): """This test checks that a custom recognizer is also enhanced by context. However this test also verifies a specific case in which the pattern also diff --git a/presidio-analyzer/tests/test_date_recognizer.py b/presidio-analyzer/tests/test_date_recognizer.py index 62a6c700d9..0b2f058857 100644 --- a/presidio-analyzer/tests/test_date_recognizer.py +++ b/presidio-analyzer/tests/test_date_recognizer.py @@ -55,7 +55,7 @@ def entities(): ("Today is5/21and it's sunny", 0, (), (),), ("Today is,5/21,and it's sunny", 1, ((9, 13),), ((0.05, 0.15),),), ("5-20-2021 is today.", 1, ((0, 9),), ((0.6, 0.81),),), - ("5-20-2021", 1, ((0, 9),), ((0.6, 0.81),),), + ("5-20-2021", 1, ((0, 9),), ((0.6, 0.81),),), # fmt: on ], ) diff --git a/presidio-analyzer/tests/test_de_bsnr_recognizer.py b/presidio-analyzer/tests/test_de_bsnr_recognizer.py index 804df45bd9..9a543e9b4a 100644 --- a/presidio-analyzer/tests/test_de_bsnr_recognizer.py +++ b/presidio-analyzer/tests/test_de_bsnr_recognizer.py @@ -18,6 +18,7 @@ 711234567 – 71 Bayern 351234567 – 35 Krankenhäuser (Anlage 8 BMV-Ä) """ + import pytest from tests import assert_result @@ -68,8 +69,12 @@ def entities(): ], ) def test_when_all_de_bsnr_numbers_then_succeed( - text, expected_len, expected_positions, expected_score, - recognizer, entities, + text, + expected_len, + expected_positions, + expected_score, + recognizer, + entities, ): results = recognizer.analyze(text, entities) assert len(results) == expected_len @@ -81,21 +86,19 @@ def test_when_all_de_bsnr_numbers_then_succeed( "number, expected", [ # Structurally valid — always None (no public checksum exists) - ("021234568", None), # whitelisted prefix 02 Hamburg - ("521234567", None), # whitelisted prefix 52 Baden-Württemberg - ("711234567", None), # whitelisted prefix 71 Bayern - ("351234567", None), # whitelisted prefix 35 Krankenhäuser - ("741234567", None), # whitelisted prefix 74 KBV - ("991234567", None), # non-whitelisted but structurally valid + ("021234568", None), # whitelisted prefix 02 Hamburg + ("521234567", None), # whitelisted prefix 52 Baden-Württemberg + ("711234567", None), # whitelisted prefix 71 Bayern + ("351234567", None), # whitelisted prefix 35 Krankenhäuser + ("741234567", None), # whitelisted prefix 74 KBV + ("991234567", None), # non-whitelisted but structurally valid ("051234567", None), # Structurally invalid — False ("000000000", False), - ("02123456", False), # 8 digits - ("0212345689", False), # 10 digits + ("02123456", False), # 8 digits + ("0212345689", False), # 10 digits ("02123456A", False), # non-numeric ], ) -def test_when_de_bsnr_validated_then_result_is_correct( - number, expected, recognizer -): +def test_when_de_bsnr_validated_then_result_is_correct(number, expected, recognizer): assert recognizer.validate_result(number) == expected diff --git a/presidio-analyzer/tests/test_de_fuehrerschein_recognizer.py b/presidio-analyzer/tests/test_de_fuehrerschein_recognizer.py index 04e9daa618..291d12f874 100644 --- a/presidio-analyzer/tests/test_de_fuehrerschein_recognizer.py +++ b/presidio-analyzer/tests/test_de_fuehrerschein_recognizer.py @@ -21,6 +21,7 @@ HH98765432C – issuing authority "HH" (Hamburg) KO12345678X – issuing authority "KO" (Koblenz) """ + import pytest from presidio_analyzer.predefined_recognizers import DeFuehrerscheinRecognizer diff --git a/presidio-analyzer/tests/test_de_handelsregister_recognizer.py b/presidio-analyzer/tests/test_de_handelsregister_recognizer.py index aef3101244..9ddba91d40 100644 --- a/presidio-analyzer/tests/test_de_handelsregister_recognizer.py +++ b/presidio-analyzer/tests/test_de_handelsregister_recognizer.py @@ -4,6 +4,7 @@ The HR[AB] prefix makes the pattern highly specific, keeping false positives low. Legal basis: §§ 9, 14 HGB; DSGVO Art. 4 Nr. 1 for HRA entries (sole traders). """ + import pytest from presidio_analyzer.predefined_recognizers import DeHandelsregisterRecognizer diff --git a/presidio-analyzer/tests/test_de_health_insurance_recognizer.py b/presidio-analyzer/tests/test_de_health_insurance_recognizer.py index 9a724cbca4..1dd9c24d76 100644 --- a/presidio-analyzer/tests/test_de_health_insurance_recognizer.py +++ b/presidio-analyzer/tests/test_de_health_insurance_recognizer.py @@ -20,6 +20,7 @@ Z000000005 – letter Z (=26), exercises upper-bound letter ordinal Z999999997 – letter Z with all-9 data, exercises Quersumme on 7 of 10 products """ + import pytest from tests import assert_result @@ -96,7 +97,7 @@ def test_when_all_de_health_insurance_numbers_then_succeed( # Starts with digit (after .upper() it's still a digit → re.match fails) ("1123456780", False), # Wrong length - ("A12345678", False), + ("A12345678", False), ("A1234567890", False), # Lowercase: .upper() converts to valid → True (IGNORECASE is global) ("a123456780", True), diff --git a/presidio-analyzer/tests/test_de_id_card_recognizer.py b/presidio-analyzer/tests/test_de_id_card_recognizer.py index ebb704e524..743c31a52f 100644 --- a/presidio-analyzer/tests/test_de_id_card_recognizer.py +++ b/presidio-analyzer/tests/test_de_id_card_recognizer.py @@ -14,6 +14,7 @@ ICAO-valid nPA → validate_result True → MAX_SCORE (1.0) Legacy T+8d → validate_result None → pattern score 0.5 """ + import pytest from tests import assert_result @@ -67,8 +68,12 @@ def entities(): ], ) def test_when_all_de_id_cards_then_succeed( - text, expected_len, expected_positions, expected_score, - recognizer, entities, + text, + expected_len, + expected_positions, + expected_score, + recognizer, + entities, ): results = recognizer.analyze(text, entities) assert len(results) == expected_len @@ -93,7 +98,7 @@ def test_when_all_de_id_cards_then_succeed( ("T22000129", None), ("T00000000", None), # Wrong length - ("L01X00T4", False), + ("L01X00T4", False), ("L01X00T440", False), # Last char must be digit (for nPA form) ("L01X00T4A", False), diff --git a/presidio-analyzer/tests/test_de_kfz_recognizer.py b/presidio-analyzer/tests/test_de_kfz_recognizer.py index ce5b4b08f1..ff49043fb4 100644 --- a/presidio-analyzer/tests/test_de_kfz_recognizer.py +++ b/presidio-analyzer/tests/test_de_kfz_recognizer.py @@ -7,6 +7,7 @@ Legal basis: Fahrzeug-Zulassungsverordnung (FZV) § 8. Data protection: DSGVO Art. 4 Nr. 1 (ECJ C-582/14, Breyer v. Germany). """ + import pytest from presidio_analyzer.predefined_recognizers import DeKfzRecognizer diff --git a/presidio-analyzer/tests/test_de_lanr_recognizer.py b/presidio-analyzer/tests/test_de_lanr_recognizer.py index 9bab439741..c615be27eb 100644 --- a/presidio-analyzer/tests/test_de_lanr_recognizer.py +++ b/presidio-analyzer/tests/test_de_lanr_recognizer.py @@ -16,6 +16,7 @@ 555555501 – physician 555555, check 5 999999901 – physician 999999, check 9 (all-9 edge case) """ + import pytest from tests import assert_result @@ -81,7 +82,7 @@ def test_when_all_de_lanr_numbers_then_succeed( ("234567601", False), ("100000401", False), # Wrong length - ("12345660", False), + ("12345660", False), ("1234566010", False), # Non-numeric ("12345660a", False), diff --git a/presidio-analyzer/tests/test_de_passport_recognizer.py b/presidio-analyzer/tests/test_de_passport_recognizer.py index be0375387d..0367fa21e1 100644 --- a/presidio-analyzer/tests/test_de_passport_recognizer.py +++ b/presidio-analyzer/tests/test_de_passport_recognizer.py @@ -17,6 +17,7 @@ C01X00T41 – matches the worked example in the recognizer docstring (products 84+0+1+231+0+0+203+12 = 531 → check 1) """ + import pytest from tests import assert_result diff --git a/presidio-analyzer/tests/test_de_plz_recognizer.py b/presidio-analyzer/tests/test_de_plz_recognizer.py index 9ef0c93de1..a9807a1b7b 100644 --- a/presidio-analyzer/tests/test_de_plz_recognizer.py +++ b/presidio-analyzer/tests/test_de_plz_recognizer.py @@ -6,6 +6,7 @@ boundary matching) only. In production, context words are required for the recognizer to produce actionable (high-confidence) results. """ + import pytest from presidio_analyzer.predefined_recognizers import DePlzRecognizer diff --git a/presidio-analyzer/tests/test_de_social_security_recognizer.py b/presidio-analyzer/tests/test_de_social_security_recognizer.py index 95e68253a0..2a39584fc7 100644 --- a/presidio-analyzer/tests/test_de_social_security_recognizer.py +++ b/presidio-analyzer/tests/test_de_social_security_recognizer.py @@ -17,6 +17,7 @@ 20151090B023 – area 20 (Westfalen-Lippe), 15.10.1990, B, serial 02, check 3 38551285K051 – Ergänzungsmerkmal day (55=5+50), 12.1985, K, serial 05, check 1 """ + import pytest from tests import assert_result @@ -81,7 +82,7 @@ def test_when_all_de_social_security_numbers_then_succeed( # Digit instead of letter at position 9 ("150706491103", False), # Wrong length - ("15070649C10", False), + ("15070649C10", False), ("15070649C1030", False), # Impossible day (42 is between 31 and 51, both excluded) ("15420649C103", False), diff --git a/presidio-analyzer/tests/test_de_tax_id_recognizer.py b/presidio-analyzer/tests/test_de_tax_id_recognizer.py index 94946a7e08..616ea448e5 100644 --- a/presidio-analyzer/tests/test_de_tax_id_recognizer.py +++ b/presidio-analyzer/tests/test_de_tax_id_recognizer.py @@ -5,6 +5,7 @@ Valid numbers are produced with the official ISO 7064 Mod 11, 10 algorithm as specified by the Bundeszentralamt für Steuern (§§ 139a–139e AO). """ + import pytest from tests import assert_result @@ -69,7 +70,7 @@ def test_when_all_de_tax_ids_then_succeed( # Non-numeric ("abcdefghijk", False), # Wrong length - ("1234567890", False), + ("1234567890", False), ("123456789030", False), # All same first 10 digits ("11111111111", False), @@ -78,5 +79,7 @@ def test_when_all_de_tax_ids_then_succeed( ("12222234567", False), ], ) -def test_when_de_tax_id_validated_then_checksum_result_is_correct(number, expected, recognizer): +def test_when_de_tax_id_validated_then_checksum_result_is_correct( + number, expected, recognizer +): assert recognizer.validate_result(number) == expected diff --git a/presidio-analyzer/tests/test_de_tax_number_recognizer.py b/presidio-analyzer/tests/test_de_tax_number_recognizer.py index 51ebeaacb9..4c6963e55e 100644 --- a/presidio-analyzer/tests/test_de_tax_number_recognizer.py +++ b/presidio-analyzer/tests/test_de_tax_number_recognizer.py @@ -4,6 +4,7 @@ Tests cover both the ELSTER unified 13-digit format and the various state-specific slash-separated formats used across German Bundesländer. """ + import pytest from presidio_analyzer.predefined_recognizers import DeTaxNumberRecognizer diff --git a/presidio-analyzer/tests/test_de_vat_id_recognizer.py b/presidio-analyzer/tests/test_de_vat_id_recognizer.py index 36a96cc66c..c8af0d9fc2 100644 --- a/presidio-analyzer/tests/test_de_vat_id_recognizer.py +++ b/presidio-analyzer/tests/test_de_vat_id_recognizer.py @@ -22,6 +22,7 @@ "DE 136 695 976", "DE-136-695-976", "DE.136.695.976", "DE 136695976", lowercase variants. """ + import pytest from tests import assert_result @@ -73,7 +74,8 @@ def entities(): ("AT123456789", 0), # wrong country ("FR12345678901", 0), # wrong country ("DE12345678", 0), # too short - ("DE1234567890", 0), # too long + ("DE1234567890", 0), + # too long # fmt: on ], ) diff --git a/presidio-analyzer/tests/test_device_detector.py b/presidio-analyzer/tests/test_device_detector.py index 9341334834..aa65dde125 100644 --- a/presidio-analyzer/tests/test_device_detector.py +++ b/presidio-analyzer/tests/test_device_detector.py @@ -76,24 +76,26 @@ class TestDeviceDetectorErrorPaths: def test_when_torch_import_fails_then_cpu_device(self): """Test that CPU is used when PyTorch import fails.""" - with patch("builtins.__import__", side_effect=ImportError("No module named 'torch'")): + with patch( + "builtins.__import__", side_effect=ImportError("No module named 'torch'") + ): detector = DeviceDetector() - + assert detector.get_device() == "cpu" def test_when_cuda_not_available_then_cpu_device(self): """Test that CPU is used when CUDA is not available.""" mock_torch = MagicMock() mock_torch.cuda.is_available.return_value = False - + def mock_import(name, *args): if name == "torch": return mock_torch return __builtins__.__import__(name, *args) - + with patch("builtins.__import__", side_effect=mock_import): detector = DeviceDetector() - + assert detector.get_device() == "cpu" def test_when_cuda_initialization_fails_then_fallback_to_cpu(self): @@ -101,15 +103,15 @@ def test_when_cuda_initialization_fails_then_fallback_to_cpu(self): mock_torch = MagicMock() mock_torch.cuda.is_available.return_value = True mock_torch.tensor.side_effect = RuntimeError("CUDA initialization error") - + def mock_import(name, *args): if name == "torch": return mock_torch return __builtins__.__import__(name, *args) - + with patch("builtins.__import__", side_effect=mock_import): detector = DeviceDetector() - + assert detector.get_device() == "cpu" def test_when_cuda_get_device_name_fails_then_fallback_to_cpu(self): @@ -118,15 +120,15 @@ def test_when_cuda_get_device_name_fails_then_fallback_to_cpu(self): mock_torch.cuda.is_available.return_value = True mock_torch.tensor.return_value = MagicMock(__str__=lambda x: "tensor") mock_torch.cuda.get_device_name.side_effect = RuntimeError("Device name error") - + def mock_import(name, *args): if name == "torch": return mock_torch return __builtins__.__import__(name, *args) - + with patch("builtins.__import__", side_effect=mock_import): detector = DeviceDetector() - + assert detector.get_device() == "cpu" def test_when_cuda_available_then_cuda_device(self): @@ -136,15 +138,15 @@ def test_when_cuda_available_then_cuda_device(self): mock_torch.tensor.return_value = MagicMock(__str__=lambda x: "tensor") mock_torch.cuda.get_device_name.return_value = "Test GPU" mock_torch.cuda.get_device_capability.return_value = (8, 0) - + def mock_import(name, *args): if name == "torch": return mock_torch return __builtins__.__import__(name, *args) - + with patch("builtins.__import__", side_effect=mock_import): detector = DeviceDetector() - + assert detector.get_device() == "cuda" def test_when_cuda_available_then_uses_cuda_not_cpu(self): @@ -154,15 +156,15 @@ def test_when_cuda_available_then_uses_cuda_not_cpu(self): mock_torch.tensor.return_value = MagicMock(__str__=lambda x: "tensor") mock_torch.cuda.get_device_name.return_value = "Test GPU" mock_torch.cuda.get_device_capability.return_value = (8, 0) - + def mock_import(name, *args): if name == "torch": return mock_torch return __builtins__.__import__(name, *args) - + with patch("builtins.__import__", side_effect=mock_import): detector = DeviceDetector() - + assert detector.get_device() == "cuda" @@ -181,7 +183,6 @@ def test_when_multiple_instances_then_same_values(self): detector1 = DeviceDetector() detector2 = DeviceDetector() - # Both should return the same device assert detector1.get_device() == detector2.get_device() @@ -192,22 +193,20 @@ class TestDeviceDetectorIntegration: def test_when_spacy_engine_loads_then_uses_device_detector(self): """Test that SpacyNlpEngine uses device_detector.""" from presidio_analyzer.nlp_engine import SpacyNlpEngine - + engine = SpacyNlpEngine( models=[{"lang_code": "en", "model_name": "en_core_web_sm"}] ) - + # Verify device_detector is accessible assert device_detector.get_device() in ["cpu", "cuda"] def test_when_stanza_engine_initializes_then_sets_device(self): """Test that StanzaNlpEngine correctly sets device from device_detector.""" from presidio_analyzer.nlp_engine import StanzaNlpEngine - - engine = StanzaNlpEngine( - models=[{"lang_code": "en", "model_name": "en"}] - ) - + + engine = StanzaNlpEngine(models=[{"lang_code": "en", "model_name": "en"}]) + # device should match device_detector expected_device = device_detector.get_device() assert engine.device == expected_device @@ -216,20 +215,18 @@ def test_when_gliner_recognizer_initializes_then_uses_correct_device(self): """Test that GLiNERRecognizer uses device from device_detector.""" pytest.importorskip("gliner") from presidio_analyzer.predefined_recognizers import GLiNERRecognizer - + recognizer = GLiNERRecognizer() - + # map_location should match device_detector.get_device() assert recognizer.map_location == device_detector.get_device() def test_when_stanza_engine_device_matches_device_detector(self): """Test that StanzaNlpEngine.device matches device_detector.""" from presidio_analyzer.nlp_engine import StanzaNlpEngine - - engine = StanzaNlpEngine( - models=[{"lang_code": "en", "model_name": "en"}] - ) - + + engine = StanzaNlpEngine(models=[{"lang_code": "en", "model_name": "en"}]) + expected_device = device_detector.get_device() assert engine.device == expected_device @@ -241,6 +238,6 @@ def test_when_creating_new_instance_then_device_consistent(self): """Test that new instances have consistent device detection.""" detector1 = DeviceDetector() detector2 = DeviceDetector() - + # Both should detect the same device - assert detector1.get_device() == detector2.get_device() \ No newline at end of file + assert detector1.get_device() == detector2.get_device() diff --git a/presidio-analyzer/tests/test_entity_mapper.py b/presidio-analyzer/tests/test_entity_mapper.py index 60f1875d44..e8b508ea02 100644 --- a/presidio-analyzer/tests/test_entity_mapper.py +++ b/presidio-analyzer/tests/test_entity_mapper.py @@ -1,4 +1,5 @@ """Tests for llm_utils.entity_mapper module.""" + import pytest from presidio_analyzer import RecognizerResult, AnalysisExplanation from presidio_analyzer.llm_utils.entity_mapper import ( @@ -14,11 +15,7 @@ def create_test_result( - entity_type="PERSON", - start=0, - end=10, - score=0.9, - recognition_metadata=None + entity_type="PERSON", start=0, end=10, score=0.9, recognition_metadata=None ): """Helper to create RecognizerResult for testing.""" return RecognizerResult( @@ -29,9 +26,9 @@ def create_test_result( analysis_explanation=AnalysisExplanation( recognizer="TestRecognizer", original_score=score, - textual_explanation="Test" + textual_explanation="Test", ), - recognition_metadata=recognition_metadata + recognition_metadata=recognition_metadata, ) @@ -93,10 +90,7 @@ def test_when_label_matching_is_case_insensitive(self): def test_when_result_has_no_entity_type_then_skips_with_warning(self): """Test that results without entity_type are skipped.""" result_without_type = RecognizerResult( - entity_type=None, - start=0, - end=10, - score=0.9 + entity_type=None, start=0, end=10, score=0.9 ) results = [ result_without_type, @@ -232,10 +226,7 @@ def test_when_result_missing_start_then_filters_out(self): """Test that results with missing start are filtered out.""" valid_result = create_test_result("PERSON", start=0, end=10) invalid_result = RecognizerResult( - entity_type="EMAIL_ADDRESS", - start=None, - end=40, - score=0.9 + entity_type="EMAIL_ADDRESS", start=None, end=40, score=0.9 ) filtered = validate_result_positions([valid_result, invalid_result]) @@ -247,10 +238,7 @@ def test_when_result_missing_end_then_filters_out(self): """Test that results with missing end are filtered out.""" valid_result = create_test_result("PERSON", start=0, end=10) invalid_result = RecognizerResult( - entity_type="EMAIL_ADDRESS", - start=20, - end=None, - score=0.9 + entity_type="EMAIL_ADDRESS", start=20, end=None, score=0.9 ) filtered = validate_result_positions([valid_result, invalid_result]) @@ -262,10 +250,7 @@ def test_when_result_missing_both_positions_then_filters_out(self): """Test that results with both positions missing are filtered out.""" valid_result = create_test_result("PERSON", start=0, end=10) invalid_result = RecognizerResult( - entity_type="EMAIL_ADDRESS", - start=None, - end=None, - score=0.9 + entity_type="EMAIL_ADDRESS", start=None, end=None, score=0.9 ) filtered = validate_result_positions([valid_result, invalid_result]) @@ -284,7 +269,9 @@ def test_when_entity_supported_then_keeps_original(self): supported_entities = ["PERSON", "EMAIL_ADDRESS"] logged_entities = set() - processed = consolidate_generic_entities(results, supported_entities, logged_entities) + processed = consolidate_generic_entities( + results, supported_entities, logged_entities + ) assert len(processed) == 1 assert processed[0].entity_type == "PERSON" @@ -297,7 +284,9 @@ def test_when_entity_unsupported_then_consolidates_to_generic(self): supported_entities = ["PERSON"] logged_entities = set() - processed = consolidate_generic_entities(results, supported_entities, logged_entities) + processed = consolidate_generic_entities( + results, supported_entities, logged_entities + ) assert len(processed) == 1 assert processed[0].entity_type == GENERIC_PII_ENTITY @@ -310,10 +299,14 @@ def test_when_entity_consolidated_then_stores_original_in_metadata(self): supported_entities = ["PERSON"] logged_entities = set() - processed = consolidate_generic_entities(results, supported_entities, logged_entities) + processed = consolidate_generic_entities( + results, supported_entities, logged_entities + ) assert processed[0].recognition_metadata is not None - assert processed[0].recognition_metadata["original_entity_type"] == "UNKNOWN_TYPE" + assert ( + processed[0].recognition_metadata["original_entity_type"] == "UNKNOWN_TYPE" + ) def test_when_entity_already_has_metadata_then_adds_original_type(self): """Test that original_entity_type is added to existing metadata.""" @@ -323,10 +316,14 @@ def test_when_entity_already_has_metadata_then_adds_original_type(self): supported_entities = ["PERSON"] logged_entities = set() - processed = consolidate_generic_entities(results, supported_entities, logged_entities) + processed = consolidate_generic_entities( + results, supported_entities, logged_entities + ) assert processed[0].recognition_metadata["key"] == "value" - assert processed[0].recognition_metadata["original_entity_type"] == "UNKNOWN_TYPE" + assert ( + processed[0].recognition_metadata["original_entity_type"] == "UNKNOWN_TYPE" + ) def test_when_unknown_entity_first_seen_then_logs_to_set(self): """Test that first occurrence of unknown entity is added to logged set.""" @@ -403,7 +400,9 @@ def test_when_consolidation_enabled_and_generic_missing_then_adds_generic(self): """Test that GENERIC_PII_ENTITY is added when consolidation is enabled.""" supported_entities = ["PERSON", "EMAIL_ADDRESS"] - result = ensure_generic_entity_support(supported_entities, enable_generic_consolidation=True) + result = ensure_generic_entity_support( + supported_entities, enable_generic_consolidation=True + ) assert GENERIC_PII_ENTITY in result assert "PERSON" in result @@ -413,16 +412,22 @@ def test_when_consolidation_disabled_then_does_not_add_generic(self): """Test that GENERIC_PII_ENTITY is not added when consolidation is disabled.""" supported_entities = ["PERSON", "EMAIL_ADDRESS"] - result = ensure_generic_entity_support(supported_entities, enable_generic_consolidation=False) + result = ensure_generic_entity_support( + supported_entities, enable_generic_consolidation=False + ) assert GENERIC_PII_ENTITY not in result assert len(result) == 2 - def test_when_generic_already_present_and_consolidation_enabled_then_no_duplicate(self): + def test_when_generic_already_present_and_consolidation_enabled_then_no_duplicate( + self, + ): """Test that GENERIC_PII_ENTITY is not duplicated.""" supported_entities = ["PERSON", GENERIC_PII_ENTITY] - result = ensure_generic_entity_support(supported_entities, enable_generic_consolidation=True) + result = ensure_generic_entity_support( + supported_entities, enable_generic_consolidation=True + ) # Count occurrences of GENERIC_PII_ENTITY generic_count = result.count(GENERIC_PII_ENTITY) @@ -431,8 +436,10 @@ def test_when_generic_already_present_and_consolidation_enabled_then_no_duplicat def test_when_original_list_not_modified(self): """Test that original list is not modified (returns a copy).""" original = ["PERSON", "EMAIL_ADDRESS"] - - result = ensure_generic_entity_support(original, enable_generic_consolidation=True) + + result = ensure_generic_entity_support( + original, enable_generic_consolidation=True + ) assert GENERIC_PII_ENTITY not in original # Original unchanged assert GENERIC_PII_ENTITY in result # Result has it diff --git a/presidio-analyzer/tests/test_entity_recognizer.py b/presidio-analyzer/tests/test_entity_recognizer.py index 3bb695eb5d..b664670750 100644 --- a/presidio-analyzer/tests/test_entity_recognizer.py +++ b/presidio-analyzer/tests/test_entity_recognizer.py @@ -120,6 +120,7 @@ def test_when_remove_duplicates_contained_shorter_length_results_removed(): results = EntityRecognizer.remove_duplicates(arr) assert len(results) == 1 + import pytest sanitizer_test_set = [ @@ -127,6 +128,7 @@ def test_when_remove_duplicates_contained_shorter_length_results_removed(): ["def", "", "def"], ] + @pytest.mark.parametrize("input_text, params, expected_output", sanitizer_test_set) def test_sanitize_value(input_text, params, expected_output): """ diff --git a/presidio-analyzer/tests/test_es_nie_recognizer.py b/presidio-analyzer/tests/test_es_nie_recognizer.py index 2f9c4e6585..97514efa7e 100644 --- a/presidio-analyzer/tests/test_es_nie_recognizer.py +++ b/presidio-analyzer/tests/test_es_nie_recognizer.py @@ -20,14 +20,41 @@ def entities(): "text, expected_len, expected_positions", [ # valid NIE scores - ("Z8078221M", 1, ((0, 9),),), - ("X9613851N", 1, ((0, 9),),), - ("Y8063915Z", 1, ((0, 9),),), - ("Y8063915-Z", 1, ((0, 10),),), - ("Mi NIE es X9613851N", 1, ((10, 19),),), - ("Z8078221M en mi NIE", 1, ((0, 9),),), - ("Mi Número de identificación de extranjero es Y8063915-Z", 1, \ - ((45, 55),),), + ( + "Z8078221M", + 1, + ((0, 9),), + ), + ( + "X9613851N", + 1, + ((0, 9),), + ), + ( + "Y8063915Z", + 1, + ((0, 9),), + ), + ( + "Y8063915-Z", + 1, + ((0, 10),), + ), + ( + "Mi NIE es X9613851N", + 1, + ((10, 19),), + ), + ( + "Z8078221M en mi NIE", + 1, + ((0, 9),), + ), + ( + "Mi Número de identificación de extranjero es Y8063915-Z", + 1, + ((45, 55),), + ), # invalid NIE scores ("Y8063915Q", 0, ()), ("Y806391Q", 0, ()), diff --git a/presidio-analyzer/tests/test_es_passport_recognizer.py b/presidio-analyzer/tests/test_es_passport_recognizer.py index cf06c38d5c..a839b293f5 100644 --- a/presidio-analyzer/tests/test_es_passport_recognizer.py +++ b/presidio-analyzer/tests/test_es_passport_recognizer.py @@ -20,27 +20,87 @@ def entities(): "text, expected_len, expected_positions, expected_score_ranges", [ # valid passport numbers - no context, base score only - ("AAA123456", 1, ((0, 9),), ((0.05, 0.05),),), - ("XYZ987654", 1, ((0, 9),), ((0.05, 0.05),),), + ( + "AAA123456", + 1, + ((0, 9),), + ((0.05, 0.05),), + ), + ( + "XYZ987654", + 1, + ((0, 9),), + ((0.05, 0.05),), + ), # valid passport numbers - with context words, score boosted - ("Mi pasaporte es AAA123456", 1, ((16, 25),), ((0.05, 1.0),),), - ("AAA123456 es mi número de pasaporte", 1, ((0, 9),), ((0.05, 1.0),),), - # valid passport numbers - no context, base score only - lowercase letters - ("aaa123456", 1, ((0, 9),), ((0.05, 0.05),),), - ("xyz987654", 1, ((0, 9),), ((0.05, 0.05),),), - # valid passport numbers - with context words, score boosted - lowercase letters - ("Mi pasaporte es aaa123456", 1, ((16, 25),), ((0.05, 1.0),),), - ("aaa123456 es mi número de pasaporte", 1, ((0, 9),), ((0.05, 1.0),),), + ( + "Mi pasaporte es AAA123456", + 1, + ((16, 25),), + ((0.05, 1.0),), + ), + ( + "AAA123456 es mi número de pasaporte", + 1, + ((0, 9),), + ((0.05, 1.0),), + ), + # valid passport numbers - no context, base score only - lowercase letters + ( + "aaa123456", + 1, + ((0, 9),), + ((0.05, 0.05),), + ), + ( + "xyz987654", + 1, + ((0, 9),), + ((0.05, 0.05),), + ), + # valid passport numbers - with context words, score boosted - lowercase letters + ( + "Mi pasaporte es aaa123456", + 1, + ((16, 25),), + ((0.05, 1.0),), + ), + ( + "aaa123456 es mi número de pasaporte", + 1, + ((0, 9),), + ((0.05, 1.0),), + ), # valid passport numbers - no context, base score only - case mixed letters - ("AaA123456", 1, ((0, 9),), ((0.05, 0.05),),), - ("XyZ987654", 1, ((0, 9),), ((0.05, 0.05),),), + ( + "AaA123456", + 1, + ((0, 9),), + ((0.05, 0.05),), + ), + ( + "XyZ987654", + 1, + ((0, 9),), + ((0.05, 0.05),), + ), # valid passport numbers - with context words, score boosted - case mixed letters - ("Mi pasaporte es AaA123456", 1, ((16, 25),), ((0.05, 1.0),),), - ("AaA123456 es mi número de pasaporte", 1, ((0, 9),), ((0.05, 1.0),),), + ( + "Mi pasaporte es AaA123456", + 1, + ((16, 25),), + ((0.05, 1.0),), + ), + ( + "AaA123456 es mi número de pasaporte", + 1, + ((0, 9),), + ((0.05, 1.0),), + ), # invalid passport numbers - ("AA123456", 0, (), ()), # only 2 letters prefix - ("AAAA12345", 0, (), ()), # 4 letters prefix - ("AAA12345", 0, (), ()), # only 5 digits + ("AA123456", 0, (), ()), # only 2 letters prefix + ("AAAA12345", 0, (), ()), # 4 letters prefix + ("AAA12345", 0, (), ()), # only 5 digits ], ) def test_when_all_es_passport_then_succeed( diff --git a/presidio-analyzer/tests/test_examples_loader.py b/presidio-analyzer/tests/test_examples_loader.py index 2f21889163..4d0e7c4a8d 100644 --- a/presidio-analyzer/tests/test_examples_loader.py +++ b/presidio-analyzer/tests/test_examples_loader.py @@ -1,4 +1,5 @@ """Tests for llm_utils.examples_loader module.""" + import pytest import tempfile from pathlib import Path @@ -13,7 +14,7 @@ class TestLoadYamlExamples: def test_when_examples_file_exists_then_loads_list(self): """Test loading examples from YAML file returns a list.""" - with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as f: + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: f.write(""" examples: - text: "John Doe works at Acme Corp" @@ -33,7 +34,7 @@ def test_when_examples_file_exists_then_loads_list(self): try: # Use absolute path to load temp file examples = load_yaml_examples(str(examples_path)) - + assert isinstance(examples, list) assert len(examples) == 2 assert examples[0]["text"] == "John Doe works at Acme Corp" @@ -41,9 +42,11 @@ def test_when_examples_file_exists_then_loads_list(self): finally: examples_path.unlink() - def test_when_examples_file_has_multiple_extractions_per_example_then_loads_all(self): + def test_when_examples_file_has_multiple_extractions_per_example_then_loads_all( + self, + ): """Test loading example with multiple extractions.""" - with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as f: + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: f.write(""" examples: - text: "John at john@example.com" @@ -59,7 +62,7 @@ def test_when_examples_file_has_multiple_extractions_per_example_then_loads_all( try: examples = load_yaml_examples(str(examples_path)) - + assert len(examples) == 1 assert len(examples[0]["extractions"]) == 2 finally: @@ -72,7 +75,7 @@ def test_when_examples_file_missing_then_raises_file_not_found_error(self): def test_when_examples_missing_from_yaml_then_raises_value_error(self): """Test that YAML without 'examples' key raises ValueError.""" - with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as f: + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: f.write(""" other_section: - text: "some text" @@ -80,7 +83,9 @@ def test_when_examples_missing_from_yaml_then_raises_value_error(self): examples_path = Path(f.name) try: - with pytest.raises(ValueError, match="Examples file must contain 'examples'"): + with pytest.raises( + ValueError, match="Examples file must contain 'examples'" + ): load_yaml_examples(str(examples_path)) finally: examples_path.unlink() @@ -91,7 +96,7 @@ def test_when_loading_actual_langextract_examples_file_then_works(self): examples = load_yaml_examples( "presidio-analyzer/presidio_analyzer/conf/langextract_prompts/default_pii_phi_examples.yaml" ) - + assert isinstance(examples, list) assert len(examples) > 0 # Each example should have text and extractions @@ -112,9 +117,9 @@ def test_when_examples_data_valid_then_converts_to_langextract_format(self): { "extraction_class": "PERSON", "extraction_text": "John Doe", - "attributes": {} + "attributes": {}, } - ] + ], } ] @@ -122,11 +127,11 @@ def test_when_examples_data_valid_then_converts_to_langextract_format(self): assert isinstance(result, list) assert len(result) == 1 - + # Check it's a LangExtract ExampleData object example = result[0] - assert hasattr(example, 'text') - assert hasattr(example, 'extractions') + assert hasattr(example, "text") + assert hasattr(example, "extractions") assert example.text == "John Doe works here" assert len(example.extractions) == 1 @@ -136,15 +141,23 @@ def test_when_multiple_examples_then_converts_all(self): { "text": "John Doe", "extractions": [ - {"extraction_class": "PERSON", "extraction_text": "John Doe", "attributes": {}} - ] + { + "extraction_class": "PERSON", + "extraction_text": "John Doe", + "attributes": {}, + } + ], }, { "text": "info@example.com", "extractions": [ - {"extraction_class": "EMAIL_ADDRESS", "extraction_text": "info@example.com", "attributes": {}} - ] - } + { + "extraction_class": "EMAIL_ADDRESS", + "extraction_text": "info@example.com", + "attributes": {}, + } + ], + }, ] result = convert_to_langextract_format(examples_data) @@ -159,9 +172,17 @@ def test_when_example_has_multiple_extractions_then_converts_all(self): { "text": "John at john@example.com", "extractions": [ - {"extraction_class": "PERSON", "extraction_text": "John", "attributes": {}}, - {"extraction_class": "EMAIL_ADDRESS", "extraction_text": "john@example.com", "attributes": {}} - ] + { + "extraction_class": "PERSON", + "extraction_text": "John", + "attributes": {}, + }, + { + "extraction_class": "EMAIL_ADDRESS", + "extraction_text": "john@example.com", + "attributes": {}, + }, + ], } ] @@ -179,9 +200,9 @@ def test_when_extraction_has_all_fields_then_preserves_data(self): { "extraction_class": "PERSON", "extraction_text": "Sample", - "attributes": {"type": "name"} + "attributes": {"type": "name"}, } - ] + ], } ] @@ -189,9 +210,9 @@ def test_when_extraction_has_all_fields_then_preserves_data(self): extraction = result[0].extractions[0] # Check the Extraction object has correct attributes - assert hasattr(extraction, 'extraction_class') - assert hasattr(extraction, 'extraction_text') - assert hasattr(extraction, 'attributes') + assert hasattr(extraction, "extraction_class") + assert hasattr(extraction, "extraction_text") + assert hasattr(extraction, "attributes") assert extraction.extraction_class == "PERSON" assert extraction.extraction_text == "Sample" assert extraction.attributes == {"type": "name"} @@ -203,12 +224,7 @@ def test_when_empty_examples_list_then_returns_empty_list(self): def test_when_example_has_no_extractions_then_creates_empty_extractions(self): """Test example with empty extractions list.""" - examples_data = [ - { - "text": "No entities here", - "extractions": [] - } - ] + examples_data = [{"text": "No entities here", "extractions": []}] result = convert_to_langextract_format(examples_data) @@ -222,7 +238,7 @@ class TestIntegration: def test_when_loading_and_converting_workflow_then_works(self): """Test complete workflow: load YAML → convert to LangExtract format.""" - with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as f: + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: f.write(""" examples: - text: "John Doe at john@example.com and Jane Smith" @@ -244,18 +260,20 @@ def test_when_loading_and_converting_workflow_then_works(self): examples_data = load_yaml_examples(str(examples_path)) assert len(examples_data) == 1 assert len(examples_data[0]["extractions"]) == 3 - + # Step 2: Convert to LangExtract format langextract_examples = convert_to_langextract_format(examples_data) assert len(langextract_examples) == 1 assert len(langextract_examples[0].extractions) == 3 - + # Verify entity types - entity_types = [e.extraction_class for e in langextract_examples[0].extractions] + entity_types = [ + e.extraction_class for e in langextract_examples[0].extractions + ] assert "PERSON" in entity_types assert "EMAIL_ADDRESS" in entity_types assert entity_types.count("PERSON") == 2 - + finally: examples_path.unlink() @@ -265,16 +283,16 @@ def test_when_using_actual_langextract_examples_then_converts_correctly(self): examples_data = load_yaml_examples( "presidio-analyzer/presidio_analyzer/conf/langextract_prompts/default_pii_phi_examples.yaml" ) - + # Convert to LangExtract format langextract_examples = convert_to_langextract_format(examples_data) - + assert isinstance(langextract_examples, list) assert len(langextract_examples) > 0 - + # Verify structure for example in langextract_examples: - assert hasattr(example, 'text') - assert hasattr(example, 'extractions') + assert hasattr(example, "text") + assert hasattr(example, "extractions") assert isinstance(example.text, str) assert isinstance(example.extractions, list) diff --git a/presidio-analyzer/tests/test_gliner_recognizer.py b/presidio-analyzer/tests/test_gliner_recognizer.py index 528634441d..0fd8794bc6 100644 --- a/presidio-analyzer/tests/test_gliner_recognizer.py +++ b/presidio-analyzer/tests/test_gliner_recognizer.py @@ -23,9 +23,7 @@ def mock_gliner(): yield mock_gliner_instance -def test_analyze_passed_entities_are_subset_of_entity_mapping( - mock_gliner -): +def test_analyze_passed_entities_are_subset_of_entity_mapping(mock_gliner): if sys.version_info < (3, 10): pytest.skip("gliner requires Python >= 3.10") @@ -71,11 +69,9 @@ def test_analyze_passed_entities_are_subset_of_entity_mapping( def test_analyze_with_unsupported_entity(mock_gliner): - if sys.version_info < (3, 10): pytest.skip("gliner requires Python >= 3.10") - # Mock GLiNER predict_entities mock_gliner.gliner.predict_entities.return_value = [ {"label": "BIRD", "start": 0, "end": 5, "score": 0.75}, @@ -128,7 +124,6 @@ def test_analyze_with_no_entities(mock_gliner): text = "No entities here." entities = [] - gliner_recognizer = GLiNERRecognizer( supported_entities=["ORG", "LOC", "PER"], ) @@ -151,10 +146,14 @@ def mock_predict_entities(text, labels, flat_ner, threshold, multi_label): entities = [] if "John Smith" in text: start = text.find("John Smith") - entities.append({"label": "person", "start": start, "end": start + 10, "score": 0.95}) + entities.append( + {"label": "person", "start": start, "end": start + 10, "score": 0.95} + ) if "Jane Doe" in text: start = text.find("Jane Doe") - entities.append({"label": "person", "start": start, "end": start + 8, "score": 0.93}) + entities.append( + {"label": "person", "start": start, "end": start + 8, "score": 0.93} + ) return entities mock_gliner.predict_entities.side_effect = mock_predict_entities @@ -168,17 +167,19 @@ def mock_predict_entities(text, labels, flat_ner, threshold, multi_label): results = gliner_recognizer.analyze(text, ["PERSON"]) # Verify chunking occurred (predict_entities called multiple times) - assert mock_gliner.predict_entities.call_count == 2, f"Expected 2 chunks, got {mock_gliner.predict_entities.call_count}" - + assert ( + mock_gliner.predict_entities.call_count == 2 + ), f"Expected 2 chunks, got {mock_gliner.predict_entities.call_count}" + # Verify exactly 2 entities were detected assert len(results) == 2, f"Expected 2 entities, found {len(results)}" - + # Verify both entities have correct offsets in original text - assert text[results[0].start:results[0].end] == "John Smith" + assert text[results[0].start : results[0].end] == "John Smith" assert results[0].entity_type == "PERSON" assert results[0].score == 0.95 - - assert text[results[1].start:results[1].end] == "Jane Doe" + + assert text[results[1].start : results[1].end] == "Jane Doe" assert results[1].entity_type == "PERSON" assert results[1].score == 0.93 @@ -196,7 +197,9 @@ def mock_predict_entities(text, labels, flat_ner, threshold, multi_label): entities = [] if "Amanda Williams" in text: start = text.find("Amanda Williams") - entities.append({"label": "person", "start": start, "end": start + 15, "score": 0.92}) + entities.append( + {"label": "person", "start": start, "end": start + 15, "score": 0.92} + ) return entities mock_gliner.predict_entities.side_effect = mock_predict_entities @@ -211,7 +214,7 @@ def mock_predict_entities(text, labels, flat_ner, threshold, multi_label): # Verify entity at boundary was detected assert len(results) == 1, f"Expected 1 entity, found {len(results)}" - assert text[results[0].start:results[0].end] == "Amanda Williams" + assert text[results[0].start : results[0].end] == "Amanda Williams" assert results[0].entity_type == "PERSON" @@ -224,6 +227,7 @@ def test_gliner_deduplicates_entities_in_overlap_region(mock_gliner): text = ("x " * 95) + "Dr. Smith" + (" x" * 100) call_count = 0 + def mock_predict_entities(text, labels, flat_ner, threshold, multi_label): nonlocal call_count call_count += 1 @@ -232,7 +236,9 @@ def mock_predict_entities(text, labels, flat_ner, threshold, multi_label): start = text.find("Dr. Smith") # Return slightly different scores to test that highest is kept score = 0.95 if call_count == 1 else 0.90 - entities.append({"label": "person", "start": start, "end": start + 9, "score": score}) + entities.append( + {"label": "person", "start": start, "end": start + 9, "score": score} + ) return entities mock_gliner.predict_entities.side_effect = mock_predict_entities @@ -246,14 +252,16 @@ def mock_predict_entities(text, labels, flat_ner, threshold, multi_label): results = gliner_recognizer.analyze(text, ["PERSON"]) # Verify: Called multiple times due to overlap - assert mock_gliner.predict_entities.call_count >= 2, "Should process multiple chunks" - + assert ( + mock_gliner.predict_entities.call_count >= 2 + ), "Should process multiple chunks" + # Verify: Only 1 result after deduplication (not 2) assert len(results) == 1, f"Expected 1 deduplicated entity, found {len(results)}" - + # Verify: Kept the one with highest score (0.95 from first chunk) assert results[0].score == 0.95 - assert text[results[0].start:results[0].end] == "Dr. Smith" + assert text[results[0].start : results[0].end] == "Dr. Smith" GLINER_MOCK_PATH = ( @@ -321,5 +329,3 @@ def test_when_model_kwargs_then_passes_to_from_pretrained(): call_kwargs = mock_gliner_class.from_pretrained.call_args[1] assert call_kwargs["custom_param1"] == "value1" assert call_kwargs["custom_param2"] == 42 - - diff --git a/presidio-analyzer/tests/test_huggingface_ner_recognizer.py b/presidio-analyzer/tests/test_huggingface_ner_recognizer.py index 7191aa6a43..65e8ded147 100644 --- a/presidio-analyzer/tests/test_huggingface_ner_recognizer.py +++ b/presidio-analyzer/tests/test_huggingface_ner_recognizer.py @@ -619,7 +619,7 @@ def test_hf_recognizer_loader_supported_entities_filtering(): ) # If the test fails here, it means the loader logic incorrectly filtered it out. - assert "supported_entities" in kwargs, ( - "supported_entities was filtered out by loader!" - ) + assert ( + "supported_entities" in kwargs + ), "supported_entities was filtered out by loader!" assert kwargs["supported_entities"] == ["STAY_PERSISTENT"] diff --git a/presidio-analyzer/tests/test_iban_recognizer.py b/presidio-analyzer/tests/test_iban_recognizer.py index 8c7eff0176..132672c841 100644 --- a/presidio-analyzer/tests/test_iban_recognizer.py +++ b/presidio-analyzer/tests/test_iban_recognizer.py @@ -4,7 +4,9 @@ import pytest from tests import assert_result -from presidio_analyzer.predefined_recognizers.generic.iban_recognizer import IbanRecognizer +from presidio_analyzer.predefined_recognizers.generic.iban_recognizer import ( + IbanRecognizer, +) @pytest.fixture(scope="module") diff --git a/presidio-analyzer/tests/test_in_aadhaar_recognizer.py b/presidio-analyzer/tests/test_in_aadhaar_recognizer.py index 6827490610..616fe6b8c3 100644 --- a/presidio-analyzer/tests/test_in_aadhaar_recognizer.py +++ b/presidio-analyzer/tests/test_in_aadhaar_recognizer.py @@ -55,6 +55,7 @@ def test_when_aadhaar_in_text_then_all_aadhaars_found( expected_score, ) + palindrome_test_set = [ ["abMA", False, False], ["abCba", False, True], @@ -77,8 +78,8 @@ def test_is_palindrome(input_text, case_sensitive, expected_output): :return: True/False """ assert ( - InAadhaarRecognizer._is_palindrome(input_text, case_sensitive) - == expected_output + InAadhaarRecognizer._is_palindrome(input_text, case_sensitive) + == expected_output ) @@ -88,6 +89,7 @@ def test_is_palindrome(input_text, case_sensitive, expected_output): [123456789012, False], ] + @pytest.mark.parametrize("input_number, is_verhoeff", verhoeff_test_set) def test_is_verhoeff(input_number, is_verhoeff) -> None: """ diff --git a/presidio-analyzer/tests/test_in_gstin_recognizer.py b/presidio-analyzer/tests/test_in_gstin_recognizer.py index 5a32a36eee..8858dcc600 100644 --- a/presidio-analyzer/tests/test_in_gstin_recognizer.py +++ b/presidio-analyzer/tests/test_in_gstin_recognizer.py @@ -1,7 +1,9 @@ import pytest from tests import assert_result -from presidio_analyzer.predefined_recognizers.country_specific.india.in_gstin_recognizer import InGstinRecognizer +from presidio_analyzer.predefined_recognizers.country_specific.india.in_gstin_recognizer import ( + InGstinRecognizer, +) @pytest.fixture(scope="module") @@ -22,22 +24,22 @@ def entities(): ("07PQRST6789K1Z2", 1, (0, 15), 1.0), ("01ABCDE1234F1Z5", 1, (0, 15), 1.0), ("37ABCDE1234F1Z5", 1, (0, 15), 1.0), - # Valid GSTINs with medium confidence (different PAN format) ("27ABCDE1234F1Z5", 1, (0, 15), 1.0), ("07PQRST6789K1Z2", 1, (0, 15), 1.0), - # Valid GSTINs with low confidence (generic pattern) ("27ABCDE1234F1Z5", 1, (0, 15), 1.0), - # GSTIN with context - ("My GSTIN number is 27ABCDE1234F1Z5 for business registration", 1, (19, 34), 1.0), + ( + "My GSTIN number is 27ABCDE1234F1Z5 for business registration", + 1, + (19, 34), + 1.0, + ), ("GST registration: 07PQRST6789K1Z2", 1, (18, 33), 1.0), ("Tax identification GSTIN: 01ABCDE1234F1Z5", 1, (26, 41), 1.0), - # Multiple GSTINs ("GSTINs: 27ABCDE1234F1Z5 and 07PQRST6789K1Z2", 2, (8, 23), 1.0), - # Invalid GSTINs (should not be detected) ("27ABCDE1234F1Z", 0, (), ()), # Too short ("27ABCDE1234F1Z55", 0, (), ()), # Too long @@ -56,7 +58,7 @@ def test_when_gstin_in_text_then_all_gstins_found( entities, ): results = recognizer.analyze(text, entities) - + assert len(results) == expected_len if results: assert_result( @@ -75,12 +77,12 @@ def test_when_gstin_in_text_then_all_gstins_found( ("", 0), ("123456789012345", 0), # All digits ("ABCDEFGHIJKLMNO", 0), # All letters - ("27ABCDE1234F1Z", 0), # Too short - ("27ABCDE1234F1Z55", 0), # Too long + ("27ABCDE1234F1Z", 0), # Too short + ("27ABCDE1234F1Z55", 0), # Too long ("00ABCDE1234F1Z5", 0), # Invalid state code (00) ("38ABCDE1234F1Z5", 0), # Invalid state code (38) ("27ABCDE1234F1Y5", 0), # Missing 'Z' at position 14 - ("27ABCDE1234F1Z", 0), # Missing checksum + ("27ABCDE1234F1Z", 0), # Missing checksum ("27ABCDE1234F1Z5", 1), # Valid GSTIN ], ) @@ -98,16 +100,15 @@ def test_gstin_validation(text, expected_len, recognizer, entities): ("07PQRST6789K1Z2", True), ("01ABCDE1234F1Z5", True), ("37ABCDE1234F1Z5", True), - # Invalid GSTINs - ("27ABCDE1234F1Z", False), # Too short - ("27ABCDE1234F1Z55", False), # Too long + ("27ABCDE1234F1Z", False), # Too short + ("27ABCDE1234F1Z55", False), # Too long ("00ABCDE1234F1Z5", False), # Invalid state code ("38ABCDE1234F1Z5", False), # Invalid state code ("27ABCDE1234F1Y5", False), # Missing 'Z' at position 14 - ("27ABCDE1234F1Z", False), # Missing checksum - ("27ABCDE1234F1Z5", True), # Valid - ("27ABCDE1234F1Z5", True), # Valid with different case + ("27ABCDE1234F1Z", False), # Missing checksum + ("27ABCDE1234F1Z5", True), # Valid + ("27ABCDE1234F1Z5", True), # Valid with different case ], ) def test_validate_result(gstin, expected, recognizer): @@ -123,13 +124,12 @@ def test_validate_result(gstin, expected, recognizer): ("ABCDE1234F", True), ("PQRST6789K", True), ("ABCDE1234F", True), - # Invalid PAN formats - ("ABCD1234F", False), # Too short - ("ABCDE12345F", False), # Too long + ("ABCD1234F", False), # Too short + ("ABCDE12345F", False), # Too long ("12345ABCDE", False), # Numbers first - ("ABCDE1234", False), # Missing last letter - ("ABCDE1234F", True), # Valid + ("ABCDE1234", False), # Missing last letter + ("ABCDE1234F", True), # Valid ], ) def test_validate_pan_format(pan, expected, recognizer): @@ -157,7 +157,7 @@ def test_sanitize_value(text, expected, recognizer): def test_gstin_recognizer_initialization(): """Test GSTIN recognizer initialization with default parameters.""" recognizer = InGstinRecognizer() - + assert recognizer.supported_entity == "IN_GSTIN" assert recognizer.supported_language == "en" assert len(recognizer.patterns) == 3 @@ -170,11 +170,9 @@ def test_gstin_recognizer_with_custom_params(): """Test GSTIN recognizer initialization with custom parameters.""" custom_context = ["custom", "context"] recognizer = InGstinRecognizer( - context=custom_context, - supported_language="hi", - supported_entity="CUSTOM_GSTIN" + context=custom_context, supported_language="hi", supported_entity="CUSTOM_GSTIN" ) - + assert recognizer.supported_entity == "CUSTOM_GSTIN" assert recognizer.supported_language == "hi" assert recognizer.context == custom_context @@ -184,9 +182,9 @@ def test_gstin_recognizer_replacement_pairs(): """Test GSTIN recognizer with custom replacement pairs.""" custom_replacement_pairs = [("-", ""), (" ", ""), (".", "")] recognizer = InGstinRecognizer(replacement_pairs=custom_replacement_pairs) - + assert recognizer.replacement_pairs == custom_replacement_pairs - + # Test sanitization with custom replacement pairs result = recognizer._sanitize_value("27-ABCDE-1234-F1-Z5") assert result == "27ABCDE1234F1Z5" diff --git a/presidio-analyzer/tests/test_in_passport_recognizer.py b/presidio-analyzer/tests/test_in_passport_recognizer.py index 327aa78d78..7c22895f81 100644 --- a/presidio-analyzer/tests/test_in_passport_recognizer.py +++ b/presidio-analyzer/tests/test_in_passport_recognizer.py @@ -1,15 +1,18 @@ -import pytest +import pytest from presidio_analyzer.predefined_recognizers import InPassportRecognizer from tests.assertions import assert_result + @pytest.fixture(scope="module") def recognizer(): return InPassportRecognizer() + @pytest.fixture(scope="module") def entities(): return ["IN_PASSPORT"] + @pytest.mark.parametrize( "text, expected_len, expected_position, expected_score", [ @@ -28,12 +31,12 @@ def entities(): ], ) def test_when_all_passport_numbers_then_succeed( - text, - expected_len, - expected_position, - expected_score, - recognizer, - entities, + text, + expected_len, + expected_position, + expected_score, + recognizer, + entities, ): results = recognizer.analyze(text, entities) assert len(results) == expected_len diff --git a/presidio-analyzer/tests/test_kr_brn_recognizer.py b/presidio-analyzer/tests/test_kr_brn_recognizer.py index 4711832135..2688599329 100644 --- a/presidio-analyzer/tests/test_kr_brn_recognizer.py +++ b/presidio-analyzer/tests/test_kr_brn_recognizer.py @@ -1,16 +1,21 @@ import pytest from tests import assert_result_within_score_range -from presidio_analyzer.predefined_recognizers.country_specific.korea import KrBrnRecognizer +from presidio_analyzer.predefined_recognizers.country_specific.korea import ( + KrBrnRecognizer, +) + @pytest.fixture(scope="module") def recognizer(): return KrBrnRecognizer() + @pytest.fixture(scope="module") def entities(): return ["KR_BRN"] + @pytest.mark.parametrize( "text, expected_len, expected_positions, expected_score_ranges", [ diff --git a/presidio-analyzer/tests/test_kr_frn_recognizer.py b/presidio-analyzer/tests/test_kr_frn_recognizer.py index 05bd40fde4..fd5cebfdd8 100644 --- a/presidio-analyzer/tests/test_kr_frn_recognizer.py +++ b/presidio-analyzer/tests/test_kr_frn_recognizer.py @@ -1,42 +1,135 @@ import pytest from tests import assert_result_within_score_range -from presidio_analyzer.predefined_recognizers.country_specific.korea.kr_frn_recognizer import KrFrnRecognizer +from presidio_analyzer.predefined_recognizers.country_specific.korea.kr_frn_recognizer import ( + KrFrnRecognizer, +) + @pytest.fixture(scope="module") def recognizer(): return KrFrnRecognizer() + @pytest.fixture(scope="module") def entities(): return ["KR_FRN"] + @pytest.mark.parametrize( "text, expected_len, expected_positions, expected_score_ranges", [ # Valid FRNs, but medium match - ("911124-5678901", 1, ((0, 14),), ((0.5, 0.5),), ), - ("9111245678901", 1, ((0, 13),), ((0.5, 0.5),), ), - ("000505-7637892", 1, ((0, 14),), ((0.5, 0.5),), ), - ("0005056637892", 1, ((0, 13),), ((0.5, 0.5),), ), - ("His Korean FRN is 911124-5678901", 1, ((18, 32),), ((0.5, 0.5),), ), - + ( + "911124-5678901", + 1, + ((0, 14),), + ((0.5, 0.5),), + ), + ( + "9111245678901", + 1, + ((0, 13),), + ((0.5, 0.5),), + ), + ( + "000505-7637892", + 1, + ((0, 14),), + ((0.5, 0.5),), + ), + ( + "0005056637892", + 1, + ((0, 13),), + ((0.5, 0.5),), + ), + ( + "His Korean FRN is 911124-5678901", + 1, + ((18, 32),), + ((0.5, 0.5),), + ), # Valid FRNs, strong match by validate_result() - ("911124-5678906", 1, ((0, 14),), ((1.0, 1.0),), ), - ("9111245678906", 1, ((0, 13),), ((1.0, 1.0),), ), - ("050912-6000012", 1, ((0, 14),), ((1.0, 1.0),), ), - ("0509126000012", 1, ((0, 13),), ((1.0, 1.0),), ), - ("His FRN is 9111245678906", 1, ((11, 24),), ((1.0, 1.0),), ), - - # Invalid FRNs - ("001332-1234567", 0, (), (),), - ("0013321234567", 0, (), (),), - ("960121+1021413", 0, (), (),), - ("960111-10214131", 0, (), (),), - ("960303-0021413", 0, (), (),), - ("760413-1212134", 0, (), (),), - ("000402-2214431", 0, (), (),), - ("051102-9234110", 0, (), (),), + ( + "911124-5678906", + 1, + ((0, 14),), + ((1.0, 1.0),), + ), + ( + "9111245678906", + 1, + ((0, 13),), + ((1.0, 1.0),), + ), + ( + "050912-6000012", + 1, + ((0, 14),), + ((1.0, 1.0),), + ), + ( + "0509126000012", + 1, + ((0, 13),), + ((1.0, 1.0),), + ), + ( + "His FRN is 9111245678906", + 1, + ((11, 24),), + ((1.0, 1.0),), + ), + # Invalid FRNs + ( + "001332-1234567", + 0, + (), + (), + ), + ( + "0013321234567", + 0, + (), + (), + ), + ( + "960121+1021413", + 0, + (), + (), + ), + ( + "960111-10214131", + 0, + (), + (), + ), + ( + "960303-0021413", + 0, + (), + (), + ), + ( + "760413-1212134", + 0, + (), + (), + ), + ( + "000402-2214431", + 0, + (), + (), + ), + ( + "051102-9234110", + 0, + (), + (), + ), ], ) def test_when_all_frns_then_succeed( diff --git a/presidio-analyzer/tests/test_kr_passport_recognizer.py b/presidio-analyzer/tests/test_kr_passport_recognizer.py index b6d7d93f35..17a985daec 100644 --- a/presidio-analyzer/tests/test_kr_passport_recognizer.py +++ b/presidio-analyzer/tests/test_kr_passport_recognizer.py @@ -1,16 +1,21 @@ import pytest from tests import assert_result_within_score_range -from presidio_analyzer.predefined_recognizers.country_specific.korea.kr_passport_recognizer import KrPassportRecognizer +from presidio_analyzer.predefined_recognizers.country_specific.korea.kr_passport_recognizer import ( + KrPassportRecognizer, +) + @pytest.fixture(scope="module") def recognizer(): return KrPassportRecognizer() + @pytest.fixture(scope="module") def entities(): return ["KR_PASSPORT"] + @pytest.mark.parametrize( "text, expected_len, expected_positions, expected_score_ranges", [ @@ -25,7 +30,6 @@ def entities(): ("My passport number is M123A4567", 1, ((22, 31),), ((0.1, 0.1),)), ("Korean passport: M456B7890", 1, ((17, 26),), ((0.1, 0.1),)), ("여권번호는 M789C1234입니다", 1, ((6, 15),), ((0.1, 0.1),)), - # Valid previous format passports (M + 8 digits) ("M12345678", 1, ((0, 9),), ((0.05, 0.05),)), ("m87654321", 1, ((0, 9),), ((0.05, 0.05),)), @@ -33,27 +37,25 @@ def entities(): ("s99887766", 1, ((0, 9),), ((0.05, 0.05),)), ("My old passport M12345678", 1, ((16, 25),), ((0.05, 0.05),)), ("대한민국 여권 S87654321", 1, ((8, 17),), ((0.05, 0.05),)), - # Multiple passport numbers ("M123A4567 and M456B7890", 2, ((0, 9), (14, 23)), ((0.1, 0.1), (0.1, 0.1))), - # Invalid formats - should not match ("A123B4567", 0, (), ()), # Wrong first letter - ("M12A4567", 0, (), ()), # Too few digits before letter + ("M12A4567", 0, (), ()), # Too few digits before letter ("M1234A567", 0, (), ()), # Too many digits before letter ("M123AB567", 0, (), ()), # Two letters - ("M123A456", 0, (), ()), # Too few final digits - ("M123A45678", 0, (), ()), # Too many final digits - ("M1234567", 0, (), ()), # Missing letter in current format - ("M123456789", 0, (), ()), # Too many digits for old format - ("M1234567", 0, (), ()), # Too few digits for old format - ("123A4567", 0, (), ()), # Missing M - ("MM123A4567", 0, (), ()), # Double M - ("M123 A4567", 0, (), ()), # Space in number - ("M123-A4567", 0, (), ()), # Hyphen in number - ("", 0, (), ()), # Empty string - ("passport", 0, (), ()), # Just text - ("M123a456", 0, (), ()), # Wrong number of digits after letter + ("M123A456", 0, (), ()), # Too few final digits + ("M123A45678", 0, (), ()), # Too many final digits + ("M1234567", 0, (), ()), # Missing letter in current format + ("M123456789", 0, (), ()), # Too many digits for old format + ("M1234567", 0, (), ()), # Too few digits for old format + ("123A4567", 0, (), ()), # Missing M + ("MM123A4567", 0, (), ()), # Double M + ("M123 A4567", 0, (), ()), # Space in number + ("M123-A4567", 0, (), ()), # Hyphen in number + ("", 0, (), ()), # Empty string + ("passport", 0, (), ()), # Just text + ("M123a456", 0, (), ()), # Wrong number of digits after letter ], ) def test_when_all_passports_then_succeed( @@ -69,14 +71,17 @@ def test_when_all_passports_then_succeed( assert len(results) == expected_len for res, (st_pos, fn_pos), (st_score, fn_score) in zip( results, expected_positions, expected_score_ranges - ): - print(f"res: {res}, st_pos: {st_pos}, fn_pos: {fn_pos}, st_score: {st_score}, fn_score: {fn_score}") + ): + print( + f"res: {res}, st_pos: {st_pos}, fn_pos: {fn_pos}, st_score: {st_score}, fn_score: {fn_score}" + ) if fn_score == "max": fn_score = max_score assert_result_within_score_range( res, entities[0], st_pos, fn_pos, st_score, fn_score ) + def test_when_passport_with_context_then_succeed(recognizer, entities): """Test passport recognition with context keywords""" text = "Korean passport number: M123A4567" @@ -86,6 +91,7 @@ def test_when_passport_with_context_then_succeed(recognizer, entities): assert results[0].end == 33 assert results[0].entity_type == "KR_PASSPORT" + def test_when_passport_korean_context_then_succeed(recognizer, entities): """Test passport recognition with Korean context""" text = "대한민국 여권번호: M456B7890" @@ -95,16 +101,17 @@ def test_when_passport_korean_context_then_succeed(recognizer, entities): assert results[0].end == 20 assert results[0].entity_type == "KR_PASSPORT" + def test_when_no_passport_then_no_results(recognizer, entities): """Test that invalid formats return no results""" invalid_texts = [ "This is just text", "A123B4567", # Wrong prefix - "M12A4567", # Wrong format - "M123456789", # Too long for old format - "123A4567", # Missing M prefix + "M12A4567", # Wrong format + "M123456789", # Too long for old format + "123A4567", # Missing M prefix ] - + for text in invalid_texts: results = recognizer.analyze(text, entities) - assert len(results) == 0, f"Expected no results for text: {text}" \ No newline at end of file + assert len(results) == 0, f"Expected no results for text: {text}" diff --git a/presidio-analyzer/tests/test_kr_rrn_recognizer.py b/presidio-analyzer/tests/test_kr_rrn_recognizer.py index 3a1719e079..ae9a4c2242 100644 --- a/presidio-analyzer/tests/test_kr_rrn_recognizer.py +++ b/presidio-analyzer/tests/test_kr_rrn_recognizer.py @@ -3,40 +3,131 @@ from tests import assert_result_within_score_range from presidio_analyzer.predefined_recognizers import KrRrnRecognizer + @pytest.fixture(scope="module") def recognizer(): return KrRrnRecognizer() + @pytest.fixture(scope="module") def entities(): return ["KR_RRN"] + @pytest.mark.parametrize( "text, expected_len, expected_positions, expected_score_ranges", [ # Valid RRNs, but medium match - ("960121-1234567", 1, ((0, 14),), ((0.5, 0.5),), ), - ("9601211234567", 1, ((0, 13),), ((0.5, 0.5),), ), - ("000505-3637892", 1, ((0, 14),), ((0.5, 0.5),), ), - ("0005053637892", 1, ((0, 13),), ((0.5, 0.5),), ), - ("His Korean RRN is 960121-1234567", 1, ((18, 32),), ((0.5, 0.5),), ), - + ( + "960121-1234567", + 1, + ((0, 14),), + ((0.5, 0.5),), + ), + ( + "9601211234567", + 1, + ((0, 13),), + ((0.5, 0.5),), + ), + ( + "000505-3637892", + 1, + ((0, 14),), + ((0.5, 0.5),), + ), + ( + "0005053637892", + 1, + ((0, 13),), + ((0.5, 0.5),), + ), + ( + "His Korean RRN is 960121-1234567", + 1, + ((18, 32),), + ((0.5, 0.5),), + ), # Valid RRNs, strong match by validate_result() - ("960121-1021413", 1, ((0, 14),), ((1.0, 1.0),), ), - ("9601211021413", 1, ((0, 13),), ((1.0, 1.0),), ), - ("050912-2000019", 1, ((0, 14),), ((1.0, 1.0),), ), - ("0509122000019", 1, ((0, 13),), ((1.0, 1.0),), ), - ("His RRN is 9601211021413", 1, ((11, 24),), ((1.0, 1.0),), ), - - # Invalid RRNs - ("001332-1234567", 0, (), (),), - ("0013321234567", 0, (), (),), - ("960121+1021413", 0, (), (),), - ("960111-10214131", 0, (), (),), - ("960303-0021413", 0, (), (),), - ("760413-5212134", 0, (), (),), - ("000402-6214431", 0, (), (),), - ("051102-9234110", 0, (), (),), + ( + "960121-1021413", + 1, + ((0, 14),), + ((1.0, 1.0),), + ), + ( + "9601211021413", + 1, + ((0, 13),), + ((1.0, 1.0),), + ), + ( + "050912-2000019", + 1, + ((0, 14),), + ((1.0, 1.0),), + ), + ( + "0509122000019", + 1, + ((0, 13),), + ((1.0, 1.0),), + ), + ( + "His RRN is 9601211021413", + 1, + ((11, 24),), + ((1.0, 1.0),), + ), + # Invalid RRNs + ( + "001332-1234567", + 0, + (), + (), + ), + ( + "0013321234567", + 0, + (), + (), + ), + ( + "960121+1021413", + 0, + (), + (), + ), + ( + "960111-10214131", + 0, + (), + (), + ), + ( + "960303-0021413", + 0, + (), + (), + ), + ( + "760413-5212134", + 0, + (), + (), + ), + ( + "000402-6214431", + 0, + (), + (), + ), + ( + "051102-9234110", + 0, + (), + (), + ), ], ) def test_when_all_rrns_then_succeed( diff --git a/presidio-analyzer/tests/test_langextract_helper.py b/presidio-analyzer/tests/test_langextract_helper.py index c04901f1c7..1640842652 100644 --- a/presidio-analyzer/tests/test_langextract_helper.py +++ b/presidio-analyzer/tests/test_langextract_helper.py @@ -1,4 +1,5 @@ """Tests for llm_utils.langextract_helper module.""" + import pytest from unittest.mock import Mock, MagicMock, patch from presidio_analyzer.llm_utils.langextract_helper import ( @@ -20,7 +21,7 @@ def test_when_lm_recognizer_section_exists_then_extracts_all_fields(self): "supported_entities": ["PERSON", "EMAIL"], "min_score": 0.7, "labels_to_ignore": ["system"], - "enable_generic_consolidation": False + "enable_generic_consolidation": False, } } @@ -44,11 +45,7 @@ def test_when_lm_recognizer_missing_then_returns_defaults(self): def test_when_partial_config_then_uses_defaults_for_missing_fields(self): """Test that defaults are used for missing fields.""" - config = { - "lm_recognizer": { - "supported_entities": ["PERSON"] - } - } + config = {"lm_recognizer": {"supported_entities": ["PERSON"]}} result = extract_lm_config(config) @@ -59,11 +56,7 @@ def test_when_partial_config_then_uses_defaults_for_missing_fields(self): def test_when_only_min_score_provided_then_uses_defaults_for_others(self): """Test partial config with only min_score.""" - config = { - "lm_recognizer": { - "min_score": 0.8 - } - } + config = {"lm_recognizer": {"min_score": 0.8}} result = extract_lm_config(config) @@ -122,7 +115,7 @@ def test_when_mapping_provided_then_creates_reverse(self): entity_mappings = { "person": "PERSON", "email": "EMAIL_ADDRESS", - "phone": "PHONE_NUMBER" + "phone": "PHONE_NUMBER", } result = create_reverse_entity_mapping(entity_mappings) @@ -139,10 +132,7 @@ def test_when_empty_mapping_then_returns_empty_dict(self): def test_when_duplicate_values_then_last_wins(self): """Test behavior with duplicate values in original mapping.""" - entity_mappings = { - "person1": "PERSON", - "person2": "PERSON" # Duplicate value - } + entity_mappings = {"person1": "PERSON", "person2": "PERSON"} # Duplicate value result = create_reverse_entity_mapping(entity_mappings) @@ -238,13 +228,12 @@ def test_when_custom_alignment_scores_provided_then_uses_custom(self): """Test using custom alignment scores.""" extraction = Mock() extraction.alignment_status = "MATCH_EXACT" - - custom_scores = { - "MATCH_EXACT": 0.99, - "MATCH_FUZZY": 0.75 - } - result = calculate_extraction_confidence(extraction, alignment_scores=custom_scores) + custom_scores = {"MATCH_EXACT": 0.99, "MATCH_FUZZY": 0.75} + + result = calculate_extraction_confidence( + extraction, alignment_scores=custom_scores + ) assert result == 0.99 @@ -252,13 +241,15 @@ def test_when_custom_scores_missing_status_then_returns_default(self): """Test that default is returned when custom scores don't have the status.""" extraction = Mock() extraction.alignment_status = "MATCH_FUZZY" - + custom_scores = { "MATCH_EXACT": 0.99 # MATCH_FUZZY missing } - result = calculate_extraction_confidence(extraction, alignment_scores=custom_scores) + result = calculate_extraction_confidence( + extraction, alignment_scores=custom_scores + ) assert result == 0.85 # Default score @@ -280,6 +271,15 @@ def test_default_alignment_scores_values_are_valid(self): def test_default_alignment_scores_are_ordered_correctly(self): """Test that scores are in descending order of confidence.""" - assert DEFAULT_ALIGNMENT_SCORES["MATCH_EXACT"] > DEFAULT_ALIGNMENT_SCORES["MATCH_FUZZY"] - assert DEFAULT_ALIGNMENT_SCORES["MATCH_FUZZY"] > DEFAULT_ALIGNMENT_SCORES["MATCH_LESSER"] - assert DEFAULT_ALIGNMENT_SCORES["MATCH_LESSER"] > DEFAULT_ALIGNMENT_SCORES["NOT_ALIGNED"] + assert ( + DEFAULT_ALIGNMENT_SCORES["MATCH_EXACT"] + > DEFAULT_ALIGNMENT_SCORES["MATCH_FUZZY"] + ) + assert ( + DEFAULT_ALIGNMENT_SCORES["MATCH_FUZZY"] + > DEFAULT_ALIGNMENT_SCORES["MATCH_LESSER"] + ) + assert ( + DEFAULT_ALIGNMENT_SCORES["MATCH_LESSER"] + > DEFAULT_ALIGNMENT_SCORES["NOT_ALIGNED"] + ) diff --git a/presidio-analyzer/tests/test_language_validation.py b/presidio-analyzer/tests/test_language_validation.py index 7cc22909ff..511b73722a 100644 --- a/presidio-analyzer/tests/test_language_validation.py +++ b/presidio-analyzer/tests/test_language_validation.py @@ -8,6 +8,7 @@ def test_configuration_validator_language_codes_no_exception(): valid_languages = ["en", "es", "fr", "en-US", "es-ES"] validate_language_codes(valid_languages) + def test_configuration_validator_language_codes_invalid(): """Test ConfigurationValidator rejects invalid language codes.""" invalid_languages = ["invalid_lang"] diff --git a/presidio-analyzer/tests/test_lemma_context_aware_enhancer.py b/presidio-analyzer/tests/test_lemma_context_aware_enhancer.py index 74df0e8e85..94f0611e53 100644 --- a/presidio-analyzer/tests/test_lemma_context_aware_enhancer.py +++ b/presidio-analyzer/tests/test_lemma_context_aware_enhancer.py @@ -18,18 +18,18 @@ def test_when_index_finding_then_succeed(): def test_when_context_word_substring_then_no_false_match(): """ Test that substring matching does not cause false positives. - + This test verifies the fix for issue #1061 where 'lic' was matching 'duplicate' as a substring. With whole-word matching, 'lic' should only match 'lic' exactly, not substrings like 'duplicate'. """ context_list = ["duplicate", "license", "driver"] recognizer_context_list = ["lic"] - + result = LemmaContextAwareEnhancer._find_supportive_word_in_context( context_list, recognizer_context_list, matching_mode="whole_word" ) - + # 'lic' should NOT match 'duplicate' (substring match - should be False) # 'lic' should NOT match 'license' (not exact match - should be False) # 'lic' should NOT match 'driver' (not exact match - should be False) @@ -39,16 +39,16 @@ def test_when_context_word_substring_then_no_false_match(): def test_when_context_word_exact_match_then_succeed(): """ Test that exact whole-word matches work correctly. - + 'lic' should match 'lic' exactly (case-insensitive). """ context_list = ["lic", "license", "driver"] recognizer_context_list = ["lic"] - + result = LemmaContextAwareEnhancer._find_supportive_word_in_context( context_list, recognizer_context_list, matching_mode="whole_word" ) - + # 'lic' should match 'lic' exactly assert result == "lic" @@ -56,27 +56,27 @@ def test_when_context_word_exact_match_then_succeed(): def test_when_context_word_case_insensitive_then_succeed(): """ Test that matching is case-insensitive. - + 'LIC' should match 'lic' and vice versa. """ context_list = ["LIC", "license", "driver"] recognizer_context_list = ["lic"] - + result = LemmaContextAwareEnhancer._find_supportive_word_in_context( context_list, recognizer_context_list, matching_mode="whole_word" ) - + # 'lic' should match 'LIC' (case-insensitive) assert result == "lic" - + # Test reverse case context_list = ["lic", "license", "driver"] recognizer_context_list = ["LIC"] - + result = LemmaContextAwareEnhancer._find_supportive_word_in_context( context_list, recognizer_context_list, matching_mode="whole_word" ) - + # 'LIC' should match 'lic' (case-insensitive) assert result == "LIC" @@ -87,11 +87,11 @@ def test_when_context_word_multiple_matches_then_first_match_returned(): """ context_list = ["driver", "license", "permit"] recognizer_context_list = ["license", "driver", "permit"] - + result = LemmaContextAwareEnhancer._find_supportive_word_in_context( context_list, recognizer_context_list, matching_mode="whole_word" ) - + # Should return the first match from recognizer_context_list assert result == "license" @@ -102,11 +102,11 @@ def test_when_context_word_no_match_then_empty_string(): """ context_list = ["random", "words", "here"] recognizer_context_list = ["lic", "driver", "license"] - + result = LemmaContextAwareEnhancer._find_supportive_word_in_context( context_list, recognizer_context_list, matching_mode="whole_word" ) - + assert result == "" @@ -115,14 +115,14 @@ def test_when_duplicate_word_in_text_then_lic_context_not_matched_with_whole_wor ): """ Integration test for issue #1061: 'lic' should not match 'duplicate' with whole_word mode. - + This test verifies that when using "whole_word" mode, the context word 'lic' (from US_DRIVER_LICENSE recognizer) should NOT cause false context enhancement when 'duplicate' appears in text. """ from presidio_analyzer import PatternRecognizer, Pattern from presidio_analyzer.context_aware_enhancers import LemmaContextAwareEnhancer - + # Create a recognizer with 'lic' as context word (similar to US_DRIVER_LICENSE) test_recognizer = PatternRecognizer( supported_entity="TEST_LICENSE", @@ -130,29 +130,27 @@ def test_when_duplicate_word_in_text_then_lic_context_not_matched_with_whole_wor context=["lic"], # This is the problematic context word patterns=[Pattern("test_pattern", r"\b[A-Z0-9]{8}\b", 0.3)], ) - + # Test case: Text with 'duplicate' but no actual license context words # The word 'duplicate' contains 'lic' as a substring, but should NOT match with whole_word mode text = "This is a duplicate document with code ABC12345" nlp_artifacts = spacy_nlp_engine.process_text(text, "en") - + recognizer_results = test_recognizer.analyze(text, nlp_artifacts) assert len(recognizer_results) > 0 - + original_score = recognizer_results[0].score - + # Use whole_word mode to prevent false positives enhancer = LemmaContextAwareEnhancer(context_matching_mode="whole_word") enhanced_results = enhancer.enhance_using_context( text, recognizer_results, nlp_artifacts, [test_recognizer] ) - + # With whole-word matching, 'lic' should NOT match 'duplicate' # So the score should NOT be enhanced and supportive_context_word should be empty assert enhanced_results[0].score == original_score - assert ( - enhanced_results[0].analysis_explanation.supportive_context_word == "" - ) + assert enhanced_results[0].analysis_explanation.supportive_context_word == "" def test_when_duplicate_word_in_text_then_lic_context_matches_with_substring( @@ -160,13 +158,13 @@ def test_when_duplicate_word_in_text_then_lic_context_matches_with_substring( ): """ Test that substring mode (default) maintains backward compatibility. - + This test verifies that with the default "substring" mode, 'lic' DOES match 'duplicate' (the original behavior), maintaining backward compatibility. """ from presidio_analyzer import PatternRecognizer, Pattern from presidio_analyzer.context_aware_enhancers import LemmaContextAwareEnhancer - + # Create a recognizer with 'lic' as context word test_recognizer = PatternRecognizer( supported_entity="TEST_LICENSE", @@ -174,55 +172,53 @@ def test_when_duplicate_word_in_text_then_lic_context_matches_with_substring( context=["lic"], patterns=[Pattern("test_pattern", r"\b[A-Z0-9]{8}\b", 0.3)], ) - + # Text with 'duplicate' - with substring mode (default), 'lic' should match text = "This is a duplicate document with code ABC12345" nlp_artifacts = spacy_nlp_engine.process_text(text, "en") - + recognizer_results = test_recognizer.analyze(text, nlp_artifacts) assert len(recognizer_results) > 0 - + original_score = recognizer_results[0].score - + # Use default substring mode (backward compatible behavior) enhancer = LemmaContextAwareEnhancer() # Default is "substring" enhanced_results = enhancer.enhance_using_context( text, recognizer_results, nlp_artifacts, [test_recognizer] ) - + # With substring matching (default), 'lic' DOES match 'duplicate' (original behavior) # So the score should be enhanced assert enhanced_results[0].score > original_score - assert ( - enhanced_results[0].analysis_explanation.supportive_context_word == "lic" - ) + assert enhanced_results[0].analysis_explanation.supportive_context_word == "lic" def test_when_substring_matching_mode_then_compound_words_match(): """ Test that substring matching mode works for compound words. - + This verifies that 'card' can match 'creditcard' when using substring mode, which is useful for handling words without spaces. """ context_list = ["creditcard", "passportnumber", "driverlicense"] recognizer_context_list = ["card", "passport", "license"] - + # With substring matching, 'card' should match 'creditcard' result = LemmaContextAwareEnhancer._find_supportive_word_in_context( context_list, recognizer_context_list, matching_mode="substring" ) - + assert result == "card" - + # Test that 'lic' also matches 'duplicate' in substring mode (the original behavior) context_list = ["duplicate", "license"] recognizer_context_list = ["lic"] - + result = LemmaContextAwareEnhancer._find_supportive_word_in_context( context_list, recognizer_context_list, matching_mode="substring" ) - + # In substring mode, 'lic' matches 'duplicate' (original behavior) assert result == "lic" @@ -232,10 +228,10 @@ def test_when_invalid_matching_mode_then_raises_error(): Test that invalid matching mode raises ValueError. """ import pytest - + with pytest.raises(ValueError, match="context_matching_mode must be one of"): LemmaContextAwareEnhancer(context_matching_mode="invalid_mode") - + # Test that "hybrid" is no longer a valid mode (it was redundant) with pytest.raises(ValueError, match="context_matching_mode must be one of"): LemmaContextAwareEnhancer(context_matching_mode="hybrid") @@ -244,13 +240,13 @@ def test_when_invalid_matching_mode_then_raises_error(): def test_when_substring_mode_then_compound_words_work_in_integration(spacy_nlp_engine): """ Integration test for substring matching with compound words. - + This test verifies that substring matching works end-to-end for cases like "passportnumber" matching "passport" context word. """ from presidio_analyzer import PatternRecognizer, Pattern from presidio_analyzer.context_aware_enhancers import LemmaContextAwareEnhancer - + # Create a recognizer with 'passport' as context word test_recognizer = PatternRecognizer( supported_entity="TEST_PASSPORT", @@ -258,22 +254,22 @@ def test_when_substring_mode_then_compound_words_work_in_integration(spacy_nlp_e context=["passport"], # Context word that should match in compound words patterns=[Pattern("test_pattern", r"\b[A-Z0-9]{8}\b", 0.3)], ) - + # Text with compound word "passportnumber" (no space) text = "My passportnumber is ABC12345" nlp_artifacts = spacy_nlp_engine.process_text(text, "en") - + recognizer_results = test_recognizer.analyze(text, nlp_artifacts) assert len(recognizer_results) > 0 - + original_score = recognizer_results[0].score - + # Use substring matching mode enhancer = LemmaContextAwareEnhancer(context_matching_mode="substring") enhanced_results = enhancer.enhance_using_context( text, recognizer_results, nlp_artifacts, [test_recognizer] ) - + # With substring matching, 'passport' should match 'passportnumber' # So the score should be enhanced assert enhanced_results[0].score > original_score diff --git a/presidio-analyzer/tests/test_lm_recognizer.py b/presidio-analyzer/tests/test_lm_recognizer.py index e6d4c39823..961dc9ef2f 100644 --- a/presidio-analyzer/tests/test_lm_recognizer.py +++ b/presidio-analyzer/tests/test_lm_recognizer.py @@ -1,4 +1,5 @@ """Tests for LMRecognizer base class.""" + import pytest from unittest.mock import Mock @@ -18,7 +19,7 @@ def __init__(self, **kwargs): def _call_llm(self, text, entities, **kwargs): """Mock implementation that returns RecognizerResult objects.""" - return self.mock_entities if hasattr(self, 'mock_entities') else [] + return self.mock_entities if hasattr(self, "mock_entities") else [] class TestLMRecognizerAnalyze: @@ -27,10 +28,10 @@ class TestLMRecognizerAnalyze: def test_when_text_is_empty_then_returns_empty_list(self): """Test that empty text returns empty results.""" recognizer = ConcreteLMRecognizer() - + results = recognizer.analyze("") assert results == [] - + results = recognizer.analyze(" ") assert results == [] @@ -40,7 +41,7 @@ def test_when_entities_is_none_then_uses_all_supported_entities(self): recognizer.mock_entities = [ RecognizerResult(entity_type="PERSON", start=0, end=4, score=0.85) ] - + results = recognizer.analyze("John", entities=None) assert len(results) == 1 assert results[0].entity_type == "PERSON" @@ -49,7 +50,7 @@ def test_when_requested_entity_not_supported_then_returns_empty(self): """Test that unsupported entities return empty results.""" recognizer = ConcreteLMRecognizer() recognizer.mock_entities = [] - + results = recognizer.analyze("test", entities=["CREDIT_CARD"]) assert results == [] @@ -62,12 +63,12 @@ def test_when_llm_returns_entities_then_returns_recognizer_results(self): start=11, end=19, score=0.9, - recognition_metadata={"source": "test"} + recognition_metadata={"source": "test"}, ) ] - + results = recognizer.analyze("My name is John Doe", entities=["PERSON"]) - + assert len(results) == 1 assert isinstance(results[0], RecognizerResult) assert results[0].entity_type == "PERSON" @@ -82,27 +83,28 @@ def test_when_entity_below_min_score_then_filters_out(self): RecognizerResult(entity_type="PERSON", start=0, end=4, score=0.9), RecognizerResult(entity_type="PERSON", start=5, end=9, score=0.7), ] - + results = recognizer.analyze("John Jane") - + assert len(results) == 1 assert results[0].score == 0.9 def test_when_llm_raises_exception_then_returns_empty_list(self): """Test that exceptions are caught and empty list returned.""" + class ErrorLMRecognizer(LMRecognizer): """Recognizer that raises error in _call_llm.""" + def __init__(self): super().__init__( - supported_entities=["PERSON"], - name="Error Test Recognizer" + supported_entities=["PERSON"], name="Error Test Recognizer" ) - + def _call_llm(self, text, entities, **kwargs): raise RuntimeError("LLM API error") - + recognizer = ErrorLMRecognizer() - + with pytest.raises(RuntimeError, match="LLM API error"): recognizer.analyze("test text", entities=["PERSON"]) @@ -110,10 +112,14 @@ def test_when_entity_missing_required_fields_then_skips(self): """Test that entities with missing required fields are skipped.""" recognizer = ConcreteLMRecognizer() recognizer.mock_entities = [ - RecognizerResult(entity_type="", start=0, end=4, score=0.85), # Missing type - RecognizerResult(entity_type="PERSON", start=None, end=4, score=0.85), # Missing start + RecognizerResult( + entity_type="", start=0, end=4, score=0.85 + ), # Missing type + RecognizerResult( + entity_type="PERSON", start=None, end=4, score=0.85 + ), # Missing start ] - + results = recognizer.analyze("test") assert results == [] @@ -130,14 +136,17 @@ def test_when_entity_has_metadata_then_preserves_it(self): start=0, end=4, score=0.9, - recognition_metadata={"alignment": "MATCH_EXACT", "source": "test"} + recognition_metadata={"alignment": "MATCH_EXACT", "source": "test"}, ) ] - + results = recognizer.analyze("John") - + assert len(results) == 1 - assert results[0].recognition_metadata == {"alignment": "MATCH_EXACT", "source": "test"} + assert results[0].recognition_metadata == { + "alignment": "MATCH_EXACT", + "source": "test", + } assert results[0].score == 0.9 def test_when_entity_has_no_metadata_then_works_correctly(self): @@ -146,9 +155,9 @@ def test_when_entity_has_no_metadata_then_works_correctly(self): recognizer.mock_entities = [ RecognizerResult(entity_type="PERSON", start=0, end=4, score=0.85) ] - + results = recognizer.analyze("John") - + assert len(results) == 1 assert results[0].entity_type == "PERSON" assert results[0].score == 0.85 @@ -165,9 +174,9 @@ def test_when_entity_in_labels_to_ignore_then_filters_out(self): RecognizerResult(entity_type="EMAIL_ADDRESS", start=5, end=20, score=0.9), RecognizerResult(entity_type="LOCATION", start=21, end=30, score=0.9), ] - + results = recognizer.analyze("test text") - + # Only EMAIL_ADDRESS should remain (PERSON and LOCATION ignored, case-insensitive) assert len(results) == 1 assert results[0].entity_type == "EMAIL_ADDRESS" @@ -179,9 +188,9 @@ def test_when_unknown_entity_and_consolidation_enabled_then_creates_generic(self RecognizerResult(entity_type="UNKNOWN_TYPE", start=0, end=10, score=0.9), RecognizerResult(entity_type="PERSON", start=11, end=15, score=0.9), ] - + results = recognizer.analyze("test text") - + assert len(results) == 2 # Unknown type should be converted to GENERIC_PII_ENTITY assert results[0].entity_type == "GENERIC_PII_ENTITY" @@ -196,9 +205,9 @@ def test_when_unknown_entity_and_consolidation_disabled_then_skips(self): RecognizerResult(entity_type="UNKNOWN_TYPE", start=0, end=10, score=0.9), RecognizerResult(entity_type="PERSON", start=11, end=15, score=0.9), ] - + results = recognizer.analyze("test text") - + # Only known entity should be returned assert len(results) == 1 assert results[0].entity_type == "PERSON" @@ -212,12 +221,12 @@ def test_when_unknown_entity_with_existing_metadata_then_preserves_metadata(self start=0, end=10, score=0.9, - recognition_metadata={"source": "custom", "confidence": "high"} + recognition_metadata={"source": "custom", "confidence": "high"}, ), ] - + results = recognizer.analyze("test text") - + assert len(results) == 1 assert results[0].entity_type == "GENERIC_PII_ENTITY" assert results[0].recognition_metadata["original_entity_type"] == "CUSTOM_TYPE" @@ -228,7 +237,7 @@ def test_when_get_supported_entities_called_then_returns_list(self): """Test get_supported_entities returns the correct list.""" recognizer = ConcreteLMRecognizer() entities = recognizer.get_supported_entities() - + assert "PERSON" in entities assert "EMAIL_ADDRESS" in entities assert "GENERIC_PII_ENTITY" in entities # Added by default @@ -237,7 +246,7 @@ def test_when_generic_consolidation_disabled_then_no_generic_in_supported(self): """Test GENERIC_PII_ENTITY not added when consolidation disabled.""" recognizer = ConcreteLMRecognizer(enable_generic_consolidation=False) entities = recognizer.get_supported_entities() - + assert "PERSON" in entities assert "EMAIL_ADDRESS" in entities assert "GENERIC_PII_ENTITY" not in entities diff --git a/presidio-analyzer/tests/test_medical_ner_recognizer.py b/presidio-analyzer/tests/test_medical_ner_recognizer.py index bf2505f7a1..ddf2c8a0df 100644 --- a/presidio-analyzer/tests/test_medical_ner_recognizer.py +++ b/presidio-analyzer/tests/test_medical_ner_recognizer.py @@ -30,8 +30,7 @@ def _make_pipeline_prediction(entity_group, score, word, start, end): ".huggingface_ner_recognizer.hf_pipeline" ) _PATCH_TORCH = ( - "presidio_analyzer.predefined_recognizers.ner" - ".huggingface_ner_recognizer.torch" + "presidio_analyzer.predefined_recognizers.ner" ".huggingface_ner_recognizer.torch" ) _PATCH_DEVICE = ( "presidio_analyzer.predefined_recognizers.ner" diff --git a/presidio-analyzer/tests/test_ner_model_configuration.py b/presidio-analyzer/tests/test_ner_model_configuration.py index 56d12070e3..2ff9c08437 100644 --- a/presidio-analyzer/tests/test_ner_model_configuration.py +++ b/presidio-analyzer/tests/test_ner_model_configuration.py @@ -77,7 +77,7 @@ def test_ner_model_configuration_validation_success(): "stride": 16, "alignment_mode": "expand", "default_score": 0.9, - "low_confidence_score_multiplier": 0.3 + "low_confidence_score_multiplier": 0.3, } config = NerModelConfiguration.from_dict(config_data) @@ -86,17 +86,17 @@ def test_ner_model_configuration_validation_success(): assert config.default_score == 0.9 assert config.low_confidence_score_multiplier == 0.3 + def test_ner_model_configuration_invalid_score(): """Test NerModelConfiguration rejects invalid score values.""" - config_data = { - "default_score": 1.5 # Invalid: > 1.0 - } + config_data = {"default_score": 1.5} # Invalid: > 1.0 with pytest.raises(ValidationError) as exc_info: NerModelConfiguration.from_dict(config_data) assert "less than or equal to 1" in str(exc_info.value) + def test_backward_compatibility_ner_config_to_dict(): """Test that NerModelConfiguration maintains backward compatibility.""" config = NerModelConfiguration(default_score=0.8, stride=20) diff --git a/presidio-analyzer/tests/test_nlp_engine_provider.py b/presidio-analyzer/tests/test_nlp_engine_provider.py index f10517c3ed..3e423c3724 100644 --- a/presidio-analyzer/tests/test_nlp_engine_provider.py +++ b/presidio-analyzer/tests/test_nlp_engine_provider.py @@ -253,66 +253,62 @@ def test_when_valid_nlp_engines_then_return_default_configuration(): def test_when_nlp_engines_type_is_not_tuple_then_fail(): """Test that nlp_engines accepts lists (not just tuples) after removing legacy validation.""" nlp_engines = [SpacyNlpEngine, StanzaNlpEngine, TransformersNlpEngine] - + # After removing legacy validation, lists are now accepted (they work just as well as tuples) provider = NlpEngineProvider(nlp_engines=nlp_engines) engine = provider.create_engine() assert isinstance(engine, SpacyNlpEngine) + def test_when_invalid_nlp_engine_types_then_fail(): """Test that invalid nlp_engine types will fail when accessing attributes.""" nlp_engines = (1, 2, 3) - + # After removing legacy validation, this fails with AttributeError when accessing .is_available with pytest.raises(AttributeError): NlpEngineProvider(nlp_engines=nlp_engines) - def test_when_valid_nlp_configuration_then_return_default_configuration(): nlp_configuration = { "nlp_engine_name": "spacy", - "models": [{"lang_code": "en", "model_name": "en_core_web_lg"}] + "models": [{"lang_code": "en", "model_name": "en_core_web_lg"}], } provider = NlpEngineProvider(nlp_configuration=nlp_configuration) engine = provider.create_engine() assert isinstance(engine, SpacyNlpEngine) assert engine.nlp is not None - + def test_when_nlp_configuration_is_passed_instead_of_nlp_engines_then_fail(): """Test that passing nlp_configuration as positional argument fails.""" nlp_configuration = { "nlp_engine_name": "stanza", - "models": [{"lang_code": "en", "model_name": "en"}] + "models": [{"lang_code": "en", "model_name": "en"}], } - + # This fails because nlp_configuration is passed as positional arg (interpreted as nlp_engines) with pytest.raises(AttributeError): NlpEngineProvider(nlp_configuration) - def test_when_nlp_configuration_is_not_dict_then_fail(): nlp_configuration = "not a dict" - + with pytest.raises(ValueError): NlpEngineProvider(nlp_configuration=nlp_configuration) - + def test_when_nlp_configuration_is_missing_nlp_engine_name_key_then_fail(): - nlp_configuration = { - "models": [{"lang_code": "en", "model_name": "en"}] - } - + nlp_configuration = {"models": [{"lang_code": "en", "model_name": "en"}]} + with pytest.raises(ValueError): NlpEngineProvider(nlp_configuration=nlp_configuration) - + + def test_when_nlp_configuration_is_missing_models_key_then_fail(): - nlp_configuration = { - "nlp_engine_name": "stanza" - } - + nlp_configuration = {"nlp_engine_name": "stanza"} + with pytest.raises(ValueError): NlpEngineProvider(nlp_configuration=nlp_configuration) @@ -325,29 +321,30 @@ def test_when_valid_conf_file_then_return_default_configuration(): assert isinstance(engine, SpacyNlpEngine) assert engine.nlp is not None + def test_when_conf_file_is_empty_string_then_fail(): - conf_file = '' - + conf_file = "" + with pytest.raises(ValueError): NlpEngineProvider(conf_file=conf_file) def test_when_conf_file_is_not_string_or_path_then_fail(): conf_file = 1 - + with pytest.raises(TypeError): NlpEngineProvider(conf_file=conf_file) def test_when_conf_file_is_directory_then_fail(): - conf_file = '/' - + conf_file = "/" + with pytest.raises(ValueError): NlpEngineProvider(conf_file=conf_file) def test_when_conf_file_does_not_exist_then_fail(): - conf_file = 'test.yml' - + conf_file = "test.yml" + with pytest.raises(ValueError): NlpEngineProvider(conf_file=conf_file) diff --git a/presidio-analyzer/tests/test_pattern.py b/presidio-analyzer/tests/test_pattern.py index 255aa9e7b6..548818f7b6 100644 --- a/presidio-analyzer/tests/test_pattern.py +++ b/presidio-analyzer/tests/test_pattern.py @@ -34,7 +34,7 @@ def test_pattern_validation_success(): pattern_data = { "name": "US ZIP Code", "regex": r"\b\d{5}(?:-\d{4})?\b", - "score": 0.85 + "score": 0.85, } pattern = Pattern.from_dict(pattern_data) @@ -42,12 +42,13 @@ def test_pattern_validation_success(): assert pattern.score == 0.85 assert pattern.regex == r"\b\d{5}(?:-\d{4})?\b" + def test_pattern_validation_invalid_regex(): """Test that Pattern class rejects invalid regex patterns.""" pattern_data = { "name": "Invalid Pattern", "regex": "[unclosed_bracket", # Invalid regex - "score": 0.5 + "score": 0.5, } with pytest.raises(ValueError) as exc_info: @@ -59,7 +60,7 @@ def test_pattern_validation_invalid_score_range(): pattern_data = { "name": "Invalid Score", "regex": r"\btest\b", - "score": 1.5 # Invalid: > 1.0 + "score": 1.5, # Invalid: > 1.0 } with pytest.raises(ValueError) as exc_info: diff --git a/presidio-analyzer/tests/test_pattern_recognizer.py b/presidio-analyzer/tests/test_pattern_recognizer.py index 61eebb6f4c..930077a38b 100644 --- a/presidio-analyzer/tests/test_pattern_recognizer.py +++ b/presidio-analyzer/tests/test_pattern_recognizer.py @@ -212,13 +212,14 @@ def test_empty_deny_list_raises_value_error(): ) def test_global_regex_flag_deny_list_returns_right_result(global_flag, expected_len): deny_list = ["MrS", "mR"] - text = "Mrs. smith \n\n" \ - "and Mr. Jones were sitting in the room." + text = "Mrs. smith \n\n" "and Mr. Jones were sitting in the room." - recognizer_ignore_case = PatternRecognizer(supported_entity="TITLE", - name="TitlesRecognizer", - deny_list=deny_list, - global_regex_flags=global_flag) + recognizer_ignore_case = PatternRecognizer( + supported_entity="TITLE", + name="TitlesRecognizer", + deny_list=deny_list, + global_regex_flags=global_flag, + ) results = recognizer_ignore_case.analyze(text=text, entities=["TITLE"]) assert len(results) == expected_len @@ -226,10 +227,11 @@ def test_global_regex_flag_deny_list_returns_right_result(global_flag, expected_ def test_pattern_recognizer_with_invalidate_result(): """Test PatternRecognizer with invalidate_result returning True.""" + class InvalidatingRecognizer(PatternRecognizer): def invalidate_result(self, pattern_text): # Invalidate if pattern starts with '0' - return pattern_text.startswith('0') + return pattern_text.startswith("0") patterns = [Pattern(name="test_pattern", regex=r"\d{3}", score=0.8)] recognizer = InvalidatingRecognizer( @@ -250,10 +252,11 @@ def invalidate_result(self, pattern_text): def test_pattern_recognizer_with_validate_result_false(): """Test PatternRecognizer with validate_result returning False.""" + class ValidatingRecognizer(PatternRecognizer): def validate_result(self, pattern_text): # Only validate if it contains digit '5' - return '5' in pattern_text + return "5" in pattern_text patterns = [Pattern(name="test_pattern", regex=r"\d{3}", score=0.5)] recognizer = ValidatingRecognizer( @@ -274,6 +277,7 @@ def validate_result(self, pattern_text): def test_pattern_recognizer_with_both_validate_and_invalidate(): """Test PatternRecognizer with both validate and invalidate logic.""" + class BothRecognizer(PatternRecognizer): def validate_result(self, pattern_text): return len(pattern_text) == 3 @@ -347,7 +351,9 @@ def test_pattern_recognizer_from_dict_with_both_supported_entity_and_entities(): "patterns": [{"name": "p1", "score": 0.5, "regex": r"\d+"}], } - with pytest.raises(ValueError, match="Both 'supported_entity' and 'supported_entities'"): + with pytest.raises( + ValueError, match="Both 'supported_entity' and 'supported_entities'" + ): PatternRecognizer.from_dict(recognizer_dict) @@ -403,7 +409,7 @@ def test_pattern_recognizer_multiple_patterns(): supported_entity="TEST", patterns=patterns, name="MultiPatternTest", - global_regex_flags=re.DOTALL | re.MULTILINE + global_regex_flags=re.DOTALL | re.MULTILINE, ) results = recognizer.analyze("Number 123 and CAPS word", ["TEST"]) @@ -564,4 +570,3 @@ def test_regex_timeout_seconds_env_var_override(): # Restore the module to default state importlib.reload(pr_module) - diff --git a/presidio-analyzer/tests/test_phone_recognizer.py b/presidio-analyzer/tests/test_phone_recognizer.py index bb47d93e61..4c12d649ed 100644 --- a/presidio-analyzer/tests/test_phone_recognizer.py +++ b/presidio-analyzer/tests/test_phone_recognizer.py @@ -1,6 +1,8 @@ import pytest -from presidio_analyzer.predefined_recognizers.generic.phone_recognizer import PhoneRecognizer +from presidio_analyzer.predefined_recognizers.generic.phone_recognizer import ( + PhoneRecognizer, +) from tests import assert_result, assert_result_with_textual_explanation @@ -11,7 +13,6 @@ def recognizer(): ) - @pytest.mark.parametrize( "text, expected_len, entities, expected_positions, score", [ @@ -137,7 +138,10 @@ def test_when_phone_with_textual_explanation_then_succeed( results = recognizer.analyze(text, entities, nlp_artifacts=nlp_artifacts) assert len(results) == expected_len for i, (res, (st_pos, fn_pos)) in enumerate(zip(results, expected_positions)): - assert_result_with_textual_explanation(res, entities[i], st_pos, fn_pos, score, expected_textual_explanations[i]) + assert_result_with_textual_explanation( + res, entities[i], st_pos, fn_pos, score, expected_textual_explanations[i] + ) + def test_get_analysis_explanation(): phone_recognizer = PhoneRecognizer() @@ -145,6 +149,7 @@ def test_get_analysis_explanation(): explanation = phone_recognizer._get_analysis_explanation(test_region) assert explanation.recognizer == "PhoneRecognizer" + def test_get_supported_entities(): default_phone_recognizer = PhoneRecognizer() default_supported_entities = default_phone_recognizer.get_supported_entities() diff --git a/presidio-analyzer/tests/test_prompt_loader.py b/presidio-analyzer/tests/test_prompt_loader.py index 7ff2132418..76e445af2a 100644 --- a/presidio-analyzer/tests/test_prompt_loader.py +++ b/presidio-analyzer/tests/test_prompt_loader.py @@ -1,4 +1,5 @@ """Tests for llm_utils.prompt_loader module.""" + import pytest from pathlib import Path from presidio_analyzer.llm_utils.prompt_loader import ( @@ -52,7 +53,9 @@ def test_when_template_has_list_iteration_then_renders_correctly(self): def test_when_template_has_conditionals_then_renders_correctly(self): """Test rendering a template with if/else conditions.""" - template = """{% if enabled %}Feature is ON{% else %}Feature is OFF{% endif %}""" + template = ( + """{% if enabled %}Feature is ON{% else %}Feature is OFF{% endif %}""" + ) result_enabled = render_jinja_template(template, enabled=True) assert result_enabled.strip() == "Feature is ON" @@ -73,10 +76,7 @@ def test_when_template_has_complex_expressions_then_renders_correctly(self): {% for item in items %} {{ loop.index }}. {{ item.name }}: {{ item.value }} {% endfor %}""" - items = [ - {"name": "Item1", "value": 10}, - {"name": "Item2", "value": 20} - ] + items = [{"name": "Item1", "value": 10}, {"name": "Item2", "value": 20}] result = render_jinja_template(template, items=items) assert "Total: 2" in result @@ -119,13 +119,13 @@ def test_when_rendering_actual_langextract_template_then_works(self): template = load_prompt_file( "presidio-analyzer/presidio_analyzer/conf/langextract_prompts/default_pii_phi_prompt.j2" ) - + # Render with typical parameters result = render_jinja_template( template, supported_entities=["PERSON", "EMAIL_ADDRESS"], enable_generic_consolidation=True, - labels_to_ignore=["metadata"] + labels_to_ignore=["metadata"], ) assert "PERSON" in result diff --git a/presidio-analyzer/tests/test_recognizer_registry.py b/presidio-analyzer/tests/test_recognizer_registry.py index e0c6b8db53..6b3c4e13af 100644 --- a/presidio-analyzer/tests/test_recognizer_registry.py +++ b/presidio-analyzer/tests/test_recognizer_registry.py @@ -53,7 +53,7 @@ def test_when_get_recognizers_then_all_recognizers_returned(mock_recognizer_regi recognizers = registry.get_recognizers(language="en", all_fields=True) # 1 custom recognizer in english + 28 predefined - 11 disabled - assert len(recognizers) == 1 + 28 - 11 + assert len(recognizers) == 1 + 29 - 11 def test_when_get_recognizers_then_return_all_fields(mock_recognizer_registry): @@ -241,8 +241,9 @@ def test_remove_recognizer_when_multiple_instances_exist(): assert len(registry.recognizers) == 1 assert registry.recognizers[0].supported_language == "es" - assert len([rec for rec in registry.recognizers - if rec.name == "SpacyRecognizer"]) == 1 + assert ( + len([rec for rec in registry.recognizers if rec.name == "SpacyRecognizer"]) == 1 + ) # --------------------------------------------------------------------------- diff --git a/presidio-analyzer/tests/test_recognizer_registry_provider.py b/presidio-analyzer/tests/test_recognizer_registry_provider.py index 17771a1d3e..f2c76209a6 100644 --- a/presidio-analyzer/tests/test_recognizer_registry_provider.py +++ b/presidio-analyzer/tests/test_recognizer_registry_provider.py @@ -5,7 +5,9 @@ from inspect import signature from presidio_analyzer.predefined_recognizers import SpacyRecognizer from presidio_analyzer.recognizer_registry import RecognizerRegistryProvider -from presidio_analyzer.recognizer_registry.recognizers_loader_utils import RecognizerConfigurationLoader +from presidio_analyzer.recognizer_registry.recognizers_loader_utils import ( + RecognizerConfigurationLoader, +) from presidio_analyzer import RecognizerRegistry @@ -13,7 +15,10 @@ def assert_default_configuration( recognizer_registry: RecognizerRegistry, mandatory_recognizers: List[str] ): assert recognizer_registry.supported_languages == ["en"] - assert recognizer_registry.global_regex_flags == re.DOTALL | re.MULTILINE | re.IGNORECASE + assert ( + recognizer_registry.global_regex_flags + == re.DOTALL | re.MULTILINE | re.IGNORECASE + ) names = [recognizer.name for recognizer in recognizer_registry.recognizers] for predefined_recognizer in mandatory_recognizers: assert predefined_recognizer in names @@ -33,14 +38,33 @@ def test_recognizer_registry_provider_configuration_file(): assert recognizer_registry.supported_languages == ["en", "es"] assert recognizer_registry.global_regex_flags == 26 assert len(recognizer_registry.recognizers) == 5 - assert [recognizer.supported_language for recognizer in recognizer_registry.recognizers if recognizer.name == "ItFiscalCodeRecognizer"] == ["en", "es"] - assert [recognizer.supported_language for recognizer in recognizer_registry.recognizers if recognizer.name == "CreditCardRecognizer"] == ["en"] - assert [recognizer.supported_language for recognizer in recognizer_registry.recognizers if recognizer.name == "ExampleCustomRecognizer"] == ["en", "es"] - spanish_recognizer = [recognizer for recognizer in recognizer_registry.recognizers if recognizer.name == "ExampleCustomRecognizer" and recognizer.supported_language == "es"][0] + assert [ + recognizer.supported_language + for recognizer in recognizer_registry.recognizers + if recognizer.name == "ItFiscalCodeRecognizer" + ] == ["en", "es"] + assert [ + recognizer.supported_language + for recognizer in recognizer_registry.recognizers + if recognizer.name == "CreditCardRecognizer" + ] == ["en"] + assert [ + recognizer.supported_language + for recognizer in recognizer_registry.recognizers + if recognizer.name == "ExampleCustomRecognizer" + ] == ["en", "es"] + spanish_recognizer = [ + recognizer + for recognizer in recognizer_registry.recognizers + if recognizer.name == "ExampleCustomRecognizer" + and recognizer.supported_language == "es" + ][0] assert spanish_recognizer.context == ["tarjeta", "credito"] -def test_recognizer_registry_provider_configuration_file_load_predefined(mandatory_recognizers): +def test_recognizer_registry_provider_configuration_file_load_predefined( + mandatory_recognizers, +): this_path = Path(__file__).parent.absolute() test_yaml = Path(this_path, "conf/test_recognizers.yaml") provider = RecognizerRegistryProvider(test_yaml) @@ -48,13 +72,28 @@ def test_recognizer_registry_provider_configuration_file_load_predefined(mandato assert recognizer_registry.supported_languages == ["en"] assert recognizer_registry.global_regex_flags == 26 assert len(recognizer_registry.recognizers) == 2 - assert [recognizer.supported_language for recognizer in recognizer_registry.recognizers if recognizer.name == "TitleRecognizer"] == ["en"] - assert [recognizer.supported_language for recognizer in recognizer_registry.recognizers if recognizer.name == "ZipCodeRecognizer"] == ["en"] - zipcode_recognizer = [recognizer for recognizer in recognizer_registry.recognizers if recognizer.name == "ZipCodeRecognizer" and recognizer.supported_language == "en"][0] + assert [ + recognizer.supported_language + for recognizer in recognizer_registry.recognizers + if recognizer.name == "TitleRecognizer" + ] == ["en"] + assert [ + recognizer.supported_language + for recognizer in recognizer_registry.recognizers + if recognizer.name == "ZipCodeRecognizer" + ] == ["en"] + zipcode_recognizer = [ + recognizer + for recognizer in recognizer_registry.recognizers + if recognizer.name == "ZipCodeRecognizer" + and recognizer.supported_language == "en" + ][0] assert zipcode_recognizer.context == ["zip", "code"] -def test_recognizer_registry_provider_missing_conf_file_expect_default_configuration(mandatory_recognizers): +def test_recognizer_registry_provider_missing_conf_file_expect_default_configuration( + mandatory_recognizers, +): test_yaml = Path("missing.yaml") provider = RecognizerRegistryProvider(test_yaml) recognizer_registry = provider.create_recognizer_registry() @@ -66,9 +105,7 @@ def test_recognizer_registry_provider_corrupt_conf_file_fail(mandatory_recognize test_yaml = Path(this_path, "conf/recognizers_error.yaml") with pytest.raises(TypeError): - RecognizerRegistryProvider( - conf_file=test_yaml - ) + RecognizerRegistryProvider(conf_file=test_yaml) def test_recognizer_registry_provider_conf_file_valid_missing_keys_fail(): @@ -81,7 +118,6 @@ def test_recognizer_registry_provider_conf_file_valid_missing_keys_fail(): RecognizerRegistryProvider(conf_file=test_yaml) - def test_recognizer_registry_provider_when_conf_file_and_registry_configuration_fail(): this_path = Path(__file__).parent.absolute() test_yaml = Path(this_path, "conf/recognizer_configuration_missing_keys.yaml") @@ -126,7 +162,10 @@ def test_recognizer_registry_provider_missing_language_config_raises(): """ Test that a recognizer configuration without language info gets the default languages. """ - from presidio_analyzer.recognizer_registry.recognizer_registry_provider import RecognizerRegistryProvider + from presidio_analyzer.recognizer_registry.recognizer_registry_provider import ( + RecognizerRegistryProvider, + ) + # Configuration with no supported_languages and no recognizer language registry_configuration = { "recognizers": [ @@ -134,9 +173,7 @@ def test_recognizer_registry_provider_missing_language_config_raises(): "name": "CustomRecognizer", "type": "custom", "supported_entity": "CUSTOM_ENTITY", - "patterns": [ - {"name": "custom", "regex": "test", "score": 0.5} - ], + "patterns": [{"name": "custom", "regex": "test", "score": 0.5}], # No supported_language or supported_languages } ] @@ -153,6 +190,7 @@ def test_recognizer_registry_provider_missing_language_config_raises(): # Tests for missing required and optional fields in YAML configuration + def test_missing_recognizers_raises_exception(): """Test that missing recognizers raises an exception.""" this_path = Path(__file__).parent.absolute() @@ -189,7 +227,9 @@ def test_valid_configuration_passes(): "global_regex_flags": 26, } - validated = ConfigurationValidator.validate_recognizer_registry_configuration(config) + validated = ConfigurationValidator.validate_recognizer_registry_configuration( + config + ) assert validated is not None assert validated["supported_languages"] == ["en", "es"] @@ -206,7 +246,9 @@ def test_valid_configuration_without_global_regex_flags(): } # Should not raise an exception - validated = ConfigurationValidator.validate_recognizer_registry_configuration(config) + validated = ConfigurationValidator.validate_recognizer_registry_configuration( + config + ) # Check default value was set assert validated["global_regex_flags"] == 26 @@ -237,7 +279,9 @@ def test_direct_validation_with_missing_global_regex_flags(): } # Should not raise an exception - validated = ConfigurationValidator.validate_recognizer_registry_configuration(config) + validated = ConfigurationValidator.validate_recognizer_registry_configuration( + config + ) # Verify default value and successful creation assert validated["global_regex_flags"] == 26 diff --git a/presidio-analyzer/tests/test_recognizers_loader_utils.py b/presidio-analyzer/tests/test_recognizers_loader_utils.py index 89a178b182..3c357aba8c 100644 --- a/presidio-analyzer/tests/test_recognizers_loader_utils.py +++ b/presidio-analyzer/tests/test_recognizers_loader_utils.py @@ -191,6 +191,7 @@ def test_country_filter_includes_tagged_custom_recognizer(): """A custom recognizer that opts in via class-level ``COUNTRY_CODE`` is included when the filter is loaded with the matching country. """ + class _BrCpfRecognizer(PatternRecognizer): COUNTRY_CODE = "br" @@ -230,6 +231,7 @@ def test_country_filter_warns_on_unknown_country(caplog): list, a WARNING is logged so silent zero-result filters are easier to debug. """ + class _XUsRecognizer(PatternRecognizer): COUNTRY_CODE = "us" @@ -364,6 +366,7 @@ def test_filter_by_countries_normalizes_case_and_whitespace(): ``" US "`` matches a ``COUNTRY_CODE = "us"`` recognizer. """ + class TaggedRecognizer(PatternRecognizer): COUNTRY_CODE = "us" diff --git a/presidio-analyzer/tests/test_se_organisationsnummer_recognizer.py b/presidio-analyzer/tests/test_se_organisationsnummer_recognizer.py index d34a70bc7e..e6597a9cc4 100644 --- a/presidio-analyzer/tests/test_se_organisationsnummer_recognizer.py +++ b/presidio-analyzer/tests/test_se_organisationsnummer_recognizer.py @@ -59,7 +59,8 @@ def entities(): ("19000309-3393", 0, ()), ("19001309-2393", 0, ()), ("55670x-7485", 0, ()), - ("556703-7r85", 0, ()),], + ("556703-7r85", 0, ()), + ], ) def test_when_all_swedish_organisationsnummer_then_succeed( text, expected_len, expected_positions, recognizer, entities, max_score @@ -68,4 +69,4 @@ def test_when_all_swedish_organisationsnummer_then_succeed( results = recognizer.analyze(text, entities) assert len(results) == expected_len for res, (st_pos, fn_pos) in zip(results, expected_positions): - assert_result(res, entities[0], st_pos, fn_pos, max_score) \ No newline at end of file + assert_result(res, entities[0], st_pos, fn_pos, max_score) diff --git a/presidio-analyzer/tests/test_se_personnummer_recognizer.py b/presidio-analyzer/tests/test_se_personnummer_recognizer.py index db81c6f57f..9a5ea4f1df 100644 --- a/presidio-analyzer/tests/test_se_personnummer_recognizer.py +++ b/presidio-analyzer/tests/test_se_personnummer_recognizer.py @@ -90,7 +90,8 @@ def entities(): ("19001309-2393", 0, ()), ("200504422381", 0, ()), ("189x09179809", 0, ()), - ("18970c17-9809", 0, ()),], + ("18970c17-9809", 0, ()), + ], ) def test_when_all_swedish_personnummer_then_succeed( text, expected_len, expected_positions, recognizer, entities, max_score diff --git a/presidio-analyzer/tests/test_sg_fin_recognizer.py b/presidio-analyzer/tests/test_sg_fin_recognizer.py index e18720abef..33df817611 100644 --- a/presidio-analyzer/tests/test_sg_fin_recognizer.py +++ b/presidio-analyzer/tests/test_sg_fin_recognizer.py @@ -64,6 +64,7 @@ def test_when_sgfins_in_text_then_all_sg_fins_found( results, expected_positions, expected_score_ranges ): import logging + logging.info(f"result: {result}") # Adjust end_score if it's marked with a placeholder value that indicates it should be considered as max_score if end_score == "max": @@ -72,4 +73,4 @@ def test_when_sgfins_in_text_then_all_sg_fins_found( # Assuming assert_result_within_score_range checks the position and verifies the score is within the specified range assert_result_within_score_range( result, entities[0], start_pos, end_pos, start_score, end_score - ) \ No newline at end of file + ) diff --git a/presidio-analyzer/tests/test_slim_spacy_nlp_engine.py b/presidio-analyzer/tests/test_slim_spacy_nlp_engine.py index 3ce2e08e23..71f42816be 100644 --- a/presidio-analyzer/tests/test_slim_spacy_nlp_engine.py +++ b/presidio-analyzer/tests/test_slim_spacy_nlp_engine.py @@ -72,9 +72,7 @@ def test_when_init_with_mixed_languages_and_fallback(self): def test_when_models_and_languages_both_given_then_models_takes_precedence(self): models = [{"lang_code": "en", "model_name": "en_core_web_lg"}] - engine = SlimSpacyNlpEngine( - models=models, supported_languages=["es"] - ) + engine = SlimSpacyNlpEngine(models=models, supported_languages=["es"]) # models parameter takes precedence assert engine.models == models @@ -147,9 +145,7 @@ def test_when_load_with_invalid_model_then_raises(self): def test_when_validate_model_params_missing_lang_code_then_raises(self): with pytest.raises(ValueError, match="lang_code"): - SlimSpacyNlpEngine._validate_model_params( - {"model_name": "en_core_web_sm"} - ) + SlimSpacyNlpEngine._validate_model_params({"model_name": "en_core_web_sm"}) def test_when_validate_model_params_missing_model_name_then_raises(self): with pytest.raises(ValueError, match="model_name"): @@ -210,9 +206,7 @@ class TestSlimSpacyNlpEngineProcessBatch: """Tests for process_batch method.""" def test_when_process_batch_then_returns_iterator(self, slim_nlp_engine): - results = slim_nlp_engine.process_batch( - ["hello", "world"], language="en" - ) + results = slim_nlp_engine.process_batch(["hello", "world"], language="en") assert isinstance(results, Iterator) def test_when_process_batch_strings_then_text_and_artifacts(self, slim_nlp_engine): @@ -225,9 +219,7 @@ def test_when_process_batch_strings_then_text_and_artifacts(self, slim_nlp_engin assert len(artifacts.tokens) == 2 assert artifacts.entities == [] - def test_when_process_batch_as_tuples_then_context_preserved( - self, slim_nlp_engine - ): + def test_when_process_batch_as_tuples_then_context_preserved(self, slim_nlp_engine): inputs = [("hello world", {"id": 1}), ("test text", {"id": 2})] results = list( slim_nlp_engine.process_batch(inputs, language="en", as_tuples=True) @@ -328,6 +320,6 @@ def test_when_default_models_then_english_present(self): def test_when_default_models_then_all_values_are_sm(self): for lang, model in DEFAULT_SLIM_MODELS.items(): - assert "_sm" in model or "sm" in model, ( - f"Default model for {lang} ({model}) should be a small model" - ) + assert ( + "_sm" in model or "sm" in model + ), f"Default model for {lang} ({model}) should be a small model" diff --git a/presidio-analyzer/tests/test_spacy_nlp_engine.py b/presidio-analyzer/tests/test_spacy_nlp_engine.py index 152032c1c1..59f5806b6b 100644 --- a/presidio-analyzer/tests/test_spacy_nlp_engine.py +++ b/presidio-analyzer/tests/test_spacy_nlp_engine.py @@ -22,7 +22,6 @@ def test_simple_process_text(spacy_nlp_engine): assert nlp_artifacts.lemmas[1] == "text" - def test_process_batch_strings(spacy_nlp_engine): nlp_artifacts_batch = spacy_nlp_engine.process_batch( ["simple text", "simple text"], language="en" @@ -72,10 +71,10 @@ def test_default_configuration_correct(): def test_get_supported_entities_doesnt_include_ignored(): - ner_config = NerModelConfiguration(labels_to_ignore=["A","B"], - model_to_presidio_entity_mapping=dict(A="A", - B="B", - C="C")) + ner_config = NerModelConfiguration( + labels_to_ignore=["A", "B"], + model_to_presidio_entity_mapping=dict(A="A", B="B", C="C"), + ) spacy_nlp_engine = SpacyNlpEngine(ner_model_configuration=ner_config) entities = spacy_nlp_engine.get_supported_entities() @@ -84,11 +83,16 @@ def test_get_supported_entities_doesnt_include_ignored(): assert "C" in entities -@pytest.mark.parametrize("texts, as_tuples", [ - (["simple text", "simple text"], False), - ([("simple text", {"key": "value"})], True), -]) -def test_batch_processing_with_as_tuples_returns_context(spacy_nlp_engine, texts, as_tuples): +@pytest.mark.parametrize( + "texts, as_tuples", + [ + (["simple text", "simple text"], False), + ([("simple text", {"key": "value"})], True), + ], +) +def test_batch_processing_with_as_tuples_returns_context( + spacy_nlp_engine, texts, as_tuples +): nlp_artifacts_batch = spacy_nlp_engine.process_batch( texts, language="en", as_tuples=as_tuples ) @@ -108,46 +112,60 @@ def test_batch_processing_with_as_tuples_returns_context(spacy_nlp_engine, texts def test_when_gpu_available_then_spacy_gpu_configured(): """Test that spaCy GPU is configured when GPU is detected.""" - with patch("presidio_analyzer.nlp_engine.spacy_nlp_engine.device_detector") as mock_detector: + with patch( + "presidio_analyzer.nlp_engine.spacy_nlp_engine.device_detector" + ) as mock_detector: mock_detector.get_device.return_value = "cuda" - + with patch("presidio_analyzer.nlp_engine.spacy_nlp_engine.spacy") as mock_spacy: mock_spacy.load.return_value = MagicMock() mock_spacy.util.is_package.return_value = True - - engine = SpacyNlpEngine(models=[{"lang_code": "en", "model_name": "en_core_web_sm"}]) + + engine = SpacyNlpEngine( + models=[{"lang_code": "en", "model_name": "en_core_web_sm"}] + ) engine.load() - + mock_spacy.require_gpu.assert_called_once() def test_when_gpu_configuration_fails_then_warning_logged(): """Test that warning is logged when GPU configuration fails.""" - with patch("presidio_analyzer.nlp_engine.spacy_nlp_engine.device_detector") as mock_detector: + with patch( + "presidio_analyzer.nlp_engine.spacy_nlp_engine.device_detector" + ) as mock_detector: mock_detector.get_device.return_value = "cuda" - + with patch("presidio_analyzer.nlp_engine.spacy_nlp_engine.spacy") as mock_spacy: mock_spacy.load.return_value = MagicMock() mock_spacy.util.is_package.return_value = True mock_spacy.require_gpu.side_effect = Exception("GPU error") - - with patch("presidio_analyzer.nlp_engine.spacy_nlp_engine.logger") as mock_logger: - engine = SpacyNlpEngine(models=[{"lang_code": "en", "model_name": "en_core_web_sm"}]) + + with patch( + "presidio_analyzer.nlp_engine.spacy_nlp_engine.logger" + ) as mock_logger: + engine = SpacyNlpEngine( + models=[{"lang_code": "en", "model_name": "en_core_web_sm"}] + ) engine.load() - + assert mock_logger.warning.called def test_when_cpu_device_then_gpu_not_configured(): """Test that GPU is not configured when CPU device is detected.""" - with patch("presidio_analyzer.nlp_engine.spacy_nlp_engine.device_detector") as mock_detector: + with patch( + "presidio_analyzer.nlp_engine.spacy_nlp_engine.device_detector" + ) as mock_detector: mock_detector.get_device.return_value = "cpu" - + with patch("presidio_analyzer.nlp_engine.spacy_nlp_engine.spacy") as mock_spacy: mock_spacy.load.return_value = MagicMock() mock_spacy.util.is_package.return_value = True - - engine = SpacyNlpEngine(models=[{"lang_code": "en", "model_name": "en_core_web_sm"}]) + + engine = SpacyNlpEngine( + models=[{"lang_code": "en", "model_name": "en_core_web_sm"}] + ) engine.load() - + mock_spacy.require_gpu.assert_not_called() diff --git a/presidio-analyzer/tests/test_spacy_recognizer.py b/presidio-analyzer/tests/test_spacy_recognizer.py index c77969cf41..4dbe7f347c 100644 --- a/presidio-analyzer/tests/test_spacy_recognizer.py +++ b/presidio-analyzer/tests/test_spacy_recognizer.py @@ -1,4 +1,3 @@ - import pytest from spacy.tokens import Span, Doc from spacy.util import get_lang_class @@ -20,19 +19,19 @@ def nlp_recognizer(nlp_recognizers): @pytest.fixture(scope="module") def mock_nlp_artifacts(): - en_vocab=get_lang_class("en")().vocab + en_vocab = get_lang_class("en")().vocab doc = Doc(en_vocab, words=["My", "name", "is", "Mitchell"]) doc.ents = [Span(doc, 2, 3, label="PERSON")] nlp_artifacts = NlpArtifacts( - entities=doc.ents, - tokens=doc, - tokens_indices=[token.idx for token in doc], - lemmas=[token.lemma_ for token in doc], - nlp_engine=None, - language="en", - scores=[0.9 for _ in doc.ents], - ) + entities=doc.ents, + tokens=doc, + tokens_indices=[token.idx for token in doc], + lemmas=[token.lemma_ for token in doc], + nlp_engine=None, + language="en", + scores=[0.9 for _ in doc.ents], + ) return nlp_artifacts diff --git a/presidio-analyzer/tests/test_stanza_batch_processing.py b/presidio-analyzer/tests/test_stanza_batch_processing.py index cffb534286..7df6373054 100644 --- a/presidio-analyzer/tests/test_stanza_batch_processing.py +++ b/presidio-analyzer/tests/test_stanza_batch_processing.py @@ -26,13 +26,13 @@ def test_when_process_batch_with_strings_then_returns_iterator( ): """Test basic batch processing with simple strings.""" texts = ["Hello world", "This is a test"] - + result = stanza_nlp_engine.process_batch(texts, language="en", batch_size=2) - + assert isinstance(result, Iterator) result_list = list(result) assert len(result_list) == 2 - + for text, nlp_artifacts in result_list: assert isinstance(text, str) assert isinstance(nlp_artifacts, NlpArtifacts) @@ -42,23 +42,20 @@ def test_when_process_batch_with_tuples_then_returns_context( self, stanza_nlp_engine ): """Test batch processing with tuples including context.""" - texts = [ - ("Hello world", {"id": 1}), - ("This is a test", {"id": 2}) - ] - + texts = [("Hello world", {"id": 1}), ("This is a test", {"id": 2})] + result = stanza_nlp_engine.process_batch( texts, language="en", batch_size=2, as_tuples=True ) - + result_list = list(result) assert len(result_list) == 2 - + text1, nlp_artifacts1, context1 = result_list[0] assert text1 == "Hello world" assert isinstance(nlp_artifacts1, NlpArtifacts) assert context1 == {"id": 1} - + text2, nlp_artifacts2, context2 = result_list[1] assert text2 == "This is a test" assert isinstance(nlp_artifacts2, NlpArtifacts) @@ -68,18 +65,15 @@ def test_when_process_batch_with_entities_then_extracts_correctly( self, stanza_nlp_engine ): """Test that batch processing correctly extracts entities.""" - texts = [ - "Barack Obama was born in Hawaii.", - "John Smith lives in New York." - ] - + texts = ["Barack Obama was born in Hawaii.", "John Smith lives in New York."] + result = stanza_nlp_engine.process_batch(texts, language="en", batch_size=2) result_list = list(result) - + # First text should have entities (Barack Obama, Hawaii) text1, nlp_artifacts1 = result_list[0] assert len(nlp_artifacts1.entities) >= 2 - + # Second text should have entities (John Smith, New York) text2, nlp_artifacts2 = result_list[1] assert len(nlp_artifacts2.entities) >= 2 @@ -89,13 +83,13 @@ def test_when_process_batch_with_different_batch_sizes_then_works( ): """Test batch processing with various batch sizes.""" texts = ["Text one", "Text two", "Text three", "Text four", "Text five"] - + for batch_size in [1, 2, 3, 10]: result = stanza_nlp_engine.process_batch( texts, language="en", batch_size=batch_size ) result_list = list(result) - + assert len(result_list) == 5 for text, nlp_artifacts in result_list: assert isinstance(nlp_artifacts, NlpArtifacts) @@ -105,21 +99,19 @@ def test_when_process_batch_with_empty_list_then_returns_empty( ): """Test batch processing with empty input.""" texts = [] - + result = stanza_nlp_engine.process_batch(texts, language="en") result_list = list(result) - + assert len(result_list) == 0 def test_when_process_batch_not_loaded_then_raises_error(self): """Test that processing without loading raises an error.""" from presidio_analyzer.nlp_engine import StanzaNlpEngine - - engine = StanzaNlpEngine( - models=[{"lang_code": "en", "model_name": "en"}] - ) + + engine = StanzaNlpEngine(models=[{"lang_code": "en", "model_name": "en"}]) # Don't call load() - + with pytest.raises(ValueError, match="NLP engine is not loaded"): list(engine.process_batch(["test"], language="en")) @@ -127,15 +119,11 @@ def test_when_process_batch_with_whitespace_then_handles_correctly( self, stanza_nlp_engine ): """Test batch processing with texts containing whitespace.""" - texts = [ - " Leading whitespace", - "Trailing whitespace ", - " Multiple spaces " - ] - + texts = [" Leading whitespace", "Trailing whitespace ", " Multiple spaces "] + result = stanza_nlp_engine.process_batch(texts, language="en", batch_size=3) result_list = list(result) - + assert len(result_list) == 3 for text, nlp_artifacts in result_list: assert isinstance(nlp_artifacts, NlpArtifacts) @@ -145,26 +133,24 @@ def test_when_process_batch_with_whitespace_then_handles_correctly( def test_when_process_batch_preserves_text_order(self, stanza_nlp_engine): """Test that batch processing preserves input order.""" texts = [f"Text number {i}" for i in range(10)] - + result = stanza_nlp_engine.process_batch(texts, language="en", batch_size=3) result_list = list(result) - + for i, (text, nlp_artifacts) in enumerate(result_list): assert text == f"Text number {i}" - def test_when_process_batch_with_special_chars_then_works( - self, stanza_nlp_engine - ): + def test_when_process_batch_with_special_chars_then_works(self, stanza_nlp_engine): """Test batch processing with special characters.""" texts = [ "Email: test@example.com", "Phone: +1-555-1234", - "URL: https://example.com" + "URL: https://example.com", ] - + result = stanza_nlp_engine.process_batch(texts, language="en", batch_size=3) result_list = list(result) - + assert len(result_list) == 3 for text, nlp_artifacts in result_list: assert isinstance(nlp_artifacts, NlpArtifacts) @@ -174,24 +160,22 @@ def test_when_process_batch_with_special_chars_then_works( class TestStanzaTokenizerConvertDoc: """Test suite for StanzaTokenizer._convert_doc() method.""" - def test_when_convert_doc_called_then_returns_spacy_doc( - self, stanza_nlp_engine - ): + def test_when_convert_doc_called_then_returns_spacy_doc(self, stanza_nlp_engine): """Test that _convert_doc() correctly converts Stanza docs to spaCy docs.""" import stanza - + # Get the tokenizer stanza_tokenizer = stanza_nlp_engine.nlp["en"].tokenizer stanza_pipeline = stanza_tokenizer.snlp - + # Process a text through Stanza text = "Barack Obama was born in Hawaii." stanza_doc = stanza.Document([], text=text) processed_doc = stanza_pipeline(stanza_doc) - + # Convert to spaCy doc spacy_doc = stanza_tokenizer._convert_doc(processed_doc) - + # Verify the conversion assert spacy_doc.text == text assert len(spacy_doc) > 0 # Has tokens @@ -205,15 +189,15 @@ def test_when_convert_doc_with_empty_text_then_returns_empty_doc( """Test _convert_doc() with empty text.""" import stanza from spacy.tokens import Doc - + stanza_tokenizer = stanza_nlp_engine.nlp["en"].tokenizer - + # Create empty Stanza doc empty_doc = stanza.Document([], text="") - + # Convert spacy_doc = stanza_tokenizer._convert_doc(empty_doc) - + assert isinstance(spacy_doc, Doc) assert len(spacy_doc) == 0 @@ -223,33 +207,31 @@ def test_when_convert_doc_with_whitespace_only_then_handles_correctly( """Test _convert_doc() with whitespace-only text.""" import stanza from spacy.tokens import Doc - + stanza_tokenizer = stanza_nlp_engine.nlp["en"].tokenizer - + # Create whitespace-only Stanza doc whitespace_doc = stanza.Document([], text=" ") - + # Convert spacy_doc = stanza_tokenizer._convert_doc(whitespace_doc) - + assert isinstance(spacy_doc, Doc) # Should handle whitespace gracefully - def test_when_convert_doc_preserves_linguistic_features( - self, stanza_nlp_engine - ): + def test_when_convert_doc_preserves_linguistic_features(self, stanza_nlp_engine): """Test that _convert_doc() preserves POS tags, lemmas, and dependencies.""" import stanza - + stanza_tokenizer = stanza_nlp_engine.nlp["en"].tokenizer stanza_pipeline = stanza_tokenizer.snlp - + text = "The quick brown fox jumps." stanza_doc = stanza.Document([], text=text) processed_doc = stanza_pipeline(stanza_doc) - + spacy_doc = stanza_tokenizer._convert_doc(processed_doc) - + # Verify linguistic features are preserved for token in spacy_doc: assert token.pos_ is not None # POS tags @@ -268,33 +250,34 @@ def test_when_process_batch_then_calls_bulk_process( ): """Test that process_batch() calls Stanza's bulk_process method.""" import stanza - + # Setup mock to return processed docs mock_bulk_process.return_value = [ Mock(text="Text 1", sentences=[], entities=[]), - Mock(text="Text 2", sentences=[], entities=[]) + Mock(text="Text 2", sentences=[], entities=[]), ] - + # Create mock for the conversion stanza_tokenizer = stanza_nlp_engine.nlp["en"].tokenizer original_convert = stanza_tokenizer._convert_doc - + def mock_convert(doc): # Return a minimal spaCy doc from spacy.tokens import Doc + return Doc(stanza_tokenizer.vocab, words=["test"]) - + stanza_tokenizer._convert_doc = mock_convert - + try: texts = ["Text 1", "Text 2"] - result = list(stanza_nlp_engine.process_batch( - texts, language="en", batch_size=2 - )) - + result = list( + stanza_nlp_engine.process_batch(texts, language="en", batch_size=2) + ) + # Verify bulk_process was called assert mock_bulk_process.called - + # Verify the input to bulk_process call_args = mock_bulk_process.call_args[0][0] assert len(call_args) == 2 @@ -309,31 +292,27 @@ def test_when_process_batch_with_large_batch_then_handles_correctly( """Test batch processing with a large number of texts.""" num_texts = 100 texts = [f"This is test text number {i}." for i in range(num_texts)] - - result = stanza_nlp_engine.process_batch( - texts, language="en", batch_size=16 - ) + + result = stanza_nlp_engine.process_batch(texts, language="en", batch_size=16) result_list = list(result) - + assert len(result_list) == num_texts - + # Verify all texts were processed for i, (text, nlp_artifacts) in enumerate(result_list): assert f"number {i}" in text assert isinstance(nlp_artifacts, NlpArtifacts) - def test_when_process_batch_batching_matches_batch_size( - self, stanza_nlp_engine - ): + def test_when_process_batch_batching_matches_batch_size(self, stanza_nlp_engine): """Test that internal batching respects batch_size parameter.""" texts = [f"Text {i}" for i in range(10)] - + # Process with different batch sizes for batch_size in [1, 3, 5, 10]: result = stanza_nlp_engine.process_batch( texts, language="en", batch_size=batch_size ) result_list = list(result) - + # Should process all texts regardless of batch size assert len(result_list) == 10 diff --git a/presidio-analyzer/tests/test_stanza_nlp_engine.py b/presidio-analyzer/tests/test_stanza_nlp_engine.py index 766dbaed43..b5dfc8a52e 100644 --- a/presidio-analyzer/tests/test_stanza_nlp_engine.py +++ b/presidio-analyzer/tests/test_stanza_nlp_engine.py @@ -17,11 +17,13 @@ def tags_equal(act, exp): def stanza_pipeline(): pytest.importorskip("stanza") import stanza + lang = "en" stanza.download(lang) nlp = load_pipeline(lang) return nlp + @pytest.mark.skip_engine("stanza_en") def test_spacy_stanza_english(stanza_pipeline): nlp = stanza_pipeline diff --git a/presidio-analyzer/tests/test_text_chunker_provider.py b/presidio-analyzer/tests/test_text_chunker_provider.py index fcb1b6464c..d95b03098a 100644 --- a/presidio-analyzer/tests/test_text_chunker_provider.py +++ b/presidio-analyzer/tests/test_text_chunker_provider.py @@ -19,30 +19,33 @@ def test_default_creates_character_chunker(self): def test_custom_params_passed_to_chunker(self): """Custom parameters are passed to chunker.""" - provider = TextChunkerProvider(chunker_configuration={ - "chunker_type": "character", - "chunk_size": 500, - "chunk_overlap": 100, - }) + provider = TextChunkerProvider( + chunker_configuration={ + "chunker_type": "character", + "chunk_size": 500, + "chunk_overlap": 100, + } + ) chunker = provider.create_chunker() assert chunker._chunk_size == 500 assert chunker._chunk_overlap == 100 def test_unknown_chunker_type_raises_error(self): """Unknown chunker_type raises ValueError.""" - provider = TextChunkerProvider(chunker_configuration={ - "chunker_type": "unknown" - }) + provider = TextChunkerProvider( + chunker_configuration={"chunker_type": "unknown"} + ) with pytest.raises(ValueError, match="Unknown chunker_type"): provider.create_chunker() def test_character_chunker_type(self): """Provider creates CharacterBasedTextChunker when type is 'character'.""" - provider = TextChunkerProvider(chunker_configuration={ - "chunker_type": "character", - "chunk_size": 300, - }) + provider = TextChunkerProvider( + chunker_configuration={ + "chunker_type": "character", + "chunk_size": 300, + } + ) chunker = provider.create_chunker() assert isinstance(chunker, CharacterBasedTextChunker) assert chunker.chunk_size == 300 - diff --git a/presidio-analyzer/tests/test_th_tnin_recognizer.py b/presidio-analyzer/tests/test_th_tnin_recognizer.py index c6f02d770c..d8c009baab 100644 --- a/presidio-analyzer/tests/test_th_tnin_recognizer.py +++ b/presidio-analyzer/tests/test_th_tnin_recognizer.py @@ -24,61 +24,232 @@ def entities(): # Valid TNINs - should get high scores due to format validation and checksum # Note: These are example TNINs created for testing purposes # with valid checksums - ("1234567890121", 1, ((0, 13),), ((0.5, 1.0),),), - ("2345678901234", 1, ((0, 13),), ((0.5, 1.0),),), - ("3456789012347", 1, ((0, 13),), ((0.5, 1.0),),), - ("4567890123459", 1, ((0, 13),), ((0.5, 1.0),),), - ("5678901234560", 1, ((0, 13),), ((0.5, 1.0),),), - + ( + "1234567890121", + 1, + ((0, 13),), + ((0.5, 1.0),), + ), + ( + "2345678901234", + 1, + ((0, 13),), + ((0.5, 1.0),), + ), + ( + "3456789012347", + 1, + ((0, 13),), + ((0.5, 1.0),), + ), + ( + "4567890123459", + 1, + ((0, 13),), + ((0.5, 1.0),), + ), + ( + "5678901234560", + 1, + ((0, 13),), + ((0.5, 1.0),), + ), # Valid TNINs in sentences - ("My Thai ID is 1234567890121", 1, ((14, 27),), ((0.5, 1.0),),), - ("TNIN: 2345678901234", 1, ((6, 19),), ((0.5, 1.0),),), - ("เลขประจำตัวประชาชน: 3456789012347", 1, ((20, 33),), ((0.5, 1.0),),), - + ( + "My Thai ID is 1234567890121", + 1, + ((14, 27),), + ((0.5, 1.0),), + ), + ( + "TNIN: 2345678901234", + 1, + ((6, 19),), + ((0.5, 1.0),), + ), + ( + "เลขประจำตัวประชาชน: 3456789012347", + 1, + ((20, 33),), + ((0.5, 1.0),), + ), # Invalid TNINs - wrong length - ("123456789012", 0, (), (),), # 12 digits - ("12345678901234", 0, (), (),), # 14 digits - + ( + "123456789012", + 0, + (), + (), + ), # 12 digits + ( + "12345678901234", + 0, + (), + (), + ), # 14 digits # Invalid TNINs - non-digits - ("123456789012a", 0, (), (),), # contains letter - ("123456789012 ", 0, (), (),), # contains space - + ( + "123456789012a", + 0, + (), + (), + ), # contains letter + ( + "123456789012 ", + 0, + (), + (), + ), # contains space # Invalid TNINs - format violations (first digit is 0) - ("0234567890124", 0, (), (),), - ("0034567890124", 0, (), (),), - + ( + "0234567890124", + 0, + (), + (), + ), + ( + "0034567890124", + 0, + (), + (), + ), # Invalid TNINs - format violations (second digit is 0) - ("1034567890124", 0, (), (),), - ("1304567890124", 0, (), (),), - + ( + "1034567890124", + 0, + (), + (), + ), + ( + "1304567890124", + 0, + (), + (), + ), # Invalid TNINs - forbidden second-third combinations - ("1284567890124", 0, (), (),), # 28 is forbidden - ("1294567890124", 0, (), (),), # 29 is forbidden - ("1594567890124", 0, (), (),), # 59 is forbidden - ("1684567890124", 0, (), (),), # 68 is forbidden - ("1694567890124", 0, (), (),), # 69 is forbidden - ("1784567890124", 0, (), (),), # 78 is forbidden - ("1794567890124", 0, (), (),), # 79 is forbidden - ("1874567890124", 0, (), (),), # 87 is forbidden - ("1884567890124", 0, (), (),), # 88 is forbidden - ("1894567890124", 0, (), (),), # 89 is forbidden - ("1974567890124", 0, (), (),), # 97 is forbidden - ("1984567890124", 0, (), (),), # 98 is forbidden - ("1994567890124", 0, (), (),), # 99 is forbidden - + ( + "1284567890124", + 0, + (), + (), + ), # 28 is forbidden + ( + "1294567890124", + 0, + (), + (), + ), # 29 is forbidden + ( + "1594567890124", + 0, + (), + (), + ), # 59 is forbidden + ( + "1684567890124", + 0, + (), + (), + ), # 68 is forbidden + ( + "1694567890124", + 0, + (), + (), + ), # 69 is forbidden + ( + "1784567890124", + 0, + (), + (), + ), # 78 is forbidden + ( + "1794567890124", + 0, + (), + (), + ), # 79 is forbidden + ( + "1874567890124", + 0, + (), + (), + ), # 87 is forbidden + ( + "1884567890124", + 0, + (), + (), + ), # 88 is forbidden + ( + "1894567890124", + 0, + (), + (), + ), # 89 is forbidden + ( + "1974567890124", + 0, + (), + (), + ), # 97 is forbidden + ( + "1984567890124", + 0, + (), + (), + ), # 98 is forbidden + ( + "1994567890124", + 0, + (), + (), + ), # 99 is forbidden # Invalid TNINs - checksum failures (but valid format) # These have valid format but wrong checksum digit - ("1234567890123", 0, (), (),), # should be 1, not 3 - ("2345678901235", 0, (), (),), # should be 4, not 5 - ("3456789012346", 0, (), (),), # should be 7, not 6 - + ( + "1234567890123", + 0, + (), + (), + ), # should be 1, not 3 + ( + "2345678901235", + 0, + (), + (), + ), # should be 4, not 5 + ( + "3456789012346", + 0, + (), + (), + ), # should be 7, not 6 # Edge cases - ("0000000000000", 0, (), (),), # all zeros (invalid format) - ("1111111111111", 0, (), (),), # all ones (valid format but wrong checksum) - + ( + "0000000000000", + 0, + (), + (), + ), # all zeros (invalid format) + ( + "1111111111111", + 0, + (), + (), + ), # all ones (valid format but wrong checksum) # Context enhancement tests - ("Thai National ID 1234567890121", 1, ((17, 30),), ((0.5, 1.0),),), - ("เลขบัตรประชาชน 2345678901234", 1, ((15, 28),), ((0.5, 1.0),),), + ( + "Thai National ID 1234567890121", + 1, + ((17, 30),), + ((0.5, 1.0),), + ), + ( + "เลขบัตรประชาชน 2345678901234", + 1, + ((15, 28),), + ((0.5, 1.0),), + ), ], ) def test_when_tnin_in_text_then_all_tnins_found( diff --git a/presidio-analyzer/tests/test_tr_national_id_recognizer.py b/presidio-analyzer/tests/test_tr_national_id_recognizer.py index b6c20e9e91..71c8bb5243 100644 --- a/presidio-analyzer/tests/test_tr_national_id_recognizer.py +++ b/presidio-analyzer/tests/test_tr_national_id_recognizer.py @@ -22,13 +22,48 @@ def entities(): "text, expected_len, expected_positions, expected_score_ranges", [ # Valid TCKNs with correct checksum - ("10000000146", 1, ((0, 11),), ((0.5, 1.0),),), - ("76543210794", 1, ((0, 11),), ((0.5, 1.0),),), - ("36493665440", 1, ((0, 11),), ((0.5, 1.0),),), - ("53857632436", 1, ((0, 11),), ((0.5, 1.0),),), - ("94357219628", 1, ((0, 11),), ((0.5, 1.0),),), - ("79059236630", 1, ((0, 11),), ((0.5, 1.0),),), - ("64625294480", 1, ((0, 11),), ((0.5, 1.0),),), + ( + "10000000146", + 1, + ((0, 11),), + ((0.5, 1.0),), + ), + ( + "76543210794", + 1, + ((0, 11),), + ((0.5, 1.0),), + ), + ( + "36493665440", + 1, + ((0, 11),), + ((0.5, 1.0),), + ), + ( + "53857632436", + 1, + ((0, 11),), + ((0.5, 1.0),), + ), + ( + "94357219628", + 1, + ((0, 11),), + ((0.5, 1.0),), + ), + ( + "79059236630", + 1, + ((0, 11),), + ((0.5, 1.0),), + ), + ( + "64625294480", + 1, + ((0, 11),), + ((0.5, 1.0),), + ), # Valid TCKNs in sentences ( "TC Kimlik No: 10000000146", @@ -46,26 +81,92 @@ def entities(): ( "Birinci kişi: 10000000146, ikinci kişi: 76543210794", 2, - ((14, 25), (40, 51),), - ((0.5, 1.0), (0.5, 1.0),), + ( + (14, 25), + (40, 51), + ), + ( + (0.5, 1.0), + (0.5, 1.0), + ), ), # Invalid TCKNs - first digit is 0 - ("00000000000", 0, (), (),), - ("02531814694", 0, (), (),), # first digit 0, rest mathematically plausible + ( + "00000000000", + 0, + (), + (), + ), + ( + "02531814694", + 0, + (), + (), + ), # first digit 0, rest mathematically plausible # Invalid TCKNs - wrong 10th digit checksum - ("12345678900", 0, (), (),), - ("76543210780", 0, (), (),), - ("83219500748", 0, (), (),), # 10th digit wrong, single-digit mismatch - ("11798724308", 0, (), (),), # random entry, both 10th and 11th wrong + ( + "12345678900", + 0, + (), + (), + ), + ( + "76543210780", + 0, + (), + (), + ), + ( + "83219500748", + 0, + (), + (), + ), # 10th digit wrong, single-digit mismatch + ( + "11798724308", + 0, + (), + (), + ), # random entry, both 10th and 11th wrong # Invalid TCKNs - correct 10th digit but wrong 11th digit - ("10000000145", 0, (), (),), - ("62286775983", 0, (), (),), # 10th correct, 11th wrong - ("97485249605", 0, (), (),), # OCR/typo scenario, single digit off + ( + "10000000145", + 0, + (), + (), + ), + ( + "62286775983", + 0, + (), + (), + ), # 10th correct, 11th wrong + ( + "97485249605", + 0, + (), + (), + ), # OCR/typo scenario, single digit off # Invalid TCKNs - wrong length - ("1234567890", 0, (), (),), - ("123456789012", 0, (), (),), + ( + "1234567890", + 0, + (), + (), + ), + ( + "123456789012", + 0, + (), + (), + ), # Invalid TCKNs - non-digits - ("abcdefghijk", 0, (), (),), + ( + "abcdefghijk", + 0, + (), + (), + ), # Context enhancement ( "Turkish ID 10000000146", diff --git a/presidio-analyzer/tests/test_tr_phone_number_recognizer.py b/presidio-analyzer/tests/test_tr_phone_number_recognizer.py index 8ab0470f9c..bf781f0ec7 100644 --- a/presidio-analyzer/tests/test_tr_phone_number_recognizer.py +++ b/presidio-analyzer/tests/test_tr_phone_number_recognizer.py @@ -125,7 +125,6 @@ def entities(): ("0232 123 45 67", 1, ((0, 14),), ((0.05, 1.0),)), ("0312 123 45 67", 1, ((0, 14),), ((0.1, 1.0),)), ("0412 123 45 67", 1, ((0, 14),), ((0.1, 1.0),)), - # False positive: embedded in longer number ("15053212345678", 0, (), ()), # Geographic number: starts with 2 (valid geographic) diff --git a/presidio-analyzer/tests/test_transformers_recognizer.py b/presidio-analyzer/tests/test_transformers_recognizer.py index 744d1a1abe..78e18785dc 100644 --- a/presidio-analyzer/tests/test_transformers_recognizer.py +++ b/presidio-analyzer/tests/test_transformers_recognizer.py @@ -76,7 +76,6 @@ def test_when_using_transformers_then_all_transformers_result_correct( ) - @pytest.mark.skip_engine("transformers_en") def test_when_person_in_text_then_person_full_name_complex_found( nlp_engine, nlp_recognizer, entities diff --git a/presidio-analyzer/tests/test_tw_national_id_recognizer.py b/presidio-analyzer/tests/test_tw_national_id_recognizer.py new file mode 100644 index 0000000000..fe307f05e3 --- /dev/null +++ b/presidio-analyzer/tests/test_tw_national_id_recognizer.py @@ -0,0 +1,60 @@ +import pytest + +from tests import assert_result_within_score_range +from presidio_analyzer.predefined_recognizers import TwNationalIdRecognizer + + +@pytest.fixture(scope="module") +def recognizer(): + return TwNationalIdRecognizer() + + +@pytest.fixture(scope="module") +def entities(): + return ["TW_NATIONAL_ID"] + + +@pytest.mark.parametrize( + "text, expected_len, expected_positions, expected_score_ranges", + [ + # fmt: off + # Valid Taiwan IDs (Passing checksum algorithm verification) + ("My ID is A123456789.", 1, ((9, 19),), ((0.2, 0.4),),), # Valid ASCII text format + ("A123456716", 1, ((0, 10),), ((0.2, 0.4),),), # Valid Checksum string + ("身分證A123456789", 1, ((3, 13),), ((0.2, 0.4),),), # CJK Chinese Character border check + ("身份證字號: H123456786", 1, ((7, 17),), ((0.2, 0.4),),), # Alternative context layout + + # Invalid Formats / Failed Checksums / Non-Matches + ("A123456788", 0, (), (),), # Valid formatting structure, but mathematically invalid checksum digit + ("A323456789", 0, (), (),), # Invalid middle format gender character + ("A12345678", 0, (), (),), # Insufficient length boundary + ("a123456789", 0, (), (),), + # Case sensitivity exclusion filter check + # fmt: on + ], +) +def test_when_tw_national_id_in_text_then_all_are_found( + text, + expected_len, + expected_positions, + expected_score_ranges, + recognizer, + entities, + max_score, +): + results = recognizer.analyze(text, entities) + results = sorted(results, key=lambda x: x.start) + assert len(results) == expected_len + + for res, st_pos, score_range in zip( + results, expected_positions, expected_score_ranges + ): + assert res.entity_type == entities[0] + assert_result_within_score_range( + res, + expected_entity_type=entities[0], + expected_start=st_pos[0], + expected_end=st_pos[1], + expected_score_min=score_range[0], + expected_score_max=score_range[1], + ) diff --git a/presidio-analyzer/tests/test_uk_nino_recognizer.py b/presidio-analyzer/tests/test_uk_nino_recognizer.py index 8499fb808c..26726c4aac 100644 --- a/presidio-analyzer/tests/test_uk_nino_recognizer.py +++ b/presidio-analyzer/tests/test_uk_nino_recognizer.py @@ -31,7 +31,7 @@ def entities(): ("nino: nt 99 88 77 a", 0, (), (), ), ("This isn't a valid national insurance number UV 98 76 54 B", 0, (), (), ), # fmt: on - ] + ], ) def test_when_nino_in_text_then_all_uk_ninos_found( text, diff --git a/presidio-analyzer/tests/test_url_recognizer.py b/presidio-analyzer/tests/test_url_recognizer.py index 085bf21733..5a9e218115 100644 --- a/presidio-analyzer/tests/test_url_recognizer.py +++ b/presidio-analyzer/tests/test_url_recognizer.py @@ -43,7 +43,7 @@ def entities(): ], ) def test_when_all_urls_then_succeed( - text, expected_len, expected_positions, expected_score, recognizer, entities + text, expected_len, expected_positions, expected_score, recognizer, entities ): results = recognizer.analyze(text, entities) assert len(results) == expected_len diff --git a/presidio-analyzer/tests/test_us_mbi_recognizer.py b/presidio-analyzer/tests/test_us_mbi_recognizer.py index 0863ba62ce..0b37211f40 100644 --- a/presidio-analyzer/tests/test_us_mbi_recognizer.py +++ b/presidio-analyzer/tests/test_us_mbi_recognizer.py @@ -96,6 +96,12 @@ def test_mbi_recognizer_supported_language(recognizer): def test_mbi_recognizer_context_words(recognizer): """Test that recognizer has appropriate context words.""" expected_context = [ - "medicare", "mbi", "beneficiary", "cms", "medicaid", "hic", "hicn" + "medicare", + "mbi", + "beneficiary", + "cms", + "medicaid", + "hic", + "hicn", ] assert recognizer.context == expected_context diff --git a/presidio-analyzer/tests/test_yaml_recognizer_models.py b/presidio-analyzer/tests/test_yaml_recognizer_models.py index 1979f3bf9c..f6227d5330 100644 --- a/presidio-analyzer/tests/test_yaml_recognizer_models.py +++ b/presidio-analyzer/tests/test_yaml_recognizer_models.py @@ -13,10 +13,7 @@ def test_language_context_config_valid(): """Test LanguageContextConfig validates correctly.""" - lang_config = LanguageContextConfig( - language="en", - context=["credit", "card"] - ) + lang_config = LanguageContextConfig(language="en", context=["credit", "card"]) assert lang_config.language == "en" assert lang_config.context == ["credit", "card"] @@ -24,8 +21,7 @@ def test_language_context_config_valid(): def test_language_context_config_valid_with_region(): """Test LanguageContextConfig with region code.""" lang_config = LanguageContextConfig( - language="en-US", - context=["social", "security"] + language="en-US", context=["social", "security"] ) assert lang_config.language == "en-US" assert lang_config.context == ["social", "security"] @@ -70,7 +66,7 @@ def test_base_recognizer_config_full(): type="custom", supported_language="en", context=["test", "context"], - supported_entity="TEST_ENTITY" + supported_entity="TEST_ENTITY", ) assert config.name == "test_recognizer" assert config.enabled is False @@ -84,20 +80,14 @@ def test_base_recognizer_config_full(): def test_language_fields_preserved(): """Test that supported_language is preserved as-is (not normalized).""" - config = BaseRecognizerConfig( - name="test", - supported_language="en" - ) + config = BaseRecognizerConfig(name="test", supported_language="en") assert config.supported_language == "en" assert config.supported_languages is None def test_entity_fields_preserved(): """Test that supported_entity is preserved as-is (not normalized).""" - config = BaseRecognizerConfig( - name="test", - supported_entity="PERSON" - ) + config = BaseRecognizerConfig(name="test", supported_entity="PERSON") assert config.supported_entity == "PERSON" assert config.supported_entities is None @@ -106,11 +96,11 @@ def test_cannot_specify_both_language_formats(): """Test that specifying both language formats raises error.""" with pytest.raises(ValidationError) as exc_info: BaseRecognizerConfig( - name="test", - supported_language="en", - supported_languages=["es", "fr"] + name="test", supported_language="en", supported_languages=["es", "fr"] ) - assert "Cannot specify both 'supported_language' and 'supported_languages'" in str(exc_info.value) + assert "Cannot specify both 'supported_language' and 'supported_languages'" in str( + exc_info.value + ) def test_cannot_specify_both_entity_formats(): @@ -119,37 +109,34 @@ def test_cannot_specify_both_entity_formats(): BaseRecognizerConfig( name="test", supported_entity="PERSON", - supported_entities=["LOCATION", "ORG"] + supported_entities=["LOCATION", "ORG"], ) - assert "has both 'supported_entity' and 'supported_entities' specified" in str(exc_info.value) + assert "has both 'supported_entity' and 'supported_entities' specified" in str( + exc_info.value + ) def test_invalid_single_language_format(): """Test validation of single language format.""" with pytest.raises(ValidationError): - BaseRecognizerConfig( - name="test", - supported_language="invalid" - ) + BaseRecognizerConfig(name="test", supported_language="invalid") def test_context_with_multiple_languages_error(): """Test that global context with multiple languages raises error.""" with pytest.raises(ValidationError) as exc_info: BaseRecognizerConfig( - name="test", - supported_languages=["en", "es"], - context=["global", "context"] + name="test", supported_languages=["en", "es"], context=["global", "context"] ) - assert "Global context can only be used with a single language" in str(exc_info.value) + assert "Global context can only be used with a single language" in str( + exc_info.value + ) def test_context_with_single_language_valid(): """Test that global context with single language is valid.""" config = BaseRecognizerConfig( - name="test", - supported_languages=["en"], - context=["global", "context"] + name="test", supported_languages=["en"], context=["global", "context"] ) assert config.context == ["global", "context"] @@ -165,8 +152,7 @@ def test_predefined_recognizer_config_defaults(): def test_predefined_recognizer_config_with_language(): """Test predefined recognizer with language specification.""" config = PredefinedRecognizerConfig( - name="CreditCardRecognizer", - supported_language="en" + name="CreditCardRecognizer", supported_language="en" ) assert config.supported_language == "en" assert config.supported_languages is None @@ -175,16 +161,10 @@ def test_predefined_recognizer_config_with_language(): def test_custom_recognizer_config_with_patterns(): """Test custom recognizer with patterns.""" patterns = [ - { - "name": "test_pattern", - "regex": r"\b\d{4}-\d{4}-\d{4}-\d{4}\b", - "score": 0.8 - } + {"name": "test_pattern", "regex": r"\b\d{4}-\d{4}-\d{4}-\d{4}\b", "score": 0.8} ] config = CustomRecognizerConfig( - name="custom_test", - supported_entity="CUSTOM_ENTITY", - patterns=patterns + name="custom_test", supported_entity="CUSTOM_ENTITY", patterns=patterns ) assert config.name == "custom_test" assert config.type == "custom" @@ -260,7 +240,7 @@ def test_custom_recognizer_config_with_deny_list(): name="custom_test", supported_entity="CUSTOM_ENTITY", deny_list=["exclude", "this"], - deny_list_score=0.1 + deny_list_score=0.1, ) assert config.deny_list == ["exclude", "this"] assert config.deny_list_score == 0.1 @@ -270,9 +250,7 @@ def test_custom_recognizer_config_invalid_patterns_not_list(): """Test that patterns must be a list.""" with pytest.raises(ValidationError) as exc_info: CustomRecognizerConfig( - name="test", - supported_entity="TEST", - patterns="not a list" + name="test", supported_entity="TEST", patterns="not a list" ) @@ -280,9 +258,7 @@ def test_custom_recognizer_config_invalid_pattern_not_dict(): """Test that each pattern must be a dict.""" with pytest.raises(ValidationError) as exc_info: CustomRecognizerConfig( - name="test", - supported_entity="TEST", - patterns=["not a dict"] + name="test", supported_entity="TEST", patterns=["not a dict"] ) @@ -296,25 +272,15 @@ def test_custom_recognizer_config_pattern_missing_fields(): with pytest.raises(ValidationError) as exc_info: CustomRecognizerConfig( - name="test", - supported_entity="TEST", - patterns=[pattern] + name="test", supported_entity="TEST", patterns=[pattern] ) def test_custom_recognizer_config_invalid_score_type(): """Test that pattern score must be float.""" - pattern = { - "name": "test", - "regex": r"\d+", - "score": "not a float" - } + pattern = {"name": "test", "regex": r"\d+", "score": "not a float"} with pytest.raises(ValidationError) as exc_info: - CustomRecognizerConfig( - name="test", - supported_entity="TEST", - patterns=[pattern] - ) + CustomRecognizerConfig(name="test", supported_entity="TEST", patterns=[pattern]) def test_custom_recognizer_config_invalid_score_range(): @@ -322,26 +288,17 @@ def test_custom_recognizer_config_invalid_score_range(): invalid_scores = [-0.1, 1.1, 2.0] for score in invalid_scores: - pattern = { - "name": "test", - "regex": r"\d+", - "score": score - } + pattern = {"name": "test", "regex": r"\d+", "score": score} with pytest.raises(ValidationError) as exc_info: CustomRecognizerConfig( - name="test", - supported_entity="TEST", - patterns=[pattern] + name="test", supported_entity="TEST", patterns=[pattern] ) def test_custom_recognizer_config_no_patterns_or_deny_list(): """Test that custom recognizer must have patterns or deny_list.""" with pytest.raises(ValidationError) as exc_info: - CustomRecognizerConfig( - name="test", - supported_entity="TEST" - ) + CustomRecognizerConfig(name="test", supported_entity="TEST") def test_custom_recognizer_config_invalid_deny_list_score(): @@ -351,7 +308,7 @@ def test_custom_recognizer_config_invalid_deny_list_score(): name="test", supported_entity="TEST", deny_list=["test"], - deny_list_score=1.5 # Invalid: > 1.0 + deny_list_score=1.5, # Invalid: > 1.0 ) with pytest.raises(ValidationError): @@ -359,7 +316,7 @@ def test_custom_recognizer_config_invalid_deny_list_score(): name="test", supported_entity="TEST", deny_list=["test"], - deny_list_score=-0.1 # Invalid: < 0.0 + deny_list_score=-0.1, # Invalid: < 0.0 ) @@ -374,8 +331,7 @@ def test_recognizer_registry_config_defaults(): def test_recognizer_registry_config_valid_languages(): """Test registry with valid languages.""" config = RecognizerRegistryConfig( - supported_languages=["en", "es", "fr-CA"], - recognizers=["CreditCardRecognizer"] + supported_languages=["en", "es", "fr-CA"], recognizers=["CreditCardRecognizer"] ) assert config.supported_languages == ["en", "es", "fr-CA"] @@ -385,15 +341,14 @@ def test_recognizer_registry_config_invalid_language(): with pytest.raises(ValidationError): RecognizerRegistryConfig( supported_languages=["en", "invalid", "es"], - recognizers=["CreditCardRecognizer"] + recognizers=["CreditCardRecognizer"], ) def test_recognizer_registry_config_empty_languages(): """Test registry with empty languages list.""" config = RecognizerRegistryConfig( - supported_languages=[], - recognizers=["CreditCardRecognizer"] + supported_languages=[], recognizers=["CreditCardRecognizer"] ) assert config.supported_languages == [] @@ -401,20 +356,14 @@ def test_recognizer_registry_config_empty_languages(): def test_recognizer_registry_config_empty_recognizers(): """Test that empty recognizers list raises a validation error.""" with pytest.raises(ValidationError) as exc_info: - RecognizerRegistryConfig( - recognizers=[], - global_regex_flags=26 - ) + RecognizerRegistryConfig(recognizers=[], global_regex_flags=26) assert "empty recognizers list" in str(exc_info.value).lower() def test_recognizer_registry_config_missing_recognizers(): """Test that missing recognizers field raises a validation error.""" with pytest.raises(ValidationError) as exc_info: - RecognizerRegistryConfig( - supported_languages=["en"], - global_regex_flags=26 - ) + RecognizerRegistryConfig(supported_languages=["en"], global_regex_flags=26) assert "empty recognizers list" in str(exc_info.value).lower() @@ -433,7 +382,7 @@ def test_recognizer_registry_config_mixed_recognizers(): "name": "custom_test", "type": "custom", "supported_entity": "TEST", - "patterns": [{"name": "test", "regex": r"\d+", "score": 0.5}] + "patterns": [{"name": "test", "regex": r"\d+", "score": 0.5}], } with pytest.raises(ValidationError) as exc_info: @@ -441,7 +390,7 @@ def test_recognizer_registry_config_mixed_recognizers(): recognizers=[ "credit_card", # string predefined {"name": "UrlRecognizer", "type": "predefined"}, # predefined - custom_config # custom without languages should trigger error + custom_config, # custom without languages should trigger error ] ) assert "Language configuration missing" in str(exc_info.value) @@ -467,7 +416,7 @@ def test_recognizer_registry_config_auto_detect_type(): "name": "auto_custom_patterns", "supported_entity": "TEST", "supported_language": "en", - "patterns": [{"name": "test", "regex": r"\d+", "score": 0.5}] + "patterns": [{"name": "test", "regex": r"\d+", "score": 0.5}], } # Should be detected as custom due to deny_list @@ -475,18 +424,19 @@ def test_recognizer_registry_config_auto_detect_type(): "name": "auto_custom_deny", "supported_entity": "TEST", "supported_language": "en", - "deny_list": ["exclude_this"] + "deny_list": ["exclude_this"], } # Should be detected as predefined (no patterns or deny_list) - predefined_config = { - "name": "UrlRecognizer", - "enabled": True - } + predefined_config = {"name": "UrlRecognizer", "enabled": True} config = RecognizerRegistryConfig( supported_languages=["en"], # Add global language to satisfy new validation - recognizers=[custom_with_patterns_config, custom_with_deny_list_config, predefined_config] + recognizers=[ + custom_with_patterns_config, + custom_with_deny_list_config, + predefined_config, + ], ) assert isinstance(config.recognizers[0], CustomRecognizerConfig) @@ -497,32 +447,23 @@ def test_recognizer_registry_config_auto_detect_type(): assert config.recognizers[2].type == "predefined" - def test_complete_registry_scenario(): """Test a complete registry configuration scenario.""" registry_config = { "supported_languages": ["en", "es"], "recognizers": [ "credit_card", # String recognizer (kept as string) - { - "name": "EmailRecognizer", - "type": "predefined", - "enabled": True - }, + {"name": "EmailRecognizer", "type": "predefined", "enabled": True}, { "name": "custom_pattern", "type": "custom", "supported_entity": "CUSTOM_ID", "supported_language": "en", "patterns": [ - { - "name": "id_pattern", - "regex": r"ID-\d{6}", - "score": 0.9 - } - ] - } - ] + {"name": "id_pattern", "regex": r"ID-\d{6}", "score": 0.9} + ], + }, + ], } config = RecognizerRegistryConfig(**registry_config) @@ -532,7 +473,6 @@ def test_complete_registry_scenario(): assert isinstance(config.recognizers[2], CustomRecognizerConfig) - def test_error_handling_cascade(): """Test that validation errors are properly cascaded.""" # This should fail at the CustomRecognizerConfig level @@ -548,9 +488,9 @@ def test_error_handling_cascade(): { "name": "test", "regex": r"\d+", - "score": 2.0 # Invalid score > 1.0 + "score": 2.0, # Invalid score > 1.0 } - ] + ], } ] ) @@ -587,11 +527,13 @@ def test_custom_recognizer_config_predefined_name_error(): name="CreditCardRecognizer", # This is a predefined recognizer type="custom", supported_entity="CREDIT_CARD", - patterns=[{"name": "test", "regex": r"\d+", "score": 0.5}] + patterns=[{"name": "test", "regex": r"\d+", "score": 0.5}], ) error_message = str(exc_info.value) - assert "Recognizer 'CreditCardRecognizer' conflicts with a predefined" in error_message + assert ( + "Recognizer 'CreditCardRecognizer' conflicts with a predefined" in error_message + ) assert "Either use type: 'predefined' or choose a different name" in error_message @@ -600,13 +542,15 @@ def test_custom_recognizer_config_predefined_name_error_without_required_fields( with pytest.raises(ValidationError) as exc_info: CustomRecognizerConfig( name="UrlRecognizer", # This is a predefined recognizer - type="custom" + type="custom", # Intentionally missing supported_entity, patterns, and deny_list ) error_message = str(exc_info.value) - assert "conflicts with a predefined recognizer" in error_message or \ - "is a predefined recognizer but is marked as 'custom'" in error_message + assert ( + "conflicts with a predefined recognizer" in error_message + or "is a predefined recognizer but is marked as 'custom'" in error_message + ) def test_custom_recognizer_config_unique_name_valid(): @@ -615,7 +559,7 @@ def test_custom_recognizer_config_unique_name_valid(): name="MyCustomRecognizer", # This should not exist as predefined type="custom", supported_entity="CUSTOM_ENTITY", - patterns=[{"name": "test", "regex": r"\d+", "score": 0.5}] + patterns=[{"name": "test", "regex": r"\d+", "score": 0.5}], ) assert config.name == "MyCustomRecognizer" assert config.type == "custom" @@ -631,7 +575,7 @@ def test_custom_recognizer_config_predefined_name_validation_with_import_error() name="SomeUniqueRecognizer", type="custom", supported_entity="TEST", - patterns=[{"name": "test", "regex": r"\d+", "score": 0.5}] + patterns=[{"name": "test", "regex": r"\d+", "score": 0.5}], ) assert config.name == "SomeUniqueRecognizer" assert config.type == "custom" @@ -646,9 +590,7 @@ def test_custom_recognizer_with_language_no_global_languages(): "type": "custom", "supported_entity": "TEST", "supported_language": "en", - "patterns": [ - {"name": "p", "regex": r"\d+", "score": 0.5} - ] + "patterns": [{"name": "p", "regex": r"\d+", "score": 0.5}], } ] } @@ -674,7 +616,7 @@ def test_recognizer_registry_config_custom_name_with_hf_class(): "type": "predefined", "supported_language": "ko", "supported_entities": ["PERSON"], - "model_name": "TestModel/Ner" + "model_name": "TestModel/Ner", } ] }