diff --git a/.env.example b/.env.example index e4e6060..c19aaed 100644 --- a/.env.example +++ b/.env.example @@ -34,6 +34,12 @@ MASKER_DEFAULT_POLICY_ID=default # Default fail mode: "closed" (block on error) or "open" (forward on error) MASKER_DEFAULT_FAIL_MODE=closed +# ======================================== +# Custom Sensitive Words +# ======================================== +# Comma-separated, case-insensitive words or phrases to redact +MASKER_SENSITIVE_WORDS= + # ======================================== # Audit Settings # ======================================== diff --git a/README.md b/README.md index f6ff8eb..d59d400 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ Masker detects and redacts: - **PHONE** — phone numbers - **CARD** — credit card numbers - **PERSON** — person names (AI-powered) +- **SENSITIVE_WORD** — custom words and phrases configured by the operator ## Quick Start @@ -86,8 +87,11 @@ docker-compose up -d MASKER_API_KEYS=sk-key1:tenant1,sk-key2:tenant2 MASKER_UPSTREAM_URL=https://api.openai.com/v1/chat/completions MASKER_DEFAULT_FAIL_MODE=closed +MASKER_SENSITIVE_WORDS=secret,confidential,internal project ``` +`MASKER_SENSITIVE_WORDS` is a comma-separated, case-insensitive list. + See [.env.example](.env.example) for all options. ## Documentation diff --git a/app/api/rapidapi/redact.py b/app/api/rapidapi/redact.py index e34d674..625205d 100644 --- a/app/api/rapidapi/redact.py +++ b/app/api/rapidapi/redact.py @@ -129,7 +129,7 @@ async def rapidapi_redact(request: RapidAPIRedactRequest) -> RapidAPIRedactRespo path=e.path, start=e.start, end=e.end, - score=1.0 if e.type in ("EMAIL", "PHONE", "CARD") else 0.85, + score=1.0 if e.type in ("EMAIL", "PHONE", "CARD", "SENSITIVE_WORD") else 0.85, ) for e in json_entities ] diff --git a/app/core/config.py b/app/core/config.py index 5fdd318..6f1c621 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -38,6 +38,15 @@ class Settings(BaseSettings): placeholder_email: str = "" placeholder_phone: str = "" placeholder_card: str = "" + placeholder_sensitive_word: str = "" + + # Sensitive word detection (comma-separated list) + sensitive_words: str = "" + + @property + def sensitive_word_list(self) -> list[str]: + """Parse comma-separated sensitive words.""" + return [w.strip() for w in self.sensitive_words.split(",") if w.strip()] # ======================================== # Safe-to-LLM Proxy Settings diff --git a/app/models/rapidapi_schemas.py b/app/models/rapidapi_schemas.py index c41241e..d432c44 100644 --- a/app/models/rapidapi_schemas.py +++ b/app/models/rapidapi_schemas.py @@ -7,7 +7,7 @@ from app.core.config import settings # Supported entity types for filtering -EntityTypeFilter = Literal["PERSON", "EMAIL", "PHONE", "CARD"] +EntityTypeFilter = Literal["PERSON", "EMAIL", "PHONE", "CARD", "SENSITIVE_WORD"] # Redaction modes RedactionMode = Literal["mask", "placeholder"] @@ -81,7 +81,8 @@ class RedactedItem(BaseModel): """Schema for a single redacted item in the response.""" entity_type: str = Field( - ..., description="Type of the detected entity (PERSON, EMAIL, PHONE, CARD)" + ..., + description=("Type of the detected entity (PERSON, EMAIL, PHONE, CARD, SENSITIVE_WORD)"), ) path: str | None = Field( default=None, description="JSON path to the field (only for JSON mode)" diff --git a/app/models/schemas.py b/app/models/schemas.py index 4508d12..78204b2 100644 --- a/app/models/schemas.py +++ b/app/models/schemas.py @@ -7,7 +7,7 @@ from app.core.config import settings # Entity types -EntityType = Literal["EMAIL", "PHONE", "CARD", "PERSON"] +EntityType = Literal["EMAIL", "PHONE", "CARD", "PERSON", "SENSITIVE_WORD"] class TextRequest(BaseModel): diff --git a/app/services/pii_detector.py b/app/services/pii_detector.py index afa9dbe..544a462 100644 --- a/app/services/pii_detector.py +++ b/app/services/pii_detector.py @@ -32,6 +32,7 @@ class PIIDetector: - PHONE: Phone numbers (international formats) - CARD: Credit/debit card numbers - PERSON: Person names (via spaCy NER) + - SENSITIVE_WORD: User-defined sensitive words (via MASKER_SENSITIVE_WORDS env var) """ # Regex patterns for PII detection @@ -154,6 +155,24 @@ def _detect_by_ner(self, text: str, language: str) -> list[DetectedEntity]: return entities + def _detect_sensitive_words(self, text: str) -> list[DetectedEntity]: + """Detect configured sensitive words using word boundary regex.""" + from app.core.config import settings + + entities = [] + for word in settings.sensitive_word_list: + pattern = re.compile(rf"(? list[DetectedEntity]: """Remove overlapping entities, preferring regex matches. @@ -169,23 +188,29 @@ def _remove_overlaps(self, entities: list[DetectedEntity]) -> list[DetectedEntit if not entities: return [] - # Sort by start position, then by priority (more specific types first) - # CARD has higher priority than PHONE to avoid card numbers being detected as phones - priority = {"EMAIL": 0, "CARD": 1, "PHONE": 2, "PERSON": 3} - sorted_entities = sorted(entities, key=lambda e: (e.start, priority.get(e.type, 99))) + # Resolve overlaps by type priority, regardless of which match starts first. + # CARD has higher priority than PHONE to avoid card numbers being detected as phones. + priority = {"EMAIL": 0, "CARD": 1, "PHONE": 2, "PERSON": 3, "SENSITIVE_WORD": 4} + sorted_entities = sorted( + entities, + key=lambda e: ( + priority.get(e.type, 99), + e.start, + -(e.end - e.start), + ), + ) - result = [] - last_end = -1 + result: list[DetectedEntity] = [] for entity in sorted_entities: - # Skip if this entity overlaps with the previous one - if entity.start < last_end: + if any( + entity.start < existing.end and existing.start < entity.end for existing in result + ): continue result.append(entity) - last_end = entity.end - return result + return sorted(result, key=lambda e: (e.start, e.end)) def detect( self, text: str, language: str = "en", entity_types: list[str] | None = None @@ -207,8 +232,11 @@ def detect( # Then, detect using NER ner_entities = self._detect_by_ner(text, language) + # Detect sensitive words + sensitive_entities = self._detect_sensitive_words(text) + # Combine and remove overlaps - all_entities = regex_entities + ner_entities + all_entities = regex_entities + ner_entities + sensitive_entities unique_entities = self._remove_overlaps(all_entities) # Filter by entity types if specified diff --git a/app/services/redaction.py b/app/services/redaction.py index 0024a7a..aae3756 100644 --- a/app/services/redaction.py +++ b/app/services/redaction.py @@ -26,6 +26,7 @@ class RedactedEntity: "EMAIL": "", "PHONE": "", "CARD": "", + "SENSITIVE_WORD": "", } # Default mask token @@ -64,8 +65,8 @@ def get_entity_score(entity: DetectedEntity) -> float: Returns: Confidence score (0.0 to 1.0) """ - # Regex-based detections (EMAIL, PHONE, CARD) get perfect score - if entity.type in ("EMAIL", "PHONE", "CARD"): + # Deterministic pattern-based detections get a perfect score + if entity.type in ("EMAIL", "PHONE", "CARD", "SENSITIVE_WORD"): return REGEX_SCORE # NER-based detections (PERSON) get a default score diff --git a/policies/default.yaml b/policies/default.yaml index 3f366de..5477fcd 100644 --- a/policies/default.yaml +++ b/policies/default.yaml @@ -14,6 +14,7 @@ categories: phone: mask card: drop person: placeholder + sensitive_word: mask # Behavior when redaction fails: # - closed: block the request (do not forward to upstream) diff --git a/tests/test_sensitive_words.py b/tests/test_sensitive_words.py new file mode 100644 index 0000000..8fe5f87 --- /dev/null +++ b/tests/test_sensitive_words.py @@ -0,0 +1,156 @@ +"""Tests for sensitive word detection.""" + +from fastapi.testclient import TestClient + +from app.core.config import settings + + +class TestSensitiveWords: + """Tests for SENSITIVE_WORD entity detection and masking.""" + + def test_detect_sensitive_word(self, client: TestClient, monkeypatch): + """Should detect configured sensitive words.""" + monkeypatch.setattr(settings, "sensitive_words", "secret,confidential") + response = client.post("/api/v1/detect", json={"text": "This is a secret document"}) + + assert response.status_code == 200 + data = response.json() + sw = [e for e in data["entities"] if e["type"] == "SENSITIVE_WORD"] + assert len(sw) == 1 + assert sw[0]["value"] == "secret" + + def test_detect_multiple_sensitive_words(self, client: TestClient, monkeypatch): + """Should detect multiple different sensitive words.""" + monkeypatch.setattr(settings, "sensitive_words", "secret,confidential") + response = client.post("/api/v1/detect", json={"text": "This is secret and confidential"}) + + assert response.status_code == 200 + data = response.json() + sw = [e for e in data["entities"] if e["type"] == "SENSITIVE_WORD"] + assert len(sw) == 2 + values = {e["value"].lower() for e in sw} + assert "secret" in values + assert "confidential" in values + + def test_case_insensitive_detection(self, client: TestClient, monkeypatch): + """Should detect words regardless of case.""" + monkeypatch.setattr(settings, "sensitive_words", "secret") + response = client.post("/api/v1/detect", json={"text": "This is SECRET and Secret"}) + + assert response.status_code == 200 + data = response.json() + sw = [e for e in data["entities"] if e["type"] == "SENSITIVE_WORD"] + assert len(sw) == 2 + + def test_word_boundary_prevents_substring_match(self, client: TestClient, monkeypatch): + """Should not match sensitive words inside other words.""" + monkeypatch.setattr(settings, "sensitive_words", "secret") + response = client.post("/api/v1/detect", json={"text": "This is a secretory gland"}) + + assert response.status_code == 200 + data = response.json() + sw = [e for e in data["entities"] if e["type"] == "SENSITIVE_WORD"] + assert len(sw) == 0 + + def test_empty_config_no_detection(self, client: TestClient, monkeypatch): + """Should not detect sensitive words when config is empty.""" + monkeypatch.setattr(settings, "sensitive_words", "") + response = client.post("/api/v1/detect", json={"text": "This is a secret document"}) + + assert response.status_code == 200 + data = response.json() + sw = [e for e in data["entities"] if e["type"] == "SENSITIVE_WORD"] + assert len(sw) == 0 + + def test_mask_sensitive_word(self, client: TestClient, monkeypatch): + """Should mask sensitive words with asterisks.""" + monkeypatch.setattr(settings, "sensitive_words", "secret") + response = client.post("/api/v1/mask", json={"text": "This is a secret document"}) + + assert response.status_code == 200 + data = response.json() + assert "***" in data["text"] + assert "secret" not in data["text"] + + def test_redact_sensitive_word(self, client: TestClient, monkeypatch): + """Should redact sensitive words with [REDACTED].""" + monkeypatch.setattr(settings, "sensitive_words", "secret") + response = client.post("/api/v1/redact", json={"text": "This is a secret document"}) + + assert response.status_code == 200 + data = response.json() + assert "[REDACTED]" in data["text"] + assert "secret" not in data["text"] + + def test_overlap_existing_entity_wins(self, client: TestClient, monkeypatch): + """Should prefer EMAIL over SENSITIVE_WORD on overlap.""" + monkeypatch.setattr(settings, "sensitive_words", "test") + response = client.post("/api/v1/detect", json={"text": "Contact test@example.com"}) + + assert response.status_code == 200 + data = response.json() + types = [e["type"] for e in data["entities"]] + assert "EMAIL" in types + assert "SENSITIVE_WORD" not in types + + def test_earlier_sensitive_phrase_does_not_override_email( + self, client: TestClient, monkeypatch + ): + """Should prefer the full EMAIL even when a sensitive phrase starts earlier.""" + monkeypatch.setattr(settings, "sensitive_words", "Contact test") + response = client.post("/api/v1/mask", json={"text": "Contact test@example.com"}) + + assert response.status_code == 200 + data = response.json() + assert data["text"] == "Contact ***" + assert "@example.com" not in data["text"] + types = [entity["type"] for entity in data["entities"]] + assert "EMAIL" in types + assert "SENSITIVE_WORD" not in types + + def test_sensitive_term_can_end_with_punctuation(self, client: TestClient, monkeypatch): + """Should detect configured terms that end with non-word characters.""" + monkeypatch.setattr(settings, "sensitive_words", "C++") + response = client.post("/api/v1/detect", json={"text": "Use C++ safely"}) + + assert response.status_code == 200 + data = response.json() + sw = [e for e in data["entities"] if e["type"] == "SENSITIVE_WORD"] + assert len(sw) == 1 + assert sw[0]["value"] == "C++" + + def test_cyrillic_word_boundary(self, client: TestClient, monkeypatch): + """Should detect Cyrillic sensitive words.""" + monkeypatch.setattr(settings, "sensitive_words", "пароль,секрет") + response = client.post( + "/api/v1/detect", + json={"text": "Введите пароль и секрет.", "language": "ru"}, + ) + + assert response.status_code == 200 + data = response.json() + sw = [e for e in data["entities"] if e["type"] == "SENSITIVE_WORD"] + assert {entity["value"] for entity in sw} == {"пароль", "секрет"} + + def test_sensitive_word_at_text_boundaries(self, client: TestClient, monkeypatch): + """Should detect words at start and end of text.""" + monkeypatch.setattr(settings, "sensitive_words", "secret") + response = client.post("/api/v1/detect", json={"text": "secret is the secret"}) + + assert response.status_code == 200 + data = response.json() + sw = [e for e in data["entities"] if e["type"] == "SENSITIVE_WORD"] + assert len(sw) == 2 + assert sw[0]["start"] == 0 + + def test_sensitive_word_entity_positions(self, client: TestClient, monkeypatch): + """Should return correct positions for sensitive words.""" + monkeypatch.setattr(settings, "sensitive_words", "secret") + text = "The secret is here" + response = client.post("/api/v1/detect", json={"text": text}) + + assert response.status_code == 200 + data = response.json() + sw = [e for e in data["entities"] if e["type"] == "SENSITIVE_WORD"] + assert len(sw) == 1 + assert text[sw[0]["start"] : sw[0]["end"]] == "secret"