From 290a35e8e8f4d5e09e88980fb6dbd2a3770fbbba Mon Sep 17 00:00:00 2001 From: CTO Agent Date: Wed, 8 Jul 2026 11:39:55 -0400 Subject: [PATCH] perf: gate anonymizer regexes with cheap substring checks (~15x faster) Run each regex family in TraceAnonymizer only when a cheap literal substring check says the text could match: emails need '@', home paths need 'home'/'Users', each API-key pattern needs its literal prefix, assignment/generic-secret scans need a sensitive word, JWTs need a lookaround-free pre-scan. Also cache the per-username owner/group patterns instead of recompiling them for every string. Output is byte-identical on a 3000-line benchmark (7.2s -> 0.48s); per-file replacement scoping is unchanged. Co-Authored-By: Claude Fable 5 --- src/teich/anonymize.py | 113 ++++++++++++++++++++++++++++++++--------- 1 file changed, 90 insertions(+), 23 deletions(-) diff --git a/src/teich/anonymize.py b/src/teich/anonymize.py index 4df3627..c360318 100644 --- a/src/teich/anonymize.py +++ b/src/teich/anonymize.py @@ -432,12 +432,62 @@ class TraceAnonymizer: "timer", } + # ponytail: literal gates — each entry pairs the regexes above with cheap + # substring checks so the expensive lookbehind patterns only run on text + # that could possibly match. Order mirrors _api_key_patterns. + _api_key_gates = ( + ("sk-or-v1-",), + ("sk-ant-api03-",), + ("sk-proj-",), + ("sk-",), + ("hf_",), + ("gsk_",), + ("github_pat_",), + ("ghp_", "gho_", "ghu_", "ghs_", "ghr_"), + ("glpat-",), + ("lin_api_",), + ("npm_",), + ("pypi-",), + ("sk_live_", "sk_test_"), + ("rk_live_", "rk_test_", "pk_live_", "pk_test_"), + ("whsec_",), + ("re_",), + ("sq0atp-", "sq0csp-"), + ("xoxb-", "xoxa-", "xoxp-", "xoxr-", "xoxs-"), + ("AIza",), + ("GOCSPX-",), + ("ctx7sk-",), + ("AKIA", "ASIA"), + ("SK",), + ("SG.",), + ) + # Every name _is_sensitive_name can fire on contains one of these + # substrings (lowercased), so assignment/generic-secret scans are skipped + # when none is present. + _sensitive_gate_words = ( + "pass", + "pwd", + "secret", + "credential", + "private", + "token", + "sig", + "jwt", + "key", + "url", + "uri", + "dsn", + "connection", + ) + _jwt_gate = re.compile(r"[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}") + def __init__(self) -> None: self.counts = {"email": 0, "username": 0, "api_key": 0} self._email_map: dict[str, str] = {} self._username_map: dict[str, str] = {} self._api_key_map: dict[str, str] = {} self._api_replacements: set[str] = set() + self._owner_group_patterns: dict[str, re.Pattern[str] | None] = {} def anonymize_value(self, value: Any) -> Any: if isinstance(value, str): @@ -500,10 +550,14 @@ def _looks_like_base64_blob(value: str) -> bool: return re.fullmatch(r"[A-Za-z0-9+/=\s]+", value) is not None def anonymize_text(self, text: str) -> str: - text = self._replace_emails(text) - text = self._replace_usernames_in_paths(text) - text = self._replace_encoded_home_usernames(text) - text = self._replace_known_unix_owner_group_usernames(text) + lowered = text.lower() + if "@" in text: + text = self._replace_emails(text) + if "home" in lowered or "users" in lowered: + text = self._replace_usernames_in_paths(text) + text = self._replace_encoded_home_usernames(text) + if self._username_map: + text = self._replace_known_unix_owner_group_usernames(text) text = self._replace_api_keys(text) return text @@ -601,32 +655,45 @@ def replace(match: re.Match[str]) -> str: def _replace_known_unix_owner_group_usernames(self, text: str) -> str: for username, replacement in list(self._username_map.items()): - if not re.fullmatch(r"[A-Za-z0-9._-]{3,}", username): - continue - if username in self._non_person_usernames: + if username not in self._owner_group_patterns: + if not re.fullmatch(r"[A-Za-z0-9._-]{3,}", username) or username in self._non_person_usernames: + self._owner_group_patterns[username] = None + else: + self._owner_group_patterns[username] = re.compile( + rf"(? str: - text = self._private_key_block_pattern.sub(self._replace_private_key_block, text) - for pattern in self._api_key_patterns: - text = pattern.sub(self._replace_prefixed_key, text) - text = self._jwe_pattern.sub(self._replace_jwt, text) - text = self._jwt_pattern.sub(self._replace_jwt, text) - text = self._bearer_pattern.sub(self._replace_bearer, text) - text = self._credential_url_pattern.sub(self._replace_credential_url_password, text) - text = self._query_secret_pattern.sub(self._replace_query_secret, text) - text = self._connection_component_secret_pattern.sub(self._replace_connection_component_secret, text) - text = self._quoted_assignment_pattern.sub(self._replace_sensitive_quoted_assignment, text) - text = self._bare_assignment_pattern.sub(self._replace_sensitive_bare_assignment, text) - text = self._generic_secret_pattern.sub(self._replace_generic_secret, text) + if "-----BEGIN " in text: + text = self._private_key_block_pattern.sub(self._replace_private_key_block, text) + for gates, pattern in zip(self._api_key_gates, self._api_key_patterns): + if any(gate in text for gate in gates): + text = pattern.sub(self._replace_prefixed_key, text) + if self._jwt_gate.search(text): + text = self._jwe_pattern.sub(self._replace_jwt, text) + text = self._jwt_pattern.sub(self._replace_jwt, text) + lowered = text.lower() + if "bearer" in lowered: + text = self._bearer_pattern.sub(self._replace_bearer, text) + if "://" in text and "@" in text: + text = self._credential_url_pattern.sub(self._replace_credential_url_password, text) + if any(word in lowered for word in self._sensitive_gate_words): + if "=" in text and ("?" in text or "&" in text): + text = self._query_secret_pattern.sub(self._replace_query_secret, text) + if "=" in text: + text = self._connection_component_secret_pattern.sub(self._replace_connection_component_secret, text) + if "=" in text or ":" in text: + text = self._quoted_assignment_pattern.sub(self._replace_sensitive_quoted_assignment, text) + text = self._bare_assignment_pattern.sub(self._replace_sensitive_bare_assignment, text) + text = self._generic_secret_pattern.sub(self._replace_generic_secret, text) return text def _map_username(self, username: str) -> str: