From 9dc3c201018c4dfe4fa221cc02fb6f7c56c2ccc9 Mon Sep 17 00:00:00 2001 From: Steven Moy Date: Thu, 17 Sep 2026 23:03:37 +0000 Subject: [PATCH 1/2] perf(security): seek token gaps in C and stop scanning confusables as a regex class Three whole-text fast paths in the security scan, all behaviour-preserving. 1. Token-gap scanning seeks the next candidate with a compiled regex instead of stepping through the text one character at a time in Python. Every token-gap character lies outside printable ASCII and tab/newline/carriage-return, so the seek class is a superset and each hit is still confirmed by the exact predicate. Verified: no code point in Unicode is a gap character that the seek class fails to match. 2. `_ASCII_CONFUSABLE_PATTERN` is replaced by a frozenset membership test. The question asked of it is only ever "does this text contain any of these", and the class holds 1,515 code points spanning 528 disjoint ranges, so as a regex it costs a bounded scan per character. On 180 KB of ASCII prose: regex character class 164.30 ms range-compressed class 44.97 ms frozenset.isdisjoint 0.70 ms The class contains no ASCII code point at all, so ordinary text answers with a single disjointness check. This lifts `_requires_normalized_security_view` from 166.31 ms to 2.78 ms on that text, a 60x improvement, and the regex accounted for 164 of those 166 ms. 3. `normalized_security_view` and the letter-spacing span scan are memoized, in the manner of #570. The view is a frozen dataclass whose `source_offsets` array is only ever read -- sliced, or copied into a fresh array -- so callers can share one instance. A caller passing `check_runtime` bypasses the cached path so runtime budgets are still enforced. Measured on 901 real skills, two alternating runs per arm on an idle 8-core host, against upstream main: p95 -8.8%, p99 -12.1%, total scan time -4.1%. Findings are byte-identical (8,152 on both arms), as are coverage-ledger outcomes (176) and error counts (0). The corpus-level gain is much smaller than the microbenchmarks because the predicate that improves 60x is already memoized by #570, so it runs about 23 times per scan rather than 780. It is still the single most expensive thing left in that predicate, and the regex class would cost proportionally more on larger files. Tests assert the seek class covers every token-gap code point in Unicode, that gap spans are unchanged across 1,500 randomized texts, that confusable membership matches the original character class across 1,500 more, that the memoized spans equal the uncached scan, and that a runtime budget is still honoured. Full suite: 5808 passed. Signed-off-by: Steven Moy --- src/skillspector/artifacts.py | 60 ++++++- .../test_security_scan_fast_paths.py | 152 ++++++++++++++++++ 2 files changed, 207 insertions(+), 5 deletions(-) create mode 100644 tests/nodes/analyzers/test_security_scan_fast_paths.py diff --git a/src/skillspector/artifacts.py b/src/skillspector/artifacts.py index 0d44476e7..62c9d2dbe 100644 --- a/src/skillspector/artifacts.py +++ b/src/skillspector/artifacts.py @@ -282,9 +282,13 @@ class _ObfuscatedIgnoreState: ) _DEFAULT_IGNORABLE_RUN_PATTERN = re.compile(_DEFAULT_IGNORABLE_PATTERN.pattern + "+") _REPEATED_CHARACTER_RUN_PATTERN = re.compile(r"(.)\1+") -_ASCII_CONFUSABLE_PATTERN = re.compile( - "[" + "".join(re.escape(chr(codepoint)) for codepoint in ASCII_CONFUSABLE_SKELETON) + "]" -) +# Membership in a 1,515-code-point class, asked as "does this text contain any". +# As a regex character class that costs a bounded scan per character against 528 +# disjoint ranges; as a set it is one C-level pass building the text's distinct +# characters. On 180 KB of ASCII prose the set form is ~230x faster, and the +# class contains no ASCII code point at all, so ordinary text answers with a +# single disjointness check. +_ASCII_CONFUSABLE_CHARS = frozenset(chr(codepoint) for codepoint in ASCII_CONFUSABLE_SKELETON) _OBFUSCATED_INSTRUCTION_ACTIONS = ( "ignore", "override", @@ -464,11 +468,40 @@ def _letter_spacing_gap_signature(gap: str) -> tuple[str, str] | None: return ("marked", marker[0]) +@lru_cache(maxsize=_TEXT_PREDICATE_CACHE_SIZE) +def _letter_spacing_run_spans_cached( + text: str, require_consistent_separator_class: bool +) -> tuple[tuple[int, int], ...]: + """Materialize the spans once per text so repeat callers reuse them.""" + return tuple( + _letter_spacing_run_spans_uncached( + text, require_consistent_separator_class=require_consistent_separator_class + ) + ) + + def _letter_spacing_run_spans( text: str, check_runtime: Callable[[], None] | None = None, *, require_consistent_separator_class: bool = True, +) -> Iterator[tuple[int, int]]: + """Yield maximal runs of six or more separator-delimited single letters.""" + if check_runtime is None: + yield from _letter_spacing_run_spans_cached(text, require_consistent_separator_class) + return + yield from _letter_spacing_run_spans_uncached( + text, + check_runtime, + require_consistent_separator_class=require_consistent_separator_class, + ) + + +def _letter_spacing_run_spans_uncached( + text: str, + check_runtime: Callable[[], None] | None = None, + *, + require_consistent_separator_class: bool = True, ) -> Iterator[tuple[int, int]]: """Yield maximal runs of six or more separator-delimited single letters.""" if check_runtime is not None: @@ -1524,6 +1557,13 @@ def _compute_token_gap_character(ch: str) -> bool: # ASCII character outside this class is settled by the table above, so a text # built only from them has no gap spans and the per-character walk below is # pure overhead. +# Every token-gap character lies outside printable ASCII and the three ASCII +# whitespace characters, so the next candidate position can be found in C rather +# than by stepping through the text one character at a time in Python. The class +# is a deliberate superset -- an accented letter matches it but is not a gap +# character -- so each hit is still confirmed by the exact predicate below. +_TOKEN_GAP_SEEK = re.compile(r"[^\t\n\r\x20-\x7e]") + _TOKEN_GAP_CANDIDATE = re.compile( "[^" + "".join(re.escape(ch) for ch in map(chr, range(128)) if ch not in _ASCII_TOKEN_GAP_CHARS) @@ -1550,6 +1590,10 @@ def _token_bridging_gap_spans( while offset < len(text): if check_runtime is not None and offset % 4096 == 0: check_runtime() + seek = _TOKEN_GAP_SEEK.search(text, offset) + if seek is None: + return + offset = seek.start() if not _is_token_gap_character(text[offset]): offset += 1 continue @@ -1703,8 +1747,14 @@ def _next_offset(offsets: Iterator[int]) -> int | None: return next(offsets, None) +@lru_cache(maxsize=_TEXT_PREDICATE_CACHE_SIZE) def normalized_security_view(text: str) -> SecurityTextView: - """Build an NFKC/UTS #39 ASCII-skeleton view with compact offsets.""" + """Build an NFKC/UTS #39 ASCII-skeleton view with compact offsets. + + Memoized: every analyzer reaches this with the same file content. The view + is a frozen dataclass and its ``source_offsets`` array is only ever read -- + sliced, or copied into a fresh array -- so callers can share one instance. + """ output = StringIO() offsets = array("I") contextual_spans = iter(_normalization_ignored_spans(text)) @@ -1987,7 +2037,7 @@ def _requires_normalized_security_view(text: str) -> bool: return True if not unicodedata.is_normalized("NFKC", text): return True - if _ASCII_CONFUSABLE_PATTERN.search(text) is not None: + if not _ASCII_CONFUSABLE_CHARS.isdisjoint(text): return True if text.isprintable(): return False diff --git a/tests/nodes/analyzers/test_security_scan_fast_paths.py b/tests/nodes/analyzers/test_security_scan_fast_paths.py new file mode 100644 index 000000000..347a2687b --- /dev/null +++ b/tests/nodes/analyzers/test_security_scan_fast_paths.py @@ -0,0 +1,152 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Equivalence tests for the whole-text fast paths in the security scan. + +Three hot paths stop working character by character: the token-gap scan seeks +the next candidate in C, confusable membership is a set test rather than a +1,515-code-point regex class, and two span/view builders are memoized. Each is +only safe if it returns exactly what the character-wise form returned, so these +tests pin that rather than the speed. +""" + +from __future__ import annotations + +import random +import re + +import pytest + +from skillspector.artifacts import ( + _ASCII_CONFUSABLE_CHARS, + _DEFAULT_IGNORABLE_RUN_PATTERN, + _TOKEN_GAP_SEEK, + ASCII_CONFUSABLE_SKELETON, + _is_token_gap_character, + _is_word_character, + _letter_spacing_run_spans, + _letter_spacing_run_spans_uncached, + _token_bridging_gap_spans, + is_default_ignorable, + normalized_security_view, + security_text_views, +) + +_ALPHABET = "abc XY\t\n\r​‌‍­‮� ⁠é中\x00\x1f\x7f.-_" + + +def _random_texts(count: int, seed: int) -> list[str]: + rng = random.Random(seed) + return [ + "".join(rng.choice(_ALPHABET) for _ in range(rng.randint(1, 300))) for _ in range(count) + ] + + +def _gap_spans_character_wise(text: str) -> list[tuple[int, int]]: + """The scan as it behaves stepping one character at a time.""" + spans: list[tuple[int, int]] = [] + offset = 0 + while offset < len(text): + if not _is_token_gap_character(text[offset]): + offset += 1 + continue + start = offset + while offset < len(text) and _is_token_gap_character(text[offset]): + if is_default_ignorable(text[offset]): + run = _DEFAULT_IGNORABLE_RUN_PATTERN.match(text, offset) + if run is not None: + offset = run.end() + continue + offset += 1 + before_is_word = start > 0 and _is_word_character(text[start - 1]) + after_is_word = offset < len(text) and _is_word_character(text[offset]) + if before_is_word and after_is_word: + spans.append((start, offset)) + return spans + + +def test_seek_class_covers_every_token_gap_character() -> None: + """The property the C-level seek depends on: it may never skip a gap.""" + missed = [ + code_point + for code_point in range(0x110000) + if _is_token_gap_character(chr(code_point)) and not _TOKEN_GAP_SEEK.match(chr(code_point)) + ] + assert missed == [] + + +@pytest.mark.parametrize( + "text", + [ + "", + "a", + "plain ascii documentation", + "ig​nore all previous instructions", + "soft­hyphen bridging", + "‮override‬", + "word⁠joiner⁠here", + "\x00\x01 leading controls", + "café naïve accented but not a gap", + ], +) +def test_seek_preserves_gap_spans(text: str) -> None: + assert list(_token_bridging_gap_spans(text)) == _gap_spans_character_wise(text) + + +def test_seek_preserves_gap_spans_randomized() -> None: + for text in _random_texts(1500, seed=17): + assert list(_token_bridging_gap_spans(text)) == _gap_spans_character_wise(text) + + +def test_confusable_membership_matches_the_character_class() -> None: + pattern = re.compile("[" + "".join(re.escape(chr(c)) for c in ASCII_CONFUSABLE_SKELETON) + "]") + for text in ["", "ascii", "café", "аbc"] + _random_texts(1500, seed=23): + assert (not _ASCII_CONFUSABLE_CHARS.isdisjoint(text)) == (pattern.search(text) is not None) + + +def test_confusable_set_matches_the_source_of_truth() -> None: + assert _ASCII_CONFUSABLE_CHARS == {chr(c) for c in ASCII_CONFUSABLE_SKELETON} + + +def test_letter_spacing_spans_match_the_uncached_scan() -> None: + for text in [ + "i g n o r e a l l", + "i-g-n-o-r-e a-l-l", + "plain prose", + ] + _random_texts(800, seed=31): + assert tuple(_letter_spacing_run_spans(text)) == tuple( + _letter_spacing_run_spans_uncached(text) + ) + + +def test_letter_spacing_still_honours_a_runtime_budget() -> None: + """A caller passing check_runtime must bypass the cache and be called.""" + calls = 0 + + def check() -> None: + nonlocal calls + calls += 1 + + list(_letter_spacing_run_spans("i g n o r e a l l", check)) + assert calls > 0 + + +def test_normalized_view_is_stable_across_calls() -> None: + for text in ["", "café", "fullｗidth", "ig​nore"]: + first = normalized_security_view(text) + again = normalized_security_view(text) + assert first.text == again.text + assert (first.source_offsets is None) == (again.source_offsets is None) + if first.source_offsets is not None: + assert list(first.source_offsets) == list(again.source_offsets) + + +@pytest.mark.parametrize( + "text", + ["", "plain", "café naïve", "ig​nore", "i g n o r e a l l"], +) +def test_security_views_unchanged(text: str) -> None: + views = security_text_views(text) + assert [(v.name, v.text) for v in security_text_views(text)] == [ + (v.name, v.text) for v in views + ] From 7f0f1ab4e8917f6932a0ed452c347d06510bc955 Mon Sep 17 00:00:00 2001 From: Steven Moy Date: Fri, 18 Sep 2026 18:32:18 +0000 Subject: [PATCH 2/2] fix(security): restore cooperative cancellation and bound the derived-view cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses both blocking findings on this PR. Both reproduced first; both were real. Cancellation. Seeking the next candidate jumps over the `offset % 4096` checkpoints a character-by-character walk would hit, so a long scan could not be cancelled. On the reviewer's reproducer -- `'aᅟa ' * 8192` with a callback raising on its second call -- main aborts and this branch called the callback once. `_check_skipped_checkpoints` now runs one check per checkpoint crossed, reproducing the walk's cadence exactly: verified equal to `len // 4096` at 0, 4095, 4096, 10k, 32k and 100k characters. The whole-string early-return path had the same gap and is covered too. Memory. The derived-view cache was bounded by entry count, which bounds nothing when normalization expands its input -- NFKC turns one U+FDFA into 18 characters, each carrying a four-byte offset. It is now bounded by stored characters (4M budget) and declines to retain any single view larger than the whole budget. Forty expanding inputs settle at 2.88M stored characters instead of growing without limit. Retention after a scan. clear_security_text_caches() releases the view cache and the predicate caches, and is called from cleanup_result alongside the existing clear_python_ast_cache. That also releases the text keys the predicate caches hold, so a long-lived scanner process no longer keeps a scanned file's content alive after the scan that produced it. Performance is preserved. Full suite, 901 real skills, two alternating runs per arm against upstream main: p95 -10.3%, p99 -12.7%, total scan time -2.6%. Findings remain byte-identical (8,152 on both arms), as do coverage-ledger outcomes (176) and error counts (0). The mean gain narrows from -4.1% to -2.6%, which is the cost of the restored checkpoints and is the right trade. Thirteen new tests cover the checkpoint cadence at six lengths, cancellation across sparse in-word gaps and on text with no candidates, the size bound, the oversized-single-view case, clearing, teardown via cleanup_result, and that an evicted-then-rebuilt view is identical to the original. Full suite: 5821 passed. Signed-off-by: Steven Moy --- src/skillspector/artifacts.py | 109 +++++++++++++++- src/skillspector/cleanup.py | 2 + .../test_security_scan_runtime_and_memory.py | 117 ++++++++++++++++++ 3 files changed, 224 insertions(+), 4 deletions(-) create mode 100644 tests/nodes/analyzers/test_security_scan_runtime_and_memory.py diff --git a/src/skillspector/artifacts.py b/src/skillspector/artifacts.py index 62c9d2dbe..e6e6008cb 100644 --- a/src/skillspector/artifacts.py +++ b/src/skillspector/artifacts.py @@ -14,6 +14,7 @@ import re import unicodedata from array import array +from collections import OrderedDict from collections.abc import Callable, Iterator from dataclasses import dataclass from enum import StrEnum @@ -1577,6 +1578,21 @@ def _is_token_gap_character(ch: str) -> bool: return _compute_token_gap_character(ch) +_RUNTIME_CHECKPOINT_STRIDE = 4096 + + +def _check_skipped_checkpoints(check_runtime: Callable[[], None], start: int, end: int) -> None: + """Run the cooperative checks a character-by-character walk would have run. + + Seeking jumps straight to the next candidate, so the ``offset % 4096`` + checkpoints between ``start`` and ``end`` would otherwise never fire and a + long scan could not be cancelled. Invoke one per checkpoint crossed, which + matches the cadence of the walk it replaces. + """ + for _ in range(start // _RUNTIME_CHECKPOINT_STRIDE + 1, end // _RUNTIME_CHECKPOINT_STRIDE + 1): + check_runtime() + + def _token_bridging_gap_spans( text: str, *, @@ -1585,6 +1601,8 @@ def _token_bridging_gap_spans( ) -> Iterator[tuple[int, int]]: """Yield contextual noise runs in one pass without crossing ASCII spaces.""" if _TOKEN_GAP_CANDIDATE.search(text) is None: + if check_runtime is not None: + _check_skipped_checkpoints(check_runtime, 0, len(text)) return offset = 0 while offset < len(text): @@ -1592,7 +1610,11 @@ def _token_bridging_gap_spans( check_runtime() seek = _TOKEN_GAP_SEEK.search(text, offset) if seek is None: + if check_runtime is not None: + _check_skipped_checkpoints(check_runtime, offset, len(text)) return + if check_runtime is not None: + _check_skipped_checkpoints(check_runtime, offset, seek.start()) offset = seek.start() if not _is_token_gap_character(text[offset]): offset += 1 @@ -1747,14 +1769,93 @@ def _next_offset(offsets: Iterator[int]) -> int | None: return next(offsets, None) -@lru_cache(maxsize=_TEXT_PREDICATE_CACHE_SIZE) +# Derived views can be far larger than their input, so this cache is bounded by +# stored characters rather than entries. The budget is a few files' worth of +# expanded text -- enough for the analyzers that revisit one file, small enough +# that a long-lived scanner process cannot accumulate. +_DERIVED_VIEW_CACHE_BUDGET_CHARS = 4_000_000 + + +class _SizeBoundedViewCache: + """A small insertion-ordered cache bounded by total stored characters.""" + + def __init__(self, budget: int) -> None: + self._budget = budget + self._entries: OrderedDict[str, SecurityTextView] = OrderedDict() + self._sizes: dict[str, int] = {} + self._total = 0 + + def get(self, key: str) -> SecurityTextView | None: + view = self._entries.get(key) + if view is not None: + self._entries.move_to_end(key) + return view + + def store(self, key: str, view: SecurityTextView, size: int) -> None: + if size > self._budget: + # A single view larger than the whole budget is never worth keeping. + return + if key in self._entries: + return + self._entries[key] = view + self._sizes[key] = size + self._total += size + while self._total > self._budget and self._entries: + evicted, _ = self._entries.popitem(last=False) + self._total -= self._sizes.pop(evicted, 0) + + def clear(self) -> None: + self._entries.clear() + self._sizes.clear() + self._total = 0 + + @property + def stored_chars(self) -> int: + return self._total + + +_NORMALIZED_VIEW_CACHE = _SizeBoundedViewCache(_DERIVED_VIEW_CACHE_BUDGET_CHARS) + + +def clear_security_text_caches() -> None: + """Release every memoized security-text derivation. + + Called from scan teardown so a long-lived scanner process does not retain a + scanned file's content -- the derived views, and the text keys the predicate + caches hold -- after the scan that produced it has finished. + """ + _NORMALIZED_VIEW_CACHE.clear() + for cached in ( + _has_letter_spacing_run, + _has_obfuscated_instruction, + _requires_normalized_security_view, + _letter_spacing_run_spans_cached, + ): + cached.cache_clear() + + def normalized_security_view(text: str) -> SecurityTextView: """Build an NFKC/UTS #39 ASCII-skeleton view with compact offsets. - Memoized: every analyzer reaches this with the same file content. The view - is a frozen dataclass and its ``source_offsets`` array is only ever read -- - sliced, or copied into a fresh array -- so callers can share one instance. + Memoized, because every analyzer reaches this with the same file content. + The view is a frozen dataclass and its ``source_offsets`` array is only ever + read -- sliced, or copied into a fresh array -- so callers can share one + instance. + + The cache is bounded by the *size* of what it stores rather than by entry + count. Normalization can expand its input several-fold -- NFKC turns a + single U+FDFA into 18 characters, each carrying a four-byte offset -- so a + count-based bound places no limit on retained memory. """ + cached = _NORMALIZED_VIEW_CACHE.get(text) + if cached is not None: + return cached + view = _build_normalized_security_view(text) + _NORMALIZED_VIEW_CACHE.store(text, view, len(view.text)) + return view + + +def _build_normalized_security_view(text: str) -> SecurityTextView: output = StringIO() offsets = array("I") contextual_spans = iter(_normalization_ignored_spans(text)) diff --git a/src/skillspector/cleanup.py b/src/skillspector/cleanup.py index 493f56c98..bd2b20ff0 100644 --- a/src/skillspector/cleanup.py +++ b/src/skillspector/cleanup.py @@ -5,11 +5,13 @@ import shutil +from skillspector.artifacts import clear_security_text_caches from skillspector.python_ast import clear_python_ast_cache def cleanup_result(result: dict[str, object]) -> None: """Release scan-local resources and remove a temp dir if set.""" + clear_security_text_caches() python_ast_cache_key = result.get("python_ast_cache_key") clear_python_ast_cache(python_ast_cache_key if isinstance(python_ast_cache_key, str) else None) temp_dir = result.get("temp_dir_for_cleanup") diff --git a/tests/nodes/analyzers/test_security_scan_runtime_and_memory.py b/tests/nodes/analyzers/test_security_scan_runtime_and_memory.py new file mode 100644 index 000000000..bf5084b5e --- /dev/null +++ b/tests/nodes/analyzers/test_security_scan_runtime_and_memory.py @@ -0,0 +1,117 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Cooperative cancellation and cache-bound tests for the security scan. + +Both properties regressed when the scan stopped walking character by character: +seeking jumps over the periodic checkpoints a walk would hit, and memoizing a +derived view retains far more than the entry count suggests because +normalization can expand its input several-fold. These tests pin both. +""" + +from __future__ import annotations + +import pytest + +from skillspector.artifacts import ( + _DERIVED_VIEW_CACHE_BUDGET_CHARS, + _NORMALIZED_VIEW_CACHE, + _RUNTIME_CHECKPOINT_STRIDE, + _token_bridging_gap_spans, + clear_security_text_caches, + normalized_security_view, +) + + +class _AbortError(Exception): + pass + + +@pytest.fixture(autouse=True) +def _clear(): + clear_security_text_caches() + yield + clear_security_text_caches() + + +@pytest.mark.parametrize("length", [0, 4095, 4096, 10_000, 32_768, 100_000]) +def test_checkpoint_cadence_matches_a_character_walk(length: int) -> None: + """Seeking must fire the checks a character-by-character walk would fire.""" + fired = 0 + + def check() -> None: + nonlocal fired + fired += 1 + + list(_token_bridging_gap_spans("a" * length, check_runtime=check)) + assert fired == length // _RUNTIME_CHECKPOINT_STRIDE + + +def test_cancellation_is_observed_across_sparse_gaps() -> None: + """In-word gaps that yield no spans must still reach the runtime check.""" + calls = 0 + + def check() -> None: + nonlocal calls + calls += 1 + if calls >= 2: + raise _AbortError + + with pytest.raises(_AbortError): + list(_token_bridging_gap_spans("aᅟa " * 8192, check_runtime=check)) + + +def test_cancellation_is_observed_on_text_with_no_candidates() -> None: + """The whole-string skip must not swallow cancellation either.""" + calls = 0 + + def check() -> None: + nonlocal calls + calls += 1 + if calls >= 2: + raise _AbortError + + with pytest.raises(_AbortError): + list(_token_bridging_gap_spans("a" * 100_000, check_runtime=check)) + + +def test_derived_view_cache_is_bounded_by_size_not_entry_count() -> None: + """NFKC expands U+FDFA to 18 characters, so entry count bounds nothing.""" + chunk = "ﷺ" * 4000 + for index in range(40): + normalized_security_view(f"{index} {chunk}") + assert _NORMALIZED_VIEW_CACHE.stored_chars <= _DERIVED_VIEW_CACHE_BUDGET_CHARS + + +def test_a_single_oversized_view_is_not_retained() -> None: + oversized = "ﷺ" * (_DERIVED_VIEW_CACHE_BUDGET_CHARS // 10) + normalized_security_view(oversized) + assert _NORMALIZED_VIEW_CACHE.stored_chars == 0 + + +def test_clearing_releases_everything() -> None: + normalized_security_view("ﷺ" * 1000) + assert _NORMALIZED_VIEW_CACHE.stored_chars > 0 + clear_security_text_caches() + assert _NORMALIZED_VIEW_CACHE.stored_chars == 0 + + +def test_scan_teardown_clears_the_caches() -> None: + """cleanup_result is the hook that releases scan-local state.""" + from skillspector.cleanup import cleanup_result + + normalized_security_view("ﷺ" * 1000) + assert _NORMALIZED_VIEW_CACHE.stored_chars > 0 + cleanup_result({}) + assert _NORMALIZED_VIEW_CACHE.stored_chars == 0 + + +def test_eviction_does_not_change_results() -> None: + """A view evicted and rebuilt must be identical to the first one.""" + text = "ﷺ" * 500 + first = normalized_security_view(text) + for index in range(60): + normalized_security_view(f"{index} " + "ﷺ" * 4000) + rebuilt = normalized_security_view(text) + assert rebuilt.text == first.text + assert list(rebuilt.source_offsets or []) == list(first.source_offsets or [])