Skip to content

Commit 0f51f99

Browse files
authored
fix(pii): bound NER memory per worker so large texts stop OOM-killing Presidio workers (#10068)
1 parent eaed9a1 commit 0f51f99

2 files changed

Lines changed: 56 additions & 46 deletions

File tree

‎apps/pii/server.py‎

Lines changed: 51 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,10 @@
88

99
import json
1010
import logging
11+
import threading
1112
import time
1213
from collections.abc import AsyncIterator
13-
from contextlib import asynccontextmanager
14+
from contextlib import asynccontextmanager, nullcontext
1415
from typing import Any
1516

1617
import regex as regex_module
@@ -184,7 +185,6 @@ def build_analyzer() -> AnalyzerEngine:
184185
anonymizer: AnonymizerEngine
185186
NER_ENTITIES: frozenset
186187
_BLANK_ARTIFACTS: dict
187-
_NLP_MAX_LENGTH: dict
188188

189189
# Every entity the spaCy NER recognizers can actually produce. A request touching
190190
# any of these must run spaCy; a request naming only non-NER (regex/checksum)
@@ -228,19 +228,22 @@ def _regex_only(entities: list[str] | None, score_threshold: float | None) -> bo
228228
)
229229

230230

231-
# spaCy refuses (error E088) any text longer than its model's `max_length`, so a
232-
# longer text runs NER in windows of this many characters, following spaCy's
233-
# guidance to segment long documents (it sizes NER memory at roughly 1GB per
234-
# 100,000 characters of one text).
231+
# A text longer than this runs NER in windows of this many characters, following
232+
# spaCy's guidance to segment long documents. One NER pass costs memory that grows
233+
# with the text's length (about 3GB for 900,000 characters), enough to get a worker
234+
# OOM-killed, so no pass sees more than one window. That also keeps every pass under
235+
# spaCy's `max_length`, which it refuses with error E088.
235236
NER_WINDOW_CHARS = 100_000
236237
# How much of the previous window each window repeats. Far longer than any entity
237238
# spaCy tags, so an entity cut by one window's edge lies whole inside the next.
238239
NER_WINDOW_OVERLAP = 1_000
239240

240-
241-
def _exceeds_nlp_length(text: str, language: str) -> bool:
242-
limit = _NLP_MAX_LENGTH.get(language)
243-
return limit is not None and len(text) > limit
241+
# One NER pass (a batch, a short text, or one window) at a time per worker. FastAPI
242+
# runs these sync endpoints on a threadpool, so without it a worker runs one pass per
243+
# request in flight and their memory adds up until it is OOM-killed. A long text
244+
# takes it per window, so it interleaves with other requests. Regex-only requests
245+
# never take it.
246+
_NER_SLOT = threading.Lock()
244247

245248

246249
def _window_entities(
@@ -265,7 +268,7 @@ def _analyze_long(
265268
return_decision_process: bool = False,
266269
ad_hoc_recognizers: list[PatternRecognizer] | None = None,
267270
) -> list[RecognizerResult]:
268-
"""Analyze a text longer than spaCy's `max_length`.
271+
"""Analyze a text longer than NER_WINDOW_CHARS.
269272
270273
Pattern, checksum, and custom recognizers have no length limit, so they run
271274
once over the whole text (against blank NLP artifacts, as the regex-only path
@@ -291,13 +294,15 @@ def _analyze_long(
291294
return results
292295
step = NER_WINDOW_CHARS - NER_WINDOW_OVERLAP
293296
for start in range(0, len(text), step):
294-
for result in analyzer.analyze(
295-
text=text[start : start + NER_WINDOW_CHARS],
296-
language=language,
297-
entities=window_entities,
298-
score_threshold=score_threshold,
299-
return_decision_process=return_decision_process,
300-
):
297+
with _NER_SLOT:
298+
window_results = analyzer.analyze(
299+
text=text[start : start + NER_WINDOW_CHARS],
300+
language=language,
301+
entities=window_entities,
302+
score_threshold=score_threshold,
303+
return_decision_process=return_decision_process,
304+
)
305+
for result in window_results:
301306
results.append(
302307
RecognizerResult(
303308
result.entity_type,
@@ -327,23 +332,24 @@ def _analyze_one(
327332
ad_hoc_recognizers: list[PatternRecognizer] | None = None,
328333
):
329334
regex_only = _regex_only(entities, score_threshold)
330-
if not regex_only and _exceeds_nlp_length(text, language):
335+
if not regex_only and len(text) > NER_WINDOW_CHARS:
331336
return _analyze_long(
332337
text, language, entities, score_threshold, return_decision_process, ad_hoc_recognizers
333338
)
334339
# Regex-only requests reuse a blank NlpArtifacts to skip the spaCy NLP pass;
335340
# otherwise analyze() computes artifacts (runs spaCy) as usual. Custom-pattern
336341
# recognizers are regex-based, so they run fine against the blank artifacts.
337342
nlp_artifacts = _BLANK_ARTIFACTS.get(language) if regex_only else None
338-
return analyzer.analyze(
339-
text=text,
340-
language=language,
341-
entities=entities or None,
342-
score_threshold=score_threshold,
343-
return_decision_process=return_decision_process,
344-
nlp_artifacts=nlp_artifacts,
345-
ad_hoc_recognizers=ad_hoc_recognizers or None,
346-
)
343+
with nullcontext() if regex_only else _NER_SLOT:
344+
return analyzer.analyze(
345+
text=text,
346+
language=language,
347+
entities=entities or None,
348+
score_threshold=score_threshold,
349+
return_decision_process=return_decision_process,
350+
nlp_artifacts=nlp_artifacts,
351+
ad_hoc_recognizers=ad_hoc_recognizers or None,
352+
)
347353

348354

349355
# Presidio's BatchAnalyzerEngine defaults nlp.pipe to batch_size=1, so a 2000-text
@@ -359,7 +365,7 @@ def _analyze_many(
359365
ad_hoc_recognizers: list[PatternRecognizer] | None = None,
360366
):
361367
"""Analyze many texts, skipping the spaCy pass for regex-only requests and
362-
analyzing any text longer than spaCy's `max_length` on its own (see
368+
analyzing any text longer than NER_WINDOW_CHARS on its own (see
363369
`_analyze_long`)."""
364370
if _regex_only(entities, score_threshold):
365371
blank = _BLANK_ARTIFACTS.get(language)
@@ -374,19 +380,20 @@ def _analyze_many(
374380
)
375381
for text in texts
376382
]
377-
long_indices = {i for i, text in enumerate(texts) if _exceeds_nlp_length(text, language)}
383+
long_indices = {i for i, text in enumerate(texts) if len(text) > NER_WINDOW_CHARS}
378384
short_indices = [i for i in range(len(texts)) if i not in long_indices]
379385
results: list[list[RecognizerResult]] = [[] for _ in texts]
380-
short_results = batch_analyzer.analyze_iterator(
381-
texts=[texts[i] for i in short_indices],
382-
language=language,
383-
batch_size=NLP_BATCH_SIZE,
384-
entities=entities or None,
385-
score_threshold=score_threshold,
386-
ad_hoc_recognizers=ad_hoc_recognizers or None,
387-
)
388-
for i, per_text in zip(short_indices, short_results):
389-
results[i] = list(per_text)
386+
with _NER_SLOT:
387+
short_results = batch_analyzer.analyze_iterator(
388+
texts=[texts[i] for i in short_indices],
389+
language=language,
390+
batch_size=NLP_BATCH_SIZE,
391+
entities=entities or None,
392+
score_threshold=score_threshold,
393+
ad_hoc_recognizers=ad_hoc_recognizers or None,
394+
)
395+
for i, per_text in zip(short_indices, short_results):
396+
results[i] = list(per_text)
390397
for i in sorted(long_indices):
391398
results[i] = _analyze_long(
392399
texts[i], language, entities, score_threshold, ad_hoc_recognizers=ad_hoc_recognizers
@@ -396,7 +403,7 @@ def _analyze_many(
396403

397404
@asynccontextmanager
398405
async def lifespan(_app: FastAPI) -> AsyncIterator[None]:
399-
global analyzer, batch_analyzer, anonymizer, NER_ENTITIES, _BLANK_ARTIFACTS, _NLP_MAX_LENGTH
406+
global analyzer, batch_analyzer, anonymizer, NER_ENTITIES, _BLANK_ARTIFACTS
400407
logger.info("building analyzer (spacy)")
401408
analyzer = build_analyzer()
402409
batch_analyzer = BatchAnalyzerEngine(analyzer_engine=analyzer)
@@ -423,10 +430,6 @@ async def lifespan(_app: FastAPI) -> AsyncIterator[None]:
423430
language: analyzer.nlp_engine.process_text("", language)
424431
for language in SUPPORTED_LANGUAGES
425432
}
426-
_NLP_MAX_LENGTH = {
427-
language: nlp.max_length
428-
for language, nlp in getattr(analyzer.nlp_engine, "nlp", {}).items()
429-
}
430433
yield
431434

432435

@@ -686,8 +689,10 @@ def run_anonymize(
686689
)
687690

688691

692+
# Async so it answers on the event loop: requests queued on _NER_SLOT hold
693+
# threadpool threads, and the load balancer's health check must not wait behind them.
689694
@app.get("/health")
690-
def health() -> dict[str, str]:
695+
async def health() -> dict[str, str]:
691696
return {"status": "ok"}
692697

693698

‎docker/pii.Dockerfile‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,11 @@ USER pii
6161
# 5001 avoids colliding with the app's 3000 in local/compose runs on one host.
6262
EXPOSE 5001
6363

64+
# glibc gives each threadpool thread its own malloc arena and rarely returns their
65+
# memory, so a worker's footprint ratchets up to one NER pass's peak per thread
66+
# that ran one. Two arenas keep it near a single pass.
67+
ENV MALLOC_ARENA_MAX=2
68+
6469
# Per-pattern regex match timeout (Presidio's `regex`-module `finditer(timeout=...)`),
6570
# an interactive backstop against a catastrophic user-supplied custom regex. Presidio
6671
# logs and skips a pattern that exceeds it — the request stays up rather than hanging.

0 commit comments

Comments
 (0)