diff --git a/packages/sie_server/src/sie_server/adapters/gliner2/adapter.py b/packages/sie_server/src/sie_server/adapters/gliner2/adapter.py index 1cfd91e9c..7c1f10dcc 100644 --- a/packages/sie_server/src/sie_server/adapters/gliner2/adapter.py +++ b/packages/sie_server/src/sie_server/adapters/gliner2/adapter.py @@ -26,7 +26,7 @@ subword_budget, ) from sie_server.adapters.gliner2.words import linear_equivalent -from sie_server.core.inference_output import ExtractOutput +from sie_server.core.inference_output import ExtractItemError, ExtractOutput from sie_server.types.inputs import InvalidInputError, Item from sie_server.types.responses import Classification, Entity, Relation @@ -280,7 +280,6 @@ def extract( # gliner2 reads a bounded window of words: pass it only the prefix holding them. reads = [self._read(text) for text in texts] windows = [(model_text, subwords) for model_text, subwords, _ in reads] - model_texts = [model_text for model_text, _ in windows] opts = options or {} effective_threshold = self._validate_threshold(opts.get("threshold", self._threshold)) classification_task = opts.get("classification_task", self._classification_task) @@ -305,8 +304,9 @@ def extract( prompt = self._prompt_tokens(specs, key=("json", tuple(specs)), extra=len(choices)) rows = self._row_tokens(windows, prompt) with torch.inference_mode(): - raw_results = self._run_planned( - model_texts, + raw_results, errors, input_token_counts = self._run_complete( + texts, + reads, rows, lambda batch: self._model.batch_extract_json( batch, @@ -317,11 +317,18 @@ def extract( include_spans=False, max_len=self._max_seq_length, ), + input_token_counts=input_token_counts, rows_per_pass=len(texts), ) return ExtractOutput( entities=[[] for _ in texts], - data=[self._flatten_structured_result(result, output_schema=output_schema) for result in raw_results], + data=[ + {} + if errors is not None and errors[index] is not None + else self._flatten_structured_result(result, output_schema=output_schema) + for index, result in enumerate(raw_results) + ], + errors=errors, input_token_counts=input_token_counts, ) @@ -342,8 +349,9 @@ def extract( ) rows = self._row_tokens(windows, prompt) with torch.inference_mode(): - raw_results = self._run_planned( - model_texts, + raw_results, errors, input_token_counts = self._run_complete( + texts, + reads, rows, lambda batch: self._model.batch_extract_relations( batch, @@ -354,10 +362,15 @@ def extract( include_spans=True, max_len=self._max_seq_length, ), + input_token_counts=input_token_counts, rows_per_pass=len(texts), ) return ExtractOutput( - entities=normalized_entities, + entities=[ + [] if errors is not None and errors[index] is not None else entities + for index, entities in enumerate(normalized_entities) + ], + errors=errors, relations=[ self._flatten_relations(result, entities=entities) for result, entities in zip(raw_results, normalized_entities) @@ -413,15 +426,72 @@ def extract_entities(batch: list[str]) -> list[Any]: check_label_chars("GLiNER2", "labels", normalized_labels) prompt = self._prompt_tokens(normalized_labels, key=("entities", tuple(normalized_labels))) with torch.inference_mode(): - raw_results = self._run_planned( - model_texts, + raw_results, errors, input_token_counts = self._run_complete( + texts, + reads, self._row_tokens(windows, prompt), extract_entities, + input_token_counts=input_token_counts, rows_per_pass=1 if len(texts) == 1 else _PACKAGE_BATCH_SIZE, ) all_entities = [self._flatten_entities(result, text=text) for text, result in zip(texts, raw_results)] - return ExtractOutput(entities=all_entities, input_token_counts=input_token_counts) + return ExtractOutput(entities=all_entities, errors=errors, input_token_counts=input_token_counts) + + def _run_complete( + self, + texts: list[str], + reads: list[tuple[str, int | None, Window | None]], + rows: list[int] | None, + run: Callable[[list[str]], list[Any]], + *, + input_token_counts: list[int] | None, + rows_per_pass: int, + ) -> tuple[list[Any], list[ExtractItemError | None] | None, list[int] | None]: + """Run extraction only for items whose entire text fits the model's word/subword window. + + Classification has its own whole-document windowing. Entity, relation, + and structured extraction must fail an overlong item rather than return + a successful partial prediction. Inspect the last piece actually read: + a long word's ``cut`` can be beyond that piece, and lowercase offsets + need not index the original text. The package's synthetic sentence end + and trailing whitespace are not unread user content. + """ + errors: list[ExtractItemError | None] = [] + for text, (_, _, window) in zip(texts, reads, strict=True): + source = text.lower() if self._lower_text_first else text + incomplete = ( + window is not None + and window.cut is not None + and bool(window.words) + and _NON_SPACE.search(source, window.words[-1][2]) is not None + ) + errors.append( + ExtractItemError( + code="INPUT_TOO_LONG", + message="GLiNER2 extraction requires the entire text to fit one word/subword window. " + "Split the text into smaller items.", + ) + if incomplete + else None + ) + accepted = [index for index, error in enumerate(errors) if error is None] + results: list[Any] = [{} for _ in texts] + if accepted: + raw = self._run_planned( + [reads[index][0] for index in accepted], + [rows[index] for index in accepted] if rows is not None else None, + run, + rows_per_pass=rows_per_pass, + ) + for index, result in zip(accepted, raw, strict=True): + results[index] = result + counts = ( + [count if error is None else 0 for count, error in zip(input_token_counts, errors, strict=True)] + if input_token_counts is not None + else ([0] * len(texts) if not accepted else None) + ) + return results, errors if len(accepted) != len(texts) else None, counts def _classify( self, diff --git a/packages/sie_server/src/sie_server/queue_executor.py b/packages/sie_server/src/sie_server/queue_executor.py index faf4e299b..b16c90842 100644 --- a/packages/sie_server/src/sie_server/queue_executor.py +++ b/packages/sie_server/src/sie_server/queue_executor.py @@ -1900,6 +1900,13 @@ def _with_images(units: UnitCounts | None, image_count: int | None) -> UnitCount ) +def _with_input_tokens(units: UnitCounts | None, token_count: int) -> UnitCounts: + """Set an authoritative input-token count while preserving other units.""" + if units is None: + return UnitCounts(input_tokens=token_count) + return msgspec.structs.replace(units, input_tokens=token_count) + + def _with_pages(units: UnitCounts | None, page_count: int | None) -> UnitCounts | None: """Fold an authoritative page count into an existing ``UnitCounts`` โ€” the ยง7 canonical parse/OCR dimension ("$ per 1k pages") โ€” minting one when only @@ -2195,6 +2202,11 @@ def _extract_success_outcome( # Adapter returned no results for a single-item request โ€” surface an # error instead of publishing an object the client reads as success. return _error_outcome(bi, _INFERENCE_ERROR_CODE, "adapter returned no extraction results") + error = extraction_results[0].get("error") + if isinstance(error, dict) and error.get("code") == ErrorCode.INPUT_TOO_LONG.value: + # Length-rejected items have zero billable input tokens, even when an + # adapter reports a count or cannot meter the valid siblings. + units = _with_input_tokens(units, 0) item_id = server_item.id if server_item.id is not None else f"item-{bi.item_index}" result_msgpack = pack_msgpack({**extraction_results[0], "id": item_id}, use_bin_type=True) diff --git a/packages/sie_server/tests/adapters/test_gliner2_complete_text.py b/packages/sie_server/tests/adapters/test_gliner2_complete_text.py new file mode 100644 index 000000000..81d445561 --- /dev/null +++ b/packages/sie_server/tests/adapters/test_gliner2_complete_text.py @@ -0,0 +1,279 @@ +from __future__ import annotations + +import math +from typing import Any +from unittest.mock import MagicMock + +import pytest +from sie_server.adapters._word_window import WindowedSplitter +from sie_server.adapters.gliner2.adapter import GLiNER2Adapter +from sie_server.adapters.gliner2.words import LinearWordSplitter +from sie_server.core.worker.handlers.extract import ExtractHandler +from sie_server.types.inputs import InvalidInputError, Item + + +class _Tokenizer: + def tokenize(self, text: str) -> list[str]: + return [text] * max(1, math.ceil(len(text) / 3)) + + def __call__(self, texts: list[str], *, max_length: int | None, **_: Any) -> dict[str, Any]: + return {"input_ids": [list(range(min(len(text.split()) + 2, max_length or 512))) for text in texts]} + + +def _adapter(*, lower: bool = True, max_words: int = 8, max_subwords: int = 1000) -> tuple[GLiNER2Adapter, MagicMock]: + adapter = GLiNER2Adapter("test", max_seq_length=max_words) + model = MagicMock() + model.processor.tokenizer = _Tokenizer() + adapter._model = model + adapter._lower_text_first = lower + adapter._word_splitter = WindowedSplitter( + LinearWordSplitter(lower_text_first=lower), + max_words=max_words, + max_subwords=max_subwords, + count_subwords=lambda words: [len(model.processor.tokenizer.tokenize(word)) for word in words], + ) + model.extract_entities.side_effect = lambda text, *_a, **_k: { + "entities": {"person": [{"text": text.split()[0], "start": 0, "end": len(text.split()[0]), "confidence": 0.9}]} + } + model.batch_extract_entities.side_effect = lambda texts, *_a, **_k: [model.extract_entities(text) for text in texts] + model.batch_extract_json.side_effect = lambda texts, *_a, **_k: [ + {"_sie_root": [{"name": text.split()[0]}]} for text in texts + ] + model.batch_extract_relations.side_effect = lambda texts, *_a, **_k: [ + { + "relation_extraction": { + "knows": [ + {"head": {"text": text.split()[0], "confidence": 0.9}, "tail": {"text": "Acme", "confidence": 0.8}} + ] + } + } + for text in texts + ] + return adapter, model + + +def _request(task: str, texts: list[str]) -> tuple[list[Item], dict[str, Any]]: + items = [Item(text=text) for text in texts] + kwargs: dict[str, Any] = {"labels": ["person"]} + if task == "structured": + kwargs = {"output_schema": {"type": "object", "properties": {"name": {"type": "string"}}, "required": ["name"]}} + elif task == "relations": + kwargs = {"labels": ["knows"]} + items = [ + Item( + text=text, + metadata={ + "entities": [ + {"text": text.split()[0], "start": 0, "end": len(text.split()[0])}, + {"text": "Acme", "start": text.index("Acme"), "end": text.index("Acme") + 4}, + ] + }, + ) + for text in texts + ] + return items, kwargs + + +@pytest.mark.parametrize("task", ["entities", "relations", "structured"]) +def test_overlong_items_fail_without_losing_short_items_or_their_positions(task: str) -> None: + adapter, model = _adapter() + long = "Alice Acme " + "word " * 20 + items, kwargs = _request(task, ["Alice Acme", long, "Bob Acme", long]) + output = adapter.extract(items, **kwargs) + + assert output.errors is not None + assert [error.code if error else None for error in output.errors] == [ + None, + "INPUT_TOO_LONG", + None, + "INPUT_TOO_LONG", + ] + assert output.input_token_counts == [4, 0, 4, 0] + assert output.entities[1] == output.entities[3] == [] + if task == "entities": + assert [output.entities[i][0]["text"] for i in [0, 2]] == ["Alice", "Bob"] + method = model.batch_extract_entities + elif task == "structured": + assert output.data == [{"name": "Alice"}, {}, {"name": "Bob"}, {}] + method = model.batch_extract_json + else: + assert output.relations is not None + assert [relations[0]["head"] if relations else None for relations in output.relations] == [ + "Alice", + None, + "Bob", + None, + ] + method = model.batch_extract_relations + assert method.call_args.args[0] == ["Alice Acme", "Bob Acme"] + + handler = ExtractHandler() + assembled = handler.assemble_output({i: handler.slice_output(output, i) for i in range(4)}, batch_size=4) + assert assembled.input_token_counts == [4, 0, 4, 0] + formatted = handler.format_output(assembled) + assert "error" not in formatted[0] + assert "error" not in formatted[2] + assert formatted[1]["error"]["code"] == formatted[3]["error"]["code"] == "INPUT_TOO_LONG" + + +@pytest.mark.parametrize("task", ["entities", "relations", "structured"]) +def test_an_all_overlong_batch_never_calls_the_model(task: str) -> None: + adapter, model = _adapter() + items, kwargs = _request(task, ["Alice Acme " + "word " * 20] * 2) + output = adapter.extract(items, **kwargs) + assert output.errors is not None + assert all(error and error.code == "INPUT_TOO_LONG" for error in output.errors) + assert output.input_token_counts == [0, 0] + assert not model.extract_entities.called + assert not model.batch_extract_entities.called + assert not model.batch_extract_relations.called + assert not model.batch_extract_json.called + + +@pytest.mark.parametrize("lower", [False, True]) +@pytest.mark.parametrize( + ("text", "max_words", "max_subwords", "too_long"), + [ + ("one two three", 3, 1000, False), # only the package's synthetic dot is unread + ("one two three \t\n", 3, 1000, False), + ("one two three.", 3, 1000, True), # the caller's punctuation is real input + ("one two three four", 3, 1000, True), + ("\u0130stanbul Ankara Izmir", 2, 1000, True), # lowercase expands offsets + ("\u039f\u0394\u039f\u03a3'\u0391 next", 1, 1000, True), + ("x" * 513, 2, 1000, True), # cut is the original word end, beyond the last read piece + ("abc " * 7, 8, 3, True), # fewer than max_words, but the subword budget is full + ], +) +def test_completeness_uses_the_actual_word_window( + text: str, max_words: int, max_subwords: int, too_long: bool, lower: bool +) -> None: + adapter, model = _adapter(lower=lower, max_words=max_words, max_subwords=max_subwords) + model.extract_entities.side_effect = None + model.extract_entities.return_value = {"entities": {}} + output = adapter.extract([Item(text=text)], labels=["person"]) + if too_long: + assert output.errors is not None + assert output.errors[0] is not None + assert output.errors[0].code == "INPUT_TOO_LONG" + assert output.input_token_counts == [0] + model.extract_entities.assert_not_called() + else: + assert output.errors is None + model.extract_entities.assert_called_once() + + +def test_invalid_request_parameters_are_still_validated_before_item_length_errors() -> None: + adapter, model = _adapter() + with pytest.raises(InvalidInputError, match="requires labels"): + adapter.extract([Item(text="word " * 20)]) + model.extract_entities.assert_not_called() + + +@pytest.mark.parametrize("index", [0, 1]) +def test_queue_outcome_preserves_length_error_and_zero_billing(index: int) -> None: + from types import SimpleNamespace + + import msgpack + from sie_server.ipc_types import ExtractBatchItem + from sie_server.queue_executor import _extract_success_outcome + + adapter, _ = _adapter() + items = [Item(text="word " * 20), Item(text="Alice Acme")] + output = adapter.extract(items, labels=["person"]) + worker_result = SimpleNamespace( + output=ExtractHandler().slice_output(output, index), + timing=SimpleNamespace(inference_ms=0, tokenization_ms=0, postprocessing_ms=0), + ) + batch_item = ExtractBatchItem( + work_item_id=f"req.{index}", + request_id="req", + item_index=index, + total_items=2, + timestamp=0, + item={"text": items[index].text}, + labels=["person"], + ) + outcome = _extract_success_outcome(adapter, batch_item, items[index], worker_result) + result = msgpack.unpackb(outcome.result_msgpack, raw=False) + assert outcome.units is not None + if index == 0: + assert result["error"]["code"] == "INPUT_TOO_LONG" + assert result["entities"] == [] + assert outcome.units.input_tokens == 0 + else: + assert "error" not in result + assert result["entities"][0]["text"] == "Alice" + assert outcome.units.input_tokens == 4 + + +def test_queue_length_error_keeps_zero_billing_when_sibling_metering_fails() -> None: + from types import SimpleNamespace + + import msgpack + from sie_server.ipc_types import ExtractBatchItem + from sie_server.queue_executor import _extract_success_outcome + + adapter, _ = _adapter() + adapter._doc_input_token_counts = lambda _texts: None + items = [Item(text="word " * 20), Item(text="Alice Acme")] + output = adapter.extract(items, labels=["person"]) + assert output.input_token_counts is None + assert output.errors is not None + assert output.errors[0] is not None + + batch_item = ExtractBatchItem( + work_item_id="req.0", + request_id="req", + item_index=0, + total_items=2, + timestamp=0, + item={"text": items[0].text}, + labels=["person"], + ) + worker_result = SimpleNamespace( + output=ExtractHandler().slice_output(output, 0), + timing=SimpleNamespace(inference_ms=0, tokenization_ms=0, postprocessing_ms=0), + ) + outcome = _extract_success_outcome(adapter, batch_item, items[0], worker_result) + result = msgpack.unpackb(outcome.result_msgpack, raw=False) + + assert result["error"]["code"] == "INPUT_TOO_LONG" + assert outcome.units is not None + assert outcome.units.input_tokens == 0 + + +@pytest.mark.parametrize(("error_code", "expected_input_tokens"), [("INPUT_TOO_LONG", 0), ("INFERENCE_ERROR", 7)]) +def test_queue_length_error_overrides_reported_positive_billing(error_code: str, expected_input_tokens: int) -> None: + from types import SimpleNamespace + + import msgpack + from sie_server.ipc_types import ExtractBatchItem + from sie_server.queue_executor import _extract_success_outcome + + adapter, _ = _adapter() + items = [Item(text="word " * 20), Item(text="Alice Acme")] + output = adapter.extract(items, labels=["person"]) + output.input_token_counts = [7, 4] + assert output.errors is not None + assert output.errors[0] is not None + output.errors[0].code = error_code + + batch_item = ExtractBatchItem( + work_item_id="req.0", + request_id="req", + item_index=0, + total_items=2, + timestamp=0, + item={"text": items[0].text}, + labels=["person"], + ) + worker_result = SimpleNamespace( + output=ExtractHandler().slice_output(output, 0), + timing=SimpleNamespace(inference_ms=0, tokenization_ms=0, postprocessing_ms=0), + ) + outcome = _extract_success_outcome(adapter, batch_item, items[0], worker_result) + result = msgpack.unpackb(outcome.result_msgpack, raw=False) + + assert result["error"]["code"] == error_code + assert outcome.units is not None + assert outcome.units.input_tokens == expected_input_tokens diff --git a/packages/sie_server/tests/adapters/test_gliner2_long_text.py b/packages/sie_server/tests/adapters/test_gliner2_long_text.py index d467c455e..2c1bf8dd6 100644 --- a/packages/sie_server/tests/adapters/test_gliner2_long_text.py +++ b/packages/sie_server/tests/adapters/test_gliner2_long_text.py @@ -306,7 +306,7 @@ def test_long_words_keep_the_encoder_input_within_the_subword_budget(cls: type[G entities = adapter.extract( [Item(text=text), Item(text="Priya Raman works at Novartis")], labels=["person"], - options={"classification_task": None}, # entity extraction, which reads the first window + options={"classification_task": None}, # entity extraction rejects an incomplete item ) extract_seconds = time.perf_counter() - started @@ -316,26 +316,32 @@ def test_long_words_keep_the_encoder_input_within_the_subword_budget(cls: type[G for batch in model.inputs for words, starts, ends in zip(batch.text_tokens, batch.start_mappings, batch.end_mappings, strict=True) ] - assert len(rows) == 2 + assert len(rows) == 1 for words, starts, ends in rows: assert sum(len(tokenize(word)) for word in words) <= budget assert all(end - start <= MAX_WORD_CHARS for start, end in zip(starts, ends, strict=True)) assert all(batch.input_ids.shape[1] <= budget + 64 for batch in model.inputs) # the task prompt and specials assert ["priya", "raman", "works", "at", "novartis", "."] in [words for words, _, _ in rows] - # Billing is unchanged: the document tokens up to max_seq_length. + # The rejected item is not inferred or billed; the short item is still metered. assert entities.input_token_counts is not None - assert entities.input_token_counts[0] == 512 + assert entities.input_token_counts[0] == 0 + assert entities.input_token_counts[1] > 0 + assert entities.errors is not None + assert entities.errors[0] is not None + assert entities.errors[0].code == "INPUT_TOO_LONG" + assert entities.errors[1] is None assert classify_seconds < 1.0 assert extract_seconds < 1.0 def test_a_batch_of_long_rows_runs_in_passes_within_the_attention_budget() -> None: adapter, model = make_adapter(encoder_config={"model_type": "deberta-v2"}) - texts = [("w" * 200 + " ") * 400 for _ in range(6)] + ["Priya Raman works at Novartis"] * 6 + texts = [("w" * 12 + " ") * 400 for _ in range(6)] + ["Priya Raman works at Novartis"] * 6 output = adapter.extract([Item(text=text) for text in texts], labels=["person"]) assert len(output.entities) == len(texts) + assert output.errors is None assert len(model.inputs) > 1 for batch in model.inputs: rows, width = batch.input_ids.shape @@ -346,7 +352,7 @@ def test_a_batch_of_long_rows_runs_in_passes_within_the_attention_budget() -> No def test_an_ordinary_batch_runs_in_one_pass() -> None: adapter, model = make_adapter(encoder_config={"model_type": "deberta-v2"}) - texts = [" ".join(f"Sentence {i} mentions Dr. Priya Raman of Novartis." for i in range(60))] * 8 + texts = [" ".join(f"Sentence {i} mentions Dr. Priya Raman of Novartis." for i in range(50))] * 8 adapter.extract([Item(text=text) for text in texts], labels=["person"]) @@ -466,13 +472,18 @@ def test_pathological_documents_run_in_bounded_time(cls: type[GLiNER2Adapter], t entities = adapter.extract( [Item(text=text), Item(text="Priya Raman works at Novartis")], labels=["person"], - options={"classification_task": None}, # entity extraction, which reads the first window + options={"classification_task": None}, # entity extraction rejects an incomplete item ) extract_seconds = time.perf_counter() - started assert entities.input_token_counts is not None - assert entities.input_token_counts[0] == 512 - assert len(model.inputs[-1].text_tokens[0]) == 512 + assert entities.input_token_counts[0] == 0 + assert entities.input_token_counts[1] > 0 + assert entities.errors is not None + assert entities.errors[0] is not None + assert entities.errors[0].code == "INPUT_TOO_LONG" + assert entities.errors[1] is None + assert model.inputs[-1].text_tokens == [["priya", "raman", "works", "at", "novartis", "."]] # gliner2's own splitter takes minutes to hours on these texts. assert classify_seconds < 1.0 assert extract_seconds < 1.0 @@ -781,3 +792,24 @@ def test_positive_label_is_validated() -> None: assert adapter._effective_positive_label({"positive_label": "block"}, ["allow", "block"]) == "block" with pytest.raises(ValueError, match="positive_label must be one of the labels"): adapter._effective_positive_label({"positive_label": "unsafe"}, ["allow", "block"]) + + +@pytest.mark.parametrize("task", ["entities", "relations", "structured"]) +def test_long_item_rejection_uses_the_pinned_package_processor(task: str) -> None: + adapter, model = make_adapter() + text = " ".join(["word"] * 512 + ["Alice"]) + kwargs: dict[str, Any] = {"labels": ["person"]} + item = Item(text=text) + if task == "structured": + kwargs = {"output_schema": {"type": "object", "properties": {"name": {"type": "string"}}}} + elif task == "relations": + kwargs = {"labels": ["knows"]} + item = Item(text=text, metadata={"entities": [{"text": "Alice", "start": len(text) - 5, "end": len(text)}]}) + + output = adapter.extract([item], **kwargs) + + assert output.errors is not None + assert output.errors[0] is not None + assert output.errors[0].code == "INPUT_TOO_LONG" + assert output.input_token_counts == [0] + assert not model.inputs diff --git a/packages/sie_server/tests/api/test_extract.py b/packages/sie_server/tests/api/test_extract.py index 75513cd74..d03ab8fc3 100644 --- a/packages/sie_server/tests/api/test_extract.py +++ b/packages/sie_server/tests/api/test_extract.py @@ -207,6 +207,30 @@ def test_extract_rejects_a_non_string_options_instruction( assert response.status_code == 400 mock_adapter.extract.assert_not_called() + def test_overlong_item_error_and_zero_tokens_are_preserved_in_a_mixed_response( + self, client: TestClient, mock_adapter: MagicMock + ) -> None: + mock_adapter.extract.side_effect = None + mock_adapter.extract.return_value = ExtractOutput( + entities=[[], [Entity(text="Alice", label="person", score=0.9, start=0, end=5)]], + errors=[ExtractItemError(code="INPUT_TOO_LONG", message="Split the text into smaller items."), None], + input_token_counts=[0, 5], + ) + + response = client.post( + "/v1/extract/test-extractor", + json={"items": [{"text": "word " * 513}, {"text": "Alice"}], "params": {"labels": ["person"]}}, + headers=JSON_HEADERS, + ) + + assert response.status_code == 200 + data = response.json() + assert data["items"][0]["error"]["code"] == "INPUT_TOO_LONG" + assert data["items"][0]["entities"] == [] + assert "error" not in data["items"][1] + assert data["items"][1]["entities"][0]["text"] == "Alice" + assert data["usage"]["input_tokens"] == 5 + def test_extract_item_error_is_preserved_in_local_response( self, client: TestClient,