From 4d0bbfdc581596803c725e6993be2cc33f8b32f3 Mon Sep 17 00:00:00 2001 From: Shai Shmuel Date: Thu, 5 Nov 2026 10:54:45 +0200 Subject: [PATCH] fix(server): report SGLang window refusals as client errors A request too long for a model's sequence window came back as HTTP 500 on /v1/extract, /v1/encode and /v1/generate. That reads as a server failure, so clients may retry a request that can never succeed. The engine reports these refusals as plain text in an event, a finish reason or a 400 body, depending on where it noticed. Recognise its exact texts and raise INPUT_TOO_LONG, which those routes already return as HTTP 400. Only exact matches qualify, and callers see a fixed message with the token counts, so other engine messages stay generic. The new error is also an InvalidInputError, so a batch shared with other requests still isolates the one at fault instead of failing them all. Co-Authored-By: Claude Opus 5.5 --- .../src/sie_server/adapters/sglang/_window.py | 48 +++ .../sie_server/adapters/sglang/embedding.py | 15 + .../sie_server/adapters/sglang/generation.py | 14 +- .../adapters/sglang_vision_extract/adapter.py | 10 +- .../adapters/test_sglang_window_refusal.py | 310 ++++++++++++++++++ .../tests/api/test_encode_endpoint.py | 34 +- packages/sie_server/tests/api/test_extract.py | 32 +- 7 files changed, 458 insertions(+), 5 deletions(-) create mode 100644 packages/sie_server/src/sie_server/adapters/sglang/_window.py create mode 100644 packages/sie_server/tests/adapters/test_sglang_window_refusal.py diff --git a/packages/sie_server/src/sie_server/adapters/sglang/_window.py b/packages/sie_server/src/sie_server/adapters/sglang/_window.py new file mode 100644 index 000000000..2af2fec80 --- /dev/null +++ b/packages/sie_server/src/sie_server/adapters/sglang/_window.py @@ -0,0 +1,48 @@ +"""Recognise the SGLang engine refusing a request that does not fit the model's window. + +The engine reports this as a plain message, in an event, a finish reason or a +400 body depending on where it noticed. A retry of such a request can never +succeed, so it must reach the caller as a client error rather than as an engine +failure. Only the exact texts below qualify: the engine's other messages can +carry request or engine details and stay generic, and the caller only ever sees +the fixed text built here. +""" + +from __future__ import annotations + +import re + +from sie_server.adapters.errors import InputTooLongError +from sie_server.types.inputs import InvalidInputError + +# Identical in the SGLang 0.5.10 and 0.5.20 the bundles pin: the first two come from +# its tokenizer process, the last two from its scheduler, which also counts image +# tokens and keeps a small margin. +_WINDOW_REFUSALS = tuple( + re.compile(pattern) + for pattern in ( + r"The input \((?P\d+) tokens\) is longer than the model's context length \((?P\d+) tokens\)\.", + r"Requested token count exceeds the model's maximum context length of (?P\d+) tokens\. " + r"You requested a total of (?P\d+) tokens: \d+ tokens from the input messages and \d+ tokens " + r"for the completion\. Please reduce the number of tokens in the input messages or the completion " + r"to fit within the limit\.", + r"Input length \((?P\d+) tokens\) exceeds the maximum allowed length \((?P\d+) tokens\)\. " + r"Use a shorter input or enable --allow-auto-truncate\.", + r"Multimodal prompt is too long after expanding multimodal tokens\. " + r"After expanding len\(req\.origin_input_ids_unpadded\)=\d+ => (?P\d+) >= (?P\d+)\.", + ) +) + + +class SGLangInputTooLongError(InputTooLongError, InvalidInputError): + """``INPUT_TOO_LONG`` that still lets a shared batch isolate the request at fault.""" + + +def window_refusal_message(message: object) -> str | None: + """The text to show the caller when ``message`` is a window refusal, else ``None``.""" + if not isinstance(message, str): + return None + for pattern in _WINDOW_REFUSALS: + if (match := pattern.fullmatch(message)) is not None: + return f"The request needs {match['tokens']} tokens, at or over this model's limit of {match['limit']}" + return None diff --git a/packages/sie_server/src/sie_server/adapters/sglang/embedding.py b/packages/sie_server/src/sie_server/adapters/sglang/embedding.py index 20cbe71f7..b9557ccf7 100644 --- a/packages/sie_server/src/sie_server/adapters/sglang/embedding.py +++ b/packages/sie_server/src/sie_server/adapters/sglang/embedding.py @@ -37,6 +37,7 @@ from sie_server.adapters.sglang import _server from sie_server.adapters.sglang._lora import SGLangLoraManager, SGLangLoraMixin, leased_encode from sie_server.adapters.sglang._memory import SGLangMemoryBudget +from sie_server.adapters.sglang._window import SGLangInputTooLongError, window_refusal_message from sie_server.core.inference_output import EncodeOutput from sie_server.core.tokenizer import load_tokenizer @@ -84,6 +85,19 @@ def _resolve_embed_concurrency(configured: int | None) -> int: return _DEFAULT_EMBED_CONCURRENCY +def _raise_for_window_refusal(response: requests.Response) -> None: + """Report an embedding request the engine refused for its length as a client error.""" + if response.status_code != 400: + return + try: + body = response.json() + except ValueError: + return + refusal = window_refusal_message(body.get("message")) if isinstance(body, dict) else None + if refusal is not None: + raise SGLangInputTooLongError(refusal) + + class SGLangEmbeddingAdapter(SGLangLoraMixin, BaseAdapter): """Adapter for LLM embedding models using SGLang HTTP server backend. @@ -766,6 +780,7 @@ def _post_embedding_batch( timeout=self._request_read_timeout_s, ) if response.status_code != 200: + _raise_for_window_refusal(response) logger.error( "SGLang error %d for %d texts: %s", response.status_code, diff --git a/packages/sie_server/src/sie_server/adapters/sglang/generation.py b/packages/sie_server/src/sie_server/adapters/sglang/generation.py index e18d7ee00..a25a201c3 100644 --- a/packages/sie_server/src/sie_server/adapters/sglang/generation.py +++ b/packages/sie_server/src/sie_server/adapters/sglang/generation.py @@ -45,6 +45,7 @@ GenerationAdapter, GenerationChunk, GenerationError, + GenerationInputTooLongError, GenerationInvalidRequestError, GenerationResult, ReasoningFormat, @@ -60,6 +61,7 @@ from sie_server.adapters.sglang._compat.sie_thinking_budget import PROCESSOR_REFERENCE as THINKING_BUDGET_PROCESSOR from sie_server.adapters.sglang._lora import SGLangLoraManager, SGLangLoraMixin, leased_generation from sie_server.adapters.sglang._memory import SGLangMemoryBudget +from sie_server.adapters.sglang._window import window_refusal_message from sie_server.observability.generation_diagnostics import GenerationStreamTimer from sie_server.types.grammar import ( OUTLINES_JSON_SCHEMA_TYPE_DIAGNOSTIC, @@ -314,6 +316,12 @@ def _raise_for_media_load_error(message: object) -> None: ) +def _raise_for_window_refusal(message: object) -> None: + refusal = window_refusal_message(message) + if refusal is not None: + raise GenerationInputTooLongError(refusal) + + def _unique_json_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]: result: dict[str, Any] = {} for key, value in pairs: @@ -352,6 +360,7 @@ async def read_error() -> bytes: ): raise GenerationInvalidRequestError("grammar", OUTLINES_JSON_SCHEMA_TYPE_MESSAGE) _raise_for_media_load_error(message) + _raise_for_window_refusal(message) response.raise_for_status() @@ -363,7 +372,9 @@ def _raise_for_sglang_event_error( raise GenerationError("SGLang /generate returned an invalid event") if "error" in event: error = event["error"] - _raise_for_media_load_error(error.get("message") if isinstance(error, dict) else None) + message = error.get("message") if isinstance(error, dict) else None + _raise_for_media_load_error(message) + _raise_for_window_refusal(message) raise GenerationError("SGLang /generate returned an in-band error") meta = event.get("meta_info") finish = meta.get("finish_reason") if isinstance(meta, dict) else None @@ -372,6 +383,7 @@ def _raise_for_sglang_event_error( raise GenerationError("SGLang /generate returned a malformed finish reason") if kind == "abort": if isinstance(finish, dict) and type(finish.get("status_code")) is int and finish["status_code"] == 400: + _raise_for_window_refusal(finish.get("message")) if grammar is not None: if ( grammar_backend == "outlines" diff --git a/packages/sie_server/src/sie_server/adapters/sglang_vision_extract/adapter.py b/packages/sie_server/src/sie_server/adapters/sglang_vision_extract/adapter.py index 40c87b025..8e77371d0 100644 --- a/packages/sie_server/src/sie_server/adapters/sglang_vision_extract/adapter.py +++ b/packages/sie_server/src/sie_server/adapters/sglang_vision_extract/adapter.py @@ -17,12 +17,13 @@ from huggingface_hub import snapshot_download -from sie_server.adapters._generation_base import GenerationResult, collect_generation +from sie_server.adapters._generation_base import GenerationInputTooLongError, GenerationResult, collect_generation from sie_server.adapters._spec import AdapterSpec from sie_server.adapters._transformers_compat import install_transformers_security_fixes from sie_server.adapters._types import ComputePrecision from sie_server.adapters.base import ModelCapabilities, ModelDims from sie_server.adapters.lighton_ocr._markdown import lightonocr3_markdown +from sie_server.adapters.sglang._window import SGLangInputTooLongError from sie_server.adapters.sglang.generation import SGLangGenerationAdapter from sie_server.core.inference_output import ExtractOutput from sie_server.types.inputs import InvalidMediaError @@ -313,7 +314,12 @@ def extract( self._extract_async(prompt, images, max_new_tokens=max_new_tokens), self._request_loop, ) - results = future.result() + # Extract callers only recognise the input-error type: the generation one + # would be reported as a 500. + try: + results = future.result() + except GenerationInputTooLongError as exc: + raise SGLangInputTooLongError(str(exc)) from exc entities = [ [ Entity( diff --git a/packages/sie_server/tests/adapters/test_sglang_window_refusal.py b/packages/sie_server/tests/adapters/test_sglang_window_refusal.py new file mode 100644 index 000000000..21d737eee --- /dev/null +++ b/packages/sie_server/tests/adapters/test_sglang_window_refusal.py @@ -0,0 +1,310 @@ +"""A request that cannot fit the model's window is a client error on every SGLang path.""" + +from __future__ import annotations + +import json +from collections.abc import Iterator +from concurrent.futures import ThreadPoolExecutor +from types import SimpleNamespace +from typing import Any +from unittest.mock import MagicMock, patch + +import httpx +import pytest +from sie_server.adapters._generation_base import ( + GenerationError, + GenerationInputTooLongError, + GenerationInvalidRequestError, +) +from sie_server.adapters.errors import InputTooLongError +from sie_server.adapters.sglang._window import SGLangInputTooLongError, window_refusal_message +from sie_server.adapters.sglang.embedding import SGLangEmbeddingAdapter +from sie_server.adapters.sglang.generation import ( + SGLangGenerationAdapter, + _raise_for_sglang_event_error, + _raise_for_sglang_http_error, +) +from sie_server.adapters.sglang_vision_extract.adapter import SGLangVisionExtractAdapter +from sie_server.queue_executor import _inference_exception_outcome +from sie_server.types.grammar import GrammarSpec +from sie_server.types.inputs import ImageInput, InvalidInputError, Item + +# The engine's four texts, with the tokens and limit each should report. +REFUSALS = [ + pytest.param( + "The input (30042 tokens) is longer than the model's context length (24576 tokens).", + "30042", + "24576", + id="tokenizer-input", + ), + pytest.param( + "Requested token count exceeds the model's maximum context length of 8192 tokens. " + "You requested a total of 8192 tokens: 8176 tokens from the input messages and 16 tokens for the " + "completion. Please reduce the number of tokens in the input messages or the completion to fit within " + "the limit.", + "8192", + "8192", + id="tokenizer-total", + ), + pytest.param( + "Input length (8189 tokens) exceeds the maximum allowed length (8186 tokens). " + "Use a shorter input or enable --allow-auto-truncate.", + "8189", + "8186", + id="scheduler-input", + ), + pytest.param( + "Multimodal prompt is too long after expanding multimodal tokens. " + "After expanding len(req.origin_input_ids_unpadded)=71 => 16454 >= 16378.", + "16454", + "16378", + id="scheduler-multimodal", + ), +] +OTHER_MESSAGES = [ + "vision processor rejected image", + "The input (12 tokens) is longer than the model's context length (10 tokens). secret", + "secret The input (12 tokens) is longer than the model's context length (10 tokens).", + "The input (twelve tokens) is longer than the model's context length (10 tokens).", + ["The input (12 tokens) is longer than the model's context length (10 tokens)."], + None, +] +ONE_REFUSAL = "The input (30042 tokens) is longer than the model's context length (24576 tokens)." +REGEX_GRAMMAR = GrammarSpec(kind="regex", value="[a-z]+") + + +def _expected(tokens: str, limit: str) -> str: + return f"The request needs {tokens} tokens, at or over this model's limit of {limit}" + + +@pytest.mark.parametrize(("message", "tokens", "limit"), REFUSALS) +def test_engine_refusals_become_fixed_text(message: str, tokens: str, limit: str) -> None: + assert window_refusal_message(message) == _expected(tokens, limit) + + +@pytest.mark.parametrize("message", OTHER_MESSAGES) +def test_other_engine_messages_are_not_window_refusals(message: object) -> None: + assert window_refusal_message(message) is None + + +# Generation: where the engine reports the refusal depends on where it noticed. + + +@pytest.mark.parametrize(("message", "tokens", "limit"), REFUSALS) +def test_in_band_refusal_is_input_too_long(message: str, tokens: str, limit: str) -> None: + with pytest.raises(GenerationInputTooLongError) as error: + _raise_for_sglang_event_error({"error": {"message": message}}) + assert error.value.code == "INPUT_TOO_LONG" + assert str(error.value) == _expected(tokens, limit) + + +def test_in_band_refusal_is_recognised_in_the_newer_engine_event() -> None: + # SGLang 0.5.20 reports the same text with these extra fields. + event = { + "error": {"message": ONE_REFUSAL, "type": "invalid_request_error", "code": 400, "retryable": False}, + } + with pytest.raises(GenerationInputTooLongError) as error: + _raise_for_sglang_event_error(event) + assert str(error.value) == _expected("30042", "24576") + + +@pytest.mark.parametrize("grammar", [None, REGEX_GRAMMAR], ids=["no-grammar", "grammar"]) +@pytest.mark.parametrize(("message", "tokens", "limit"), REFUSALS) +def test_abort_refusal_is_input_too_long_even_with_a_grammar( + message: str, tokens: str, limit: str, grammar: GrammarSpec | None +) -> None: + event = {"meta_info": {"finish_reason": {"type": "abort", "status_code": 400, "message": message}}} + with pytest.raises(GenerationInputTooLongError) as error: + _raise_for_sglang_event_error(event, grammar=grammar) + assert str(error.value) == _expected(tokens, limit) + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("message", "tokens", "limit"), REFUSALS) +async def test_http_400_refusal_is_input_too_long(message: str, tokens: str, limit: str) -> None: + response = httpx.Response( + 400, json={"error": {"message": message}}, request=httpx.Request("POST", "http://localhost/generate") + ) + with pytest.raises(GenerationInputTooLongError) as error: + await _raise_for_sglang_http_error(response, grammar=None, grammar_backend=None) + assert str(error.value) == _expected(tokens, limit) + + +@pytest.mark.parametrize("message", OTHER_MESSAGES) +def test_other_in_band_errors_stay_engine_failures(message: object) -> None: + with pytest.raises(GenerationError) as error: + _raise_for_sglang_event_error({"error": {"message": message}}) + assert not isinstance(error.value, GenerationInputTooLongError) + assert error.value.code == "inference_error" + assert "secret" not in str(error.value) + + +@pytest.mark.asyncio +async def test_a_refusal_text_on_an_engine_failure_status_stays_a_failure() -> None: + response = httpx.Response( + 500, json={"error": {"message": ONE_REFUSAL}}, request=httpx.Request("POST", "http://localhost/generate") + ) + with pytest.raises(httpx.HTTPStatusError): + await _raise_for_sglang_http_error(response, grammar=None, grammar_backend=None) + + event = {"meta_info": {"finish_reason": {"type": "abort", "status_code": 500, "message": ONE_REFUSAL}}} + with pytest.raises(GenerationError) as error: + _raise_for_sglang_event_error(event) + assert not isinstance(error.value, GenerationInputTooLongError) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("n", "stream", "best_of"), [(1, False, None), (2, True, None), (2, False, None), (1, False, 2)] +) +async def test_generate_reports_a_refusal_on_every_request_shape(n: int, stream: bool, best_of: int | None) -> None: + adapter = SGLangGenerationAdapter(model_name_or_path="org/model", max_seq_length=8192, mem_fraction_static=0.85) + + def respond(request: httpx.Request) -> httpx.Response: + if json.loads(request.content)["stream"]: + line = json.dumps({"error": {"message": ONE_REFUSAL}}) + return httpx.Response(200, content=f"data: {line}\n\ndata: [DONE]\n\n".encode()) + return httpx.Response(400, json={"error": {"message": ONE_REFUSAL}}) + + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client: + adapter._http_client = client + adapter._server_url = "http://localhost:30005" + with pytest.raises(GenerationInputTooLongError): + async for _ in adapter.generate(prompt="word", max_new_tokens=8, n=n, stream=stream, best_of=best_of): + pass + + +# Extract: the same refusal comes through a generation that extract callers do not know. + + +class _Processor: + def apply_chat_template(self, *_args: object, **_kwargs: object) -> str: + return "rendered prompt" + + +def _engine_stream(event: dict[str, Any]) -> httpx.MockTransport: + return httpx.MockTransport(lambda _: httpx.Response(200, content=f"data: {json.dumps(event)}\n\n".encode())) + + +@pytest.fixture +def extract_adapter() -> Iterator[SGLangVisionExtractAdapter]: + adapter = SGLangVisionExtractAdapter("org/ocr", max_concurrent_requests=2) + adapter._processor = _Processor() + adapter._server_url = "http://sglang.test" + adapter._start_request_loop() + yield adapter + adapter.unload() + + +def _page() -> Item: + return Item(images=[ImageInput(data=b"page", format="png")]) + + +@pytest.mark.parametrize( + "event", + [ + pytest.param({"error": {"message": ONE_REFUSAL}}, id="in-band"), + pytest.param( + {"meta_info": {"finish_reason": {"type": "abort", "status_code": 400, "message": ONE_REFUSAL}}}, + id="abort", + ), + ], +) +def test_extract_reports_a_refusal_as_input_too_long( + extract_adapter: SGLangVisionExtractAdapter, event: dict[str, Any] +) -> None: + extract_adapter._http_client = httpx.AsyncClient(transport=_engine_stream(event)) + + with pytest.raises(InputTooLongError) as error: + extract_adapter.extract([_page(), _page()]) + + assert str(error.value) == _expected("30042", "24576") + # Also an input error, so a batch shared with other requests isolates this one. + assert isinstance(error.value, InvalidInputError) + + +def test_extract_leaves_other_engine_failures_alone(extract_adapter: SGLangVisionExtractAdapter) -> None: + event = {"error": {"message": "vision processor rejected image"}} + extract_adapter._http_client = httpx.AsyncClient(transport=_engine_stream(event)) + + with pytest.raises(GenerationError) as error: + extract_adapter.extract([_page()]) + + assert not isinstance(error.value, GenerationInvalidRequestError) + assert not isinstance(error.value, ValueError) + + +def test_extract_refusal_keeps_the_input_too_long_code_on_the_queue_path() -> None: + batch_item: Any = SimpleNamespace(work_item_id="w", request_id="r", item_index=0) + outcome = _inference_exception_outcome(batch_item, SGLangInputTooLongError(_expected("30042", "24576"))) + + assert outcome.error_code == "INPUT_TOO_LONG" + assert outcome.error == _expected("30042", "24576") + + +# Encode: SGLang's OpenAI surface returns the same text in its own error body. + + +def _embedding_refusal(status_code: int = 400, message: object = ONE_REFUSAL) -> MagicMock: + response = MagicMock(status_code=status_code, text="engine error body") + response.json.return_value = { + "object": "error", + "message": message, + "type": "BadRequestError", + "param": None, + "code": status_code, + } + response.raise_for_status.side_effect = RuntimeError(f"status {status_code}") + return response + + +def test_encode_reports_a_refusal_as_input_too_long() -> None: + adapter = SGLangEmbeddingAdapter("org/embedding", normalize=False, embed_concurrency=1) + adapter._server_url = "http://localhost:30000" + + with patch("sie_server.adapters.sglang.embedding.requests.post", return_value=_embedding_refusal()): + with pytest.raises(InputTooLongError) as error: + adapter.encode([Item(text="hello"), Item(text="world")], output_types=["dense"]) + + assert str(error.value) == _expected("30042", "24576") + # Also an input error, so a batch shared with other requests isolates this one. + assert isinstance(error.value, InvalidInputError) + + +def test_encode_reports_a_refusal_from_a_concurrent_shard() -> None: + adapter = SGLangEmbeddingAdapter("org/embedding", normalize=False, embed_concurrency=2) + adapter._server_url = "http://localhost:30000" + adapter._session = MagicMock() + adapter._session.post.return_value = _embedding_refusal() + + with ThreadPoolExecutor(2) as executor: + adapter._post_executor = executor + with pytest.raises(InputTooLongError): + adapter.encode([Item(text="hello"), Item(text="world")], output_types=["dense"]) + + assert adapter._session.post.call_count == 2 + + +@pytest.mark.parametrize( + ("status_code", "message"), + [(400, "unknown model"), (400, ["not text"]), (500, ONE_REFUSAL)], +) +def test_encode_leaves_other_engine_errors_alone(status_code: int, message: object) -> None: + adapter = SGLangEmbeddingAdapter("org/embedding", normalize=False, embed_concurrency=1) + adapter._server_url = "http://localhost:30000" + refused = _embedding_refusal(status_code, message) + + with patch("sie_server.adapters.sglang.embedding.requests.post", return_value=refused): + with pytest.raises(RuntimeError, match=f"status {status_code}"): + adapter.encode([Item(text="hello")], output_types=["dense"]) + + +def test_encode_tolerates_an_engine_error_that_is_not_json() -> None: + adapter = SGLangEmbeddingAdapter("org/embedding", normalize=False, embed_concurrency=1) + adapter._server_url = "http://localhost:30000" + refused = _embedding_refusal() + refused.json.side_effect = ValueError("not json") + + with patch("sie_server.adapters.sglang.embedding.requests.post", return_value=refused): + with pytest.raises(RuntimeError, match="status 400"): + adapter.encode([Item(text="hello")], output_types=["dense"]) diff --git a/packages/sie_server/tests/api/test_encode_endpoint.py b/packages/sie_server/tests/api/test_encode_endpoint.py index 7b10b48ef..daf94609a 100644 --- a/packages/sie_server/tests/api/test_encode_endpoint.py +++ b/packages/sie_server/tests/api/test_encode_endpoint.py @@ -1,13 +1,16 @@ +import json import time from typing import Any -from unittest.mock import AsyncMock, MagicMock +from unittest.mock import AsyncMock, MagicMock, patch import msgpack import msgpack_numpy as m import numpy as np import pytest +import requests from fastapi import FastAPI from fastapi.testclient import TestClient +from sie_server.adapters.sglang.embedding import SGLangEmbeddingAdapter from sie_server.api.encode import router as encode_router from sie_server.config.model import ( AdapterOptions, @@ -126,6 +129,35 @@ def client(mock_registry: MagicMock) -> TestClient: class TestEncodeEndpoint: """Tests for POST /v1/encode/{model}.""" + def test_encode_engine_window_refusal_is_input_too_long(self, client: TestClient, mock_adapter: MagicMock) -> None: + """A text the SGLang engine refuses for its length is the caller's to change, so it is not a 500.""" + engine = SGLangEmbeddingAdapter("org/embedding", normalize=False, embed_concurrency=1) + engine._server_url = "http://localhost:30000" + mock_adapter.encode.side_effect = lambda items, output_types, **_kwargs: engine.encode( + items, output_types=output_types + ) + refused = requests.Response() + refused.status_code = 400 + refused._content = json.dumps( + { + "object": "error", + "message": "The input (40000 tokens) is longer than the model's context length (32768 tokens).", + "type": "BadRequestError", + "param": None, + "code": 400, + } + ).encode() + + with patch("sie_server.adapters.sglang.embedding.requests.post", return_value=refused): + response = client.post( + "/v1/encode/test-model", + json={"items": [{"text": "Hello world"}]}, + headers=JSON_HEADERS, + ) + + assert response.status_code == 400 + assert response.json()["detail"]["code"] == "INPUT_TOO_LONG" + def test_encode_basic_json(self, client: TestClient) -> None: """Basic encode request returns JSON when Accept header set.""" response = client.post( diff --git a/packages/sie_server/tests/api/test_extract.py b/packages/sie_server/tests/api/test_extract.py index ef38cebee..fe93077d4 100644 --- a/packages/sie_server/tests/api/test_extract.py +++ b/packages/sie_server/tests/api/test_extract.py @@ -1,9 +1,11 @@ """Tests for extract endpoint.""" import asyncio +import json from typing import Any, ClassVar from unittest.mock import AsyncMock, MagicMock +import httpx import msgpack import msgpack_numpy as m import msgspec @@ -13,6 +15,7 @@ from sie_server.adapters._base_adapter import BaseAdapter from sie_server.adapters._spec import AdapterSpec from sie_server.adapters.base import ModelCapabilities, ModelDims +from sie_server.adapters.sglang_vision_extract.adapter import SGLangVisionExtractAdapter from sie_server.api.extract import _extract_via_worker from sie_server.api.extract import router as extract_router from sie_server.config.model import ( @@ -39,7 +42,7 @@ from sie_server.core.timing import RequestTiming from sie_server.core.worker import WorkerResult from sie_server.core.worker.handlers.extract import ExtractHandler -from sie_server.types.inputs import MAX_ITEM_TEXT_BYTES, Item +from sie_server.types.inputs import MAX_ITEM_TEXT_BYTES, ImageInput, Item from sie_server.types.responses import Classification, Entity # Patch msgpack for numpy support @@ -873,6 +876,33 @@ def test_invalid_profile_returns_400(self, client: TestClient) -> None: assert data["detail"]["code"] == "INVALID_INPUT" assert "nonexistent_profile" in data["detail"]["message"] + def test_extract_engine_window_refusal_is_input_too_long(self, client: TestClient, mock_adapter: MagicMock) -> None: + """A request too long for the SGLang engine is the caller's to change, so it is not a 500.""" + ocr = SGLangVisionExtractAdapter("org/ocr") + ocr._processor = MagicMock() + ocr._processor.apply_chat_template.return_value = "prompt" + ocr._server_url = "http://sglang.test" + ocr._start_request_loop() + event = json.dumps( + {"error": {"message": "The input (30042 tokens) is longer than the model's context length (24576 tokens)."}} + ) + ocr._http_client = httpx.AsyncClient( + transport=httpx.MockTransport(lambda _: httpx.Response(200, content=f"data: {event}\n\n".encode())) + ) + page = Item(images=[ImageInput(data=b"page", format="png")]) + mock_adapter.extract.side_effect = lambda *_args, **_kwargs: ocr.extract([page]) + try: + response = client.post( + "/v1/extract/test-extractor", + json={"items": [{"text": "Text"}], "params": {"labels": ["entity"]}}, + headers=JSON_HEADERS, + ) + finally: + ocr.unload() + + assert response.status_code == 400 + assert response.json()["detail"]["code"] == "INPUT_TOO_LONG" + def test_extract_adapter_runtime_error(self, client: TestClient, mock_adapter: MagicMock) -> None: """RuntimeError from adapter returns 500.""" mock_adapter.extract.side_effect = RuntimeError("Inference failed")